From bc6067d19550743f3efb3b2709f22c14f4f1f901 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 1 Oct 2021 11:50:00 -0700 Subject: [PATCH] Add admin inspect Glob support (#13328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add admin Glob support Allow returning multiple files on inspect calls. ``` λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/* ... λ unzip -l inspect.5f0643b2.zip Archive: inspect.5f0643b2.zip Length Date Time Name --------- ---------- ----- ---- 0 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 802 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta 0 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 802 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta 0 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 802 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta 0 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 802 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta --------- ------- 3208 8 files ``` Using fully recursive: ``` λ mc admin inspect local2/testbucket/nyc-taxi-data-10M.csv.zst/** ... Archive: inspect.79c261cb.zip Length Date Time Name --------- ---------- ----- ---- 0 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/ 0 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35 3439368 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8 4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9 802 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta 0 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/ 0 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35 3439368 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8 4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9 802 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta 0 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/ 0 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35 3439368 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8 4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9 802 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta 0 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/ 0 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/ 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35 3439368 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8 4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9 802 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta --------- ------- 601034920 156 files ``` Furthermore allow `inspect` to do direct decode from `mc`, for example: ``` λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*|inspect -json Output decrypted to inspect.5f0643b2.zip ``` - Correct error, forward non-EOF errors. - Add some extra safety. Log FNF when no files. - Add `xl-meta` zip support. For `xl-meta` multiple inputs output object with names as key. Automatically switches `xl-meta` to single-line output when multiple objects. Add double-star wildcard support to xl-meta input. Co-authored-by: Harshavardhana --- cmd/admin-handlers.go | 19 ++- cmd/erasure-healing_test.go | 4 +- cmd/erasure-server-pool.go | 33 +++-- cmd/fs-v1.go | 4 +- cmd/naughty-disk_test.go | 4 +- cmd/storage-interface.go | 2 +- cmd/storage-rest-client.go | 18 ++- cmd/storage-rest-common.go | 1 + cmd/storage-rest-server.go | 9 +- cmd/storage-rest_test.go | 4 +- cmd/xl-storage-disk-id-check.go | 8 +- cmd/xl-storage-format-v1.go | 2 + cmd/xl-storage-format-v1_gen.go | 218 +++++++++++++----------------- cmd/xl-storage.go | 36 +++-- cmd/xl-storage_test.go | 2 +- cmd/xl-storage_unix_test.go | 2 +- docs/bucket/versioning/xl-meta.go | 182 ++++++++++++++++++------- docs/debugging/inspect/main.go | 47 ++++++- go.mod | 1 + go.sum | 4 +- 20 files changed, 379 insertions(+), 221 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 02c3396f7..bd8e7c527 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -38,7 +38,7 @@ import ( "strings" "time" - humanize "github.com/dustin/go-humanize" + "github.com/dustin/go-humanize" "github.com/gorilla/mux" "github.com/klauspost/compress/zip" "github.com/minio/madmin-go" @@ -2144,7 +2144,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error { // getRawDataer provides an interface for getting raw FS files. type getRawDataer interface { - GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error + GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error } // InspectDataHandler - GET /minio/admin/v3/inspect-data @@ -2177,6 +2177,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) return } + file = strings.ReplaceAll(file, string(os.PathSeparator), "/") + + // Reject attempts to traverse parent or absolute paths. + if strings.Contains(file, "..") || strings.Contains(volume, "..") { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } var key [32]byte // MUST use crypto/rand @@ -2214,15 +2221,19 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ zipWriter := zip.NewWriter(encw) defer zipWriter.Close() - err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error { + err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time, isDir bool) error { // Prefix host+disk filename = path.Join(host, disk, filename) + if isDir { + filename += "/" + size = 0 + } header, zerr := zip.FileInfoHeader(dummyFileInfo{ name: filename, size: size, mode: 0600, modTime: modtime, - isDir: false, + isDir: isDir, sys: nil, }) if zerr != nil { diff --git a/cmd/erasure-healing_test.go b/cmd/erasure-healing_test.go index ea1f650ee..9d0708490 100644 --- a/cmd/erasure-healing_test.go +++ b/cmd/erasure-healing_test.go @@ -421,7 +421,7 @@ func TestHealObjectCorrupted(t *testing.T) { t.Fatalf("Failed to getLatestFileInfo - %v", err) } - if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil { + if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil { t.Errorf("Expected er.meta file to be present but stat failed - %v", err) } @@ -565,7 +565,7 @@ func TestHealObjectErasure(t *testing.T) { t.Fatalf("Failed to heal object - %v", err) } - if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil { + if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil { t.Errorf("Expected er.meta file to be present but stat failed - %v", err) } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index e5f9d8c71..fcb8a2cea 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -18,6 +18,7 @@ package cmd import ( + "bytes" "context" "errors" "fmt" @@ -150,33 +151,45 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI { // GetRawData will return all files with a given raw path to the callback. // Errors are ignored, only errors from the callback are returned. // For now only direct file paths are supported. -func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error { +func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error { + found := 0 for _, s := range z.serverPools { for _, disks := range s.erasureDisks { for i, disk := range disks { if disk == OfflineDisk { continue } - si, err := disk.StatInfoFile(ctx, volume, file) + stats, err := disk.StatInfoFile(ctx, volume, file, true) if err != nil { continue } - r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size) - if err != nil { - continue - } - defer r.Close() did, err := disk.GetDiskID() if err != nil { did = fmt.Sprintf("disk-%d", i) } - err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime) - if err != nil { - return err + for _, si := range stats { + found++ + var r io.ReadCloser + if !si.Dir { + r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size) + if err != nil { + continue + } + } else { + r = io.NopCloser(bytes.NewBuffer([]byte{})) + } + err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si.Size, si.ModTime, si.Dir) + r.Close() + if err != nil { + return err + } } } } } + if found == 0 { + return errFileNotFound + } return nil } diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 44b45038c..c0863ea2b 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -1508,7 +1508,7 @@ func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, obje // GetRawData returns raw file data to the callback. // Errors are ignored, only errors from the callback are returned. // For now only direct file paths are supported. -func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error { +func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error { f, err := os.Open(filepath.Join(fs.fsPath, volume, file)) if err != nil { return nil @@ -1518,5 +1518,5 @@ func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn fun if err != nil || st.IsDir() { return nil } - return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime()) + return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime(), st.IsDir()) } diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index e59806361..6ceae732e 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -285,9 +285,9 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi return d.disk.VerifyFile(ctx, volume, path, fi) } -func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) { +func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { if err := d.calcError(); err != nil { return stat, err } - return d.disk.StatInfoFile(ctx, volume, path) + return d.disk.StatInfoFile(ctx, volume, path, glob) } diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index 99a6566e5..d17352c11 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -73,7 +73,7 @@ type StorageAPI interface { CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error Delete(ctx context.Context, volume string, path string, recursive bool) (err error) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error - StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) + StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) // Write all data, syncs the data to disk. // Should be used for smaller payloads. diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 4fff5da07..4635a9b16 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -23,6 +23,7 @@ import ( "encoding/gob" "encoding/hex" "errors" + "fmt" "io" "io/ioutil" "net/url" @@ -680,10 +681,11 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st return toStorageErr(verifyResp.Err) } -func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) { +func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { values := make(url.Values) values.Set(storageRESTVolume, volume) values.Set(storageRESTFilePath, path) + values.Set(storageRESTGlob, fmt.Sprint(glob)) respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1) if err != nil { return stat, err @@ -693,7 +695,19 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path if err != nil { return stat, err } - err = stat.DecodeMsg(msgpNewReader(respReader)) + rd := msgpNewReader(respReader) + for { + var st StatInfo + err = st.DecodeMsg(rd) + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + break + } + stat = append(stat, st) + } + return stat, err } diff --git a/cmd/storage-rest-common.go b/cmd/storage-rest-common.go index c78dc5da5..af7e82e48 100644 --- a/cmd/storage-rest-common.go +++ b/cmd/storage-rest-common.go @@ -78,4 +78,5 @@ const ( storageRESTBitrotHash = "bitrot-hash" storageRESTDiskID = "disk-id" storageRESTForceDelete = "force-delete" + storageRESTGlob = "glob" ) diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index a5b9f196c..a6efe0a0a 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -1122,13 +1122,16 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request) vars := mux.Vars(r) volume := vars[storageRESTVolume] filePath := vars[storageRESTFilePath] + glob := vars[storageRESTGlob] done := keepHTTPResponseAlive(w) - si, err := s.storage.StatInfoFile(r.Context(), volume, filePath) + stats, err := s.storage.StatInfoFile(r.Context(), volume, filePath, glob == "true") done(err) if err != nil { return } - msgp.Encode(w, &si) + for _, si := range stats { + msgp.Encode(w, &si) + } } // registerStorageRPCRouter - register storage rpc router. @@ -1221,7 +1224,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)). Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...) subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)). - Queries(restQueries(storageRESTVolume, storageRESTFilePath)...) + Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTGlob)...) } } } diff --git a/cmd/storage-rest_test.go b/cmd/storage-rest_test.go index 16f2d8c53..b501db5d3 100644 --- a/cmd/storage-rest_test.go +++ b/cmd/storage-rest_test.go @@ -186,11 +186,11 @@ func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) { } for i, testCase := range testCases { - _, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile) + _, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile, false) expectErr := (err != nil) if expectErr != testCase.expectErr { - t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + t.Fatalf("case %v: error: expected: %v, got: %v, err: %v", i+1, expectErr, testCase.expectErr, err) } } } diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index 7d9eb94ae..241002be5 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -547,18 +547,18 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path return p.storage.ReadAll(ctx, volume, path) } -func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) { +func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { defer p.updateStorageMetrics(storageStatInfoFile, volume, path)() if contextCanceled(ctx) { - return StatInfo{}, ctx.Err() + return nil, ctx.Err() } if err = p.checkDiskStale(); err != nil { - return StatInfo{}, err + return nil, err } - return p.storage.StatInfoFile(ctx, volume, path) + return p.storage.StatInfoFile(ctx, volume, path, glob) } func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo { diff --git a/cmd/xl-storage-format-v1.go b/cmd/xl-storage-format-v1.go index dc7e2f5ad..ec2c17467 100644 --- a/cmd/xl-storage-format-v1.go +++ b/cmd/xl-storage-format-v1.go @@ -81,6 +81,8 @@ type xlMetaV1Object struct { type StatInfo struct { Size int64 `json:"size"` // Size of the object `xl.meta`. ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`. + Name string `json:"name"` + Dir bool `json:"dir"` } // ErasureInfo holds erasure coding and bitrot related information. diff --git a/cmd/xl-storage-format-v1_gen.go b/cmd/xl-storage-format-v1_gen.go index 6c88e0daf..5e78a774f 100644 --- a/cmd/xl-storage-format-v1_gen.go +++ b/cmd/xl-storage-format-v1_gen.go @@ -759,6 +759,18 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "ModTime") return } + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Dir": + z.Dir, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Dir") + return + } default: err = dc.Skip() if err != nil { @@ -771,10 +783,10 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 +func (z *StatInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 // write "Size" - err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65) + err = en.Append(0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65) if err != nil { return } @@ -793,19 +805,45 @@ func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "ModTime") return } + // write "Name" + err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Dir" + err = en.Append(0xa3, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.Dir) + if err != nil { + err = msgp.WrapError(err, "Dir") + return + } return } // MarshalMsg implements msgp.Marshaler -func (z StatInfo) MarshalMsg(b []byte) (o []byte, err error) { +func (z *StatInfo) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 2 + // map header, size 4 // string "Size" - o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = append(o, 0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65) o = msgp.AppendInt64(o, z.Size) // string "ModTime" o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendTime(o, z.ModTime) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Dir" + o = append(o, 0xa3, 0x44, 0x69, 0x72) + o = msgp.AppendBool(o, z.Dir) return } @@ -839,6 +877,18 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "ModTime") return } + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Dir": + z.Dir, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Dir") + return + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -852,8 +902,8 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z StatInfo) Msgsize() (s int) { - s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize +func (z *StatInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.Name) + 4 + msgp.BoolSize return } @@ -1041,40 +1091,11 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) { return } case "Stat": - var zb0002 uint32 - zb0002, err = dc.ReadMapHeader() + err = z.Stat.DecodeMsg(dc) if err != nil { err = msgp.WrapError(err, "Stat") return } - for zb0002 > 0 { - zb0002-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err, "Stat") - return - } - switch msgp.UnsafeString(field) { - case "Size": - z.Stat.Size, err = dc.ReadInt64() - if err != nil { - err = msgp.WrapError(err, "Stat", "Size") - return - } - case "ModTime": - z.Stat.ModTime, err = dc.ReadTime() - if err != nil { - err = msgp.WrapError(err, "Stat", "ModTime") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err, "Stat") - return - } - } - } case "Erasure": err = z.Erasure.DecodeMsg(dc) if err != nil { @@ -1082,14 +1103,14 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) { return } case "Minio": - var zb0003 uint32 - zb0003, err = dc.ReadMapHeader() + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() if err != nil { err = msgp.WrapError(err, "Minio") return } - for zb0003 > 0 { - zb0003-- + for zb0002 > 0 { + zb0002-- field, err = dc.ReadMapKeyPtr() if err != nil { err = msgp.WrapError(err, "Minio") @@ -1111,21 +1132,21 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) { } } case "Meta": - var zb0004 uint32 - zb0004, err = dc.ReadMapHeader() + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() if err != nil { err = msgp.WrapError(err, "Meta") return } if z.Meta == nil { - z.Meta = make(map[string]string, zb0004) + z.Meta = make(map[string]string, zb0003) } else if len(z.Meta) > 0 { for key := range z.Meta { delete(z.Meta, key) } } - for zb0004 > 0 { - zb0004-- + for zb0003 > 0 { + zb0003-- var za0001 string var za0002 string za0001, err = dc.ReadString() @@ -1141,16 +1162,16 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) { z.Meta[za0001] = za0002 } case "Parts": - var zb0005 uint32 - zb0005, err = dc.ReadArrayHeader() + var zb0004 uint32 + zb0004, err = dc.ReadArrayHeader() if err != nil { err = msgp.WrapError(err, "Parts") return } - if cap(z.Parts) >= int(zb0005) { - z.Parts = (z.Parts)[:zb0005] + if cap(z.Parts) >= int(zb0004) { + z.Parts = (z.Parts)[:zb0004] } else { - z.Parts = make([]ObjectPartInfo, zb0005) + z.Parts = make([]ObjectPartInfo, zb0004) } for za0003 := range z.Parts { err = z.Parts[za0003].DecodeMsg(dc) @@ -1210,25 +1231,9 @@ func (z *xlMetaV1Object) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - // map header, size 2 - // write "Size" - err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65) + err = z.Stat.EncodeMsg(en) if err != nil { - return - } - err = en.WriteInt64(z.Stat.Size) - if err != nil { - err = msgp.WrapError(err, "Stat", "Size") - return - } - // write "ModTime" - err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) - if err != nil { - return - } - err = en.WriteTime(z.Stat.ModTime) - if err != nil { - err = msgp.WrapError(err, "Stat", "ModTime") + err = msgp.WrapError(err, "Stat") return } // write "Erasure" @@ -1331,13 +1336,11 @@ func (z *xlMetaV1Object) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.AppendString(o, z.Format) // string "Stat" o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74) - // map header, size 2 - // string "Size" - o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65) - o = msgp.AppendInt64(o, z.Stat.Size) - // string "ModTime" - o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) - o = msgp.AppendTime(o, z.Stat.ModTime) + o, err = z.Stat.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Stat") + return + } // string "Erasure" o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65) o, err = z.Erasure.MarshalMsg(o) @@ -1408,40 +1411,11 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "Stat": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + bts, err = z.Stat.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "Stat") return } - for zb0002 > 0 { - zb0002-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "Stat") - return - } - switch msgp.UnsafeString(field) { - case "Size": - z.Stat.Size, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Stat", "Size") - return - } - case "ModTime": - z.Stat.ModTime, bts, err = msgp.ReadTimeBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Stat", "ModTime") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err, "Stat") - return - } - } - } case "Erasure": bts, err = z.Erasure.UnmarshalMsg(bts) if err != nil { @@ -1449,14 +1423,14 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "Minio": - var zb0003 uint32 - zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "Minio") return } - for zb0003 > 0 { - zb0003-- + for zb0002 > 0 { + zb0002-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err, "Minio") @@ -1478,23 +1452,23 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) { } } case "Meta": - var zb0004 uint32 - zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "Meta") return } if z.Meta == nil { - z.Meta = make(map[string]string, zb0004) + z.Meta = make(map[string]string, zb0003) } else if len(z.Meta) > 0 { for key := range z.Meta { delete(z.Meta, key) } } - for zb0004 > 0 { + for zb0003 > 0 { var za0001 string var za0002 string - zb0004-- + zb0003-- za0001, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err, "Meta") @@ -1508,16 +1482,16 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) { z.Meta[za0001] = za0002 } case "Parts": - var zb0005 uint32 - zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + var zb0004 uint32 + zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "Parts") return } - if cap(z.Parts) >= int(zb0005) { - z.Parts = (z.Parts)[:zb0005] + if cap(z.Parts) >= int(zb0004) { + z.Parts = (z.Parts)[:zb0004] } else { - z.Parts = make([]ObjectPartInfo, zb0005) + z.Parts = make([]ObjectPartInfo, zb0004) } for za0003 := range z.Parts { bts, err = z.Parts[za0003].UnmarshalMsg(bts) @@ -1552,7 +1526,7 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *xlMetaV1Object) Msgsize() (s int) { - s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize + s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + z.Stat.Msgsize() + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize if z.Meta != nil { for za0001, za0002 := range z.Meta { _ = za0002 diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 9603d928b..5a095220d 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -46,6 +46,7 @@ import ( "github.com/minio/minio/internal/logger" "github.com/minio/pkg/console" "github.com/minio/pkg/env" + "github.com/yargevad/filepathx" ) const ( @@ -1132,7 +1133,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str // Check the data path if there is a part with data. partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number) dataPath := pathJoin(path, fi.DataDir, partPath) - _, err = s.StatInfoFile(ctx, volume, dataPath) + _, err = s.StatInfoFile(ctx, volume, dataPath, false) if err != nil { // Set the inline header, our inlined data is fine. fi.SetInlineData() @@ -2204,7 +2205,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File return nil } -func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) { +func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { volumeDir, err := s.getVolDir(volume) if err != nil { return stat, err @@ -2221,14 +2222,29 @@ func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat } return stat, err } - filePath := pathJoin(volumeDir, path) - if err := checkPathLength(filePath); err != nil { - return stat, err + var files = []string{pathJoin(volumeDir, path)} + if glob { + files, err = filepathx.Glob(pathJoin(volumeDir, path)) + if err != nil { + return nil, err + } } - st, _ := Lstat(filePath) - if st == nil { - return stat, errPathNotFound + for _, filePath := range files { + if err := checkPathLength(filePath); err != nil { + return stat, err + } + st, _ := Lstat(filePath) + if st == nil { + return stat, errPathNotFound + } + name, err := filepath.Rel(volumeDir, filePath) + if err != nil { + name = filePath + } + if os.PathSeparator != '/' { + name = strings.Replace(name, string(os.PathSeparator), "/", -1) + } + stat = append(stat, StatInfo{ModTime: st.ModTime(), Size: st.Size(), Name: name, Dir: st.IsDir()}) } - - return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil + return stat, nil } diff --git a/cmd/xl-storage_test.go b/cmd/xl-storage_test.go index 9b0ede45a..e3ec8a774 100644 --- a/cmd/xl-storage_test.go +++ b/cmd/xl-storage_test.go @@ -1711,7 +1711,7 @@ func TestXLStorageStatInfoFile(t *testing.T) { } for i, testCase := range testCases { - _, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile) + _, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile, false) if err != testCase.expectedErr { t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } diff --git a/cmd/xl-storage_unix_test.go b/cmd/xl-storage_unix_test.go index 8a8da2893..bd8e1b668 100644 --- a/cmd/xl-storage_unix_test.go +++ b/cmd/xl-storage_unix_test.go @@ -114,7 +114,7 @@ func TestIsValidUmaskFile(t *testing.T) { } // CheckFile - stat the file. - if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile); err != nil { + if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile, false); err != nil { t.Fatalf("Stat failed with %s expected to pass.", err) } } diff --git a/docs/bucket/versioning/xl-meta.go b/docs/bucket/versioning/xl-meta.go index 425bb6780..4a03cf9ff 100644 --- a/docs/bucket/versioning/xl-meta.go +++ b/docs/bucket/versioning/xl-meta.go @@ -27,10 +27,12 @@ import ( "io/ioutil" "log" "os" - "path/filepath" + "strings" + "github.com/klauspost/compress/zip" "github.com/minio/cli" "github.com/tinylib/msgp/msgp" + "github.com/yargevad/filepathx" ) func main() { @@ -44,9 +46,10 @@ func main() { USAGE: {{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES... -Multiple files can be added. +Multiple files can be added. Files ending in ".zip" will be searched for 'xl.meta' files. Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt +Double stars means full recursive. testdir/**/xl.meta will search for all xl.meta recursively. {{if .VisibleFlags}} GLOBAL FLAGS: @@ -72,43 +75,15 @@ GLOBAL FLAGS: } app.Action = func(c *cli.Context) error { - args := c.Args() - if len(args) == 0 { - // If no args, assume xl.meta - args = []string{"xl.meta"} - } - var files []string - for _, pattern := range args { - found, err := filepath.Glob(pattern) - if err != nil { - return err - } - if len(found) == 0 { - return fmt.Errorf("unable to find file %v", pattern) - } - files = append(files, found...) - } - for _, file := range files { - var r io.Reader - switch file { - case "-": - r = os.Stdin - default: - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - r = f - } - + ndjson := c.Bool("ndjson") + decode := func(r io.Reader, file string) ([]byte, error) { b, err := ioutil.ReadAll(r) if err != nil { - return err + return nil, err } b, _, minor, err := checkXL2V1(b) if err != nil { - return err + return nil, err } buf := bytes.NewBuffer(nil) @@ -117,12 +92,12 @@ GLOBAL FLAGS: case 0: _, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b)) if err != nil { - return err + return nil, err } case 1, 2: v, b, err := msgp.ReadBytesZC(b) if err != nil { - return err + return nil, err } if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil { // Read metadata CRC (added in v2, ignore if not found) @@ -131,31 +106,47 @@ GLOBAL FLAGS: _, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v)) if err != nil { - return err + return nil, err } data = b default: - return fmt.Errorf("unknown metadata version %d", minor) + return nil, fmt.Errorf("unknown metadata version %d", minor) } if c.Bool("data") { b, err := data.json() if err != nil { - return err + return nil, err } buf = bytes.NewBuffer(b) } if c.Bool("export") { + file := strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z': + return r + case r >= 'A' && r <= 'Z': + return r + case r >= '0' && r <= '9': + return r + case strings.ContainsAny(string(r), "+=-_()!@."): + return r + default: + return '_' + } + }, file) err := data.files(func(name string, data []byte) { - ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm) + err = ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm) + if err != nil { + fmt.Println(err) + } }) if err != nil { - return err + return nil, err } } - if c.Bool("ndjson") { - fmt.Println(buf.String()) - continue + if ndjson { + return buf.Bytes(), nil } var msi map[string]interface{} dec := json.NewDecoder(buf) @@ -163,14 +154,113 @@ GLOBAL FLAGS: dec.UseNumber() err = dec.Decode(&msi) if err != nil { - return err + return nil, err } b, err = json.MarshalIndent(msi, "", " ") + if err != nil { + return nil, err + } + return b, nil + } + + args := c.Args() + if len(args) == 0 { + // If no args, assume xl.meta + args = []string{"xl.meta"} + } + var files []string + + for _, pattern := range args { + if pattern == "-" { + files = append(files, pattern) + continue + } + found, err := filepathx.Glob(pattern) if err != nil { return err } - fmt.Println(string(b)) + if len(found) == 0 { + return fmt.Errorf("unable to find file %v", pattern) + } + files = append(files, found...) } + if len(files) == 0 { + return fmt.Errorf("no files found") + } + multiple := len(files) > 1 || strings.HasSuffix(files[0], ".zip") + if multiple { + ndjson = true + fmt.Println("{") + } + + hasWritten := false + for _, file := range files { + var r io.Reader + var sz int64 + switch file { + case "-": + r = os.Stdin + default: + f, err := os.Open(file) + if err != nil { + return err + } + if st, err := f.Stat(); err == nil { + sz = st.Size() + } + defer f.Close() + r = f + } + if strings.HasSuffix(file, ".zip") { + zr, err := zip.NewReader(r.(io.ReaderAt), sz) + if err != nil { + return err + } + for _, file := range zr.File { + if !file.FileInfo().IsDir() && strings.HasSuffix(file.Name, "xl.meta") { + r, err := file.Open() + if err != nil { + return err + } + // Quote string... + b, _ := json.Marshal(file.Name) + if hasWritten { + fmt.Print(",\n") + } + fmt.Printf("\t%s: ", string(b)) + + b, err = decode(r, file.Name) + if err != nil { + return err + } + fmt.Print(string(b)) + hasWritten = true + } + } + } else { + if multiple { + // Quote string... + b, _ := json.Marshal(file) + if hasWritten { + fmt.Print(",\n") + } + fmt.Printf("\t%s: ", string(b)) + } + + b, err := decode(r, file) + if err != nil { + return err + } + + hasWritten = true + fmt.Print(string(b)) + } + } + fmt.Println("") + if multiple { + fmt.Println("}") + } + return nil } err := app.Run(os.Args) diff --git a/docs/debugging/inspect/main.go b/docs/debugging/inspect/main.go index 0c53257dd..7570b53f2 100644 --- a/docs/debugging/inspect/main.go +++ b/docs/debugging/inspect/main.go @@ -21,10 +21,13 @@ import ( "bufio" "encoding/binary" "encoding/hex" + "encoding/json" + "errors" "flag" "fmt" "hash/crc32" "io" + "io/ioutil" "log" "os" "strings" @@ -34,25 +37,48 @@ import ( var ( key = flag.String("key", "", "decryption string") - //js = flag.Bool("json", false, "expect json input") + js = flag.Bool("json", false, "expect json input from stdin") ) func main() { flag.Parse() + + if *js { + // Match struct in https://github.com/minio/mc/blob/b3ce21fb72a914f50522358668e6464eb97de8d1/cmd/admin-inspect.go#L135 + input := struct { + File string `json:"file"` + Key string `json:"key"` + }{} + got, err := ioutil.ReadAll(os.Stdin) + if err != nil { + fatalErr(err) + } + fatalErr(json.Unmarshal(got, &input)) + r, err := os.Open(input.File) + fatalErr(err) + defer r.Close() + dstName := strings.TrimSuffix(input.File, ".enc") + ".zip" + w, err := os.Create(dstName) + fatalErr(err) + defer w.Close() + decrypt(input.Key, r, w) + fmt.Println("Output decrypted to", dstName) + return + } args := flag.Args() switch len(flag.Args()) { case 0: // Read from stdin, write to stdout. + if *key == "" { + flag.Usage() + fatalErr(errors.New("no key supplied")) + } decrypt(*key, os.Stdin, os.Stdout) return case 1: r, err := os.Open(args[0]) fatalErr(err) defer r.Close() - dstName := strings.TrimSuffix(args[0], ".enc") + ".zip" - w, err := os.Create(dstName) - fatalErr(err) - defer w.Close() if len(*key) == 0 { reader := bufio.NewReader(os.Stdin) fmt.Print("Enter Decryption Key: ") @@ -61,18 +87,25 @@ func main() { // convert CRLF to LF *key = strings.Replace(text, "\n", "", -1) } + *key = strings.TrimSpace(*key) + fatalIf(len(*key) != 72, "Unexpected key length: %d, want 72", len(*key)) + + dstName := strings.TrimSuffix(args[0], ".enc") + ".zip" + w, err := os.Create(dstName) + fatalErr(err) + defer w.Close() + decrypt(*key, r, w) fmt.Println("Output decrypted to", dstName) return default: + flag.Usage() fatalIf(true, "Only 1 file can be decrypted") os.Exit(1) } } func decrypt(keyHex string, r io.Reader, w io.Writer) { - keyHex = strings.TrimSpace(keyHex) - fatalIf(len(keyHex) != 72, "Unexpected key length: %d, want 72", len(keyHex)) id, err := hex.DecodeString(keyHex[:8]) fatalErr(err) key, err := hex.DecodeString(keyHex[8:]) diff --git a/go.mod b/go.mod index 626c78c6d..b18fb36c8 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c + github.com/yargevad/filepathx v1.0.0 go.etcd.io/etcd/api/v3 v3.5.0-beta.4 go.etcd.io/etcd/client/v3 v3.5.0-beta.4 go.opencensus.io v0.22.5 // indirect diff --git a/go.sum b/go.sum index 1e1539de7..4226d7cc2 100644 --- a/go.sum +++ b/go.sum @@ -1030,8 +1030,6 @@ github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw= github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs= -github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788 h1:O+/N9vxhoObjuCuQczycuzdG240SoLrrdnyipJ5JJc0= -github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8 h1:+8oaaj9Gm/yJFvR09Mz4iefX7pSYcFw6d6fMfvr1Mow= github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= github.com/minio/operator v0.0.0-20210812082324-26350f153661 h1:dGAJHpfmhNukFg0M0wDqH+G1OB2YPgZCcT6uv4n9YQk= @@ -1440,6 +1438,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=