Add admin inspect Glob support (#13328)

* Add admin Glob support

Allow returning multiple files on inspect calls.

```
λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*

...

λ unzip -l inspect.5f0643b2.zip

Archive:  inspect.5f0643b2.zip
  Length      Date    Time    Name
---------  ---------- -----   ----
        0  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
      802  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
---------                     -------
     3208                     8 files
```

Using fully recursive:

```
λ  mc admin inspect local2/testbucket/nyc-taxi-data-10M.csv.zst/**

...

Archive:  inspect.79c261cb.zip
  Length      Date    Time    Name
---------  ---------- -----   ----
        0  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-03 12:50   192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-03 12:50   192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-03 12:50   192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
        0  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/
        0  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
  3439368  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
  4194816  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
      802  2021-09-09 15:56   192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
---------                     -------
601034920                     156 files

```

Furthermore allow `inspect` to do direct decode from `mc`, for example:

```
λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*|inspect -json
Output decrypted to inspect.5f0643b2.zip
```

- Correct error, forward non-EOF errors.
- Add some extra safety. Log FNF when no files.
- Add `xl-meta` zip support.
For `xl-meta` multiple inputs output object with names as key.
Automatically switches `xl-meta` to single-line output when multiple objects.
Add double-star wildcard support to xl-meta input.

Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Klaus Post
2021-10-01 11:50:00 -07:00
committed by GitHub
parent 7203d93fb3
commit bc6067d195
20 changed files with 379 additions and 221 deletions

View File

@@ -38,7 +38,7 @@ import (
"strings"
"time"
humanize "github.com/dustin/go-humanize"
"github.com/dustin/go-humanize"
"github.com/gorilla/mux"
"github.com/klauspost/compress/zip"
"github.com/minio/madmin-go"
@@ -2144,7 +2144,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
// getRawDataer provides an interface for getting raw FS files.
type getRawDataer interface {
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error
}
// InspectDataHandler - GET /minio/admin/v3/inspect-data
@@ -2177,6 +2177,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
}
file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
// Reject attempts to traverse parent or absolute paths.
if strings.Contains(file, "..") || strings.Contains(volume, "..") {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
return
}
var key [32]byte
// MUST use crypto/rand
@@ -2214,15 +2221,19 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
zipWriter := zip.NewWriter(encw)
defer zipWriter.Close()
err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time, isDir bool) error {
// Prefix host+disk
filename = path.Join(host, disk, filename)
if isDir {
filename += "/"
size = 0
}
header, zerr := zip.FileInfoHeader(dummyFileInfo{
name: filename,
size: size,
mode: 0600,
modTime: modtime,
isDir: false,
isDir: isDir,
sys: nil,
})
if zerr != nil {

View File

@@ -421,7 +421,7 @@ func TestHealObjectCorrupted(t *testing.T) {
t.Fatalf("Failed to getLatestFileInfo - %v", err)
}
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
}
@@ -565,7 +565,7 @@ func TestHealObjectErasure(t *testing.T) {
t.Fatalf("Failed to heal object - %v", err)
}
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
}

View File

@@ -18,6 +18,7 @@
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
@@ -150,33 +151,45 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
// GetRawData will return all files with a given raw path to the callback.
// Errors are ignored, only errors from the callback are returned.
// For now only direct file paths are supported.
func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
found := 0
for _, s := range z.serverPools {
for _, disks := range s.erasureDisks {
for i, disk := range disks {
if disk == OfflineDisk {
continue
}
si, err := disk.StatInfoFile(ctx, volume, file)
stats, err := disk.StatInfoFile(ctx, volume, file, true)
if err != nil {
continue
}
r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
if err != nil {
continue
}
defer r.Close()
did, err := disk.GetDiskID()
if err != nil {
did = fmt.Sprintf("disk-%d", i)
}
err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
if err != nil {
return err
for _, si := range stats {
found++
var r io.ReadCloser
if !si.Dir {
r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size)
if err != nil {
continue
}
} else {
r = io.NopCloser(bytes.NewBuffer([]byte{}))
}
err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si.Size, si.ModTime, si.Dir)
r.Close()
if err != nil {
return err
}
}
}
}
}
if found == 0 {
return errFileNotFound
}
return nil
}

View File

@@ -1508,7 +1508,7 @@ func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, obje
// GetRawData returns raw file data to the callback.
// Errors are ignored, only errors from the callback are returned.
// For now only direct file paths are supported.
func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
if err != nil {
return nil
@@ -1518,5 +1518,5 @@ func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn fun
if err != nil || st.IsDir() {
return nil
}
return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime(), st.IsDir())
}

View File

@@ -285,9 +285,9 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi
return d.disk.VerifyFile(ctx, volume, path, fi)
}
func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
if err := d.calcError(); err != nil {
return stat, err
}
return d.disk.StatInfoFile(ctx, volume, path)
return d.disk.StatInfoFile(ctx, volume, path, glob)
}

View File

@@ -73,7 +73,7 @@ type StorageAPI interface {
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)
StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
// Write all data, syncs the data to disk.
// Should be used for smaller payloads.

View File

@@ -23,6 +23,7 @@ import (
"encoding/gob"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
@@ -680,10 +681,11 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
return toStorageErr(verifyResp.Err)
}
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
values := make(url.Values)
values.Set(storageRESTVolume, volume)
values.Set(storageRESTFilePath, path)
values.Set(storageRESTGlob, fmt.Sprint(glob))
respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
if err != nil {
return stat, err
@@ -693,7 +695,19 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
if err != nil {
return stat, err
}
err = stat.DecodeMsg(msgpNewReader(respReader))
rd := msgpNewReader(respReader)
for {
var st StatInfo
err = st.DecodeMsg(rd)
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
break
}
stat = append(stat, st)
}
return stat, err
}

View File

@@ -78,4 +78,5 @@ const (
storageRESTBitrotHash = "bitrot-hash"
storageRESTDiskID = "disk-id"
storageRESTForceDelete = "force-delete"
storageRESTGlob = "glob"
)

View File

@@ -1122,13 +1122,16 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request)
vars := mux.Vars(r)
volume := vars[storageRESTVolume]
filePath := vars[storageRESTFilePath]
glob := vars[storageRESTGlob]
done := keepHTTPResponseAlive(w)
si, err := s.storage.StatInfoFile(r.Context(), volume, filePath)
stats, err := s.storage.StatInfoFile(r.Context(), volume, filePath, glob == "true")
done(err)
if err != nil {
return
}
msgp.Encode(w, &si)
for _, si := range stats {
msgp.Encode(w, &si)
}
}
// registerStorageRPCRouter - register storage rpc router.
@@ -1221,7 +1224,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTGlob)...)
}
}
}

View File

@@ -186,11 +186,11 @@ func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) {
}
for i, testCase := range testCases {
_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile)
_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile, false)
expectErr := (err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
t.Fatalf("case %v: error: expected: %v, got: %v, err: %v", i+1, expectErr, testCase.expectErr, err)
}
}
}

View File

@@ -547,18 +547,18 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
return p.storage.ReadAll(ctx, volume, path)
}
func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
if contextCanceled(ctx) {
return StatInfo{}, ctx.Err()
return nil, ctx.Err()
}
if err = p.checkDiskStale(); err != nil {
return StatInfo{}, err
return nil, err
}
return p.storage.StatInfoFile(ctx, volume, path)
return p.storage.StatInfoFile(ctx, volume, path, glob)
}
func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {

View File

@@ -81,6 +81,8 @@ type xlMetaV1Object struct {
type StatInfo struct {
Size int64 `json:"size"` // Size of the object `xl.meta`.
ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`.
Name string `json:"name"`
Dir bool `json:"dir"`
}
// ErasureInfo holds erasure coding and bitrot related information.

View File

@@ -759,6 +759,18 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "ModTime")
return
}
case "Name":
z.Name, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
case "Dir":
z.Dir, err = dc.ReadBool()
if err != nil {
err = msgp.WrapError(err, "Dir")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -771,10 +783,10 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
}
// EncodeMsg implements msgp.Encodable
func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
func (z *StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 4
// write "Size"
err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
err = en.Append(0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
if err != nil {
return
}
@@ -793,19 +805,45 @@ func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "ModTime")
return
}
// write "Name"
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteString(z.Name)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
// write "Dir"
err = en.Append(0xa3, 0x44, 0x69, 0x72)
if err != nil {
return
}
err = en.WriteBool(z.Dir)
if err != nil {
err = msgp.WrapError(err, "Dir")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
func (z *StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
// map header, size 4
// string "Size"
o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = append(o, 0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.Size)
// string "ModTime"
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendTime(o, z.ModTime)
// string "Name"
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = msgp.AppendString(o, z.Name)
// string "Dir"
o = append(o, 0xa3, 0x44, 0x69, 0x72)
o = msgp.AppendBool(o, z.Dir)
return
}
@@ -839,6 +877,18 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "ModTime")
return
}
case "Name":
z.Name, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Name")
return
}
case "Dir":
z.Dir, bts, err = msgp.ReadBoolBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Dir")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -852,8 +902,8 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z StatInfo) Msgsize() (s int) {
s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize
func (z *StatInfo) Msgsize() (s int) {
s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.Name) + 4 + msgp.BoolSize
return
}
@@ -1041,40 +1091,11 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
case "Stat":
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
err = z.Stat.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
for zb0002 > 0 {
zb0002--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
switch msgp.UnsafeString(field) {
case "Size":
z.Stat.Size, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "Stat", "Size")
return
}
case "ModTime":
z.Stat.ModTime, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "Stat", "ModTime")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
}
}
case "Erasure":
err = z.Erasure.DecodeMsg(dc)
if err != nil {
@@ -1082,14 +1103,14 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
case "Minio":
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Minio")
return
}
for zb0003 > 0 {
zb0003--
for zb0002 > 0 {
zb0002--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "Minio")
@@ -1111,21 +1132,21 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
}
}
case "Meta":
var zb0004 uint32
zb0004, err = dc.ReadMapHeader()
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Meta")
return
}
if z.Meta == nil {
z.Meta = make(map[string]string, zb0004)
z.Meta = make(map[string]string, zb0003)
} else if len(z.Meta) > 0 {
for key := range z.Meta {
delete(z.Meta, key)
}
}
for zb0004 > 0 {
zb0004--
for zb0003 > 0 {
zb0003--
var za0001 string
var za0002 string
za0001, err = dc.ReadString()
@@ -1141,16 +1162,16 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
z.Meta[za0001] = za0002
}
case "Parts":
var zb0005 uint32
zb0005, err = dc.ReadArrayHeader()
var zb0004 uint32
zb0004, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "Parts")
return
}
if cap(z.Parts) >= int(zb0005) {
z.Parts = (z.Parts)[:zb0005]
if cap(z.Parts) >= int(zb0004) {
z.Parts = (z.Parts)[:zb0004]
} else {
z.Parts = make([]ObjectPartInfo, zb0005)
z.Parts = make([]ObjectPartInfo, zb0004)
}
for za0003 := range z.Parts {
err = z.Parts[za0003].DecodeMsg(dc)
@@ -1210,25 +1231,9 @@ func (z *xlMetaV1Object) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil {
return
}
// map header, size 2
// write "Size"
err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
err = z.Stat.EncodeMsg(en)
if err != nil {
return
}
err = en.WriteInt64(z.Stat.Size)
if err != nil {
err = msgp.WrapError(err, "Stat", "Size")
return
}
// write "ModTime"
err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteTime(z.Stat.ModTime)
if err != nil {
err = msgp.WrapError(err, "Stat", "ModTime")
err = msgp.WrapError(err, "Stat")
return
}
// write "Erasure"
@@ -1331,13 +1336,11 @@ func (z *xlMetaV1Object) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.AppendString(o, z.Format)
// string "Stat"
o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74)
// map header, size 2
// string "Size"
o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.Stat.Size)
// string "ModTime"
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendTime(o, z.Stat.ModTime)
o, err = z.Stat.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
// string "Erasure"
o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65)
o, err = z.Erasure.MarshalMsg(o)
@@ -1408,40 +1411,11 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
case "Stat":
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
bts, err = z.Stat.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
for zb0002 > 0 {
zb0002--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
switch msgp.UnsafeString(field) {
case "Size":
z.Stat.Size, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Stat", "Size")
return
}
case "ModTime":
z.Stat.ModTime, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Stat", "ModTime")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, "Stat")
return
}
}
}
case "Erasure":
bts, err = z.Erasure.UnmarshalMsg(bts)
if err != nil {
@@ -1449,14 +1423,14 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
case "Minio":
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Minio")
return
}
for zb0003 > 0 {
zb0003--
for zb0002 > 0 {
zb0002--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "Minio")
@@ -1478,23 +1452,23 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
}
case "Meta":
var zb0004 uint32
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Meta")
return
}
if z.Meta == nil {
z.Meta = make(map[string]string, zb0004)
z.Meta = make(map[string]string, zb0003)
} else if len(z.Meta) > 0 {
for key := range z.Meta {
delete(z.Meta, key)
}
}
for zb0004 > 0 {
for zb0003 > 0 {
var za0001 string
var za0002 string
zb0004--
zb0003--
za0001, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Meta")
@@ -1508,16 +1482,16 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
z.Meta[za0001] = za0002
}
case "Parts":
var zb0005 uint32
zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts)
var zb0004 uint32
zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Parts")
return
}
if cap(z.Parts) >= int(zb0005) {
z.Parts = (z.Parts)[:zb0005]
if cap(z.Parts) >= int(zb0004) {
z.Parts = (z.Parts)[:zb0004]
} else {
z.Parts = make([]ObjectPartInfo, zb0005)
z.Parts = make([]ObjectPartInfo, zb0004)
}
for za0003 := range z.Parts {
bts, err = z.Parts[za0003].UnmarshalMsg(bts)
@@ -1552,7 +1526,7 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV1Object) Msgsize() (s int) {
s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + z.Stat.Msgsize() + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
if z.Meta != nil {
for za0001, za0002 := range z.Meta {
_ = za0002

View File

@@ -46,6 +46,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/console"
"github.com/minio/pkg/env"
"github.com/yargevad/filepathx"
)
const (
@@ -1132,7 +1133,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
// Check the data path if there is a part with data.
partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
dataPath := pathJoin(path, fi.DataDir, partPath)
_, err = s.StatInfoFile(ctx, volume, dataPath)
_, err = s.StatInfoFile(ctx, volume, dataPath, false)
if err != nil {
// Set the inline header, our inlined data is fine.
fi.SetInlineData()
@@ -2204,7 +2205,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
return nil
}
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
volumeDir, err := s.getVolDir(volume)
if err != nil {
return stat, err
@@ -2221,14 +2222,29 @@ func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat
}
return stat, err
}
filePath := pathJoin(volumeDir, path)
if err := checkPathLength(filePath); err != nil {
return stat, err
var files = []string{pathJoin(volumeDir, path)}
if glob {
files, err = filepathx.Glob(pathJoin(volumeDir, path))
if err != nil {
return nil, err
}
}
st, _ := Lstat(filePath)
if st == nil {
return stat, errPathNotFound
for _, filePath := range files {
if err := checkPathLength(filePath); err != nil {
return stat, err
}
st, _ := Lstat(filePath)
if st == nil {
return stat, errPathNotFound
}
name, err := filepath.Rel(volumeDir, filePath)
if err != nil {
name = filePath
}
if os.PathSeparator != '/' {
name = strings.Replace(name, string(os.PathSeparator), "/", -1)
}
stat = append(stat, StatInfo{ModTime: st.ModTime(), Size: st.Size(), Name: name, Dir: st.IsDir()})
}
return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil
return stat, nil
}

View File

@@ -1711,7 +1711,7 @@ func TestXLStorageStatInfoFile(t *testing.T) {
}
for i, testCase := range testCases {
_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile)
_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile, false)
if err != testCase.expectedErr {
t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
}

View File

@@ -114,7 +114,7 @@ func TestIsValidUmaskFile(t *testing.T) {
}
// CheckFile - stat the file.
if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile); err != nil {
if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile, false); err != nil {
t.Fatalf("Stat failed with %s expected to pass.", err)
}
}