mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add admin inspect Glob support (#13328)
* Add admin Glob support
Allow returning multiple files on inspect calls.
```
λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*
...
λ unzip -l inspect.5f0643b2.zip
Archive: inspect.5f0643b2.zip
Length Date Time Name
--------- ---------- ----- ----
0 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
802 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
0 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
802 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
0 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
802 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
0 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
802 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
--------- -------
3208 8 files
```
Using fully recursive:
```
λ mc admin inspect local2/testbucket/nyc-taxi-data-10M.csv.zst/**
...
Archive: inspect.79c261cb.zip
Length Date Time Name
--------- ---------- ----- ----
0 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/
0 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
3439368 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
4194816 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
802 2021-09-03 12:50 192.168.1.78:9001/a221edde-48fe-45f5-ad32-3bc7131c7659/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
0 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/
0 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
3439368 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
4194816 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
802 2021-09-03 12:50 192.168.1.78:9001/cb7440ef-f0d9-42a8-b137-f00f519276ca/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
0 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/
0 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
3439368 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
4194816 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
802 2021-09-03 12:50 192.168.1.78:9001/759cd5ac-7860-4cf3-acad-a375fcbae338/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
0 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/
0 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.1
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.10
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.11
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.12
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.13
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.14
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.15
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.16
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.17
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.18
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.19
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.2
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.20
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.21
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.22
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.23
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.24
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.25
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.26
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.27
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.28
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.29
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.3
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.30
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.31
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.32
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.33
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.34
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.35
3439368 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.36
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.4
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.5
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.6
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.7
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.8
4194816 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/18a50b3e-3c56-418e-a045-ad5c58c1d44b/part.9
802 2021-09-09 15:56 192.168.1.78:9001/2b48619c-c2fa-4e69-839e-58fc82c1b43e/testbucket/nyc-taxi-data-10M.csv.zst/xl.meta
--------- -------
601034920 156 files
```
Furthermore allow `inspect` to do direct decode from `mc`, for example:
```
λ mc admin inspect --json local2/testbucket/nyc-taxi-data-10M.csv.zst/*|inspect -json
Output decrypted to inspect.5f0643b2.zip
```
- Correct error, forward non-EOF errors.
- Add some extra safety. Log FNF when no files.
- Add `xl-meta` zip support.
For `xl-meta` multiple inputs output object with names as key.
Automatically switches `xl-meta` to single-line output when multiple objects.
Add double-star wildcard support to xl-meta input.
Co-authored-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Klaus Post
GitHub
parent
7203d93fb3
commit
bc6067d195
@ -38,7 +38,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
humanize "github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/klauspost/compress/zip"
|
"github.com/klauspost/compress/zip"
|
||||||
"github.com/minio/madmin-go"
|
"github.com/minio/madmin-go"
|
||||||
@ -2144,7 +2144,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
|
|||||||
|
|
||||||
// getRawDataer provides an interface for getting raw FS files.
|
// getRawDataer provides an interface for getting raw FS files.
|
||||||
type getRawDataer interface {
|
type getRawDataer interface {
|
||||||
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
|
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectDataHandler - GET /minio/admin/v3/inspect-data
|
// InspectDataHandler - GET /minio/admin/v3/inspect-data
|
||||||
@ -2177,6 +2177,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
|
||||||
|
|
||||||
|
// Reject attempts to traverse parent or absolute paths.
|
||||||
|
if strings.Contains(file, "..") || strings.Contains(volume, "..") {
|
||||||
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
var key [32]byte
|
var key [32]byte
|
||||||
// MUST use crypto/rand
|
// MUST use crypto/rand
|
||||||
@ -2214,15 +2221,19 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
zipWriter := zip.NewWriter(encw)
|
zipWriter := zip.NewWriter(encw)
|
||||||
defer zipWriter.Close()
|
defer zipWriter.Close()
|
||||||
|
|
||||||
err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
|
err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time, isDir bool) error {
|
||||||
// Prefix host+disk
|
// Prefix host+disk
|
||||||
filename = path.Join(host, disk, filename)
|
filename = path.Join(host, disk, filename)
|
||||||
|
if isDir {
|
||||||
|
filename += "/"
|
||||||
|
size = 0
|
||||||
|
}
|
||||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||||
name: filename,
|
name: filename,
|
||||||
size: size,
|
size: size,
|
||||||
mode: 0600,
|
mode: 0600,
|
||||||
modTime: modtime,
|
modTime: modtime,
|
||||||
isDir: false,
|
isDir: isDir,
|
||||||
sys: nil,
|
sys: nil,
|
||||||
})
|
})
|
||||||
if zerr != nil {
|
if zerr != nil {
|
||||||
|
|||||||
@ -421,7 +421,7 @@ func TestHealObjectCorrupted(t *testing.T) {
|
|||||||
t.Fatalf("Failed to getLatestFileInfo - %v", err)
|
t.Fatalf("Failed to getLatestFileInfo - %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
|
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
|
||||||
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
|
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -565,7 +565,7 @@ func TestHealObjectErasure(t *testing.T) {
|
|||||||
t.Fatalf("Failed to heal object - %v", err)
|
t.Fatalf("Failed to heal object - %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
|
if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
|
||||||
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
|
t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -18,6 +18,7 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -150,33 +151,45 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
|
|||||||
// GetRawData will return all files with a given raw path to the callback.
|
// GetRawData will return all files with a given raw path to the callback.
|
||||||
// Errors are ignored, only errors from the callback are returned.
|
// Errors are ignored, only errors from the callback are returned.
|
||||||
// For now only direct file paths are supported.
|
// For now only direct file paths are supported.
|
||||||
func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
|
func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
|
||||||
|
found := 0
|
||||||
for _, s := range z.serverPools {
|
for _, s := range z.serverPools {
|
||||||
for _, disks := range s.erasureDisks {
|
for _, disks := range s.erasureDisks {
|
||||||
for i, disk := range disks {
|
for i, disk := range disks {
|
||||||
if disk == OfflineDisk {
|
if disk == OfflineDisk {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
si, err := disk.StatInfoFile(ctx, volume, file)
|
stats, err := disk.StatInfoFile(ctx, volume, file, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
defer r.Close()
|
|
||||||
did, err := disk.GetDiskID()
|
did, err := disk.GetDiskID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
did = fmt.Sprintf("disk-%d", i)
|
did = fmt.Sprintf("disk-%d", i)
|
||||||
}
|
}
|
||||||
err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
|
for _, si := range stats {
|
||||||
if err != nil {
|
found++
|
||||||
return err
|
var r io.ReadCloser
|
||||||
|
if !si.Dir {
|
||||||
|
r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
r = io.NopCloser(bytes.NewBuffer([]byte{}))
|
||||||
|
}
|
||||||
|
err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si.Size, si.ModTime, si.Dir)
|
||||||
|
r.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if found == 0 {
|
||||||
|
return errFileNotFound
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1508,7 +1508,7 @@ func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, obje
|
|||||||
// GetRawData returns raw file data to the callback.
|
// GetRawData returns raw file data to the callback.
|
||||||
// Errors are ignored, only errors from the callback are returned.
|
// Errors are ignored, only errors from the callback are returned.
|
||||||
// For now only direct file paths are supported.
|
// For now only direct file paths are supported.
|
||||||
func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
|
func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
|
||||||
f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
|
f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
@ -1518,5 +1518,5 @@ func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn fun
|
|||||||
if err != nil || st.IsDir() {
|
if err != nil || st.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
|
return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime(), st.IsDir())
|
||||||
}
|
}
|
||||||
|
|||||||
@ -285,9 +285,9 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi
|
|||||||
return d.disk.VerifyFile(ctx, volume, path, fi)
|
return d.disk.VerifyFile(ctx, volume, path, fi)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
|
func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
|
||||||
if err := d.calcError(); err != nil {
|
if err := d.calcError(); err != nil {
|
||||||
return stat, err
|
return stat, err
|
||||||
}
|
}
|
||||||
return d.disk.StatInfoFile(ctx, volume, path)
|
return d.disk.StatInfoFile(ctx, volume, path, glob)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -73,7 +73,7 @@ type StorageAPI interface {
|
|||||||
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
|
CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
|
||||||
Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
|
Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
|
||||||
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
|
VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
|
||||||
StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)
|
StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
|
||||||
|
|
||||||
// Write all data, syncs the data to disk.
|
// Write all data, syncs the data to disk.
|
||||||
// Should be used for smaller payloads.
|
// Should be used for smaller payloads.
|
||||||
|
|||||||
@ -23,6 +23,7 @@ import (
|
|||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net/url"
|
"net/url"
|
||||||
@ -680,10 +681,11 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
|
|||||||
return toStorageErr(verifyResp.Err)
|
return toStorageErr(verifyResp.Err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
|
func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
|
||||||
values := make(url.Values)
|
values := make(url.Values)
|
||||||
values.Set(storageRESTVolume, volume)
|
values.Set(storageRESTVolume, volume)
|
||||||
values.Set(storageRESTFilePath, path)
|
values.Set(storageRESTFilePath, path)
|
||||||
|
values.Set(storageRESTGlob, fmt.Sprint(glob))
|
||||||
respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
|
respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return stat, err
|
return stat, err
|
||||||
@ -693,7 +695,19 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return stat, err
|
return stat, err
|
||||||
}
|
}
|
||||||
err = stat.DecodeMsg(msgpNewReader(respReader))
|
rd := msgpNewReader(respReader)
|
||||||
|
for {
|
||||||
|
var st StatInfo
|
||||||
|
err = st.DecodeMsg(rd)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
err = nil
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
stat = append(stat, st)
|
||||||
|
}
|
||||||
|
|
||||||
return stat, err
|
return stat, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -78,4 +78,5 @@ const (
|
|||||||
storageRESTBitrotHash = "bitrot-hash"
|
storageRESTBitrotHash = "bitrot-hash"
|
||||||
storageRESTDiskID = "disk-id"
|
storageRESTDiskID = "disk-id"
|
||||||
storageRESTForceDelete = "force-delete"
|
storageRESTForceDelete = "force-delete"
|
||||||
|
storageRESTGlob = "glob"
|
||||||
)
|
)
|
||||||
|
|||||||
@ -1122,13 +1122,16 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request)
|
|||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
volume := vars[storageRESTVolume]
|
volume := vars[storageRESTVolume]
|
||||||
filePath := vars[storageRESTFilePath]
|
filePath := vars[storageRESTFilePath]
|
||||||
|
glob := vars[storageRESTGlob]
|
||||||
done := keepHTTPResponseAlive(w)
|
done := keepHTTPResponseAlive(w)
|
||||||
si, err := s.storage.StatInfoFile(r.Context(), volume, filePath)
|
stats, err := s.storage.StatInfoFile(r.Context(), volume, filePath, glob == "true")
|
||||||
done(err)
|
done(err)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
msgp.Encode(w, &si)
|
for _, si := range stats {
|
||||||
|
msgp.Encode(w, &si)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// registerStorageRPCRouter - register storage rpc router.
|
// registerStorageRPCRouter - register storage rpc router.
|
||||||
@ -1221,7 +1224,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
|
|||||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
|
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
|
||||||
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
|
Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
|
||||||
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
|
subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
|
||||||
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
|
Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTGlob)...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -186,11 +186,11 @@ func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile)
|
_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile, false)
|
||||||
expectErr := (err != nil)
|
expectErr := (err != nil)
|
||||||
|
|
||||||
if expectErr != testCase.expectErr {
|
if expectErr != testCase.expectErr {
|
||||||
t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
t.Fatalf("case %v: error: expected: %v, got: %v, err: %v", i+1, expectErr, testCase.expectErr, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -547,18 +547,18 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
|
|||||||
return p.storage.ReadAll(ctx, volume, path)
|
return p.storage.ReadAll(ctx, volume, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
|
func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
|
||||||
defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
|
defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
|
||||||
|
|
||||||
if contextCanceled(ctx) {
|
if contextCanceled(ctx) {
|
||||||
return StatInfo{}, ctx.Err()
|
return nil, ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = p.checkDiskStale(); err != nil {
|
if err = p.checkDiskStale(); err != nil {
|
||||||
return StatInfo{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return p.storage.StatInfoFile(ctx, volume, path)
|
return p.storage.StatInfoFile(ctx, volume, path, glob)
|
||||||
}
|
}
|
||||||
|
|
||||||
func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {
|
func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {
|
||||||
|
|||||||
@ -81,6 +81,8 @@ type xlMetaV1Object struct {
|
|||||||
type StatInfo struct {
|
type StatInfo struct {
|
||||||
Size int64 `json:"size"` // Size of the object `xl.meta`.
|
Size int64 `json:"size"` // Size of the object `xl.meta`.
|
||||||
ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`.
|
ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`.
|
||||||
|
Name string `json:"name"`
|
||||||
|
Dir bool `json:"dir"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErasureInfo holds erasure coding and bitrot related information.
|
// ErasureInfo holds erasure coding and bitrot related information.
|
||||||
|
|||||||
@ -759,6 +759,18 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
err = msgp.WrapError(err, "ModTime")
|
err = msgp.WrapError(err, "ModTime")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
case "Name":
|
||||||
|
z.Name, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Dir":
|
||||||
|
z.Dir, err = dc.ReadBool()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Dir")
|
||||||
|
return
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
err = dc.Skip()
|
err = dc.Skip()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -771,10 +783,10 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// EncodeMsg implements msgp.Encodable
|
// EncodeMsg implements msgp.Encodable
|
||||||
func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
func (z *StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
// map header, size 2
|
// map header, size 4
|
||||||
// write "Size"
|
// write "Size"
|
||||||
err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
err = en.Append(0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -793,19 +805,45 @@ func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
|||||||
err = msgp.WrapError(err, "ModTime")
|
err = msgp.WrapError(err, "ModTime")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// write "Name"
|
||||||
|
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Name)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Dir"
|
||||||
|
err = en.Append(0xa3, 0x44, 0x69, 0x72)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteBool(z.Dir)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Dir")
|
||||||
|
return
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalMsg implements msgp.Marshaler
|
// MarshalMsg implements msgp.Marshaler
|
||||||
func (z StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
func (z *StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
o = msgp.Require(b, z.Msgsize())
|
o = msgp.Require(b, z.Msgsize())
|
||||||
// map header, size 2
|
// map header, size 4
|
||||||
// string "Size"
|
// string "Size"
|
||||||
o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
o = append(o, 0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||||
o = msgp.AppendInt64(o, z.Size)
|
o = msgp.AppendInt64(o, z.Size)
|
||||||
// string "ModTime"
|
// string "ModTime"
|
||||||
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
|
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
|
||||||
o = msgp.AppendTime(o, z.ModTime)
|
o = msgp.AppendTime(o, z.ModTime)
|
||||||
|
// string "Name"
|
||||||
|
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||||
|
o = msgp.AppendString(o, z.Name)
|
||||||
|
// string "Dir"
|
||||||
|
o = append(o, 0xa3, 0x44, 0x69, 0x72)
|
||||||
|
o = msgp.AppendBool(o, z.Dir)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -839,6 +877,18 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
err = msgp.WrapError(err, "ModTime")
|
err = msgp.WrapError(err, "ModTime")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
case "Name":
|
||||||
|
z.Name, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Dir":
|
||||||
|
z.Dir, bts, err = msgp.ReadBoolBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Dir")
|
||||||
|
return
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
bts, err = msgp.Skip(bts)
|
bts, err = msgp.Skip(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -852,8 +902,8 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
func (z StatInfo) Msgsize() (s int) {
|
func (z *StatInfo) Msgsize() (s int) {
|
||||||
s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize
|
s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.Name) + 4 + msgp.BoolSize
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1041,40 +1091,11 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
case "Stat":
|
case "Stat":
|
||||||
var zb0002 uint32
|
err = z.Stat.DecodeMsg(dc)
|
||||||
zb0002, err = dc.ReadMapHeader()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Stat")
|
err = msgp.WrapError(err, "Stat")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for zb0002 > 0 {
|
|
||||||
zb0002--
|
|
||||||
field, err = dc.ReadMapKeyPtr()
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch msgp.UnsafeString(field) {
|
|
||||||
case "Size":
|
|
||||||
z.Stat.Size, err = dc.ReadInt64()
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat", "Size")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case "ModTime":
|
|
||||||
z.Stat.ModTime, err = dc.ReadTime()
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat", "ModTime")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
err = dc.Skip()
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case "Erasure":
|
case "Erasure":
|
||||||
err = z.Erasure.DecodeMsg(dc)
|
err = z.Erasure.DecodeMsg(dc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1082,14 +1103,14 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
case "Minio":
|
case "Minio":
|
||||||
var zb0003 uint32
|
var zb0002 uint32
|
||||||
zb0003, err = dc.ReadMapHeader()
|
zb0002, err = dc.ReadMapHeader()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Minio")
|
err = msgp.WrapError(err, "Minio")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for zb0003 > 0 {
|
for zb0002 > 0 {
|
||||||
zb0003--
|
zb0002--
|
||||||
field, err = dc.ReadMapKeyPtr()
|
field, err = dc.ReadMapKeyPtr()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Minio")
|
err = msgp.WrapError(err, "Minio")
|
||||||
@ -1111,21 +1132,21 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
case "Meta":
|
case "Meta":
|
||||||
var zb0004 uint32
|
var zb0003 uint32
|
||||||
zb0004, err = dc.ReadMapHeader()
|
zb0003, err = dc.ReadMapHeader()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Meta")
|
err = msgp.WrapError(err, "Meta")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if z.Meta == nil {
|
if z.Meta == nil {
|
||||||
z.Meta = make(map[string]string, zb0004)
|
z.Meta = make(map[string]string, zb0003)
|
||||||
} else if len(z.Meta) > 0 {
|
} else if len(z.Meta) > 0 {
|
||||||
for key := range z.Meta {
|
for key := range z.Meta {
|
||||||
delete(z.Meta, key)
|
delete(z.Meta, key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for zb0004 > 0 {
|
for zb0003 > 0 {
|
||||||
zb0004--
|
zb0003--
|
||||||
var za0001 string
|
var za0001 string
|
||||||
var za0002 string
|
var za0002 string
|
||||||
za0001, err = dc.ReadString()
|
za0001, err = dc.ReadString()
|
||||||
@ -1141,16 +1162,16 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
z.Meta[za0001] = za0002
|
z.Meta[za0001] = za0002
|
||||||
}
|
}
|
||||||
case "Parts":
|
case "Parts":
|
||||||
var zb0005 uint32
|
var zb0004 uint32
|
||||||
zb0005, err = dc.ReadArrayHeader()
|
zb0004, err = dc.ReadArrayHeader()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Parts")
|
err = msgp.WrapError(err, "Parts")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cap(z.Parts) >= int(zb0005) {
|
if cap(z.Parts) >= int(zb0004) {
|
||||||
z.Parts = (z.Parts)[:zb0005]
|
z.Parts = (z.Parts)[:zb0004]
|
||||||
} else {
|
} else {
|
||||||
z.Parts = make([]ObjectPartInfo, zb0005)
|
z.Parts = make([]ObjectPartInfo, zb0004)
|
||||||
}
|
}
|
||||||
for za0003 := range z.Parts {
|
for za0003 := range z.Parts {
|
||||||
err = z.Parts[za0003].DecodeMsg(dc)
|
err = z.Parts[za0003].DecodeMsg(dc)
|
||||||
@ -1210,25 +1231,9 @@ func (z *xlMetaV1Object) EncodeMsg(en *msgp.Writer) (err error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// map header, size 2
|
err = z.Stat.EncodeMsg(en)
|
||||||
// write "Size"
|
|
||||||
err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
err = msgp.WrapError(err, "Stat")
|
||||||
}
|
|
||||||
err = en.WriteInt64(z.Stat.Size)
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat", "Size")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// write "ModTime"
|
|
||||||
err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
err = en.WriteTime(z.Stat.ModTime)
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat", "ModTime")
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// write "Erasure"
|
// write "Erasure"
|
||||||
@ -1331,13 +1336,11 @@ func (z *xlMetaV1Object) MarshalMsg(b []byte) (o []byte, err error) {
|
|||||||
o = msgp.AppendString(o, z.Format)
|
o = msgp.AppendString(o, z.Format)
|
||||||
// string "Stat"
|
// string "Stat"
|
||||||
o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74)
|
o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74)
|
||||||
// map header, size 2
|
o, err = z.Stat.MarshalMsg(o)
|
||||||
// string "Size"
|
if err != nil {
|
||||||
o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
err = msgp.WrapError(err, "Stat")
|
||||||
o = msgp.AppendInt64(o, z.Stat.Size)
|
return
|
||||||
// string "ModTime"
|
}
|
||||||
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
|
|
||||||
o = msgp.AppendTime(o, z.Stat.ModTime)
|
|
||||||
// string "Erasure"
|
// string "Erasure"
|
||||||
o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65)
|
o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65)
|
||||||
o, err = z.Erasure.MarshalMsg(o)
|
o, err = z.Erasure.MarshalMsg(o)
|
||||||
@ -1408,40 +1411,11 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
case "Stat":
|
case "Stat":
|
||||||
var zb0002 uint32
|
bts, err = z.Stat.UnmarshalMsg(bts)
|
||||||
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Stat")
|
err = msgp.WrapError(err, "Stat")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for zb0002 > 0 {
|
|
||||||
zb0002--
|
|
||||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch msgp.UnsafeString(field) {
|
|
||||||
case "Size":
|
|
||||||
z.Stat.Size, bts, err = msgp.ReadInt64Bytes(bts)
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat", "Size")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case "ModTime":
|
|
||||||
z.Stat.ModTime, bts, err = msgp.ReadTimeBytes(bts)
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat", "ModTime")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
bts, err = msgp.Skip(bts)
|
|
||||||
if err != nil {
|
|
||||||
err = msgp.WrapError(err, "Stat")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case "Erasure":
|
case "Erasure":
|
||||||
bts, err = z.Erasure.UnmarshalMsg(bts)
|
bts, err = z.Erasure.UnmarshalMsg(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1449,14 +1423,14 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
case "Minio":
|
case "Minio":
|
||||||
var zb0003 uint32
|
var zb0002 uint32
|
||||||
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
|
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Minio")
|
err = msgp.WrapError(err, "Minio")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for zb0003 > 0 {
|
for zb0002 > 0 {
|
||||||
zb0003--
|
zb0002--
|
||||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Minio")
|
err = msgp.WrapError(err, "Minio")
|
||||||
@ -1478,23 +1452,23 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
case "Meta":
|
case "Meta":
|
||||||
var zb0004 uint32
|
var zb0003 uint32
|
||||||
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Meta")
|
err = msgp.WrapError(err, "Meta")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if z.Meta == nil {
|
if z.Meta == nil {
|
||||||
z.Meta = make(map[string]string, zb0004)
|
z.Meta = make(map[string]string, zb0003)
|
||||||
} else if len(z.Meta) > 0 {
|
} else if len(z.Meta) > 0 {
|
||||||
for key := range z.Meta {
|
for key := range z.Meta {
|
||||||
delete(z.Meta, key)
|
delete(z.Meta, key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for zb0004 > 0 {
|
for zb0003 > 0 {
|
||||||
var za0001 string
|
var za0001 string
|
||||||
var za0002 string
|
var za0002 string
|
||||||
zb0004--
|
zb0003--
|
||||||
za0001, bts, err = msgp.ReadStringBytes(bts)
|
za0001, bts, err = msgp.ReadStringBytes(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Meta")
|
err = msgp.WrapError(err, "Meta")
|
||||||
@ -1508,16 +1482,16 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
z.Meta[za0001] = za0002
|
z.Meta[za0001] = za0002
|
||||||
}
|
}
|
||||||
case "Parts":
|
case "Parts":
|
||||||
var zb0005 uint32
|
var zb0004 uint32
|
||||||
zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Parts")
|
err = msgp.WrapError(err, "Parts")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cap(z.Parts) >= int(zb0005) {
|
if cap(z.Parts) >= int(zb0004) {
|
||||||
z.Parts = (z.Parts)[:zb0005]
|
z.Parts = (z.Parts)[:zb0004]
|
||||||
} else {
|
} else {
|
||||||
z.Parts = make([]ObjectPartInfo, zb0005)
|
z.Parts = make([]ObjectPartInfo, zb0004)
|
||||||
}
|
}
|
||||||
for za0003 := range z.Parts {
|
for za0003 := range z.Parts {
|
||||||
bts, err = z.Parts[za0003].UnmarshalMsg(bts)
|
bts, err = z.Parts[za0003].UnmarshalMsg(bts)
|
||||||
@ -1552,7 +1526,7 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
|
|
||||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
func (z *xlMetaV1Object) Msgsize() (s int) {
|
func (z *xlMetaV1Object) Msgsize() (s int) {
|
||||||
s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
|
s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + z.Stat.Msgsize() + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
|
||||||
if z.Meta != nil {
|
if z.Meta != nil {
|
||||||
for za0001, za0002 := range z.Meta {
|
for za0001, za0002 := range z.Meta {
|
||||||
_ = za0002
|
_ = za0002
|
||||||
|
|||||||
@ -46,6 +46,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/console"
|
"github.com/minio/pkg/console"
|
||||||
"github.com/minio/pkg/env"
|
"github.com/minio/pkg/env"
|
||||||
|
"github.com/yargevad/filepathx"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -1132,7 +1133,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
|
|||||||
// Check the data path if there is a part with data.
|
// Check the data path if there is a part with data.
|
||||||
partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
|
partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
|
||||||
dataPath := pathJoin(path, fi.DataDir, partPath)
|
dataPath := pathJoin(path, fi.DataDir, partPath)
|
||||||
_, err = s.StatInfoFile(ctx, volume, dataPath)
|
_, err = s.StatInfoFile(ctx, volume, dataPath, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Set the inline header, our inlined data is fine.
|
// Set the inline header, our inlined data is fine.
|
||||||
fi.SetInlineData()
|
fi.SetInlineData()
|
||||||
@ -2204,7 +2205,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
|
func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
|
||||||
volumeDir, err := s.getVolDir(volume)
|
volumeDir, err := s.getVolDir(volume)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return stat, err
|
return stat, err
|
||||||
@ -2221,14 +2222,29 @@ func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat
|
|||||||
}
|
}
|
||||||
return stat, err
|
return stat, err
|
||||||
}
|
}
|
||||||
filePath := pathJoin(volumeDir, path)
|
var files = []string{pathJoin(volumeDir, path)}
|
||||||
if err := checkPathLength(filePath); err != nil {
|
if glob {
|
||||||
return stat, err
|
files, err = filepathx.Glob(pathJoin(volumeDir, path))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
st, _ := Lstat(filePath)
|
for _, filePath := range files {
|
||||||
if st == nil {
|
if err := checkPathLength(filePath); err != nil {
|
||||||
return stat, errPathNotFound
|
return stat, err
|
||||||
|
}
|
||||||
|
st, _ := Lstat(filePath)
|
||||||
|
if st == nil {
|
||||||
|
return stat, errPathNotFound
|
||||||
|
}
|
||||||
|
name, err := filepath.Rel(volumeDir, filePath)
|
||||||
|
if err != nil {
|
||||||
|
name = filePath
|
||||||
|
}
|
||||||
|
if os.PathSeparator != '/' {
|
||||||
|
name = strings.Replace(name, string(os.PathSeparator), "/", -1)
|
||||||
|
}
|
||||||
|
stat = append(stat, StatInfo{ModTime: st.ModTime(), Size: st.Size(), Name: name, Dir: st.IsDir()})
|
||||||
}
|
}
|
||||||
|
return stat, nil
|
||||||
return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1711,7 +1711,7 @@ func TestXLStorageStatInfoFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile)
|
_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile, false)
|
||||||
if err != testCase.expectedErr {
|
if err != testCase.expectedErr {
|
||||||
t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -114,7 +114,7 @@ func TestIsValidUmaskFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CheckFile - stat the file.
|
// CheckFile - stat the file.
|
||||||
if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile); err != nil {
|
if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile, false); err != nil {
|
||||||
t.Fatalf("Stat failed with %s expected to pass.", err)
|
t.Fatalf("Stat failed with %s expected to pass.", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -27,10 +27,12 @@ import (
|
|||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/klauspost/compress/zip"
|
||||||
"github.com/minio/cli"
|
"github.com/minio/cli"
|
||||||
"github.com/tinylib/msgp/msgp"
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
"github.com/yargevad/filepathx"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@ -44,9 +46,10 @@ func main() {
|
|||||||
USAGE:
|
USAGE:
|
||||||
{{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES...
|
{{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES...
|
||||||
|
|
||||||
Multiple files can be added.
|
Multiple files can be added. Files ending in ".zip" will be searched for 'xl.meta' files.
|
||||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||||
|
Double stars means full recursive. testdir/**/xl.meta will search for all xl.meta recursively.
|
||||||
|
|
||||||
{{if .VisibleFlags}}
|
{{if .VisibleFlags}}
|
||||||
GLOBAL FLAGS:
|
GLOBAL FLAGS:
|
||||||
@ -72,43 +75,15 @@ GLOBAL FLAGS:
|
|||||||
}
|
}
|
||||||
|
|
||||||
app.Action = func(c *cli.Context) error {
|
app.Action = func(c *cli.Context) error {
|
||||||
args := c.Args()
|
ndjson := c.Bool("ndjson")
|
||||||
if len(args) == 0 {
|
decode := func(r io.Reader, file string) ([]byte, error) {
|
||||||
// If no args, assume xl.meta
|
|
||||||
args = []string{"xl.meta"}
|
|
||||||
}
|
|
||||||
var files []string
|
|
||||||
for _, pattern := range args {
|
|
||||||
found, err := filepath.Glob(pattern)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(found) == 0 {
|
|
||||||
return fmt.Errorf("unable to find file %v", pattern)
|
|
||||||
}
|
|
||||||
files = append(files, found...)
|
|
||||||
}
|
|
||||||
for _, file := range files {
|
|
||||||
var r io.Reader
|
|
||||||
switch file {
|
|
||||||
case "-":
|
|
||||||
r = os.Stdin
|
|
||||||
default:
|
|
||||||
f, err := os.Open(file)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
r = f
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := ioutil.ReadAll(r)
|
b, err := ioutil.ReadAll(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
b, _, minor, err := checkXL2V1(b)
|
b, _, minor, err := checkXL2V1(b)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
buf := bytes.NewBuffer(nil)
|
buf := bytes.NewBuffer(nil)
|
||||||
@ -117,12 +92,12 @@ GLOBAL FLAGS:
|
|||||||
case 0:
|
case 0:
|
||||||
_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b))
|
_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
case 1, 2:
|
case 1, 2:
|
||||||
v, b, err := msgp.ReadBytesZC(b)
|
v, b, err := msgp.ReadBytesZC(b)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
|
if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
|
||||||
// Read metadata CRC (added in v2, ignore if not found)
|
// Read metadata CRC (added in v2, ignore if not found)
|
||||||
@ -131,31 +106,47 @@ GLOBAL FLAGS:
|
|||||||
|
|
||||||
_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v))
|
_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
data = b
|
data = b
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unknown metadata version %d", minor)
|
return nil, fmt.Errorf("unknown metadata version %d", minor)
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.Bool("data") {
|
if c.Bool("data") {
|
||||||
b, err := data.json()
|
b, err := data.json()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
buf = bytes.NewBuffer(b)
|
buf = bytes.NewBuffer(b)
|
||||||
}
|
}
|
||||||
if c.Bool("export") {
|
if c.Bool("export") {
|
||||||
|
file := strings.Map(func(r rune) rune {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z':
|
||||||
|
return r
|
||||||
|
case r >= 'A' && r <= 'Z':
|
||||||
|
return r
|
||||||
|
case r >= '0' && r <= '9':
|
||||||
|
return r
|
||||||
|
case strings.ContainsAny(string(r), "+=-_()!@."):
|
||||||
|
return r
|
||||||
|
default:
|
||||||
|
return '_'
|
||||||
|
}
|
||||||
|
}, file)
|
||||||
err := data.files(func(name string, data []byte) {
|
err := data.files(func(name string, data []byte) {
|
||||||
ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm)
|
err = ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if c.Bool("ndjson") {
|
if ndjson {
|
||||||
fmt.Println(buf.String())
|
return buf.Bytes(), nil
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
var msi map[string]interface{}
|
var msi map[string]interface{}
|
||||||
dec := json.NewDecoder(buf)
|
dec := json.NewDecoder(buf)
|
||||||
@ -163,14 +154,113 @@ GLOBAL FLAGS:
|
|||||||
dec.UseNumber()
|
dec.UseNumber()
|
||||||
err = dec.Decode(&msi)
|
err = dec.Decode(&msi)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
b, err = json.MarshalIndent(msi, "", " ")
|
b, err = json.MarshalIndent(msi, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
args := c.Args()
|
||||||
|
if len(args) == 0 {
|
||||||
|
// If no args, assume xl.meta
|
||||||
|
args = []string{"xl.meta"}
|
||||||
|
}
|
||||||
|
var files []string
|
||||||
|
|
||||||
|
for _, pattern := range args {
|
||||||
|
if pattern == "-" {
|
||||||
|
files = append(files, pattern)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found, err := filepathx.Glob(pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Println(string(b))
|
if len(found) == 0 {
|
||||||
|
return fmt.Errorf("unable to find file %v", pattern)
|
||||||
|
}
|
||||||
|
files = append(files, found...)
|
||||||
}
|
}
|
||||||
|
if len(files) == 0 {
|
||||||
|
return fmt.Errorf("no files found")
|
||||||
|
}
|
||||||
|
multiple := len(files) > 1 || strings.HasSuffix(files[0], ".zip")
|
||||||
|
if multiple {
|
||||||
|
ndjson = true
|
||||||
|
fmt.Println("{")
|
||||||
|
}
|
||||||
|
|
||||||
|
hasWritten := false
|
||||||
|
for _, file := range files {
|
||||||
|
var r io.Reader
|
||||||
|
var sz int64
|
||||||
|
switch file {
|
||||||
|
case "-":
|
||||||
|
r = os.Stdin
|
||||||
|
default:
|
||||||
|
f, err := os.Open(file)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if st, err := f.Stat(); err == nil {
|
||||||
|
sz = st.Size()
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
r = f
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(file, ".zip") {
|
||||||
|
zr, err := zip.NewReader(r.(io.ReaderAt), sz)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, file := range zr.File {
|
||||||
|
if !file.FileInfo().IsDir() && strings.HasSuffix(file.Name, "xl.meta") {
|
||||||
|
r, err := file.Open()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Quote string...
|
||||||
|
b, _ := json.Marshal(file.Name)
|
||||||
|
if hasWritten {
|
||||||
|
fmt.Print(",\n")
|
||||||
|
}
|
||||||
|
fmt.Printf("\t%s: ", string(b))
|
||||||
|
|
||||||
|
b, err = decode(r, file.Name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Print(string(b))
|
||||||
|
hasWritten = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if multiple {
|
||||||
|
// Quote string...
|
||||||
|
b, _ := json.Marshal(file)
|
||||||
|
if hasWritten {
|
||||||
|
fmt.Print(",\n")
|
||||||
|
}
|
||||||
|
fmt.Printf("\t%s: ", string(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := decode(r, file)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
hasWritten = true
|
||||||
|
fmt.Print(string(b))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("")
|
||||||
|
if multiple {
|
||||||
|
fmt.Println("}")
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
err := app.Run(os.Args)
|
err := app.Run(os.Args)
|
||||||
|
|||||||
@ -21,10 +21,13 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"hash/crc32"
|
"hash/crc32"
|
||||||
"io"
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
@ -34,25 +37,48 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
key = flag.String("key", "", "decryption string")
|
key = flag.String("key", "", "decryption string")
|
||||||
//js = flag.Bool("json", false, "expect json input")
|
js = flag.Bool("json", false, "expect json input from stdin")
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
if *js {
|
||||||
|
// Match struct in https://github.com/minio/mc/blob/b3ce21fb72a914f50522358668e6464eb97de8d1/cmd/admin-inspect.go#L135
|
||||||
|
input := struct {
|
||||||
|
File string `json:"file"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
}{}
|
||||||
|
got, err := ioutil.ReadAll(os.Stdin)
|
||||||
|
if err != nil {
|
||||||
|
fatalErr(err)
|
||||||
|
}
|
||||||
|
fatalErr(json.Unmarshal(got, &input))
|
||||||
|
r, err := os.Open(input.File)
|
||||||
|
fatalErr(err)
|
||||||
|
defer r.Close()
|
||||||
|
dstName := strings.TrimSuffix(input.File, ".enc") + ".zip"
|
||||||
|
w, err := os.Create(dstName)
|
||||||
|
fatalErr(err)
|
||||||
|
defer w.Close()
|
||||||
|
decrypt(input.Key, r, w)
|
||||||
|
fmt.Println("Output decrypted to", dstName)
|
||||||
|
return
|
||||||
|
}
|
||||||
args := flag.Args()
|
args := flag.Args()
|
||||||
switch len(flag.Args()) {
|
switch len(flag.Args()) {
|
||||||
case 0:
|
case 0:
|
||||||
// Read from stdin, write to stdout.
|
// Read from stdin, write to stdout.
|
||||||
|
if *key == "" {
|
||||||
|
flag.Usage()
|
||||||
|
fatalErr(errors.New("no key supplied"))
|
||||||
|
}
|
||||||
decrypt(*key, os.Stdin, os.Stdout)
|
decrypt(*key, os.Stdin, os.Stdout)
|
||||||
return
|
return
|
||||||
case 1:
|
case 1:
|
||||||
r, err := os.Open(args[0])
|
r, err := os.Open(args[0])
|
||||||
fatalErr(err)
|
fatalErr(err)
|
||||||
defer r.Close()
|
defer r.Close()
|
||||||
dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
|
|
||||||
w, err := os.Create(dstName)
|
|
||||||
fatalErr(err)
|
|
||||||
defer w.Close()
|
|
||||||
if len(*key) == 0 {
|
if len(*key) == 0 {
|
||||||
reader := bufio.NewReader(os.Stdin)
|
reader := bufio.NewReader(os.Stdin)
|
||||||
fmt.Print("Enter Decryption Key: ")
|
fmt.Print("Enter Decryption Key: ")
|
||||||
@ -61,18 +87,25 @@ func main() {
|
|||||||
// convert CRLF to LF
|
// convert CRLF to LF
|
||||||
*key = strings.Replace(text, "\n", "", -1)
|
*key = strings.Replace(text, "\n", "", -1)
|
||||||
}
|
}
|
||||||
|
*key = strings.TrimSpace(*key)
|
||||||
|
fatalIf(len(*key) != 72, "Unexpected key length: %d, want 72", len(*key))
|
||||||
|
|
||||||
|
dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
|
||||||
|
w, err := os.Create(dstName)
|
||||||
|
fatalErr(err)
|
||||||
|
defer w.Close()
|
||||||
|
|
||||||
decrypt(*key, r, w)
|
decrypt(*key, r, w)
|
||||||
fmt.Println("Output decrypted to", dstName)
|
fmt.Println("Output decrypted to", dstName)
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
|
flag.Usage()
|
||||||
fatalIf(true, "Only 1 file can be decrypted")
|
fatalIf(true, "Only 1 file can be decrypted")
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func decrypt(keyHex string, r io.Reader, w io.Writer) {
|
func decrypt(keyHex string, r io.Reader, w io.Writer) {
|
||||||
keyHex = strings.TrimSpace(keyHex)
|
|
||||||
fatalIf(len(keyHex) != 72, "Unexpected key length: %d, want 72", len(keyHex))
|
|
||||||
id, err := hex.DecodeString(keyHex[:8])
|
id, err := hex.DecodeString(keyHex[:8])
|
||||||
fatalErr(err)
|
fatalErr(err)
|
||||||
key, err := hex.DecodeString(keyHex[8:])
|
key, err := hex.DecodeString(keyHex[8:])
|
||||||
|
|||||||
1
go.mod
1
go.mod
@ -76,6 +76,7 @@ require (
|
|||||||
github.com/valyala/bytebufferpool v1.0.0
|
github.com/valyala/bytebufferpool v1.0.0
|
||||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
|
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
|
||||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
|
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
|
||||||
|
github.com/yargevad/filepathx v1.0.0
|
||||||
go.etcd.io/etcd/api/v3 v3.5.0-beta.4
|
go.etcd.io/etcd/api/v3 v3.5.0-beta.4
|
||||||
go.etcd.io/etcd/client/v3 v3.5.0-beta.4
|
go.etcd.io/etcd/client/v3 v3.5.0-beta.4
|
||||||
go.opencensus.io v0.22.5 // indirect
|
go.opencensus.io v0.22.5 // indirect
|
||||||
|
|||||||
4
go.sum
4
go.sum
@ -1030,8 +1030,6 @@ github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8
|
|||||||
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
|
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
|
||||||
github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA=
|
github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA=
|
||||||
github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
|
github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
|
||||||
github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788 h1:O+/N9vxhoObjuCuQczycuzdG240SoLrrdnyipJ5JJc0=
|
|
||||||
github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
|
|
||||||
github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8 h1:+8oaaj9Gm/yJFvR09Mz4iefX7pSYcFw6d6fMfvr1Mow=
|
github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8 h1:+8oaaj9Gm/yJFvR09Mz4iefX7pSYcFw6d6fMfvr1Mow=
|
||||||
github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
|
github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
|
||||||
github.com/minio/operator v0.0.0-20210812082324-26350f153661 h1:dGAJHpfmhNukFg0M0wDqH+G1OB2YPgZCcT6uv4n9YQk=
|
github.com/minio/operator v0.0.0-20210812082324-26350f153661 h1:dGAJHpfmhNukFg0M0wDqH+G1OB2YPgZCcT6uv4n9YQk=
|
||||||
@ -1440,6 +1438,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
|
|||||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
|
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
|
||||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||||
|
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
|
||||||
|
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
|
||||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user