MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rename crawler config option to heal (#10678)

This commit is contained in:
Harshavardhana 2020-10-14 13:51:51 -07:00 committed by GitHub
parent f9be783f3e
commit 2042d4873c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 30 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/config-current.go
View File

@ -25,9 +25,9 @@ import (
"github.com/minio/minio/cmd/config/api" "github.com/minio/minio/cmd/config/api"
"github.com/minio/minio/cmd/config/cache" "github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/config/compress" "github.com/minio/minio/cmd/config/compress"
"github.com/minio/minio/cmd/config/crawler"
"github.com/minio/minio/cmd/config/dns" "github.com/minio/minio/cmd/config/dns"
"github.com/minio/minio/cmd/config/etcd" "github.com/minio/minio/cmd/config/etcd"
"github.com/minio/minio/cmd/config/heal"
xldap "github.com/minio/minio/cmd/config/identity/ldap" xldap "github.com/minio/minio/cmd/config/identity/ldap"
"github.com/minio/minio/cmd/config/identity/openid" "github.com/minio/minio/cmd/config/identity/openid"
"github.com/minio/minio/cmd/config/notify" "github.com/minio/minio/cmd/config/notify"
@ -56,7 +56,7 @@ func initHelp() {
config.KmsKesSubSys: crypto.DefaultKesKVS, config.KmsKesSubSys: crypto.DefaultKesKVS,
config.LoggerWebhookSubSys: logger.DefaultKVS, config.LoggerWebhookSubSys: logger.DefaultKVS,
config.AuditWebhookSubSys: logger.DefaultAuditKVS, config.AuditWebhookSubSys: logger.DefaultAuditKVS,
config.CrawlerSubSys: crawler.DefaultKVS, config.HealSubSys: heal.DefaultKVS,
} }
for k, v := range notify.DefaultNotificationKVS { for k, v := range notify.DefaultNotificationKVS {
kvs[k] = v kvs[k] = v
@ -109,8 +109,8 @@ func initHelp() {
Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.", Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
}, },
config.HelpKV{ config.HelpKV{
Key: config.CrawlerSubSys, Key: config.HealSubSys,
Description: "manage continuous disk crawling for bucket disk usage, lifecycle, quota and data integrity checks", Description: "manage object healing frequency and bitrot verification checks",
}, },
config.HelpKV{ config.HelpKV{
Key: config.LoggerWebhookSubSys, Key: config.LoggerWebhookSubSys,
@ -191,7 +191,7 @@ func initHelp() {
config.EtcdSubSys: etcd.Help, config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help, config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help, config.CompressionSubSys: compress.Help,
config.CrawlerSubSys: crawler.Help, config.HealSubSys: heal.Help,
config.IdentityOpenIDSubSys: openid.Help, config.IdentityOpenIDSubSys: openid.Help,
config.IdentityLDAPSubSys: xldap.Help, config.IdentityLDAPSubSys: xldap.Help,
config.PolicyOPASubSys: opa.Help, config.PolicyOPASubSys: opa.Help,
@ -253,7 +253,7 @@ func validateConfig(s config.Config, setDriveCount int) error {
return err return err
} }
if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil { if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
return err return err
} }
@ -438,9 +438,9 @@ func lookupConfigs(s config.Config, setDriveCount int) {
} }
} }
} }
globalCrawlerConfig, err = crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]) globalHealConfig, err = heal.LookupConfig(s[config.HealSubSys][config.Default])
if err != nil { if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to read crawler config: %w", err)) logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
} }
kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport()) kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())

6
cmd/config/config.go
View File

@ -76,7 +76,7 @@ const (
KmsKesSubSys = "kms_kes" KmsKesSubSys = "kms_kes"
LoggerWebhookSubSys = "logger_webhook" LoggerWebhookSubSys = "logger_webhook"
AuditWebhookSubSys = "audit_webhook" AuditWebhookSubSys = "audit_webhook"
CrawlerSubSys = "crawler" HealSubSys = "heal"
// Add new constants here if you add new fields to config. // Add new constants here if you add new fields to config.
) )
@ -113,7 +113,7 @@ var SubSystems = set.CreateStringSet([]string{
PolicyOPASubSys, PolicyOPASubSys,
IdentityLDAPSubSys, IdentityLDAPSubSys,
IdentityOpenIDSubSys, IdentityOpenIDSubSys,
CrawlerSubSys, HealSubSys,
NotifyAMQPSubSys, NotifyAMQPSubSys,
NotifyESSubSys, NotifyESSubSys,
NotifyKafkaSubSys, NotifyKafkaSubSys,
@ -140,7 +140,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
PolicyOPASubSys, PolicyOPASubSys,
IdentityLDAPSubSys, IdentityLDAPSubSys,
IdentityOpenIDSubSys, IdentityOpenIDSubSys,
CrawlerSubSys, HealSubSys,
}...) }...)
// Constant separators // Constant separators

18
cmd/config/crawler/crawler.go → cmd/config/heal/heal.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package crawler package heal
import ( import (
"errors" "errors"
@ -24,20 +24,20 @@ import (
// Compression environment variables // Compression environment variables
const ( const (
BitrotScan = "bitrotscan" Bitrot = "bitrotscan"
) )
// Config represents the crawler settings. // Config represents the heal settings.
type Config struct { type Config struct {
// Bitrot will perform bitrot scan on local disk when checking objects. // Bitrot will perform bitrot scan on local disk when checking objects.
Bitrot bool `json:"bitrotscan"` Bitrot bool `json:"bitrotscan"`
} }
var ( var (
// DefaultKVS - default KV config for crawler settings // DefaultKVS - default KV config for heal settings
DefaultKVS = config.KVS{ DefaultKVS = config.KVS{
config.KV{ config.KV{
Key: BitrotScan, Key: Bitrot,
Value: config.EnableOff, Value: config.EnableOff,
}, },
} }
@ -45,7 +45,7 @@ var (
// Help provides help for config values // Help provides help for config values
Help = config.HelpKVS{ Help = config.HelpKVS{
config.HelpKV{ config.HelpKV{
Key: BitrotScan, Key: Bitrot,
Description: `perform bitrot scan on disks when checking objects during crawl`, Description: `perform bitrot scan on disks when checking objects during crawl`,
Optional: true, Optional: true,
Type: "on|off", Type: "on|off",
@ -55,12 +55,12 @@ var (
// LookupConfig - lookup config and override with valid environment settings if any. // LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) { func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil { if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
return cfg, err return cfg, err
} }
bitrot := kvs.Get(BitrotScan) bitrot := kvs.Get(Bitrot)
if bitrot != config.EnableOn && bitrot != config.EnableOff { if bitrot != config.EnableOn && bitrot != config.EnableOff {
return cfg, errors.New(BitrotScan + ": must be 'on' or 'off'") return cfg, errors.New(Bitrot + ": must be 'on' or 'off'")
} }
cfg.Bitrot = bitrot == config.EnableOn cfg.Bitrot = bitrot == config.EnableOn
return cfg, nil return cfg, nil

4
cmd/data-crawler.go
View File

@ -29,7 +29,7 @@ import (
"time" "time"
"github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/crawler" "github.com/minio/minio/cmd/config/heal"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/bucket/lifecycle" "github.com/minio/minio/pkg/bucket/lifecycle"
"github.com/minio/minio/pkg/bucket/replication" "github.com/minio/minio/pkg/bucket/replication"
@ -53,7 +53,7 @@ const (
) )
var ( var (
globalCrawlerConfig crawler.Config globalHealConfig heal.Config
dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second) dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
) )

1
cmd/gateway/nas/gateway-nas.go
View File

@ -52,7 +52,6 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.HelpName}} /shared/nasvol {{.Prompt}} {{.HelpName}} /shared/nasvol
` `

3
cmd/server-main.go
View File

@ -98,6 +98,9 @@ func serverCmdArgs(ctx *cli.Context) []string {
v = env.Get(config.EnvEndpoints, "") v = env.Get(config.EnvEndpoints, "")
} }
if v == "" { if v == "" {
if !ctx.Args().Present() || ctx.Args().First() == "help" {
cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1)
}
return ctx.Args() return ctx.Args()
} }
return strings.Fields(v) return strings.Fields(v)

7
cmd/xl-storage.go
View File

@ -370,7 +370,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
if objAPI == nil { if objAPI == nil {
return cache, errServerNotInitialized return cache, errServerNotInitialized
} }
opts := globalCrawlerConfig opts := globalHealConfig
dataUsageInfo, err := crawlDataFolder(ctx, s.diskPath, cache, func(item crawlItem) (int64, error) { dataUsageInfo, err := crawlDataFolder(ctx, s.diskPath, cache, func(item crawlItem) (int64, error) {
// Look for `xl.meta/xl.json' at the leaf. // Look for `xl.meta/xl.json' at the leaf.
@ -414,7 +414,10 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
err := s.VerifyFile(ctx, item.bucket, item.objectPath(), version) err := s.VerifyFile(ctx, item.bucket, item.objectPath(), version)
switch err { switch err {
case errFileCorrupt: case errFileCorrupt:
res, err := objAPI.HealObject(ctx, item.bucket, item.objectPath(), oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling, ScanMode: madmin.HealDeepScan}) res, err := objAPI.HealObject(ctx, item.bucket, item.objectPath(), oi.VersionID, madmin.HealOpts{
Remove: healDeleteDangling,
ScanMode: madmin.HealDeepScan,
})
if err != nil { if err != nil {
if !errors.Is(err, NotImplemented{}) { if !errors.Is(err, NotImplemented{}) {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)