Add detailed scanner metrics (#15161)

internal/config/heal/heal.go

@@ -71,7 +71,7 @@ func (opts Config) BitrotScanCycle() (d time.Duration) {
 // Wait waits for IOCount to go down or max sleep to elapse before returning.
 // usually used in healing paths to wait for specified amount of time to
 // throttle healing.
-func (opts Config) Wait(currentIO func() int, systemIO func() int) {
+func (opts Config) Wait(currentIO func() int, activeListeners func() int) {
 	configMutex.RLock()
 	maxIO, maxWait := opts.IOCount, opts.Sleep
 	configMutex.RUnlock()
@@ -87,7 +87,7 @@ func (opts Config) Wait(currentIO func() int, systemIO func() int) {
 	tmpMaxWait := maxWait
 
 	if currentIO != nil {
-		for currentIO() >= maxIO+systemIO() {
+		for currentIO() >= maxIO+activeListeners() {
 			if tmpMaxWait > 0 {
 				if tmpMaxWait < waitTick {
 					time.Sleep(tmpMaxWait)

internal/event/event.go

@@ -17,6 +17,10 @@
 
 package event
 
+import (
+	"github.com/minio/madmin-go"
+)
+
 const (
 	// NamespaceFormat - namespace log format used in some event targets.
 	NamespaceFormat = "namespace"
@@ -79,6 +83,12 @@ type Event struct {
 	ResponseElements  map[string]string `json:"responseElements"`
 	S3                Metadata          `json:"s3"`
 	Source            Source            `json:"source"`
+	Type              madmin.TraceType  `json:"-"`
+}
+
+// Mask returns the type as mask.
+func (e Event) Mask() uint64 {
+	return e.EventName.Mask()
 }
 
 // Log represents event information for some event targets.

internal/event/name.go

@@ -30,12 +30,12 @@ type Name int
 
 // Values of event Name
 const (
-	ObjectAccessedAll Name = 1 + iota
-	ObjectAccessedGet
+	// Single event types (does not require expansion)
+
+	ObjectAccessedGet Name = 1 + iota
 	ObjectAccessedGetRetention
 	ObjectAccessedGetLegalHold
 	ObjectAccessedHead
-	ObjectCreatedAll
 	ObjectCreatedCompleteMultipartUpload
 	ObjectCreatedCopy
 	ObjectCreatedPost
@@ -44,12 +44,10 @@ const (
 	ObjectCreatedPutLegalHold
 	ObjectCreatedPutTagging
 	ObjectCreatedDeleteTagging
-	ObjectRemovedAll
 	ObjectRemovedDelete
 	ObjectRemovedDeleteMarkerCreated
 	BucketCreated
 	BucketRemoved
-	ObjectReplicationAll
 	ObjectReplicationFailed
 	ObjectReplicationComplete
 	ObjectReplicationMissedThreshold
@@ -57,19 +55,28 @@ const (
 	ObjectReplicationNotTracked
 	ObjectRestorePostInitiated
 	ObjectRestorePostCompleted
-	ObjectRestorePostAll
-	ObjectTransitionAll
 	ObjectTransitionFailed
 	ObjectTransitionComplete
+
+	objectSingleTypesEnd
+	// Start Compound types that require expansion:
+
+	ObjectAccessedAll
+	ObjectCreatedAll
+	ObjectRemovedAll
+	ObjectReplicationAll
+	ObjectRestorePostAll
+	ObjectTransitionAll
 )
 
+// The number of single names should not exceed 64.
+// This will break masking. Use bit 63 as extension.
+var _ = uint64(1 << objectSingleTypesEnd)
+
 // Expand - returns expanded values of abbreviated event type.
 func (name Name) Expand() []Name {
 	switch name {
-	case BucketCreated:
-		return []Name{BucketCreated}
-	case BucketRemoved:
-		return []Name{BucketRemoved}
+
 	case ObjectAccessedAll:
 		return []Name{
 			ObjectAccessedGet, ObjectAccessedHead,
@@ -110,6 +117,19 @@ func (name Name) Expand() []Name {
 	}
 }
 
+// Mask returns the type as mask.
+// Compound "All" types are expanded.
+func (name Name) Mask() uint64 {
+	if name < objectSingleTypesEnd {
+		return 1 << (name - 1)
+	}
+	var mask uint64
+	for _, n := range name.Expand() {
+		mask |= 1 << (n - 1)
+	}
+	return mask
+}
+
 // String - returns string representation of event type.
 func (name Name) String() string {
 	switch name {

internal/ioutil/read_file.go

@@ -25,12 +25,21 @@ import (
 	"github.com/minio/minio/internal/disk"
 )
 
+var (
+	// OpenFileDirectIO allows overriding default function.
+	OpenFileDirectIO = disk.OpenFileDirectIO
+	// OsOpen allows overriding default function.
+	OsOpen = os.Open
+	// OsOpenFile allows overriding default function.
+	OsOpenFile = os.OpenFile
+)
+
 // ReadFileWithFileInfo reads the named file and returns the contents.
 // A successful call returns err == nil, not err == EOF.
 // Because ReadFile reads the whole file, it does not treat an EOF from Read
 // as an error to be reported, additionall returns os.FileInfo
 func ReadFileWithFileInfo(name string) ([]byte, fs.FileInfo, error) {
-	f, err := os.Open(name)
+	f, err := OsOpen(name)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -53,11 +62,11 @@ func ReadFileWithFileInfo(name string) ([]byte, fs.FileInfo, error) {
 //
 // passes NOATIME flag for reads on Unix systems to avoid atime updates.
 func ReadFile(name string) ([]byte, error) {
-	f, err := disk.OpenFileDirectIO(name, readMode, 0o666)
+	f, err := OpenFileDirectIO(name, readMode, 0o666)
 	if err != nil {
 		// fallback if there is an error to read
 		// 'name' with O_DIRECT
-		f, err = os.OpenFile(name, readMode, 0o666)
+		f, err = OsOpenFile(name, readMode, 0o666)
 		if err != nil {
 			return nil, err
 		}

internal/logger/audit.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/klauspost/compress/gzhttp"
+	"github.com/minio/madmin-go"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger/message/audit"
 )
@@ -234,8 +235,8 @@ func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqCl
 
 	// Send audit logs only to http targets.
 	for _, t := range auditTgts {
-		if err := t.Send(entry, string(All)); err != nil {
-			LogAlwaysIf(context.Background(), fmt.Errorf("event(%v) was not sent to Audit target (%v): %v", entry, t, err), All)
+		if err := t.Send(entry); err != nil {
+			LogAlwaysIf(context.Background(), fmt.Errorf("event(%v) was not sent to Audit target (%v): %v", entry, t, err), madmin.LogKindAll)
 		}
 	}
 }

internal/logger/logger.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/minio/highwayhash"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger/message/log"
@@ -50,6 +51,10 @@ const (
 	InformationLvl Level = iota + 1
 	ErrorLvl
 	FatalLvl
+
+	Application = madmin.LogKindApplication
+	Minio       = madmin.LogKindMinio
+	All         = madmin.LogKindAll
 )
 
 var trimStrings []string
@@ -65,16 +70,15 @@ var matchingFuncNames = [...]string{
 }
 
 func (level Level) String() string {
-	var lvlStr string
 	switch level {
 	case InformationLvl:
-		lvlStr = "INFO"
+		return "INFO"
 	case ErrorLvl:
-		lvlStr = "ERROR"
+		return "ERROR"
 	case FatalLvl:
-		lvlStr = "FATAL"
+		return "FATAL"
 	}
-	return lvlStr
+	return ""
 }
 
 // quietFlag: Hide startup messages if enabled
@@ -237,18 +241,6 @@ func hashString(input string) string {
 	return hex.EncodeToString(hh.Sum(nil))
 }
 
-// Kind specifies the kind of error log
-type Kind string
-
-const (
-	// Minio errors
-	Minio Kind = "MINIO"
-	// Application errors
-	Application Kind = "APPLICATION"
-	// All errors
-	All Kind = "ALL"
-)
-
 // LogAlwaysIf prints a detailed error message during
 // the execution of the server.
 func LogAlwaysIf(ctx context.Context, err error, errKind ...interface{}) {
@@ -279,10 +271,10 @@ func LogIf(ctx context.Context, err error, errKind ...interface{}) {
 }
 
 func errToEntry(ctx context.Context, err error, errKind ...interface{}) log.Entry {
-	logKind := string(Minio)
+	logKind := madmin.LogKindAll
 	if len(errKind) > 0 {
-		if ek, ok := errKind[0].(Kind); ok {
-			logKind = string(ek)
+		if ek, ok := errKind[0].(madmin.LogKind); ok {
+			logKind = ek
 		}
 	}
 	req := GetReqInfo(ctx)
@@ -366,7 +358,7 @@ func consoleLogIf(ctx context.Context, err error, errKind ...interface{}) {
 
 	if consoleTgt != nil {
 		entry := errToEntry(ctx, err, errKind...)
-		consoleTgt.Send(entry, entry.LogKind)
+		consoleTgt.Send(entry)
 	}
 }
 
@@ -385,10 +377,10 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 	entry := errToEntry(ctx, err, errKind...)
 	// Iterate over all logger targets to send the log entry
 	for _, t := range systemTgts {
-		if err := t.Send(entry, entry.LogKind); err != nil {
+		if err := t.Send(entry); err != nil {
 			if consoleTgt != nil {
 				entry.Trace.Message = fmt.Sprintf("event(%#v) was not sent to Logger target (%#v): %#v", entry, t, err)
-				consoleTgt.Send(entry, entry.LogKind)
+				consoleTgt.Send(entry)
 			}
 		}
 	}

internal/logger/message/log/entry.go

@@ -20,6 +20,8 @@ package log
 import (
 	"strings"
 	"time"
+
+	"github.com/minio/madmin-go"
 )
 
 // ObjectVersion object version key/versionId
@@ -52,17 +54,17 @@ type API struct {
 
 // Entry - defines fields and values of each log entry.
 type Entry struct {
-	DeploymentID string    `json:"deploymentid,omitempty"`
-	Level        string    `json:"level"`
-	LogKind      string    `json:"errKind"`
-	Time         time.Time `json:"time"`
-	API          *API      `json:"api,omitempty"`
-	RemoteHost   string    `json:"remotehost,omitempty"`
-	Host         string    `json:"host,omitempty"`
-	RequestID    string    `json:"requestID,omitempty"`
-	UserAgent    string    `json:"userAgent,omitempty"`
-	Message      string    `json:"message,omitempty"`
-	Trace        *Trace    `json:"error,omitempty"`
+	DeploymentID string         `json:"deploymentid,omitempty"`
+	Level        string         `json:"level"`
+	LogKind      madmin.LogKind `json:"errKind"`
+	Time         time.Time      `json:"time"`
+	API          *API           `json:"api,omitempty"`
+	RemoteHost   string         `json:"remotehost,omitempty"`
+	Host         string         `json:"host,omitempty"`
+	RequestID    string         `json:"requestID,omitempty"`
+	UserAgent    string         `json:"userAgent,omitempty"`
+	Message      string         `json:"message,omitempty"`
+	Trace        *Trace         `json:"error,omitempty"`
 }
 
 // Info holds console log messages
@@ -73,9 +75,15 @@ type Info struct {
 	Err        error  `json:"-"`
 }
 
-// SendLog returns true if log pertains to node specified in args.
-func (l Info) SendLog(node, logKind string) bool {
-	nodeFltr := (node == "" || strings.EqualFold(node, l.NodeName))
-	typeFltr := strings.EqualFold(logKind, "all") || strings.EqualFold(l.LogKind, logKind)
-	return nodeFltr && typeFltr
+// Mask returns the mask based on the error level.
+func (l Info) Mask() uint64 {
+	return l.LogKind.LogMask().Mask()
+}
+
+// SendLog returns true if log pertains to node specified in args.
+func (l Info) SendLog(node string, logKind madmin.LogMask) bool {
+	if logKind.Contains(l.LogKind.LogMask()) {
+		return node == "" || strings.EqualFold(node, l.NodeName)
+	}
+	return false
 }

internal/logger/target/http/http.go

@@ -203,7 +203,7 @@ func New(config Config) *Target {
 }
 
 // Send log message 'e' to http target.
-func (h *Target) Send(entry interface{}, errKind string) error {
+func (h *Target) Send(entry interface{}) error {
 	select {
 	case <-h.doneCh:
 		return nil

internal/logger/target/kafka/kafka.go

@@ -48,7 +48,7 @@ type Target struct {
 }
 
 // Send log message 'e' to kafka target.
-func (h *Target) Send(entry interface{}, errKind string) error {
+func (h *Target) Send(entry interface{}) error {
 	select {
 	case <-h.doneCh:
 		return nil

internal/logger/targets.go

@@ -33,7 +33,7 @@ type Target interface {
 	Endpoint() string
 	Init() error
 	Cancel()
-	Send(entry interface{}, errKind string) error
+	Send(entry interface{}) error
 	Type() types.TargetType
 }
 

internal/pubsub/mask.go

@@ -0,0 +1,66 @@
+package pubsub
+
+import (
+	"math"
+	"math/bits"
+)
+
+// Mask allows filtering by a bitset mask.
+type Mask uint64
+
+const (
+	// MaskAll is the mask for all entries.
+	MaskAll Mask = math.MaxUint64
+)
+
+// MaskFromMaskable extracts mask from an interface.
+func MaskFromMaskable(m Maskable) Mask {
+	return Mask(m.Mask())
+}
+
+// Contains returns whether *all* flags in other is present in t.
+func (t Mask) Contains(other Mask) bool {
+	return t&other == other
+}
+
+// Overlaps returns whether *any* flags in t overlaps with other.
+func (t Mask) Overlaps(other Mask) bool {
+	return t&other != 0
+}
+
+// SingleType returns whether t has a single type set.
+func (t Mask) SingleType() bool {
+	return bits.OnesCount64(uint64(t)) == 1
+}
+
+// FromUint64 will set a mask to the uint64 value.
+func (t *Mask) FromUint64(m uint64) {
+	*t = Mask(m)
+}
+
+// Merge will merge other into t.
+func (t *Mask) Merge(other Mask) {
+	*t |= other
+}
+
+// MergeMaskable will merge other into t.
+func (t *Mask) MergeMaskable(other Maskable) {
+	*t |= Mask(other.Mask())
+}
+
+// SetIf will add other if b is true.
+func (t *Mask) SetIf(b bool, other Mask) {
+	if b {
+		*t |= other
+	}
+}
+
+// Mask returns the mask as a uint64.
+func (t Mask) Mask() uint64 {
+	return uint64(t)
+}
+
+// Maskable implementations must return their mask as a 64 bit uint.
+type Maskable interface {
+	Mask() uint64
+}

internal/pubsub/pubsub.go

@@ -25,27 +25,31 @@ import (
 
 // Sub - subscriber entity.
 type Sub struct {
-	ch     chan interface{}
-	filter func(entry interface{}) bool
+	ch     chan Maskable
+	types  Mask
+	filter func(entry Maskable) bool
 }
 
 // PubSub holds publishers and subscribers
 type PubSub struct {
-	subs           []*Sub
+	// atomics, keep at top:
+	types          uint64
 	numSubscribers int32
 	maxSubscribers int32
+
+	// not atomics:
+	subs []*Sub
 	sync.RWMutex
 }
 
 // Publish message to the subscribers.
 // Note that publish is always nob-blocking send so that we don't block on slow receivers.
 // Hence receivers should use buffered channel so as not to miss the published events.
-func (ps *PubSub) Publish(item interface{}) {
+func (ps *PubSub) Publish(item Maskable) {
 	ps.RLock()
 	defer ps.RUnlock()
-
 	for _, sub := range ps.subs {
-		if sub.filter == nil || sub.filter(item) {
+		if sub.types.Contains(Mask(item.Mask())) && (sub.filter == nil || sub.filter(item)) {
 			select {
 			case sub.ch <- item:
 			default:
@@ -55,38 +59,54 @@ func (ps *PubSub) Publish(item interface{}) {
 }
 
 // Subscribe - Adds a subscriber to pubsub system
-func (ps *PubSub) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, filter func(entry interface{}) bool) error {
+func (ps *PubSub) Subscribe(mask Mask, subCh chan Maskable, doneCh <-chan struct{}, filter func(entry Maskable) bool) error {
 	totalSubs := atomic.AddInt32(&ps.numSubscribers, 1)
 	if ps.maxSubscribers > 0 && totalSubs > ps.maxSubscribers {
 		atomic.AddInt32(&ps.numSubscribers, -1)
 		return fmt.Errorf("the limit of `%d` subscribers is reached", ps.maxSubscribers)
 	}
-
 	ps.Lock()
 	defer ps.Unlock()
 
-	sub := &Sub{subCh, filter}
+	sub := &Sub{ch: subCh, types: mask, filter: filter}
 	ps.subs = append(ps.subs, sub)
 
+	// We hold a lock, so we are safe to update
+	combined := Mask(atomic.LoadUint64(&ps.types))
+	combined.Merge(mask)
+	atomic.StoreUint64(&ps.types, uint64(combined))
+
 	go func() {
 		<-doneCh
 
 		ps.Lock()
 		defer ps.Unlock()
-
+		var remainTypes Mask
 		for i, s := range ps.subs {
 			if s == sub {
 				ps.subs = append(ps.subs[:i], ps.subs[i+1:]...)
+			} else {
+				remainTypes.Merge(s.types)
 			}
 		}
+		atomic.StoreUint64(&ps.types, uint64(remainTypes))
 		atomic.AddInt32(&ps.numSubscribers, -1)
 	}()
 
 	return nil
 }
 
-// NumSubscribers returns the number of current subscribers
-func (ps *PubSub) NumSubscribers() int32 {
+// NumSubscribers returns the number of current subscribers,
+// If t is non-nil, the type is checked against the active subscribed types,
+// and 0 will be returned if nobody is subscribed for the type,
+// otherwise the *total* number of subscribers is returned.
+func (ps *PubSub) NumSubscribers(m Maskable) int32 {
+	if m != nil {
+		types := Mask(atomic.LoadUint64(&ps.types))
+		if !types.Overlaps(Mask(m.Mask())) {
+			return 0
+		}
+	}
 	return atomic.LoadInt32(&ps.numSubscribers)
 }
 

internal/pubsub/pubsub_test.go

@@ -25,51 +25,87 @@ import (
 
 func TestSubscribe(t *testing.T) {
 	ps := New(2)
-	ch1 := make(chan interface{}, 1)
-	ch2 := make(chan interface{}, 1)
+	ch1 := make(chan Maskable, 1)
+	ch2 := make(chan Maskable, 1)
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	if err := ps.Subscribe(ch1, doneCh, nil); err != nil {
+	if err := ps.Subscribe(MaskAll, ch1, doneCh, nil); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if err := ps.Subscribe(ch2, doneCh, nil); err != nil {
+	if err := ps.Subscribe(MaskAll, ch2, doneCh, nil); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 	ps.Lock()
 	defer ps.Unlock()
+
+	if len(ps.subs) != 2 || ps.NumSubscribers(nil) != 2 {
+		t.Fatalf("expected 2 subscribers")
+	}
+}
+
+func TestNumSubscribersMask(t *testing.T) {
+	ps := New(2)
+	ch1 := make(chan Maskable, 1)
+	ch2 := make(chan Maskable, 1)
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+	if err := ps.Subscribe(1, ch1, doneCh, nil); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if err := ps.Subscribe(2, ch2, doneCh, nil); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	ps.Lock()
+	defer ps.Unlock()
+
 	if len(ps.subs) != 2 {
 		t.Fatalf("expected 2 subscribers")
 	}
+	if want, got := int32(2), ps.NumSubscribers(Mask(1)); got != want {
+		t.Fatalf("want %d subscribers, got %d", want, got)
+	}
+	if want, got := int32(2), ps.NumSubscribers(Mask(2)); got != want {
+		t.Fatalf("want %d subscribers, got %d", want, got)
+	}
+	if want, got := int32(2), ps.NumSubscribers(Mask(1|2)); got != want {
+		t.Fatalf("want %d subscribers, got %d", want, got)
+	}
+	if want, got := int32(2), ps.NumSubscribers(nil); got != want {
+		t.Fatalf("want %d subscribers, got %d", want, got)
+	}
+	if want, got := int32(0), ps.NumSubscribers(Mask(4)); got != want {
+		t.Fatalf("want %d subscribers, got %d", want, got)
+	}
 }
 
 func TestSubscribeExceedingLimit(t *testing.T) {
 	ps := New(2)
-	ch1 := make(chan interface{}, 1)
-	ch2 := make(chan interface{}, 1)
-	ch3 := make(chan interface{}, 1)
+	ch1 := make(chan Maskable, 1)
+	ch2 := make(chan Maskable, 1)
+	ch3 := make(chan Maskable, 1)
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	if err := ps.Subscribe(ch1, doneCh, nil); err != nil {
+	if err := ps.Subscribe(MaskAll, ch1, doneCh, nil); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if err := ps.Subscribe(ch2, doneCh, nil); err != nil {
+	if err := ps.Subscribe(MaskAll, ch2, doneCh, nil); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if err := ps.Subscribe(ch3, doneCh, nil); err == nil {
+	if err := ps.Subscribe(MaskAll, ch3, doneCh, nil); err == nil {
 		t.Fatalf("unexpected nil err")
 	}
 }
 
 func TestUnsubscribe(t *testing.T) {
 	ps := New(2)
-	ch1 := make(chan interface{}, 1)
-	ch2 := make(chan interface{}, 1)
+	ch1 := make(chan Maskable, 1)
+	ch2 := make(chan Maskable, 1)
 	doneCh1 := make(chan struct{})
 	doneCh2 := make(chan struct{})
-	if err := ps.Subscribe(ch1, doneCh1, nil); err != nil {
+	if err := ps.Subscribe(MaskAll, ch1, doneCh1, nil); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if err := ps.Subscribe(ch2, doneCh2, nil); err != nil {
+	if err := ps.Subscribe(MaskAll, ch2, doneCh2, nil); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
@@ -84,40 +120,81 @@ func TestUnsubscribe(t *testing.T) {
 	close(doneCh2)
 }
 
+type maskString string
+
+func (m maskString) Mask() uint64 {
+	return 1
+}
+
 func TestPubSub(t *testing.T) {
 	ps := New(1)
-	ch1 := make(chan interface{}, 1)
+	ch1 := make(chan Maskable, 1)
 	doneCh1 := make(chan struct{})
 	defer close(doneCh1)
-	if err := ps.Subscribe(ch1, doneCh1, func(entry interface{}) bool { return true }); err != nil {
+	if err := ps.Subscribe(MaskAll, ch1, doneCh1, func(entry Maskable) bool { return true }); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	val := "hello"
+	val := maskString("hello")
 	ps.Publish(val)
 	msg := <-ch1
-	if msg != "hello" {
+	if msg != val {
 		t.Fatalf(fmt.Sprintf("expected %s , found %s", val, msg))
 	}
 }
 
 func TestMultiPubSub(t *testing.T) {
 	ps := New(2)
-	ch1 := make(chan interface{}, 1)
-	ch2 := make(chan interface{}, 1)
+	ch1 := make(chan Maskable, 1)
+	ch2 := make(chan Maskable, 1)
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	if err := ps.Subscribe(ch1, doneCh, func(entry interface{}) bool { return true }); err != nil {
+	if err := ps.Subscribe(MaskAll, ch1, doneCh, func(entry Maskable) bool { return true }); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if err := ps.Subscribe(ch2, doneCh, func(entry interface{}) bool { return true }); err != nil {
+	if err := ps.Subscribe(MaskAll, ch2, doneCh, func(entry Maskable) bool { return true }); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	val := "hello"
+	val := maskString("hello")
 	ps.Publish(val)
 
 	msg1 := <-ch1
 	msg2 := <-ch2
-	if msg1 != "hello" && msg2 != "hello" {
+	if msg1 != val && msg2 != val {
 		t.Fatalf(fmt.Sprintf("expected both subscribers to have%s , found %s and  %s", val, msg1, msg2))
 	}
 }
+
+func TestMultiPubSubMask(t *testing.T) {
+	ps := New(3)
+	ch1 := make(chan Maskable, 1)
+	ch2 := make(chan Maskable, 1)
+	ch3 := make(chan Maskable, 1)
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+	// Mask matches maskString, should get result
+	if err := ps.Subscribe(Mask(1), ch1, doneCh, func(entry Maskable) bool { return true }); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	// Mask matches maskString, should get result
+	if err := ps.Subscribe(Mask(1|2), ch2, doneCh, func(entry Maskable) bool { return true }); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	// Does NOT overlap maskString
+	if err := ps.Subscribe(Mask(2), ch3, doneCh, func(entry Maskable) bool { return true }); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	val := maskString("hello")
+	ps.Publish(val)
+
+	msg1 := <-ch1
+	msg2 := <-ch2
+	if msg1 != val && msg2 != val {
+		t.Fatalf(fmt.Sprintf("expected both subscribers to have%s , found %s and  %s", val, msg1, msg2))
+	}
+
+	select {
+	case msg := <-ch3:
+		t.Fatalf(fmt.Sprintf("unexpect msg, f got %s", msg))
+	default:
+	}
+}