minio/internal/dsync/dsync-server_test.go

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package dsync

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gorilla/mux"
)

const numberOfNodes = 5

var (
	ds          *Dsync
	nodes       = make([]*httptest.Server, numberOfNodes) // list of node IP addrs or hostname with ports.
	lockServers = make([]*lockServer, numberOfNodes)
)

func getLockArgs(r *http.Request) (args LockArgs, err error) {
	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return args, err
	}
	_, err = args.UnmarshalMsg(buf)
	return args, err
}

type lockServerHandler struct {
	lsrv *lockServer
}

func (lh *lockServerHandler) writeErrorResponse(w http.ResponseWriter, err error) {
	w.WriteHeader(http.StatusForbidden)
	w.Write([]byte(err.Error()))
}

func (lh *lockServerHandler) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
	args, err := getLockArgs(r)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}

	if _, err = lh.lsrv.ForceUnlock(&args); err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
}

func (lh *lockServerHandler) RefreshHandler(w http.ResponseWriter, r *http.Request) {
	args, err := getLockArgs(r)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}

	reply, err := lh.lsrv.Refresh(&args)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}

	if !reply {
		lh.writeErrorResponse(w, errLockNotFound)
		return
	}
}

func (lh *lockServerHandler) LockHandler(w http.ResponseWriter, r *http.Request) {
	args, err := getLockArgs(r)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
	reply, err := lh.lsrv.Lock(&args)
	if err == nil && !reply {
		err = errLockConflict
	}
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
}

func (lh *lockServerHandler) UnlockHandler(w http.ResponseWriter, r *http.Request) {
	args, err := getLockArgs(r)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
	_, err = lh.lsrv.Unlock(&args)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
}

func (lh *lockServerHandler) RUnlockHandler(w http.ResponseWriter, r *http.Request) {
	args, err := getLockArgs(r)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
	_, err = lh.lsrv.RUnlock(&args)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
}

func (lh *lockServerHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {}

func (lh *lockServerHandler) RLockHandler(w http.ResponseWriter, r *http.Request) {
	args, err := getLockArgs(r)
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}

	reply, err := lh.lsrv.RLock(&args)
	if err == nil && !reply {
		err = errLockConflict
	}
	if err != nil {
		lh.writeErrorResponse(w, err)
		return
	}
}

func stopLockServers() {
	for i := 0; i < numberOfNodes; i++ {
		nodes[i].Close()
	}
}

func startLockServers() {
	for i := 0; i < numberOfNodes; i++ {
		lsrv := &lockServer{
			mutex:   sync.Mutex{},
			lockMap: make(map[string]int64),
		}
		lockServer := lockServerHandler{
			lsrv: lsrv,
		}
		lockServers[i] = lsrv

		router := mux.NewRouter().SkipClean(true)
		subrouter := router.PathPrefix("/").Subrouter()
		subrouter.Methods(http.MethodPost).Path("/v1/health").HandlerFunc(lockServer.HealthHandler)
		subrouter.Methods(http.MethodPost).Path("/v1/refresh").HandlerFunc(lockServer.RefreshHandler)
		subrouter.Methods(http.MethodPost).Path("/v1/lock").HandlerFunc(lockServer.LockHandler)
		subrouter.Methods(http.MethodPost).Path("/v1/rlock").HandlerFunc(lockServer.RLockHandler)
		subrouter.Methods(http.MethodPost).Path("/v1/unlock").HandlerFunc(lockServer.UnlockHandler)
		subrouter.Methods(http.MethodPost).Path("/v1/runlock").HandlerFunc(lockServer.RUnlockHandler)
		subrouter.Methods(http.MethodPost).Path("/v1/force-unlock").HandlerFunc(lockServer.ForceUnlockHandler)

		nodes[i] = httptest.NewServer(router)
	}
}

const WriteLock = -1

type lockServer struct {
	mutex sync.Mutex
	// Map of locks, with negative value indicating (exclusive) write lock
	// and positive values indicating number of read locks
	lockMap map[string]int64

	// Refresh returns lock not found if set to true
	lockNotFound bool

	// Set to true if you want peers servers to do not respond
	responseDelay int64
}

func (l *lockServer) setRefreshReply(refreshed bool) {
	l.mutex.Lock()
	defer l.mutex.Unlock()
	l.lockNotFound = !refreshed
}

func (l *lockServer) setResponseDelay(responseDelay time.Duration) {
	atomic.StoreInt64(&l.responseDelay, int64(responseDelay))
}

func (l *lockServer) Lock(args *LockArgs) (reply bool, err error) {
	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
		time.Sleep(time.Duration(d))
	}

	l.mutex.Lock()
	defer l.mutex.Unlock()
	if _, reply = l.lockMap[args.Resources[0]]; !reply {
		l.lockMap[args.Resources[0]] = WriteLock // No locks held on the given name, so claim write lock
	}
	reply = !reply // Negate *reply to return true when lock is granted or false otherwise
	return reply, nil
}

func (l *lockServer) Unlock(args *LockArgs) (reply bool, err error) {
	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
		time.Sleep(time.Duration(d))
	}

	l.mutex.Lock()
	defer l.mutex.Unlock()
	var locksHeld int64
	if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { // No lock is held on the given name
		return false, fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Resources[0])
	}
	if reply = locksHeld == WriteLock; !reply { // Unless it is a write lock
		return false, fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Resources[0], locksHeld)
	}
	delete(l.lockMap, args.Resources[0]) // Remove the write lock
	return true, nil
}

const ReadLock = 1

func (l *lockServer) RLock(args *LockArgs) (reply bool, err error) {
	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
		time.Sleep(time.Duration(d))
	}

	l.mutex.Lock()
	defer l.mutex.Unlock()
	var locksHeld int64
	if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply {
		l.lockMap[args.Resources[0]] = ReadLock // No locks held on the given name, so claim (first) read lock
		reply = true
	} else if reply = locksHeld != WriteLock; reply { // Unless there is a write lock
		l.lockMap[args.Resources[0]] = locksHeld + ReadLock // Grant another read lock
	}
	return reply, nil
}

func (l *lockServer) RUnlock(args *LockArgs) (reply bool, err error) {
	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
		time.Sleep(time.Duration(d))
	}

	l.mutex.Lock()
	defer l.mutex.Unlock()
	var locksHeld int64
	if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { // No lock is held on the given name
		return false, fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Resources[0])
	}
	if reply = locksHeld != WriteLock; !reply { // A write-lock is held, cannot release a read lock
		return false, fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Resources[0])
	}
	if locksHeld > ReadLock {
		l.lockMap[args.Resources[0]] = locksHeld - ReadLock // Remove one of the read locks held
	} else {
		delete(l.lockMap, args.Resources[0]) // Remove the (last) read lock
	}
	return reply, nil
}

func (l *lockServer) Refresh(args *LockArgs) (reply bool, err error) {
	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
		time.Sleep(time.Duration(d))
	}

	l.mutex.Lock()
	defer l.mutex.Unlock()
	reply = !l.lockNotFound
	return reply, nil
}

func (l *lockServer) ForceUnlock(args *LockArgs) (reply bool, err error) {
	if d := atomic.LoadInt64(&l.responseDelay); d != 0 {
		time.Sleep(time.Duration(d))
	}

	l.mutex.Lock()
	defer l.mutex.Unlock()
	if len(args.UID) != 0 {
		return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
	}
	delete(l.lockMap, args.Resources[0]) // Remove the lock (irrespective of write or read lock)
	reply = true
	return reply, nil
}