minio/internal/deadlineconn/deadlineconn_test.go
Klaus Post 55f5c18fd9
Harden internode DeadlineConn (#20631)
Since DeadlineConn would send deadline updates directly upstream,
it would race with Read/Write operations. The stdlib will perform a read, 
but do an async SetReadDeadLine(unix(1)) to cancel the Read in 
`abortPendingRead`. In this case, the Read may override the 
deadline intended to cancel the read.

Stop updating deadlines if a deadline in the past is seen and when Close is called. 
A mutex now protects all upstream deadline calls to avoid races. 

This should fix the short-term buildup of...

```
365 @ 0x44112e 0x4756b9 0x475699 0x483525 0x732286 0x737407 0x73816b 0x479601
#	0x475698	sync.runtime_notifyListWait+0x138		runtime/sema.go:569
#	0x483524	sync.(*Cond).Wait+0x84				sync/cond.go:70
#	0x732285	net/http.(*connReader).abortPendingRead+0xa5	net/http/server.go:729
#	0x737406	net/http.(*response).finishRequest+0x86		net/http/server.go:1676
#	0x73816a	net/http.(*conn).serve+0x62a			net/http/server.go:2050
```

AFAICT Only affects internode calls that create a connection (non-grid).
2024-11-11 09:15:17 -08:00

193 lines
4.8 KiB
Go

// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deadlineconn
import (
"bufio"
"fmt"
"io"
"net"
"sync"
"testing"
"time"
)
// Test deadlineconn handles read timeout properly by reading two messages beyond deadline.
func TestBuffConnReadTimeout(t *testing.T) {
l, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("unable to create listener. %v", err)
}
defer l.Close()
serverAddr := l.Addr().String()
tcpListener, ok := l.(*net.TCPListener)
if !ok {
t.Fatalf("failed to assert to net.TCPListener")
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
tcpConn, terr := tcpListener.AcceptTCP()
if terr != nil {
t.Errorf("failed to accept new connection. %v", terr)
return
}
deadlineconn := New(tcpConn)
deadlineconn.WithReadDeadline(time.Second)
deadlineconn.WithWriteDeadline(time.Second)
defer deadlineconn.Close()
// Read a line
b := make([]byte, 12)
_, terr = deadlineconn.Read(b)
if terr != nil {
t.Errorf("failed to read from client. %v", terr)
return
}
received := string(b)
if received != "message one\n" {
t.Errorf(`server: expected: "message one\n", got: %v`, received)
return
}
// Wait for more than read timeout to simulate processing.
time.Sleep(3 * time.Second)
_, terr = deadlineconn.Read(b)
if terr != nil {
t.Errorf("failed to read from client. %v", terr)
return
}
received = string(b)
if received != "message two\n" {
t.Errorf(`server: expected: "message two\n", got: %v`, received)
return
}
// Send a response.
_, terr = io.WriteString(deadlineconn, "messages received\n")
if terr != nil {
t.Errorf("failed to write to client. %v", terr)
return
}
}()
c, err := net.Dial("tcp", serverAddr)
if err != nil {
t.Fatalf("unable to connect to server. %v", err)
}
defer c.Close()
_, err = io.WriteString(c, "message one\n")
if err != nil {
t.Fatalf("failed to write to server. %v", err)
}
_, err = io.WriteString(c, "message two\n")
if err != nil {
t.Fatalf("failed to write to server. %v", err)
}
received, err := bufio.NewReader(c).ReadString('\n')
if err != nil {
t.Fatalf("failed to read from server. %v", err)
}
if received != "messages received\n" {
t.Fatalf(`client: expected: "messages received\n", got: %v`, received)
}
wg.Wait()
}
// Test deadlineconn handles read timeout properly by reading two messages beyond deadline.
func TestBuffConnReadCheckTimeout(t *testing.T) {
l, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("unable to create listener. %v", err)
}
defer l.Close()
serverAddr := l.Addr().String()
tcpListener, ok := l.(*net.TCPListener)
if !ok {
t.Fatalf("failed to assert to net.TCPListener")
}
var cerr error
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
tcpConn, terr := tcpListener.AcceptTCP()
if terr != nil {
cerr = fmt.Errorf("failed to accept new connection. %v", terr)
return
}
deadlineconn := New(tcpConn)
deadlineconn.WithReadDeadline(time.Second)
deadlineconn.WithWriteDeadline(time.Second)
defer deadlineconn.Close()
// Read a line
b := make([]byte, 12)
_, terr = deadlineconn.Read(b)
if terr != nil {
cerr = fmt.Errorf("failed to read from client. %v", terr)
return
}
received := string(b)
if received != "message one\n" {
cerr = fmt.Errorf(`server: expected: "message one\n", got: %v`, received)
return
}
// Set a deadline in the past to indicate we want the next read to fail.
// Ensure we don't override it on read.
deadlineconn.SetReadDeadline(time.Unix(1, 0))
// Be sure to exceed update interval
time.Sleep(updateInterval * 2)
_, terr = deadlineconn.Read(b)
if terr == nil {
cerr = fmt.Errorf("could read from client, expected error, got %v", terr)
return
}
}()
c, err := net.Dial("tcp", serverAddr)
if err != nil {
t.Fatalf("unable to connect to server. %v", err)
}
defer c.Close()
_, err = io.WriteString(c, "message one\n")
if err != nil {
t.Fatalf("failed to write to server. %v", err)
}
_, _ = io.WriteString(c, "message two\n")
wg.Wait()
if cerr != nil {
t.Fatal(cerr)
}
}