Look for network errors appropriately for RemoteStorageAPI (#8128)

net.Error is very unreliable in providing better error
handling, we need to ensure that we always have a fallback
option in case of network failures.

This fixes an important issue in our distributed server
setups when one of the servers is down, all deployments
out there are recommended to upgrade after this fix is
merged to ensure that availability is not lost.

Fixes #8127
Fixes #8016
Fixes #7964
This commit is contained in:
Harshavardhana 2019-08-25 13:32:49 -07:00 committed by GitHub
parent d6dd98e597
commit 70136fb55b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -29,6 +29,7 @@ import (
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
@ -446,10 +447,25 @@ func isNetworkOrHostDown(err error) bool {
// or a non-temporary error.
e, ok := err.(net.Error)
if ok {
return e.Timeout()
urlErr, ok := e.(*url.Error)
if ok {
switch urlErr.Err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
return true
}
}
if e.Timeout() {
return true
}
}
ok = false
// Fallback to other mechanisms.
if strings.Contains(err.Error(), "i/o timeout") {
if strings.Contains(err.Error(), "Connection closed by foreign host") {
ok = true
} else if strings.Contains(err.Error(), "TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError.
ok = true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError.
ok = true
} else if strings.Contains(err.Error(), "connection timed out") {