Add Support for Manta Object Storage as a Gateway (#5025)

Manta is an Object Storage by [Joyent](https://www.joyent.com/)

This PR adds initial support for Manta. It is intended as non-production 
ready so that feedback can be obtained.
This commit is contained in:
Paul Stack
2017-12-20 10:07:56 +02:00
committed by Nitish Tiwari
parent 1f77708a30
commit 7d75d61621
69 changed files with 20974 additions and 0 deletions

View File

@@ -0,0 +1,72 @@
package authentication
// DON'T USE THIS OUTSIDE TESTING ~ This key was only created to use for
// internal unit testing. It should never be used for acceptance testing either.
//
// This is just a randomly generated key pair.
var Dummy = struct {
Fingerprint string
PrivateKey []byte
PublicKey []byte
Signer Signer
}{
"9f:d6:65:fc:d6:60:dc:d0:4e:db:2d:75:f7:92:8c:31",
[]byte(`-----BEGIN RSA PRIVATE KEY-----
MIIJKAIBAAKCAgEAui9lNjCJahHeFSFC6HXi/CNX588C/L2gJUx65bnNphVC98hW
1wzoRvPXHx5aWnb7lEbpNhP6B0UoCBDTaPgt9hHfD/oNQ+6HT1QpDIGfZmXI91/t
cjGVSBbxN7WaYt/HsPrGjbalwvQPChN53sMVmFkMTEDR5G3zOBOAGrOimlCT80wI
2S5Xg0spd8jjKM5I1swDR0xtuDWnHTR1Ohin+pEQIE6glLTfYq7oQx6nmMXXBNmk
+SaPD1FAyjkF/81im2EHXBygNEwraVrDcAxK2mKlU2XMJiogQKNYWlm3UkbNB6WP
Le12+Ka02rmIVsSqIpc/ZCBraAlCaSWlYCkU+vJ2hH/+ypy5bXNlbaTiWZK+vuI7
PC87T50yLNeXVuNZAynzDpBCvsjiiHrB/ZFRfVfF6PviV8CV+m7GTzfAwJhVeSbl
rR6nts16K0HTD48v57DU0b0t5VOvC7cWPShs+afdSL3Z8ReL5EWMgU1wfvtycRKe
hiDVGj3Ms2cf83RIANr387G+1LcTQYP7JJuB7Svy5j+R6+HjI0cgu4EMUPdWfCNG
GyrlxwJNtPmUSfasH1xUKpqr7dC+0sN4/gfJw75WTAYrATkPzexoYNaMsGDfhuoh
kYa3Tn2q1g3kqhsX/R0Fd5d8d5qc137qcRCxiZYz9f3bVkXQbhYmO9da3KsCAwEA
AQKCAgAeEAURqOinPddUJhi9nDtYZwSMo3piAORY4W5+pW+1P32esLSE6MqgmkLD
/YytSsT4fjKtzq/yeJIsKztXmasiLmSMGd4Gd/9VKcuu/0cTq5+1gcG/TI5EI6Az
VJlnGacOxo9E1pcRUYMUJ2zoMSvNe6NmtJivf6lkBpIKvbKlpBkfkclj9/2db4d0
lfVH43cTZ8Gnw4l70v320z+Sb+S/qqil7swy9rmTH5bVL5/0JQ3A9LuUl0tGN+J0
RJzZXvprCFG958leaGYiDsu7zeBQPtlfC/LYvriSd02O2SmmmVQFxg/GZK9vGsvc
/VQsXnjyOOW9bxaop8YXYELBsiB21ipTHzOwoqHT8wFnjgU9Y/7iZIv7YbZKQsCS
DrwdlZ/Yw90wiif+ldYryIVinWfytt6ERv4Dgezc98+1XPi1Z/WB74/lIaDXFl3M
3ypjtvLYbKew2IkIjeAwjvZJg/QpC/50RrrPtVDgeAI1Ni01ikixUhMYsHJ1kRih
0tqLvLqSPoHmr6luFlaoKdc2eBqb+8U6K/TrXhKtT7BeUFiSbvnVfdbrH9r+AY/2
zYtG6llzkE5DH8ZR3Qp+dx7QEDtvYhGftWhx9uasd79AN7CuGYnL54YFLKGRrWKN
ylysqfUyOQYiitdWdNCw9PP2vGRx5JAsMMSy+ft18jjTJvNQ0QKCAQEA28M11EE6
MpnHxfyP00Dl1+3wl2lRyNXZnZ4hgkk1f83EJGpoB2amiMTF8P1qJb7US1fXtf7l
gkJMMk6t6iccexV1/NBh/7tDZHH/v4HPirFTXQFizflaghD8dEADy9DY4BpQYFRe
8zGsv4/4U0txCXkUIfKcENt/FtXv2T9blJT6cDV0yTx9IAyd4Kor7Ly2FIYroSME
uqnOQt5PwB+2qkE+9hdg4xBhFs9sW5dvyBvQvlBfX/xOmMw2ygH6vsaJlNfZ5VPa
EP/wFP/qHyhDlCfbHdL6qF2//wUoM2QM9RgBdZNhcKU7zWuf7Ev199tmlLC5O14J
PkQxUGftMfmWxQKCAQEA2OLKD8dwOzpwGJiPQdBmGpwCamfcCY4nDwqEaCu4vY1R
OJR+rpYdC2hgl5PTXWH7qzJVdT/ZAz2xUQOgB1hD3Ltk7DQ+EZIA8+vJdaicQOme
vfpMPNDxCEX9ee0AXAmAC3aET82B4cMFnjXjl1WXLLTowF/Jp/hMorm6tl2m15A2
oTyWlB/i/W/cxHl2HFWK7o8uCNoKpKJjheNYn+emEcH1bkwrk8sxQ78cBNmqe/gk
MLgu8qfXQ0LLKIL7wqmIUHeUpkepOod8uXcTmmN2X9saCIwFKx4Jal5hh5v5cy0G
MkyZcUIhhnmzr7lXbepauE5V2Sj5Qp040AfRVjZcrwKCAQANe8OwuzPL6P2F20Ij
zwaLIhEx6QdYkC5i6lHaAY3jwoc3SMQLODQdjh0q9RFvMW8rFD+q7fG89T5hk8w9
4ppvvthXY52vqBixcAEmCdvnAYxA15XtV1BDTLGAnHDfL3gu/85QqryMpU6ZDkdJ
LQbJcwFWN+F1c1Iv335w0N9YlW9sNQtuUWTH8544K5i4VLfDOJwyrchbf5GlLqir
/AYkGg634KVUKSwbzywxzm/QUkyTcLD5Xayg2V6/NDHjRKEqXbgDxwpJIrrjPvRp
ZvoGfA+Im+o/LElcZz+ZL5lP7GIiiaFf3PN3XhQY1mxIAdEgbFthFhrxFBQGf+ng
uBSVAoIBAHl12K8pg8LHoUtE9MVoziWMxRWOAH4ha+JSg4BLK/SLlbbYAnIHg1CG
LcH1eWNMokJnt9An54KXJBw4qYAzgB23nHdjcncoivwPSg1oVclMjCfcaqGMac+2
UpPblF32vAyvXL3MWzZxn03Q5Bo2Rqk0zzwc6LP2rARdeyDyJaOHEfEOG03s5ZQE
91/YnbqUdW/QI3m1kkxM3Ot4PIOgmTJMqwQQCD+GhZppBmn49C7k8m+OVkxyjm0O
lPOlFxUXGE3oCgltDGrIwaKj+wh1Ny/LZjLvJ13UPnWhUYE+al6EEnpMx4nT/S5w
LZ71bu8RVajtxcoN1jnmDpECL8vWOeUCggEBAIEuKoY7pVHfs5gr5dXfQeVZEtqy
LnSdsd37/aqQZRlUpVmBrPNl1JBLiEVhk2SL3XJIDU4Er7f0idhtYLY3eE7wqZ4d
38Iaj5tv3zBc/wb1bImPgOgXCH7QrrbW7uTiYMLScuUbMR4uSpfubLaV8Zc9WHT8
kTJ2pKKtA1GPJ4V7HCIxuTjD2iyOK1CRkaqSC+5VUuq5gHf92CEstv9AIvvy5cWg
gnfBQoS89m3aO035henSfRFKVJkHaEoasj8hB3pwl9FGZUJp1c2JxiKzONqZhyGa
6tcIAM3od0QtAfDJ89tWJ5D31W8KNNysobFSQxZ62WgLUUtXrkN1LGodxGQ=
-----END RSA PRIVATE KEY-----`),
[]byte(`ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC6L2U2MIlqEd4VIULodeL8I1fnzwL8vaAlTHrluc2mFUL3yFbXDOhG89cfHlpadvuURuk2E/oHRSgIENNo+C32Ed8P+g1D7odPVCkMgZ9mZcj3X+1yMZVIFvE3tZpi38ew+saNtqXC9A8KE3newxWYWQxMQNHkbfM4E4Aas6KaUJPzTAjZLleDSyl3yOMozkjWzANHTG24NacdNHU6GKf6kRAgTqCUtN9iruhDHqeYxdcE2aT5Jo8PUUDKOQX/zWKbYQdcHKA0TCtpWsNwDEraYqVTZcwmKiBAo1haWbdSRs0HpY8t7Xb4prTauYhWxKoilz9kIGtoCUJpJaVgKRT68naEf/7KnLltc2VtpOJZkr6+4js8LztPnTIs15dW41kDKfMOkEK+yOKIesH9kVF9V8Xo++JXwJX6bsZPN8DAmFV5JuWtHqe2zXorQdMPjy/nsNTRvS3lU68LtxY9KGz5p91IvdnxF4vkRYyBTXB++3JxEp6GINUaPcyzZx/zdEgA2vfzsb7UtxNBg/skm4HtK/LmP5Hr4eMjRyC7gQxQ91Z8I0YbKuXHAk20+ZRJ9qwfXFQqmqvt0L7Sw3j+B8nDvlZMBisBOQ/N7Ghg1oywYN+G6iGRhrdOfarWDeSqGxf9HQV3l3x3mpzXfupxELGJljP1/dtWRdBuFiY711rcqw== test-dummy-20171002140848`),
nil,
}
func init() {
testSigner, _ := NewTestSigner()
Dummy.Signer = testSigner
}

View File

@@ -0,0 +1,66 @@
package authentication
import (
"encoding/asn1"
"encoding/base64"
"fmt"
"math/big"
"github.com/hashicorp/errwrap"
"golang.org/x/crypto/ssh"
)
type ecdsaSignature struct {
hashAlgorithm string
R *big.Int
S *big.Int
}
func (s *ecdsaSignature) SignatureType() string {
return fmt.Sprintf("ecdsa-%s", s.hashAlgorithm)
}
func (s *ecdsaSignature) String() string {
toEncode := struct {
R *big.Int
S *big.Int
}{
R: s.R,
S: s.S,
}
signatureBytes, err := asn1.Marshal(toEncode)
if err != nil {
panic(fmt.Sprintf("Error marshaling signature: %s", err))
}
return base64.StdEncoding.EncodeToString(signatureBytes)
}
func newECDSASignature(signatureBlob []byte) (*ecdsaSignature, error) {
var ecSig struct {
R *big.Int
S *big.Int
}
if err := ssh.Unmarshal(signatureBlob, &ecSig); err != nil {
return nil, errwrap.Wrapf("Error unmarshaling signature: {{err}}", err)
}
rValue := ecSig.R.Bytes()
var hashAlgorithm string
switch len(rValue) {
case 31, 32:
hashAlgorithm = "sha256"
case 65, 66:
hashAlgorithm = "sha512"
default:
return nil, fmt.Errorf("Unsupported key length: %d", len(rValue))
}
return &ecdsaSignature{
hashAlgorithm: hashAlgorithm,
R: ecSig.R,
S: ecSig.S,
}, nil
}

View File

@@ -0,0 +1,106 @@
package authentication
import (
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
"strings"
"github.com/hashicorp/errwrap"
"golang.org/x/crypto/ssh"
)
type PrivateKeySigner struct {
formattedKeyFingerprint string
keyFingerprint string
algorithm string
accountName string
hashFunc crypto.Hash
privateKey *rsa.PrivateKey
}
func NewPrivateKeySigner(keyFingerprint string, privateKeyMaterial []byte, accountName string) (*PrivateKeySigner, error) {
keyFingerprintMD5 := strings.Replace(keyFingerprint, ":", "", -1)
block, _ := pem.Decode(privateKeyMaterial)
if block == nil {
return nil, errors.New("Error PEM-decoding private key material: nil block received")
}
rsakey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
return nil, errwrap.Wrapf("Error parsing private key: {{err}}", err)
}
sshPublicKey, err := ssh.NewPublicKey(rsakey.Public())
if err != nil {
return nil, errwrap.Wrapf("Error parsing SSH key from private key: {{err}}", err)
}
matchKeyFingerprint := formatPublicKeyFingerprint(sshPublicKey, false)
displayKeyFingerprint := formatPublicKeyFingerprint(sshPublicKey, true)
if matchKeyFingerprint != keyFingerprintMD5 {
return nil, errors.New("Private key file does not match public key fingerprint")
}
signer := &PrivateKeySigner{
formattedKeyFingerprint: displayKeyFingerprint,
keyFingerprint: keyFingerprint,
accountName: accountName,
hashFunc: crypto.SHA1,
privateKey: rsakey,
}
_, algorithm, err := signer.SignRaw("HelloWorld")
if err != nil {
return nil, fmt.Errorf("Cannot sign using ssh agent: %s", err)
}
signer.algorithm = algorithm
return signer, nil
}
func (s *PrivateKeySigner) Sign(dateHeader string) (string, error) {
const headerName = "date"
hash := s.hashFunc.New()
hash.Write([]byte(fmt.Sprintf("%s: %s", headerName, dateHeader)))
digest := hash.Sum(nil)
signed, err := rsa.SignPKCS1v15(rand.Reader, s.privateKey, s.hashFunc, digest)
if err != nil {
return "", errwrap.Wrapf("Error signing date header: {{err}}", err)
}
signedBase64 := base64.StdEncoding.EncodeToString(signed)
keyID := fmt.Sprintf("/%s/keys/%s", s.accountName, s.formattedKeyFingerprint)
return fmt.Sprintf(authorizationHeaderFormat, keyID, "rsa-sha1", headerName, signedBase64), nil
}
func (s *PrivateKeySigner) SignRaw(toSign string) (string, string, error) {
hash := s.hashFunc.New()
hash.Write([]byte(toSign))
digest := hash.Sum(nil)
signed, err := rsa.SignPKCS1v15(rand.Reader, s.privateKey, s.hashFunc, digest)
if err != nil {
return "", "", errwrap.Wrapf("Error signing date header: {{err}}", err)
}
signedBase64 := base64.StdEncoding.EncodeToString(signed)
return signedBase64, "rsa-sha1", nil
}
func (s *PrivateKeySigner) KeyFingerprint() string {
return s.formattedKeyFingerprint
}
func (s *PrivateKeySigner) DefaultAlgorithm() string {
return s.algorithm
}

View File

@@ -0,0 +1,25 @@
package authentication
import (
"encoding/base64"
)
type rsaSignature struct {
hashAlgorithm string
signature []byte
}
func (s *rsaSignature) SignatureType() string {
return s.hashAlgorithm
}
func (s *rsaSignature) String() string {
return base64.StdEncoding.EncodeToString(s.signature)
}
func newRSASignature(signatureBlob []byte) (*rsaSignature, error) {
return &rsaSignature{
hashAlgorithm: "rsa-sha1",
signature: signatureBlob,
}, nil
}

View File

@@ -0,0 +1,27 @@
package authentication
import (
"regexp"
"fmt"
)
type httpAuthSignature interface {
SignatureType() string
String() string
}
func keyFormatToKeyType(keyFormat string) (string, error) {
if keyFormat == "ssh-rsa" {
return "rsa", nil
}
if keyFormat == "ssh-ed25519" {
return "ed25519", nil
}
if regexp.MustCompile("^ecdsa-sha2-*").Match([]byte(keyFormat)) {
return "ecdsa", nil
}
return "", fmt.Errorf("Unknown key format: %s", keyFormat)
}

View File

@@ -0,0 +1,10 @@
package authentication
const authorizationHeaderFormat = `Signature keyId="%s",algorithm="%s",headers="%s",signature="%s"`
type Signer interface {
DefaultAlgorithm() string
KeyFingerprint() string
Sign(dateHeader string) (string, error)
SignRaw(toSign string) (string, string, error)
}

View File

@@ -0,0 +1,170 @@
package authentication
import (
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"net"
"os"
"strings"
"github.com/hashicorp/errwrap"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
)
var (
ErrUnsetEnvVar = errors.New("SSH_AUTH_SOCK is not set")
)
type SSHAgentSigner struct {
formattedKeyFingerprint string
keyFingerprint string
algorithm string
accountName string
keyIdentifier string
agent agent.Agent
key ssh.PublicKey
}
func NewSSHAgentSigner(keyFingerprint, accountName string) (*SSHAgentSigner, error) {
sshAgentAddress, agentOk := os.LookupEnv("SSH_AUTH_SOCK")
if !agentOk {
return nil, ErrUnsetEnvVar
}
conn, err := net.Dial("unix", sshAgentAddress)
if err != nil {
return nil, errwrap.Wrapf("Error dialing SSH agent: {{err}}", err)
}
ag := agent.NewClient(conn)
signer := &SSHAgentSigner{
keyFingerprint: keyFingerprint,
accountName: accountName,
agent: ag,
}
matchingKey, err := signer.MatchKey()
if err != nil {
return nil, err
}
signer.key = matchingKey
signer.formattedKeyFingerprint = formatPublicKeyFingerprint(signer.key, true)
signer.keyIdentifier = fmt.Sprintf("/%s/keys/%s", signer.accountName, signer.formattedKeyFingerprint)
_, algorithm, err := signer.SignRaw("HelloWorld")
if err != nil {
return nil, fmt.Errorf("Cannot sign using ssh agent: %s", err)
}
signer.algorithm = algorithm
return signer, nil
}
func (s *SSHAgentSigner) MatchKey() (ssh.PublicKey, error) {
keys, err := s.agent.List()
if err != nil {
return nil, errwrap.Wrapf("Error listing keys in SSH Agent: %s", err)
}
keyFingerprintStripped := strings.TrimPrefix(s.keyFingerprint, "MD5:")
keyFingerprintStripped = strings.TrimPrefix(keyFingerprintStripped, "SHA256:")
keyFingerprintStripped = strings.Replace(keyFingerprintStripped, ":", "", -1)
var matchingKey ssh.PublicKey
for _, key := range keys {
keyMD5 := md5.New()
keyMD5.Write(key.Marshal())
finalizedMD5 := fmt.Sprintf("%x", keyMD5.Sum(nil))
keySHA256 := sha256.New()
keySHA256.Write(key.Marshal())
finalizedSHA256 := base64.RawStdEncoding.EncodeToString(keySHA256.Sum(nil))
if keyFingerprintStripped == finalizedMD5 || keyFingerprintStripped == finalizedSHA256 {
matchingKey = key
}
}
if matchingKey == nil {
return nil, fmt.Errorf("No key in the SSH Agent matches fingerprint: %s", s.keyFingerprint)
}
return matchingKey, nil
}
func (s *SSHAgentSigner) Sign(dateHeader string) (string, error) {
const headerName = "date"
signature, err := s.agent.Sign(s.key, []byte(fmt.Sprintf("%s: %s", headerName, dateHeader)))
if err != nil {
return "", errwrap.Wrapf("Error signing date header: {{err}}", err)
}
keyFormat, err := keyFormatToKeyType(signature.Format)
if err != nil {
return "", errwrap.Wrapf("Error reading signature: {{err}}", err)
}
var authSignature httpAuthSignature
switch keyFormat {
case "rsa":
authSignature, err = newRSASignature(signature.Blob)
if err != nil {
return "", errwrap.Wrapf("Error reading signature: {{err}}", err)
}
case "ecdsa":
authSignature, err = newECDSASignature(signature.Blob)
if err != nil {
return "", errwrap.Wrapf("Error reading signature: {{err}}", err)
}
default:
return "", fmt.Errorf("Unsupported algorithm from SSH agent: %s", signature.Format)
}
return fmt.Sprintf(authorizationHeaderFormat, s.keyIdentifier,
authSignature.SignatureType(), headerName, authSignature.String()), nil
}
func (s *SSHAgentSigner) SignRaw(toSign string) (string, string, error) {
signature, err := s.agent.Sign(s.key, []byte(toSign))
if err != nil {
return "", "", errwrap.Wrapf("Error signing string: {{err}}", err)
}
keyFormat, err := keyFormatToKeyType(signature.Format)
if err != nil {
return "", "", errwrap.Wrapf("Error reading signature: {{err}}", err)
}
var authSignature httpAuthSignature
switch keyFormat {
case "rsa":
authSignature, err = newRSASignature(signature.Blob)
if err != nil {
return "", "", errwrap.Wrapf("Error reading signature: {{err}}", err)
}
case "ecdsa":
authSignature, err = newECDSASignature(signature.Blob)
if err != nil {
return "", "", errwrap.Wrapf("Error reading signature: {{err}}", err)
}
default:
return "", "", fmt.Errorf("Unsupported algorithm from SSH agent: %s", signature.Format)
}
return authSignature.String(), authSignature.SignatureType(), nil
}
func (s *SSHAgentSigner) KeyFingerprint() string {
return s.formattedKeyFingerprint
}
func (s *SSHAgentSigner) DefaultAlgorithm() string {
return s.algorithm
}

View File

@@ -0,0 +1,27 @@
package authentication
// TestSigner represents an authentication key signer which we can use for
// testing purposes only. This will largely be a stub to send through client
// unit tests.
type TestSigner struct{}
// NewTestSigner constructs a new instance of test signer
func NewTestSigner() (Signer, error) {
return &TestSigner{}, nil
}
func (s *TestSigner) DefaultAlgorithm() string {
return ""
}
func (s *TestSigner) KeyFingerprint() string {
return ""
}
func (s *TestSigner) Sign(dateHeader string) (string, error) {
return "", nil
}
func (s *TestSigner) SignRaw(toSign string) (string, string, error) {
return "", "", nil
}

View File

@@ -0,0 +1,29 @@
package authentication
import (
"crypto/md5"
"fmt"
"strings"
"golang.org/x/crypto/ssh"
)
// formatPublicKeyFingerprint produces the MD5 fingerprint of the given SSH
// public key. If display is true, the fingerprint is formatted with colons
// between each byte, as per the output of OpenSSL.
func formatPublicKeyFingerprint(key ssh.PublicKey, display bool) string {
publicKeyFingerprint := md5.New()
publicKeyFingerprint.Write(key.Marshal())
publicKeyFingerprintString := fmt.Sprintf("%x", publicKeyFingerprint.Sum(nil))
if !display {
return publicKeyFingerprintString
}
formatted := ""
for i := 0; i < len(publicKeyFingerprintString); i = i + 2 {
formatted = fmt.Sprintf("%s%s:", formatted, publicKeyFingerprintString[i:i+2])
}
return strings.TrimSuffix(formatted, ":")
}