Adding minio-hash with streaming crypto hashes

pkgs/crypto/md5/md5.go

@@ -0,0 +1,22 @@
+package md5
+
+import (
+	"crypto/md5"
+	"io"
+)
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := md5.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/md5/md5_test.go

@@ -0,0 +1,23 @@
+package md5
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestMd5sum(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("0fd3dbec9730101bff92acc820befc34")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

pkgs/crypto/sha1/sha1.go

@@ -23,9 +23,12 @@ package sha1
 // void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
 import "C"
 import (
+	gosha1 "crypto/sha1"
 	"errors"
-	"github.com/minio-io/minio/pkgs/cpu"
+	"io"
 	"unsafe"
+
+	"github.com/minio-io/minio/pkgs/cpu"
 )
 
 const (
@@ -69,3 +72,19 @@ func Sha1(buffer []byte) ([]int32, error) {
 
 	return shbuf, nil
 }
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := gosha1.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/sha1/sha1_test.go

@@ -1,8 +1,11 @@
 package sha1
 
 import (
-	. "gopkg.in/check.v1"
+	"bytes"
+	"encoding/hex"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -24,3 +27,11 @@ func (s *MySuite) TestSha1(c *C) {
 
 	c.Assert(sha, DeepEquals, newsha)
 }
+
+func (s *MySuite) TestStreamingSha1(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("18af819125b70879d36378431c4e8d9bfa6a2599")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

pkgs/crypto/sha256/sha256.go

@@ -17,6 +17,10 @@ package sha256
 // #define SHA256_H6       0x1f83d9abUL
 // #define SHA256_H7       0x5be0cd19UL
 import "C"
+import (
+	gosha256 "crypto/sha256"
+	"io"
+)
 
 /*
 func Sha256(buffer []byte) ([]uint32, error) {
@@ -37,3 +41,19 @@ func Sha256(buffer []byte) ([]uint32, error) {
 	}
 }
 */
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := gosha256.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/sha256/sha256_test.go

@@ -0,0 +1,23 @@
+package sha256
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestSha256Stream(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("a3e49d843df13c2e2a7786f6ecd7e0d184f45d718d1ac1a8a63e570466e489dd")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

pkgs/crypto/sha512/sha512.go

@@ -7,3 +7,23 @@ package sha512
 // void sha512_transform_ssse3(const void* M, void* D, uint64_t L);
 // void sha512_transform_rorx(const void* M, void* D, uint64_t L);
 import "C"
+import (
+	gosha512 "crypto/sha512"
+	"io"
+)
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := gosha512.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/sha512/sha512_test.go

@@ -0,0 +1,23 @@
+package sha512
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestSha512Stream(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("811aa0c53c0039b6ead0ca878b096eed1d39ed873fd2d2d270abfb9ca620d3ed561c565d6dbd1114c323d38e3f59c00df475451fc9b30074f2abda3529df2fa7")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}