Adding minio-hash with streaming crypto hashes

cmd/minio-hash/minio-hash-options.go

@@ -0,0 +1,126 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/codegangsta/cli"
+	"github.com/minio-io/minio/pkgs/crypto/md5"
+	"github.com/minio-io/minio/pkgs/crypto/sha1"
+	"github.com/minio-io/minio/pkgs/crypto/sha256"
+	"github.com/minio-io/minio/pkgs/crypto/sha512"
+)
+
+var Options = []cli.Command{
+	Md5sum,
+	Sha1sum,
+	// Sha1sumFast, // not working
+	Sha256sum,
+	Sha512sum,
+}
+
+var Md5sum = cli.Command{
+	Name:  "md5sum",
+	Usage: "",
+	Description: `
+`,
+	Action: doMd5sum,
+}
+
+var Sha1sum = cli.Command{
+	Name:  "sha1sum",
+	Usage: "",
+	Description: `
+`,
+	Action: doSha1sum,
+}
+
+var Sha1sumFast = cli.Command{
+	Name:  "sha1sum-fast",
+	Usage: "",
+	Description: `
+`,
+	Action: doSha1sumFast,
+}
+
+var Sha256sum = cli.Command{
+	Name:  "sha256sum",
+	Usage: "",
+	Description: `
+`,
+	Action: doSha256sum,
+}
+
+var Sha512sum = cli.Command{
+	Name:  "sha512sum",
+	Usage: "",
+	Description: `
+`,
+	Action: doSha512sum,
+}
+
+func doMd5sum(c *cli.Context) {
+	hash, err := md5.Sum(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%x", hash)
+}
+
+func doSha1sum(c *cli.Context) {
+	hash, err := sha1.Sum(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%x", hash)
+}
+
+func doSha1sumFast(c *cli.Context) {
+	buffer, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	hash, err := sha1.Sha1(buffer)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var bytesBuffer bytes.Buffer
+	binary.Write(&bytesBuffer, binary.LittleEndian, hash)
+	fmt.Printf("%x", bytesBuffer.Bytes())
+}
+
+func doSha256sum(c *cli.Context) {
+	hash, err := sha256.Sum(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%x", hash)
+}
+
+func doSha512sum(c *cli.Context) {
+	hash, err := sha512.Sum(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%x", hash)
+}

cmd/minio-hash/minio-hash.go

@@ -0,0 +1,31 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+
+	"github.com/codegangsta/cli"
+)
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "minio-hash"
+	app.Usage = ""
+	app.Commands = Options
+	app.Run(os.Args)
+}

cmd/minio-hash/minio-hash.md

@@ -0,0 +1,13 @@
+% MINIO(1) Minio Manual
+% Minio community
+% December 2014
+# NAME
+md5sum - 
+
+# SYNOPSIS
+
+# DESCRIPTION
+
+# EXAMPLES
+
+# AUTHORS

pkgs/crypto/md5/md5.go

@@ -0,0 +1,22 @@
+package md5
+
+import (
+	"crypto/md5"
+	"io"
+)
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := md5.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/md5/md5_test.go

@@ -0,0 +1,23 @@
+package md5
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestMd5sum(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("0fd3dbec9730101bff92acc820befc34")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

pkgs/crypto/sha1/sha1.go

@@ -23,9 +23,12 @@ package sha1
 // void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
 import "C"
 import (
+	gosha1 "crypto/sha1"
 	"errors"
-	"github.com/minio-io/minio/pkgs/cpu"
+	"io"
 	"unsafe"
+
+	"github.com/minio-io/minio/pkgs/cpu"
 )
 
 const (
@@ -69,3 +72,19 @@ func Sha1(buffer []byte) ([]int32, error) {
 
 	return shbuf, nil
 }
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := gosha1.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/sha1/sha1_test.go

@@ -1,8 +1,11 @@
 package sha1
 
 import (
-	. "gopkg.in/check.v1"
+	"bytes"
+	"encoding/hex"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -24,3 +27,11 @@ func (s *MySuite) TestSha1(c *C) {
 
 	c.Assert(sha, DeepEquals, newsha)
 }
+
+func (s *MySuite) TestStreamingSha1(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("18af819125b70879d36378431c4e8d9bfa6a2599")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

pkgs/crypto/sha256/sha256.go

@@ -17,6 +17,10 @@ package sha256
 // #define SHA256_H6       0x1f83d9abUL
 // #define SHA256_H7       0x5be0cd19UL
 import "C"
+import (
+	gosha256 "crypto/sha256"
+	"io"
+)
 
 /*
 func Sha256(buffer []byte) ([]uint32, error) {
@@ -37,3 +41,19 @@ func Sha256(buffer []byte) ([]uint32, error) {
 	}
 }
 */
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := gosha256.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/sha256/sha256_test.go

@@ -0,0 +1,23 @@
+package sha256
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestSha256Stream(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("a3e49d843df13c2e2a7786f6ecd7e0d184f45d718d1ac1a8a63e570466e489dd")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}

pkgs/crypto/sha512/sha512.go

@@ -7,3 +7,23 @@ package sha512
 // void sha512_transform_ssse3(const void* M, void* D, uint64_t L);
 // void sha512_transform_rorx(const void* M, void* D, uint64_t L);
 import "C"
+import (
+	gosha512 "crypto/sha512"
+	"io"
+)
+
+func Sum(reader io.Reader) ([]byte, error) {
+	hash := gosha512.New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		hash.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return hash.Sum(nil), nil
+}

pkgs/crypto/sha512/sha512_test.go

@@ -0,0 +1,23 @@
+package sha512
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestSha512Stream(c *C) {
+	testString := []byte("Test string")
+	expectedHash, _ := hex.DecodeString("811aa0c53c0039b6ead0ca878b096eed1d39ed873fd2d2d270abfb9ca620d3ed561c565d6dbd1114c323d38e3f59c00df475451fc9b30074f2abda3529df2fa7")
+	hash, err := Sum(bytes.NewBuffer(testString))
+	c.Assert(err, IsNil)
+	c.Assert(bytes.Equal(expectedHash, hash), Equals, true)
+}