Merge pull request #259 from harshavardhana/pr_out_now_able_to_use_crc32c_and_sha1_on_darwin_with_osx_specific_code

pkg/utils/checksum/crc32c/crc32c_darwin.go

@@ -17,7 +17,7 @@
 package crc32c
 
 import (
-	"errors"
+	"io"
 	"hash/crc32"
 )
 
@@ -35,7 +35,7 @@ func Sum32(buffer []byte) uint32 {
 
 // Sum - io.Reader based crc helper
 func Sum(reader io.Reader) (uint32, error) {
-	h := New()
+	h := crc32.New(castanagoliTable)
 	var err error
 	for err == nil {
 		length := 0
@@ -45,7 +45,7 @@ func Sum(reader io.Reader) (uint32, error) {
 		h.Write(byteBuffer)
 	}
 	if err != io.EOF {
-		return nil, err
+		return 0, err
 	}
 	return h.Sum32(), nil
 }

pkg/utils/crypto/sha1/sha1_darwin.go

@@ -0,0 +1,167 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174.
+package sha1
+
+import (
+	"hash"
+	"io"
+
+	"github.com/minio-io/minio/pkg/utils/cpu"
+)
+
+// The size of a SHA1 checksum in bytes.
+const Size = 20
+
+// The blocksize of SHA1 in bytes.
+const BlockSize = 64
+
+const (
+	chunk = 64
+	init0 = 0x67452301
+	init1 = 0xEFCDAB89
+	init2 = 0x98BADCFE
+	init3 = 0x10325476
+	init4 = 0xC3D2E1F0
+)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	h   [5]uint32
+	x   [chunk]byte
+	nx  int
+	len uint64
+}
+
+// Reset digest
+func (d *digest) Reset() {
+	d.h[0] = init0
+	d.h[1] = init1
+	d.h[2] = init2
+	d.h[3] = init3
+	d.h[4] = init4
+	d.nx = 0
+	d.len = 0
+}
+
+// New returns a new hash.Hash computing the SHA1 checksum.
+func New() hash.Hash {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+func block(dig *digest, p []byte) {
+	switch true {
+	case cpu.HasSSE41() == true:
+		blockSSE3(dig, p)
+	default:
+		blockGeneric(dig, p)
+	}
+}
+
+// Return output size
+func (d *digest) Size() int { return Size }
+
+// Return checksum blocksize
+func (d *digest) BlockSize() int { return BlockSize }
+
+// Write to digest
+func (d *digest) Write(p []byte) (nn int, err error) {
+	nn = len(p)
+	d.len += uint64(nn)
+	if d.nx > 0 {
+		n := copy(d.x[d.nx:], p)
+		d.nx += n
+		if d.nx == chunk {
+			block(d, d.x[:])
+			d.nx = 0
+		}
+		p = p[n:]
+	}
+	if len(p) >= chunk {
+		n := len(p) &^ (chunk - 1)
+		block(d, p[:n])
+		p = p[n:]
+	}
+	if len(p) > 0 {
+		d.nx = copy(d.x[:], p)
+	}
+	return
+}
+
+// Return checksum bytes
+func (d0 *digest) Sum(in []byte) []byte {
+	// Make a copy of d0 so that caller can keep writing and summing.
+	d := *d0
+	hash := d.checkSum()
+	return append(in, hash[:]...)
+}
+
+// Intermediate checksum function
+func (d *digest) checkSum() [Size]byte {
+	len := d.len
+	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
+	var tmp [64]byte
+	tmp[0] = 0x80
+	if len%64 < 56 {
+		d.Write(tmp[0 : 56-len%64])
+	} else {
+		d.Write(tmp[0 : 64+56-len%64])
+	}
+
+	// Length in bits.
+	len <<= 3
+	for i := uint(0); i < 8; i++ {
+		tmp[i] = byte(len >> (56 - 8*i))
+	}
+	d.Write(tmp[0:8])
+
+	if d.nx != 0 {
+		panic("d.nx != 0")
+	}
+
+	var digest [Size]byte
+	for i, s := range d.h {
+		digest[i*4] = byte(s >> 24)
+		digest[i*4+1] = byte(s >> 16)
+		digest[i*4+2] = byte(s >> 8)
+		digest[i*4+3] = byte(s)
+	}
+
+	return digest
+}
+
+/// Convenience functions
+
+// Sum1 - single caller sha1 helper
+func Sum1(data []byte) [Size]byte {
+	var d digest
+	d.Reset()
+	d.Write(data)
+	return d.checkSum()
+}
+
+// Sum - io.Reader based streaming sha1 helper
+func Sum(reader io.Reader) ([]byte, error) {
+	h := New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		h.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return h.Sum(nil), nil
+}

pkg/utils/crypto/sha1/sha1_yasm_darwin.go

@@ -2,4 +2,4 @@
 
 package sha1
 
-//go:generate yasm -f macho64 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso
+//go:generate yasm -f macho64 -DINTEL_SHA1_UPDATE_FUNCNAME=_sha1_update_intel sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

pkg/utils/crypto/sha1/sha1block_darwin.go

@@ -0,0 +1,13 @@
+// +build amd64
+
+package sha1
+
+// #include <stdint.h>
+// #include <stdlib.h>
+// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
+import "C"
+import "unsafe"
+
+func blockSSE3(dig *digest, p []byte) {
+	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
+}

pkg/utils/crypto/sha1/sha1block_generic.go

@@ -2,14 +2,6 @@
 
 package sha1
 
-// #cgo CFLAGS: -DHAS_AVX2
-// #include <stdint.h>
-// #include <stdlib.h>
-// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
-// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks );
-import "C"
-import "unsafe"
-
 const (
 	_K0 = 0x5A827999
 	_K1 = 0x6ED9EBA1
@@ -17,14 +9,6 @@ const (
 	_K3 = 0xCA62C1D6
 )
 
-func blockAVX2(dig *digest, p []byte) {
-	C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
-}
-
-func blockSSE3(dig *digest, p []byte) {
-	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
-}
-
 // blockGeneric is a portable, pure Go version of the SHA1 block step.
 // It's used by sha1block_generic.go and tests.
 func blockGeneric(dig *digest, p []byte) {

pkg/utils/crypto/sha1/sha1block_linux.go

@@ -0,0 +1,21 @@
+// +build amd64
+
+package sha1
+
+// #cgo CFLAGS: -DHAS_AVX2
+// #include <stdint.h>
+// #include <stdlib.h>
+// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
+// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
+import "C"
+import "unsafe"
+
+func blockAVX2(dig *digest, p []byte) {
+	C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
+}
+
+func blockSSE3(dig *digest, p []byte) {
+	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
+}
+
+