From 7cd6f89c4b579b28b8c2a652b46a6c33993644fb Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 27 Apr 2021 09:49:26 -0700 Subject: [PATCH] move argon2,csvparser into their repos --- contrib/pkg/argon2/argon2.go | 333 ----------------- contrib/pkg/argon2/argon2_test.go | 280 -------------- contrib/pkg/argon2/blake2b.go | 53 --- contrib/pkg/argon2/blamka_amd64.go | 60 --- contrib/pkg/argon2/blamka_amd64.s | 243 ------------ contrib/pkg/argon2/blamka_generic.go | 163 --------- contrib/pkg/argon2/blamka_ref.go | 15 - contrib/pkg/csvparser/example_test.go | 131 ------- contrib/pkg/csvparser/fuzz.go | 70 ---- contrib/pkg/csvparser/reader.go | 458 ----------------------- contrib/pkg/csvparser/reader_test.go | 509 -------------------------- contrib/pkg/csvparser/writer.go | 179 --------- contrib/pkg/csvparser/writer_test.go | 102 ------ go.mod | 2 + go.sum | 4 + pkg/madmin/encrypt.go | 2 +- pkg/s3select/csv/reader.go | 2 +- pkg/s3select/csv/record.go | 2 +- pkg/s3select/json/record.go | 2 +- pkg/s3select/simdj/record.go | 2 +- 20 files changed, 11 insertions(+), 2601 deletions(-) delete mode 100644 contrib/pkg/argon2/argon2.go delete mode 100644 contrib/pkg/argon2/argon2_test.go delete mode 100644 contrib/pkg/argon2/blake2b.go delete mode 100644 contrib/pkg/argon2/blamka_amd64.go delete mode 100644 contrib/pkg/argon2/blamka_amd64.s delete mode 100644 contrib/pkg/argon2/blamka_generic.go delete mode 100644 contrib/pkg/argon2/blamka_ref.go delete mode 100644 contrib/pkg/csvparser/example_test.go delete mode 100644 contrib/pkg/csvparser/fuzz.go delete mode 100644 contrib/pkg/csvparser/reader.go delete mode 100644 contrib/pkg/csvparser/reader_test.go delete mode 100644 contrib/pkg/csvparser/writer.go delete mode 100644 contrib/pkg/csvparser/writer_test.go diff --git a/contrib/pkg/argon2/argon2.go b/contrib/pkg/argon2/argon2.go deleted file mode 100644 index 5953f10ed..000000000 --- a/contrib/pkg/argon2/argon2.go +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the https://golang.org/LICENSE file. - -// Package argon2 implements the key derivation function Argon2. -// Argon2 was selected as the winner of the Password Hashing Competition and can -// be used to derive cryptographic keys from passwords. -// -// For a detailed specification of Argon2 see [1]. -// -// If you aren't sure which function you need, use Argon2id (IDKey) and -// the parameter recommendations for your scenario. -// -// -// Argon2i -// -// Argon2i (implemented by Key) is the side-channel resistant version of Argon2. -// It uses data-independent memory access, which is preferred for password -// hashing and password-based key derivation. Argon2i requires more passes over -// memory than Argon2id to protect from trade-off attacks. The recommended -// parameters (taken from [2]) for non-interactive operations are time=3 and to -// use the maximum available memory. -// -// -// Argon2id -// -// Argon2id (implemented by IDKey) is a hybrid version of Argon2 combining -// Argon2i and Argon2d. It uses data-independent memory access for the first -// half of the first iteration over the memory and data-dependent memory access -// for the rest. Argon2id is side-channel resistant and provides better brute- -// force cost savings due to time-memory tradeoffs than Argon2i. The recommended -// parameters for non-interactive operations (taken from [2]) are time=1 and to -// use the maximum available memory. -// -// [1] https://github.com/P-H-C/phc-winner-argon2/blob/master/argon2-specs.pdf -// [2] https://tools.ietf.org/html/draft-irtf-cfrg-argon2-03#section-9.3 -// -// Modified to be used with MinIO. Modification here specifically adds -// sync.Pool reusable buffers to avoid large memory build up with frequent -// allocations done by memory hard PBKDF. -package argon2 - -import ( - "encoding/binary" - "sync" - - "golang.org/x/crypto/blake2b" -) - -// The Argon2 version implemented by this package. -const Version = 0x13 - -const ( - argon2d = iota - argon2i - argon2id -) - -// Key derives a key from the password, salt, and cost parameters using Argon2i -// returning a byte slice of length keyLen that can be used as cryptographic -// key. The CPU cost and parallelism degree must be greater than zero. -// -// For example, you can get a derived key for e.g. AES-256 (which needs a -// 32-byte key) by doing: -// -// key := argon2.Key([]byte("some password"), salt, 3, 32*1024, 4, 32) -// -// The draft RFC recommends[2] time=3, and memory=32*1024 is a sensible number. -// If using that amount of memory (32 MB) is not possible in some contexts then -// the time parameter can be increased to compensate. -// -// The time parameter specifies the number of passes over the memory and the -// memory parameter specifies the size of the memory in KiB. For example -// memory=32*1024 sets the memory cost to ~32 MB. The number of threads can be -// adjusted to the number of available CPUs. The cost parameters should be -// increased as memory latency and CPU parallelism increases. Remember to get a -// good random salt. -func Key(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte { - return deriveKey(argon2i, password, salt, nil, nil, time, memory, threads, keyLen) -} - -// IDKey derives a key from the password, salt, and cost parameters using -// Argon2id returning a byte slice of length keyLen that can be used as -// cryptographic key. The CPU cost and parallelism degree must be greater than -// zero. -// -// For example, you can get a derived key for e.g. AES-256 (which needs a -// 32-byte key) by doing: -// -// key := argon2.IDKey([]byte("some password"), salt, 1, 64*1024, 4, 32) -// -// The draft RFC recommends[2] time=1, and memory=64*1024 is a sensible number. -// If using that amount of memory (64 MB) is not possible in some contexts then -// the time parameter can be increased to compensate. -// -// The time parameter specifies the number of passes over the memory and the -// memory parameter specifies the size of the memory in KiB. For example -// memory=64*1024 sets the memory cost to ~64 MB. The number of threads can be -// adjusted to the numbers of available CPUs. The cost parameters should be -// increased as memory latency and CPU parallelism increases. Remember to get a -// good random salt. -func IDKey(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte { - return deriveKey(argon2id, password, salt, nil, nil, time, memory, threads, keyLen) -} - -func clearBlocks(B []block) { - for i := range B { - B[i] = block{} - } -} - -// NewIDKey returns an argon2 PBKDF backend by sync.Pool -func NewIDKey(time, memory uint32, threads uint8) func([]byte, []byte, []byte, []byte, uint32) []byte { - if time < 1 { - panic("argon2: number of rounds too small") - } - if threads < 1 { - panic("argon2: parallelism degree too low") - } - - hashMemory := memory - - memory = memory / (syncPoints * uint32(threads)) * (syncPoints * uint32(threads)) - if memory < 2*syncPoints*uint32(threads) { - memory = 2 * syncPoints * uint32(threads) - } - - pool := sync.Pool{ - New: func() interface{} { - b := make([]block, memory) - return &b - }, - } - - return func(password, salt, secret, data []byte, keyLen uint32) []byte { - B := pool.Get().(*[]block) - defer func() { - clearBlocks(*B) - pool.Put(B) - }() - - h0 := initHash(password, salt, secret, data, time, hashMemory, uint32(threads), keyLen, argon2id) - B1 := initBlocks(&h0, *B, uint32(threads)) - processBlocks(B1, time, memory, uint32(threads), argon2id) - return extractKey(B1, memory, uint32(threads), keyLen) - } -} - -func deriveKey(mode int, password, salt, secret, data []byte, time, memory uint32, threads uint8, keyLen uint32) []byte { - if time < 1 { - panic("argon2: number of rounds too small") - } - if threads < 1 { - panic("argon2: parallelism degree too low") - } - h0 := initHash(password, salt, secret, data, time, memory, uint32(threads), keyLen, mode) - - memory = memory / (syncPoints * uint32(threads)) * (syncPoints * uint32(threads)) - if memory < 2*syncPoints*uint32(threads) { - memory = 2 * syncPoints * uint32(threads) - } - B := make([]block, memory) - B = initBlocks(&h0, B, uint32(threads)) - processBlocks(B, time, memory, uint32(threads), mode) - return extractKey(B, memory, uint32(threads), keyLen) -} - -const ( - blockLength = 128 - syncPoints = 4 -) - -type block [blockLength]uint64 - -func initHash(password, salt, key, data []byte, time, memory, threads, keyLen uint32, mode int) [blake2b.Size + 8]byte { - var ( - h0 [blake2b.Size + 8]byte - params [24]byte - tmp [4]byte - ) - - b2, _ := blake2b.New512(nil) - binary.LittleEndian.PutUint32(params[0:4], threads) - binary.LittleEndian.PutUint32(params[4:8], keyLen) - binary.LittleEndian.PutUint32(params[8:12], memory) - binary.LittleEndian.PutUint32(params[12:16], time) - binary.LittleEndian.PutUint32(params[16:20], uint32(Version)) - binary.LittleEndian.PutUint32(params[20:24], uint32(mode)) - b2.Write(params[:]) - binary.LittleEndian.PutUint32(tmp[:], uint32(len(password))) - b2.Write(tmp[:]) - b2.Write(password) - binary.LittleEndian.PutUint32(tmp[:], uint32(len(salt))) - b2.Write(tmp[:]) - b2.Write(salt) - binary.LittleEndian.PutUint32(tmp[:], uint32(len(key))) - b2.Write(tmp[:]) - b2.Write(key) - binary.LittleEndian.PutUint32(tmp[:], uint32(len(data))) - b2.Write(tmp[:]) - b2.Write(data) - b2.Sum(h0[:0]) - return h0 -} - -func initBlocks(h0 *[blake2b.Size + 8]byte, blocks []block, threads uint32) []block { - var block0 [1024]byte - B := blocks - for lane := uint32(0); lane < threads; lane++ { - j := lane * (uint32(len(B)) / threads) - binary.LittleEndian.PutUint32(h0[blake2b.Size+4:], lane) - - binary.LittleEndian.PutUint32(h0[blake2b.Size:], 0) - blake2bHash(block0[:], h0[:]) - for i := range B[j+0] { - B[j+0][i] = binary.LittleEndian.Uint64(block0[i*8:]) - } - - binary.LittleEndian.PutUint32(h0[blake2b.Size:], 1) - blake2bHash(block0[:], h0[:]) - for i := range B[j+1] { - B[j+1][i] = binary.LittleEndian.Uint64(block0[i*8:]) - } - } - return B -} - -func processBlocks(B []block, time, memory, threads uint32, mode int) { - lanes := memory / threads - segments := lanes / syncPoints - - processSegment := func(n, slice, lane uint32, wg *sync.WaitGroup) { - var addresses, in, zero block - if mode == argon2i || (mode == argon2id && n == 0 && slice < syncPoints/2) { - in[0] = uint64(n) - in[1] = uint64(lane) - in[2] = uint64(slice) - in[3] = uint64(memory) - in[4] = uint64(time) - in[5] = uint64(mode) - } - - index := uint32(0) - if n == 0 && slice == 0 { - index = 2 // we have already generated the first two blocks - if mode == argon2i || mode == argon2id { - in[6]++ - processBlock(&addresses, &in, &zero) - processBlock(&addresses, &addresses, &zero) - } - } - - offset := lane*lanes + slice*segments + index - var random uint64 - for index < segments { - prev := offset - 1 - if index == 0 && slice == 0 { - prev += lanes // last block in lane - } - if mode == argon2i || (mode == argon2id && n == 0 && slice < syncPoints/2) { - if index%blockLength == 0 { - in[6]++ - processBlock(&addresses, &in, &zero) - processBlock(&addresses, &addresses, &zero) - } - random = addresses[index%blockLength] - } else { - random = B[prev][0] - } - newOffset := indexAlpha(random, lanes, segments, threads, n, slice, lane, index) - processBlockXOR(&B[offset], &B[prev], &B[newOffset]) - index, offset = index+1, offset+1 - } - wg.Done() - } - - for n := uint32(0); n < time; n++ { - for slice := uint32(0); slice < syncPoints; slice++ { - var wg sync.WaitGroup - for lane := uint32(0); lane < threads; lane++ { - wg.Add(1) - go processSegment(n, slice, lane, &wg) - } - wg.Wait() - } - } - -} - -func extractKey(B []block, memory, threads, keyLen uint32) []byte { - lanes := memory / threads - for lane := uint32(0); lane < threads-1; lane++ { - for i, v := range B[(lane*lanes)+lanes-1] { - B[memory-1][i] ^= v - } - } - - var block [1024]byte - for i, v := range B[memory-1] { - binary.LittleEndian.PutUint64(block[i*8:], v) - } - key := make([]byte, keyLen) - blake2bHash(key, block[:]) - return key -} - -func indexAlpha(rand uint64, lanes, segments, threads, n, slice, lane, index uint32) uint32 { - refLane := uint32(rand>>32) % threads - if n == 0 && slice == 0 { - refLane = lane - } - m, s := 3*segments, ((slice+1)%syncPoints)*segments - if lane == refLane { - m += index - } - if n == 0 { - m, s = slice*segments, 0 - if slice == 0 || lane == refLane { - m += index - } - } - if index == 0 || lane == refLane { - m-- - } - return phi(rand, uint64(m), uint64(s), refLane, lanes) -} - -func phi(rand, m, s uint64, lane, lanes uint32) uint32 { - p := rand & 0xFFFFFFFF - p = (p * p) >> 32 - p = (p * m) >> 32 - return lane*lanes + uint32((s+m-(p+1))%uint64(lanes)) -} diff --git a/contrib/pkg/argon2/argon2_test.go b/contrib/pkg/argon2/argon2_test.go deleted file mode 100644 index eeaa38a51..000000000 --- a/contrib/pkg/argon2/argon2_test.go +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package argon2 - -import ( - "bytes" - "encoding/hex" - "testing" -) - -var ( - genKatPassword = []byte{ - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - } - genKatSalt = []byte{0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02} - genKatSecret = []byte{0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03} - genKatAAD = []byte{0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04} -) - -func TestArgon2(t *testing.T) { - defer func(sse4 bool) { useSSE4 = sse4 }(useSSE4) - - if useSSE4 { - t.Log("SSE4.1 version") - testArgon2i(t) - testArgon2d(t) - testArgon2id(t) - useSSE4 = false - } - t.Log("generic version") - testArgon2i(t) - testArgon2d(t) - testArgon2id(t) -} - -func testArgon2d(t *testing.T) { - want := []byte{ - 0x51, 0x2b, 0x39, 0x1b, 0x6f, 0x11, 0x62, 0x97, - 0x53, 0x71, 0xd3, 0x09, 0x19, 0x73, 0x42, 0x94, - 0xf8, 0x68, 0xe3, 0xbe, 0x39, 0x84, 0xf3, 0xc1, - 0xa1, 0x3a, 0x4d, 0xb9, 0xfa, 0xbe, 0x4a, 0xcb, - } - hash := deriveKey(argon2d, genKatPassword, genKatSalt, genKatSecret, genKatAAD, 3, 32, 4, 32) - if !bytes.Equal(hash, want) { - t.Errorf("derived key does not match - got: %s , want: %s", hex.EncodeToString(hash), hex.EncodeToString(want)) - } -} - -func testArgon2i(t *testing.T) { - want := []byte{ - 0xc8, 0x14, 0xd9, 0xd1, 0xdc, 0x7f, 0x37, 0xaa, - 0x13, 0xf0, 0xd7, 0x7f, 0x24, 0x94, 0xbd, 0xa1, - 0xc8, 0xde, 0x6b, 0x01, 0x6d, 0xd3, 0x88, 0xd2, - 0x99, 0x52, 0xa4, 0xc4, 0x67, 0x2b, 0x6c, 0xe8, - } - hash := deriveKey(argon2i, genKatPassword, genKatSalt, genKatSecret, genKatAAD, 3, 32, 4, 32) - if !bytes.Equal(hash, want) { - t.Errorf("derived key does not match - got: %s , want: %s", hex.EncodeToString(hash), hex.EncodeToString(want)) - } -} - -func testArgon2id(t *testing.T) { - want := []byte{ - 0x0d, 0x64, 0x0d, 0xf5, 0x8d, 0x78, 0x76, 0x6c, - 0x08, 0xc0, 0x37, 0xa3, 0x4a, 0x8b, 0x53, 0xc9, - 0xd0, 0x1e, 0xf0, 0x45, 0x2d, 0x75, 0xb6, 0x5e, - 0xb5, 0x25, 0x20, 0xe9, 0x6b, 0x01, 0xe6, 0x59, - } - hash := deriveKey(argon2id, genKatPassword, genKatSalt, genKatSecret, genKatAAD, 3, 32, 4, 32) - if !bytes.Equal(hash, want) { - t.Errorf("derived key does not match - got: %s , want: %s", hex.EncodeToString(hash), hex.EncodeToString(want)) - } -} - -func TestVectorsIDKey(t *testing.T) { - password, salt := []byte("password"), []byte("somesalt") - for _, v := range testVectors { - v := v - t.Run("", func(t *testing.T) { - var fn func(password, salt, secret, data []byte, keyLen uint32) []byte - switch v.mode { - case argon2id: - fn = NewIDKey(v.time, v.memory, v.threads) - default: - t.Skip() - } - want, err := hex.DecodeString(v.hash) - if err != nil { - t.Fatalf("failed to decode hash: %v", err) - } - hash := fn(password, salt, nil, nil, uint32(len(want))) - if !bytes.Equal(hash, want) { - t.Errorf("got: %s want: %s", hex.EncodeToString(hash), hex.EncodeToString(want)) - } - }) - } - -} - -func TestVectors(t *testing.T) { - password, salt := []byte("password"), []byte("somesalt") - for _, v := range testVectors { - v := v - t.Run("", func(t *testing.T) { - want, err := hex.DecodeString(v.hash) - if err != nil { - t.Fatalf("failed to decode hash: %v", err) - } - hash := deriveKey(v.mode, password, salt, nil, nil, v.time, v.memory, v.threads, uint32(len(want))) - if !bytes.Equal(hash, want) { - t.Errorf("got: %s want: %s", hex.EncodeToString(hash), hex.EncodeToString(want)) - } - }) - } -} - -func benchmarkArgon2(mode int, time, memory uint32, threads uint8, keyLen uint32, b *testing.B) { - password := []byte("password") - salt := []byte("choosing random salts is hard") - b.ReportAllocs() - for i := 0; i < b.N; i++ { - deriveKey(mode, password, salt, nil, nil, time, memory, threads, keyLen) - } -} - -func BenchmarkArgon2i(b *testing.B) { - b.Run(" Time: 3 Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2i, 3, 32*1024, 1, 32, b) }) - b.Run(" Time: 4 Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2i, 4, 32*1024, 1, 32, b) }) - b.Run(" Time: 5 Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2i, 5, 32*1024, 1, 32, b) }) - b.Run(" Time: 3 Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2i, 3, 64*1024, 4, 32, b) }) - b.Run(" Time: 4 Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2i, 4, 64*1024, 4, 32, b) }) - b.Run(" Time: 5 Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2i, 5, 64*1024, 4, 32, b) }) -} - -func BenchmarkArgon2d(b *testing.B) { - b.Run(" Time: 3, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2d, 3, 32*1024, 1, 32, b) }) - b.Run(" Time: 4, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2d, 4, 32*1024, 1, 32, b) }) - b.Run(" Time: 5, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2d, 5, 32*1024, 1, 32, b) }) - b.Run(" Time: 3, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2d, 3, 64*1024, 4, 32, b) }) - b.Run(" Time: 4, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2d, 4, 64*1024, 4, 32, b) }) - b.Run(" Time: 5, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2d, 5, 64*1024, 4, 32, b) }) -} - -func BenchmarkArgon2id(b *testing.B) { - b.Run(" Time: 3, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2id, 3, 32*1024, 1, 32, b) }) - b.Run(" Time: 4, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2id, 4, 32*1024, 1, 32, b) }) - b.Run(" Time: 5, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2(argon2id, 5, 32*1024, 1, 32, b) }) - b.Run(" Time: 3, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2id, 3, 64*1024, 4, 32, b) }) - b.Run(" Time: 4, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2id, 4, 64*1024, 4, 32, b) }) - b.Run(" Time: 5, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2(argon2id, 5, 64*1024, 4, 32, b) }) -} - -func benchmarkArgon2idNew(time, memory uint32, threads uint8, keyLen uint32, b *testing.B) { - f := NewIDKey(time, memory, threads) - password := []byte("password") - salt := []byte("choosing random salts is hard") - b.ReportAllocs() - for i := 0; i < b.N; i++ { - _ = f(password, salt, nil, nil, keyLen) - } -} - -func BenchmarkArgon2idNew(b *testing.B) { - b.Run(" Time: 3, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2idNew(3, 32*1024, 1, 32, b) }) - b.Run(" Time: 4, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2idNew(4, 32*1024, 1, 32, b) }) - b.Run(" Time: 5, Memory: 32 MB, Threads: 1", func(b *testing.B) { benchmarkArgon2idNew(5, 32*1024, 1, 32, b) }) - b.Run(" Time: 3, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2idNew(3, 64*1024, 4, 32, b) }) - b.Run(" Time: 4, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2idNew(4, 64*1024, 4, 32, b) }) - b.Run(" Time: 5, Memory: 64 MB, Threads: 4", func(b *testing.B) { benchmarkArgon2idNew(5, 64*1024, 4, 32, b) }) -} - -// Generated with the CLI of https://github.com/P-H-C/phc-winner-argon2/blob/master/argon2-specs.pdf -var testVectors = []struct { - mode int - time, memory uint32 - threads uint8 - hash string -}{ - { - mode: argon2i, time: 1, memory: 64, threads: 1, - hash: "b9c401d1844a67d50eae3967dc28870b22e508092e861a37", - }, - { - mode: argon2d, time: 1, memory: 64, threads: 1, - hash: "8727405fd07c32c78d64f547f24150d3f2e703a89f981a19", - }, - { - mode: argon2id, time: 1, memory: 64, threads: 1, - hash: "655ad15eac652dc59f7170a7332bf49b8469be1fdb9c28bb", - }, - { - mode: argon2i, time: 2, memory: 64, threads: 1, - hash: "8cf3d8f76a6617afe35fac48eb0b7433a9a670ca4a07ed64", - }, - { - mode: argon2d, time: 2, memory: 64, threads: 1, - hash: "3be9ec79a69b75d3752acb59a1fbb8b295a46529c48fbb75", - }, - { - mode: argon2id, time: 2, memory: 64, threads: 1, - hash: "068d62b26455936aa6ebe60060b0a65870dbfa3ddf8d41f7", - }, - { - mode: argon2i, time: 2, memory: 64, threads: 2, - hash: "2089f3e78a799720f80af806553128f29b132cafe40d059f", - }, - { - mode: argon2d, time: 2, memory: 64, threads: 2, - hash: "68e2462c98b8bc6bb60ec68db418ae2c9ed24fc6748a40e9", - }, - { - mode: argon2id, time: 2, memory: 64, threads: 2, - hash: "350ac37222f436ccb5c0972f1ebd3bf6b958bf2071841362", - }, - { - mode: argon2i, time: 3, memory: 256, threads: 2, - hash: "f5bbf5d4c3836af13193053155b73ec7476a6a2eb93fd5e6", - }, - { - mode: argon2d, time: 3, memory: 256, threads: 2, - hash: "f4f0669218eaf3641f39cc97efb915721102f4b128211ef2", - }, - { - mode: argon2id, time: 3, memory: 256, threads: 2, - hash: "4668d30ac4187e6878eedeacf0fd83c5a0a30db2cc16ef0b", - }, - { - mode: argon2i, time: 4, memory: 4096, threads: 4, - hash: "a11f7b7f3f93f02ad4bddb59ab62d121e278369288a0d0e7", - }, - { - mode: argon2d, time: 4, memory: 4096, threads: 4, - hash: "935598181aa8dc2b720914aa6435ac8d3e3a4210c5b0fb2d", - }, - { - mode: argon2id, time: 4, memory: 4096, threads: 4, - hash: "145db9733a9f4ee43edf33c509be96b934d505a4efb33c5a", - }, - { - mode: argon2i, time: 4, memory: 1024, threads: 8, - hash: "0cdd3956aa35e6b475a7b0c63488822f774f15b43f6e6e17", - }, - { - mode: argon2d, time: 4, memory: 1024, threads: 8, - hash: "83604fc2ad0589b9d055578f4d3cc55bc616df3578a896e9", - }, - { - mode: argon2id, time: 4, memory: 1024, threads: 8, - hash: "8dafa8e004f8ea96bf7c0f93eecf67a6047476143d15577f", - }, - { - mode: argon2i, time: 2, memory: 64, threads: 3, - hash: "5cab452fe6b8479c8661def8cd703b611a3905a6d5477fe6", - }, - { - mode: argon2d, time: 2, memory: 64, threads: 3, - hash: "22474a423bda2ccd36ec9afd5119e5c8949798cadf659f51", - }, - { - mode: argon2id, time: 2, memory: 64, threads: 3, - hash: "4a15b31aec7c2590b87d1f520be7d96f56658172deaa3079", - }, - { - mode: argon2i, time: 3, memory: 1024, threads: 6, - hash: "d236b29c2b2a09babee842b0dec6aa1e83ccbdea8023dced", - }, - { - mode: argon2d, time: 3, memory: 1024, threads: 6, - hash: "a3351b0319a53229152023d9206902f4ef59661cdca89481", - }, - { - mode: argon2id, time: 3, memory: 1024, threads: 6, - hash: "1640b932f4b60e272f5d2207b9a9c626ffa1bd88d2349016", - }, -} diff --git a/contrib/pkg/argon2/blake2b.go b/contrib/pkg/argon2/blake2b.go deleted file mode 100644 index 10f46948d..000000000 --- a/contrib/pkg/argon2/blake2b.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package argon2 - -import ( - "encoding/binary" - "hash" - - "golang.org/x/crypto/blake2b" -) - -// blake2bHash computes an arbitrary long hash value of in -// and writes the hash to out. -func blake2bHash(out []byte, in []byte) { - var b2 hash.Hash - if n := len(out); n < blake2b.Size { - b2, _ = blake2b.New(n, nil) - } else { - b2, _ = blake2b.New512(nil) - } - - var buffer [blake2b.Size]byte - binary.LittleEndian.PutUint32(buffer[:4], uint32(len(out))) - b2.Write(buffer[:4]) - b2.Write(in) - - if len(out) <= blake2b.Size { - b2.Sum(out[:0]) - return - } - - outLen := len(out) - b2.Sum(buffer[:0]) - b2.Reset() - copy(out, buffer[:32]) - out = out[32:] - for len(out) > blake2b.Size { - b2.Write(buffer[:]) - b2.Sum(buffer[:0]) - copy(out, buffer[:32]) - out = out[32:] - b2.Reset() - } - - if outLen%blake2b.Size > 0 { // outLen > 64 - r := ((outLen + 31) / 32) - 2 // ⌈τ /32⌉-2 - b2, _ = blake2b.New(outLen-32*r, nil) - } - b2.Write(buffer[:]) - b2.Sum(out[:0]) -} diff --git a/contrib/pkg/argon2/blamka_amd64.go b/contrib/pkg/argon2/blamka_amd64.go deleted file mode 100644 index 2fc1ec031..000000000 --- a/contrib/pkg/argon2/blamka_amd64.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build amd64,!gccgo,!appengine - -package argon2 - -import "golang.org/x/sys/cpu" - -func init() { - useSSE4 = cpu.X86.HasSSE41 -} - -//go:noescape -func mixBlocksSSE2(out, a, b, c *block) - -//go:noescape -func xorBlocksSSE2(out, a, b, c *block) - -//go:noescape -func blamkaSSE4(b *block) - -func processBlockSSE(out, in1, in2 *block, xor bool) { - var t block - mixBlocksSSE2(&t, in1, in2, &t) - if useSSE4 { - blamkaSSE4(&t) - } else { - for i := 0; i < blockLength; i += 16 { - blamkaGeneric( - &t[i+0], &t[i+1], &t[i+2], &t[i+3], - &t[i+4], &t[i+5], &t[i+6], &t[i+7], - &t[i+8], &t[i+9], &t[i+10], &t[i+11], - &t[i+12], &t[i+13], &t[i+14], &t[i+15], - ) - } - for i := 0; i < blockLength/8; i += 2 { - blamkaGeneric( - &t[i], &t[i+1], &t[16+i], &t[16+i+1], - &t[32+i], &t[32+i+1], &t[48+i], &t[48+i+1], - &t[64+i], &t[64+i+1], &t[80+i], &t[80+i+1], - &t[96+i], &t[96+i+1], &t[112+i], &t[112+i+1], - ) - } - } - if xor { - xorBlocksSSE2(out, in1, in2, &t) - } else { - mixBlocksSSE2(out, in1, in2, &t) - } -} - -func processBlock(out, in1, in2 *block) { - processBlockSSE(out, in1, in2, false) -} - -func processBlockXOR(out, in1, in2 *block) { - processBlockSSE(out, in1, in2, true) -} diff --git a/contrib/pkg/argon2/blamka_amd64.s b/contrib/pkg/argon2/blamka_amd64.s deleted file mode 100644 index 74a6e7332..000000000 --- a/contrib/pkg/argon2/blamka_amd64.s +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build amd64,!gccgo,!appengine - -#include "textflag.h" - -DATA ·c40<>+0x00(SB)/8, $0x0201000706050403 -DATA ·c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b -GLOBL ·c40<>(SB), (NOPTR+RODATA), $16 - -DATA ·c48<>+0x00(SB)/8, $0x0100070605040302 -DATA ·c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a -GLOBL ·c48<>(SB), (NOPTR+RODATA), $16 - -#define SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2) \ - MOVO v4, t1; \ - MOVO v5, v4; \ - MOVO t1, v5; \ - MOVO v6, t1; \ - PUNPCKLQDQ v6, t2; \ - PUNPCKHQDQ v7, v6; \ - PUNPCKHQDQ t2, v6; \ - PUNPCKLQDQ v7, t2; \ - MOVO t1, v7; \ - MOVO v2, t1; \ - PUNPCKHQDQ t2, v7; \ - PUNPCKLQDQ v3, t2; \ - PUNPCKHQDQ t2, v2; \ - PUNPCKLQDQ t1, t2; \ - PUNPCKHQDQ t2, v3 - -#define SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2) \ - MOVO v4, t1; \ - MOVO v5, v4; \ - MOVO t1, v5; \ - MOVO v2, t1; \ - PUNPCKLQDQ v2, t2; \ - PUNPCKHQDQ v3, v2; \ - PUNPCKHQDQ t2, v2; \ - PUNPCKLQDQ v3, t2; \ - MOVO t1, v3; \ - MOVO v6, t1; \ - PUNPCKHQDQ t2, v3; \ - PUNPCKLQDQ v7, t2; \ - PUNPCKHQDQ t2, v6; \ - PUNPCKLQDQ t1, t2; \ - PUNPCKHQDQ t2, v7 - -#define HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48) \ - MOVO v0, t0; \ - PMULULQ v2, t0; \ - PADDQ v2, v0; \ - PADDQ t0, v0; \ - PADDQ t0, v0; \ - PXOR v0, v6; \ - PSHUFD $0xB1, v6, v6; \ - MOVO v4, t0; \ - PMULULQ v6, t0; \ - PADDQ v6, v4; \ - PADDQ t0, v4; \ - PADDQ t0, v4; \ - PXOR v4, v2; \ - PSHUFB c40, v2; \ - MOVO v0, t0; \ - PMULULQ v2, t0; \ - PADDQ v2, v0; \ - PADDQ t0, v0; \ - PADDQ t0, v0; \ - PXOR v0, v6; \ - PSHUFB c48, v6; \ - MOVO v4, t0; \ - PMULULQ v6, t0; \ - PADDQ v6, v4; \ - PADDQ t0, v4; \ - PADDQ t0, v4; \ - PXOR v4, v2; \ - MOVO v2, t0; \ - PADDQ v2, t0; \ - PSRLQ $63, v2; \ - PXOR t0, v2; \ - MOVO v1, t0; \ - PMULULQ v3, t0; \ - PADDQ v3, v1; \ - PADDQ t0, v1; \ - PADDQ t0, v1; \ - PXOR v1, v7; \ - PSHUFD $0xB1, v7, v7; \ - MOVO v5, t0; \ - PMULULQ v7, t0; \ - PADDQ v7, v5; \ - PADDQ t0, v5; \ - PADDQ t0, v5; \ - PXOR v5, v3; \ - PSHUFB c40, v3; \ - MOVO v1, t0; \ - PMULULQ v3, t0; \ - PADDQ v3, v1; \ - PADDQ t0, v1; \ - PADDQ t0, v1; \ - PXOR v1, v7; \ - PSHUFB c48, v7; \ - MOVO v5, t0; \ - PMULULQ v7, t0; \ - PADDQ v7, v5; \ - PADDQ t0, v5; \ - PADDQ t0, v5; \ - PXOR v5, v3; \ - MOVO v3, t0; \ - PADDQ v3, t0; \ - PSRLQ $63, v3; \ - PXOR t0, v3 - -#define LOAD_MSG_0(block, off) \ - MOVOU 8*(off+0)(block), X0; \ - MOVOU 8*(off+2)(block), X1; \ - MOVOU 8*(off+4)(block), X2; \ - MOVOU 8*(off+6)(block), X3; \ - MOVOU 8*(off+8)(block), X4; \ - MOVOU 8*(off+10)(block), X5; \ - MOVOU 8*(off+12)(block), X6; \ - MOVOU 8*(off+14)(block), X7 - -#define STORE_MSG_0(block, off) \ - MOVOU X0, 8*(off+0)(block); \ - MOVOU X1, 8*(off+2)(block); \ - MOVOU X2, 8*(off+4)(block); \ - MOVOU X3, 8*(off+6)(block); \ - MOVOU X4, 8*(off+8)(block); \ - MOVOU X5, 8*(off+10)(block); \ - MOVOU X6, 8*(off+12)(block); \ - MOVOU X7, 8*(off+14)(block) - -#define LOAD_MSG_1(block, off) \ - MOVOU 8*off+0*8(block), X0; \ - MOVOU 8*off+16*8(block), X1; \ - MOVOU 8*off+32*8(block), X2; \ - MOVOU 8*off+48*8(block), X3; \ - MOVOU 8*off+64*8(block), X4; \ - MOVOU 8*off+80*8(block), X5; \ - MOVOU 8*off+96*8(block), X6; \ - MOVOU 8*off+112*8(block), X7 - -#define STORE_MSG_1(block, off) \ - MOVOU X0, 8*off+0*8(block); \ - MOVOU X1, 8*off+16*8(block); \ - MOVOU X2, 8*off+32*8(block); \ - MOVOU X3, 8*off+48*8(block); \ - MOVOU X4, 8*off+64*8(block); \ - MOVOU X5, 8*off+80*8(block); \ - MOVOU X6, 8*off+96*8(block); \ - MOVOU X7, 8*off+112*8(block) - -#define BLAMKA_ROUND_0(block, off, t0, t1, c40, c48) \ - LOAD_MSG_0(block, off); \ - HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \ - SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1); \ - HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \ - SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1); \ - STORE_MSG_0(block, off) - -#define BLAMKA_ROUND_1(block, off, t0, t1, c40, c48) \ - LOAD_MSG_1(block, off); \ - HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \ - SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1); \ - HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \ - SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1); \ - STORE_MSG_1(block, off) - -// func blamkaSSE4(b *block) -TEXT ·blamkaSSE4(SB), 4, $0-8 - MOVQ b+0(FP), AX - - MOVOU ·c40<>(SB), X10 - MOVOU ·c48<>(SB), X11 - - BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11) - BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11) - - BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11) - BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11) - RET - -// func mixBlocksSSE2(out, a, b, c *block) -TEXT ·mixBlocksSSE2(SB), 4, $0-32 - MOVQ out+0(FP), DX - MOVQ a+8(FP), AX - MOVQ b+16(FP), BX - MOVQ a+24(FP), CX - MOVQ $128, BP - -loop: - MOVOU 0(AX), X0 - MOVOU 0(BX), X1 - MOVOU 0(CX), X2 - PXOR X1, X0 - PXOR X2, X0 - MOVOU X0, 0(DX) - ADDQ $16, AX - ADDQ $16, BX - ADDQ $16, CX - ADDQ $16, DX - SUBQ $2, BP - JA loop - RET - -// func xorBlocksSSE2(out, a, b, c *block) -TEXT ·xorBlocksSSE2(SB), 4, $0-32 - MOVQ out+0(FP), DX - MOVQ a+8(FP), AX - MOVQ b+16(FP), BX - MOVQ a+24(FP), CX - MOVQ $128, BP - -loop: - MOVOU 0(AX), X0 - MOVOU 0(BX), X1 - MOVOU 0(CX), X2 - MOVOU 0(DX), X3 - PXOR X1, X0 - PXOR X2, X0 - PXOR X3, X0 - MOVOU X0, 0(DX) - ADDQ $16, AX - ADDQ $16, BX - ADDQ $16, CX - ADDQ $16, DX - SUBQ $2, BP - JA loop - RET diff --git a/contrib/pkg/argon2/blamka_generic.go b/contrib/pkg/argon2/blamka_generic.go deleted file mode 100644 index a481b2243..000000000 --- a/contrib/pkg/argon2/blamka_generic.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package argon2 - -var useSSE4 bool - -func processBlockGeneric(out, in1, in2 *block, xor bool) { - var t block - for i := range t { - t[i] = in1[i] ^ in2[i] - } - for i := 0; i < blockLength; i += 16 { - blamkaGeneric( - &t[i+0], &t[i+1], &t[i+2], &t[i+3], - &t[i+4], &t[i+5], &t[i+6], &t[i+7], - &t[i+8], &t[i+9], &t[i+10], &t[i+11], - &t[i+12], &t[i+13], &t[i+14], &t[i+15], - ) - } - for i := 0; i < blockLength/8; i += 2 { - blamkaGeneric( - &t[i], &t[i+1], &t[16+i], &t[16+i+1], - &t[32+i], &t[32+i+1], &t[48+i], &t[48+i+1], - &t[64+i], &t[64+i+1], &t[80+i], &t[80+i+1], - &t[96+i], &t[96+i+1], &t[112+i], &t[112+i+1], - ) - } - if xor { - for i := range t { - out[i] ^= in1[i] ^ in2[i] ^ t[i] - } - } else { - for i := range t { - out[i] = in1[i] ^ in2[i] ^ t[i] - } - } -} - -func blamkaGeneric(t00, t01, t02, t03, t04, t05, t06, t07, t08, t09, t10, t11, t12, t13, t14, t15 *uint64) { - v00, v01, v02, v03 := *t00, *t01, *t02, *t03 - v04, v05, v06, v07 := *t04, *t05, *t06, *t07 - v08, v09, v10, v11 := *t08, *t09, *t10, *t11 - v12, v13, v14, v15 := *t12, *t13, *t14, *t15 - - v00 += v04 + 2*uint64(uint32(v00))*uint64(uint32(v04)) - v12 ^= v00 - v12 = v12>>32 | v12<<32 - v08 += v12 + 2*uint64(uint32(v08))*uint64(uint32(v12)) - v04 ^= v08 - v04 = v04>>24 | v04<<40 - - v00 += v04 + 2*uint64(uint32(v00))*uint64(uint32(v04)) - v12 ^= v00 - v12 = v12>>16 | v12<<48 - v08 += v12 + 2*uint64(uint32(v08))*uint64(uint32(v12)) - v04 ^= v08 - v04 = v04>>63 | v04<<1 - - v01 += v05 + 2*uint64(uint32(v01))*uint64(uint32(v05)) - v13 ^= v01 - v13 = v13>>32 | v13<<32 - v09 += v13 + 2*uint64(uint32(v09))*uint64(uint32(v13)) - v05 ^= v09 - v05 = v05>>24 | v05<<40 - - v01 += v05 + 2*uint64(uint32(v01))*uint64(uint32(v05)) - v13 ^= v01 - v13 = v13>>16 | v13<<48 - v09 += v13 + 2*uint64(uint32(v09))*uint64(uint32(v13)) - v05 ^= v09 - v05 = v05>>63 | v05<<1 - - v02 += v06 + 2*uint64(uint32(v02))*uint64(uint32(v06)) - v14 ^= v02 - v14 = v14>>32 | v14<<32 - v10 += v14 + 2*uint64(uint32(v10))*uint64(uint32(v14)) - v06 ^= v10 - v06 = v06>>24 | v06<<40 - - v02 += v06 + 2*uint64(uint32(v02))*uint64(uint32(v06)) - v14 ^= v02 - v14 = v14>>16 | v14<<48 - v10 += v14 + 2*uint64(uint32(v10))*uint64(uint32(v14)) - v06 ^= v10 - v06 = v06>>63 | v06<<1 - - v03 += v07 + 2*uint64(uint32(v03))*uint64(uint32(v07)) - v15 ^= v03 - v15 = v15>>32 | v15<<32 - v11 += v15 + 2*uint64(uint32(v11))*uint64(uint32(v15)) - v07 ^= v11 - v07 = v07>>24 | v07<<40 - - v03 += v07 + 2*uint64(uint32(v03))*uint64(uint32(v07)) - v15 ^= v03 - v15 = v15>>16 | v15<<48 - v11 += v15 + 2*uint64(uint32(v11))*uint64(uint32(v15)) - v07 ^= v11 - v07 = v07>>63 | v07<<1 - - v00 += v05 + 2*uint64(uint32(v00))*uint64(uint32(v05)) - v15 ^= v00 - v15 = v15>>32 | v15<<32 - v10 += v15 + 2*uint64(uint32(v10))*uint64(uint32(v15)) - v05 ^= v10 - v05 = v05>>24 | v05<<40 - - v00 += v05 + 2*uint64(uint32(v00))*uint64(uint32(v05)) - v15 ^= v00 - v15 = v15>>16 | v15<<48 - v10 += v15 + 2*uint64(uint32(v10))*uint64(uint32(v15)) - v05 ^= v10 - v05 = v05>>63 | v05<<1 - - v01 += v06 + 2*uint64(uint32(v01))*uint64(uint32(v06)) - v12 ^= v01 - v12 = v12>>32 | v12<<32 - v11 += v12 + 2*uint64(uint32(v11))*uint64(uint32(v12)) - v06 ^= v11 - v06 = v06>>24 | v06<<40 - - v01 += v06 + 2*uint64(uint32(v01))*uint64(uint32(v06)) - v12 ^= v01 - v12 = v12>>16 | v12<<48 - v11 += v12 + 2*uint64(uint32(v11))*uint64(uint32(v12)) - v06 ^= v11 - v06 = v06>>63 | v06<<1 - - v02 += v07 + 2*uint64(uint32(v02))*uint64(uint32(v07)) - v13 ^= v02 - v13 = v13>>32 | v13<<32 - v08 += v13 + 2*uint64(uint32(v08))*uint64(uint32(v13)) - v07 ^= v08 - v07 = v07>>24 | v07<<40 - - v02 += v07 + 2*uint64(uint32(v02))*uint64(uint32(v07)) - v13 ^= v02 - v13 = v13>>16 | v13<<48 - v08 += v13 + 2*uint64(uint32(v08))*uint64(uint32(v13)) - v07 ^= v08 - v07 = v07>>63 | v07<<1 - - v03 += v04 + 2*uint64(uint32(v03))*uint64(uint32(v04)) - v14 ^= v03 - v14 = v14>>32 | v14<<32 - v09 += v14 + 2*uint64(uint32(v09))*uint64(uint32(v14)) - v04 ^= v09 - v04 = v04>>24 | v04<<40 - - v03 += v04 + 2*uint64(uint32(v03))*uint64(uint32(v04)) - v14 ^= v03 - v14 = v14>>16 | v14<<48 - v09 += v14 + 2*uint64(uint32(v09))*uint64(uint32(v14)) - v04 ^= v09 - v04 = v04>>63 | v04<<1 - - *t00, *t01, *t02, *t03 = v00, v01, v02, v03 - *t04, *t05, *t06, *t07 = v04, v05, v06, v07 - *t08, *t09, *t10, *t11 = v08, v09, v10, v11 - *t12, *t13, *t14, *t15 = v12, v13, v14, v15 -} diff --git a/contrib/pkg/argon2/blamka_ref.go b/contrib/pkg/argon2/blamka_ref.go deleted file mode 100644 index baf7b551d..000000000 --- a/contrib/pkg/argon2/blamka_ref.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !amd64 appengine gccgo - -package argon2 - -func processBlock(out, in1, in2 *block) { - processBlockGeneric(out, in1, in2, false) -} - -func processBlockXOR(out, in1, in2 *block) { - processBlockGeneric(out, in1, in2, true) -} diff --git a/contrib/pkg/csvparser/example_test.go b/contrib/pkg/csvparser/example_test.go deleted file mode 100644 index 6f20f1f57..000000000 --- a/contrib/pkg/csvparser/example_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in https://golang.org/LICENSE - -package csv_test - -import ( - "encoding/csv" - "fmt" - "io" - "log" - "os" - "strings" -) - -func ExampleReader() { - in := `first_name,last_name,username -"Rob","Pike",rob -Ken,Thompson,ken -"Robert","Griesemer","gri" -` - r := csv.NewReader(strings.NewReader(in)) - - for { - record, err := r.Read() - if err == io.EOF { - break - } - if err != nil { - log.Fatal(err) - } - - fmt.Println(record) - } - // Output: - // [first_name last_name username] - // [Rob Pike rob] - // [Ken Thompson ken] - // [Robert Griesemer gri] -} - -// This example shows how csv.Reader can be configured to handle other -// types of CSV files. -func ExampleReader_options() { - in := `first_name;last_name;username -"Rob";"Pike";rob -# lines beginning with a # character are ignored -Ken;Thompson;ken -"Robert";"Griesemer";"gri" -` - r := csv.NewReader(strings.NewReader(in)) - r.Comma = ';' - r.Comment = '#' - - records, err := r.ReadAll() - if err != nil { - log.Fatal(err) - } - - fmt.Print(records) - // Output: - // [[first_name last_name username] [Rob Pike rob] [Ken Thompson ken] [Robert Griesemer gri]] -} - -func ExampleReader_ReadAll() { - in := `first_name,last_name,username -"Rob","Pike",rob -Ken,Thompson,ken -"Robert","Griesemer","gri" -` - r := csv.NewReader(strings.NewReader(in)) - - records, err := r.ReadAll() - if err != nil { - log.Fatal(err) - } - - fmt.Print(records) - // Output: - // [[first_name last_name username] [Rob Pike rob] [Ken Thompson ken] [Robert Griesemer gri]] -} - -func ExampleWriter() { - records := [][]string{ - {"first_name", "last_name", "username"}, - {"Rob", "Pike", "rob"}, - {"Ken", "Thompson", "ken"}, - {"Robert", "Griesemer", "gri"}, - } - - w := csv.NewWriter(os.Stdout) - - for _, record := range records { - if err := w.Write(record); err != nil { - log.Fatalln("error writing record to csv:", err) - } - } - - // Write any buffered data to the underlying writer (standard output). - w.Flush() - - if err := w.Error(); err != nil { - log.Fatal(err) - } - // Output: - // first_name,last_name,username - // Rob,Pike,rob - // Ken,Thompson,ken - // Robert,Griesemer,gri -} - -func ExampleWriter_WriteAll() { - records := [][]string{ - {"first_name", "last_name", "username"}, - {"Rob", "Pike", "rob"}, - {"Ken", "Thompson", "ken"}, - {"Robert", "Griesemer", "gri"}, - } - - w := csv.NewWriter(os.Stdout) - w.WriteAll(records) // calls Flush internally - - if err := w.Error(); err != nil { - log.Fatalln("error writing csv:", err) - } - // Output: - // first_name,last_name,username - // Rob,Pike,rob - // Ken,Thompson,ken - // Robert,Griesemer,gri -} diff --git a/contrib/pkg/csvparser/fuzz.go b/contrib/pkg/csvparser/fuzz.go deleted file mode 100644 index 643f0336e..000000000 --- a/contrib/pkg/csvparser/fuzz.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in https://golang.org/LICENSE - -// +build gofuzz - -package csv - -import ( - "bytes" - "fmt" - "reflect" -) - -func Fuzz(data []byte) int { - score := 0 - buf := new(bytes.Buffer) - - for _, tt := range []Reader{ - {}, - {Comma: ';'}, - {Comma: '\t'}, - {LazyQuotes: true}, - {TrimLeadingSpace: true}, - {Comment: '#'}, - {Comment: ';'}, - } { - r := NewReader(bytes.NewReader(data)) - r.Comma = tt.Comma - r.Comment = tt.Comment - r.LazyQuotes = tt.LazyQuotes - r.TrimLeadingSpace = tt.TrimLeadingSpace - - records, err := r.ReadAll() - if err != nil { - continue - } - score = 1 - - buf.Reset() - w := NewWriter(buf) - w.Comma = tt.Comma - err = w.WriteAll(records) - if err != nil { - fmt.Printf("writer = %#v\n", w) - fmt.Printf("records = %v\n", records) - panic(err) - } - - r = NewReader(buf) - r.Comma = tt.Comma - r.Comment = tt.Comment - r.LazyQuotes = tt.LazyQuotes - r.TrimLeadingSpace = tt.TrimLeadingSpace - result, err := r.ReadAll() - if err != nil { - fmt.Printf("reader = %#v\n", r) - fmt.Printf("records = %v\n", records) - panic(err) - } - - if !reflect.DeepEqual(records, result) { - fmt.Println("records = \n", records) - fmt.Println("result = \n", records) - panic("not equal") - } - } - - return score -} diff --git a/contrib/pkg/csvparser/reader.go b/contrib/pkg/csvparser/reader.go deleted file mode 100644 index 39195bf68..000000000 --- a/contrib/pkg/csvparser/reader.go +++ /dev/null @@ -1,458 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in https://golang.org/LICENSE - -// Package csv reads and writes comma-separated values (CSV) files. -// There are many kinds of CSV files; this package supports the format -// described in RFC 4180. -// -// A csv file contains zero or more records of one or more fields per record. -// Each record is separated by the newline character. The final record may -// optionally be followed by a newline character. -// -// field1,field2,field3 -// -// White space is considered part of a field. -// -// Carriage returns before newline characters are silently removed. -// -// Blank lines are ignored. A line with only whitespace characters (excluding -// the ending newline character) is not considered a blank line. -// -// Fields which start and stop with the quote character " are called -// quoted-fields. The beginning and ending quote are not part of the -// field. -// -// The source: -// -// normal string,"quoted-field" -// -// results in the fields -// -// {`normal string`, `quoted-field`} -// -// Within a quoted-field a quote character followed by a second quote -// character is considered a single quote. -// -// "the ""word"" is true","a ""quoted-field""" -// -// results in -// -// {`the "word" is true`, `a "quoted-field"`} -// -// Newlines and commas may be included in a quoted-field -// -// "Multi-line -// field","comma is ," -// -// results in -// -// {`Multi-line -// field`, `comma is ,`} -// -// Modified to be used with MinIO. Main modifications include -// - Configurable 'quote' parameter -// - Performance improvements -// benchmark old ns/op new ns/op delta -// BenchmarkRead-8 2807 2189 -22.02% -// BenchmarkReadWithFieldsPerRecord-8 2802 2179 -22.23% -// BenchmarkReadWithoutFieldsPerRecord-8 2824 2181 -22.77% -// BenchmarkReadLargeFields-8 3584 3371 -5.94% -// BenchmarkReadReuseRecord-8 2044 1480 -27.59% -// BenchmarkReadReuseRecordWithFieldsPerRecord-8 2056 1483 -27.87% -// BenchmarkReadReuseRecordWithoutFieldsPerRecord-8 2047 1482 -27.60% -// BenchmarkReadReuseRecordLargeFields-8 2777 2594 -6.59% -package csv - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "unicode" - "unicode/utf8" -) - -// A ParseError is returned for parsing errors. -// Line numbers are 1-indexed and columns are 0-indexed. -type ParseError struct { - StartLine int // Line where the record starts - Line int // Line where the error occurred - Column int // Column (rune index) where the error occurred - Err error // The actual error -} - -func (e *ParseError) Error() string { - if e.Err == ErrFieldCount { - return fmt.Sprintf("record on line %d: %v", e.Line, e.Err) - } - if e.StartLine != e.Line { - return fmt.Sprintf("record on line %d; parse error on line %d, column %d: %v", e.StartLine, e.Line, e.Column, e.Err) - } - return fmt.Sprintf("parse error on line %d, column %d: %v", e.Line, e.Column, e.Err) -} - -// Unwrap returns the underlying error -func (e *ParseError) Unwrap() error { return e.Err } - -// These are the errors that can be returned in ParseError.Err. -var ( - ErrTrailingComma = errors.New("extra delimiter at end of line") // Deprecated: No longer used. - ErrBareQuote = errors.New("bare \" in non-quoted-field") - ErrQuote = errors.New("extraneous or missing \" in quoted-field") - ErrFieldCount = errors.New("wrong number of fields") -) - -var errInvalidDelim = errors.New("csv: invalid field or comment delimiter") - -func validDelim(r rune) bool { - return r != 0 && r != '"' && r != '\r' && r != '\n' && utf8.ValidRune(r) && r != utf8.RuneError -} - -// A Reader reads records from a CSV-encoded file. -// -// As returned by NewReader, a Reader expects input conforming to RFC 4180. -// The exported fields can be changed to customize the details before the -// first call to Read or ReadAll. -// -// The Reader converts all \r\n sequences in its input to plain \n, -// including in multiline field values, so that the returned data does -// not depend on which line-ending convention an input file uses. -type Reader struct { - // Comma is the field delimiter. - // It is set to comma (',') by NewReader. - // Comma must be a valid rune and must not be \r, \n, - // or the Unicode replacement character (0xFFFD). - Comma rune - - // Quote is a single rune used for marking fields limits - Quote []rune - - // QuoteEscape is a single rune to escape the quote character - QuoteEscape rune - - // Comment, if not 0, is the comment character. Lines beginning with the - // Comment character without preceding whitespace are ignored. - // With leading whitespace the Comment character becomes part of the - // field, even if TrimLeadingSpace is true. - // Comment must be a valid rune and must not be \r, \n, - // or the Unicode replacement character (0xFFFD). - // It must also not be equal to Comma. - Comment rune - - // FieldsPerRecord is the number of expected fields per record. - // If FieldsPerRecord is positive, Read requires each record to - // have the given number of fields. If FieldsPerRecord is 0, Read sets it to - // the number of fields in the first record, so that future records must - // have the same field count. If FieldsPerRecord is negative, no check is - // made and records may have a variable number of fields. - FieldsPerRecord int - - // If LazyQuotes is true, a quote may appear in an unquoted field and a - // non-doubled quote may appear in a quoted field. - LazyQuotes bool - - // If TrimLeadingSpace is true, leading white space in a field is ignored. - // This is done even if the field delimiter, Comma, is white space. - TrimLeadingSpace bool - - // ReuseRecord controls whether calls to Read may return a slice sharing - // the backing array of the previous call's returned slice for performance. - // By default, each call to Read returns newly allocated memory owned by the caller. - ReuseRecord bool - - TrailingComma bool // Deprecated: No longer used. - - r *bufio.Reader - - // numLine is the current line being read in the CSV file. - numLine int - - // rawBuffer is a line buffer only used by the readLine method. - rawBuffer []byte - - // recordBuffer holds the unescaped fields, one after another. - // The fields can be accessed by using the indexes in fieldIndexes. - // E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de` - // and fieldIndexes will contain the indexes [1, 2, 5, 6]. - recordBuffer []byte - - // fieldIndexes is an index of fields inside recordBuffer. - // The i'th field ends at offset fieldIndexes[i] in recordBuffer. - fieldIndexes []int - - // lastRecord is a record cache and only used when ReuseRecord == true. - lastRecord []string - - // Caching some values between Read() calls for performance gain - cached bool - cachedQuoteEscapeLen int - cachedQuoteLen int - cachedEncodedQuote []byte - cachedCommaLen int - cachedQuotes string -} - -// NewReader returns a new Reader that reads from r. -func NewReader(r io.Reader) *Reader { - return &Reader{ - Comma: ',', - Quote: []rune(`"`), - QuoteEscape: '"', - r: bufio.NewReader(r), - } -} - -// Read reads one record (a slice of fields) from r. -// If the record has an unexpected number of fields, -// Read returns the record along with the error ErrFieldCount. -// Except for that case, Read always returns either a non-nil -// record or a non-nil error, but not both. -// If there is no data left to be read, Read returns nil, io.EOF. -// If ReuseRecord is true, the returned slice may be shared -// between multiple calls to Read. -func (r *Reader) Read() (record []string, err error) { - if r.ReuseRecord { - record, err = r.readRecord(r.lastRecord) - r.lastRecord = record - } else { - record, err = r.readRecord(nil) - } - return record, err -} - -// ReadAll reads all the remaining records from r. -// Each record is a slice of fields. -// A successful call returns err == nil, not err == io.EOF. Because ReadAll is -// defined to read until EOF, it does not treat end of file as an error to be -// reported. -func (r *Reader) ReadAll() (records [][]string, err error) { - for { - record, err := r.readRecord(nil) - if err == io.EOF { - return records, nil - } - if err != nil { - return nil, err - } - records = append(records, record) - } -} - -// readLine reads the next line (with the trailing endline). -// If EOF is hit without a trailing endline, it will be omitted. -// If some bytes were read, then the error is never io.EOF. -// The result is only valid until the next call to readLine. -func (r *Reader) readLine() ([]byte, error) { - line, err := r.r.ReadSlice('\n') - if err == bufio.ErrBufferFull { - r.rawBuffer = append(r.rawBuffer[:0], line...) - for err == bufio.ErrBufferFull { - line, err = r.r.ReadSlice('\n') - r.rawBuffer = append(r.rawBuffer, line...) - } - line = r.rawBuffer - } - if len(line) > 0 && err == io.EOF { - err = nil - // For backwards compatibility, drop trailing \r before EOF. - if line[len(line)-1] == '\r' { - line = line[:len(line)-1] - } - } - r.numLine++ - // Normalize \r\n to \n on all input lines. - if n := len(line); n >= 2 && line[n-2] == '\r' && line[n-1] == '\n' { - line[n-2] = '\n' - line = line[:n-1] - } - return line, err -} - -// lengthNL reports the number of bytes for the trailing \n. -func lengthNL(b []byte) int { - if len(b) > 0 && b[len(b)-1] == '\n' { - return 1 - } - return 0 -} - -// nextRune returns the next rune in b or utf8.RuneError. -func nextRune(b []byte) rune { - r, _ := utf8.DecodeRune(b) - return r -} - -func encodeRune(r rune) []byte { - rlen := utf8.RuneLen(r) - p := make([]byte, rlen) - _ = utf8.EncodeRune(p, r) - return p -} - -func (r *Reader) readRecord(dst []string) ([]string, error) { - if r.Comma == r.Comment || !validDelim(r.Comma) || (r.Comment != 0 && !validDelim(r.Comment)) { - return nil, errInvalidDelim - } - - // Read line (automatically skipping past empty lines and any comments). - var line, fullLine []byte - var errRead error - for errRead == nil { - line, errRead = r.readLine() - if r.Comment != 0 && nextRune(line) == r.Comment { - line = nil - continue // Skip comment lines - } - if errRead == nil && len(line) == lengthNL(line) { - line = nil - continue // Skip empty lines - } - fullLine = line - break - } - if errRead == io.EOF { - return nil, errRead - } - - if !r.cached { - r.cachedQuoteEscapeLen = utf8.RuneLen(r.QuoteEscape) - if len(r.Quote) > 0 { - r.cachedQuoteLen = utf8.RuneLen(r.Quote[0]) - r.cachedEncodedQuote = encodeRune(r.Quote[0]) - r.cachedQuotes += string(r.Quote[0]) - } - r.cachedCommaLen = utf8.RuneLen(r.Comma) - r.cachedQuotes += string(r.QuoteEscape) - r.cached = true - } - - // Parse each field in the record. - var err error - recLine := r.numLine // Starting line for record - r.recordBuffer = r.recordBuffer[:0] - r.fieldIndexes = r.fieldIndexes[:0] -parseField: - for { - if r.TrimLeadingSpace { - line = bytes.TrimLeftFunc(line, unicode.IsSpace) - } - if len(line) == 0 || r.cachedQuoteLen == 0 || nextRune(line) != r.Quote[0] { - // Non-quoted string field - i := bytes.IndexRune(line, r.Comma) - field := line - if i >= 0 { - field = field[:i] - } else { - field = field[:len(field)-lengthNL(field)] - } - // Check to make sure a quote does not appear in field. - if !r.LazyQuotes { - if j := bytes.IndexRune(field, r.Quote[0]); j >= 0 { - col := utf8.RuneCount(fullLine[:len(fullLine)-len(line[j:])]) - err = &ParseError{StartLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote} - break parseField - } - } - r.recordBuffer = append(r.recordBuffer, field...) - r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) - if i >= 0 { - line = line[i+r.cachedCommaLen:] - continue parseField - } - break parseField - } else { - // Quoted string field - line = line[r.cachedQuoteLen:] - for { - i := bytes.IndexAny(line, r.cachedQuotes) - if i >= 0 { - // Hit next quote or escape quote - r.recordBuffer = append(r.recordBuffer, line[:i]...) - - escape := nextRune(line[i:]) == r.QuoteEscape - if escape { - line = line[i+r.cachedQuoteEscapeLen:] - } else { - line = line[i+r.cachedQuoteLen:] - } - - switch rn := nextRune(line); { - case escape && r.QuoteEscape != r.Quote[0]: - r.recordBuffer = append(r.recordBuffer, encodeRune(rn)...) - line = line[utf8.RuneLen(rn):] - case rn == r.Quote[0]: - // `""` sequence (append quote). - r.recordBuffer = append(r.recordBuffer, r.cachedEncodedQuote...) - line = line[r.cachedQuoteLen:] - case rn == r.Comma: - // `",` sequence (end of field). - line = line[r.cachedCommaLen:] - r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) - continue parseField - case lengthNL(line) == len(line): - // `"\n` sequence (end of line). - r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) - break parseField - case r.LazyQuotes: - // `"` sequence (bare quote). - r.recordBuffer = append(r.recordBuffer, r.cachedEncodedQuote...) - default: - // `"*` sequence (invalid non-escaped quote). - col := utf8.RuneCount(fullLine[:len(fullLine)-len(line)-r.cachedQuoteLen]) - err = &ParseError{StartLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote} - break parseField - } - } else if len(line) > 0 { - // Hit end of line (copy all data so far). - r.recordBuffer = append(r.recordBuffer, line...) - if errRead != nil { - break parseField - } - line, errRead = r.readLine() - if errRead == io.EOF { - errRead = nil - } - fullLine = line - } else { - // Abrupt end of file (EOF or error). - if !r.LazyQuotes && errRead == nil { - col := utf8.RuneCount(fullLine) - err = &ParseError{StartLine: recLine, Line: r.numLine, Column: col, Err: ErrQuote} - break parseField - } - r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer)) - break parseField - } - } - } - } - if err == nil { - err = errRead - } - - // Create a single string and create slices out of it. - // This pins the memory of the fields together, but allocates once. - str := string(r.recordBuffer) // Convert to string once to batch allocations - dst = dst[:0] - if cap(dst) < len(r.fieldIndexes) { - dst = make([]string, len(r.fieldIndexes)) - } - dst = dst[:len(r.fieldIndexes)] - var preIdx int - for i, idx := range r.fieldIndexes { - dst[i] = str[preIdx:idx] - preIdx = idx - } - - // Check or update the expected fields per record. - if r.FieldsPerRecord > 0 { - if len(dst) != r.FieldsPerRecord && err == nil { - err = &ParseError{StartLine: recLine, Line: recLine, Err: ErrFieldCount} - } - } else if r.FieldsPerRecord == 0 { - r.FieldsPerRecord = len(dst) - } - return dst, err -} diff --git a/contrib/pkg/csvparser/reader_test.go b/contrib/pkg/csvparser/reader_test.go deleted file mode 100644 index b129cfc60..000000000 --- a/contrib/pkg/csvparser/reader_test.go +++ /dev/null @@ -1,509 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in https://golang.org/LICENSE - -package csv - -import ( - "io" - "reflect" - "strings" - "testing" - "unicode/utf8" -) - -func TestRead(t *testing.T) { - tests := []struct { - Name string - Input string - Output [][]string - Error error - - // These fields are copied into the Reader - Comma rune - Comment rune - UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1 - FieldsPerRecord int - LazyQuotes bool - TrimLeadingSpace bool - ReuseRecord bool - }{{ - Name: "Simple", - Input: "a,b,c\n", - Output: [][]string{{"a", "b", "c"}}, - }, { - Name: "CRLF", - Input: "a,b\r\nc,d\r\n", - Output: [][]string{{"a", "b"}, {"c", "d"}}, - }, { - Name: "BareCR", - Input: "a,b\rc,d\r\n", - Output: [][]string{{"a", "b\rc", "d"}}, - }, { - Name: "RFC4180test", - Input: `#field1,field2,field3 -"aaa","bb -b","ccc" -"a,a","b""bb","ccc" -zzz,yyy,xxx -`, - Output: [][]string{ - {"#field1", "field2", "field3"}, - {"aaa", "bb\nb", "ccc"}, - {"a,a", `b"bb`, "ccc"}, - {"zzz", "yyy", "xxx"}, - }, - UseFieldsPerRecord: true, - FieldsPerRecord: 0, - }, { - Name: "NoEOLTest", - Input: "a,b,c", - Output: [][]string{{"a", "b", "c"}}, - }, { - Name: "Semicolon", - Input: "a;b;c\n", - Output: [][]string{{"a", "b", "c"}}, - Comma: ';', - }, { - Name: "MultiLine", - Input: `"two -line","one line","three -line -field"`, - Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}}, - }, { - Name: "BlankLine", - Input: "a,b,c\n\nd,e,f\n\n", - Output: [][]string{ - {"a", "b", "c"}, - {"d", "e", "f"}, - }, - }, { - Name: "BlankLineFieldCount", - Input: "a,b,c\n\nd,e,f\n\n", - Output: [][]string{ - {"a", "b", "c"}, - {"d", "e", "f"}, - }, - UseFieldsPerRecord: true, - FieldsPerRecord: 0, - }, { - Name: "TrimSpace", - Input: " a, b, c\n", - Output: [][]string{{"a", "b", "c"}}, - TrimLeadingSpace: true, - }, { - Name: "LeadingSpace", - Input: " a, b, c\n", - Output: [][]string{{" a", " b", " c"}}, - }, { - Name: "Comment", - Input: "#1,2,3\na,b,c\n#comment", - Output: [][]string{{"a", "b", "c"}}, - Comment: '#', - }, { - Name: "NoComment", - Input: "#1,2,3\na,b,c", - Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}}, - }, { - Name: "LazyQuotes", - Input: `a "word","1"2",a","b`, - Output: [][]string{{`a "word"`, `1"2`, `a"`, `b`}}, - LazyQuotes: true, - }, { - Name: "BareQuotes", - Input: `a "word","1"2",a"`, - Output: [][]string{{`a "word"`, `1"2`, `a"`}}, - LazyQuotes: true, - }, { - Name: "BareDoubleQuotes", - Input: `a""b,c`, - Output: [][]string{{`a""b`, `c`}}, - LazyQuotes: true, - }, { - Name: "BadDoubleQuotes", - Input: `a""b,c`, - Error: &ParseError{StartLine: 1, Line: 1, Column: 1, Err: ErrBareQuote}, - }, { - Name: "TrimQuote", - Input: ` "a"," b",c`, - Output: [][]string{{"a", " b", "c"}}, - TrimLeadingSpace: true, - }, { - Name: "BadBareQuote", - Input: `a "word","b"`, - Error: &ParseError{StartLine: 1, Line: 1, Column: 2, Err: ErrBareQuote}, - }, { - Name: "BadTrailingQuote", - Input: `"a word",b"`, - Error: &ParseError{StartLine: 1, Line: 1, Column: 10, Err: ErrBareQuote}, - }, { - Name: "ExtraneousQuote", - Input: `"a "word","b"`, - Error: &ParseError{StartLine: 1, Line: 1, Column: 3, Err: ErrQuote}, - }, { - Name: "BadFieldCount", - Input: "a,b,c\nd,e", - Error: &ParseError{StartLine: 2, Line: 2, Err: ErrFieldCount}, - UseFieldsPerRecord: true, - FieldsPerRecord: 0, - }, { - Name: "BadFieldCount1", - Input: `a,b,c`, - Error: &ParseError{StartLine: 1, Line: 1, Err: ErrFieldCount}, - UseFieldsPerRecord: true, - FieldsPerRecord: 2, - }, { - Name: "FieldCount", - Input: "a,b,c\nd,e", - Output: [][]string{{"a", "b", "c"}, {"d", "e"}}, - }, { - Name: "TrailingCommaEOF", - Input: "a,b,c,", - Output: [][]string{{"a", "b", "c", ""}}, - }, { - Name: "TrailingCommaEOL", - Input: "a,b,c,\n", - Output: [][]string{{"a", "b", "c", ""}}, - }, { - Name: "TrailingCommaSpaceEOF", - Input: "a,b,c, ", - Output: [][]string{{"a", "b", "c", ""}}, - TrimLeadingSpace: true, - }, { - Name: "TrailingCommaSpaceEOL", - Input: "a,b,c, \n", - Output: [][]string{{"a", "b", "c", ""}}, - TrimLeadingSpace: true, - }, { - Name: "TrailingCommaLine3", - Input: "a,b,c\nd,e,f\ng,hi,", - Output: [][]string{{"a", "b", "c"}, {"d", "e", "f"}, {"g", "hi", ""}}, - TrimLeadingSpace: true, - }, { - Name: "NotTrailingComma3", - Input: "a,b,c, \n", - Output: [][]string{{"a", "b", "c", " "}}, - }, { - Name: "CommaFieldTest", - Input: `x,y,z,w -x,y,z, -x,y,, -x,,, -,,, -"x","y","z","w" -"x","y","z","" -"x","y","","" -"x","","","" -"","","","" -`, - Output: [][]string{ - {"x", "y", "z", "w"}, - {"x", "y", "z", ""}, - {"x", "y", "", ""}, - {"x", "", "", ""}, - {"", "", "", ""}, - {"x", "y", "z", "w"}, - {"x", "y", "z", ""}, - {"x", "y", "", ""}, - {"x", "", "", ""}, - {"", "", "", ""}, - }, - }, { - Name: "TrailingCommaIneffective1", - Input: "a,b,\nc,d,e", - Output: [][]string{ - {"a", "b", ""}, - {"c", "d", "e"}, - }, - TrimLeadingSpace: true, - }, { - Name: "ReadAllReuseRecord", - Input: "a,b\nc,d", - Output: [][]string{ - {"a", "b"}, - {"c", "d"}, - }, - ReuseRecord: true, - }, { - Name: "StartLine1", // Issue 19019 - Input: "a,\"b\nc\"d,e", - Error: &ParseError{StartLine: 1, Line: 2, Column: 1, Err: ErrQuote}, - }, { - Name: "StartLine2", - Input: "a,b\n\"d\n\n,e", - Error: &ParseError{StartLine: 2, Line: 5, Column: 0, Err: ErrQuote}, - }, { - Name: "CRLFInQuotedField", // Issue 21201 - Input: "A,\"Hello\r\nHi\",B\r\n", - Output: [][]string{ - {"A", "Hello\nHi", "B"}, - }, - }, { - Name: "BinaryBlobField", // Issue 19410 - Input: "x09\x41\xb4\x1c,aktau", - Output: [][]string{{"x09A\xb4\x1c", "aktau"}}, - }, { - Name: "TrailingCR", - Input: "field1,field2\r", - Output: [][]string{{"field1", "field2"}}, - }, { - Name: "QuotedTrailingCR", - Input: "\"field\"\r", - Output: [][]string{{"field"}}, - }, { - Name: "QuotedTrailingCRCR", - Input: "\"field\"\r\r", - Error: &ParseError{StartLine: 1, Line: 1, Column: 6, Err: ErrQuote}, - }, { - Name: "FieldCR", - Input: "field\rfield\r", - Output: [][]string{{"field\rfield"}}, - }, { - Name: "FieldCRCR", - Input: "field\r\rfield\r\r", - Output: [][]string{{"field\r\rfield\r"}}, - }, { - Name: "FieldCRCRLF", - Input: "field\r\r\nfield\r\r\n", - Output: [][]string{{"field\r"}, {"field\r"}}, - }, { - Name: "FieldCRCRLFCR", - Input: "field\r\r\n\rfield\r\r\n\r", - Output: [][]string{{"field\r"}, {"\rfield\r"}}, - }, { - Name: "FieldCRCRLFCRCR", - Input: "field\r\r\n\r\rfield\r\r\n\r\r", - Output: [][]string{{"field\r"}, {"\r\rfield\r"}, {"\r"}}, - }, { - Name: "MultiFieldCRCRLFCRCR", - Input: "field1,field2\r\r\n\r\rfield1,field2\r\r\n\r\r,", - Output: [][]string{ - {"field1", "field2\r"}, - {"\r\rfield1", "field2\r"}, - {"\r\r", ""}, - }, - }, { - Name: "NonASCIICommaAndComment", - Input: "a£b,c£ \td,e\n€ comment\n", - Output: [][]string{{"a", "b,c", "d,e"}}, - TrimLeadingSpace: true, - Comma: '£', - Comment: '€', - }, { - Name: "NonASCIICommaAndCommentWithQuotes", - Input: "a€\" b,\"€ c\nλ comment\n", - Output: [][]string{{"a", " b,", " c"}}, - Comma: '€', - Comment: 'λ', - }, { - // λ and θ start with the same byte. - // This tests that the parser doesn't confuse such characters. - Name: "NonASCIICommaConfusion", - Input: "\"abθcd\"λefθgh", - Output: [][]string{{"abθcd", "efθgh"}}, - Comma: 'λ', - Comment: '€', - }, { - Name: "NonASCIICommentConfusion", - Input: "λ\nλ\nθ\nλ\n", - Output: [][]string{{"λ"}, {"λ"}, {"λ"}}, - Comment: 'θ', - }, { - Name: "QuotedFieldMultipleLF", - Input: "\"\n\n\n\n\"", - Output: [][]string{{"\n\n\n\n"}}, - }, { - Name: "MultipleCRLF", - Input: "\r\n\r\n\r\n\r\n", - }, { - // The implementation may read each line in several chunks if it doesn't fit entirely - // in the read buffer, so we should test the code to handle that condition. - Name: "HugeLines", - Input: strings.Repeat("#ignore\n", 10000) + strings.Repeat("@", 5000) + "," + strings.Repeat("*", 5000), - Output: [][]string{{strings.Repeat("@", 5000), strings.Repeat("*", 5000)}}, - Comment: '#', - }, { - Name: "QuoteWithTrailingCRLF", - Input: "\"foo\"bar\"\r\n", - Error: &ParseError{StartLine: 1, Line: 1, Column: 4, Err: ErrQuote}, - }, { - Name: "LazyQuoteWithTrailingCRLF", - Input: "\"foo\"bar\"\r\n", - Output: [][]string{{`foo"bar`}}, - LazyQuotes: true, - }, { - Name: "DoubleQuoteWithTrailingCRLF", - Input: "\"foo\"\"bar\"\r\n", - Output: [][]string{{`foo"bar`}}, - }, { - Name: "EvenQuotes", - Input: `""""""""`, - Output: [][]string{{`"""`}}, - }, { - Name: "OddQuotes", - Input: `"""""""`, - Error: &ParseError{StartLine: 1, Line: 1, Column: 7, Err: ErrQuote}, - }, { - Name: "LazyOddQuotes", - Input: `"""""""`, - Output: [][]string{{`"""`}}, - LazyQuotes: true, - }, { - Name: "BadComma1", - Comma: '\n', - Error: errInvalidDelim, - }, { - Name: "BadComma2", - Comma: '\r', - Error: errInvalidDelim, - }, { - Name: "BadComma3", - Comma: '"', - Error: errInvalidDelim, - }, { - Name: "BadComma4", - Comma: utf8.RuneError, - Error: errInvalidDelim, - }, { - Name: "BadComment1", - Comment: '\n', - Error: errInvalidDelim, - }, { - Name: "BadComment2", - Comment: '\r', - Error: errInvalidDelim, - }, { - Name: "BadComment3", - Comment: utf8.RuneError, - Error: errInvalidDelim, - }, { - Name: "BadCommaComment", - Comma: 'X', - Comment: 'X', - Error: errInvalidDelim, - }} - - for _, tt := range tests { - t.Run(tt.Name, func(t *testing.T) { - r := NewReader(strings.NewReader(tt.Input)) - - if tt.Comma != 0 { - r.Comma = tt.Comma - } - r.Comment = tt.Comment - if tt.UseFieldsPerRecord { - r.FieldsPerRecord = tt.FieldsPerRecord - } else { - r.FieldsPerRecord = -1 - } - r.LazyQuotes = tt.LazyQuotes - r.TrimLeadingSpace = tt.TrimLeadingSpace - r.ReuseRecord = tt.ReuseRecord - - out, err := r.ReadAll() - if !reflect.DeepEqual(err, tt.Error) { - t.Errorf("ReadAll() error:\ngot %v\nwant %v", err, tt.Error) - } else if !reflect.DeepEqual(out, tt.Output) { - t.Errorf("ReadAll() output:\ngot %q\nwant %q", out, tt.Output) - } - }) - } -} - -// nTimes is an io.Reader which yields the string s n times. -type nTimes struct { - s string - n int - off int -} - -func (r *nTimes) Read(p []byte) (n int, err error) { - for { - if r.n <= 0 || r.s == "" { - return n, io.EOF - } - n0 := copy(p, r.s[r.off:]) - p = p[n0:] - n += n0 - r.off += n0 - if r.off == len(r.s) { - r.off = 0 - r.n-- - } - if len(p) == 0 { - return - } - } -} - -// benchmarkRead measures reading the provided CSV rows data. -// initReader, if non-nil, modifies the Reader before it's used. -func benchmarkRead(b *testing.B, initReader func(*Reader), rows string) { - b.ReportAllocs() - r := NewReader(&nTimes{s: rows, n: b.N}) - if initReader != nil { - initReader(r) - } - for { - _, err := r.Read() - if err == io.EOF { - break - } - if err != nil { - b.Fatal(err) - } - } -} - -const benchmarkCSVData = `x,y,z,w -x,y,z, -x,y,, -x,,, -,,, -"x","y","z","w" -"x","y","z","" -"x","y","","" -"x","","","" -"","","","" -` - -func BenchmarkRead(b *testing.B) { - benchmarkRead(b, nil, benchmarkCSVData) -} - -func BenchmarkReadWithFieldsPerRecord(b *testing.B) { - benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = 4 }, benchmarkCSVData) -} - -func BenchmarkReadWithoutFieldsPerRecord(b *testing.B) { - benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = -1 }, benchmarkCSVData) -} - -func BenchmarkReadLargeFields(b *testing.B) { - benchmarkRead(b, nil, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv -xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv -,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv -xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv -`, 3)) -} - -func BenchmarkReadReuseRecord(b *testing.B) { - benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, benchmarkCSVData) -} - -func BenchmarkReadReuseRecordWithFieldsPerRecord(b *testing.B) { - benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = 4 }, benchmarkCSVData) -} - -func BenchmarkReadReuseRecordWithoutFieldsPerRecord(b *testing.B) { - benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true; r.FieldsPerRecord = -1 }, benchmarkCSVData) -} - -func BenchmarkReadReuseRecordLargeFields(b *testing.B) { - benchmarkRead(b, func(r *Reader) { r.ReuseRecord = true }, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv -xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv -,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv -xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv -`, 3)) -} diff --git a/contrib/pkg/csvparser/writer.go b/contrib/pkg/csvparser/writer.go deleted file mode 100644 index 2376a4769..000000000 --- a/contrib/pkg/csvparser/writer.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in https://golang.org/LICENSE - -package csv - -import ( - "bufio" - "io" - "strings" - "unicode" - "unicode/utf8" -) - -// A Writer writes records using CSV encoding. -// -// As returned by NewWriter, a Writer writes records terminated by a -// newline and uses ',' as the field delimiter. The exported fields can be -// changed to customize the details before the first call to Write or WriteAll. -// -// Comma is the field delimiter. -// -// If UseCRLF is true, the Writer ends each output line with \r\n instead of \n. -// -// The writes of individual records are buffered. -// After all data has been written, the client should call the -// Flush method to guarantee all data has been forwarded to -// the underlying io.Writer. Any errors that occurred should -// be checked by calling the Error method. -type Writer struct { - Comma rune // Field delimiter (set to ',' by NewWriter) - Quote rune // Fields quote character - QuoteEscape rune - AlwaysQuote bool // True to quote all fields - UseCRLF bool // True to use \r\n as the line terminator - w *bufio.Writer -} - -// NewWriter returns a new Writer that writes to w. -func NewWriter(w io.Writer) *Writer { - return &Writer{ - Comma: ',', - Quote: '"', - QuoteEscape: '"', - w: bufio.NewWriter(w), - } -} - -// Write writes a single CSV record to w along with any necessary quoting. -// A record is a slice of strings with each string being one field. -// Writes are buffered, so Flush must eventually be called to ensure -// that the record is written to the underlying io.Writer. -func (w *Writer) Write(record []string) error { - if !validDelim(w.Comma) { - return errInvalidDelim - } - - for n, field := range record { - if n > 0 { - if _, err := w.w.WriteRune(w.Comma); err != nil { - return err - } - } - - // If we don't have to have a quoted field then just - // write out the field and continue to the next field. - if !w.AlwaysQuote && !w.fieldNeedsQuotes(field) { - if _, err := w.w.WriteString(field); err != nil { - return err - } - continue - } - - if _, err := w.w.WriteRune(w.Quote); err != nil { - return err - } - - specialChars := "\r\n" + string(w.Quote) - - for len(field) > 0 { - // Search for special characters. - i := strings.IndexAny(field, specialChars) - if i < 0 { - i = len(field) - } - - // Copy verbatim everything before the special character. - if _, err := w.w.WriteString(field[:i]); err != nil { - return err - } - field = field[i:] - - // Encode the special character. - if len(field) > 0 { - var err error - switch nextRune([]byte(field)) { - case w.Quote: - _, err = w.w.WriteRune(w.QuoteEscape) - if err != nil { - break - } - _, err = w.w.WriteRune(w.Quote) - case '\r': - if !w.UseCRLF { - err = w.w.WriteByte('\r') - } - case '\n': - if w.UseCRLF { - _, err = w.w.WriteString("\r\n") - } else { - err = w.w.WriteByte('\n') - } - } - field = field[1:] - if err != nil { - return err - } - } - } - if _, err := w.w.WriteRune(w.Quote); err != nil { - return err - } - } - var err error - if w.UseCRLF { - _, err = w.w.WriteString("\r\n") - } else { - err = w.w.WriteByte('\n') - } - return err -} - -// Flush writes any buffered data to the underlying io.Writer. -// To check if an error occurred during the Flush, call Error. -func (w *Writer) Flush() { - w.w.Flush() -} - -// Error reports any error that has occurred during a previous Write or Flush. -func (w *Writer) Error() error { - _, err := w.w.Write(nil) - return err -} - -// WriteAll writes multiple CSV records to w using Write and then calls Flush, -// returning any error from the Flush. -func (w *Writer) WriteAll(records [][]string) error { - for _, record := range records { - err := w.Write(record) - if err != nil { - return err - } - } - return w.w.Flush() -} - -// fieldNeedsQuotes reports whether our field must be enclosed in quotes. -// Fields with a Comma, fields with a quote or newline, and -// fields which start with a space must be enclosed in quotes. -// We used to quote empty strings, but we do not anymore (as of Go 1.4). -// The two representations should be equivalent, but Postgres distinguishes -// quoted vs non-quoted empty string during database imports, and it has -// an option to force the quoted behavior for non-quoted CSV but it has -// no option to force the non-quoted behavior for quoted CSV, making -// CSV with quoted empty strings strictly less useful. -// Not quoting the empty string also makes this package match the behavior -// of Microsoft Excel and Google Drive. -// For Postgres, quote the data terminating string `\.`. -func (w *Writer) fieldNeedsQuotes(field string) bool { - if field == "" { - return false - } - if field == `\.` || strings.ContainsAny(field, "\r\n"+string(w.Quote)+string(w.Comma)) { - return true - } - - r1, _ := utf8.DecodeRuneInString(field) - return unicode.IsSpace(r1) -} diff --git a/contrib/pkg/csvparser/writer_test.go b/contrib/pkg/csvparser/writer_test.go deleted file mode 100644 index e64c15400..000000000 --- a/contrib/pkg/csvparser/writer_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in https://golang.org/LICENSE - -package csv - -import ( - "bytes" - "errors" - "testing" -) - -var writeTests = []struct { - Input [][]string - Output string - Error error - UseCRLF bool - Comma rune - Quote rune - AlwaysQuote bool -}{ - {Input: [][]string{{"abc"}}, Output: "abc\n"}, - {Input: [][]string{{"abc"}}, Output: "abc\r\n", UseCRLF: true}, - {Input: [][]string{{`"abc"`}}, Output: `"""abc"""` + "\n"}, - {Input: [][]string{{`a"b`}}, Output: `"a""b"` + "\n"}, - {Input: [][]string{{`"a"b"`}}, Output: `"""a""b"""` + "\n"}, - {Input: [][]string{{" abc"}}, Output: `" abc"` + "\n"}, - {Input: [][]string{{"abc,def"}}, Output: `"abc,def"` + "\n"}, - {Input: [][]string{{"abc", "def"}}, Output: "abc,def\n"}, - {Input: [][]string{{"abc"}, {"def"}}, Output: "abc\ndef\n"}, - {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\ndef\"\n"}, - {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\r\ndef\"\r\n", UseCRLF: true}, - {Input: [][]string{{"abc\rdef"}}, Output: "\"abcdef\"\r\n", UseCRLF: true}, - {Input: [][]string{{"abc\rdef"}}, Output: "\"abc\rdef\"\n", UseCRLF: false}, - {Input: [][]string{{""}}, Output: "\n"}, - {Input: [][]string{{"", ""}}, Output: ",\n"}, - {Input: [][]string{{"", "", ""}}, Output: ",,\n"}, - {Input: [][]string{{"", "", "a"}}, Output: ",,a\n"}, - {Input: [][]string{{"", "a", ""}}, Output: ",a,\n"}, - {Input: [][]string{{"", "a", "a"}}, Output: ",a,a\n"}, - {Input: [][]string{{"a", "", ""}}, Output: "a,,\n"}, - {Input: [][]string{{"a", "", "a"}}, Output: "a,,a\n"}, - {Input: [][]string{{"a", "a", ""}}, Output: "a,a,\n"}, - {Input: [][]string{{"a", "a", "a"}}, Output: "a,a,a\n"}, - {Input: [][]string{{`\.`}}, Output: "\"\\.\"\n"}, - {Input: [][]string{{"x09\x41\xb4\x1c", "aktau"}}, Output: "x09\x41\xb4\x1c,aktau\n"}, - {Input: [][]string{{",x09\x41\xb4\x1c", "aktau"}}, Output: "\",x09\x41\xb4\x1c\",aktau\n"}, - {Input: [][]string{{"a", "a", ""}}, Output: "a|a|\n", Comma: '|'}, - {Input: [][]string{{",", ",", ""}}, Output: ",|,|\n", Comma: '|'}, - {Input: [][]string{{"foo"}}, Comma: '"', Error: errInvalidDelim}, - {Input: [][]string{{"a", "a", ""}}, Quote: '"', AlwaysQuote: true, Output: "\"a\"|\"a\"|\"\"\n", Comma: '|'}, -} - -func TestWrite(t *testing.T) { - for n, tt := range writeTests { - b := &bytes.Buffer{} - f := NewWriter(b) - f.UseCRLF = tt.UseCRLF - if tt.Comma != 0 { - f.Comma = tt.Comma - } - if tt.Quote != 0 { - f.Quote = tt.Quote - } - f.AlwaysQuote = tt.AlwaysQuote - err := f.WriteAll(tt.Input) - if err != tt.Error { - t.Errorf("Unexpected error:\ngot %v\nwant %v", err, tt.Error) - } - out := b.String() - if out != tt.Output { - t.Errorf("#%d: out=%q want %q", n, out, tt.Output) - } - } -} - -type errorWriter struct{} - -func (e errorWriter) Write(b []byte) (int, error) { - return 0, errors.New("Test") -} - -func TestError(t *testing.T) { - b := &bytes.Buffer{} - f := NewWriter(b) - f.Write([]string{"abc"}) - f.Flush() - err := f.Error() - - if err != nil { - t.Errorf("Unexpected error: %s\n", err) - } - - f = NewWriter(errorWriter{}) - f.Write([]string{"abc"}) - f.Flush() - err = f.Error() - - if err == nil { - t.Error("Error should not be nil") - } -} diff --git a/go.mod b/go.mod index a48cd6527..d0bb8ad19 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,9 @@ require ( github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.12 github.com/miekg/dns v1.1.35 + github.com/minio/argon2 v0.0.0-20210427164258-0025d10c2c04 github.com/minio/cli v1.22.0 + github.com/minio/csvparser v0.0.0-20210427163918-ee4f0ffa388d github.com/minio/highwayhash v1.0.2 github.com/minio/md5-simd v1.1.1 // indirect github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 diff --git a/go.sum b/go.sum index 4bc17c54c..042befb84 100644 --- a/go.sum +++ b/go.sum @@ -430,8 +430,12 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/minio/argon2 v0.0.0-20210427164258-0025d10c2c04 h1:4mrboT3YeVBHjAFia8IzmEUnzLRkemKeR0rjQ6+M9B4= +github.com/minio/argon2 v0.0.0-20210427164258-0025d10c2c04/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ= github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68= github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/csvparser v0.0.0-20210427163918-ee4f0ffa388d h1:Q/oXgmaJluRgq4zIht5CbAOJphFbiG+wsQcR5WmDVYM= +github.com/minio/csvparser v0.0.0-20210427163918-ee4f0ffa388d/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= diff --git a/pkg/madmin/encrypt.go b/pkg/madmin/encrypt.go index 74bb7d6a2..ea5f656c0 100644 --- a/pkg/madmin/encrypt.go +++ b/pkg/madmin/encrypt.go @@ -24,7 +24,7 @@ import ( "io" "io/ioutil" - "github.com/minio/minio/contrib/pkg/argon2" + "github.com/minio/argon2" "github.com/minio/minio/pkg/fips" "github.com/secure-io/sio-go" "github.com/secure-io/sio-go/sioutil" diff --git a/pkg/s3select/csv/reader.go b/pkg/s3select/csv/reader.go index 787426ae9..0b2be9035 100644 --- a/pkg/s3select/csv/reader.go +++ b/pkg/s3select/csv/reader.go @@ -26,7 +26,7 @@ import ( "sync" "unicode/utf8" - csv "github.com/minio/minio/contrib/pkg/csvparser" + csv "github.com/minio/csvparser" "github.com/minio/minio/pkg/s3select/sql" ) diff --git a/pkg/s3select/csv/record.go b/pkg/s3select/csv/record.go index 73eebb1d9..07890192a 100644 --- a/pkg/s3select/csv/record.go +++ b/pkg/s3select/csv/record.go @@ -24,7 +24,7 @@ import ( "io" "github.com/bcicen/jstream" - csv "github.com/minio/minio/contrib/pkg/csvparser" + csv "github.com/minio/csvparser" "github.com/minio/minio/pkg/s3select/sql" ) diff --git a/pkg/s3select/json/record.go b/pkg/s3select/json/record.go index 707e61efc..31ce7e17a 100644 --- a/pkg/s3select/json/record.go +++ b/pkg/s3select/json/record.go @@ -27,7 +27,7 @@ import ( "strings" "github.com/bcicen/jstream" - csv "github.com/minio/minio/contrib/pkg/csvparser" + csv "github.com/minio/csvparser" "github.com/minio/minio/pkg/s3select/sql" ) diff --git a/pkg/s3select/simdj/record.go b/pkg/s3select/simdj/record.go index 404e44de8..b68857bc9 100644 --- a/pkg/s3select/simdj/record.go +++ b/pkg/s3select/simdj/record.go @@ -22,7 +22,7 @@ import ( "io" "github.com/bcicen/jstream" - csv "github.com/minio/minio/contrib/pkg/csvparser" + csv "github.com/minio/csvparser" "github.com/minio/minio/pkg/s3select/json" "github.com/minio/minio/pkg/s3select/sql" "github.com/minio/simdjson-go"