rename all remaining packages to internal/ (#12418)

This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`

internal/s3select/simdj/reader_amd64_test.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package simdj
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/minio/minio/internal/s3select/json"
+	"github.com/minio/minio/internal/s3select/sql"
+	"github.com/minio/simdjson-go"
+)
+
+type tester interface {
+	Fatal(args ...interface{})
+}
+
+func loadCompressed(t tester, file string) (js []byte) {
+	dec, err := zstd.NewReader(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dec.Close()
+	js, err = ioutil.ReadFile(filepath.Join("testdata", file+".json.zst"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	js, err = dec.DecodeAll(js, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return js
+}
+
+var testCases = []struct {
+	name  string
+	array bool
+}{
+	{
+		name: "parking-citations-10",
+	},
+}
+
+func TestNDJSON(t *testing.T) {
+	if !simdjson.SupportedCPU() {
+		t.Skip("Unsupported cpu")
+	}
+
+	for _, tt := range testCases {
+
+		t.Run(tt.name, func(t *testing.T) {
+			ref := loadCompressed(t, tt.name)
+
+			var err error
+			dst := make(chan simdjson.Object, 100)
+			dec := NewElementReader(dst, &err, &json.ReaderArgs{ContentType: "json"})
+			pj, err := simdjson.ParseND(ref, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			i := pj.Iter()
+			cpy := i
+			b, err := cpy.MarshalJSON()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if false {
+				t.Log(string(b))
+			}
+			//_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".json"), b, os.ModePerm)
+
+		parser:
+			for {
+				var next simdjson.Iter
+				typ, err := i.AdvanceIter(&next)
+				if err != nil {
+					t.Fatal(err)
+				}
+				switch typ {
+				case simdjson.TypeNone:
+					close(dst)
+					break parser
+				case simdjson.TypeRoot:
+					typ, obj, err := next.Root(nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					if typ != simdjson.TypeObject {
+						if typ == simdjson.TypeNone {
+							close(dst)
+							break parser
+						}
+						t.Fatal("Unexpected type:", typ.String())
+					}
+
+					o, err := obj.Object(nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					dst <- *o
+				default:
+					t.Fatal("unexpected type:", typ.String())
+				}
+			}
+			refDec := json.NewReader(ioutil.NopCloser(bytes.NewBuffer(ref)), &json.ReaderArgs{ContentType: "json"})
+
+			for {
+				rec, err := dec.Read(nil)
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					t.Error(err)
+				}
+				want, err := refDec.Read(nil)
+				if err != nil {
+					t.Error(err)
+				}
+				var gotB, wantB bytes.Buffer
+				opts := sql.WriteCSVOpts{
+					FieldDelimiter: ',',
+					Quote:          '"',
+					QuoteEscape:    '"',
+					AlwaysQuote:    false,
+				}
+				err = rec.WriteCSV(&gotB, opts)
+				if err != nil {
+					t.Error(err)
+				}
+				err = want.WriteCSV(&wantB, opts)
+				if err != nil {
+					t.Error(err)
+				}
+
+				if !bytes.Equal(gotB.Bytes(), wantB.Bytes()) {
+					t.Errorf("CSV output mismatch.\nwant: %s(%x)\ngot:  %s(%x)", wantB.String(), wantB.Bytes(), gotB.String(), gotB.Bytes())
+				}
+				gotB.Reset()
+				wantB.Reset()
+
+				err = rec.WriteJSON(&gotB)
+				if err != nil {
+					t.Error(err)
+				}
+				err = want.WriteJSON(&wantB)
+				if err != nil {
+					t.Error(err)
+				}
+				// truncate newline from 'want'
+				wantB.Truncate(wantB.Len() - 1)
+				if !bytes.Equal(gotB.Bytes(), wantB.Bytes()) {
+					t.Errorf("JSON output mismatch.\nwant: %s\ngot:  %s", wantB.String(), gotB.String())
+				}
+			}
+		})
+	}
+}