diff --git a/cmd/object-api-getobjectinfo_test.go b/cmd/object-api-getobjectinfo_test.go index b65a1190d..ced333ff8 100644 --- a/cmd/object-api-getobjectinfo_test.go +++ b/cmd/object-api-getobjectinfo_test.go @@ -57,19 +57,19 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { {"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, {"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false}, {"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, - // Test cases with valid but non-existing bucket names (Test number 5-7). + // Test cases with valid but non-existing bucket names (Test number 5-6). {"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, {"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, - // Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9). + // Test cases with valid but non-existing bucket names and invalid object name (Test number 7-8). {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, - // Test cases with non-existing object name with existing bucket (Test number 10-12). + // Test cases with non-existing object name with existing bucket (Test number 9-11). {"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, {"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, {"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false}, - // Test case with existing bucket but object name set to a directory (Test number 13). + // Test case with existing bucket but object name set to a directory (Test number 12). {"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false}, - // Valid case with existing object (Test number 14). + // Valid case with existing object (Test number 13). {"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, } for i, testCase := range testCases { diff --git a/cmd/xl-v1-metadata.go b/cmd/xl-v1-metadata.go index d769e2de5..0208a91ff 100644 --- a/cmd/xl-v1-metadata.go +++ b/cmd/xl-v1-metadata.go @@ -136,9 +136,9 @@ func (m xlMetaV1) IsValid() bool { return m.Version == "1.0.0" && m.Format == "xl" } -// ObjectPartIndex - returns the index of matching object part number. -func (m xlMetaV1) ObjectPartIndex(partNumber int) int { - for i, part := range m.Parts { +// objectPartIndex - returns the index of matching object part number. +func objectPartIndex(parts []objectPartInfo, partNumber int) int { + for i, part := range parts { if partNumber == part.Number { return i } @@ -214,16 +214,15 @@ var objMetadataOpIgnoredErrs = []error{ errFileNotFound, } -// readXLMetadata - returns the object metadata `xl.json` content from -// one of the disks picked at random. -func (xl xlObjects) readXLMetadata(bucket, object string) (xlMeta xlMetaV1, err error) { +// readXLMetaParts - returns the XL Metadata Parts from xl.json of one of the disks picked at random. +func (xl xlObjects) readXLMetaParts(bucket, object string) (xlMetaParts []objectPartInfo, err error) { for _, disk := range xl.getLoadBalancedDisks() { if disk == nil { continue } - xlMeta, err = readXLMeta(disk, bucket, object) + xlMetaParts, err = readXLMetaParts(disk, bucket, object) if err == nil { - return xlMeta, nil + return xlMetaParts, nil } // For any reason disk or bucket is not available continue // and read from other disks. @@ -233,7 +232,29 @@ func (xl xlObjects) readXLMetadata(bucket, object string) (xlMeta xlMetaV1, err break } // Return error here. - return xlMetaV1{}, err + return nil, err +} + +// readXLMetaStat - return xlMetaV1.Stat and xlMetaV1.Meta from one of the disks picked at random. +func (xl xlObjects) readXLMetaStat(bucket, object string) (xlStat statInfo, xlMeta map[string]string, err error) { + for _, disk := range xl.getLoadBalancedDisks() { + if disk == nil { + continue + } + // parses only xlMetaV1.Meta and xlMeta.Stat + xlStat, xlMeta, err = readXLMetaStat(disk, bucket, object) + if err == nil { + return xlStat, xlMeta, nil + } + // For any reason disk or bucket is not available continue + // and read from other disks. + if isErrIgnored(err, objMetadataOpIgnoredErrs) { + continue + } + break + } + // Return error here. + return statInfo{}, nil, err } // deleteXLMetadata - deletes `xl.json` on a single disk. diff --git a/cmd/xl-v1-metadata_test.go b/cmd/xl-v1-metadata_test.go index 3671f0fb4..8c9fc0f39 100644 --- a/cmd/xl-v1-metadata_test.go +++ b/cmd/xl-v1-metadata_test.go @@ -55,13 +55,14 @@ func TestAddObjectPart(t *testing.T) { xlMeta.AddObjectPart(testCase.partNum, "part."+partNumString, "etag."+partNumString, int64(testCase.partNum+MiB)) } - if index := xlMeta.ObjectPartIndex(testCase.partNum); index != testCase.expectedIndex { + if index := objectPartIndex(xlMeta.Parts, testCase.partNum); index != testCase.expectedIndex { t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index) } } } -// Test xlMetaV1.ObjectPartIndex() +// Test objectPartIndex(). +// generates a sample xlMeta data and asserts the output of objectPartIndex() with the expected value. func TestObjectPartIndex(t *testing.T) { testCases := []struct { partNum int @@ -94,7 +95,7 @@ func TestObjectPartIndex(t *testing.T) { // Test them. for _, testCase := range testCases { - if index := xlMeta.ObjectPartIndex(testCase.partNum); index != testCase.expectedIndex { + if index := objectPartIndex(xlMeta.Parts, testCase.partNum); index != testCase.expectedIndex { t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index) } } diff --git a/cmd/xl-v1-multipart.go b/cmd/xl-v1-multipart.go index db6e3214b..1f083850d 100644 --- a/cmd/xl-v1-multipart.go +++ b/cmd/xl-v1-multipart.go @@ -512,7 +512,7 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID) - xlMeta, err := xl.readXLMetadata(minioMetaBucket, uploadIDPath) + xlParts, err := xl.readXLMetaParts(minioMetaBucket, uploadIDPath) if err != nil { return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath) } @@ -524,7 +524,7 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM result.MaxParts = maxParts // For empty number of parts or maxParts as zero, return right here. - if len(xlMeta.Parts) == 0 || maxParts == 0 { + if len(xlParts) == 0 || maxParts == 0 { return result, nil } @@ -534,10 +534,10 @@ func (xl xlObjects) listObjectParts(bucket, object, uploadID string, partNumberM } // Only parts with higher part numbers will be listed. - partIdx := xlMeta.ObjectPartIndex(partNumberMarker) - parts := xlMeta.Parts + partIdx := objectPartIndex(xlParts, partNumberMarker) + parts := xlParts if partIdx != -1 { - parts = xlMeta.Parts[partIdx+1:] + parts = xlParts[partIdx+1:] } count := maxParts for _, part := range parts { @@ -675,7 +675,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Validate each part and then commit to disk. for i, part := range parts { - partIdx := currentXLMeta.ObjectPartIndex(part.PartNumber) + partIdx := objectPartIndex(currentXLMeta.Parts, part.PartNumber) // All parts should have same part number. if partIdx == -1 { return "", traceError(InvalidPart{}) @@ -779,7 +779,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Remove parts that weren't present in CompleteMultipartUpload request. for _, curpart := range currentXLMeta.Parts { - if xlMeta.ObjectPartIndex(curpart.Number) == -1 { + if objectPartIndex(xlMeta.Parts, curpart.Number) == -1 { // Delete the missing part files. e.g, // Request 1: NewMultipart // Request 2: PutObjectPart 1 diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go index 8785442b9..08d9edb85 100644 --- a/cmd/xl-v1-object.go +++ b/cmd/xl-v1-object.go @@ -375,22 +375,23 @@ func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) { - var xlMeta xlMetaV1 - xlMeta, err = xl.readXLMetadata(bucket, object) + // returns xl meta map and stat info. + xlStat, xlMetaMap, err := xl.readXLMetaStat(bucket, object) if err != nil { // Return error. return ObjectInfo{}, err } + objInfo = ObjectInfo{ IsDir: false, Bucket: bucket, Name: object, - Size: xlMeta.Stat.Size, - ModTime: xlMeta.Stat.ModTime, - MD5Sum: xlMeta.Meta["md5Sum"], - ContentType: xlMeta.Meta["content-type"], - ContentEncoding: xlMeta.Meta["content-encoding"], - UserDefined: xlMeta.Meta, + Size: xlStat.Size, + ModTime: xlStat.ModTime, + MD5Sum: xlMetaMap["md5Sum"], + ContentType: xlMetaMap["content-type"], + ContentEncoding: xlMetaMap["content-encoding"], + UserDefined: xlMetaMap, } return objInfo, nil } diff --git a/cmd/xl-v1-utils.go b/cmd/xl-v1-utils.go index 35cc58c61..4813bbec5 100644 --- a/cmd/xl-v1-utils.go +++ b/cmd/xl-v1-utils.go @@ -17,10 +17,12 @@ package cmd import ( - "encoding/json" "hash/crc32" "path" "sync" + "time" + + "github.com/tidwall/gjson" ) // Returns number of errors that occurred the most (incl. nil) and the @@ -99,19 +101,160 @@ func hashOrder(key string, cardinality int) []int { return nums } +func parseXLStat(xlMetaBuf []byte) (statInfo, error) { + // obtain stat info. + stat := statInfo{} + // fetching modTime. + modTime, err := time.Parse(time.RFC3339, gjson.GetBytes(xlMetaBuf, "stat.modTime").String()) + if err != nil { + return statInfo{}, err + } + stat.ModTime = modTime + // obtain Stat.Size . + stat.Size = gjson.GetBytes(xlMetaBuf, "stat.size").Int() + return stat, nil +} + +func parseXLVersion(xlMetaBuf []byte) string { + return gjson.GetBytes(xlMetaBuf, "version").String() +} + +func parseXLFormat(xlMetaBuf []byte) string { + return gjson.GetBytes(xlMetaBuf, "format").String() +} + +func parseXLRelease(xlMetaBuf []byte) string { + return gjson.GetBytes(xlMetaBuf, "minio.release").String() +} + +func parseXLErasureInfo(xlMetaBuf []byte) erasureInfo { + erasure := erasureInfo{} + erasureResult := gjson.GetBytes(xlMetaBuf, "erasure") + // parse the xlV1Meta.Erasure.Distribution. + disResult := erasureResult.Get("distribution").Array() + + distribution := make([]int, len(disResult)) + for i, dis := range disResult { + distribution[i] = int(dis.Int()) + } + erasure.Distribution = distribution + + erasure.Algorithm = erasureResult.Get("algorithm").String() + erasure.DataBlocks = int(erasureResult.Get("data").Int()) + erasure.ParityBlocks = int(erasureResult.Get("parity").Int()) + erasure.BlockSize = erasureResult.Get("blockSize").Int() + erasure.Index = int(erasureResult.Get("index").Int()) + // Pare xlMetaV1.Erasure.Checksum array. + checkSumsResult := erasureResult.Get("checksum").Array() + checkSums := make([]checkSumInfo, len(checkSumsResult)) + for i, checkSumResult := range checkSumsResult { + checkSum := checkSumInfo{} + checkSum.Name = checkSumResult.Get("name").String() + checkSum.Algorithm = checkSumResult.Get("algorithm").String() + checkSum.Hash = checkSumResult.Get("hash").String() + checkSums[i] = checkSum + } + erasure.Checksum = checkSums + + return erasure +} + +func parseXLParts(xlMetaBuf []byte) []objectPartInfo { + // Parse the XL Parts. + partsResult := gjson.GetBytes(xlMetaBuf, "parts").Array() + partInfo := make([]objectPartInfo, len(partsResult)) + for i, p := range partsResult { + info := objectPartInfo{} + info.Number = int(p.Get("number").Int()) + info.Name = p.Get("name").String() + info.ETag = p.Get("etag").String() + info.Size = p.Get("size").Int() + partInfo[i] = info + } + return partInfo +} + +func parseXLMetaMap(xlMetaBuf []byte) map[string]string { + // Get xlMetaV1.Meta map. + metaMapResult := gjson.GetBytes(xlMetaBuf, "meta").Map() + metaMap := make(map[string]string) + for key, valResult := range metaMapResult { + metaMap[key] = valResult.String() + } + return metaMap +} + +// Constructs XLMetaV1 using `gjson` lib to retrieve each field. +func xlMetaV1UnmarshalJSON(xlMetaBuf []byte) (xlMetaV1, error) { + xlMeta := xlMetaV1{} + // obtain version. + xlMeta.Version = parseXLVersion(xlMetaBuf) + // obtain format. + xlMeta.Format = parseXLFormat(xlMetaBuf) + // Parse xlMetaV1.Stat . + stat, err := parseXLStat(xlMetaBuf) + if err != nil { + return xlMetaV1{}, err + } + + xlMeta.Stat = stat + // parse the xlV1Meta.Erasure fields. + xlMeta.Erasure = parseXLErasureInfo(xlMetaBuf) + + // Parse the XL Parts. + xlMeta.Parts = parseXLParts(xlMetaBuf) + // Get the xlMetaV1.Realse field. + xlMeta.Minio.Release = parseXLRelease(xlMetaBuf) + // parse xlMetaV1. + xlMeta.Meta = parseXLMetaMap(xlMetaBuf) + + return xlMeta, nil +} + +// read xl.json from the given disk, parse and return xlV1MetaV1.Parts. +func readXLMetaParts(disk StorageAPI, bucket string, object string) ([]objectPartInfo, error) { + // Reads entire `xl.json`. + xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) + if err != nil { + return nil, traceError(err) + } + // obtain xlMetaV1{}.Partsusing `github.com/tidwall/gjson`. + xlMetaParts := parseXLParts(xlMetaBuf) + + return xlMetaParts, nil +} + +// read xl.json from the given disk and parse xlV1Meta.Stat and xlV1Meta.Meta using gjson. +func readXLMetaStat(disk StorageAPI, bucket string, object string) (statInfo, map[string]string, error) { + // Reads entire `xl.json`. + xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) + if err != nil { + return statInfo{}, nil, traceError(err) + } + // obtain xlMetaV1{}.Meta using `github.com/tidwall/gjson`. + xlMetaMap := parseXLMetaMap(xlMetaBuf) + + // obtain xlMetaV1{}.Stat using `github.com/tidwall/gjson`. + xlStat, err := parseXLStat(xlMetaBuf) + if err != nil { + return statInfo{}, nil, traceError(err) + } + // Return structured `xl.json`. + return xlStat, xlMetaMap, nil +} + // readXLMeta reads `xl.json` and returns back XL metadata structure. func readXLMeta(disk StorageAPI, bucket string, object string) (xlMeta xlMetaV1, err error) { // Reads entire `xl.json`. - buf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) + xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile)) if err != nil { return xlMetaV1{}, traceError(err) } - - // Unmarshal xl metadata. - if err = json.Unmarshal(buf, &xlMeta); err != nil { + // obtain xlMetaV1{} using `github.com/tidwall/gjson`. + xlMeta, err = xlMetaV1UnmarshalJSON(xlMetaBuf) + if err != nil { return xlMetaV1{}, traceError(err) } - // Return structured `xl.json`. return xlMeta, nil } diff --git a/cmd/xl-v1-utils_test.go b/cmd/xl-v1-utils_test.go index 209a23040..422550aaa 100644 --- a/cmd/xl-v1-utils_test.go +++ b/cmd/xl-v1-utils_test.go @@ -17,8 +17,11 @@ package cmd import ( + "encoding/json" "reflect" + "strconv" "testing" + "time" ) // Test for reduceErrs, reduceErr reduces collection @@ -93,3 +96,201 @@ func TestHashOrder(t *testing.T) { t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder) } } + +// newTestXLMetaV1 - initializes new xlMetaV1, adds version, allocates a fresh erasure info and metadata. +func newTestXLMetaV1() xlMetaV1 { + xlMeta := xlMetaV1{} + xlMeta.Version = "1.0.0" + xlMeta.Format = "xl" + xlMeta.Minio.Release = "1.0.0" + xlMeta.Erasure = erasureInfo{ + Algorithm: "klauspost/reedsolomon/vandermonde", + DataBlocks: 5, + ParityBlocks: 5, + BlockSize: 10485760, + Index: 10, + Distribution: []int{9, 10, 1, 2, 3, 4, 5, 6, 7, 8}, + } + xlMeta.Stat = statInfo{ + Size: int64(20), + ModTime: time.Now().UTC(), + } + // Set meta data. + xlMeta.Meta = make(map[string]string) + xlMeta.Meta["testKey1"] = "val1" + xlMeta.Meta["testKey2"] = "val2" + return xlMeta +} + +func (m *xlMetaV1) AddTestObjectCheckSum(checkSumNum int, name string, hash string, algo string) { + checkSum := checkSumInfo{ + Name: name, + Algorithm: algo, + Hash: hash, + } + m.Erasure.Checksum[checkSumNum] = checkSum +} + +// AddTestObjectPart - add a new object part in order. +func (m *xlMetaV1) AddTestObjectPart(partNumber int, partName string, partETag string, partSize int64) { + partInfo := objectPartInfo{ + Number: partNumber, + Name: partName, + ETag: partETag, + Size: partSize, + } + + // Proceed to include new part info. + m.Parts[partNumber] = partInfo +} + +// Constructs xlMetaV1{} for given number of parts and converts it into bytes. +func getXLMetaBytes(totalParts int) []byte { + xlSampleMeta := getSampleXLMeta(totalParts) + xlMetaBytes, err := json.Marshal(xlSampleMeta) + if err != nil { + panic(err) + } + return xlMetaBytes +} + +// Returns sample xlMetaV1{} for number of parts. +func getSampleXLMeta(totalParts int) xlMetaV1 { + xlMeta := newTestXLMetaV1() + // Number of checksum info == total parts. + xlMeta.Erasure.Checksum = make([]checkSumInfo, totalParts) + // total number of parts. + xlMeta.Parts = make([]objectPartInfo, totalParts) + for i := 0; i < totalParts; i++ { + partName := "part." + strconv.Itoa(i+1) + // hard coding hash and algo value for the checksum, Since we are benchmarking the parsing of xl.json the magnitude doesn't affect the test, + // The magnitude doesn't make a difference, only the size does. + xlMeta.AddTestObjectCheckSum(i, partName, "a23f5eff248c4372badd9f3b2455a285cd4ca86c3d9a570b091d3fc5cd7ca6d9484bbea3f8c5d8d4f84daae96874419eda578fd736455334afbac2c924b3915a", "blake2b") + xlMeta.AddTestObjectPart(i, partName, "d3fdd79cc3efd5fe5c068d7be397934b", 67108864) + } + return xlMeta +} + +// Compare the unmarshaled XLMetaV1 with the one obtained from gjson parsing. +func compareXLMetaV1(t *testing.T, unMarshalXLMeta, gjsonXLMeta xlMetaV1) { + + // Start comparing the fields of xlMetaV1 obtained from gjson parsing with one parsed using json unmarshaling. + if unMarshalXLMeta.Version != gjsonXLMeta.Version { + t.Errorf("Expected the Version to be \"%s\", but got \"%s\".", unMarshalXLMeta.Version, gjsonXLMeta.Version) + } + if unMarshalXLMeta.Format != gjsonXLMeta.Format { + t.Errorf("Expected the format to be \"%s\", but got \"%s\".", unMarshalXLMeta.Format, gjsonXLMeta.Format) + } + if unMarshalXLMeta.Stat.Size != gjsonXLMeta.Stat.Size { + t.Errorf("Expected the stat size to be %v, but got %v.", unMarshalXLMeta.Stat.Size, gjsonXLMeta.Stat.Size) + } + if unMarshalXLMeta.Stat.ModTime != gjsonXLMeta.Stat.ModTime { + t.Errorf("Expected the modTime to be \"%v\", but got \"%v\".", unMarshalXLMeta.Stat.ModTime, gjsonXLMeta.Stat.ModTime) + } + if unMarshalXLMeta.Erasure.Algorithm != gjsonXLMeta.Erasure.Algorithm { + t.Errorf("Expected the erasure algorithm to be \"%v\", but got \"%v\".", unMarshalXLMeta.Erasure.Algorithm, gjsonXLMeta.Erasure.Algorithm) + } + if unMarshalXLMeta.Erasure.DataBlocks != gjsonXLMeta.Erasure.DataBlocks { + t.Errorf("Expected the erasure data blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.DataBlocks, gjsonXLMeta.Erasure.DataBlocks) + } + if unMarshalXLMeta.Erasure.ParityBlocks != gjsonXLMeta.Erasure.ParityBlocks { + t.Errorf("Expected the erasure parity blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.ParityBlocks, gjsonXLMeta.Erasure.ParityBlocks) + } + if unMarshalXLMeta.Erasure.BlockSize != gjsonXLMeta.Erasure.BlockSize { + t.Errorf("Expected the erasure block size to be %v, but got %v.", unMarshalXLMeta.Erasure.BlockSize, gjsonXLMeta.Erasure.BlockSize) + } + if unMarshalXLMeta.Erasure.Index != gjsonXLMeta.Erasure.Index { + t.Errorf("Expected the erasure index to be %v, but got %v.", unMarshalXLMeta.Erasure.Index, gjsonXLMeta.Erasure.Index) + } + if len(unMarshalXLMeta.Erasure.Distribution) != len(gjsonXLMeta.Erasure.Distribution) { + t.Errorf("Expected the size of Erasure Distribution to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Distribution), len(gjsonXLMeta.Erasure.Distribution)) + } else { + for i := 0; i < len(unMarshalXLMeta.Erasure.Distribution); i++ { + if unMarshalXLMeta.Erasure.Distribution[i] != gjsonXLMeta.Erasure.Distribution[i] { + t.Errorf("Expected the Erasure Distribution to be %d, got %d.", unMarshalXLMeta.Erasure.Distribution[i], gjsonXLMeta.Erasure.Distribution[i]) + } + } + } + + if len(unMarshalXLMeta.Erasure.Checksum) != len(gjsonXLMeta.Erasure.Checksum) { + t.Errorf("Expected the size of Erasure Checksum to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Checksum), len(gjsonXLMeta.Erasure.Checksum)) + } else { + for i := 0; i < len(unMarshalXLMeta.Erasure.Checksum); i++ { + if unMarshalXLMeta.Erasure.Checksum[i].Name != gjsonXLMeta.Erasure.Checksum[i].Name { + t.Errorf("Expected the Erasure Checksum Name to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksum[i].Name, gjsonXLMeta.Erasure.Checksum[i].Name) + } + if unMarshalXLMeta.Erasure.Checksum[i].Algorithm != gjsonXLMeta.Erasure.Checksum[i].Algorithm { + t.Errorf("Expected the Erasure Checksum Algorithm to be \"%s\", got \"%s.\"", unMarshalXLMeta.Erasure.Checksum[i].Algorithm, gjsonXLMeta.Erasure.Checksum[i].Algorithm) + } + if unMarshalXLMeta.Erasure.Checksum[i] != gjsonXLMeta.Erasure.Checksum[i] { + t.Errorf("Expected the Erasure Checksum Hash to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksum[i].Hash, gjsonXLMeta.Erasure.Checksum[i].Hash) + } + } + } + if unMarshalXLMeta.Minio.Release != gjsonXLMeta.Minio.Release { + t.Errorf("Expected the Release string to be \"%s\", but got \"%s\".", unMarshalXLMeta.Minio.Release, gjsonXLMeta.Minio.Release) + } + if len(unMarshalXLMeta.Parts) != len(gjsonXLMeta.Parts) { + t.Errorf("Expected info of %d parts to be present, but got %d instead.", len(unMarshalXLMeta.Parts), len(gjsonXLMeta.Parts)) + } else { + for i := 0; i < len(unMarshalXLMeta.Parts); i++ { + if unMarshalXLMeta.Parts[i].Name != gjsonXLMeta.Parts[i].Name { + t.Errorf("Expected the name of part %d to be \"%s\", got \"%s\".", i+1, unMarshalXLMeta.Parts[i].Name, gjsonXLMeta.Parts[i].Name) + } + if unMarshalXLMeta.Parts[i].ETag != gjsonXLMeta.Parts[i].ETag { + t.Errorf("Expected the ETag of part %d to be \"%s\", got \"%s\".", i+1, unMarshalXLMeta.Parts[i].ETag, gjsonXLMeta.Parts[i].ETag) + } + if unMarshalXLMeta.Parts[i].Number != gjsonXLMeta.Parts[i].Number { + t.Errorf("Expected the number of part %d to be \"%d\", got \"%d\".", i+1, unMarshalXLMeta.Parts[i].Number, gjsonXLMeta.Parts[i].Number) + } + if unMarshalXLMeta.Parts[i].Size != gjsonXLMeta.Parts[i].Size { + t.Errorf("Expected the size of part %d to be %v, got %v.", i+1, unMarshalXLMeta.Parts[i].Size, gjsonXLMeta.Parts[i].Size) + } + } + } + + for key, val := range unMarshalXLMeta.Meta { + gjsonVal, exists := gjsonXLMeta.Meta[key] + if !exists { + t.Errorf("No meta data entry for Key \"%s\" exists.", key) + } + if val != gjsonVal { + t.Errorf("Expected the value for Meta data key \"%s\" to be \"%s\", but got \"%s\".", key, val, gjsonVal) + } + + } +} + +// Tests the correctness of constructing XLMetaV1 using gjson lib. +// The result will be compared with the result obtained from json.unMarshal of the byte data. +func TestGetXLMetaV1GJson1(t *testing.T) { + xlMetaJSON := getXLMetaBytes(1) + + var unMarshalXLMeta xlMetaV1 + if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { + t.Errorf("Unmarshalling failed") + } + + gjsonXLMeta, err := xlMetaV1UnmarshalJSON(xlMetaJSON) + if err != nil { + t.Errorf("gjson parsing of XLMeta failed") + } + compareXLMetaV1(t, unMarshalXLMeta, gjsonXLMeta) +} + +// Tests the correctness of constructing XLMetaV1 using gjson lib for XLMetaV1 of size 10 parts. +// The result will be compared with the result obtained from json.unMarshal of the byte data. +func TestGetXLMetaV1GJson10(t *testing.T) { + + xlMetaJSON := getXLMetaBytes(10) + + var unMarshalXLMeta xlMetaV1 + if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { + t.Errorf("Unmarshalling failed") + } + gjsonXLMeta, err := xlMetaV1UnmarshalJSON(xlMetaJSON) + if err != nil { + t.Errorf("gjson parsing of XLMeta failed") + } + compareXLMetaV1(t, unMarshalXLMeta, gjsonXLMeta) +} diff --git a/vendor/github.com/tidwall/gjson/LICENSE b/vendor/github.com/tidwall/gjson/LICENSE new file mode 100644 index 000000000..58f5819a4 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md new file mode 100644 index 000000000..1ee5ae3dd --- /dev/null +++ b/vendor/github.com/tidwall/gjson/README.md @@ -0,0 +1,278 @@ +

+GJSON +
+Build Status +GoDoc +

+ +

get a json value quickly

+ +GJSON is a Go package the provides a [very fast](#performance) and simple way to get a value from a json document. The reason for this library it to give efficient json indexing for the [BuntDB](https://github.com/tidwall/buntdb) project. + +Getting Started +=============== + +## Installing + +To start using GJSON, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/gjson +``` + +This will retrieve the library. + +## Get a value +Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed and validates. Invalid json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. + +```go +package main + +import "github.com/tidwall/gjson" + +const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}` + +func main() { + value := gjson.Get(json, "name.last") + println(value.String()) +} +``` + +This will print: + +``` +Prichard +``` + +## Path Syntax + +A path is a series of keys separated by a dot. +A key may contain special wildcard characters '\*' and '?'. +To access an array value use the index as the key. +To get the number of elements in an array or to access a child path, use the '#' character. +The dot and wildcard characters can be escaped with '\'. + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "James", "last": "Murphy"}, + {"first": "Roger", "last": "Craig"} + ] +} +``` +``` +"name.last" >> "Anderson" +"age" >> 37 +"children.#" >> 3 +"children.1" >> "Alex" +"child*.2" >> "Jack" +"c?ildren.0" >> "Sara" +"fav\.movie" >> "Deer Hunter" +"friends.#.first" >> [ "James", "Roger" ] +"friends.1.last" >> "Craig" +``` +To query an array: +``` +`friends.#[last="Murphy"].first` >> "James" +``` + +## Result Type + +GJSON supports the json types `string`, `number`, `bool`, and `null`. +Arrays and Objects are returned as their raw json types. + +The `Result` type holds one of these: + +``` +bool, for JSON booleans +float64, for JSON numbers +string, for JSON string literals +nil, for JSON null +``` + +To directly access the value: + +```go +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Multi // holds nested array values +``` + +There are a variety of handy functions that work on a result: + +```go +result.Value() interface{} +result.Int() int64 +result.Float() float64 +result.String() string +result.Bool() bool +result.Array() []gjson.Result +result.Map() map[string]gjson.Result +result.Get(path string) Result +``` + +The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: + +```go +boolean >> bool +number >> float64 +string >> string +null >> nil +array >> []interface{} +object >> map[string]interface{} +``` + +## Get nested array values + +Suppose you want all the last names from the following json: + +```json +{ + "programmers": [ + { + "firstName": "Janet", + "lastName": "McLaughlin", + }, { + "firstName": "Elliotte", + "lastName": "Hunter", + }, { + "firstName": "Jason", + "lastName": "Harold", + } + ] +}` +``` + +You would use the path "programmers.#.lastName" like such: + +```go +result := gjson.Get(json, "programmers.#.lastName") +for _,name := range result.Array() { + println(name.String()) +} +``` + +You can also query an object inside an array: + +```go +name := gjson.Get(json, `programmers.#[lastName="Hunter"].firstName`) +println(name.String()) // prints "Elliotte" +``` + + +## Simple Parse and Get + +There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result. + +For example, all of these will return the same result: + +```go +gjson.Parse(json).Get("name").Get("last") +gjson.Get(json, "name").Get("last") +gjson.Get(json, "name.last") +``` + +## Check for the existence of a value + +Sometimes you just want to know you if a value exists. + +```go +value := gjson.Get(json, "name.last") +if !value.Exists() { + println("no last name") +} else { + println(value.String()) +} + +// Or as one step +if gjson.Get(json, "name.last").Exists(){ + println("has a last name") +} +``` + +## Unmarshal to a map + +To unmarshal to a `map[string]interface{}`: + +```go +m, ok := gjson.Parse(json).Value().(map[string]interface{}) +if !ok{ + // not a map +} +``` + +## Performance + +Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), +[ffjson](https://github.com/pquerna/ffjson), +[EasyJSON](https://github.com/mailru/easyjson), +and [jsonparser](https://github.com/buger/jsonparser) + +``` +BenchmarkGJSONGet-8 15000000 333 ns/op 0 B/op 0 allocs/op +BenchmarkGJSONUnmarshalMap-8 900000 4188 ns/op 1920 B/op 26 allocs/op +BenchmarkJSONUnmarshalMap-8 600000 8908 ns/op 3048 B/op 69 allocs/op +BenchmarkJSONUnmarshalStruct-8 600000 9026 ns/op 1832 B/op 69 allocs/op +BenchmarkJSONDecoder-8 300000 14339 ns/op 4224 B/op 184 allocs/op +BenchmarkFFJSONLexer-8 1500000 3156 ns/op 896 B/op 8 allocs/op +BenchmarkEasyJSONLexer-8 3000000 938 ns/op 613 B/op 6 allocs/op +BenchmarkJSONParserGet-8 3000000 442 ns/op 21 B/op 0 allocs/op +``` + +JSON document used: + +```json +{ + "widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + } +} +``` + +Each operation was rotated though one of the following search paths: + +``` +widget.window.name +widget.image.hOffset +widget.text.onMouseUp +``` + + +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* + +## Contact +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +GJSON source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go new file mode 100644 index 000000000..5ad877455 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -0,0 +1,1291 @@ +// Package gjson provides searching for json strings. +package gjson + +import ( + "reflect" + "strconv" + "unsafe" + + "github.com/tidwall/match" +) + +// Type is Result type +type Type int + +const ( + // Null is a null json value + Null Type = iota + // False is a json false boolean + False + // Number is json number + Number + // String is a json string + String + // True is a json true boolean + True + // JSON is a raw block of JSON + JSON +) + +// Result represents a json value that is returned from Get(). +type Result struct { + // Type is the json type + Type Type + // Raw is the raw json + Raw string + // Str is the json string + Str string + // Num is the json number + Num float64 +} + +// String returns a string representation of the value. +func (t Result) String() string { + switch t.Type { + default: + return "null" + case False: + return "false" + case Number: + return strconv.FormatFloat(t.Num, 'f', -1, 64) + case String: + return t.Str + case JSON: + return t.Raw + case True: + return "true" + } +} + +// Bool returns an boolean representation. +func (t Result) Bool() bool { + switch t.Type { + default: + return false + case True: + return true + case String: + return t.Str != "" && t.Str != "0" + case Number: + return t.Num != 0 + } +} + +// Int returns an integer representation. +func (t Result) Int() int64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := strconv.ParseInt(t.Str, 10, 64) + return n + case Number: + return int64(t.Num) + } +} + +// Float returns an float64 representation. +func (t Result) Float() float64 { + switch t.Type { + default: + return 0 + case True: + return 1 + case String: + n, _ := strconv.ParseFloat(t.Str, 64) + return n + case Number: + return t.Num + } +} + +// Array returns back an array of children. The result must be a JSON array. +func (t Result) Array() []Result { + if t.Type != JSON { + return nil + } + r := t.arrayOrMap('[', false) + return r.a +} + +// Map returns back an map of children. The result should be a JSON array. +func (t Result) Map() map[string]Result { + if t.Type != JSON { + return map[string]Result{} + } + r := t.arrayOrMap('{', false) + return r.o +} + +// Get searches result for the specified path. +// The result should be a JSON array or object. +func (t Result) Get(path string) Result { + return Get(t.Raw, path) +} + +type arrayOrMapResult struct { + a []Result + ai []interface{} + o map[string]Result + oi map[string]interface{} + vc byte +} + +func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { + var json = t.Raw + var i int + var value Result + var count int + var key Result + if vc == 0 { + for ; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + r.vc = json[i] + i++ + break + } + if json[i] > ' ' { + goto end + } + } + } else { + for ; i < len(json); i++ { + if json[i] == vc { + i++ + break + } + if json[i] > ' ' { + goto end + } + } + r.vc = vc + } + if r.vc == '{' { + if valueize { + r.oi = make(map[string]interface{}) + } else { + r.o = make(map[string]Result) + } + } else { + if valueize { + r.ai = make([]interface{}, 0) + } else { + r.a = make([]Result, 0) + } + } + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + // get next value + if json[i] == ']' || json[i] == '}' { + break + } + switch json[i] { + default: + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + } else { + continue + } + case '{', '[': + value.Type = JSON + value.Raw = squash(json[i:]) + case 'n': + value.Type = Null + value.Raw = tolit(json[i:]) + case 't': + value.Type = True + value.Raw = tolit(json[i:]) + case 'f': + value.Type = False + value.Raw = tolit(json[i:]) + case '"': + value.Type = String + value.Raw, value.Str = tostr(json[i:]) + } + i += len(value.Raw) - 1 + + if r.vc == '{' { + if count%2 == 0 { + key = value + } else { + if valueize { + r.oi[key.Str] = value.Value() + } else { + r.o[key.Str] = value + } + } + count++ + } else { + if valueize { + r.ai = append(r.ai, value.Value()) + } else { + r.a = append(r.a, value) + } + } + } +end: + return +} + +// Parse parses the json and returns a result +func Parse(json string) Result { + var value Result + for i := 0; i < len(json); i++ { + if json[i] == '{' || json[i] == '[' { + value.Type = JSON + value.Raw = json[i:] // just take the entire raw + break + } + if json[i] <= ' ' { + continue + } + switch json[i] { + default: + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + } else { + return Result{} + } + case 'n': + value.Type = Null + value.Raw = tolit(json[i:]) + case 't': + value.Type = True + value.Raw = tolit(json[i:]) + case 'f': + value.Type = False + value.Raw = tolit(json[i:]) + case '"': + value.Type = String + value.Raw, value.Str = tostr(json[i:]) + } + break + } + return value +} + +func squash(json string) string { + // expects that the lead character is a '[' or '{' + // squash the value, ignoring all nested arrays and objects. + // the first '[' or '{' has already been read + depth := 1 + for i := 1; i < len(json); i++ { + if json[i] >= '"' && json[i] <= '}' { + switch json[i] { + case '"': + i++ + s2 := i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + case '{', '[': + depth++ + case '}', ']': + depth-- + if depth == 0 { + return json[:i+1] + } + } + } + } + return json +} + +func tonum(json string) (raw string, num float64) { + for i := 1; i < len(json); i++ { + // less than dash might have valid characters + if json[i] <= '-' { + if json[i] <= ' ' || json[i] == ',' { + // break on whitespace and comma + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return + } + // could be a '+' or '-'. let's assume so. + continue + } + if json[i] < ']' { + // probably a valid number + continue + } + if json[i] == 'e' || json[i] == 'E' { + // allow for exponential numbers + continue + } + // likely a ']' or '}' + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return + } + raw = json + num, _ = strconv.ParseFloat(raw, 64) + return +} + +func tolit(json string) (raw string) { + for i := 1; i < len(json); i++ { + if json[i] <= 'a' || json[i] >= 'z' { + return json[:i] + } + } + return json +} + +func tostr(json string) (raw string, str string) { + // expects that the lead character is a '"' + for i := 1; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + return json[:i+1], json[1:i] + } + if json[i] == '\\' { + i++ + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + return json[:i+1], unescape(json[1:i]) + } + } + return json, json[1:] +} + +// Exists returns true if value exists. +// +// if gjson.Get(json, "name.last").Exists(){ +// println("value exists") +// } +func (t Result) Exists() bool { + return t.Type != Null || len(t.Raw) != 0 +} + +// Value returns one of these types: +// +// bool, for JSON booleans +// float64, for JSON numbers +// Number, for JSON numbers +// string, for JSON string literals +// nil, for JSON null +// +func (t Result) Value() interface{} { + if t.Type == String { + return t.Str + } + switch t.Type { + default: + return nil + case False: + return false + case Number: + return t.Num + case JSON: + r := t.arrayOrMap(0, true) + if r.vc == '{' { + return r.oi + } else if r.vc == '[' { + return r.ai + } + return nil + case True: + return true + } +} + +func parseString(json string, i int) (int, string, bool, bool) { + var s = i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + return i + 1, json[s-1 : i+1], false, true + } + if json[i] == '\\' { + i++ + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + return i + 1, json[s-1 : i+1], true, true + } + } + break + } + } + return i, json[s-1:], false, false +} + +func parseNumber(json string, i int) (int, string) { + var s = i + i++ + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ']' || json[i] == '}' { + return i, json[s:i] + } + } + return i, json[s:] +} + +func parseLiteral(json string, i int) (int, string) { + var s = i + i++ + for ; i < len(json); i++ { + if json[i] < 'a' || json[i] > 'z' { + return i, json[s:i] + } + } + return i, json[s:] +} + +type arrayPathResult struct { + part string + path string + more bool + alogok bool + arrch bool + alogkey string + query struct { + on bool + path string + op string + value string + } +} + +func parseArrayPath(path string) (r arrayPathResult) { + for i := 0; i < len(path); i++ { + if path[i] == '.' { + r.part = path[:i] + r.path = path[i+1:] + r.more = true + return + } + if path[i] == '#' { + r.arrch = true + if i == 0 && len(path) > 1 { + if path[1] == '.' { + r.alogok = true + r.alogkey = path[2:] + r.path = path[:1] + } else if path[1] == '[' { + r.query.on = true + // query + i += 2 + // whitespace + for ; i < len(path); i++ { + if path[i] > ' ' { + break + } + } + s := i + for ; i < len(path); i++ { + if path[i] <= ' ' || path[i] == '=' || + path[i] == '<' || path[i] == '>' || + path[i] == ']' { + break + } + } + r.query.path = path[s:i] + // whitespace + for ; i < len(path); i++ { + if path[i] > ' ' { + break + } + } + if i < len(path) { + s = i + if path[i] == '<' || path[i] == '>' { + if i < len(path)-1 && path[i+1] == '=' { + i++ + } + } else if path[i] == '=' { + if i < len(path)-1 && path[i+1] == '=' { + s++ + i++ + } + } + i++ + r.query.op = path[s:i] + // whitespace + for ; i < len(path); i++ { + if path[i] > ' ' { + break + } + } + s = i + for ; i < len(path); i++ { + if path[i] == '"' { + i++ + s2 := i + for ; i < len(path); i++ { + if path[i] > '\\' { + continue + } + if path[i] == '"' { + // look for an escaped slash + if path[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if path[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + } else if path[i] == ']' { + break + } + } + if i > len(path) { + i = len(path) + } + v := path[s:i] + for len(v) > 0 && v[len(v)-1] <= ' ' { + v = v[:len(v)-1] + } + r.query.value = v + } + } + } + continue + } + } + r.part = path + r.path = "" + return +} + +type objectPathResult struct { + part string + path string + wild bool + more bool +} + +func parseObjectPath(path string) (r objectPathResult) { + for i := 0; i < len(path); i++ { + if path[i] == '.' { + r.part = path[:i] + r.path = path[i+1:] + r.more = true + return + } + if path[i] == '*' || path[i] == '?' { + r.wild = true + continue + } + if path[i] == '\\' { + // go into escape mode. this is a slower path that + // strips off the escape character from the part. + epart := []byte(path[:i]) + i++ + if i < len(path) { + epart = append(epart, path[i]) + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + if i < len(path) { + epart = append(epart, path[i]) + } + continue + } else if path[i] == '.' { + r.part = string(epart) + r.path = path[i+1:] + r.more = true + return + } else if path[i] == '*' || path[i] == '?' { + r.wild = true + } + epart = append(epart, path[i]) + } + } + // append the last part + r.part = string(epart) + return + } + } + r.part = path + return +} + +func parseSquash(json string, i int) (int, string) { + // expects that the lead character is a '[' or '{' + // squash the value, ignoring all nested arrays and objects. + // the first '[' or '{' has already been read + s := i + i++ + depth := 1 + for ; i < len(json); i++ { + if json[i] >= '"' && json[i] <= '}' { + switch json[i] { + case '"': + i++ + s2 := i + for ; i < len(json); i++ { + if json[i] > '\\' { + continue + } + if json[i] == '"' { + // look for an escaped slash + if json[i-1] == '\\' { + n := 0 + for j := i - 2; j > s2-1; j-- { + if json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + break + } + } + case '{', '[': + depth++ + case '}', ']': + depth-- + if depth == 0 { + i++ + return i, json[s:i] + } + } + } + } + return i, json[s:] +} + +func parseObject(c *parseContext, i int, path string) (int, bool) { + var pmatch, kesc, vesc, ok, hit bool + var key, val string + rp := parseObjectPath(path) + for i < len(c.json) { + for ; i < len(c.json); i++ { + if c.json[i] == '"' { + // parse_key_string + // this is slightly different from getting s string value + // because we don't need the outer quotes. + i++ + var s = i + for ; i < len(c.json); i++ { + if c.json[i] > '\\' { + continue + } + if c.json[i] == '"' { + i, key, kesc, ok = i+1, c.json[s:i], false, true + goto parse_key_string_done + } + if c.json[i] == '\\' { + i++ + for ; i < len(c.json); i++ { + if c.json[i] > '\\' { + continue + } + if c.json[i] == '"' { + // look for an escaped slash + if c.json[i-1] == '\\' { + n := 0 + for j := i - 2; j > 0; j-- { + if c.json[j] != '\\' { + break + } + n++ + } + if n%2 == 0 { + continue + } + } + i, key, kesc, ok = i+1, c.json[s:i], true, true + goto parse_key_string_done + } + } + break + } + } + i, key, kesc, ok = i, c.json[s:], false, false + parse_key_string_done: + break + } + if c.json[i] == '}' { + return i + 1, false + } + } + if !ok { + return i, false + } + if rp.wild { + if kesc { + pmatch = match.Match(unescape(key), rp.part) + } else { + pmatch = match.Match(key, rp.part) + } + } else { + if kesc { + pmatch = rp.part == unescape(key) + } else { + pmatch = rp.part == key + } + } + hit = pmatch && !rp.more + for ; i < len(c.json); i++ { + switch c.json[i] { + default: + continue + case '"': + i++ + i, val, vesc, ok = parseString(c.json, i) + if !ok { + return i, false + } + if hit { + if vesc { + c.value.Str = unescape(val[1 : len(val)-1]) + } else { + c.value.Str = val[1 : len(val)-1] + } + c.value.Raw = val + c.value.Type = String + return i, true + } + case '{': + if pmatch && !hit { + i, hit = parseObject(c, i+1, rp.path) + if hit { + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '[': + if pmatch && !hit { + i, hit = parseArray(c, i+1, rp.path) + if hit { + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i, val = parseNumber(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + case 't', 'f', 'n': + vc := c.json[i] + i, val = parseLiteral(c.json, i) + if hit { + c.value.Raw = val + switch vc { + case 't': + c.value.Type = True + case 'f': + c.value.Type = False + } + return i, true + } + } + break + } + } + return i, false +} +func queryMatches(rp *arrayPathResult, value Result) bool { + rpv := rp.query.value + if len(rpv) > 2 && rpv[0] == '"' && rpv[len(rpv)-1] == '"' { + rpv = rpv[1 : len(rpv)-1] + } + switch value.Type { + case String: + switch rp.query.op { + case "=": + return value.Str == rpv + case "<": + return value.Str < rpv + case "<=": + return value.Str <= rpv + case ">": + return value.Str > rpv + case ">=": + return value.Str >= rpv + } + case Number: + rpvn, _ := strconv.ParseFloat(rpv, 64) + switch rp.query.op { + case "=": + return value.Num == rpvn + case "<": + return value.Num < rpvn + case "<=": + return value.Num <= rpvn + case ">": + return value.Num > rpvn + case ">=": + return value.Num >= rpvn + } + case True: + switch rp.query.op { + case "=": + return rpv == "true" + case ">": + return rpv == "false" + case ">=": + return true + } + case False: + switch rp.query.op { + case "=": + return rpv == "false" + case "<": + return rpv == "true" + case "<=": + return true + } + } + return false +} +func parseArray(c *parseContext, i int, path string) (int, bool) { + var pmatch, vesc, ok, hit bool + var val string + var h int + var alog []int + var partidx int + rp := parseArrayPath(path) + if !rp.arrch { + n, err := strconv.ParseUint(rp.part, 10, 64) + if err != nil { + partidx = -1 + } else { + partidx = int(n) + } + } + for i < len(c.json) { + if !rp.arrch { + pmatch = partidx == h + hit = pmatch && !rp.more + } + h++ + if rp.alogok { + alog = append(alog, i) + } + for ; i < len(c.json); i++ { + switch c.json[i] { + default: + continue + case '"': + i++ + i, val, vesc, ok = parseString(c.json, i) + if !ok { + return i, false + } + if hit { + if rp.alogok { + break + } + if vesc { + c.value.Str = unescape(val[1 : len(val)-1]) + } else { + c.value.Str = val[1 : len(val)-1] + } + c.value.Raw = val + c.value.Type = String + return i, true + } + case '{': + if pmatch && !hit { + i, hit = parseObject(c, i+1, rp.path) + if hit { + if rp.alogok { + break + } + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if rp.query.on { + res := Get(val, rp.query.path) + if queryMatches(&rp, res) { + if rp.more { + c.value = Get(val, rp.path) + } else { + c.value.Raw = val + c.value.Type = JSON + } + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '[': + if pmatch && !hit { + i, hit = parseArray(c, i+1, rp.path) + if hit { + if rp.alogok { + break + } + return i, true + } + } else { + i, val = parseSquash(c.json, i) + if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = JSON + return i, true + } + } + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i, val = parseNumber(c.json, i) + if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + case 't', 'f', 'n': + vc := c.json[i] + i, val = parseLiteral(c.json, i) + if hit { + if rp.alogok { + break + } + c.value.Raw = val + switch vc { + case 't': + c.value.Type = True + case 'f': + c.value.Type = False + } + return i, true + } + case ']': + if rp.arrch && rp.part == "#" { + if rp.alogok { + var jsons = make([]byte, 0, 64) + jsons = append(jsons, '[') + for j := 0; j < len(alog); j++ { + res := Get(c.json[alog[j]:], rp.alogkey) + if res.Exists() { + if j > 0 { + jsons = append(jsons, ',') + } + jsons = append(jsons, []byte(res.Raw)...) + } + } + jsons = append(jsons, ']') + c.value.Type = JSON + c.value.Raw = string(jsons) + return i + 1, true + } else { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num = float64(h - 1) + return i + 1, true + } + } + return i + 1, false + } + break + } + } + return i, false +} + +type parseContext struct { + json string + value Result +} + +// Get searches json for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// When the value is found it's returned immediately. +// +// A path is a series of keys seperated by a dot. +// A key may contain special wildcard characters '*' and '?'. +// To access an array value use the index as the key. +// To get the number of elements in an array or to access a child path, use the '#' character. +// The dot and wildcard character can be escaped with '\'. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children.#" >> 3 +// "children.1" >> "Alex" +// "child*.2" >> "Jack" +// "c?ildren.0" >> "Sara" +// "friends.#.first" >> [ "James", "Roger" ] +// +func Get(json, path string) Result { + var i int + var c = &parseContext{json: json} + for ; i < len(c.json); i++ { + if c.json[i] == '{' { + i++ + parseObject(c, i, path) + break + } + if c.json[i] == '[' { + i++ + parseArray(c, i, path) + break + } + } + return c.value +} + +// GetBytes searches json for the specified path. +// If working with bytes, this method preferred over Get(string(data), path) +func GetBytes(json []byte, path string) Result { + var result Result + if json != nil { + // unsafe cast to string + result = Get(*(*string)(unsafe.Pointer(&json)), path) + // copy of string data for safety. + rawh := *(*reflect.SliceHeader)(unsafe.Pointer(&result.Raw)) + strh := *(*reflect.SliceHeader)(unsafe.Pointer(&result.Str)) + if strh.Data == 0 { + if rawh.Data == 0 { + result.Raw = "" + } else { + result.Raw = string(*(*[]byte)(unsafe.Pointer(&result.Raw))) + } + result.Str = "" + } else if rawh.Data == 0 { + result.Raw = "" + result.Str = string(*(*[]byte)(unsafe.Pointer(&result.Str))) + } else if strh.Data >= rawh.Data && + int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + // Str is a substring of Raw. + start := int(strh.Data - rawh.Data) + result.Raw = string(*(*[]byte)(unsafe.Pointer(&result.Raw))) + result.Str = result.Raw[start : start+strh.Len] + } else { + result.Raw = string(*(*[]byte)(unsafe.Pointer(&result.Raw))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&result.Str))) + } + } + return result +} + +// unescape unescapes a string +func unescape(json string) string { //, error) { + var str = make([]byte, 0, len(json)) + for i := 0; i < len(json); i++ { + switch { + default: + str = append(str, json[i]) + case json[i] < ' ': + return "" //, errors.New("invalid character in string") + case json[i] == '\\': + i++ + if i >= len(json) { + return "" //, errors.New("invalid escape sequence") + } + switch json[i] { + default: + return "" //, errors.New("invalid escape sequence") + case '\\': + str = append(str, '\\') + case '/': + str = append(str, '/') + case 'b': + str = append(str, '\b') + case 'f': + str = append(str, '\f') + case 'n': + str = append(str, '\n') + case 'r': + str = append(str, '\r') + case 't': + str = append(str, '\t') + case '"': + str = append(str, '"') + case 'u': + if i+5 > len(json) { + return "" //, errors.New("invalid escape sequence") + } + i++ + // extract the codepoint + var code int + for j := i; j < i+4; j++ { + switch { + default: + return "" //, errors.New("invalid escape sequence") + case json[j] >= '0' && json[j] <= '9': + code += (int(json[j]) - '0') << uint(12-(j-i)*4) + case json[j] >= 'a' && json[j] <= 'f': + code += (int(json[j]) - 'a' + 10) << uint(12-(j-i)*4) + case json[j] >= 'a' && json[j] <= 'f': + code += (int(json[j]) - 'a' + 10) << uint(12-(j-i)*4) + } + } + str = append(str, []byte(string(code))...) + i += 3 // only 3 because we will increment on the for-loop + } + } + } + return string(str) //, nil +} + +// Less return true if a token is less than another token. +// The caseSensitive paramater is used when the tokens are Strings. +// The order when comparing two different type is: +// +// Null < False < Number < String < True < JSON +// +func (t Result) Less(token Result, caseSensitive bool) bool { + if t.Type < token.Type { + return true + } + if t.Type > token.Type { + return false + } + if t.Type == String { + if caseSensitive { + return t.Str < token.Str + } + return stringLessInsensitive(t.Str, token.Str) + } + if t.Type == Number { + return t.Num < token.Num + } + return t.Raw < token.Raw +} + +func stringLessInsensitive(a, b string) bool { + for i := 0; i < len(a) && i < len(b); i++ { + if a[i] >= 'A' && a[i] <= 'Z' { + if b[i] >= 'A' && b[i] <= 'Z' { + // both are uppercase, do nothing + if a[i] < b[i] { + return true + } else if a[i] > b[i] { + return false + } + } else { + // a is uppercase, convert a to lowercase + if a[i]+32 < b[i] { + return true + } else if a[i]+32 > b[i] { + return false + } + } + } else if b[i] >= 'A' && b[i] <= 'Z' { + // b is uppercase, convert b to lowercase + if a[i] < b[i]+32 { + return true + } else if a[i] > b[i]+32 { + return false + } + } else { + // neither are uppercase + if a[i] < b[i] { + return true + } else if a[i] > b[i] { + return false + } + } + } + return len(a) < len(b) +} diff --git a/vendor/github.com/tidwall/gjson/logo.png b/vendor/github.com/tidwall/gjson/logo.png new file mode 100644 index 000000000..17a8bbe9d Binary files /dev/null and b/vendor/github.com/tidwall/gjson/logo.png differ diff --git a/vendor/github.com/tidwall/match/LICENSE b/vendor/github.com/tidwall/match/LICENSE new file mode 100644 index 000000000..58f5819a4 --- /dev/null +++ b/vendor/github.com/tidwall/match/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/match/README.md b/vendor/github.com/tidwall/match/README.md new file mode 100644 index 000000000..04b0aaa9d --- /dev/null +++ b/vendor/github.com/tidwall/match/README.md @@ -0,0 +1,31 @@ +Match +===== +Build Status +GoDoc + +Match is a very simple pattern matcher where '*' matches on any +number characters and '?' matches on any one character. +Installing +---------- + +``` +go get -u github.com/tidwall/match +``` + +Example +------- + +```go +match.Match("hello", "*llo") +match.Match("jello", "?ello") +match.Match("hello", "h*o") +``` + + +Contact +------- +Josh Baker [@tidwall](http://twitter.com/tidwall) + +License +------- +Redcon source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go new file mode 100644 index 000000000..8885add63 --- /dev/null +++ b/vendor/github.com/tidwall/match/match.go @@ -0,0 +1,192 @@ +// Match provides a simple pattern matcher with unicode support. +package match + +import "unicode/utf8" + +// Match returns true if str matches pattern. This is a very +// simple wildcard match where '*' matches on any number characters +// and '?' matches on any one character. + +// pattern: +// { term } +// term: +// '*' matches any sequence of non-Separator characters +// '?' matches any single non-Separator character +// c matches character c (c != '*', '?', '\\') +// '\\' c matches character c +// +func Match(str, pattern string) bool { + if pattern == "*" { + return true + } + return deepMatch(str, pattern) +} +func deepMatch(str, pattern string) bool { + for len(pattern) > 0 { + if pattern[0] > 0x7f { + return deepMatchRune(str, pattern) + } + switch pattern[0] { + default: + if len(str) == 0 { + return false + } + if str[0] > 0x7f { + return deepMatchRune(str, pattern) + } + if str[0] != pattern[0] { + return false + } + case '?': + if len(str) == 0 { + return false + } + case '*': + return deepMatch(str, pattern[1:]) || + (len(str) > 0 && deepMatch(str[1:], pattern)) + } + str = str[1:] + pattern = pattern[1:] + } + return len(str) == 0 && len(pattern) == 0 +} + +func deepMatchRune(str, pattern string) bool { + var sr, pr rune + var srsz, prsz int + + // read the first rune ahead of time + if len(str) > 0 { + if str[0] > 0x7f { + sr, srsz = utf8.DecodeRuneInString(str) + } else { + sr, srsz = rune(str[0]), 1 + } + } else { + sr, srsz = utf8.RuneError, 0 + } + if len(pattern) > 0 { + if pattern[0] > 0x7f { + pr, prsz = utf8.DecodeRuneInString(pattern) + } else { + pr, prsz = rune(pattern[0]), 1 + } + } else { + pr, prsz = utf8.RuneError, 0 + } + // done reading + for pr != utf8.RuneError { + switch pr { + default: + if srsz == utf8.RuneError { + return false + } + if sr != pr { + return false + } + case '?': + if srsz == utf8.RuneError { + return false + } + case '*': + return deepMatchRune(str, pattern[prsz:]) || + (srsz > 0 && deepMatchRune(str[srsz:], pattern)) + } + str = str[srsz:] + pattern = pattern[prsz:] + // read the next runes + if len(str) > 0 { + if str[0] > 0x7f { + sr, srsz = utf8.DecodeRuneInString(str) + } else { + sr, srsz = rune(str[0]), 1 + } + } else { + sr, srsz = utf8.RuneError, 0 + } + if len(pattern) > 0 { + if pattern[0] > 0x7f { + pr, prsz = utf8.DecodeRuneInString(pattern) + } else { + pr, prsz = rune(pattern[0]), 1 + } + } else { + pr, prsz = utf8.RuneError, 0 + } + // done reading + } + + return srsz == 0 && prsz == 0 +} + +var maxRuneBytes = func() []byte { + b := make([]byte, 4) + if utf8.EncodeRune(b, '\U0010FFFF') != 4 { + panic("invalid rune encoding") + } + return b +}() + +// Allowable parses the pattern and determines the minimum and maximum allowable +// values that the pattern can represent. +// When the max cannot be determined, 'true' will be returned +// for infinite. +func Allowable(pattern string) (min, max string) { + if pattern == "" || pattern[0] == '*' { + return "", "" + } + + minb := make([]byte, 0, len(pattern)) + maxb := make([]byte, 0, len(pattern)) + var wild bool + for i := 0; i < len(pattern); i++ { + if pattern[i] == '*' { + wild = true + break + } + if pattern[i] == '?' { + minb = append(minb, 0) + maxb = append(maxb, maxRuneBytes...) + } else { + minb = append(minb, pattern[i]) + maxb = append(maxb, pattern[i]) + } + } + if wild { + r, n := utf8.DecodeLastRune(maxb) + if r != utf8.RuneError { + if r < utf8.MaxRune { + r++ + if r > 0x7f { + b := make([]byte, 4) + nn := utf8.EncodeRune(b, r) + maxb = append(maxb[:len(maxb)-n], b[:nn]...) + } else { + maxb = append(maxb[:len(maxb)-n], byte(r)) + } + } + } + } + return string(minb), string(maxb) + /* + return + if wild { + r, n := utf8.DecodeLastRune(maxb) + if r != utf8.RuneError { + if r < utf8.MaxRune { + infinite = true + } else { + r++ + if r > 0x7f { + b := make([]byte, 4) + nn := utf8.EncodeRune(b, r) + maxb = append(maxb[:len(maxb)-n], b[:nn]...) + } else { + maxb = append(maxb[:len(maxb)-n], byte(r)) + } + } + } + } + return string(minb), string(maxb), infinite + */ +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 1ad37a23e..0d9b10232 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -169,6 +169,18 @@ "revision": "2e25825abdbd7752ff08b270d313b93519a0a232", "revisionTime": "2016-03-11T21:55:03Z" }, + { + "checksumSHA1": "+Pcohsuq0Mi/y8bgaDFjb/CGzkk=", + "path": "github.com/tidwall/gjson", + "revision": "7c631e98686a791e5fc60ff099512968122afb52", + "revisionTime": "2016-09-08T16:02:40Z" + }, + { + "checksumSHA1": "qmePMXEDYGwkAfT9QvtMC58JN/E=", + "path": "github.com/tidwall/match", + "revision": "173748da739a410c5b0b813b956f89ff94730b4c", + "revisionTime": "2016-08-30T17:39:30Z" + }, { "path": "golang.org/x/crypto/bcrypt", "revision": "7b85b097bf7527677d54d3220065e966a0e3b613",