From 80a351633f96b54b1e02b1254eaa7afa995d21e9 Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Wed, 20 Feb 2019 23:59:23 -0800 Subject: [PATCH] Update vendorized bcicen/jstream (#7257) - Includes an error handling fix that is waiting to be merged upstream - Uses order-preserving (un)marshalling for JSON objects. --- pkg/s3select/json/reader.go | 2 +- vendor/github.com/bcicen/jstream/decoder.go | 110 +++++++++++++++++++- vendor/vendor.json | 6 +- 3 files changed, 110 insertions(+), 8 deletions(-) diff --git a/pkg/s3select/json/reader.go b/pkg/s3select/json/reader.go index 06971e2ab..e9d4d1f45 100644 --- a/pkg/s3select/json/reader.go +++ b/pkg/s3select/json/reader.go @@ -71,7 +71,7 @@ func (r *Reader) Close() error { // NewReader - creates new JSON reader using readCloser. func NewReader(readCloser io.ReadCloser, args *ReaderArgs) *Reader { - d := jstream.NewDecoder(readCloser, 0) + d := jstream.NewDecoder(readCloser, 0).ObjectAsKVS() return &Reader{ args: args, decoder: d, diff --git a/vendor/github.com/bcicen/jstream/decoder.go b/vendor/github.com/bcicen/jstream/decoder.go index 1c596e22c..22efe1472 100644 --- a/vendor/github.com/bcicen/jstream/decoder.go +++ b/vendor/github.com/bcicen/jstream/decoder.go @@ -42,8 +42,8 @@ type KV struct { // KVS - represents key values in an JSON object type KVS []KV -// MarshalJSON - implements converting slice of KVS into a JSON object -// with multiple keys and values. +// MarshalJSON - implements converting a KVS datastructure into a JSON +// object with multiple keys and values. func (kvs KVS) MarshalJSON() ([]byte, error) { b := new(bytes.Buffer) b.Write([]byte("{")) @@ -69,6 +69,7 @@ type Decoder struct { emitDepth int emitKV bool emitRecursive bool + objectAsKVS bool depth int scratch *scratch @@ -97,6 +98,15 @@ func NewDecoder(r io.Reader, emitDepth int) *Decoder { return d } +// ObjectAsKVS - by default JSON returns map[string]interface{} this +// is usually fine in most cases, but when you need to preserve the +// input order its not a right data structure. To preserve input +// order please use this option. +func (d *Decoder) ObjectAsKVS() *Decoder { + d.objectAsKVS = true + return d +} + // EmitKV enables emitting a jstream.KV struct when the items(s) parsed // at configured emit depth are within a JSON object. By default, only // the object values are emitted. @@ -217,7 +227,13 @@ func (d *Decoder) any() (interface{}, ValueType, error) { i, err := d.array() return i, Array, err case '{': - i, err := d.object() + var i interface{} + var err error + if d.objectAsKVS { + i, err = d.objectOrdered() + } else { + i, err = d.object() + } return i, Object, err default: return nil, Unknown, d.mkError(ErrSyntax, "looking for beginning of value") @@ -442,7 +458,92 @@ out: } // object accept valid JSON array value -func (d *Decoder) object() (KVS, error) { +func (d *Decoder) object() (map[string]interface{}, error) { + d.depth++ + + var ( + c byte + k string + v interface{} + t ValueType + err error + obj map[string]interface{} + ) + + // skip allocating map if it will not be emitted + if d.depth > d.emitDepth { + obj = make(map[string]interface{}) + } + + // if the object has no keys + if c = d.skipSpaces(); c == '}' { + goto out + } + +scan: + for { + offset := d.pos - 1 + + // read string key + if c != '"' { + err = d.mkError(ErrSyntax, "looking for beginning of object key string") + break + } + if k, err = d.string(); err != nil { + break + } + + // read colon before value + if c = d.skipSpaces(); c != ':' { + err = d.mkError(ErrSyntax, "after object key") + break + } + + // read value + d.skipSpaces() + if d.emitKV { + if v, t, err = d.any(); err != nil { + break + } + if d.willEmit() { + d.metaCh <- &MetaValue{ + Offset: int(offset), + Length: int(d.pos - offset), + Depth: d.depth, + Value: KV{k, v}, + ValueType: t, + } + } + } else { + if v, err = d.emitAny(); err != nil { + break + } + } + + if obj != nil { + obj[k] = v + } + + // next token must be ',' or '}' + switch c = d.skipSpaces(); c { + case '}': + goto out + case ',': + c = d.skipSpaces() + goto scan + default: + err = d.mkError(ErrSyntax, "after object key:value pair") + goto out + } + } + +out: + d.depth-- + return obj, err +} + +// object (ordered) accept valid JSON array value +func (d *Decoder) objectOrdered() (KVS, error) { d.depth++ var ( @@ -517,6 +618,7 @@ scan: goto scan default: err = d.mkError(ErrSyntax, "after object key:value pair") + goto out } } diff --git a/vendor/vendor.json b/vendor/vendor.json index 2c9c85175..3134901fe 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -116,10 +116,10 @@ "revisionTime": "2017-09-25T03:23:15Z" }, { - "checksumSHA1": "mSo9Sti7F6+laUs4NLx/p23rHD0=", + "checksumSHA1": "YPGDETCyuMkceQvAXafFnj6ndkY=", "path": "github.com/bcicen/jstream", - "revision": "f306cd3e1fa602a1b513114521a0d6a5a9d5c919", - "revisionTime": "2019-02-06T02:23:53Z" + "revision": "16c1f8af81c2a9967b30d365a29472126274f998", + "revisionTime": "2019-02-20T04:59:26Z" }, { "checksumSHA1": "0rido7hYHQtfq3UJzVT5LClLAWc=",