diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 7c11face2..5111dcde3 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -5,6 +5,14 @@ "./..." ], "Deps": [ + { + "ImportPath": "github.com/clbanning/mxj", + "Rev": "e11b85050263aff26728fb9863bf2ebaf6591279" + }, + { + "ImportPath": "github.com/fatih/structs", + "Rev": "c00d27128bb88e9c1adab1a53cda9c72c6d1ff9b" + }, { "ImportPath": "github.com/gorilla/context", "Rev": "50c25fb3b2b3b3cc724e9b6ac75fb44b3bccd0da" diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/LICENSE b/Godeps/_workspace/src/github.com/clbanning/mxj/LICENSE new file mode 100644 index 000000000..67563f958 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/LICENSE @@ -0,0 +1,55 @@ +Copyright (c) 2012-2014 Charles Banning . All rights reserved. + +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== + +Go Language Copyright & License - + +Copyright 2009 The Go Authors. All rights reserved. +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/anyxml.go b/Godeps/_workspace/src/github.com/clbanning/mxj/anyxml.go new file mode 100644 index 000000000..4eb6001d1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/anyxml.go @@ -0,0 +1,178 @@ +package mxj + +import ( + "encoding/xml" + "reflect" +) + +const ( + DefaultElementTag = "element" +) + +// Encode arbitrary value as XML. +// +// Note: unmarshaling the resultant +// XML may not return the original value, since tag labels may have been injected +// to create the XML representation of the value. +/* + Encode an arbitrary JSON object. + package main + + import ( + "encoding/json" + "fmt" + "github/clbanning/mxj" + ) + + func main() { + jsondata := []byte(`[ + { "somekey":"somevalue" }, + "string", + 3.14159265, + true + ]`) + var i interface{} + err := json.Unmarshal(jsondata, &i) + if err != nil { + // do something + } + x, err := anyxml.XmlIndent(i, "", " ", "mydoc") + if err != nil { + // do something else + } + fmt.Println(string(x)) + } + + output: + + somevalue + string + 3.14159265 + true + +*/ +// Alternative values for DefaultRootTag and DefaultElementTag can be set as: +// AnyXmlIndent( v, myRootTag, myElementTag). +func AnyXml(v interface{}, tags ...string) ([]byte, error) { + if reflect.TypeOf(v).Kind() == reflect.Struct { + return xml.Marshal(v) + } + + var err error + s := new(string) + p := new(pretty) + + var rt, et string + if len(tags) == 1 || len(tags) == 2 { + rt = tags[0] + } else { + rt = DefaultRootTag + } + if len(tags) == 2 { + et = tags[1] + } else { + et = DefaultElementTag + } + + var ss string + var b []byte + switch v.(type) { + case []interface{}: + ss = "<" + rt + ">" + for _, vv := range v.([]interface{}) { + switch vv.(type) { + case map[string]interface{}: + m := vv.(map[string]interface{}) + if len(m) == 1 { + for tag, val := range m { + err = mapToXmlIndent(false, s, tag, val, p) + } + } else { + err = mapToXmlIndent(false, s, et, vv, p) + } + default: + err = mapToXmlIndent(false, s, et, vv, p) + } + if err != nil { + break + } + } + ss += *s + "" + b = []byte(ss) + case map[string]interface{}: + m := Map(v.(map[string]interface{})) + b, err = m.Xml(rt) + default: + err = mapToXmlIndent(false, s, rt, v, p) + b = []byte(*s) + } + + return b, err +} + + +// Encode an arbitrary value as a pretty XML string. +// Alternative values for DefaultRootTag and DefaultElementTag can be set as: +// AnyXmlIndent( v, "", " ", myRootTag, myElementTag). +func AnyXmlIndent(v interface{}, prefix, indent string, tags ...string) ([]byte, error) { + if reflect.TypeOf(v).Kind() == reflect.Struct { + return xml.MarshalIndent(v, prefix, indent) + } + + var err error + s := new(string) + p := new(pretty) + p.indent = indent + p.padding = prefix + + var rt, et string + if len(tags) == 1 || len(tags) == 2 { + rt = tags[0] + } else { + rt = DefaultRootTag + } + if len(tags) == 2 { + et = tags[1] + } else { + et = DefaultElementTag + } + + var ss string + var b []byte + switch v.(type) { + case []interface{}: + ss = "<" + rt + ">\n" + p.Indent() + for _, vv := range v.([]interface{}) { + switch vv.(type) { + case map[string]interface{}: + m := vv.(map[string]interface{}) + if len(m) == 1 { + for tag, val := range m { + err = mapToXmlIndent(true, s, tag, val, p) + } + } else { + p.start = 1 // we 1 tag in + err = mapToXmlIndent(true, s, et, vv, p) + *s += "\n" + } + default: + p.start = 0 // in case trailing p.start = 1 + err = mapToXmlIndent(true, s, et, vv, p) + } + if err != nil { + break + } + } + ss += *s + "" + b = []byte(ss) + case map[string]interface{}: + m := Map(v.(map[string]interface{})) + b, err = m.XmlIndent(prefix, indent, rt) + default: + err = mapToXmlIndent(true, s, rt, v, p) + b = []byte(*s) + } + + return b, err +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/anyxml_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/anyxml_test.go new file mode 100644 index 000000000..f4c80a513 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/anyxml_test.go @@ -0,0 +1,110 @@ +package mxj + +import ( + "encoding/json" + "fmt" + "testing" +) + +func TestAnyXmlHeader(t *testing.T) { + fmt.Println("\n---------------- anyxml_test.go ...\n") +} + +var anydata = []byte(`[ + { + "somekey": "somevalue" + }, + { + "somekey": "somevalue" + }, + { + "somekey": "somevalue", + "someotherkey": "someothervalue" + }, + "string", + 3.14159265, + true +]`) + +type MyStruct struct { + Somekey string `xml:"somekey"` + B float32 `xml:"floatval"` +} + +func TestAnyXml(t *testing.T) { + var i interface{} + err := json.Unmarshal(anydata, &i) + if err != nil { + t.Fatal(err) + } + x, err := AnyXml(i) + if err != nil { + t.Fatal(err) + } + fmt.Println("[]->x:", string(x)) + + a := []interface{}{ "try", "this", 3.14159265, true } + x, err = AnyXml(a) + if err != nil { + t.Fatal(err) + } + fmt.Println("a->x:", string(x)) + + x, err = AnyXml(a, "myRootTag", "myElementTag") + if err != nil { + t.Fatal(err) + } + fmt.Println("a->x:", string(x)) + + x, err = AnyXml(3.14159625) + if err != nil { + t.Fatal(err) + } + fmt.Println("f->x:", string(x)) + + s := MyStruct{"somevalue", 3.14159625} + x, err = AnyXml(s) + if err != nil { + t.Fatal(err) + } + fmt.Println("s->x:", string(x)) +} + +func TestAnyXmlIndent(t *testing.T) { + var i interface{} + err := json.Unmarshal(anydata, &i) + if err != nil { + t.Fatal(err) + } + x, err := AnyXmlIndent(i, "", " ") + if err != nil { + t.Fatal(err) + } + fmt.Println("[]->x:\n", string(x)) + + a := []interface{}{ "try", "this", 3.14159265, true } + x, err = AnyXmlIndent(a, "", " ") + if err != nil { + t.Fatal(err) + } + fmt.Println("a->x:\n", string(x)) + + x, err = AnyXmlIndent(3.14159625, "", " ") + if err != nil { + t.Fatal(err) + } + fmt.Println("f->x:\n", string(x)) + + x, err = AnyXmlIndent(3.14159625, "", " ", "myRootTag", "myElementTag") + if err != nil { + t.Fatal(err) + } + fmt.Println("f->x:\n", string(x)) + + s := MyStruct{"somevalue", 3.14159625} + x, err = AnyXmlIndent(s, "", " ") + if err != nil { + t.Fatal(err) + } + fmt.Println("s->x:\n", string(x)) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/bulk_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/bulk_test.go new file mode 100644 index 000000000..bb64f9c00 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/bulk_test.go @@ -0,0 +1,107 @@ +// bulk_test.go - uses Handler and Writer functions to process some streams as a demo. + +package mxj + +import ( + "bytes" + "fmt" + "testing" +) + +func TestBulkHeader(t *testing.T) { + fmt.Println("\n---------------- bulk_test.go ...\n") +} + +var jsonWriter = new(bytes.Buffer) +var xmlWriter = new(bytes.Buffer) + +var jsonErrLog = new(bytes.Buffer) +var xmlErrLog = new(bytes.Buffer) + +func TestXmlReader(t *testing.T) { + // create Reader for xmldata + xmlReader := bytes.NewReader(xmldata) + + // read XML from Readerand pass Map value with the raw XML to handler + err := HandleXmlReader(xmlReader, bxmaphandler, bxerrhandler) + if err != nil { + t.Fatal("err:", err.Error()) + } + + // get the JSON + j := make([]byte, jsonWriter.Len()) + _, _ = jsonWriter.Read(j) + + // get the errors + e := make([]byte, xmlErrLog.Len()) + _, _ = xmlErrLog.Read(e) + + // print the input + fmt.Println("XmlReader, xmldata:\n", string(xmldata)) + // print the result + fmt.Println("XmlReader, result :\n", string(j)) + // print the errors + fmt.Println("XmlReader, errors :\n", string(e)) +} + +func bxmaphandler(m Map) bool { + j, err := m.JsonIndent("", " ", true) + if err != nil { + return false + } + + _, _ = jsonWriter.Write(j) + // put in a NL to pretty up printing the Writer + _, _ = jsonWriter.Write([]byte("\n")) + return true +} + +func bxerrhandler(err error) bool { + // write errors to file + _, _ = xmlErrLog.Write([]byte(err.Error())) + _, _ = xmlErrLog.Write([]byte("\n")) // pretty up + return true +} + +func TestJsonReader(t *testing.T) { + jsonReader := bytes.NewReader(jsondata) + + // read all the JSON + err := HandleJsonReader(jsonReader, bjmaphandler, bjerrhandler) + if err != nil { + t.Fatal("err:", err.Error()) + } + + // get the XML + x := make([]byte, xmlWriter.Len()) + _, _ = xmlWriter.Read(x) + + // get the errors + e := make([]byte, jsonErrLog.Len()) + _, _ = jsonErrLog.Read(e) + + // print the input + fmt.Println("JsonReader, jsondata:\n", string(jsondata)) + // print the result + fmt.Println("JsonReader, result :\n", string(x)) + // print the errors + fmt.Println("JsonReader, errors :\n", string(e)) +} + +func bjmaphandler(m Map) bool { + x, err := m.XmlIndent(" ", " ") + if err != nil { + return false + } + _, _ = xmlWriter.Write(x) + // put in a NL to pretty up printing the Writer + _, _ = xmlWriter.Write([]byte("\n")) + return true +} + +func bjerrhandler(err error) bool { + // write errors to file + _, _ = jsonErrLog.Write([]byte(err.Error())) + _, _ = jsonErrLog.Write([]byte("\n")) // pretty up + return true +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/bulkraw_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/bulkraw_test.go new file mode 100644 index 000000000..61c61fd5f --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/bulkraw_test.go @@ -0,0 +1,113 @@ +// bulk_test.go - uses Handler and Writer functions to process some streams as a demo. + +package mxj + +import ( + "bytes" + "fmt" + "testing" +) + +func TestBulkRawHeader(t *testing.T) { + fmt.Println("\n---------------- bulkraw_test.go ...\n") +} + +// use data from bulk_test.go + +var jsonWriterRaw = new(bytes.Buffer) +var xmlWriterRaw = new(bytes.Buffer) + +var jsonErrLogRaw = new(bytes.Buffer) +var xmlErrLogRaw = new(bytes.Buffer) + +func TestXmlReaderRaw(t *testing.T) { + // create Reader for xmldata + xmlReader := bytes.NewReader(xmldata) + + // read XML from Reader and pass Map value with the raw XML to handler + err := HandleXmlReaderRaw(xmlReader, bxmaphandlerRaw, bxerrhandlerRaw) + if err != nil { + t.Fatal("err:", err.Error()) + } + + // get the JSON + j := make([]byte, jsonWriterRaw.Len()) + _, _ = jsonWriterRaw.Read(j) + + // get the errors + e := make([]byte, xmlErrLogRaw.Len()) + _, _ = xmlErrLogRaw.Read(e) + + // print the input + fmt.Println("XmlReaderRaw, xmldata:\n", string(xmldata)) + // print the result + fmt.Println("XmlReaderRaw, result :\n", string(j)) + // print the errors + fmt.Println("XmlReaderRaw, errors :\n", string(e)) +} + +func bxmaphandlerRaw(m Map, raw []byte) bool { + j, err := m.JsonIndent("", " ", true) + if err != nil { + return false + } + + _, _ = jsonWriterRaw.Write(j) + // put in a NL to pretty up printing the Writer + _, _ = jsonWriterRaw.Write([]byte("\n")) + return true +} + +func bxerrhandlerRaw(err error, raw []byte) bool { + // write errors to file + _, _ = xmlErrLogRaw.Write([]byte(err.Error())) + _, _ = xmlErrLogRaw.Write([]byte("\n")) // pretty up + _, _ = xmlErrLogRaw.Write(raw) + _, _ = xmlErrLogRaw.Write([]byte("\n")) // pretty up + return true +} + +func TestJsonReaderRaw(t *testing.T) { + jsonReader := bytes.NewReader(jsondata) + + // read all the JSON + err := HandleJsonReaderRaw(jsonReader, bjmaphandlerRaw, bjerrhandlerRaw) + if err != nil { + t.Fatal("err:", err.Error()) + } + + // get the XML + x := make([]byte, xmlWriterRaw.Len()) + _, _ = xmlWriterRaw.Read(x) + + // get the errors + e := make([]byte, jsonErrLogRaw.Len()) + _, _ = jsonErrLogRaw.Read(e) + + // print the input + fmt.Println("JsonReaderRaw, jsondata:\n", string(jsondata)) + // print the result + fmt.Println("JsonReaderRaw, result :\n", string(x)) + // print the errors + fmt.Println("JsonReaderRaw, errors :\n", string(e)) +} + +func bjmaphandlerRaw(m Map, raw []byte) bool { + x, err := m.XmlIndent(" ", " ") + if err != nil { + return false + } + _, _ = xmlWriterRaw.Write(x) + // put in a NL to pretty up printing the Writer + _, _ = xmlWriterRaw.Write([]byte("\n")) + return true +} + +func bjerrhandlerRaw(err error, raw []byte) bool { + // write errors to file + _, _ = jsonErrLogRaw.Write([]byte(err.Error())) + _, _ = jsonErrLogRaw.Write([]byte("\n")) // pretty up, Error() from json.Unmarshal !NL + _, _ = jsonErrLogRaw.Write(raw) + _, _ = jsonErrLogRaw.Write([]byte("\n")) // pretty up + return true +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/data_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/data_test.go new file mode 100644 index 000000000..b3a4aaec9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/data_test.go @@ -0,0 +1,39 @@ +package mxj + +var xmldata = []byte(` + + William H. Gaddis + The Recognitions + One of the seminal American novels of the 20th century. + + + William H. Gaddis + JR + Won the National Book Award. + + + Austin Tappan Wright + Islandia + An example of earlier 20th century American utopian fiction. + + + John Hawkes + The Beetle Leg + A lyrical novel about the construction of Ft. Peck Dam in Montana. + + + + T.E. + Porter + + King's Day + A magical novella. +`) + +var jsondata = []byte(` + {"book":{"author":"William H. Gaddis","review":"One of the great seminal American novels of the 20th century.","title":"The Recognitions"}} +{"book":{"author":"Austin Tappan Wright","review":"An example of earlier 20th century American utopian fiction.","title":"Islandia"}} +{"book":{"author":"John Hawkes","review":"A lyrical novel about the construction of Ft. Peck Dam in Montana.","title":"The Beetle Leg"}} +{"book":{"author":{"first_name":"T.E.","last_name":"Porter"},"review":"A magical novella.","title":"King's Day"}} +{ "here":"we", "put":"in", "an":error }`) + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/doc.go b/Godeps/_workspace/src/github.com/clbanning/mxj/doc.go new file mode 100644 index 000000000..cb9813106 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/doc.go @@ -0,0 +1,84 @@ +// mxj - A collection of map[string]interface{} and associated XML and JSON utilities. +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +/* +Marshal/Unmarshal XML to/from JSON and map[string]interface{} values, and extract/modify values from maps by key or key-path, including wildcards. + +mxj supplants the legacy x2j and j2x packages. If you want the old syntax, use mxj/x2j or mxj/j2x packages. + +Note: this library was designed for processing ad hoc anonymous messages. Bulk processing large data sets may be much more efficiently performed using the encoding/xml or encoding/json packages from Go's standard library directly. + +Note: + 2014-08-02: AnyXml() and AnyXmlIndent() will try to marshal arbitrary values to XML. + +SUMMARY + + type Map map[string]interface{} + + Create a Map value, 'm', from any map[string]interface{} value, 'v': + m := Map(v) + + Unmarshal / marshal XML as a Map value, 'm': + m, err := NewMapXml(xmlValue) // unmarshal + xmlValue, err := m.Xml() // marshal + + Unmarshal XML from an io.Reader as a Map value, 'm': + m, err := NewMapReader(xmlReader) // repeated calls, as with an os.File Reader, will process stream + m, raw, err := NewMapReaderRaw(xmlReader) // 'raw' is the raw XML that was decoded + + Marshal Map value, 'm', to an XML Writer (io.Writer): + err := m.XmlWriter(xmlWriter) + raw, err := m.XmlWriterRaw(xmlWriter) // 'raw' is the raw XML that was written on xmlWriter + + Also, for prettified output: + xmlValue, err := m.XmlIndent(prefix, indent, ...) + err := m.XmlIndentWriter(xmlWriter, prefix, indent, ...) + raw, err := m.XmlIndentWriterRaw(xmlWriter, prefix, indent, ...) + + Bulk process XML with error handling (note: handlers must return a boolean value): + err := HandleXmlReader(xmlReader, mapHandler(Map), errHandler(error)) + err := HandleXmlReaderRaw(xmlReader, mapHandler(Map, []byte), errHandler(error, []byte)) + + Converting XML to JSON: see Examples for NewMapXml and HandleXmlReader. + + There are comparable functions and methods for JSON processing. + + Arbitrary structure values can be decoded to / encoded from Map values: + m, err := NewMapStruct(structVal) + err := m.Struct(structPointer) + + To work with XML tag values, JSON or Map key values or structure field values, decode the XML, JSON + or structure to a Map value, 'm', or cast a map[string]interface{} value to a Map value, 'm', then: + paths := m.PathsForKey(key) + path := m.PathForKeyShortest(key) + values, err := m.ValuesForKey(key, subkeys) + values, err := m.ValuesForPath(path, subkeys) // 'path' can be dot-notation with wildcards and indexed arrays. + count, err := m.UpdateValuesForPath(newVal, path, subkeys) + + Get everything at once, irrespective of path depth: + leafnodes := m.LeafNodes() + leafvalues := m.LeafValues() + + A new Map with whatever keys are desired can be created from the current Map and then encoded in XML + or JSON. (Note: keys can use dot-notation. 'oldKey' can also use wildcards and indexed arrays.) + newMap := m.NewMap("oldKey_1:newKey_1", "oldKey_2:newKey_2", ..., "oldKey_N:newKey_N") + newXml := newMap.Xml() // for example + newJson := newMap.Json() // ditto + +XML PARSING CONVENTIONS + + - Attributes are parsed to map[string]interface{} values by prefixing a hyphen, '-', + to the attribute label. (PrependAttrWithHyphen(false) will override this.) + - If the element is a simple element and has attributes, the element value + is given the key '#text' for its map[string]interface{} representation. + +XML ENCODING CONVENTIONS + + - 'nil' Map values, which may represent 'null' JSON values, are encoded as "". + NOTE: the operation is not symmetric as "" elements are decoded as 'tag:""' Map values, + which, then, encode in JSON as '"tag":""' values.. + +*/ +package mxj diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/example_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/example_test.go new file mode 100644 index 000000000..5bcb0ab08 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/example_test.go @@ -0,0 +1,346 @@ +// note - "// Output:" is a key for "go test" to match function ouput with the lines that follow. +// It is also use by "godoc" to build the Output block of the function / method documentation. +// To skip processing Example* functions, use: go test -run "Test*" +// or make sure example function output matches // Output: documentation EXACTLY. + +package mxj_test + +import ( + "bytes" + "fmt" + "io" + "github.com/clbanning/mxj" +) + +func ExampleHandleXmlReader() { + /* + Bulk processing XML to JSON seems to be a common requirement. + See: bulk_test.go for working example. + Run "go test" in package directory then scroll back to find output. + + The logic is as follows. + + // need somewhere to write the JSON. + var jsonWriter io.Writer + + // probably want to log any errors in reading the XML stream + var xmlErrLogger io.Writer + + // func to handle Map value from XML Reader + func maphandler(m mxj.Map) bool { + // marshal Map as JSON + jsonVal, err := m.Json() + if err != nil { + // log error + return false // stops further processing of XML Reader + } + + // write JSON somewhere + _, err = jsonWriter.Write(jsonVal) + if err != nil { + // log error + return false // stops further processing of XML Reader + } + + // continue - get next XML from Reader + return true + } + + // func to handle error from unmarshaling XML Reader + func errhandler(errVal error) bool { + // log error somewhere + _, err := xmlErrLogger.Write([]byte(errVal.Error())) + if err != nil { + // log error + return false // stops further processing of XML Reader + } + + // continue processing + return true + } + + // func that starts bulk processing of the XML + ... + // set up io.Reader for XML data - perhaps an os.File + ... + err := mxj.HandleXmlReader(xmlReader, maphandler, errhandler) + if err != nil { + // handle error + } + ... + */ +} + +func ExampleHandleXmlReaderRaw() { + /* + See: bulkraw_test.go for working example. + Run "go test" in package directory then scroll back to find output. + + Basic logic for bulk XML to JSON processing is in HandleXmlReader example; + the only major difference is in handler function signatures so they are passed + the raw XML. (Read documentation on NewXmlReader regarding performance.) + */ +} + +func ExampleHandleJsonReader() { + /* + See: bulk_test.go for working example. + Run "go test" in package directory then scroll back to find output. + + Basic logic for bulk JSON to XML processing is similar to that for + bulk XML to JSON processing as outlined in the HandleXmlReader example. + The test case is also a good example. + */ +} + +func ExampleHandleJsonReaderRaw() { + /* + See: bulkraw_test.go for working example. + Run "go test" in package directory then scroll back to find output. + + Basic logic for bulk JSON to XML processing is similar to that for + bulk XML to JSON processing as outlined in the HandleXmlReader example. + The test case is also a good example. + */ +} + +/* +func ExampleNewMapXmlReaderRaw() { + // in an http.Handler + + mapVal, raw, err := mxj.NewMapXmlReader(req.Body) + if err != nil { + // handle error + } + logger.Print(string(*raw)) + // do something with mapVal + +} +*/ + +func ExampleNewMapStruct() { + type str struct { + IntVal int `json:"int"` + StrVal string `json:"str"` + FloatVal float64 `json:"float"` + BoolVal bool `json:"bool"` + private string + } + strVal := str{IntVal: 4, StrVal: "now's the time", FloatVal: 3.14159, BoolVal: true, private: "Skies are blue"} + + mapVal, merr := mxj.NewMapStruct(strVal) + if merr != nil { + // handle error + } + + fmt.Printf("strVal: %#v\n", strVal) + fmt.Printf("mapVal: %#v\n", mapVal) + // Note: example output is conformed to pass "go test". "mxj_test" is example_test.go package name. + + // Output: + // strVal: mxj_test.str{IntVal:4, StrVal:"now's the time", FloatVal:3.14159, BoolVal:true, private:"Skies are blue"} + // mapVal: mxj.Map{"int":4, "str":"now's the time", "float":3.14159, "bool":true} +} + +func ExampleMap_Struct() { + type str struct { + IntVal int `json:"int"` + StrVal string `json:"str"` + FloatVal float64 `json:"float"` + BoolVal bool `json:"bool"` + private string + } + + mapVal := mxj.Map{"int": 4, "str": "now's the time", "float": 3.14159, "bool": true, "private": "Somewhere over the rainbow"} + + var strVal str + mverr := mapVal.Struct(&strVal) + if mverr != nil { + // handle error + } + + fmt.Printf("mapVal: %#v\n", mapVal) + fmt.Printf("strVal: %#v\n", strVal) + // Note: example output is conformed to pass "go test". "mxj_test" is example_test.go package name. + + // Output: + // mapVal: mxj.Map{"int":4, "str":"now's the time", "float":3.14159, "bool":true, "private":"Somewhere over the rainbow"} + // strVal: mxj_test.str{IntVal:4, StrVal:"now's the time", FloatVal:3.14159, BoolVal:true, private:""} +} + +func ExampleMap_ValuesForKeyPath() { + // a snippet from examples/gonuts1.go + // How to compensate for irregular tag labels in data. + // Need to extract from an XML stream the values for "netid" and "idnet". + // Solution: use a wildcard path "data.*" to anonymize the "netid" and "idnet" tags. + + var msg1 = []byte(` + + + + no + default:text + default:word + + +`) + + var msg2 = []byte(` + + + + yes + default:text + default:word + + +`) + + // let's create a message stream + buf := new(bytes.Buffer) + // load a couple of messages into it + _, _ = buf.Write(msg1) + _, _ = buf.Write(msg2) + + n := 0 + for { + n++ + // Read the stream as Map values - quit on io.EOF. + // Get the raw XML as well as the Map value. + m, merr := mxj.NewMapXmlReader(buf) + if merr != nil && merr != io.EOF { + // handle error - for demo we just print it and continue + fmt.Printf("msg: %d - merr: %s\n", n, merr.Error()) + continue + } else if merr == io.EOF { + break + } + + // get the values for "netid" or "idnet" key using path == "data.*" + values, _ := m.ValuesForPath("data.*") + fmt.Println("\nmsg:", n, "> path == data.* - got array of values, len:", len(values)) + for i, val := range values { + fmt.Println("ValuesForPath result array member -", i, ":", val) + fmt.Println(" k:v pairs for array member:", i) + for key, val := range val.(map[string]interface{}) { + // You'd probably want to process the value, as appropriate. + // Here we just print it out. + fmt.Println("\t\t", key, ":", val) + } + } + } + // Output: + // msg: 1 > path == data.* - got array of values, len: 1 + // ValuesForPath result array member - 0 : map[disable:no text1:default:text word1:default:word] + // k:v pairs for array member: 0 + // disable : no + // text1 : default:text + // word1 : default:word + // + // msg: 2 > path == data.* - got array of values, len: 1 + // ValuesForPath result array member - 0 : map[disable:yes text1:default:text word1:default:word] + // k:v pairs for array member: 0 + // disable : yes + // text1 : default:text + // word1 : default:word +} + +func ExampleMap_UpdateValuesForPath() { +/* + +var biblioDoc = []byte(` + + + William Gaddis + + + The Recognitions + 1955 + A novel that changed the face of American literature. + + + JR + 1975 + Winner of National Book Award for Fiction. + + + +`) + + ... + m, merr := mxj.NewMapXml(biblioDoc) + if merr != nil { + // handle error + } + + // change 'review' for a book + count, err := m.UpdateValuesForPath("review:National Book Award winner." "*.*.*.*", "title:JR") + if err != nil { + // handle error + } + ... + + // change 'date' value from string type to float64 type + // Note: the following is equivalent to m, merr := NewMapXml(biblioDoc, mxj.Cast). + path := m.PathForKeyShortest("date") + v, err := m.ValuesForPath(path) + if err != nil { + // handle error + } + var total int + for _, vv := range v { + oldVal := "date:" + vv.(string) + newVal := "date:" + vv.(string) + ":num" + n, err := m.UpdateValuesForPath(newVal, path, oldVal) + if err != nil { + // handle error + } + total += n + } + ... +*/ +} + +func ExampleMap_Copy() { + // Hand-crafted Map values that include structures do NOT Copy() as expected, + // since to simulate a deep copy the original Map value is JSON encoded then decoded. + + type str struct { + IntVal int `json:"int"` + StrVal string `json:"str"` + FloatVal float64 `json:"float"` + BoolVal bool `json:"bool"` + private string + } + s := str{IntVal: 4, StrVal: "now's the time", FloatVal: 3.14159, BoolVal: true, private: "Skies are blue"} + m := make(map[string]interface{},0) + m["struct"] = interface{}(s) + m["struct_ptr"] = interface{}(&s) + m["misc"] = interface{}(`Now is the time`) + + + mv := mxj.Map(m) + cp,_ := mv.Copy() + + fmt.Printf("mv:%s\n", mv.StringIndent(2)) + fmt.Printf("cp:%s\n", cp.StringIndent(2)) + +// Output: +// mv: +// struct :[unknown] mxj_test.str{IntVal:4, StrVal:"now's the time", FloatVal:3.14159, BoolVal:true, private:"Skies are blue"} +// struct_ptr :[unknown] &mxj_test.str{IntVal:4, StrVal:"now's the time", FloatVal:3.14159, BoolVal:true, private:"Skies are blue"} +// misc :[string] Now is the time +// cp: +// misc :[string] Now is the time +// struct : +// int :[float64] 4.00e+00 +// str :[string] now's the time +// float :[float64] 3.14e+00 +// bool :[bool] true +// struct_ptr : +// int :[float64] 4.00e+00 +// str :[string] now's the time +// float :[float64] 3.14e+00 +// bool :[bool] true +} + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/README b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/README new file mode 100644 index 000000000..e14580f7b --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/README @@ -0,0 +1,124 @@ +Examples of using ValuesFromTagPath(). + +A number of interesting examples have shown up in the gonuts discussion group +that could be handled - after a fashion - using the ValuesFromTagPath() function. + +gonuts1.go - + +Here we see that the message stream has a problem with multiple tag spellings, +though the message structure remains constant. In this example we 'anonymize' +the tag for the variant spellings. + values := m.ValuesForPath("data.*) + where '*' is any possible spelling - "netid" or "idnet" + and the result is a list with 1 member of map[string]interface{} type. + +Once we've retrieved the Map, we can parse it using the known keys - "disable", +"text1" and "word1". + + +gonuts1a.go - (03-mar-14) + +Here we just permute the tag labels using m.NewMap() to make all the messages +consistent. Then they can be decoded into a single structure definition. + + +gonuts2.go - + +This is an interesting case where there was a need to handle messages with lists +of "ClaimStatusCodeRecord" entries as well as messages with NONE. (Here we see +some of the vagaries of dealing with mixed messages that are verging on becoming +anonymous.) + + msg1 - the message with two ClaimStatusCodeRecord entries + msg2 - the message with one ClaimStatusCodeRecord entry + msg3 - the message with NO ClaimStatusCodeRecord entries + +ValuesForPath options: + + path == "Envelope.Body.GetClaimStatusCodesResponse.GetClaimStatusCodesResult.ClaimStatusCodeRecord" + for msg == msg1: + returns: a list - []interface{} - with two values of map[string]interface{} type + for msg == msg2: + returns: a list - []interface{} - with one map[string]interface{} type + for msg == msg3: + returns 'nil' - no values + + path == "*.*.*.*.*" + for msg == msg1: + returns: a list - []interface{} - with two values of map[string]interface{} type + + path == "*.*.*.*.*.Description + for msg == msg1: + returns: a list - []interface{} - with two values of string type, the individual + values from parsing the two map[string]interface{} values where key=="Description" + + path == "*.*.*.*.*.*" + for msg == msg1: + returns: a list - []interface{} - with six values of string type, the individual + values from parsing all keys in the two map[string]interface{} values + +Think of the wildcard character "*" as anonymizing the tag in the position of the path where +it occurs. The books.go example has a range of use cases. + + +gonuts3.go - + +Uses the ValuesForKey method to extract a list of image "src" file names that are encoded as +attribute values. + + +gonuts4.go - + +Here we use the ValuesForPath to extract attribute values for country names. The attribute +is included in the 'path' argument by prepending it with a hyphen: ""doc.some_tag.geoInfo.country.-name". + + +gonuts5.go (10-mar-14) - + +Extract a node of values using ValuesForPath based on name="list3-1-1-1". Then get the values +for the 'int' entries based on attribute 'name' values - mv.ValuesForKey("int", "-name:"+n). + +gonuts5a.go (10-mar-14) - + +Extract a node of values using ValuesForPath based on name="list3-1-1-1". Then get the values +for the 'int' entries based on attribute 'name' values - mv.ValuesForKey("*", "-name:"+n). +(Same as gonuts5.go but with wildcarded key value, since we're matching elements on subkey.) + + +EAT YOUR OWN DOG FOOD ... + +I needed to convert a large (14.9 MB) XML data set from an Eclipse metrics report on an +application that had 355,100 lines of code in 211 packages into CSV data sets. The report +included application-, package-, class- and method-level metrics reported in an element, +"Value", with varying attributes. + + + + + +In addition, the metrics were reported with two different "Metric" compound elements: + + + + + ... + + + ... + + + + ... + + +Using the mxj package seemed a more straightforward approach than using Go vernacular +and the standard xml package. I wrote the program getmetrics.go to do this. Here are +three version to illustrate using + getmetrics1.go - pass os.File handle for metrics_data.xml to NewMapXmlReader. + getmetrics2.go - load metrics_data.xml into an in-memory buffer, then pass it to NewMapXml. + getmetrics3.go - demonstrates overhead of extracting the raw XML while decoding with NewMapXmlReaderRaw. + +To run example getmetrics1.go, extract a 120,000+ row data set from metrics_data.zip. Then: + go run getmetrics1.go -file=metrics_data.xml + + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/books.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/books.go new file mode 100644 index 000000000..8c43d05fa --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/books.go @@ -0,0 +1,71 @@ +// Note: this illustrates ValuesForKey() and ValuesForPath() methods + +package main + +import ( + "fmt" + "log" + "github.com/clbanning/mxj" +) + +var xmldata = []byte(` + + + William H. Gaddis + The Recognitions + One of the great seminal American novels of the 20th century. + + + Austin Tappan Wright + Islandia + An example of earlier 20th century American utopian fiction. + + + John Hawkes + The Beetle Leg + A lyrical novel about the construction of Ft. Peck Dam in Montana. + + + T.E. Porter + King's Day + A magical novella. + + +`) + +func main() { + fmt.Println(string(xmldata)) + + m, err := mxj.NewMapXml(xmldata) + if err != nil { + log.Fatal("err:", err.Error()) + } + + v, _ := m.ValuesForKey("books") + fmt.Println("path: books; len(v):", len(v)) + fmt.Printf("\t%+v\n", v) + + v, _ = m.ValuesForPath("books.book") + fmt.Println("path: books.book; len(v):", len(v)) + for _, vv := range v { + fmt.Printf("\t%+v\n", vv) + } + + v, _ = m.ValuesForPath("books.*") + fmt.Println("path: books.*; len(v):", len(v)) + for _, vv := range v { + fmt.Printf("\t%+v\n", vv) + } + + v, _ = m.ValuesForPath("books.*.title") + fmt.Println("path: books.*.title len(v):", len(v)) + for _, vv := range v { + fmt.Printf("\t%+v\n", vv) + } + + v, _ = m.ValuesForPath("books.*.*") + fmt.Println("path: books.*.*; len(v):", len(v)) + for _, vv := range v { + fmt.Printf("\t%+v\n", vv) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics1.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics1.go new file mode 100644 index 000000000..91c624613 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics1.go @@ -0,0 +1,176 @@ +// getmetrics1.go - transform Eclipse Metrics (v3) XML report into CSV files for each metric +// Uses NewMapXmlReader on os.File without copying the raw XML into a buffer while decoding.. +// Shows no significant overhead for not first buffering large XML file as in getmetrics2.go. + +/* +I needed to convert a large (14.9 MB) XML data set from an Eclipse metrics report on an +application that had 355,100 lines of code in 211 packages into CSV data sets. The report +included application-, package-, class- and method-level metrics reported in an element, +"Value", with varying attributes. + + + + + +In addition, the metrics were reported with two different "Metric" compound elements: + + + + + ... + + + ... + + + + ... + + +To run this example, extract the metrics_data.xml file from metrics_data.zip, then: + > go run getmetrics1 -file=metrics_data.xml + +The output will be a set of "csv" files. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + "github.com/clbanning/mxj" +) + +func main() { + var file string + flag.StringVar(&file, "file", "", "file to process") + flag.Parse() + + fh, fherr := os.Open(file) + if fherr != nil { + fmt.Println("fherr:", fherr.Error()) + return + } + defer fh.Close() + fmt.Println(time.Now().String(), "... File Opened:", file) + + /* + // Get the XML data set from the file. + fs, _ := fh.Stat() + xmldata := make([]byte, fs.Size()) + n, frerr := fh.Read(xmldata) + if frerr != nil { + fmt.Println("frerr:", frerr.Error()) + return + } + if int64(n) != fs.Size() { + fmt.Println("n:", n, "fs.Size():", fs.Size()) + return + } + fmt.Println(time.Now().String(), "... File Read - size:", fs.Size()) + + // load XML into a Map value + m, merr := mxj.NewMapXml(xmldata) + */ + // Consume the file using os.File Reader. + // Note: there is a single record with root tag of "Metrics". + m, merr := mxj.NewMapXmlReader(fh) + if merr != nil { + log.Fatal("merr:", merr.Error()) + } + fmt.Println(time.Now().String(), "... XML Unmarshaled - len:", len(m)) + + // Get just the key values of interest. + // Could also use m.ValuesForKey("Metric"), + // since there's just the one path. + metricVals, err := m.ValuesForPath("Metrics.Metric") + if err != nil { + log.Fatal("err:", err.Error()) + } + fmt.Println(time.Now().String(), "... ValuesFromKeyPath - len:", len(metricVals)) + + // now just manipulate Map entries returned as []interface{} array. + for _, v := range metricVals { + aMetricVal := v.(map[string]interface{}) + + // create file to hold csv data sets + id := aMetricVal["-id"].(string) + desc := aMetricVal["-description"].(string) + mf, mferr := os.OpenFile(id+".csv", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if mferr != nil { + fmt.Println("mferr:", mferr.Error()) + return + } + + fmt.Print(time.Now().String(), " id: ", id, " desc: ", desc) + mf.WriteString(id + "," + desc + "\n") + + // rescan looking for keys with data: Values or Value + for key, val := range aMetricVal { + switch key { + case "Values": + // extract the list of "Value" from map + values := val.(map[string]interface{})["Value"].([]interface{}) + fmt.Println(" len(Values):", len(values)) + + // first line in file is the metric label values (keys) + var gotKeys bool + for _, vval := range values { + valueEntry := vval.(map[string]interface{}) + + // no guarantee that range on map will follow any sequence + lv := len(valueEntry) + type ev [2]string + list := make([]ev, lv) + var i int + for k, v := range valueEntry { + list[i][0] = k + list[i][1] = v.(string) + i++ + } + + // extract keys as column header on first pass + if !gotKeys { + // print out the keys + var gotFirstKey bool + // for kk, _ := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstKey { + mf.WriteString(",") + } else { + gotFirstKey = true + } + // strip prepended hyphen + mf.WriteString((list[i][0])[1:]) + } + mf.WriteString("\n") + gotKeys = true + } + + // print out values + var gotFirstVal bool + // for _, vv := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstVal { + mf.WriteString(",") + } else { + gotFirstVal = true + } + mf.WriteString(list[i][1]) + } + + // terminate row of data + mf.WriteString("\n") + } + case "Value": + vv := val.(map[string]interface{}) + fmt.Println(" len(Value):", len(vv)) + mf.WriteString("value\n" + vv["-value"].(string) + "\n") + } + } + mf.Close() + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics2.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics2.go new file mode 100644 index 000000000..a11cbd756 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics2.go @@ -0,0 +1,173 @@ +// getmetrics2.go - transform Eclipse Metrics (v3) XML report into CSV files for each metric +// Uses an in-memory buffer for the XML data and direct XML decoding of the buffer into a Map. +// Not significantly faster than getmetrics1.go that uses an io.Reader (os.File) to directly +// decode the XML from the file. + +/* +I needed to convert a large (14.9 MB) XML data set from an Eclipse metrics report on an +application that had 355,100 lines of code in 211 packages into CSV data sets. The report +included application-, package-, class- and method-level metrics reported in an element, +"Value", with varying attributes. + + + + + +In addition, the metrics were reported with two different "Metric" compound elements: + + + + + ... + + + ... + + + + ... + + +To run this example, extract the metrics_data.xml file from metrics_data.zip, then: + > go run getmetrics -file=metrics_data.xml + +The output will be a set of "csv" files. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + "github.com/clbanning/mxj" +) + +func main() { + var file string + flag.StringVar(&file, "file", "", "file to process") + flag.Parse() + + fh, fherr := os.Open(file) + if fherr != nil { + fmt.Println("fherr:", fherr.Error()) + return + } + defer fh.Close() + fmt.Println(time.Now().String(), "... File Opened:", file) + + // Get the XML data set from the file. + fs, _ := fh.Stat() + xmldata := make([]byte, fs.Size()) + n, frerr := fh.Read(xmldata) + if frerr != nil { + fmt.Println("frerr:", frerr.Error()) + return + } + if int64(n) != fs.Size() { + fmt.Println("n:", n, "fs.Size():", fs.Size()) + return + } + fmt.Println(time.Now().String(), "... File Read - size:", fs.Size()) + + // load XML into a Map value + // Note: there is a single record with root tag of "Metrics". + m, merr := mxj.NewMapXml(xmldata) + if merr != nil { + log.Fatal("merr:", merr.Error()) + } + fmt.Println(time.Now().String(), "... XML Unmarshaled - len:", len(m)) + + // Get just the key values of interest. + // Could also use m.ValuesForKey("Metric"), + // since there's just the one path. + metricVals, err := m.ValuesForPath("Metrics.Metric") + if err != nil { + log.Fatal("err:", err.Error()) + } + fmt.Println(time.Now().String(), "... ValuesFromKeyPath - len:", len(metricVals)) + + // now just manipulate Map entries returned as []interface{} array. + for _, v := range metricVals { + aMetricVal := v.(map[string]interface{}) + + // create file to hold csv data sets + id := aMetricVal["-id"].(string) + desc := aMetricVal["-description"].(string) + mf, mferr := os.OpenFile(id+".csv", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if mferr != nil { + fmt.Println("mferr:", mferr.Error()) + return + } + + fmt.Print(time.Now().String(), " id: ", id, " desc: ", desc) + mf.WriteString(id + "," + desc + "\n") + + // rescan looking for keys with data: Values or Value + for key, val := range aMetricVal { + switch key { + case "Values": + // extract the list of "Value" from map + values := val.(map[string]interface{})["Value"].([]interface{}) + fmt.Println(" len(Values):", len(values)) + + // first line in file is the metric label values (keys) + var gotKeys bool + for _, vval := range values { + valueEntry := vval.(map[string]interface{}) + + // no guarantee that range on map will follow any sequence + lv := len(valueEntry) + type ev [2]string + list := make([]ev, lv) + var i int + for k, v := range valueEntry { + list[i][0] = k + list[i][1] = v.(string) + i++ + } + + // extract keys as column header on first pass + if !gotKeys { + // print out the keys + var gotFirstKey bool + // for kk, _ := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstKey { + mf.WriteString(",") + } else { + gotFirstKey = true + } + // strip prepended hyphen + mf.WriteString((list[i][0])[1:]) + } + mf.WriteString("\n") + gotKeys = true + } + + // print out values + var gotFirstVal bool + // for _, vv := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstVal { + mf.WriteString(",") + } else { + gotFirstVal = true + } + mf.WriteString(list[i][1]) + } + + // terminate row of data + mf.WriteString("\n") + } + case "Value": + vv := val.(map[string]interface{}) + fmt.Println(" len(Value):", len(vv)) + mf.WriteString("value\n" + vv["-value"].(string) + "\n") + } + } + mf.Close() + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics3.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics3.go new file mode 100644 index 000000000..370702fbd --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics3.go @@ -0,0 +1,180 @@ +// getmetrics3.go - transform Eclipse Metrics (v3) XML report into CSV files for each metric +// Uses NewMapXmlReaderRaw that requires loading raw XML into a []byte buffer using a ByteReader. +// Show's performance impact of copying the raw XML while simultaneously decoding it from on os.File +// Reader. (vs. getmetrics1.go) If you're processing a file and need a copy of the raw XML and SPEED, +// should buffer the file in memory and decode using mxj.NewMapXmlReaderRaw as in getmetrics4.go. + +/* +I needed to convert a large (14.9 MB) XML data set from an Eclipse metrics report on an +application that had 355,100 lines of code in 211 packages into CSV data sets. The report +included application-, package-, class- and method-level metrics reported in an element, +"Value", with varying attributes. + + + + + +In addition, the metrics were reported with two different "Metric" compound elements: + + + + + ... + + + ... + + + + ... + + +To run this example, extract the metrics_data.xml file from metrics_data.zip, then: + > go run getmetrics3 -file=metrics_data.xml + +The output will be a set of "csv" files. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + "github.com/clbanning/mxj" +) + +func main() { + var file string + flag.StringVar(&file, "file", "", "file to process") + flag.Parse() + + fh, fherr := os.Open(file) + if fherr != nil { + fmt.Println("fherr:", fherr.Error()) + return + } + defer fh.Close() + fmt.Println(time.Now().String(), "... File Opened:", file) + + /* + // Get the XML data set from the file. + fs, _ := fh.Stat() + xmldata := make([]byte, fs.Size()) + n, frerr := fh.Read(xmldata) + if frerr != nil { + fmt.Println("frerr:", frerr.Error()) + return + } + if int64(n) != fs.Size() { + fmt.Println("n:", n, "fs.Size():", fs.Size()) + return + } + fmt.Println(time.Now().String(), "... File Read - size:", fs.Size()) + + // load XML into a Map value + m, merr := mxj.NewMapXml(xmldata) + */ + // Consume the file using os.File Reader. + // Note: there is a single record with root tag of "Metrics". + // Also: this is MUCH slower than using buffer or not loading raw XML. + m, raw, merr := mxj.NewMapXmlReaderRaw(fh) + if merr != nil { + log.Fatal("merr:", merr.Error()) + } + fmt.Println(time.Now().String(), "... XML Unmarshaled - len:", len(m)) + fmt.Println("raw XML buffer size (should be same as File size):", len(*raw)) + + // Get just the key values of interest. + // Could also use m.ValuesForKey("Metric"), + // since there's just the one path. + metricVals, err := m.ValuesForPath("Metrics.Metric") + if err != nil { + log.Fatal("err:", err.Error()) + } + fmt.Println(time.Now().String(), "... ValuesFromKeyPath - len:", len(metricVals)) + + // now just manipulate Map entries returned as []interface{} array. + for _, v := range metricVals { + aMetricVal := v.(map[string]interface{}) + + // create file to hold csv data sets + id := aMetricVal["-id"].(string) + desc := aMetricVal["-description"].(string) + mf, mferr := os.OpenFile(id+".csv", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if mferr != nil { + fmt.Println("mferr:", mferr.Error()) + return + } + + fmt.Print(time.Now().String(), " id: ", id, " desc: ", desc) + mf.WriteString(id + "," + desc + "\n") + + // rescan looking for keys with data: Values or Value + for key, val := range aMetricVal { + switch key { + case "Values": + // extract the list of "Value" from map + values := val.(map[string]interface{})["Value"].([]interface{}) + fmt.Println(" len(Values):", len(values)) + + // first line in file is the metric label values (keys) + var gotKeys bool + for _, vval := range values { + valueEntry := vval.(map[string]interface{}) + + // no guarantee that range on map will follow any sequence + lv := len(valueEntry) + type ev [2]string + list := make([]ev, lv) + var i int + for k, v := range valueEntry { + list[i][0] = k + list[i][1] = v.(string) + i++ + } + + // extract keys as column header on first pass + if !gotKeys { + // print out the keys + var gotFirstKey bool + // for kk, _ := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstKey { + mf.WriteString(",") + } else { + gotFirstKey = true + } + // strip prepended hyphen + mf.WriteString((list[i][0])[1:]) + } + mf.WriteString("\n") + gotKeys = true + } + + // print out values + var gotFirstVal bool + // for _, vv := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstVal { + mf.WriteString(",") + } else { + gotFirstVal = true + } + mf.WriteString(list[i][1]) + } + + // terminate row of data + mf.WriteString("\n") + } + case "Value": + vv := val.(map[string]interface{}) + fmt.Println(" len(Value):", len(vv)) + mf.WriteString("value\n" + vv["-value"].(string) + "\n") + } + } + mf.Close() + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics4.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics4.go new file mode 100644 index 000000000..a8db0b7bf --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/getmetrics4.go @@ -0,0 +1,179 @@ +// getmetrics2.go - transform Eclipse Metrics (v3) XML report into CSV files for each metric +// Uses an in-memory buffer for the XML data and direct XML decoding of the buffer into a Map. +// Then XML buffer is decoded into a Map while the raw XML is copied using NewMapXmlReaderRaw() +// to illustrate processing overhead relative to getmetrics2.go. Not a practical example, +// but confirms the getmetrics1.go vs. getmetrics3.go use case. + +/* +I needed to convert a large (14.9 MB) XML data set from an Eclipse metrics report on an +application that had 355,100 lines of code in 211 packages into CSV data sets. The report +included application-, package-, class- and method-level metrics reported in an element, +"Value", with varying attributes. + + + + + +In addition, the metrics were reported with two different "Metric" compound elements: + + + + + ... + + + ... + + + + ... + + +To run this example, extract the metrics_data.xml file from metrics_data.zip, then: + > go run getmetrics -file=metrics_data.xml + +The output will be a set of "csv" files. +*/ + +package main + +import ( + "bytes" + "flag" + "fmt" + "log" + "os" + "time" + "github.com/clbanning/mxj" +) + +func main() { + var file string + flag.StringVar(&file, "file", "", "file to process") + flag.Parse() + + fh, fherr := os.Open(file) + if fherr != nil { + fmt.Println("fherr:", fherr.Error()) + return + } + defer fh.Close() + fmt.Println(time.Now().String(), "... File Opened:", file) + + // Get the XML data set from the file. + fs, _ := fh.Stat() + xmldata := make([]byte, fs.Size()) + n, frerr := fh.Read(xmldata) + if frerr != nil { + fmt.Println("frerr:", frerr.Error()) + return + } + if int64(n) != fs.Size() { + fmt.Println("n:", n, "fs.Size():", fs.Size()) + return + } + fmt.Println(time.Now().String(), "... File Read - size:", fs.Size()) + + // wrap the buffer in an io.Reader + xmlReader := bytes.NewBuffer(xmldata) + + // load XML into a Map value + // Note: there is a single record with root tag of "Metrics". + m, raw, merr := mxj.NewMapXmlReaderRaw(xmlReader) // don't catch the pointer to raw XML + if merr != nil { + log.Fatal("merr:", merr.Error()) + } + fmt.Println(time.Now().String(), "... XML Unmarshaled - len:", len(m)) + fmt.Println("raw XML buffer size (should be same as File size):", len(*raw)) + + // Get just the key values of interest. + // Could also use m.ValuesForKey("Metric"), + // since there's just the one path. + metricVals, err := m.ValuesForPath("Metrics.Metric") + if err != nil { + log.Fatal("err:", err.Error()) + } + fmt.Println(time.Now().String(), "... ValuesFromKeyPath - len:", len(metricVals)) + + // now just manipulate Map entries returned as []interface{} array. + for _, v := range metricVals { + aMetricVal := v.(map[string]interface{}) + + // create file to hold csv data sets + id := aMetricVal["-id"].(string) + desc := aMetricVal["-description"].(string) + mf, mferr := os.OpenFile(id+".csv", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if mferr != nil { + fmt.Println("mferr:", mferr.Error()) + return + } + + fmt.Print(time.Now().String(), " id: ", id, " desc: ", desc) + mf.WriteString(id + "," + desc + "\n") + + // rescan looking for keys with data: Values or Value + for key, val := range aMetricVal { + switch key { + case "Values": + // extract the list of "Value" from map + values := val.(map[string]interface{})["Value"].([]interface{}) + fmt.Println(" len(Values):", len(values)) + + // first line in file is the metric label values (keys) + var gotKeys bool + for _, vval := range values { + valueEntry := vval.(map[string]interface{}) + + // no guarantee that range on map will follow any sequence + lv := len(valueEntry) + type ev [2]string + list := make([]ev, lv) + var i int + for k, v := range valueEntry { + list[i][0] = k + list[i][1] = v.(string) + i++ + } + + // extract keys as column header on first pass + if !gotKeys { + // print out the keys + var gotFirstKey bool + // for kk, _ := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstKey { + mf.WriteString(",") + } else { + gotFirstKey = true + } + // strip prepended hyphen + mf.WriteString((list[i][0])[1:]) + } + mf.WriteString("\n") + gotKeys = true + } + + // print out values + var gotFirstVal bool + // for _, vv := range valueEntry { + for i := 0; i < lv; i++ { + if gotFirstVal { + mf.WriteString(",") + } else { + gotFirstVal = true + } + mf.WriteString(list[i][1]) + } + + // terminate row of data + mf.WriteString("\n") + } + case "Value": + vv := val.(map[string]interface{}) + fmt.Println(" len(Value):", len(vv)) + mf.WriteString("value\n" + vv["-value"].(string) + "\n") + } + } + mf.Close() + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts1.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts1.go new file mode 100644 index 000000000..0564fbc80 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts1.go @@ -0,0 +1,82 @@ +// https://groups.google.com/forum/#!searchin/golang-nuts/idnet$20netid/golang-nuts/guM3ZHHqSF0/K1pBpMqQSSwJ +// http://play.golang.org/p/BFFDxphKYK + +package main + +import ( + "bytes" + "fmt" + "io" + "github.com/clbanning/mxj" +) + +// Demo how to compensate for irregular tag labels in data. +// Need to extract from an XML stream the values for "netid" and "idnet". +// Solution: use a wildcard path "data.*" to anonymize the "netid" and "idnet" tags. + +var msg1 = []byte(` + + + + no + default:text + default:word + + +`) + +var msg2 = []byte(` + + + + yes + default:text + default:word + + +`) + +func main() { + // let's create a message stream + buf := new(bytes.Buffer) + // load a couple of messages into it + _, _ = buf.Write(msg1) + _, _ = buf.Write(msg2) + + n := 0 + for { + n++ + // read the stream as Map values - quit on io.EOF + m, raw, merr := mxj.NewMapXmlReaderRaw(buf) + if merr != nil && merr != io.EOF { + // handle error - for demo we just print it and continue + fmt.Printf("msg: %d - merr: %s\n", n, merr.Error()) + continue + } else if merr == io.EOF { + break + } + fmt.Println("\nMessage to parse:", string(*raw)) + fmt.Println("Map value for XML message:", m.StringIndent()) + + // get the values for "netid" or "idnet" key using path == "data.*" + values, _ := m.ValuesForPath("data.*") + fmt.Println("\nmsg:", n, "> path == data.* - got array of values, len:", len(values)) + for i, val := range values { + fmt.Println("ValuesForPath result array member -", i, ":", val) + fmt.Println(" k:v pairs for array member:", i) + for key, val := range val.(map[string]interface{}) { + // You'd probably want to process the value, as appropriate. + // Here we just print it out. + fmt.Println("\t\t", key, ":", val) + } + } + + // This shows what happens if you wildcard the value keys for "idnet" and "netid" + values, _ = m.ValuesForPath("data.*.*") + fmt.Println("\npath == data.*.* - got an array of values, len(v):", len(values)) + fmt.Println("(Note: values returned by ValuesForPath are at maximum depth of the tree. So just have values.)") + for i, val := range values { + fmt.Println("ValuesForPath array member -", i, ":", val) + } + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts1a.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts1a.go new file mode 100644 index 000000000..fbe58ffad --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts1a.go @@ -0,0 +1,68 @@ +// https://groups.google.com/forum/#!searchin/golang-nuts/idnet$20netid/golang-nuts/guM3ZHHqSF0/K1pBpMqQSSwJ +// http://play.golang.org/p/BFFDxphKYK + +package main + +import ( + "bytes" + "fmt" + "github.com/clbanning/mxj" + "io" +) + +// Demo how to re-label a key using mv.NewMap(). +// Need to normalize from an XML stream the tags "netid" and "idnet". +// Solution: make everything "netid". + +var msg1 = []byte(` + + + + no + default:text + default:word + + +`) + +var msg2 = []byte(` + + + + yes + default:text + default:word + + +`) + +func main() { + // let's create a message stream + buf := new(bytes.Buffer) + // load a couple of messages into it + _, _ = buf.Write(msg1) + _, _ = buf.Write(msg2) + + n := 0 + for { + n++ + // read the stream as Map values - quit on io.EOF + m, raw, merr := mxj.NewMapXmlReaderRaw(buf) + if merr != nil && merr != io.EOF { + // handle error - for demo we just print it and continue + fmt.Printf("msg: %d - merr: %s\n", n, merr.Error()) + continue + } else if merr == io.EOF { + break + } + + // the first keypair retains values if data correct + // the second keypair relabels "idnet" to "netid" + n, _ := m.NewMap("data.netid", "data.idnet:data.netid") + x, _ := n.XmlIndent("", " ") + + fmt.Println("original value:", string(raw)) + fmt.Println("new value:") + fmt.Println(string(x)) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts2.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts2.go new file mode 100644 index 000000000..8cacafeeb --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts2.go @@ -0,0 +1,240 @@ +// https://groups.google.com/forum/#!topic/golang-nuts/V83jUKluLnM +// http://play.golang.org/p/alWGk4MDBc + +// Here messsages come in one of three forms: +// ... list of ClaimStatusCodeRecord ... +// ... one instance of ClaimStatusCodeRecord ... +// ... empty element ... +// ValueForPath + +package main + +import ( + "fmt" + "github.com/clbanning/mxj" +) + +var xmlmsg1 = []byte(` + + + + +true +A +Initial Claim Review/Screening + + +true +B +Initial Contact Made w/ Provider + + + + + +`) + +var xmlmsg2 = []byte(` + + + + +true +A +Initial Claim Review/Screening + + + + + +`) + +var xmlmsg3 = []byte(` + + + + + + + +`) + +func main() { + xmldata := [][]byte{xmlmsg1, xmlmsg2, xmlmsg3} + fullPath(xmldata) + partPath1(xmlmsg1) + partPath2(xmlmsg1) + partPath3(xmlmsg1) + partPath4(xmlmsg1) + partPath5(xmlmsg1) + partPath6(xmlmsg1) +} + +func fullPath(xmldata [][]byte) { + for i, msg := range xmldata { + fmt.Println("\ndoc:", i) + fmt.Println(string(msg)) + // decode the XML + m, _ := mxj.NewMapXml(msg) + + // get the value for the key path of interest + path := "Envelope.Body.GetClaimStatusCodesResponse.GetClaimStatusCodesResult.ClaimStatusCodeRecord" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + continue + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + fmt.Println("values:", values, "\n") + for _, v := range values { + switch v.(type) { + case map[string]interface{}: + fmt.Println("map[string]interface{}:", v.(map[string]interface{})) + case []map[string]interface{}: + fmt.Println("[]map[string]interface{}:", v.([]map[string]interface{})) + case []interface{}: + fmt.Println("[]interface{}:", v.([]interface{})) + case interface{}: + fmt.Println("interface{}:", v.(interface{})) + } + } + } +} + +func partPath1(msg []byte) { + fmt.Println("\nmsg:",string(msg)) + m, _ := mxj.NewMapXml(msg) + fmt.Println("m:",m.StringIndent()) + path := "Envelope.Body.*.*.ClaimStatusCodeRecord" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + return + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + for n, v := range values { + fmt.Printf("\t#%d: %v\n", n, v) + } +} + +func partPath2(msg []byte) { + fmt.Println("\nmsg:",string(msg)) + m, _ := mxj.NewMapXml(msg) + fmt.Println("m:",m.StringIndent()) + path := "Envelope.Body.*.*.*" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + return + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + for n, v := range values { + fmt.Printf("\t#%d: %v\n", n, v) + } +} + +func partPath3(msg []byte) { + fmt.Println("\nmsg:",string(msg)) + m, _ := mxj.NewMapXml(msg) + fmt.Println("m:",m.StringIndent()) + path := "*.*.*.*.*" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + return + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + for n, v := range values { + fmt.Printf("\t#%d: %v\n", n, v) + } +} + +func partPath4(msg []byte) { + fmt.Println("\nmsg:",string(msg)) + m, _ := mxj.NewMapXml(msg) + fmt.Println("m:",m.StringIndent()) + path := "*.*.*.*.*.Description" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + return + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + for n, v := range values { + fmt.Printf("\t#%d: %v\n", n, v) + } +} + +func partPath5(msg []byte) { + fmt.Println("\nmsg:",string(msg)) + m, _ := mxj.NewMapXml(msg) + fmt.Println("m:",m.StringIndent()) + path := "*.*.*.*.*.*" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + return + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + for n, v := range values { + fmt.Printf("\t#%d: %v\n", n, v) + } +} + +func partPath6(msg []byte) { + fmt.Println("\nmsg:",string(msg)) + m, _ := mxj.NewMapXml(msg) + fmt.Println("m:",m.StringIndent()) + path := "*.*.*.*.*.*.*" + values, err := m.ValueForPath(path) + if err != nil { + fmt.Println("err:", err.Error()) + return + } + if values == nil { + fmt.Println("path:", path) + fmt.Println("No ClaimStatusCodesResult code records.") + return + } + fmt.Println("\nPath:", path) + fmt.Println("Number of code records:", len(values)) + for n, v := range values { + fmt.Printf("\t#%d: %v\n", n, v) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts3.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts3.go new file mode 100644 index 000000000..5bfe0628c --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts3.go @@ -0,0 +1,47 @@ +// https://groups.google.com/forum/#!topic/golang-nuts/cok6xasvI3w +// retrieve 'src' values from 'image' tags + +package main + +import ( + "fmt" + "github.com/clbanning/mxj" +) + +var xmldata = []byte(` + + + + + something else + + + + + + + +`) + +func main() { + fmt.Println("xmldata:", string(xmldata)) + + // get all image tag values - []interface{} + m, merr := mxj.NewMapXml(xmldata) + if merr != nil { + fmt.Println("merr:", merr.Error()) + return + } + + // grab all values for attribute "src" + // Note: attributes are prepended with a hyphen, '-'. + sources, err := m.ValuesForKey("-src") + if err != nil { + fmt.Println("err:", err.Error()) + return + } + + for _, src := range sources { + fmt.Println(src) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts4.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts4.go new file mode 100644 index 000000000..8dfc30fd4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts4.go @@ -0,0 +1,50 @@ +// https://groups.google.com/forum/#!topic/golang-nuts/-N9Toa6qlu8 +// shows that you can extract attribute values directly from tag/key path. +// NOTE: attribute values are encoded by prepending a hyphen, '-'. + +package main + +import ( + "fmt" + "github.com/clbanning/mxj" +) + +var xmldata = []byte(` + + + + + + + + + + + + + + + + + + `) + +func main() { + fmt.Println("xmldata:", string(xmldata)) + + m, merr := mxj.NewMapXml(xmldata) + if merr != nil { + fmt.Println("merr:", merr) + return + } + + // Attributes are keys with prepended hyphen, '-'. + values, err := m.ValuesForPath("doc.some_tag.geoInfo.country.-name") + if err != nil { + fmt.Println("err:", err.Error()) + } + + for _, v := range values { + fmt.Println("v:",v) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts5.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts5.go new file mode 100644 index 000000000..a1b8d16a5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts5.go @@ -0,0 +1,75 @@ +// gonuts5.go - from https://groups.google.com/forum/#!topic/golang-nuts/MWoYY19of3o +// problem is to extract entries from by "int name=" + +package main + +import ( + "fmt" + "github.com/clbanning/mxj" +) + +var xmlData = []byte(` + + + + + + + 1 + + + + 1 + 2 + 3 + 4 + 5 + + + + + 1 + 2 + 3 + 4 + 5 + + + + +`) + +func main() { + // parse XML into a Map + m, merr := mxj.NewMapXml(xmlData) + if merr != nil { + fmt.Println("merr:", merr.Error()) + return + } + + // extract the 'list3-1-1-1' node - there'll be just 1? + // NOTE: attribute keys are prepended with '-' + lstVal, lerr := m.ValuesForPath("*.*.*.*.*", "-name:list3-1-1-1") + if lerr != nil { + fmt.Println("ierr:", lerr.Error()) + return + } + + // assuming just one value returned - create a new Map + mv := mxj.Map(lstVal[0].(map[string]interface{})) + + // extract the 'int' values by 'name' attribute: "-name" + // interate over list of 'name' values of interest + var names = []string{"field1", "field2", "field3", "field4", "field5"} + for _, n := range names { + vals, verr := mv.ValuesForKey("int", "-name:"+n) + if verr != nil { + fmt.Println("verr:", verr.Error(), len(vals)) + return + } + + // values for simple elements have key '#text' + // NOTE: there can be only one value for key '#text' + fmt.Println(n, ":", vals[0].(map[string]interface{})["#text"]) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts5a.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts5a.go new file mode 100644 index 000000000..de60688fa --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts5a.go @@ -0,0 +1,75 @@ +// gonuts5.go - from https://groups.google.com/forum/#!topic/golang-nuts/MWoYY19of3o +// problem is to extract entries from by "int name=" + +package main + +import ( + "fmt" + "github.com/clbanning/mxj" +) + +var xmlData = []byte(` + + + + + + + 1 + + + + 1 + 2 + 3 + 4 + 5 + + + + + 1 + 2 + 3 + 4 + 5 + + + + +`) + +func main() { + // parse XML into a Map + m, merr := mxj.NewMapXml(xmlData) + if merr != nil { + fmt.Println("merr:", merr.Error()) + return + } + + // extract the 'list3-1-1-1' node - there'll be just 1? + // NOTE: attribute keys are prepended with '-' + lstVal, lerr := m.ValuesForPath("*.*.*.*.*", "-name:list3-1-1-1") + if lerr != nil { + fmt.Println("ierr:", lerr.Error()) + return + } + + // assuming just one value returned - create a new Map + mv := mxj.Map(lstVal[0].(map[string]interface{})) + + // extract the 'int' values by 'name' attribute: "-name" + // interate over list of 'name' values of interest + var names = []string{"field1", "field2", "field3", "field4", "field5"} + for _, n := range names { + vals, verr := mv.ValuesForKey("*", "-name:"+n) + if verr != nil { + fmt.Println("verr:", verr.Error(), len(vals)) + return + } + + // values for simple elements have key '#text' + // NOTE: there can be only one value for key '#text' + fmt.Println(n, ":", vals[0].(map[string]interface{})["#text"]) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts6.go b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts6.go new file mode 100644 index 000000000..ee2b89803 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/gonuts6.go @@ -0,0 +1,44 @@ +/* https://groups.google.com/forum/#!topic/golang-nuts/EMXHB1nJoBA + +package main + +import "fmt" +import "encoding/json" + +func main() { + var v struct { + DBInstances []struct { + Endpoint struct { + Address string + } + } + } + json.Unmarshal(s, &v) + fmt.Println(v.DBInstances[0].Endpoint.Address) +} +*/ + +package main + +import "fmt" +import "github.com/clbanning/mxj" + +func main() { + m, err := mxj.NewMapJson(s) + if err != nil { + fmt.Println("err:", err.Error()) + } + v, err := m.ValuesForKey("Address") + // v, err := m.ValuesForPath("DBInstances[0].Endpoint.Address") + if err != nil { + fmt.Println("err:", err.Error()) + } + if len(v) > 0 { + fmt.Println(v[0].(string)) + } else { + fmt.Println("No value.") + } +} + +var s = []byte(`{ "DBInstances": [ { "PubliclyAccessible": true, "MasterUsername": "postgres", "LicenseModel": "postgresql-license", "VpcSecurityGroups": [ { "Status": "active", "VpcSecurityGroupId": "sg-e72a4282" } ], "InstanceCreateTime": "2014-06-29T03:52:59.268Z", "OptionGroupMemberships": [ { "Status": "in-sync", "OptionGroupName": "default:postgres-9-3" } ], "PendingModifiedValues": {}, "Engine": "postgres", "MultiAZ": true, "LatestRestorableTime": "2014-06-29T12:00:34Z", "DBSecurityGroups": [ { "Status": "active", "DBSecurityGroupName": "production-dbsecuritygroup-q4f0ugxpjck8" } ], "DBParameterGroups": [ { "DBParameterGroupName": "default.postgres9.3", "ParameterApplyStatus": "in-sync" } ], "AutoMinorVersionUpgrade": true, "PreferredBackupWindow": "06:59-07:29", "DBSubnetGroup": { "Subnets": [ { "SubnetStatus": "Active", "SubnetIdentifier": "subnet-34e5d01c", "SubnetAvailabilityZone": { "Name": "us-east-1b", "ProvisionedIopsCapable": false } }, { "SubnetStatus": "Active", "SubnetIdentifier": "subnet-50759d27", "SubnetAvailabilityZone": { "Name": "us-east-1c", "ProvisionedIopsCapable": false } }, { "SubnetStatus": "Active", "SubnetIdentifier": "subnet-450a1f03", "SubnetAvailabilityZone": { "Name": "us-east-1d", "ProvisionedIopsCapable": false } } ], "DBSubnetGroupName": "default", "VpcId": "vpc-acb86cc9", "DBSubnetGroupDescription": "default", "SubnetGroupStatus": "Complete" }, "SecondaryAvailabilityZone": "us-east-1b", "ReadReplicaDBInstanceIdentifiers": [], "AllocatedStorage": 15, "BackupRetentionPeriod": 1, "DBName": "deis", "PreferredMaintenanceWindow": "fri:05:52-fri:06:22", "Endpoint": { "Port": 5432, "Address": "production.cfk8mskkbkeu.us-east-1.rds.amazonaws.com" }, "DBInstanceStatus": "available", "EngineVersion": "9.3.3", "AvailabilityZone": "us-east-1c", "DBInstanceClass": "db.m1.small", "DBInstanceIdentifier": "production" } ] }`) + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/examples/metrics_data.zip b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/metrics_data.zip new file mode 100644 index 000000000..386837491 Binary files /dev/null and b/Godeps/_workspace/src/github.com/clbanning/mxj/examples/metrics_data.zip differ diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files.go b/Godeps/_workspace/src/github.com/clbanning/mxj/files.go new file mode 100644 index 000000000..340302674 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files.go @@ -0,0 +1,301 @@ +package mxj + +import ( + "fmt" + "io" + "os" +) + +type Maps []Map + +func NewMaps() Maps { + return make(Maps,0) +} + +type MapRaw struct { + M Map + R []byte +} + +// NewMapsFromXmlFile - creates an array from a file of JSON values. +func NewMapsFromJsonFile(name string) (Maps, error) { + fi, err := os.Stat(name) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("file %s is not a regular file", name) + } + + fh, err := os.Open(name) + if err != nil { + return nil, err + } + defer fh.Close() + + am := make([]Map,0) + for { + m, raw, err := NewMapJsonReaderRaw(fh) + if err != nil && err != io.EOF { + return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(raw)) + } + if len(m) > 0 { + am = append(am, m) + } + if err == io.EOF { + break + } + } + return am, nil +} + +// ReadMapsFromJsonFileRaw - creates an array of MapRaw from a file of JSON values. +func NewMapsFromJsonFileRaw(name string) ([]MapRaw, error) { + fi, err := os.Stat(name) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("file %s is not a regular file", name) + } + + fh, err := os.Open(name) + if err != nil { + return nil, err + } + defer fh.Close() + + am := make([]MapRaw,0) + for { + mr := new(MapRaw) + mr.M, mr.R, err = NewMapJsonReaderRaw(fh) + if err != nil && err != io.EOF { + return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(mr.R)) + } + if len(mr.M) > 0 { + am = append(am, *mr) + } + if err == io.EOF { + break + } + } + return am, nil +} + +// NewMapsFromXmlFile - creates an array from a file of XML values. +func NewMapsFromXmlFile(name string) (Maps, error) { + x := XmlWriterBufSize + XmlWriterBufSize = 0 + defer func() { + XmlWriterBufSize = x + }() + + fi, err := os.Stat(name) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("file %s is not a regular file", name) + } + + fh, err := os.Open(name) + if err != nil { + return nil, err + } + defer fh.Close() + + am := make([]Map,0) + for { + m, raw, err := NewMapXmlReaderRaw(fh) + if err != nil && err != io.EOF { + return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(raw)) + } + if len(m) > 0 { + am = append(am, m) + } + if err == io.EOF { + break + } + } + return am, nil +} + +// NewMapsFromXmlFileRaw - creates an array of MapRaw from a file of XML values. +// NOTE: the slice with the raw XML is clean with no extra capacity - unlike NewMapXmlReaderRaw(). +// It is slow at parsing a file from disk and is intended for relatively small utility files. +func NewMapsFromXmlFileRaw(name string) ([]MapRaw, error) { + x := XmlWriterBufSize + XmlWriterBufSize = 0 + defer func() { + XmlWriterBufSize = x + }() + + + fi, err := os.Stat(name) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("file %s is not a regular file", name) + } + + fh, err := os.Open(name) + if err != nil { + return nil, err + } + defer fh.Close() + + am := make([]MapRaw,0) + for { + mr := new(MapRaw) + mr.M, mr.R, err = NewMapXmlReaderRaw(fh) + if err != nil && err != io.EOF { + return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(mr.R)) + } + if len(mr.M) > 0 { + am = append(am, *mr) + } + if err == io.EOF { + break + } + } + return am, nil +} + +// ------------------------ Maps writing ------------------------- +// These are handy-dandy methods for dumping configuration data, etc. + +// JsonString - analogous to mv.Json() +func (mvs Maps) JsonString(safeEncoding ...bool) (string, error) { + var s string + for _, v := range mvs { + j, err := v.Json() + if err != nil { + return s, err + } + s += string(j) + } + return s, nil +} + +// JsonStringIndent - analogous to mv.JsonIndent() +func (mvs Maps) JsonStringIndent(prefix, indent string, safeEncoding ...bool) (string, error) { + var s string + var haveFirst bool + for _, v := range mvs { + j, err := v.JsonIndent(prefix, indent) + if err != nil { + return s, err + } + if haveFirst { + s += "\n" + } else { + haveFirst = true + } + s += string(j) + } + return s, nil +} + +// XmlString - analogous to mv.Xml() +func (mvs Maps) XmlString() (string, error) { + var s string + for _, v := range mvs { + x, err := v.Xml() + if err != nil { + return s, err + } + s += string(x) + } + return s, nil +} + +// XmlStringIndent - analogous to mv.XmlIndent() +func (mvs Maps) XmlStringIndent(prefix, indent string) (string, error) { + var s string + for _, v := range mvs { + x, err := v.XmlIndent(prefix, indent) + if err != nil { + return s, err + } + s += string(x) + } + return s, nil +} + +// JsonFile - write Maps to named file as JSON +// Note: the file will be created, if necessary; if it exists it will be truncated. +// If you need to append to a file, open it and use JsonWriter method. +func (mvs Maps) JsonFile(file string, safeEncoding ...bool) error { + var encoding bool + if len(safeEncoding) == 1 { + encoding = safeEncoding[0] + } + s, err := mvs.JsonString(encoding) + if err != nil { + return err + } + fh, err := os.Create(file) + if err != nil { + return err + } + defer fh.Close() + fh.WriteString(s) + return nil +} + +// JsonFileIndent - write Maps to named file as pretty JSON +// Note: the file will be created, if necessary; if it exists it will be truncated. +// If you need to append to a file, open it and use JsonIndentWriter method. +func (mvs Maps) JsonFileIndent(file, prefix, indent string, safeEncoding ...bool) error { + var encoding bool + if len(safeEncoding) == 1 { + encoding = safeEncoding[0] + } + s, err := mvs.JsonStringIndent(prefix, indent, encoding) + if err != nil { + return err + } + fh, err := os.Create(file) + if err != nil { + return err + } + defer fh.Close() + fh.WriteString(s) + return nil +} + +// XmlFile - write Maps to named file as XML +// Note: the file will be created, if necessary; if it exists it will be truncated. +// If you need to append to a file, open it and use XmlWriter method. +func (mvs Maps) XmlFile(file string) error { + s, err := mvs.XmlString() + if err != nil { + return err + } + fh, err := os.Create(file) + if err != nil { + return err + } + defer fh.Close() + fh.WriteString(s) + return nil +} + +// XmlFileIndent - write Maps to named file as pretty XML +// Note: the file will be created,if necessary; if it exists it will be truncated. +// If you need to append to a file, open it and use XmlIndentWriter method. +func (mvs Maps) XmlFileIndent(file, prefix, indent string) error { + s, err := mvs.XmlStringIndent(prefix, indent) + if err != nil { + return err + } + fh, err := os.Create(file) + if err != nil { + return err + } + defer fh.Close() + fh.WriteString(s) + return nil +} + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.badjson b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.badjson new file mode 100644 index 000000000..d18720044 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.badjson @@ -0,0 +1,2 @@ +{ "this":"is", "a":"test", "file":"for", "files_test.go":"case" } +{ "with":"some", "bad":JSON, "in":"it" } diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.badxml b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.badxml new file mode 100644 index 000000000..4736ef973 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.badxml @@ -0,0 +1,9 @@ + + test + for files.go + + + some + doc + test case + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.go new file mode 100644 index 000000000..2850902f9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.go @@ -0,0 +1,168 @@ +package mxj + +import ( + "fmt" + "testing" +) + +func TestFilesHeader(t *testing.T) { + fmt.Println("\n---------------- files_test.go ...\n") +} + +func TestNewJsonFile(t *testing.T) { + fmt.Println("NewMapsFromJsonFile()") + am, err := NewMapsFromJsonFile("files_test.json") + if err != nil { + t.Fatal(err.Error()) + } + for _, v := range am { + fmt.Printf("%v\n", v) + } + + am, err = NewMapsFromJsonFile("nil") + if err == nil { + t.Fatal("no error returned for read of nil file") + } + fmt.Println("caught error: ", err.Error()) + + am, err = NewMapsFromJsonFile("files_test.badjson") + if err == nil { + t.Fatal("no error returned for read of badjson file") + } + fmt.Println("caught error: ", err.Error()) +} + +func TestNewJsonFileRaw(t *testing.T) { + fmt.Println("NewMapsFromJsonFileRaw()") + mr, err := NewMapsFromJsonFileRaw("files_test.json") + if err != nil { + t.Fatal(err.Error()) + } + for _, v := range mr { + fmt.Printf("%v\n", v) + } + + mr, err = NewMapsFromJsonFileRaw("nil") + if err == nil { + t.Fatal("no error returned for read of nil file") + } + fmt.Println("caught error: ", err.Error()) + + mr, err = NewMapsFromJsonFileRaw("files_test.badjson") + if err == nil { + t.Fatal("no error returned for read of badjson file") + } + fmt.Println("caught error: ", err.Error()) +} + +func TestNewXmFile(t *testing.T) { + fmt.Println("NewMapsFromXmlFile()") + am, err := NewMapsFromXmlFile("files_test.xml") + if err != nil { + t.Fatal(err.Error()) + } + for _, v := range am { + fmt.Printf("%v\n", v) + } + + am, err = NewMapsFromXmlFile("nil") + if err == nil { + t.Fatal("no error returned for read of nil file") + } + fmt.Println("caught error: ", err.Error()) + + am, err = NewMapsFromXmlFile("files_test.badxml") + if err == nil { + t.Fatal("no error returned for read of badjson file") + } + fmt.Println("caught error: ", err.Error()) +} + +func TestNewXmFileRaw(t *testing.T) { + fmt.Println("NewMapsFromXmlFileRaw()") + mr, err := NewMapsFromXmlFileRaw("files_test.xml") + if err != nil { + t.Fatal(err.Error()) + } + for _, v := range mr { + fmt.Printf("%v\n", v) + } + + mr, err = NewMapsFromXmlFileRaw("nil") + if err == nil { + t.Fatal("no error returned for read of nil file") + } + fmt.Println("caught error: ", err.Error()) + + mr, err = NewMapsFromXmlFileRaw("files_test.badxml") + if err == nil { + t.Fatal("no error returned for read of badjson file") + } + fmt.Println("caught error: ", err.Error()) +} + +func TestMaps(t *testing.T) { + fmt.Println("TestMaps()") + mvs := NewMaps() + for i := 0 ; i < 2 ; i++ { + m, _ := NewMapJson([]byte(`{ "this":"is", "a":"test" }`)) + mvs = append(mvs, m) + } + fmt.Println("mvs:", mvs) + + s, _ := mvs.JsonString() + fmt.Println("JsonString():", s) + + s, _ = mvs.JsonStringIndent("", " ") + fmt.Println("JsonStringIndent():", s) + + s, _ = mvs.XmlString() + fmt.Println("XmlString():", s) + + s, _ = mvs.XmlStringIndent("", " ") + fmt.Println("XmlStringIndent():", s) +} + +func TestJsonFile(t *testing.T) { + am, err := NewMapsFromJsonFile("files_test.json") + if err != nil { + t.Fatal(err.Error()) + } + for _, v := range am { + fmt.Printf("%v\n", v) + } + + err = am.JsonFile("files_test_dup.json") + if err != nil { + t.Fatal(err.Error()) + } + fmt.Println("files_test_dup.json written") + + err = am.JsonFileIndent("files_test_indent.json","", " ") + if err != nil { + t.Fatal(err.Error()) + } + fmt.Println("files_test_indent.json written") +} + +func TestXmlFile(t *testing.T) { + am, err := NewMapsFromXmlFile("files_test.xml") + if err != nil { + t.Fatal(err.Error()) + } + for _, v := range am { + fmt.Printf("%v\n", v) + } + + err = am.XmlFile("files_test_dup.xml") + if err != nil { + t.Fatal(err.Error()) + } + fmt.Println("files_test_dup.xml written") + + err = am.XmlFileIndent("files_test_indent.xml","", " ") + if err != nil { + t.Fatal(err.Error()) + } + fmt.Println("files_test_indent.xml written") +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.json b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.json new file mode 100644 index 000000000..e9a3ddf40 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.json @@ -0,0 +1,2 @@ +{ "this":"is", "a":"test", "file":"for", "files_test.go":"case" } +{ "with":"just", "two":2, "JSON":"values", "true":true } diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.xml b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.xml new file mode 100644 index 000000000..65cf021fb --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test.xml @@ -0,0 +1,9 @@ + + test + for files.go + + + some + doc + test case + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_dup.json b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_dup.json new file mode 100644 index 000000000..2becb6a45 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_dup.json @@ -0,0 +1 @@ +{"a":"test","file":"for","files_test.go":"case","this":"is"}{"JSON":"values","true":true,"two":2,"with":"just"} \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_dup.xml b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_dup.xml new file mode 100644 index 000000000..aa63dda77 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_dup.xml @@ -0,0 +1 @@ +testfor files.gosomedoctest case \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_indent.json b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_indent.json new file mode 100644 index 000000000..6fde15634 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_indent.json @@ -0,0 +1,12 @@ +{ + "a": "test", + "file": "for", + "files_test.go": "case", + "this": "is" +} +{ + "JSON": "values", + "true": true, + "two": 2, + "with": "just" +} \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_indent.xml b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_indent.xml new file mode 100644 index 000000000..febee213c --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/files_test_indent.xml @@ -0,0 +1,8 @@ + + test + for files.go + + some + doc + test case + \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/j2x/j2x.go b/Godeps/_workspace/src/github.com/clbanning/mxj/j2x/j2x.go new file mode 100644 index 000000000..afb7109fb --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/j2x/j2x.go @@ -0,0 +1,179 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +// j2x.go - For (mostly) backwards compatibility with legacy j2x package. +// Wrappers for end-to-end JSON to XML transformation and value manipulation. +package j2x + + +import ( + . "github.com/clbanning/mxj" + "io" +) + +// FromJson() --> map[string]interface{} +func JsonToMap(jsonVal []byte) (map[string]interface{}, error) { + return NewMapJson(jsonVal) +} + +// interface{} --> ToJson (w/o safe encoding, default) { +func MapToJson(m map[string]interface{}, safeEncoding ...bool) ([]byte, error) { + return Map(m).Json() +} + +// FromJson() --> ToXml(). +func JsonToXml(jsonVal []byte) ([]byte, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.Xml() +} + +// FromJson() --> ToXmlWriter(). +func JsonToXmlWriter(jsonVal []byte, xmlWriter io.Writer) ([]byte, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.XmlWriterRaw(xmlWriter) +} + +// FromJsonReader() --> ToXml(). +func JsonReaderToXml(jsonReader io.Reader) ([]byte, []byte, error) { + m, jraw, err := NewMapJsonReaderRaw(jsonReader) + if err != nil { + return jraw, nil, err + } + x, xerr := m.Xml() + return jraw, x, xerr +} + +// FromJsonReader() --> ToXmlWriter(). Handy for transforming bulk message sets. +func JsonReaderToXmlWriter(jsonReader io.Reader, xmlWriter io.Writer) ([]byte, []byte, error) { + m, jraw, err := NewMapJsonReaderRaw(jsonReader) + if err != nil { + return jraw, nil, err + } + xraw, xerr := m.XmlWriterRaw(xmlWriter) + return jraw, xraw, xerr +} + +// JSON wrappers for Map methods implementing key path and value functions. + +// Wrap PathsForKey for JSON. +func JsonPathsForKey(jsonVal []byte, key string) ([]string, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + paths := m.PathsForKey(key) + return paths, nil +} + +// Wrap PathForKeyShortest for JSON. +func JsonPathForKeyShortest(jsonVal []byte, key string) (string, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return "", err + } + path := m.PathForKeyShortest(key) + return path, nil +} + +// Wrap ValuesForKey for JSON. +func JsonValuesForKey(jsonVal []byte, key string, subkeys ...string) ([]interface{}, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.ValuesForKey(key, subkeys...) +} + +// Wrap ValuesForKeyPath for JSON. +func JsonValuesForKeyPath(jsonVal []byte, path string, subkeys ...string) ([]interface{}, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.ValuesForPath(path, subkeys...) +} + +// Wrap UpdateValuesForPath for JSON +// 'jsonVal' is XML value +// 'newKeyValue' is the value to replace an existing value at the end of 'path' +// 'path' is the dot-notation path with the key whose value is to be replaced at the end +// (can include wildcard character, '*') +// 'subkeys' are key:value pairs of key:values that must match for the key +func JsonUpdateValsForPath(jsonVal []byte, newKeyValue interface{}, path string, subkeys ...string) ([]byte, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + _, err = m.UpdateValuesForPath(newKeyValue, path, subkeys...) + if err != nil { + return nil, err + } + return m.Json() +} + +// Wrap NewMap for JSON and return as JSON +// 'jsonVal' is an JSON value +// 'keypairs' are "oldKey:newKey" values that conform to 'keypairs' in (Map)NewMap. +func JsonNewJson(jsonVal []byte, keypairs ...string) ([]byte, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + n, err := m.NewMap(keypairs...) + if err != nil { + return nil, err + } + return n.Json() +} + +// Wrap NewMap for JSON and return as XML +// 'jsonVal' is an JSON value +// 'keypairs' are "oldKey:newKey" values that conform to 'keypairs' in (Map)NewMap. +func JsonNewXml(jsonVal []byte, keypairs ...string) ([]byte, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + n, err := m.NewMap(keypairs...) + if err != nil { + return nil, err + } + return n.Xml() +} + +// Wrap LeafNodes for JSON. +// 'jsonVal' is an JSON value +func JsonLeafNodes(jsonVal []byte) ([]LeafNode, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.LeafNodes(), nil +} + +// Wrap LeafValues for JSON. +// 'jsonVal' is an JSON value +func JsonLeafValues(jsonVal []byte) ([]interface{}, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.LeafValues(), nil +} + +// Wrap LeafPath for JSON. +// 'xmlVal' is an JSON value +func JsonLeafPath(jsonVal []byte) ([]string, error) { + m, err := NewMapJson(jsonVal) + if err != nil { + return nil, err + } + return m.LeafPaths(), nil +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/j2x/j2x_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/j2x/j2x_test.go new file mode 100644 index 000000000..098f48220 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/j2x/j2x_test.go @@ -0,0 +1,67 @@ +// thanks to Chris Malek (chris.r.malek@gmail.com) for suggestion to handle JSON list docs. + +package j2x + +import ( + "bytes" + "io/ioutil" + "fmt" + "testing" +) + +func TestJsonToXml_1(t *testing.T) { + // mimic a io.Reader + // Body := bytes.NewReader([]byte(`{"some-null-value":"", "a-non-null-value":"bar"}`)) + Body := bytes.NewReader([]byte(`[{"some-null-value":"", "a-non-null-value":"bar"}]`)) + + //body, err := ioutil.ReadAll(req.Body) + body, err := ioutil.ReadAll(Body) + if err != nil { + t.Fatal(err) + } + fmt.Println(string(body)) + //if err != nil { + // t.Fatal(err) + //} + + var xmloutput []byte + //xmloutput, err = j2x.JsonToXml(body) + xmloutput, err = JsonToXml(body) + + //log.Println(string(xmloutput)) + + if err != nil { + t.Fatal(err) + // log.Println(err) + // http.Error(rw, "Could not convert to xml", 400) + } + fmt.Println("xmloutput:", string(xmloutput)) +} + +func TestJsonToXml_2(t *testing.T) { + // mimic a io.Reader + Body := bytes.NewReader([]byte(`{"somekey":[{"value":"1st"},{"value":"2nd"}]}`)) + + //body, err := ioutil.ReadAll(req.Body) + body, err := ioutil.ReadAll(Body) + if err != nil { + t.Fatal(err) + } + fmt.Println(string(body)) + //if err != nil { + // t.Fatal(err) + //} + + var xmloutput []byte + //xmloutput, err = j2x.JsonToXml(body) + xmloutput, err = JsonToXml(body) + + //log.Println(string(xmloutput)) + + if err != nil { + t.Fatal(err) + // log.Println(err) + // http.Error(rw, "Could not convert to xml", 400) + } + fmt.Println("xmloutput:", string(xmloutput)) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/j2x_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/j2x_test.go new file mode 100644 index 000000000..4bbc7cb16 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/j2x_test.go @@ -0,0 +1,30 @@ +package mxj + +import ( + "fmt" + "testing" +) + +var jjdata = []byte(`{ "key1":"string", "key2":34, "key3":true, "key4":"unsafe: <>&", "key5":null }`) + +func TestJ2XHeader(t *testing.T) { + fmt.Println("\n---------------- j2x_test .go ...\n") +} + +func TestJ2X(t *testing.T) { + + m, err := NewMapJson(jjdata) + if err != nil { + t.Fatal("NewMapJson, err:", err) + } + + x, err := m.Xml() + if err != nil { + t.Fatal("m.Xml(), err:", err) + } + + fmt.Println("j2x, jdata:", string(jjdata)) + fmt.Println("j2x, m :", m) + fmt.Println("j2x, xml :", string(x)) +} + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/json.go b/Godeps/_workspace/src/github.com/clbanning/mxj/json.go new file mode 100644 index 000000000..b7bf03309 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/json.go @@ -0,0 +1,319 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +package mxj + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "time" +) + +// ------------------------------ write JSON ----------------------- + +// Just a wrapper on json.Marshal. +// If option safeEncoding is'true' then safe encoding of '<', '>' and '&' +// is preserved. (see encoding/json#Marshal, encoding/json#Encode) +func (mv Map) Json(safeEncoding ...bool) ([]byte, error) { + var s bool + if len(safeEncoding) == 1 { + s = safeEncoding[0] + } + + b, err := json.Marshal(mv) + + if !s { + b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1) + b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1) + b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1) + } + return b, err +} + +// Just a wrapper on json.MarshalIndent. +// If option safeEncoding is'true' then safe encoding of '<' , '>' and '&' +// is preserved. (see encoding/json#Marshal, encoding/json#Encode) +func (mv Map) JsonIndent(prefix, indent string, safeEncoding ...bool) ([]byte, error) { + var s bool + if len(safeEncoding) == 1 { + s = safeEncoding[0] + } + + b, err := json.MarshalIndent(mv, prefix, indent) + if !s { + b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1) + b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1) + b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1) + } + return b, err +} + +// The following implementation is provided for symmetry with NewMapJsonReader[Raw] +// The names will also provide a key for the number of return arguments. + +// Writes the Map as JSON on the Writer. +// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved. +func (mv Map) JsonWriter(jsonWriter io.Writer, safeEncoding ...bool) error { + b, err := mv.Json(safeEncoding...) + if err != nil { + return err + } + + _, err = jsonWriter.Write(b) + return err +} + +// Writes the Map as JSON on the Writer. []byte is the raw JSON that was written. +// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved. +func (mv Map) JsonWriterRaw(jsonWriter io.Writer, safeEncoding ...bool) ([]byte, error) { + b, err := mv.Json(safeEncoding...) + if err != nil { + return b, err + } + + _, err = jsonWriter.Write(b) + return b, err +} + +// Writes the Map as pretty JSON on the Writer. +// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved. +func (mv Map) JsonIndentWriter(jsonWriter io.Writer, prefix, indent string, safeEncoding ...bool) error { + b, err := mv.JsonIndent(prefix, indent, safeEncoding...) + if err != nil { + return err + } + + _, err = jsonWriter.Write(b) + return err +} + +// Writes the Map as pretty JSON on the Writer. []byte is the raw JSON that was written. +// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved. +func (mv Map) JsonIndentWriterRaw(jsonWriter io.Writer, prefix, indent string, safeEncoding ...bool) ([]byte, error) { + b, err := mv.JsonIndent(prefix, indent, safeEncoding...) + if err != nil { + return b, err + } + + _, err = jsonWriter.Write(b) + return b, err +} + +// --------------------------- read JSON ----------------------------- + +// Parse numeric values as json.Number types - see encoding/json#Number +var JsonUseNumber bool + +// Just a wrapper on json.Unmarshal +// Converting JSON to XML is a simple as: +// ... +// mapVal, merr := mxj.NewMapJson(jsonVal) +// if merr != nil { +// // handle error +// } +// xmlVal, xerr := mapVal.Xml() +// if xerr != nil { +// // handle error +// } +// NOTE: as a special case, passing a list, e.g., [{"some-null-value":"", "a-non-null-value":"bar"}], +// will be interpreted as having the root key 'object' prepended - {"object":[ ... ]} - to unmarshal to a Map. +// See mxj/j2x/j2x_test.go. +func NewMapJson(jsonVal []byte) (Map, error) { + // empty or nil begets empty + if len(jsonVal) == 0 { + m := make(map[string]interface{},0) + return m, nil + } + // handle a goofy case ... + if jsonVal[0] == '[' { + jsonVal = []byte(`{"object":` + string(jsonVal) + `}`) + } + m := make(map[string]interface{}) + // err := json.Unmarshal(jsonVal, &m) + buf := bytes.NewReader(jsonVal) + dec := json.NewDecoder(buf) + if JsonUseNumber { + dec.UseNumber() + } + err := dec.Decode(&m) + return m, err +} + +// Retrieve a Map value from an io.Reader. +// NOTE: The raw JSON off the reader is buffered to []byte using a ByteReader. If the io.Reader is an +// os.File, there may be significant performance impact. If the io.Reader is wrapping a []byte +// value in-memory, however, such as http.Request.Body you CAN use it to efficiently unmarshal +// a JSON object. +func NewMapJsonReader(jsonReader io.Reader) (Map, error) { + jb, err := getJson(jsonReader) + if err != nil || len(*jb) == 0 { + return nil, err + } + + // Unmarshal the 'presumed' JSON string + return NewMapJson(*jb) +} + +// Retrieve a Map value and raw JSON - []byte - from an io.Reader. +// NOTE: The raw JSON off the reader is buffered to []byte using a ByteReader. If the io.Reader is an +// os.File, there may be significant performance impact. If the io.Reader is wrapping a []byte +// value in-memory, however, such as http.Request.Body you CAN use it to efficiently unmarshal +// a JSON object and retrieve the raw JSON in a single call. +func NewMapJsonReaderRaw(jsonReader io.Reader) (Map, []byte, error) { + jb, err := getJson(jsonReader) + if err != nil || len(*jb) == 0 { + return nil, *jb, err + } + + // Unmarshal the 'presumed' JSON string + m, merr := NewMapJson(*jb) + return m, *jb, merr +} + +// Pull the next JSON string off the stream: just read from first '{' to its closing '}'. +// Returning a pointer to the slice saves 16 bytes - maybe unnecessary, but internal to package. +func getJson(rdr io.Reader) (*[]byte, error) { + bval := make([]byte, 1) + jb := make([]byte, 0) + var inQuote, inJson bool + var parenCnt int + var previous byte + + // scan the input for a matched set of {...} + // json.Unmarshal will handle syntax checking. + for { + _, err := rdr.Read(bval) + if err != nil { + if err == io.EOF && inJson && parenCnt > 0 { + return &jb, fmt.Errorf("no closing } for JSON string: %s", string(jb)) + } + return &jb, err + } + switch bval[0] { + case '{': + if !inQuote { + parenCnt++ + inJson = true + } + case '}': + if !inQuote { + parenCnt-- + } + if parenCnt < 0 { + return nil, fmt.Errorf("closing } without opening {: %s", string(jb)) + } + case '"': + if inQuote { + if previous == '\\' { + break + } + inQuote = false + } else { + inQuote = true + } + case '\n', '\r', '\t', ' ': + if !inQuote { + continue + } + } + if inJson { + jb = append(jb, bval[0]) + if parenCnt == 0 { + break + } + } + previous = bval[0] + } + + return &jb, nil +} + +// ------------------------------- JSON Reader handler via Map values ----------------------- + +// Default poll delay to keep Handler from spinning on an open stream +// like sitting on os.Stdin waiting for imput. +var jhandlerPollInterval = time.Duration(1e6) + +// While unnecessary, we make HandleJsonReader() have the same signature as HandleXmlReader(). +// This avoids treating one or other as a special case and discussing the underlying stdlib logic. + +// Bulk process JSON using handlers that process a Map value. +// 'rdr' is an io.Reader for the JSON (stream). +// 'mapHandler' is the Map processing handler. Return of 'false' stops io.Reader processing. +// 'errHandler' is the error processor. Return of 'false' stops io.Reader processing and returns the error. +// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized. +// This means that you can stop reading the file on error or after processing a particular message. +// To have reading and handling run concurrently, pass argument to a go routine in handler and return 'true'. +func HandleJsonReader(jsonReader io.Reader, mapHandler func(Map) bool, errHandler func(error) bool) error { + var n int + for { + m, merr := NewMapJsonReader(jsonReader) + n++ + + // handle error condition with errhandler + if merr != nil && merr != io.EOF { + merr = fmt.Errorf("[jsonReader: %d] %s", n, merr.Error()) + if ok := errHandler(merr); !ok { + // caused reader termination + return merr + } + continue + } + + // pass to maphandler + if len(m) != 0 { + if ok := mapHandler(m); !ok { + break + } + } else if merr != io.EOF { + <-time.After(jhandlerPollInterval) + } + + if merr == io.EOF { + break + } + } + return nil +} + +// Bulk process JSON using handlers that process a Map value and the raw JSON. +// 'rdr' is an io.Reader for the JSON (stream). +// 'mapHandler' is the Map and raw JSON - []byte - processor. Return of 'false' stops io.Reader processing. +// 'errHandler' is the error and raw JSON processor. Return of 'false' stops io.Reader processing and returns the error. +// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized. +// This means that you can stop reading the file on error or after processing a particular message. +// To have reading and handling run concurrently, pass argument(s) to a go routine in handler and return 'true'. +func HandleJsonReaderRaw(jsonReader io.Reader, mapHandler func(Map, []byte) bool, errHandler func(error, []byte) bool) error { + var n int + for { + m, raw, merr := NewMapJsonReaderRaw(jsonReader) + n++ + + // handle error condition with errhandler + if merr != nil && merr != io.EOF { + merr = fmt.Errorf("[jsonReader: %d] %s", n, merr.Error()) + if ok := errHandler(merr, raw); !ok { + // caused reader termination + return merr + } + continue + } + + // pass to maphandler + if len(m) != 0 { + if ok := mapHandler(m, raw); !ok { + break + } + } else if merr != io.EOF { + <-time.After(jhandlerPollInterval) + } + + if merr == io.EOF { + break + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/json_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/json_test.go new file mode 100644 index 000000000..7c97d2f79 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/json_test.go @@ -0,0 +1,137 @@ +package mxj + +import ( + "bytes" + "fmt" + "io" + "testing" +) + +var jdata = []byte(`{ "key1":"string", "key2":34, "key3":true, "key4":"unsafe: <>&", "key5":null }`) +var jdata2 = []byte(`{ "key1":"string", "key2":34, "key3":true, "key4":"unsafe: <>&" }, + { "key":"value in new JSON string" }`) + +func TestJsonHeader(t *testing.T) { + fmt.Println("\n---------------- json_test.go ...\n") +} + +func TestNewMapJson(t *testing.T) { + + m, merr := NewMapJson(jdata) + if merr != nil { + t.Fatal("NewMapJson, merr:", merr.Error()) + } + + fmt.Println("NewMapJson, jdata:", string(jdata)) + fmt.Printf("NewMapJson, m : %#v\n", m) +} + +func TestNewMapJsonNumber(t *testing.T) { + + JsonUseNumber = true + + m, merr := NewMapJson(jdata) + if merr != nil { + t.Fatal("NewMapJson, merr:", merr.Error()) + } + + fmt.Println("NewMapJson, jdata:", string(jdata)) + fmt.Printf("NewMapJson, m : %#v\n", m) + + JsonUseNumber = false +} + +func TestNewMapJsonError(t *testing.T) { + + m, merr := NewMapJson(jdata[:len(jdata)-2]) + if merr == nil { + t.Fatal("NewMapJsonError, m:", m) + } + + fmt.Println("NewMapJsonError, jdata :", string(jdata[:len(jdata)-2])) + fmt.Println("NewMapJsonError, merror:", merr.Error()) + + newData := []byte(`{ "this":"is", "in":error }`) + m, merr = NewMapJson(newData) + if merr == nil { + t.Fatal("NewMapJsonError, m:", m) + } + + fmt.Println("NewMapJsonError, newData :", string(newData)) + fmt.Println("NewMapJsonError, merror :", merr.Error()) +} + +func TestNewMapJsonReader(t *testing.T) { + + rdr := bytes.NewBuffer(jdata2) + + for { + m, jb, merr := NewMapJsonReaderRaw(rdr) + if merr != nil && merr != io.EOF { + t.Fatal("NewMapJsonReader, merr:", merr.Error()) + } + if merr == io.EOF { + break + } + + fmt.Println("NewMapJsonReader, jb:", string(jb)) + fmt.Printf("NewMapJsonReader, m : %#v\n", m) + } +} + +func TestNewMapJsonReaderNumber(t *testing.T) { + + JsonUseNumber = true + + rdr := bytes.NewBuffer(jdata2) + + for { + m, jb, merr := NewMapJsonReaderRaw(rdr) + if merr != nil && merr != io.EOF { + t.Fatal("NewMapJsonReader, merr:", merr.Error()) + } + if merr == io.EOF { + break + } + + fmt.Println("NewMapJsonReader, jb:", string(jb)) + fmt.Printf("NewMapJsonReader, m : %#v\n", m) + } + + JsonUseNumber = false +} + +func TestJson(t *testing.T) { + + m, _ := NewMapJson(jdata) + + j, jerr := m.Json() + if jerr != nil { + t.Fatal("Json, jerr:", jerr.Error()) + } + + fmt.Println("Json, jdata:", string(jdata)) + fmt.Println("Json, j :", string(j)) + + j, _ = m.Json(true) + fmt.Println("Json, j safe:", string(j)) +} + +func TestJsonWriter(t *testing.T) { + mv := Map(map[string]interface{}{"this": "is a", "float": 3.14159, "and": "a", "bool": true}) + + w := new(bytes.Buffer) + raw, err := mv.JsonWriterRaw(w) + if err != nil { + t.Fatal("err:", err.Error()) + } + + b := make([]byte, w.Len()) + _, err = w.Read(b) + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("JsonWriter, raw:", string(raw)) + fmt.Println("JsonWriter, b :", string(b)) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/keyvalues.go b/Godeps/_workspace/src/github.com/clbanning/mxj/keyvalues.go new file mode 100644 index 000000000..4fc951853 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/keyvalues.go @@ -0,0 +1,620 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +// keyvalues.go: Extract values from an arbitrary XML doc. Tag path can include wildcard characters. + +package mxj + +import ( + "fmt" + "strconv" + "strings" +) + +// ----------------------------- get everything FOR a single key ------------------------- + +const ( + minArraySize = 32 +) + +var defaultArraySize int = minArraySize + +// Adjust the buffers for expected number of values to return from ValuesForKey() and ValuesForPath(). +// This can have the effect of significantly reducing memory allocation-copy functions for large data sets. +// Returns the initial buffer size. +func SetArraySize(size int) int { + if size > minArraySize { + defaultArraySize = size + } else { + defaultArraySize = minArraySize + } + return defaultArraySize +} + +// Return all values in Map, 'mv', associated with a 'key'. If len(returned_values) == 0, then no match. +// On error, the returned array is 'nil'. NOTE: 'key' can be wildcard, "*". +// 'subkeys' (optional) are "key:val[:type]" strings representing attributes or elements in a list. +// - By default 'val' is of type string. "key:val:bool" and "key:val:float" to coerce them. +// - For attributes prefix the label with a hyphen, '-', e.g., "-seq:3". +// - If the 'key' refers to a list, then "key:value" could select a list member of the list. +// - The subkey can be wildcarded - "key:*" - to require that it's there with some value. +// - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an +// exclusion critera - e.g., "!author:William T. Gaddis". +func (mv Map) ValuesForKey(key string, subkeys ...string) ([]interface{}, error) { + m := map[string]interface{}(mv) + var subKeyMap map[string]interface{} + if len(subkeys) > 0 { + var err error + subKeyMap, err = getSubKeyMap(subkeys...) + if err != nil { + return nil, err + } + } + + ret := make([]interface{}, 0, defaultArraySize) + var cnt int + hasKey(m, key, &ret, &cnt, subKeyMap) + return ret[:cnt], nil + // ret := make([]interface{}, 0) + // hasKey(m, key, &ret, subKeyMap) + // return ret, nil +} + +// hasKey - if the map 'key' exists append it to array +// if it doesn't do nothing except scan array and map values +func hasKey(iv interface{}, key string, ret *[]interface{}, cnt *int, subkeys map[string]interface{}) { + // func hasKey(iv interface{}, key string, ret *[]interface{}, subkeys map[string]interface{}) { + switch iv.(type) { + case map[string]interface{}: + vv := iv.(map[string]interface{}) + // see if the current value is of interest + if v, ok := vv[key]; ok { + switch v.(type) { + case map[string]interface{}: + if hasSubKeys(v, subkeys) { + *ret = append(*ret, v) + *cnt++ + } + case []interface{}: + for _, av := range v.([]interface{}) { + if hasSubKeys(av, subkeys) { + *ret = append(*ret, av) + *cnt++ + } + } + default: + if len(subkeys) == 0 { + *ret = append(*ret, v) + *cnt++ + } + } + } + + // wildcard case + if key == "*" { + for _, v := range vv { + switch v.(type) { + case map[string]interface{}: + if hasSubKeys(v, subkeys) { + *ret = append(*ret, v) + *cnt++ + } + case []interface{}: + for _, av := range v.([]interface{}) { + if hasSubKeys(av, subkeys) { + *ret = append(*ret, av) + *cnt++ + } + } + default: + if len(subkeys) == 0 { + *ret = append(*ret, v) + *cnt++ + } + } + } + } + + // scan the rest + for _, v := range vv { + hasKey(v, key, ret, cnt, subkeys) + // hasKey(v, key, ret, subkeys) + } + case []interface{}: + for _, v := range iv.([]interface{}) { + hasKey(v, key, ret, cnt, subkeys) + // hasKey(v, key, ret, subkeys) + } + } +} + +// ----------------------- get everything for a node in the Map --------------------------- + +// Allow indexed arrays in "path" specification. (Request from Abhijit Kadam - abhijitk100@gmail.com.) +// 2014.04.28 - implementation note. +// Implemented as a wrapper of (old)ValuesForPath() because we need look-ahead logic to handle expansion +// of wildcards and unindexed arrays. Embedding such logic into valuesForKeyPath() would have made the +// code much more complicated; this wrapper is straightforward, easy to debug, and doesn't add significant overhead. + +// Retrieve all values for a path from the Map. If len(returned_values) == 0, then no match. +// On error, the returned array is 'nil'. +// 'path' is a dot-separated path of key values. +// - If a node in the path is '*', then everything beyond is walked. +// - 'path' can contain indexed array references, such as, "*.data[1]" and "msgs[2].data[0].field" - +// even "*[2].*[0].field". +// 'subkeys' (optional) are "key:val[:type]" strings representing attributes or elements in a list. +// - By default 'val' is of type string. "key:val:bool" and "key:val:float" to coerce them. +// - For attributes prefix the label with a hyphen, '-', e.g., "-seq:3". +// - If the 'path' refers to a list, then "tag:value" would return member of the list. +// - The subkey can be wildcarded - "key:*" - to require that it's there with some value. +// - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an +// exclusion critera - e.g., "!author:William T. Gaddis". +func (mv Map) ValuesForPath(path string, subkeys ...string) ([]interface{}, error) { + // If there are no array indexes in path, use legacy ValuesForPath() logic. + if strings.Index(path, "[") < 0 { + return mv.oldValuesForPath(path, subkeys...) + } + + var subKeyMap map[string]interface{} + if len(subkeys) > 0 { + var err error + subKeyMap, err = getSubKeyMap(subkeys...) + if err != nil { + return nil, err + } + } + + keys, kerr := parsePath(path) + if kerr != nil { + return nil, kerr + } + + vals, verr := valuesForArray(keys, mv) + if verr != nil { + return nil, verr // Vals may be nil, but return empty array. + } + + // Need to handle subkeys ... only return members of vals that satisfy conditions. + retvals := make([]interface{}, 0) + for _, v := range vals { + if hasSubKeys(v, subKeyMap) { + retvals = append(retvals, v) + } + } + return retvals, nil +} + +func valuesForArray(keys []*key, m Map) ([]interface{}, error) { + var tmppath string + var haveFirst bool + var vals []interface{} + var verr error + + lastkey := len(keys) - 1 + for i := 0; i <= lastkey; i++ { + if !haveFirst { + tmppath = keys[i].name + haveFirst = true + } else { + tmppath += "." + keys[i].name + } + + // Look-ahead: explode wildcards and unindexed arrays. + // Need to handle un-indexed list recursively: + // e.g., path is "stuff.data[0]" rather than "stuff[0].data[0]". + // Need to treat it as "stuff[0].data[0]", "stuff[1].data[0]", ... + if !keys[i].isArray && i < lastkey && keys[i+1].isArray { + // Can't pass subkeys because we may not be at literal end of path. + vv, vverr := m.oldValuesForPath(tmppath) + if vverr != nil { + return nil, vverr + } + for _, v := range vv { + // See if we can walk the value. + am, ok := v.(map[string]interface{}) + if !ok { + continue + } + // Work the backend. + nvals, nvalserr := valuesForArray(keys[i+1:], Map(am)) + if nvalserr != nil { + return nil, nvalserr + } + vals = append(vals, nvals...) + } + break // have recursed the whole path - return + } + + if keys[i].isArray || i == lastkey { + // Don't pass subkeys because may not be at literal end of path. + vals, verr = m.oldValuesForPath(tmppath) + } else { + continue + } + if verr != nil { + return nil, verr + } + + if i == lastkey && !keys[i].isArray { + break + } + + // Now we're looking at an array - supposedly. + // Is index in range of vals? + if len(vals) <= keys[i].position { + vals = nil + break + } + + // Return the array member of interest, if at end of path. + if i == lastkey { + vals = vals[keys[i].position:(keys[i].position + 1)] + break + } + + // Extract the array member of interest. + am := vals[keys[i].position:(keys[i].position + 1)] + + // must be a map[string]interface{} value so we can keep walking the path + amm, ok := am[0].(map[string]interface{}) + if !ok { + vals = nil + break + } + + m = Map(amm) + haveFirst = false + } + + return vals, nil +} + +type key struct { + name string + isArray bool + position int +} + +func parsePath(s string) ([]*key, error) { + keys := strings.Split(s, ".") + + ret := make([]*key, 0) + + for i := 0; i < len(keys); i++ { + if keys[i] == "" { + continue + } + + newkey := new(key) + if strings.Index(keys[i], "[") < 0 { + newkey.name = keys[i] + ret = append(ret, newkey) + continue + } + + p := strings.Split(keys[i], "[") + newkey.name = p[0] + p = strings.Split(p[1], "]") + if p[0] == "" { // no right bracket + return nil, fmt.Errorf("no right bracket on key index: %s", keys[i]) + } + // convert p[0] to a int value + pos, nerr := strconv.ParseInt(p[0], 10, 32) + if nerr != nil { + return nil, fmt.Errorf("cannot convert index to int value: %s", p[0]) + } + newkey.position = int(pos) + newkey.isArray = true + ret = append(ret, newkey) + } + + return ret, nil +} + +// legacy ValuesForPath() - now wrapped to handle special case of indexed arrays in 'path'. +func (mv Map) oldValuesForPath(path string, subkeys ...string) ([]interface{}, error) { + m := map[string]interface{}(mv) + var subKeyMap map[string]interface{} + if len(subkeys) > 0 { + var err error + subKeyMap, err = getSubKeyMap(subkeys...) + if err != nil { + return nil, err + } + } + + keys := strings.Split(path, ".") + if keys[len(keys)-1] == "" { + keys = keys[:len(keys)-1] + } + // ivals := make([]interface{}, 0) + // valuesForKeyPath(&ivals, m, keys, subKeyMap) + // return ivals, nil + ivals := make([]interface{}, 0, defaultArraySize) + var cnt int + valuesForKeyPath(&ivals, &cnt, m, keys, subKeyMap) + return ivals[:cnt], nil +} + +func valuesForKeyPath(ret *[]interface{}, cnt *int, m interface{}, keys []string, subkeys map[string]interface{}) { + lenKeys := len(keys) + + // load 'm' values into 'ret' + // expand any lists + if lenKeys == 0 { + switch m.(type) { + case map[string]interface{}: + if subkeys != nil { + if ok := hasSubKeys(m, subkeys); !ok { + return + } + } + *ret = append(*ret, m) + *cnt++ + case []interface{}: + for i, v := range m.([]interface{}) { + if subkeys != nil { + if ok := hasSubKeys(v, subkeys); !ok { + continue // only load list members with subkeys + } + } + *ret = append(*ret, (m.([]interface{}))[i]) + *cnt++ + } + default: + if subkeys != nil { + return // must be map[string]interface{} if there are subkeys + } + *ret = append(*ret, m) + *cnt++ + } + return + } + + // key of interest + key := keys[0] + switch key { + case "*": // wildcard - scan all values + switch m.(type) { + case map[string]interface{}: + for _, v := range m.(map[string]interface{}) { + // valuesForKeyPath(ret, v, keys[1:], subkeys) + valuesForKeyPath(ret, cnt, v, keys[1:], subkeys) + } + case []interface{}: + for _, v := range m.([]interface{}) { + switch v.(type) { + // flatten out a list of maps - keys are processed + case map[string]interface{}: + for _, vv := range v.(map[string]interface{}) { + // valuesForKeyPath(ret, vv, keys[1:], subkeys) + valuesForKeyPath(ret, cnt, vv, keys[1:], subkeys) + } + default: + // valuesForKeyPath(ret, v, keys[1:], subkeys) + valuesForKeyPath(ret, cnt, v, keys[1:], subkeys) + } + } + } + default: // key - must be map[string]interface{} + switch m.(type) { + case map[string]interface{}: + if v, ok := m.(map[string]interface{})[key]; ok { + // valuesForKeyPath(ret, v, keys[1:], subkeys) + valuesForKeyPath(ret, cnt, v, keys[1:], subkeys) + } + case []interface{}: // may be buried in list + for _, v := range m.([]interface{}) { + switch v.(type) { + case map[string]interface{}: + if vv, ok := v.(map[string]interface{})[key]; ok { + // valuesForKeyPath(ret, vv, keys[1:], subkeys) + valuesForKeyPath(ret, cnt, vv, keys[1:], subkeys) + } + } + } + } + } +} + +// hasSubKeys() - interface{} equality works for string, float64, bool +// 'v' must be a map[string]interface{} value to have subkeys +// 'a' can have k:v pairs with v.(string) == "*", which is treated like a wildcard. +func hasSubKeys(v interface{}, subkeys map[string]interface{}) bool { + if len(subkeys) == 0 { + return true + } + + switch v.(type) { + case map[string]interface{}: + // do all subKey name:value pairs match? + mv := v.(map[string]interface{}) + for skey, sval := range subkeys { + isNotKey := false + if skey[:1] == "!" { // a NOT-key + skey = skey[1:] + isNotKey = true + } + vv, ok := mv[skey] + if !ok { // key doesn't exist + if isNotKey { // key not there, but that's what we want + if kv, ok := sval.(string); ok && kv == "*" { + continue + } + } + return false + } + // wildcard check + if kv, ok := sval.(string); ok && kv == "*" { + if isNotKey { // key is there, and we don't want it + return false + } + continue + } + switch sval.(type) { + case string: + if s, ok := vv.(string); ok && s == sval.(string) { + if isNotKey { + return false + } + continue + } + case bool: + if b, ok := vv.(bool); ok && b == sval.(bool) { + if isNotKey { + return false + } + continue + } + case float64: + if f, ok := vv.(float64); ok && f == sval.(float64) { + if isNotKey { + return false + } + continue + } + } + // key there but didn't match subkey value + if isNotKey { // that's what we want + continue + } + return false + } + // all subkeys matched + return true + } + + // not a map[string]interface{} value, can't have subkeys + return false +} + +// Generate map of key:value entries as map[string]string. +// 'kv' arguments are "name:value" pairs: attribute keys are designated with prepended hyphen, '-'. +// If len(kv) == 0, the return is (nil, nil). +func getSubKeyMap(kv ...string) (map[string]interface{}, error) { + if len(kv) == 0 { + return nil, nil + } + m := make(map[string]interface{}, 0) + for _, v := range kv { + vv := strings.Split(v, ":") + switch len(vv) { + case 2: + m[vv[0]] = interface{}(vv[1]) + case 3: + switch vv[3] { + case "string", "char", "text": + m[vv[0]] = interface{}(vv[1]) + case "bool", "boolean": + // ParseBool treats "1"==true & "0"==false + b, err := strconv.ParseBool(vv[1]) + if err != nil { + return nil, fmt.Errorf("can't convert subkey value to bool: %s", vv[1]) + } + m[vv[0]] = interface{}(b) + case "float", "float64", "num", "number", "numeric": + f, err := strconv.ParseFloat(vv[1], 64) + if err != nil { + return nil, fmt.Errorf("can't convert subkey value to float: %s", vv[1]) + } + m[vv[0]] = interface{}(f) + default: + return nil, fmt.Errorf("unknown subkey conversion spec: %s", v) + } + default: + return nil, fmt.Errorf("unknown subkey spec: %s", v) + } + } + return m, nil +} + +// ------------------------------- END of valuesFor ... ---------------------------- + +// ----------------------- locate where a key value is in the tree ------------------- + +//----------------------------- find all paths to a key -------------------------------- + +// Get all paths through Map, 'mv', (in dot-notation) that terminate with the specified key. +// Results can be used with ValuesForPath. +func (mv Map) PathsForKey(key string) []string { + m := map[string]interface{}(mv) + breadbasket := make(map[string]bool, 0) + breadcrumbs := "" + + hasKeyPath(breadcrumbs, m, key, breadbasket) + if len(breadbasket) == 0 { + return nil + } + + // unpack map keys to return + res := make([]string, len(breadbasket)) + var i int + for k, _ := range breadbasket { + res[i] = k + i++ + } + + return res +} + +// Extract the shortest path from all possible paths - from PathsForKey() - in Map, 'mv'.. +// Paths are strings using dot-notation. +func (mv Map) PathForKeyShortest(key string) string { + paths := mv.PathsForKey(key) + + lp := len(paths) + if lp == 0 { + return "" + } + if lp == 1 { + return paths[0] + } + + shortest := paths[0] + shortestLen := len(strings.Split(shortest, ".")) + + for i := 1; i < len(paths); i++ { + vlen := len(strings.Split(paths[i], ".")) + if vlen < shortestLen { + shortest = paths[i] + shortestLen = vlen + } + } + + return shortest +} + +// hasKeyPath - if the map 'key' exists append it to KeyPath.path and increment KeyPath.depth +// This is really just a breadcrumber that saves all trails that hit the prescribed 'key'. +func hasKeyPath(crumbs string, iv interface{}, key string, basket map[string]bool) { + switch iv.(type) { + case map[string]interface{}: + vv := iv.(map[string]interface{}) + if _, ok := vv[key]; ok { + if crumbs == "" { + crumbs = key + } else { + crumbs += "." + key + } + // *basket = append(*basket, crumb) + basket[crumbs] = true + } + // walk on down the path, key could occur again at deeper node + for k, v := range vv { + // create a new breadcrumb, intialized with the one we have + var nbc string + if crumbs == "" { + nbc = k + } else { + nbc = crumbs + "." + k + } + hasKeyPath(nbc, v, key, basket) + } + case []interface{}: + // crumb-trail doesn't change, pass it on + for _, v := range iv.([]interface{}) { + hasKeyPath(crumbs, v, key, basket) + } + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/keyvalues_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/keyvalues_test.go new file mode 100644 index 000000000..a43bcb803 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/keyvalues_test.go @@ -0,0 +1,412 @@ +// keyvalues_test.go - test keyvalues.go methods + +package mxj + +import ( + // "bytes" + "fmt" + // "io" + "testing" +) + +func TestKVHeader(t *testing.T) { + fmt.Println("\n---------------- keyvalues_test.go ...\n") +} + +var doc1 = []byte(` + + + + William T. Gaddis + The Recognitions + One of the great seminal American novels of the 20th century. + + + Austin Tappan Wright + Islandia + An example of earlier 20th century American utopian fiction. + + + John Hawkes + The Beetle Leg + A lyrical novel about the construction of Ft. Peck Dam in Montana. + + + + T.E. + Porter + + King's Day + A magical novella. + + + +`) + +var doc2 = []byte(` + + + + William T. Gaddis + The Recognitions + One of the great seminal American novels of the 20th century. + + + Something else. + +`) + +// the basic demo/test case - a small bibliography with mixed element types +func TestPathsForKey(t *testing.T) { + fmt.Println("PathsForKey, doc1 ...") + m, merr := NewMapXml(doc1) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println("PathsForKey, doc1#author") + ss := m.PathsForKey("author") + fmt.Println("... ss:", ss) + + fmt.Println("PathsForKey, doc1#books") + ss = m.PathsForKey("books") + fmt.Println("... ss:", ss) + + fmt.Println("PathsForKey, doc2 ...") + m, merr = NewMapXml(doc2) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println("PathForKey, doc2#book") + ss = m.PathsForKey("book") + fmt.Println("... ss:", ss) + + fmt.Println("PathForKeyShortest, doc2#book") + s := m.PathForKeyShortest("book") + fmt.Println("... s :", s) +} + +func TestValuesForKey(t *testing.T) { + fmt.Println("ValuesForKey ...") + m, merr := NewMapXml(doc1) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println("ValuesForKey, doc1#author") + ss, sserr := m.ValuesForKey("author") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForKey, doc1#book") + ss, sserr = m.ValuesForKey("book") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForKey, doc1#book,-seq:3") + ss, sserr = m.ValuesForKey("book", "-seq:3") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForKey, doc1#book, author:William T. Gaddis") + ss, sserr = m.ValuesForKey("book", "author:William T. Gaddis") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForKey, doc1#author, -seq:1") + ss, sserr = m.ValuesForKey("author", "-seq:1") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { // should be len(ss) == 0 + fmt.Println("... ss.v:", v) + } +} + +func TestValuesForPath(t *testing.T) { + fmt.Println("ValuesForPath ...") + m, merr := NewMapXml(doc1) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println("ValuesForPath, doc.books.book.author") + ss, sserr := m.ValuesForPath("doc.books.book.author") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForPath, doc.books.book") + ss, sserr = m.ValuesForPath("doc.books.book") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForPath, doc.books.book -seq=3") + ss, sserr = m.ValuesForPath("doc.books.book", "-seq:3") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForPath, doc.books.* -seq=3") + ss, sserr = m.ValuesForPath("doc.books.*", "-seq:3") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForPath, doc.*.* -seq=3") + ss, sserr = m.ValuesForPath("doc.*.*", "-seq:3") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } +} + +func TestValuesForNotKey( t *testing.T) { + fmt.Println("ValuesForNotKey ...") + m, merr := NewMapXml(doc1) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println("ValuesForPath, doc.books.book !author:William T. Gaddis") + ss, sserr := m.ValuesForPath("doc.books.book", "!author:William T. Gaddis") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForPath, doc.books.book !author:*") + ss, sserr = m.ValuesForPath("doc.books.book", "!author:*") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { // expect len(ss) == 0 + fmt.Println("... ss.v:", v) + } + + fmt.Println("ValuesForPath, doc.books.book !unknown:*") + ss, sserr = m.ValuesForPath("doc.books.book", "!unknown:*") + if sserr != nil { + t.Fatal("sserr:", sserr.Error()) + } + for _, v := range ss { + fmt.Println("... ss.v:", v) + } +} + +func TestIAHeader(t *testing.T) { + fmt.Println("\n---------------- indexedarray_test.go ...\n") +} + +var ak_data = []byte(`{ "section1":{"data" : [{"F1" : "F1 data","F2" : "F2 data"},{"F1" : "demo 123","F2" : "abc xyz"}]}}`) +var j_data = []byte(`{ "stuff":[ { "data":[ { "F":1 }, { "F":2 }, { "F":3 } ] }, { "data":[ 4, 5, 6 ] } ] }`) +var x_data = []byte(` + + + + 1 + + + 2 + + + 3 + + + + + 4 + + + 5 + + + 6 + + +`) + +func TestValuesForIndexedArray(t *testing.T) { + j_main(t) + x_main(t) + ak_main(t) +} + +func ak_main(t *testing.T) { + fmt.Println("\nak_data:", string(ak_data)) + m, merr := NewMapJson(ak_data) + if merr != nil { + t.Fatal("merr:", merr.Error()) + return + } + fmt.Println("m:", m) + + v, verr := m.ValuesForPath("section1.data[0].F1") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("section1.data[0].F1:", v) +} + +func j_main(t *testing.T) { + fmt.Println("j_data:", string(j_data)) + m, merr := NewMapJson(j_data) + if merr != nil { + t.Fatal("merr:", merr.Error()) + return + } + fmt.Println("m:", m) + + v, verr := m.ValuesForPath("stuff[0]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff[0]:", v) + + v, verr = m.ValuesForPath("stuff.data") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff.data:", v) + + v, verr = m.ValuesForPath("stuff[0].data") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff[0].data:", v) + + v, verr = m.ValuesForPath("stuff.data[0]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff.data[0]:", v) + + v, verr = m.ValuesForPath("stuff.*[2]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff.*[2]:", v) + + v, verr = m.ValuesForPath("stuff.data.F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff.data.F:", v) + + v, verr = m.ValuesForPath("*.*.F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("*.*.F:", v) + + v, verr = m.ValuesForPath("stuff.data[0].F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff.data[0].F:", v) + + v, verr = m.ValuesForPath("stuff.data[1].F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff.data[1].F:", v) + + v, verr = m.ValuesForPath("stuff[0].data[2]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff[0].data[2]:", v) + + v, verr = m.ValuesForPath("stuff[1].data[1]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff[1].data[1]:", v) + + v, verr = m.ValuesForPath("stuff[1].data[1].F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff[1].data[1].F", v) + + v, verr = m.ValuesForPath("stuff[1].data.F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("stuff[1].data.F:", v) +} + +func x_main(t *testing.T) { + fmt.Println("\nx_data:", string(x_data)) + m, merr := NewMapXml(x_data) + if merr != nil { + t.Fatal("merr:", merr.Error()) + return + } + fmt.Println("m:", m) + + v, verr := m.ValuesForPath("doc.stuff[0]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("doc.stuff[0]:", v) + + v, verr = m.ValuesForPath("doc.stuff.data[0]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("doc.stuff.data[0]:", v) + + v, verr = m.ValuesForPath("doc.stuff.data[0]", "-seq:2.1") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("doc.stuff.data[0] -seq:2.1:", v) + + v, verr = m.ValuesForPath("doc.stuff.data[0].F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("doc.stuff.data[0].F:", v) + + v, verr = m.ValuesForPath("doc.stuff[0].data[2]") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("doc.stuff[0].data[2]:", v) + + v, verr = m.ValuesForPath("doc.stuff[1].data[1].F") + if verr != nil { + t.Fatal("verr:", verr.Error()) + } + fmt.Println("doc.stuff[1].data[1].F:", v) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/leafnode.go b/Godeps/_workspace/src/github.com/clbanning/mxj/leafnode.go new file mode 100644 index 000000000..bbed685d0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/leafnode.go @@ -0,0 +1,82 @@ +package mxj + +// leafnode.go - return leaf nodes with paths and values for the Map +// inspired by: https://groups.google.com/forum/#!topic/golang-nuts/3JhuVKRuBbw + +import ( + "strconv" +) + +const ( + NoAttributes = true // suppress LeafNode values that are attributes +) + +// LeafNode - a terminal path value in a Map. +// For XML Map values it represents an attribute or simple element value - of type +// string unless Map was created using Cast flag. For JSON Map values it represents +// a string, numeric, boolean, or null value. +type LeafNode struct { + Path string // a dot-notation representation of the path with array subscripting + Value interface{} // the value at the path termination +} + +// LeafNodes - returns an array of all LeafNode values for the Map. +// The option no_attr argument suppresses attribute values (keys with prepended hyphen, '-') +// as well as the "#text" key for the associated simple element value. +func (mv Map)LeafNodes(no_attr ...bool) []LeafNode { + var a bool + if len(no_attr) == 1 { + a = no_attr[0] + } + + l := make([]LeafNode, 0) + getLeafNodes("", "", map[string]interface{}(mv), &l, a) + return l +} + +func getLeafNodes(path, node string, mv interface{}, l *[]LeafNode, noattr bool) { + // if stripping attributes, then also strip "#text" key + if !noattr || node != "#text" { + if path != "" && node[:1] != "[" { + path += "." + } + path += node + } + switch mv.(type) { + case map[string]interface{}: + for k, v := range mv.(map[string]interface{}) { + if noattr && k[:1] == "-" { + continue + } + getLeafNodes(path, k, v, l, noattr) + } + case []interface{}: + for i, v := range mv.([]interface{}) { + getLeafNodes(path, "["+strconv.Itoa(i)+"]", v, l, noattr) + } + default: + // can't walk any further, so create leaf + n := LeafNode{path, mv} + *l = append(*l, n) + } +} + +// LeafPaths - all paths that terminate in LeafNode values. +func (mv Map) LeafPaths(no_attr ...bool) []string { + ln := mv.LeafNodes() + ss := make([]string,len(ln)) + for i := 0 ; i < len(ln); i++ { + ss[i] = ln[i].Path + } + return ss +} + +// LeafValues - all terminal values in the Map. +func (mv Map) LeafValues(no_attr ...bool) []interface{} { + ln := mv.LeafNodes() + vv := make([]interface{},len(ln)) + for i := 0 ; i < len(ln); i++ { + vv[i] = ln[i].Value + } + return vv +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/leafnode_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/leafnode_test.go new file mode 100644 index 000000000..7bf59b781 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/leafnode_test.go @@ -0,0 +1,98 @@ +package mxj + +import ( + "fmt" + "testing" +) + +func TestLNHeader(t *testing.T) { + fmt.Println("\n---------------- leafnode_test.go ...") +} + +func TestLeafNodes(t *testing.T) { + json1 := []byte(`{ + "friends": [ + { + "skills": [ + 44, 12 + ] + } + ] + }`) + + json2 := []byte(`{ + "friends": + { + "skills": [ + 44, 12 + ] + } + + }`) + + m, _ := NewMapJson(json1) + ln := m.LeafNodes() + fmt.Println("\njson1-LeafNodes:") + for _, v := range ln { + fmt.Printf("%#v\n", v) + } + p := m.LeafPaths() + fmt.Println("\njson1-LeafPaths:") + for _, v := range p { + fmt.Printf("%#v\n", v) + } + + m, _ = NewMapJson(json2) + ln = m.LeafNodes() + fmt.Println("\njson2-LeafNodes:") + for _, v := range ln { + fmt.Printf("%#v\n", v) + } + v := m.LeafValues() + fmt.Println("\njson1-LeafValues:") + for _, v := range v { + fmt.Printf("%#v\n", v) + } + + json3 := []byte(`{ "a":"list", "of":["data", "of", 3, "types", true]}`) + m, _ = NewMapJson(json3) + ln = m.LeafNodes() + fmt.Println("\njson3-LeafNodes:") + for _, v := range ln { + fmt.Printf("%#v\n", v) + } + v = m.LeafValues() + fmt.Println("\njson3-LeafValues:") + for _, v := range v { + fmt.Printf("%#v\n", v) + } + p = m.LeafPaths() + fmt.Println("\njson3-LeafPaths:") + for _, v := range p { + fmt.Printf("%#v\n", v) + } + + + xmldata2 := []byte(` + + Item 2 is blue + + + + + `) + m, err := NewMapXml(xmldata2) + if err != nil { + t.Fatal(err.Error()) + } + fmt.Println("\nxml2data2-LeafValues:") + ln = m.LeafNodes() + for _, v := range ln { + fmt.Printf("%#v\n", v) + } + fmt.Println("\nxml2data2-LeafValues(NoAttributes):") + ln = m.LeafNodes(NoAttributes) + for _, v := range ln { + fmt.Printf("%#v\n", v) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/mxj.go b/Godeps/_workspace/src/github.com/clbanning/mxj/mxj.go new file mode 100644 index 000000000..01a6754ef --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/mxj.go @@ -0,0 +1,107 @@ +// mxj - A collection of map[string]interface{} and associated XML and JSON utilities. +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +package mxj + +import ( + "fmt" + "strconv" +) + +const ( + Cast = true // for clarity - e.g., mxj.NewMapXml(doc, mxj.Cast) + SafeEncoding = true // ditto - e.g., mv.Json(mxj.SafeEncoding) +) + +type Map map[string]interface{} + +// Allocate a Map. +func New() Map { + m := make(map[string]interface{}, 0) + return m +} + +// Cast a Map to map[string]interface{} +func (mv Map) Old() map[string]interface{} { + return mv +} + +// Return a copy of mv as a newly allocated Map. If the Map only contains string, +// numeric, map[string]interface{}, and []interface{} values, then it can be thought +// of as a "deep copy." Copying a structure (or structure reference) value is subject +// to the noted restrictions. +// NOTE: If 'mv' includes structure values with, possibly, JSON encoding tags +// then only public fields of the structure are in the new Map - and with +// keys that conform to any encoding tag instructions. The structure itself will +// be represented as a map[string]interface{} value. +func (mv Map) Copy() (Map, error) { + // this is the poor-man's deep copy + // not efficient, but it works + j, jerr := mv.Json() + // must handle, we don't know how mv got built + if jerr != nil { + return nil, jerr + } + return NewMapJson(j) +} + +// --------------- StringIndent ... from x2j.WriteMap ------------- + +// Pretty print a Map. +func (mv Map) StringIndent(offset ...int) string { + return writeMap(map[string]interface{}(mv), offset...) +} + +// writeMap - dumps the map[string]interface{} for examination. +// 'offset' is initial indentation count; typically: Write(m). +func writeMap(m interface{}, offset ...int) string { + var indent int + if len(offset) == 1 { + indent = offset[0] + } + + var s string + switch m.(type) { + case nil: + return "[nil] nil" + case string: + return "[string] " + m.(string) + case float64: + return "[float64] " + strconv.FormatFloat(m.(float64), 'e', 2, 64) + case bool: + return "[bool] " + strconv.FormatBool(m.(bool)) + case []interface{}: + s += "[[]interface{}]" + for i, v := range m.([]interface{}) { + s += "\n" + for i := 0; i < indent; i++ { + s += " " + } + s += "[item: " + strconv.FormatInt(int64(i), 10) + "]" + switch v.(type) { + case string, float64, bool: + s += "\n" + default: + // noop + } + for i := 0; i < indent; i++ { + s += " " + } + s += writeMap(v, indent+1) + } + case map[string]interface{}: + for k, v := range m.(map[string]interface{}) { + s += "\n" + for i := 0; i < indent; i++ { + s += " " + } + s += k + " :" + writeMap(v, indent+1) + } + default: + // shouldn't ever be here ... + s += fmt.Sprintf("[unknown] %#v", m) + } + return s +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/mxj_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/mxj_test.go new file mode 100644 index 000000000..a0295b5ec --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/mxj_test.go @@ -0,0 +1,38 @@ +package mxj + +import ( + "fmt" + "testing" +) + +func TestMxjHeader(t *testing.T) { + fmt.Println("\n---------------- mxj_test.go ...\n") +} + +func TestMap(t *testing.T) { + m := New() + + m["key"] = interface{}("value") + v := map[string]interface{}{ "bool":true, "float":3.14159, "string":"Now is the time" } + vv := []interface{}{ 3.1415962535, false, "for all good men" } + v["listkey"] = interface{}(vv) + m["newkey"] = interface{}(v) + + fmt.Println("TestMap, m:", m) + fmt.Println("TestMap, StringIndent:", m.StringIndent()) + + o := interface{}(m.Old()) + switch o.(type) { + case map[string]interface{}: + // do nothing + default: + t.Fatal("invalid type for m.Old()") + } + + m, _ = NewMapXml([]byte(`HelloWorld`)) + fmt.Println("TestMap, m_fromXML:",m) + fmt.Println("TestMap, StringIndent:", m.StringIndent()) + + mm, _ := m.Copy() + fmt.Println("TestMap, m.Copy():", mm) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/newmap.go b/Godeps/_workspace/src/github.com/clbanning/mxj/newmap.go new file mode 100644 index 000000000..e9c7f0cff --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/newmap.go @@ -0,0 +1,183 @@ +// mxj - A collection of map[string]interface{} and associated XML and JSON utilities. +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +// remap.go - build a new Map from the current Map based on keyOld:keyNew mapppings +// keys can use dot-notation, keyOld can use wildcard, '*' +// +// Computational strategy - +// Using the key path - []string - traverse a new map[string]interface{} and +// insert the oldVal as the newVal when we arrive at the end of the path. +// If the type at the end is nil, then that is newVal +// If the type at the end is a singleton (string, float64, bool) an array is created. +// If the type at the end is an array, newVal is just appended. +// If the type at the end is a map, it is inserted if possible or the map value +// is converted into an array if necessary. + +package mxj + +import ( + "errors" + "strings" +) + +// (Map)NewMap - create a new Map from data in the current Map. +// 'keypairs' are key mappings "oldKey:newKey" and specify that the current value of 'oldKey' +// should be the value for 'newKey' in the returned Map. +// - 'oldKey' supports dot-notation as described for (Map)ValuesForPath() +// - 'newKey' supports dot-notation but with no wildcards, '*', or indexed arrays +// - "oldKey" is shorthand for for the keypair value "oldKey:oldKey" +// - "oldKey:" and ":newKey" are invalid keypair values +// - if 'oldKey' does not exist in the current Map, it is not written to the new Map. +// "null" is not supported unless it is the current Map. +// - see newmap_test.go for several syntax examples +// +// NOTE: mv.NewMap() == mxj.New(). +func (mv Map) NewMap(keypairs ...string) (Map, error) { + n := make(map[string]interface{}, 0) + if len(keypairs) == 0 { + return n, nil + } + + // loop through the pairs + var oldKey, newKey string + var path []string + for _, v := range keypairs { + if len(v) == 0 { + continue // just skip over empty keypair arguments + } + + // initialize oldKey, newKey and check + vv := strings.Split(v, ":") + if len(vv) > 2 { + return n, errors.New("oldKey:newKey keypair value not valid - " + v) + } + if len(vv) == 1 { + oldKey, newKey = vv[0], vv[0] + } else { + oldKey, newKey = vv[0], vv[1] + } + strings.TrimSpace(oldKey) + strings.TrimSpace(newKey) + if i := strings.Index(newKey, "*"); i > -1 { + return n, errors.New("newKey value cannot contain wildcard character - " + v) + } + if i := strings.Index(newKey, "["); i > -1 { + return n, errors.New("newKey value cannot contain indexed arrays - " + v) + } + if oldKey == "" || newKey == "" { + return n, errors.New("oldKey or newKey is not specified - " + v) + } + + // get oldKey value + oldVal, err := mv.ValuesForPath(oldKey) + if err != nil { + return n, err + } + if len(oldVal) == 0 { + continue // oldKey has no value, may not exist in mv + } + + // break down path + path = strings.Split(newKey, ".") + if path[len(path)-1] == "" { // ignore a trailing dot in newKey spec + path = path[:len(path)-1] + } + + addNewVal(&n, path, oldVal) + } + + return n, nil +} + +// navigate 'n' to end of path and add val +func addNewVal(n *map[string]interface{}, path []string, val []interface{}) { + // newVal - either singleton or array + var newVal interface{} + if len(val) == 1 { + newVal = val[0] // is type interface{} + } else { + newVal = interface{}(val) + } + + // walk to the position of interest, create it if necessary + m := (*n) // initialize map walker + var k string // key for m + lp := len(path) - 1 // when to stop looking + for i := 0; i < len(path); i++ { + k = path[i] + if i == lp { + break + } + var nm map[string]interface{} // holds position of next-map + switch m[k].(type) { + case nil: // need a map for next node in path, so go there + nm = make(map[string]interface{}, 0) + m[k] = interface{}(nm) + m = m[k].(map[string]interface{}) + case map[string]interface{}: + // OK - got somewhere to walk to, go there + m = m[k].(map[string]interface{}) + case []interface{}: + // add a map and nm points to new map unless there's already + // a map in the array, then nm points there + // The placement of the next value in the array is dependent + // on the sequence of members - could land on a map or a nil + // value first. TODO: how to test this. + a := make([]interface{},0) + var foundmap bool + for _, vv := range m[k].([]interface{}) { + switch vv.(type) { + case nil: // doesn't appear that this occurs, need a test case + if foundmap { // use the first one in array + a = append(a, vv) + continue + } + nm = make(map[string]interface{}, 0) + a = append(a,interface{}(nm)) + foundmap = true + case map[string]interface{}: + if foundmap { // use the first one in array + a = append(a, vv) + continue + } + nm = vv.(map[string]interface{}) + a = append(a, vv) + foundmap = true + default: + a = append(a, vv) + } + } + // no map found in array + if !foundmap { + nm = make(map[string]interface{},0) + a = append(a, interface{}(nm)) + } + m[k] = interface{}(a) // must insert in map + m = nm + default: // it's a string, float, bool, etc. + aa := make([]interface{}, 0) + nm = make(map[string]interface{}, 0) + aa = append(aa, m[k], nm) + m[k] = interface{}(aa) + m = nm + } + } + + // value is nil, array or a singleton of some kind + // initially m.(type) == map[string]interface{} + v := m[k] + switch v.(type) { + case nil: // initialized + m[k] = newVal + case []interface{}: + a := m[k].([]interface{}) + a = append(a, newVal) + m[k] = interface{}(a) + default: // v exists:string, float64, bool, map[string]interface, etc. + a := make([]interface{}, 0) + a = append(a, v, newVal) + m[k] =interface{}(a) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/newmap_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/newmap_test.go new file mode 100644 index 000000000..0b2504612 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/newmap_test.go @@ -0,0 +1,114 @@ +package mxj + +import ( + "io" + "bytes" + "fmt" + "testing" +) + +func TestNewMapHeader(t *testing.T) { + fmt.Println("\n---------------- newmap_test.go ...\n") +} + +func TestNewMap(t *testing.T) { + j := []byte(`{ "A":"this", "B":"is", "C":"a", "D":"test" }`) + fmt.Println("j:", string(j)) + + m, _ := NewMapJson(j) + fmt.Printf("m: %#v\n", m) + + fmt.Println("\n",`eval - m.NewMap("A:AA", "B:BB", "C:cc", "D:help")`) + n, err := m.NewMap("A:AA", "B:BB", "C:cc", "D:help") + if err != nil { + fmt.Println("err:", err.Error()) + } + j, _ = n.Json() + fmt.Println("n.Json():", string(j)) + x, _ := n.Xml() + fmt.Println("n.Xml():\n",string(x)) + x, _ = n.XmlIndent("", " ") + fmt.Println("n.XmlIndent():\n",string(x)) + + fmt.Println("\n",`eval - m.NewMap("A:AA.A", "B:AA.B", "C:AA.B.cc", "D:hello.help")`) + n, err = m.NewMap("A:AA.A", "B:AA.B", "C:AA.B.cc", "D:hello.help") + if err != nil { + fmt.Println("err:", err.Error()) + } + j, _ = n.Json() + fmt.Println("n.Json():", string(j)) + x, _ = n.Xml() + fmt.Println("n.Xml():\n",string(x)) + x, _ = n.XmlIndent("", " ") + fmt.Println("n.XmlIndent():\n",string(x)) + + var keypairs = []string{ "A:xml.AA", "B:xml.AA.hello.again", "C:xml.AA", "D:xml.AA.hello.help" } + fmt.Println("\n",`eval - m.NewMap keypairs:`,keypairs) + n, err = m.NewMap(keypairs...) + if err != nil { + fmt.Println("err:", err.Error()) + } + j, _ = n.Json() + fmt.Println("n.Json():", string(j)) + x, _ = n.Xml() + fmt.Println("n.Xml():\n",string(x)) + x, _ = n.XmlIndent("", " ") + fmt.Println("n.XmlIndent():\n",string(x)) +} + +// Need to normalize from an XML stream the values for "netid" and "idnet". +// Solution: make everything "netid" +// Demo how to re-label a key using mv.NewMap() + +var msg1 = []byte(` + + + + no + default:text + default:word + + +`) + +var msg2 = []byte(` + + + + yes + default:text + default:word + + +`) + +func TestNetId(t *testing.T) { + // let's create a message stream + buf := new(bytes.Buffer) + // load a couple of messages into it + _, _ = buf.Write(msg1) + _, _ = buf.Write(msg2) + + n := 0 + for { + n++ + // read the stream as Map values - quit on io.EOF + m, raw, merr := NewMapXmlReaderRaw(buf) + if merr != nil && merr != io.EOF { + // handle error - for demo we just print it and continue + fmt.Printf("msg: %d - merr: %s\n", n, merr.Error()) + continue + } else if merr == io.EOF { + break + } + + // the first keypair retains values if data correct + // the second keypair relabels "idnet" to "netid" + n, _ := m.NewMap("data.netid", "data.idnet:data.netid") + x, _ := n.XmlIndent("", " ") + + fmt.Println("original value:", string(raw)) + fmt.Println("new value:") + fmt.Println(string(x)) + } +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/readme.md b/Godeps/_workspace/src/github.com/clbanning/mxj/readme.md new file mode 100644 index 000000000..76cff5df9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/readme.md @@ -0,0 +1,113 @@ +

mxj - to/from maps, XML and JSON

+Marshal/Unmarshal XML to/from JSON and `map[string]interface{}` values, and extract/modify values from maps by key or key-path, including wildcards. + +mxj supplants the legacy x2j and j2x packages. If you want the old syntax, use mxj/x2j and mxj/j2x packages. + +

Notices

+ 2014-11-09: IncludeTagSeqNum() adds "_seq" key with XML doc positional information. + (NOTE: PreserveXmlList() is similar and will be here soon.) + 2014-09-18: inspired by NYTimes fork, added PrependAttrWithHyphen() to allow stripping hyphen from attribute tag. + 2014-08-02: AnyXml() and AnyXmlIndent() will try to marshal arbitrary values to XML. + 2014-04-28: ValuesForPath() and NewMap() now accept path with indexed array references. + +

Basic Unmarshal XML / JSON / struct

+
type Map map[string]interface{}
+ +Create a `Map` value, 'm', from any `map[string]interface{}` value, 'v': +
m := Map(v)
+ +Unmarshal / marshal XML as a `Map` value, 'm': +
m, err := NewMapXml(xmlValue) // unmarshal
+xmlValue, err := m.Xml()      // marshal
+ +Unmarshal XML from an `io.Reader` as a `Map` value, 'm': +
m, err := NewMapReader(xmlReader)         // repeated calls, as with an os.File Reader, will process stream
+m, raw, err := NewMapReaderRaw(xmlReader) // 'raw' is the raw XML that was decoded
+ +Marshal `Map` value, 'm', to an XML Writer (`io.Writer`): +
err := m.XmlWriter(xmlWriter)
+raw, err := m.XmlWriterRaw(xmlWriter) // 'raw' is the raw XML that was written on xmlWriter
+ +Also, for prettified output: +
xmlValue, err := m.XmlIndent(prefix, indent, ...)
+err := m.XmlIndentWriter(xmlWriter, prefix, indent, ...)
+raw, err := m.XmlIndentWriterRaw(xmlWriter, prefix, indent, ...)
+ +Bulk process XML with error handling (note: handlers must return a boolean value): +
err := HandleXmlReader(xmlReader, mapHandler(Map), errHandler(error))
+err := HandleXmlReaderRaw(xmlReader, mapHandler(Map, []byte), errHandler(error, []byte))
+ +Converting XML to JSON: see Examples for `NewMapXml` and `HandleXmlReader`. + +There are comparable functions and methods for JSON processing. + +Arbitrary structure values can be decoded to / encoded from `Map` values: +
m, err := NewMapStruct(structVal)
+err := m.Struct(structPointer)
+ +

Extract / modify Map values

+To work with XML tag values, JSON or Map key values or structure field values, decode the XML, JSON +or structure to a `Map` value, 'm', or cast a `map[string]interface{}` value to a `Map` value, 'm', then: +
paths := m.PathsForKey(key)
+path := m.PathForKeyShortest(key)
+values, err := m.ValuesForKey(key, subkeys)
+values, err := m.ValuesForPath(path, subkeys)
+count, err := m.UpdateValuesForPath(newVal, path, subkeys)
+ +Get everything at once, irrespective of path depth: +
leafnodes := m.LeafNodes()
+leafvalues := m.LeafValues()
+ +A new `Map` with whatever keys are desired can be created from the current `Map` and then encoded in XML +or JSON. (Note: keys can use dot-notation.) +
newMap := m.NewMap("oldKey_1:newKey_1", "oldKey_2:newKey_2", ..., "oldKey_N:newKey_N")
+newXml := newMap.Xml()   // for example
+newJson := newMap.Json() // ditto
+ +

Usage

+ +The package is fairly well self-documented with examples. (http://godoc.org/github.com/clbanning/mxj) + +Also, the subdirectory "examples" contains a wide range of examples, several taken from golang-nuts discussions. + +

XML parsing conventions

+ + - Attributes are parsed to `map[string]interface{}` values by prefixing a hyphen, `-`, + to the attribute label. (Unless overridden by `PrependAttrWithHyphen(false)`.) + - If the element is a simple element and has attributes, the element value + is given the key `#text` for its `map[string]interface{}` representation. (See + the 'atomFeedString.xml' test data, below.) + +

XML encoding conventions

+ + - 'nil' `Map` values, which may represent 'null' JSON values, are encoded as ``. + NOTE: the operation is not symmetric as `` elements are decoded as `tag:""` `Map` values, + which, then, encode in JSON as `"tag":""` values. + +

Running "go test"

+ +Because there are no guarantees on the sequence map elements are retrieved, the tests have been +written for visual verification in most cases. One advantage is that you can easily use the +output from running "go test" as examples of calling the various functions and methods. + +

Motivation

+ +I make extensive use of JSON for messaging and typically unmarshal the messages into +`map[string]interface{}` variables. This is easily done using `json.Unmarshal` from the +standard Go libraries. Unfortunately, many legacy solutions use structured +XML messages; in those environments the applications would have to be refitted to +interoperate with my components. + +The better solution is to just provide an alternative HTTP handler that receives +XML messages and parses it into a `map[string]interface{}` variable and then reuse +all the JSON-based code. The Go `xml.Unmarshal()` function does not provide the same +option of unmarshaling XML messages into `map[string]interface{}` variables. So I wrote +a couple of small functions to fill this gap and released them as the x2j package. + +Over the next year and a half additional features were added, and the companion j2x +package was released to address XML encoding of arbitrary JSON and `map[string]interface{}` +values. As part of a refactoring of our production system and looking at how we had been +using the x2j and j2x packages we found that we rarely performed direct XML-to-JSON or +JSON-to_XML conversion and that working with the XML or JSON as `map[string]interface{}` +values was the primary value. Thus, everything was refactored into the mxj package. + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/seqnum_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/seqnum_test.go new file mode 100644 index 000000000..208eb27f3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/seqnum_test.go @@ -0,0 +1,52 @@ +// seqnum.go + +package mxj + +import ( + "fmt" + "testing" +) + +var seqdata1 = []byte(` + + + + + `) + +var seqdata2 = []byte(` + + + + 1 + hello + true + + + `) + +func TestSeqNumHeader(t *testing.T) { + fmt.Println("\n---------------- seqnum_test.go ...\n") +} + +func TestSeqNum(t *testing.T) { + IncludeTagSeqNum( true ) + + m, err := NewMapXml(seqdata1, Cast) + if err != nil { + t.Fatal(err) + } + fmt.Printf("m1: %#v\n", m) + j, _ := m.JsonIndent("", " ") + fmt.Println(string(j)) + + m, err = NewMapXml(seqdata2, Cast) + if err != nil { + t.Fatal(err) + } + fmt.Printf("m2: %#v\n", m) + j, _ = m.JsonIndent("", " ") + fmt.Println(string(j)) + + IncludeTagSeqNum( false ) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/struct.go b/Godeps/_workspace/src/github.com/clbanning/mxj/struct.go new file mode 100644 index 000000000..3bb2ef7c0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/struct.go @@ -0,0 +1,40 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +package mxj + +import ( + "encoding/json" + "errors" + "github.com/fatih/structs" + "reflect" +) + +// Create a new Map value from a structure. Error returned if argument is not a structure +// or if there is a json.Marshal or json.Unmarshal error. +// Only public structure fields are decoded in the Map value. Also, json.Marshal structure encoding rules +// are followed for decoding the structure fields. +func NewMapStruct(structVal interface{}) (Map, error) { + if !structs.IsStruct(structVal) { + return nil, errors.New("NewMapStruct() error: argument is not type Struct") + } + return structs.Map(structVal), nil +} + +// Marshal a map[string]interface{} into a structure referenced by 'structPtr'. Error returned +// if argument is not a pointer or if json.Unmarshal returns an error. +// json.Unmarshal structure encoding rules are followed to encode public structure fields. +func (mv Map) Struct(structPtr interface{}) error { + m := map[string]interface{}(mv) + j, err := json.Marshal(m) + if err != nil { + return err + } + + // should check that we're getting a pointer. + if reflect.ValueOf(structPtr).Kind() != reflect.Ptr { + return errors.New("mv.Struct() error: argument is not type Ptr") + } + return json.Unmarshal(j, structPtr) +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/struct_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/struct_test.go new file mode 100644 index 000000000..91af0f6ea --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/struct_test.go @@ -0,0 +1,85 @@ +package mxj + +import ( + "fmt" + "testing" +) + +func TestStructHeader(t *testing.T) { + fmt.Println("\n---------------- struct_test.go ...\n") +} + +func TestNewMapStruct(t *testing.T) { + type str struct { + IntVal int `json:"int"` + StrVal string `json:"str"` + FloatVal float64 `json:"float"` + BoolVal bool `json:"bool"` + private string + } + s := str{ IntVal:4, StrVal:"now's the time", FloatVal:3.14159, BoolVal:true, private:"It's my party" } + + m, merr := NewMapStruct(s) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + + fmt.Printf("NewMapStruct, s: %#v\n",s) + fmt.Printf("NewMapStruct, m: %#v\n",m) + + m, merr = NewMapStruct(s) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + + fmt.Printf("NewMapStruct, s: %#v\n",s) + fmt.Printf("NewMapStruct, m: %#v\n",m) +} + +func TestNewMapStructError(t *testing.T) { + var s string + _, merr := NewMapStruct(s) + if merr == nil { + t.Fatal("NewMapStructError, merr is nil") + } + + fmt.Println("NewMapStructError, merr:",merr.Error()) +} + +func TestStruct(t *testing.T) { + type str struct { + IntVal int `json:"int"` + StrVal string `json:"str"` + FloatVal float64 `json:"float"` + BoolVal bool `json:"bool"` + private string + } + var s str + m := Map{ "int":4, "str":"now's the time", "float":3.14159, "bool":true, "private":"Somewhere over the rainbow" } + + mverr := m.Struct(&s) + if mverr != nil { + t.Fatal("mverr:", mverr.Error()) + } + + fmt.Printf("Struct, m: %#v\n",m) + fmt.Printf("Struct, s: %#v\n",s) +} + +func TestStructError(t *testing.T) { + type str struct { + IntVal int `json:"int"` + StrVal string `json:"str"` + FloatVal float64 `json:"float"` + BoolVal bool `json:"bool"` + } + var s str + mv := Map{ "int":4, "str":"now's the time", "float":3.14159, "bool":true } + + mverr := mv.Struct(s) + if mverr == nil { + t.Fatal("StructError, no error returned") + } + fmt.Println("StructError, mverr:", mverr.Error()) +} + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/updatevalues.go b/Godeps/_workspace/src/github.com/clbanning/mxj/updatevalues.go new file mode 100644 index 000000000..72413d2d4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/updatevalues.go @@ -0,0 +1,249 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +// updatevalues.go - modify a value based on path and possibly sub-keys + +package mxj + +import ( + "fmt" + "strconv" + "strings" +) + +// Update value based on path and possible sub-key values. +// A count of the number of values changed and any error are returned. +// If the count == 0, then no path (and subkeys) matched. +// 'newVal' can be a Map or map[string]interface{} value with a single 'key' that is the key to be modified +// or a string value "key:value[:type]" where type is "bool" or "num" to cast the value. +// 'path' is dot-notation list of keys to traverse; last key in path can be newVal key +// NOTE: 'path' spec does not currently support indexed array references. +// 'subkeys' are "key:value[:type]" entries that must match for path node +// The subkey can be wildcarded - "key:*" - to require that it's there with some value. +// If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an +// exclusion critera - e.g., "!author:William T. Gaddis". +func (mv Map) UpdateValuesForPath(newVal interface{}, path string, subkeys ...string) (int, error) { + m := map[string]interface{}(mv) + + // extract the subkeys + var subKeyMap map[string]interface{} + if len(subkeys) > 0 { + var err error + subKeyMap, err = getSubKeyMap(subkeys...) + if err != nil { + return 0, err + } + } + + // extract key and value from newVal + var key string + var val interface{} + switch newVal.(type) { + case map[string]interface{}, Map: + switch newVal.(type) { // "fallthrough is not permitted in type switch" (Spec) + case Map: + newVal = newVal.(Map).Old() + } + if len(newVal.(map[string]interface{})) != 1 { + return 0, fmt.Errorf("newVal map can only have len == 1 - %+v", newVal) + } + for key, val = range newVal.(map[string]interface{}) { + } + case string: // split it as a key:value pair + ss := strings.Split(newVal.(string), ":") + n := len(ss) + if n < 2 || n > 3 { + return 0, fmt.Errorf("unknown newVal spec - %+v", newVal) + } + key = ss[0] + if n == 2 { + val = interface{}(ss[1]) + } else if n == 3 { + switch ss[2] { + case "bool", "boolean": + nv, err := strconv.ParseBool(ss[1]) + if err != nil { + return 0, fmt.Errorf("can't convert newVal to bool - %+v", newVal) + } + val = interface{}(nv) + case "num", "numeric", "float", "int": + nv, err := strconv.ParseFloat(ss[1], 64) + if err != nil { + return 0, fmt.Errorf("can't convert newVal to float64 - %+v", newVal) + } + val = interface{}(nv) + default: + return 0, fmt.Errorf("unknown type for newVal value - %+v", newVal) + } + } + default: + return 0, fmt.Errorf("invalid newVal type - %+v", newVal) + } + + // parse path + keys := strings.Split(path, ".") + + var count int + updateValuesForKeyPath(key, val, m, keys, subKeyMap, &count) + + return count, nil +} + +// navigate the path +func updateValuesForKeyPath(key string, value interface{}, m interface{}, keys []string, subkeys map[string]interface{}, cnt *int) { + // ----- at end node: looking at possible node to get 'key' ---- + if len(keys) == 1 { + updateValue(key, value, m, keys[0], subkeys, cnt) + return + } + + // ----- here we are navigating the path thru the penultimate node -------- + // key of interest is keys[0] - the next in the path + switch keys[0] { + case "*": // wildcard - scan all values + switch m.(type) { + case map[string]interface{}: + for _, v := range m.(map[string]interface{}) { + updateValuesForKeyPath(key, value, v, keys[1:], subkeys, cnt) + } + case []interface{}: + for _, v := range m.([]interface{}) { + switch v.(type) { + // flatten out a list of maps - keys are processed + case map[string]interface{}: + for _, vv := range v.(map[string]interface{}) { + updateValuesForKeyPath(key, value, vv, keys[1:], subkeys, cnt) + } + default: + updateValuesForKeyPath(key, value, v, keys[1:], subkeys, cnt) + } + } + } + default: // key - must be map[string]interface{} + switch m.(type) { + case map[string]interface{}: + if v, ok := m.(map[string]interface{})[keys[0]]; ok { + updateValuesForKeyPath(key, value, v, keys[1:], subkeys, cnt) + } + case []interface{}: // may be buried in list + for _, v := range m.([]interface{}) { + switch v.(type) { + case map[string]interface{}: + if vv, ok := v.(map[string]interface{})[keys[0]]; ok { + updateValuesForKeyPath(key, value, vv, keys[1:], subkeys, cnt) + } + } + } + } + } +} + +// change value if key and subkeys are present +func updateValue(key string, value interface{}, m interface{}, keys0 string, subkeys map[string]interface{}, cnt *int) { + // there are two possible options for the value of 'keys0': map[string]interface, []interface{} + // and 'key' is a key in the map or is a key in a map in a list. + switch m.(type) { + case map[string]interface{}: // gotta have the last key + if keys0 == "*" { + for k, _ := range m.(map[string]interface{}) { + updateValue(key, value, m, k, subkeys, cnt) + } + return + } + endVal, _ := m.(map[string]interface{})[keys0] + + // if newV key is the end of path, replace the value for path-end + // may be []interface{} - means replace just an entry w/ subkeys + // otherwise replace the keys0 value if subkeys are there + // NOTE: this will replace the subkeys, also + if key == keys0 { + switch endVal.(type) { + case map[string]interface{}: + if ok := hasSubKeys(m, subkeys); ok { + (m.(map[string]interface{}))[keys0] = value + (*cnt)++ + } + case []interface{}: + // without subkeys can't select list member to modify + // so key:value spec is it ... + if len(subkeys) == 0 { + (m.(map[string]interface{}))[keys0] = value + (*cnt)++ + break + } + nv := make([]interface{}, 0) + var valmodified bool + for _, v := range endVal.([]interface{}) { + // check entry subkeys + if ok := hasSubKeys(v, subkeys); ok { + // replace v with value + nv = append(nv, value) + valmodified = true + (*cnt)++ + continue + } + nv = append(nv, v) + } + if valmodified { + (m.(map[string]interface{}))[keys0] = interface{}(nv) + } + default: // anything else is a strict replacement + if len(subkeys) == 0 { + (m.(map[string]interface{}))[keys0] = value + (*cnt)++ + } + } + return + } + + // so value is for an element of endVal + // if endVal is a map then 'key' must be there w/ subkeys + // if endVal is a list then 'key' must be in a list member w/ subkeys + switch endVal.(type) { + case map[string]interface{}: + if ok := hasSubKeys(endVal, subkeys); !ok { + return + } + if _, ok := (endVal.(map[string]interface{}))[key]; ok { + (endVal.(map[string]interface{}))[key] = value + (*cnt)++ + } + case []interface{}: // keys0 points to a list, check subkeys + for _, v := range endVal.([]interface{}) { + // got to be a map so we can replace value for 'key' + vv, vok := v.(map[string]interface{}) + if !vok { + continue + } + if _, ok := vv[key]; !ok { + continue + } + if !hasSubKeys(vv, subkeys) { + continue + } + vv[key] = value + (*cnt)++ + } + } + case []interface{}: // key may be in a list member + // don't need to handle keys0 == "*"; we're looking at everything, anyway. + for _, v := range m.([]interface{}) { + // only map values - we're looking for 'key' + mm, ok := v.(map[string]interface{}) + if !ok { + continue + } + if _, ok := mm[key]; !ok { + continue + } + if !hasSubKeys(mm, subkeys) { + continue + } + mm[key] = value + (*cnt)++ + } + } + + // return +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/updatevalues_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/updatevalues_test.go new file mode 100644 index 000000000..53f084716 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/updatevalues_test.go @@ -0,0 +1,190 @@ +// modifyvalues_test.go - test keyvalues.go methods + +package mxj + +import ( + "fmt" + "testing" +) + +func TestUVHeader(t *testing.T) { + fmt.Println("\n---------------- updatevalues_test.go ...\n") +} + + +func TestUpdateValuesForPath_Author(t *testing.T) { + m, merr := NewMapXml(doc1) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println("m:", m) + + ss, _ := m.ValuesForPath("doc.books.book.author") + for _, v := range ss { + fmt.Println("v:", v) + } + fmt.Println("m.UpdateValuesForPath(\"author:NoName\", \"doc.books.book.author\")") + n, err := m.UpdateValuesForPath("author:NoName", "doc.books.book.author") + if err != nil { + t.Fatal("err:", err.Error()) + } + fmt.Println(n, "updates") + ss, _ = m.ValuesForPath("doc.books.book.author") + for _, v := range ss { + fmt.Println("v:", v) + } + + fmt.Println("m.UpdateValuesForPath(\"author:William Gadddis\", \"doc.books.book.author\", \"title:The Recognitions\")") + n, err = m.UpdateValuesForPath("author:William Gadddis", "doc.books.book.author", "title:The Recognitions") + o, _ := m.UpdateValuesForPath("author:Austin Tappen Wright", "doc.books.book", "title:Islandia") + p, _ := m.UpdateValuesForPath("author:John Hawkes", "doc.books.book", "title:The Beetle Leg") + q, _ := m.UpdateValuesForPath("author:T. E. Porter", "doc.books.book", "title:King's Day") + if err != nil { + t.Fatal("err:", err.Error()) + } + fmt.Println(n+o+p+q, "updates") + ss, _ = m.ValuesForPath("doc.books.book.author") + for _, v := range ss { + fmt.Println("v:", v) + } + + fmt.Println("m.UpdateValuesForPath(\"author:William T. Gaddis\", \"doc.books.book.*\", \"title:The Recognitions\")") + n, _ = m.UpdateValuesForPath("author:William T. Gaddis", "doc.books.book.*", "title:The Recognitions") + fmt.Println(n, "updates") + ss, _ = m.ValuesForPath("doc.books.book.author") + for _, v := range ss { + fmt.Println("v:", v) + } + + fmt.Println("m.UpdateValuesForPath(\"title:The Cannibal\", \"doc.books.book.title\", \"author:John Hawkes\")") + n, _ = m.UpdateValuesForPath("title:The Cannibal", "doc.books.book.title", "author:John Hawkes") + o, _ = m.UpdateValuesForPath("review:A novel on his experiences in WWII.", "doc.books.book.review", "title:The Cannibal") + fmt.Println(n+o, "updates") + ss, _ = m.ValuesForPath("doc.books.book") + for _, v := range ss { + fmt.Println("v:", v) + } + + fmt.Println("m.UpdateValuesForPath(\"books:\", \"doc.books\")") + n, _ = m.UpdateValuesForPath("books:", "doc.books") + fmt.Println(n, "updates") + fmt.Println("m:", m) + + fmt.Println("m.UpdateValuesForPath(mm, \"*\")") + mm, _ := NewMapXml(doc1) + n, err = m.UpdateValuesForPath(mm, "*") + if err != nil { + t.Fatal("err:", err.Error()) + } + fmt.Println(n, "updates") + fmt.Println("m:", m) + + // ---------------------- newDoc + var newDoc = []byte(`simple element`) + m, merr = NewMapXml(newDoc) + if merr != nil { + t.Fatal("merr:",merr.Error()) + } + fmt.Println("\nnewDoc:", string(newDoc)) + fmt.Println("m:", m) + fmt.Println("m.UpdateValuesForPath(\"#text:maybe not so simple element\", \"tag\")") + n, _ = m.UpdateValuesForPath("#text:maybe not so simple element", "tag") + fmt.Println("n:", n, "m:", m) + fmt.Println("m.UpdateValuesForPath(\"#text:simple element again\", \"*\")") + n, _ = m.UpdateValuesForPath("#text:simple element again", "*") + fmt.Println("n:", n, "m:", m) + +/* + fmt.Println("UpdateValuesForPath, doc.books.book, title:The Recognitions : NoBook") + n, err = m.UpdateValuesForPath("NoBook", "doc.books.book", "title:The Recognitions") + if err != nil { + t.Fatal("err:", err.Error()) + } + fmt.Println(n, "updates") + ss, _ = m.ValuesForPath("doc.books.book") + for _, v := range ss { + fmt.Println("v:", v) + } + + fmt.Println("UpdateValuesForPath, doc.books.book.title -seq=3: The Blood Oranges") + n, err = m.UpdateValuesForPath("The Blood Oranges", "doc.books.book.title", "-seq:3") + if err != nil { + t.Fatal("err:", err.Error()) + } + fmt.Println(n, "updates") + ss, _ = m.ValuesForPath("doc.books.book.title") + for _, v := range ss { + fmt.Println("v:", v) + } +*/ +} + +var authorDoc = []byte(` + + + William Gaddis + + + The Recognitions + 1955 + A novel that changed the face of American literature. + + + JR + 1975 + Winner of National Book Award for Fiction. + + + + + John Hawkes + + + The Cannibal + 1949 + A novel on his experiences in WWII. + + + The Beetle Leg + 1951 + A lyrical novel about the construction of Ft. Peck Dam in Montana. + + + The Blood Oranges + 1970 + Where everyone wants to vacation. + + + +`) + +func TestAuthorDoc(t *testing.T) { + m, merr := NewMapXml(authorDoc) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + fmt.Println(m.StringIndent()) + + fmt.Println("m.UpdateValuesForPath(\"review:National Book Award winner.\", \"*.*.*.*\", \"title:JR\")") + n, _ := m.UpdateValuesForPath("review:National Book Award winner.", "*.*.*.*", "title:JR") + fmt.Println(n, "updates") + ss, _ := m.ValuesForPath("biblio.author", "name:William Gaddis") + for _, v := range ss { + fmt.Println("v:", v) + } + + fmt.Println("m.UpdateValuesForPath(newVal, path, oldVal)") + path := m.PathForKeyShortest("date") + v,_ := m.ValuesForPath(path) + var counter int + for _, vv := range v { + oldVal := "date:" + vv.(string) + newVal := "date:" + vv.(string) + ":num" + n, _ = m.UpdateValuesForPath(newVal, path, oldVal) + counter += n + } + fmt.Println(counter, "updates") + fmt.Println(m.StringIndent()) +} + + diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/x2j/x2j.go b/Godeps/_workspace/src/github.com/clbanning/mxj/x2j/x2j.go new file mode 100644 index 000000000..dad3f0aa7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/x2j/x2j.go @@ -0,0 +1,184 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +// x2j - For (mostly) backwards compatibility with legacy x2j package. +// Wrappers for end-to-end XML to JSON transformation and value manipulation. +package x2j + +import ( + . "github.com/clbanning/mxj" + "io" +) + +// FromXml() --> map[string]interface{} +func XmlToMap(xmlVal []byte) (map[string]interface{}, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return map[string]interface{}(m), nil +} + +// map[string]interface{} --> ToXml() +func MapToXml(m map[string]interface{}) ([]byte, error) { + return Map(m).Xml() +} + +// FromXml() --> ToJson(). +func XmlToJson(xmlVal []byte, safeEncoding ...bool) ([]byte, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.Json(safeEncoding...) +} + +// FromXml() --> ToJsonWriterRaw(). +func XmlToJsonWriter(xmlVal []byte, jsonWriter io.Writer, safeEncoding ...bool) ([]byte, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.JsonWriterRaw(jsonWriter, safeEncoding...) +} + +// FromXmlReaderRaw() --> ToJson(). +func XmlReaderToJson(xmlReader io.Reader, safeEncoding ...bool) ([]byte, []byte, error) { + m, xraw, err := NewMapXmlReaderRaw(xmlReader) + if err != nil { + return xraw, nil, err + } + j, jerr := m.Json(safeEncoding...) + return xraw, j, jerr +} + +// FromXmlReader() --> ToJsonWriter(). Handy for bulk transformation of documents. +func XmlReaderToJsonWriter(xmlReader io.Reader, jsonWriter io.Writer, safeEncoding ...bool) ([]byte, []byte, error) { + m, xraw, err := NewMapXmlReaderRaw(xmlReader) + if err != nil { + return xraw, nil, err + } + jraw, jerr := m.JsonWriterRaw(jsonWriter, safeEncoding...) + return xraw, jraw, jerr +} + +// XML wrappers for Map methods implementing tag path and value functions. + +// Wrap PathsForKey for XML. +func XmlPathsForTag(xmlVal []byte, tag string) ([]string, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + paths := m.PathsForKey(tag) + return paths, nil +} + +// Wrap PathForKeyShortest for XML. +func XmlPathForTagShortest(xmlVal []byte, tag string) (string, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return "", err + } + path := m.PathForKeyShortest(tag) + return path, nil +} + +// Wrap ValuesForKey for XML. +// 'attrs' are key:value pairs for attributes, where key is attribute label prepended with a hypen, '-'. +func XmlValuesForTag(xmlVal []byte, tag string, attrs ...string) ([]interface{}, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.ValuesForKey(tag, attrs...) +} + +// Wrap ValuesForPath for XML. +// 'attrs' are key:value pairs for attributes, where key is attribute label prepended with a hypen, '-'. +func XmlValuesForPath(xmlVal []byte, path string, attrs ...string) ([]interface{}, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.ValuesForPath(path, attrs...) +} + +// Wrap UpdateValuesForPath for XML +// 'xmlVal' is XML value +// 'newTagValue' is the value to replace an existing value at the end of 'path' +// 'path' is the dot-notation path with the tag whose value is to be replaced at the end +// (can include wildcard character, '*') +// 'subkeys' are key:value pairs of tag:values that must match for the tag +func XmlUpdateValsForPath(xmlVal []byte, newTagValue interface{}, path string, subkeys ...string) ([]byte, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + _, err = m.UpdateValuesForPath(newTagValue, path, subkeys...) + if err != nil { + return nil, err + } + return m.Xml() +} + +// Wrap NewMap for XML and return as XML +// 'xmlVal' is an XML value +// 'tagpairs' are "oldTag:newTag" values that conform to 'keypairs' in (Map)NewMap. +func XmlNewXml(xmlVal []byte, tagpairs ...string) ([]byte, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + n, err := m.NewMap(tagpairs...) + if err != nil { + return nil, err + } + return n.Xml() +} + +// Wrap NewMap for XML and return as JSON +// 'xmlVal' is an XML value +// 'tagpairs' are "oldTag:newTag" values that conform to 'keypairs' in (Map)NewMap. +func XmlNewJson(xmlVal []byte, tagpairs ...string) ([]byte, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + n, err := m.NewMap(tagpairs...) + if err != nil { + return nil, err + } + return n.Json() +} + +// Wrap LeafNodes for XML. +// 'xmlVal' is an XML value +func XmlLeafNodes(xmlVal []byte) ([]LeafNode, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.LeafNodes(), nil +} + +// Wrap LeafValues for XML. +// 'xmlVal' is an XML value +func XmlLeafValues(xmlVal []byte) ([]interface{}, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.LeafValues(), nil +} + +// Wrap LeafPath for XML. +// 'xmlVal' is an XML value +func XmlLeafPath(xmlVal []byte) ([]string, error) { + m, err := NewMapXml(xmlVal) + if err != nil { + return nil, err + } + return m.LeafPaths(), nil +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/xml.go b/Godeps/_workspace/src/github.com/clbanning/mxj/xml.go new file mode 100644 index 000000000..691bc0216 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/xml.go @@ -0,0 +1,838 @@ +// Copyright 2012-2014 Charles Banning. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file + +// xml.go - basically the core of X2j for map[string]interface{} values. +// NewMapXml, NewMapXmlReader, mv.Xml, mv.XmlWriter +// see x2j and j2x for wrappers to provide end-to-end transformation of XML and JSON messages. + +package mxj + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "time" +) + +// ------------------- NewMapXml & NewMapXmlReader ... from x2j2 ------------------------- + +// If XmlCharsetReader != nil, it will be used to decode the XML, if required. +// import ( +// charset "code.google.com/p/go-charset/charset" +// github.com/clbanning/mxj +// ) +// ... +// mxj.XmlCharsetReader = charset.NewReader +// m, merr := mxj.NewMapXml(xmlValue) +var XmlCharsetReader func(charset string, input io.Reader) (io.Reader, error) + +// NewMapXml - convert a XML doc into a Map +// (This is analogous to unmarshalling a JSON string to map[string]interface{} using json.Unmarshal().) +// If the optional argument 'cast' is 'true', then values will be converted to boolean or float64 if possible. +// +// Converting XML to JSON is a simple as: +// ... +// mapVal, merr := mxj.NewMapXml(xmlVal) +// if merr != nil { +// // handle error +// } +// jsonVal, jerr := mapVal.Json() +// if jerr != nil { +// // handle error +// } +func NewMapXml(xmlVal []byte, cast ...bool) (Map, error) { + var r bool + if len(cast) == 1 { + r = cast[0] + } + n, err := xmlToTree(xmlVal) + if err != nil { + return nil, err + } + + m := make(map[string]interface{}, 0) + m[n.key] = n.treeToMap(r) + + return m, nil +} + +// Get next XML doc from an io.Reader as a Map value. Returns Map value. +func NewMapXmlReader(xmlReader io.Reader, cast ...bool) (Map, error) { + var r bool + if len(cast) == 1 { + r = cast[0] + } + + // build the node tree + n, err := xmlReaderToTree(xmlReader) + if err != nil { + return nil, err + } + + // create the Map value + m := make(map[string]interface{}) + m[n.key] = n.treeToMap(r) + + return m, nil +} + +// XmlWriterBufSize - set the size of io.Writer for the TeeReader used by NewMapXmlReaderRaw() +// and HandleXmlReaderRaw(). This reduces repeated memory allocations and copy() calls in most cases. +var XmlWriterBufSize int = 256 + +// Get next XML doc from an io.Reader as a Map value. Returns Map value and slice with the raw XML. +// NOTES: 1. Due to the implementation of xml.Decoder, the raw XML off the reader is buffered to []byte +// using a ByteReader. If the io.Reader is an os.File, there may be significant performance impact. +// See the examples - getmetrics1.go through getmetrics4.go - for comparative use cases on a large +// data set. If the io.Reader is wrapping a []byte value in-memory, however, such as http.Request.Body +// you CAN use it to efficiently unmarshal a XML doc and retrieve the raw XML in a single call. +// 2. The 'raw' return value may be larger than the XML text value. To log it, cast it to a string. +func NewMapXmlReaderRaw(xmlReader io.Reader, cast ...bool) (Map, []byte, error) { + var r bool + if len(cast) == 1 { + r = cast[0] + } + // create TeeReader so we can retrieve raw XML + buf := make([]byte, XmlWriterBufSize) + wb := bytes.NewBuffer(buf) + trdr := myTeeReader(xmlReader, wb) // see code at EOF + + // build the node tree + n, err := xmlReaderToTree(trdr) + + // retrieve the raw XML that was decoded + b := make([]byte, wb.Len()) + _, _ = wb.Read(b) + + if err != nil { + return nil, b, err + } + + // create the Map value + m := make(map[string]interface{}) + m[n.key] = n.treeToMap(r) + + return m, b, nil +} + +// xmlReaderToTree() - parse a XML io.Reader to a tree of nodes +func xmlReaderToTree(rdr io.Reader) (*node, error) { + // parse the Reader + p := xml.NewDecoder(rdr) + p.CharsetReader = XmlCharsetReader + return xmlToTreeParser("", nil, p) +} + +// for building the parse tree +type node struct { + dup bool // is member of a list + attr bool // is an attribute + key string // XML tag + val string // element value + nodes []*node +} + +// xmlToTree - convert a XML doc into a tree of nodes. +func xmlToTree(doc []byte) (*node, error) { + // xml.Decoder doesn't properly handle whitespace in some doc + // see songTextString.xml test case ... + reg, _ := regexp.Compile("[ \t\n\r]*<") + doc = reg.ReplaceAll(doc, []byte("<")) + + b := bytes.NewReader(doc) + p := xml.NewDecoder(b) + p.CharsetReader = XmlCharsetReader + n, berr := xmlToTreeParser("", nil, p) + if berr != nil { + return nil, berr + } + + return n, nil +} + +// we allow people to drop hyphen when unmarshaling the XML doc. +var useHyphen bool = true + +// PrependAttrWithHyphen. Prepend attribute tags with a hyphen. +// Default is 'true'. +// Note: +// If 'false', unmarshaling and marshaling is not symmetric. Attributes will be +// marshal'd as attr and may be part of a list. +func PrependAttrWithHyphen(v bool) { + useHyphen = v +} + +// Include sequence id with inner tags. - per Sean Murphy, murphysean84@gmail.com. +var includeTagSeqNum bool + +// IncludeTagSeqNum - include a "_seq":N key:value pair with each inner tag, denoting +// its position when parsed. E.g., +/* + + + + + hello + + + parses as: + + { + Obj:{ + "-c":"la", + "-h":"da", + "-x":"dee", + "intObj":[ + { + "-id"="3", + "_seq":"0" // if mxj.Cast is passed, then: "_seq":0 + }, + { + "-id"="2", + "_seq":"2" + }], + "intObj1":{ + "-id":"1", + "_seq":"1" + }, + "StrObj":{ + "#text":"hello", // simple element value gets "#text" tag + "_seq":"3" + } + } + } +*/ +func IncludeTagSeqNum(b bool) { + includeTagSeqNum = b +} + +// xmlToTreeParser - load a 'clean' XML doc into a tree of *node. +func xmlToTreeParser(skey string, a []xml.Attr, p *xml.Decoder) (*node, error) { + n := new(node) + n.nodes = make([]*node, 0) + var seq int // for includeTagSeqNum + + if skey != "" { + n.key = skey + if len(a) > 0 { + for _, v := range a { + na := new(node) + na.attr = true + if useHyphen { + na.key = `-` + v.Name.Local + } else { + na.key = v.Name.Local + } + na.val = v.Value + n.nodes = append(n.nodes, na) + } + } + } + for { + t, err := p.Token() + if err != nil { + if err != io.EOF { + return nil, errors.New("xml.Decoder.Token() - " + err.Error()) + } + return nil, err + } + switch t.(type) { + case xml.StartElement: + tt := t.(xml.StartElement) + // handle root + if n.key == "" { + n.key = tt.Name.Local + if len(tt.Attr) > 0 { + for _, v := range tt.Attr { + na := new(node) + na.attr = true + if useHyphen { + na.key = `-` + v.Name.Local + } else { + na.key = v.Name.Local + } + na.val = v.Value + n.nodes = append(n.nodes, na) + } + } + } else { + nn, nnerr := xmlToTreeParser(tt.Name.Local, tt.Attr, p) + if nnerr != nil { + return nil, nnerr + } + n.nodes = append(n.nodes, nn) + if includeTagSeqNum { // 2014.11.09 + sn := &node{false, false, "_seq", strconv.Itoa(seq), nil} + nn.nodes = append(nn.nodes, sn) + seq++ + } + } + case xml.EndElement: + // scan n.nodes for duplicate n.key values + n.markDuplicateKeys() + return n, nil + case xml.CharData: + tt := string(t.(xml.CharData)) + // clean up possible noise + tt = strings.Trim(tt, "\t\r\b\n ") + if len(n.nodes) > 0 && len(tt) > 0 { + // if len(n.nodes) > 0 { + nn := new(node) + nn.key = "#text" + nn.val = tt + n.nodes = append(n.nodes, nn) + } else { + n.val = tt + } + if includeTagSeqNum { // 2014.11.09 + if len(n.nodes) == 0 { // treat like a simple element with attributes + nn := new(node) + nn.key = "#text" + nn.val = tt + n.nodes = append(n.nodes, nn) + } + sn := &node{false, false, "_seq", strconv.Itoa(seq), nil} + n.nodes = append(n.nodes, sn) + seq++ + } + default: + // noop + } + } + // Logically we can't get here, but provide an error message anyway. + return nil, fmt.Errorf("Unknown parse error in xmlToTree() for: %s", n.key) +} + +// (*node)markDuplicateKeys - set node.dup flag for loading map[string]interface{}. +func (n *node) markDuplicateKeys() { + l := len(n.nodes) + for i := 0; i < l; i++ { + if n.nodes[i].dup { + continue + } + for j := i + 1; j < l; j++ { + if n.nodes[i].key == n.nodes[j].key { + n.nodes[i].dup = true + n.nodes[j].dup = true + } + } + } +} + +// (*node)treeToMap - convert a tree of nodes into a map[string]interface{}. +// (Parses to map that is structurally the same as from json.Unmarshal().) +// Note: root is not instantiated; call with: "m[n.key] = n.treeToMap(cast)". +func (n *node) treeToMap(r bool) interface{} { + if len(n.nodes) == 0 { + return cast(n.val, r) + } + + m := make(map[string]interface{}, 0) + for _, v := range n.nodes { + // 2014.11.9 - may have to back out + if includeTagSeqNum { + if len(v.nodes) == 1 { + m[v.key] = cast(v.val, r) + continue + } + } + + // just a value + if !v.dup && len(v.nodes) == 0 { + m[v.key] = cast(v.val, r) + continue + } + // a list of values + if v.dup { + var a []interface{} + if vv, ok := m[v.key]; ok { + a = vv.([]interface{}) + } else { + a = make([]interface{}, 0) + } + a = append(a, v.treeToMap(r)) + m[v.key] = interface{}(a) + continue + } + + // it's a unique key + m[v.key] = v.treeToMap(r) + } + + return interface{}(m) +} + +// cast - try to cast string values to bool or float64 +func cast(s string, r bool) interface{} { + if r { + // handle numeric strings ahead of boolean + if f, err := strconv.ParseFloat(s, 64); err == nil { + return interface{}(f) + } + // ParseBool treats "1"==true & "0"==false + // but be more strick - only allow TRUE, True, true, FALSE, False, false + if s != "t" && s != "T" && s != "f" && s != "F" { + if b, err := strconv.ParseBool(s); err == nil { + return interface{}(b) + } + } + } + return interface{}(s) +} + +// ------------------ END: NewMapXml & NewMapXmlReader ------------------------- + +// ------------------ mv.Xml & mv.XmlWriter - from j2x ------------------------ + +const ( + DefaultRootTag = "doc" +) + +var useGoXmlEmptyElemSyntax bool + +// XmlGoEmptyElemSyntax() - rather than . +// Go's encoding/xml package marshals empty XML elements as . By default this package +// encodes empty elements as . If you're marshaling Map values that include structures +// (which are passed to xml.Marshal for encoding), this will let you conform to the standard package. +// +// Alternatively, you can replace the encoding/xml/marshal.go file in the standard libary with the +// patched version in the "xml_marshal" folder in this package. Then use xml.SetUseNullEndTag(true) +// to have all XML encoding use for empty elements. +func XmlGoEmptyElemSyntax() { + useGoXmlEmptyElemSyntax = true +} + +// XmlDefaultEmptyElemSyntax() - rather than . +// Return XML encoding for empty elements to the default package setting. +// Reverses effect of XmlGoEmptyElemSyntax(). +func XmlDefaultEmptyElemSyntax() { + useGoXmlEmptyElemSyntax = false +} + +// Encode a Map as XML. The companion of NewMapXml(). +// The following rules apply. +// - The key label "#text" is treated as the value for a simple element with attributes. +// - Map keys that begin with a hyphen, '-', are interpreted as attributes. +// It is an error if the attribute doesn't have a []byte, string, number, or boolean value. +// - Map value type encoding: +// > string, bool, float64, int, int32, int64, float32: per "%v" formating +// > []bool, []uint8: by casting to string +// > structures, etc.: handed to xml.Marshal() - if there is an error, the element +// value is "UNKNOWN" +// - Elements with only attribute values or are null are terminated using "/>". +// - If len(mv) == 1 and no rootTag is provided, then the map key is used as the root tag, possible. +// Thus, `{ "key":"value" }` encodes as "value". +// - To encode empty elements in a syntax consistent with encoding/xml call UseGoXmlEmptyElementSyntax(). +func (mv Map) Xml(rootTag ...string) ([]byte, error) { + m := map[string]interface{}(mv) + var err error + s := new(string) + p := new(pretty) // just a stub + + if len(m) == 1 && len(rootTag) == 0 { + for key, value := range m { + // if it an array, see if all values are map[string]interface{} + // we force a new root tag if we'll end up with no key:value in the list + // so: key:[string_val, bool:true] --> string_valtrue + switch value.(type) { + case []interface{}: + for _, v := range value.([]interface{}) { + switch v.(type) { + case map[string]interface{}: // noop + default: // anything else + err = mapToXmlIndent(false, s, DefaultRootTag, m, p) + goto done + } + } + } + err = mapToXmlIndent(false, s, key, value, p) + } + } else if len(rootTag) == 1 { + err = mapToXmlIndent(false, s, rootTag[0], m, p) + } else { + err = mapToXmlIndent(false, s, DefaultRootTag, m, p) + } +done: + return []byte(*s), err +} + +// The following implementation is provided only for symmetry with NewMapXmlReader[Raw] +// The names will also provide a key for the number of return arguments. + +// Writes the Map as XML on the Writer. +// See Xml() for encoding rules. +func (mv Map) XmlWriter(xmlWriter io.Writer, rootTag ...string) error { + x, err := mv.Xml(rootTag...) + if err != nil { + return err + } + + _, err = xmlWriter.Write(x) + return err +} + +// Writes the Map as XML on the Writer. []byte is the raw XML that was written. +// See Xml() for encoding rules. +func (mv Map) XmlWriterRaw(xmlWriter io.Writer, rootTag ...string) ([]byte, error) { + x, err := mv.Xml(rootTag...) + if err != nil { + return x, err + } + + _, err = xmlWriter.Write(x) + return x, err +} + +// Writes the Map as pretty XML on the Writer. +// See Xml() for encoding rules. +func (mv Map) XmlIndentWriter(xmlWriter io.Writer, prefix, indent string, rootTag ...string) error { + x, err := mv.XmlIndent(prefix, indent, rootTag...) + if err != nil { + return err + } + + _, err = xmlWriter.Write(x) + return err +} + +// Writes the Map as pretty XML on the Writer. []byte is the raw XML that was written. +// See Xml() for encoding rules. +func (mv Map) XmlIndentWriterRaw(xmlWriter io.Writer, prefix, indent string, rootTag ...string) ([]byte, error) { + x, err := mv.XmlIndent(prefix, indent, rootTag...) + if err != nil { + return x, err + } + + _, err = xmlWriter.Write(x) + return x, err +} + +// -------------------- END: mv.Xml & mv.XmlWriter ------------------------------- + +// -------------- Handle XML stream by processing Map value -------------------- + +// Default poll delay to keep Handler from spinning on an open stream +// like sitting on os.Stdin waiting for imput. +var xhandlerPollInterval = time.Duration(1e6) + +// Bulk process XML using handlers that process a Map value. +// 'rdr' is an io.Reader for XML (stream) +// 'mapHandler' is the Map processor. Return of 'false' stops io.Reader processing. +// 'errHandler' is the error processor. Return of 'false' stops io.Reader processing and returns the error. +// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized. +// This means that you can stop reading the file on error or after processing a particular message. +// To have reading and handling run concurrently, pass argument to a go routine in handler and return 'true'. +func HandleXmlReader(xmlReader io.Reader, mapHandler func(Map) bool, errHandler func(error) bool) error { + var n int + for { + m, merr := NewMapXmlReader(xmlReader) + n++ + + // handle error condition with errhandler + if merr != nil && merr != io.EOF { + merr = fmt.Errorf("[xmlReader: %d] %s", n, merr.Error()) + if ok := errHandler(merr); !ok { + // caused reader termination + return merr + } + continue + } + + // pass to maphandler + if len(m) != 0 { + if ok := mapHandler(m); !ok { + break + } + } else if merr != io.EOF { + <-time.After(xhandlerPollInterval) + } + + if merr == io.EOF { + break + } + } + return nil +} + +// Bulk process XML using handlers that process a Map value and the raw XML. +// 'rdr' is an io.Reader for XML (stream) +// 'mapHandler' is the Map and raw XML - []byte - processor. Return of 'false' stops io.Reader processing. +// 'errHandler' is the error and raw XML processor. Return of 'false' stops io.Reader processing and returns the error. +// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized. +// This means that you can stop reading the file on error or after processing a particular message. +// To have reading and handling run concurrently, pass argument(s) to a go routine in handler and return 'true'. +// See NewMapXmlReaderRaw for comment on performance associated with retrieving raw XML from a Reader. +func HandleXmlReaderRaw(xmlReader io.Reader, mapHandler func(Map, []byte) bool, errHandler func(error, []byte) bool) error { + var n int + for { + m, raw, merr := NewMapXmlReaderRaw(xmlReader) + n++ + + // handle error condition with errhandler + if merr != nil && merr != io.EOF { + merr = fmt.Errorf("[xmlReader: %d] %s", n, merr.Error()) + if ok := errHandler(merr, raw); !ok { + // caused reader termination + return merr + } + continue + } + + // pass to maphandler + if len(m) != 0 { + if ok := mapHandler(m, raw); !ok { + break + } + } else if merr != io.EOF { + <-time.After(xhandlerPollInterval) + } + + if merr == io.EOF { + break + } + } + return nil +} + +// ----------------- END: Handle XML stream by processing Map value -------------- + +// -------- a hack of io.TeeReader ... need one that's an io.ByteReader for xml.NewDecoder() ---------- + +// This is a clone of io.TeeReader with the additional method t.ReadByte(). +// Thus, this TeeReader is also an io.ByteReader. +// This is necessary because xml.NewDecoder uses a ByteReader not a Reader. It appears to have been written +// with bufio.Reader or bytes.Reader in mind ... not a generic io.Reader, which doesn't have to have ReadByte().. +// If NewDecoder is passed a Reader that does not satisfy ByteReader() it wraps the Reader with +// bufio.NewReader and uses ReadByte rather than Read that runs the TeeReader pipe logic. + +type teeReader struct { + r io.Reader + w io.Writer + b []byte +} + +func myTeeReader(r io.Reader, w io.Writer) io.Reader { + b := make([]byte, 1) + return &teeReader{r, w, b} +} + +// need for io.Reader - but we don't use it ... +func (t *teeReader) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (t *teeReader) ReadByte() (c byte, err error) { + n, err := t.r.Read(t.b) + if n > 0 { + if _, err := t.w.Write(t.b[:1]); err != nil { + return t.b[0], err + } + } + return t.b[0], err +} + +// ----------------------- END: io.TeeReader hack ----------------------------------- + +// ---------------------- XmlIndent - from j2x package ---------------------------- + +// Encode a map[string]interface{} as a pretty XML string. +// See Xml for encoding rules. +func (mv Map) XmlIndent(prefix, indent string, rootTag ...string) ([]byte, error) { + m := map[string]interface{}(mv) + + var err error + s := new(string) + p := new(pretty) + p.indent = indent + p.padding = prefix + + if len(m) == 1 && len(rootTag) == 0 { + // this can extract the key for the single map element + // use it if it isn't a key for a list + for key, value := range m { + if _, ok := value.([]interface{}); ok { + err = mapToXmlIndent(true, s, DefaultRootTag, m, p) + } else { + err = mapToXmlIndent(true, s, key, value, p) + } + } + } else if len(rootTag) == 1 { + err = mapToXmlIndent(true, s, rootTag[0], m, p) + } else { + err = mapToXmlIndent(true, s, DefaultRootTag, m, p) + } + return []byte(*s), err +} + +type pretty struct { + indent string + cnt int + padding string + mapDepth int + start int +} + +func (p *pretty) Indent() { + p.padding += p.indent + p.cnt++ +} + +func (p *pretty) Outdent() { + if p.cnt > 0 { + p.padding = p.padding[:len(p.padding)-len(p.indent)] + p.cnt-- + } +} + +// where the work actually happens +// returns an error if an attribute is not atomic +func mapToXmlIndent(doIndent bool, s *string, key string, value interface{}, pp *pretty) error { + var endTag bool + var isSimple bool + p := &pretty{pp.indent, pp.cnt, pp.padding, pp.mapDepth, pp.start} + + switch value.(type) { + case map[string]interface{}, []byte, string, float64, bool, int, int32, int64, float32: + if doIndent { + *s += p.padding + } + *s += `<` + key + } + switch value.(type) { + case map[string]interface{}: + vv := value.(map[string]interface{}) + lenvv := len(vv) + // scan out attributes - keys have prepended hyphen, '-' + var cntAttr int + for k, v := range vv { + if k[:1] == "-" { + switch v.(type) { + case string, float64, bool, int, int32, int64, float32: + *s += ` ` + k[1:] + `="` + fmt.Sprintf("%v", v) + `"` + cntAttr++ + case []byte: // allow standard xml pkg []byte transform, as below + *s += ` ` + k[1:] + `="` + fmt.Sprintf("%v", string(v.([]byte))) + `"` + cntAttr++ + default: + return fmt.Errorf("invalid attribute value for: %s", k) + } + } + } + // only attributes? + if cntAttr == lenvv { + break + } + // simple element? Note: '#text" is an invalid XML tag. + if v, ok := vv["#text"]; ok { + if cntAttr+1 < lenvv { + return errors.New("#text key occurs with other non-attribute keys") + } + *s += ">" + fmt.Sprintf("%v", v) + endTag = true + break + } + // close tag with possible attributes + *s += ">" + if doIndent { + *s += "\n" + } + // something more complex + p.mapDepth++ + var i int + for k, v := range vv { + if k[:1] == "-" { + continue + } + switch v.(type) { + case []interface{}: + default: + if i == 0 && doIndent { + p.Indent() + } + } + i++ + mapToXmlIndent(doIndent, s, k, v, p) + switch v.(type) { + case []interface{}: // handled in []interface{} case + default: + if doIndent { + p.Outdent() + } + } + i-- + } + p.mapDepth-- + endTag = true + case []interface{}: + for _, v := range value.([]interface{}) { + if doIndent { + p.Indent() + } + mapToXmlIndent(doIndent, s, key, v, p) + if doIndent { + p.Outdent() + } + } + return nil + case nil: + // terminate the tag + *s += "<" + key + break + default: // handle anything - even goofy stuff + switch value.(type) { + case string, float64, bool, int, int32, int64, float32: + *s += ">" + fmt.Sprintf("%v", value) + case []byte: // NOTE: byte is just an alias for uint8 + // similar to how xml.Marshal handles []byte structure members + *s += ">" + string(value.([]byte)) + default: + var v []byte + var err error + if doIndent { + v, err = xml.MarshalIndent(value, p.padding, p.indent) + } else { + v, err = xml.Marshal(value) + } + if err != nil { + *s += ">UNKNOWN" + } else { + *s += string(v) + } + } + isSimple = true + endTag = true + } + + if endTag { + if doIndent { + if !isSimple { + // if p.mapDepth == 0 { + // p.Outdent() + // } + *s += p.padding + } + } + switch value.(type) { + case map[string]interface{}, []byte, string, float64, bool, int, int32, int64, float32: + *s += `" + } + } else if useGoXmlEmptyElemSyntax { + *s += ">" + } else { + *s += "/>" + } + if doIndent { + if p.cnt > p.start { + *s += "\n" + } + p.Outdent() + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/README b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/README new file mode 100644 index 000000000..ca36c45b5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/README @@ -0,0 +1,26 @@ +The mxj package terminates empty elements using '/>' rather than ''. + +The Xml(), XmlIndent(), XmlWriter() marshals Map values that can have structures or any other Go type +that xml.Marshal() can encode. If you want to have xml.Marshal() encode empty elements in a manner +consist with maputil, then you need to hack the src/pkg/encoding/xml/marshal.go file to support that +convention. + +The marshal.go.v1_2 file in this repo does that for Go v1.2; use it in place of the standard library +marshal.go file. + +The example_test.go.v1_2 file extends the package example_test.go file in the src/pkg/encoding/xml +directory to provide an example of the SetUseNullEndTag() function that you'll see in the new godoc +documentation for encoding/xml. xml.SetUseNullEndTag(true) causes xml.Marshal() to encode empty XML +elements as , just like this package. + +With the new marshal.go, you can then force all marshaling that uses encoding/xml to use mxj's +syntax rather than the Go standard syntax. + +NOTE: +If you install this patch then only use either + + - xml.SetUseNullEndTag() +or + - mxj.XmlGoEmptyElemSyntax() + +to have consistent encoding of empty XML elements. diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/example_test.go.v1_2 b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/example_test.go.v1_2 new file mode 100644 index 000000000..2e035a430 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/example_test.go.v1_2 @@ -0,0 +1,211 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml_test + +import ( + "encoding/xml" + "fmt" + "os" +) + +func ExampleMarshalIndent() { + type Address struct { + City, State string + } + type Person struct { + XMLName xml.Name `xml:"person"` + Id int `xml:"id,attr"` + FirstName string `xml:"name>first"` + LastName string `xml:"name>last"` + Age int `xml:"age"` + Height float32 `xml:"height,omitempty"` + Married bool + Address + Comment string `xml:",comment"` + } + + v := &Person{Id: 13, FirstName: "John", LastName: "Doe", Age: 42} + v.Comment = " Need more details. " + v.Address = Address{"Hanga Roa", "Easter Island"} + + output, err := xml.MarshalIndent(v, " ", " ") + if err != nil { + fmt.Printf("error: %v\n", err) + } + + os.Stdout.Write(output) + // Output: + // + // + // John + // Doe + // + // 42 + // false + // Hanga Roa + // Easter Island + // + // +} + +func ExampleEncoder() { + type Address struct { + City, State string + } + type Person struct { + XMLName xml.Name `xml:"person"` + Id int `xml:"id,attr"` + FirstName string `xml:"name>first"` + LastName string `xml:"name>last"` + Age int `xml:"age"` + Height float32 `xml:"height,omitempty"` + Married bool + Address + Comment string `xml:",comment"` + } + + v := &Person{Id: 13, FirstName: "John", LastName: "Doe", Age: 42} + v.Comment = " Need more details. " + v.Address = Address{"Hanga Roa", "Easter Island"} + + enc := xml.NewEncoder(os.Stdout) + enc.Indent(" ", " ") + if err := enc.Encode(v); err != nil { + fmt.Printf("error: %v\n", err) + } + + // Output: + // + // + // John + // Doe + // + // 42 + // false + // Hanga Roa + // Easter Island + // + // +} + +// This example demonstrates unmarshaling an XML excerpt into a value with +// some preset fields. Note that the Phone field isn't modified and that +// the XML element is ignored. Also, the Groups field is assigned +// considering the element path provided in its tag. +func ExampleUnmarshal() { + type Email struct { + Where string `xml:"where,attr"` + Addr string + } + type Address struct { + City, State string + } + type Result struct { + XMLName xml.Name `xml:"Person"` + Name string `xml:"FullName"` + Phone string + Email []Email + Groups []string `xml:"Group>Value"` + Address + } + v := Result{Name: "none", Phone: "none"} + + data := ` + + Grace R. Emlin + Example Inc. + + gre@example.com + + + gre@work.com + + + Friends + Squash + + Hanga Roa + Easter Island + + ` + err := xml.Unmarshal([]byte(data), &v) + if err != nil { + fmt.Printf("error: %v", err) + return + } + fmt.Printf("XMLName: %#v\n", v.XMLName) + fmt.Printf("Name: %q\n", v.Name) + fmt.Printf("Phone: %q\n", v.Phone) + fmt.Printf("Email: %v\n", v.Email) + fmt.Printf("Groups: %v\n", v.Groups) + fmt.Printf("Address: %v\n", v.Address) + // Output: + // XMLName: xml.Name{Space:"", Local:"Person"} + // Name: "Grace R. Emlin" + // Phone: "none" + // Email: [{home gre@example.com} {work gre@work.com}] + // Groups: [Friends Squash] + // Address: {Hanga Roa Easter Island} +} + +// This example demonstrates how SetUseNullEndTag changes the end tag syntax for Marshal. +func ExampleSetUseNullEndTag() { + type NullValStruct struct { + I int + B []byte + S string + } + s := new(NullValStruct) + + v, err := xml.Marshal(s) + if err != nil { + fmt.Println("err:", err.Error()) + } else { + fmt.Println("s:", string(v)) + } + + xml.SetUseNullEndTag(true) + v, err = xml.Marshal(s) + if err != nil { + fmt.Println("err:", err.Error()) + } else { + fmt.Println("s:", string(v)) + } + + type NewStruct struct { + NVS NullValStruct + S string + F float64 + } + ss := new(NewStruct) + + v, err = xml.Marshal(ss) + if err != nil { + fmt.Println("err:", err.Error()) + } else { + fmt.Println("ss:", string(v)) + } + + v, err = xml.MarshalIndent(ss," "," ") + if err != nil { + fmt.Println("err:", err.Error()) + } else { + fmt.Println("ss indent:\n",string(v)) + } + // Output: + // s: 0 + // s: 0 + // ss: 00 + // ss indent: + // + // + // 0 + // + // + // + // + // 0 + // +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/marshal.go.v1_2 b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/marshal.go.v1_2 new file mode 100644 index 000000000..b51d0c1e8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_marshal/marshal.go.v1_2 @@ -0,0 +1,1034 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xml + +import ( + "bufio" + "bytes" + "encoding" + "fmt" + "io" + "reflect" + "strconv" + "strings" +) + +const ( + // A generic XML header suitable for use with the output of Marshal. + // This is not automatically added to any output of this package, + // it is provided as a convenience. + Header = `` + "\n" +) + +// Marshal returns the XML encoding of v. +// +// Marshal handles an array or slice by marshalling each of the elements. +// Marshal handles a pointer by marshalling the value it points at or, if the +// pointer is nil, by writing nothing. Marshal handles an interface value by +// marshalling the value it contains or, if the interface value is nil, by +// writing nothing. Marshal handles all other data by writing one or more XML +// elements containing the data. +// +// The name for the XML elements is taken from, in order of preference: +// - the tag on the XMLName field, if the data is a struct +// - the value of the XMLName field of type xml.Name +// - the tag of the struct field used to obtain the data +// - the name of the struct field used to obtain the data +// - the name of the marshalled type +// +// The XML element for a struct contains marshalled elements for each of the +// exported fields of the struct, with these exceptions: +// - the XMLName field, described above, is omitted. +// - a field with tag "-" is omitted. +// - a field with tag "name,attr" becomes an attribute with +// the given name in the XML element. +// - a field with tag ",attr" becomes an attribute with the +// field name in the XML element. +// - a field with tag ",chardata" is written as character data, +// not as an XML element. +// - a field with tag ",innerxml" is written verbatim, not subject +// to the usual marshalling procedure. +// - a field with tag ",comment" is written as an XML comment, not +// subject to the usual marshalling procedure. It must not contain +// the "--" string within it. +// - a field with a tag including the "omitempty" option is omitted +// if the field value is empty. The empty values are false, 0, any +// nil pointer or interface value, and any array, slice, map, or +// string of length zero. +// - an anonymous struct field is handled as if the fields of its +// value were part of the outer struct. +// +// If a field uses a tag "a>b>c", then the element c will be nested inside +// parent elements a and b. Fields that appear next to each other that name +// the same parent will be enclosed in one XML element. +// +// See MarshalIndent for an example. +// +// Marshal will return an error if asked to marshal a channel, function, or map. +func Marshal(v interface{}) ([]byte, error) { + var b bytes.Buffer + if err := NewEncoder(&b).Encode(v); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// Marshaler is the interface implemented by objects that can marshal +// themselves into valid XML elements. +// +// MarshalXML encodes the receiver as zero or more XML elements. +// By convention, arrays or slices are typically encoded as a sequence +// of elements, one per entry. +// Using start as the element tag is not required, but doing so +// will enable Unmarshal to match the XML elements to the correct +// struct field. +// One common implementation strategy is to construct a separate +// value with a layout corresponding to the desired XML and then +// to encode it using e.EncodeElement. +// Another common strategy is to use repeated calls to e.EncodeToken +// to generate the XML output one token at a time. +// The sequence of encoded tokens must make up zero or more valid +// XML elements. +type Marshaler interface { + MarshalXML(e *Encoder, start StartElement) error +} + +// MarshalerAttr is the interface implemented by objects that can marshal +// themselves into valid XML attributes. +// +// MarshalXMLAttr returns an XML attribute with the encoded value of the receiver. +// Using name as the attribute name is not required, but doing so +// will enable Unmarshal to match the attribute to the correct +// struct field. +// If MarshalXMLAttr returns the zero attribute Attr{}, no attribute +// will be generated in the output. +// MarshalXMLAttr is used only for struct fields with the +// "attr" option in the field tag. +type MarshalerAttr interface { + MarshalXMLAttr(name Name) (Attr, error) +} + +// MarshalIndent works like Marshal, but each XML element begins on a new +// indented line that starts with prefix and is followed by one or more +// copies of indent according to the nesting depth. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + var b bytes.Buffer + enc := NewEncoder(&b) + enc.Indent(prefix, indent) + if err := enc.Encode(v); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// An Encoder writes XML data to an output stream. +type Encoder struct { + p printer +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + e := &Encoder{printer{Writer: bufio.NewWriter(w)}} + e.p.encoder = e + return e +} + +// Indent sets the encoder to generate XML in which each element +// begins on a new indented line that starts with prefix and is followed by +// one or more copies of indent according to the nesting depth. +func (enc *Encoder) Indent(prefix, indent string) { + enc.p.prefix = prefix + enc.p.indent = indent +} + +// Encode writes the XML encoding of v to the stream. +// +// See the documentation for Marshal for details about the conversion +// of Go values to XML. +// +// Encode calls Flush before returning. +func (enc *Encoder) Encode(v interface{}) error { + err := enc.p.marshalValue(reflect.ValueOf(v), nil, nil) + if err != nil { + return err + } + return enc.p.Flush() +} + +// EncodeElement writes the XML encoding of v to the stream, +// using start as the outermost tag in the encoding. +// +// See the documentation for Marshal for details about the conversion +// of Go values to XML. +// +// EncodeElement calls Flush before returning. +func (enc *Encoder) EncodeElement(v interface{}, start StartElement) error { + err := enc.p.marshalValue(reflect.ValueOf(v), nil, &start) + if err != nil { + return err + } + return enc.p.Flush() +} + +var ( + endComment = []byte("-->") + endProcInst = []byte("?>") + endDirective = []byte(">") +) + +// EncodeToken writes the given XML token to the stream. +// It returns an error if StartElement and EndElement tokens are not properly matched. +// +// EncodeToken does not call Flush, because usually it is part of a larger operation +// such as Encode or EncodeElement (or a custom Marshaler's MarshalXML invoked +// during those), and those will call Flush when finished. +// +// Callers that create an Encoder and then invoke EncodeToken directly, without +// using Encode or EncodeElement, need to call Flush when finished to ensure +// that the XML is written to the underlying writer. +func (enc *Encoder) EncodeToken(t Token) error { + p := &enc.p + switch t := t.(type) { + case StartElement: + if err := p.writeStart(&t); err != nil { + return err + } + case EndElement: + if err := p.writeEnd(t.Name); err != nil { + return err + } + case CharData: + EscapeText(p, t) + case Comment: + if bytes.Contains(t, endComment) { + return fmt.Errorf("xml: EncodeToken of Comment containing --> marker") + } + p.WriteString("") + return p.cachedWriteError() + case ProcInst: + if t.Target == "xml" || !isNameString(t.Target) { + return fmt.Errorf("xml: EncodeToken of ProcInst with invalid Target") + } + if bytes.Contains(t.Inst, endProcInst) { + return fmt.Errorf("xml: EncodeToken of ProcInst containing ?> marker") + } + p.WriteString(" 0 { + p.WriteByte(' ') + p.Write(t.Inst) + } + p.WriteString("?>") + case Directive: + if bytes.Contains(t, endDirective) { + return fmt.Errorf("xml: EncodeToken of Directive containing > marker") + } + p.WriteString("") + } + return p.cachedWriteError() +} + +// Flush flushes any buffered XML to the underlying writer. +// See the EncodeToken documentation for details about when it is necessary. +func (enc *Encoder) Flush() error { + return enc.p.Flush() +} + +type printer struct { + *bufio.Writer + encoder *Encoder + seq int + indent string + prefix string + depth int + indentedIn bool + putNewline bool + attrNS map[string]string // map prefix -> name space + attrPrefix map[string]string // map name space -> prefix + prefixes []string + tags []Name +} + +// createAttrPrefix finds the name space prefix attribute to use for the given name space, +// defining a new prefix if necessary. It returns the prefix. +func (p *printer) createAttrPrefix(url string) string { + if prefix := p.attrPrefix[url]; prefix != "" { + return prefix + } + + // The "http://www.w3.org/XML/1998/namespace" name space is predefined as "xml" + // and must be referred to that way. + // (The "http://www.w3.org/2000/xmlns/" name space is also predefined as "xmlns", + // but users should not be trying to use that one directly - that's our job.) + if url == xmlURL { + return "xml" + } + + // Need to define a new name space. + if p.attrPrefix == nil { + p.attrPrefix = make(map[string]string) + p.attrNS = make(map[string]string) + } + + // Pick a name. We try to use the final element of the path + // but fall back to _. + prefix := strings.TrimRight(url, "/") + if i := strings.LastIndex(prefix, "/"); i >= 0 { + prefix = prefix[i+1:] + } + if prefix == "" || !isName([]byte(prefix)) || strings.Contains(prefix, ":") { + prefix = "_" + } + if strings.HasPrefix(prefix, "xml") { + // xmlanything is reserved. + prefix = "_" + prefix + } + if p.attrNS[prefix] != "" { + // Name is taken. Find a better one. + for p.seq++; ; p.seq++ { + if id := prefix + "_" + strconv.Itoa(p.seq); p.attrNS[id] == "" { + prefix = id + break + } + } + } + + p.attrPrefix[url] = prefix + p.attrNS[prefix] = url + + p.WriteString(`xmlns:`) + p.WriteString(prefix) + p.WriteString(`="`) + EscapeText(p, []byte(url)) + p.WriteString(`" `) + + p.prefixes = append(p.prefixes, prefix) + + return prefix +} + +// deleteAttrPrefix removes an attribute name space prefix. +func (p *printer) deleteAttrPrefix(prefix string) { + delete(p.attrPrefix, p.attrNS[prefix]) + delete(p.attrNS, prefix) +} + +func (p *printer) markPrefix() { + p.prefixes = append(p.prefixes, "") +} + +func (p *printer) popPrefix() { + for len(p.prefixes) > 0 { + prefix := p.prefixes[len(p.prefixes)-1] + p.prefixes = p.prefixes[:len(p.prefixes)-1] + if prefix == "" { + break + } + p.deleteAttrPrefix(prefix) + } +} + +var ( + marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() + marshalerAttrType = reflect.TypeOf((*MarshalerAttr)(nil)).Elem() + textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() +) + +// CLB, 11/30/13 +var useNullEndTag bool + +// SetUseNullEndTag - 'true': have Marshal/MarshalIndent use in place of for empty element. +func SetUseNullEndTag(b bool) { + useNullEndTag = b +} + +// marshalValue writes one or more XML elements representing val. +// If val was obtained from a struct field, finfo must have its details. +func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplate *StartElement) error { + if startTemplate != nil && startTemplate.Name.Local == "" { + return fmt.Errorf("xml: EncodeElement of StartElement with missing name") + } + + if !val.IsValid() { + return nil + } + if finfo != nil && finfo.flags&fOmitEmpty != 0 && isEmptyValue(val) { + return nil + } + + // Drill into interfaces and pointers. + // This can turn into an infinite loop given a cyclic chain, + // but it matches the Go 1 behavior. + for val.Kind() == reflect.Interface || val.Kind() == reflect.Ptr { + if val.IsNil() { + return nil + } + val = val.Elem() + } + + kind := val.Kind() + typ := val.Type() + + // Check for marshaler. + if val.CanInterface() && typ.Implements(marshalerType) { + return p.marshalInterface(val.Interface().(Marshaler), defaultStart(typ, finfo, startTemplate)) + } + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(marshalerType) { + return p.marshalInterface(pv.Interface().(Marshaler), defaultStart(pv.Type(), finfo, startTemplate)) + } + } + + // Check for text marshaler. + if val.CanInterface() && typ.Implements(textMarshalerType) { + return p.marshalTextInterface(val.Interface().(encoding.TextMarshaler), defaultStart(typ, finfo, startTemplate)) + } + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + return p.marshalTextInterface(pv.Interface().(encoding.TextMarshaler), defaultStart(pv.Type(), finfo, startTemplate)) + } + } + + // Slices and arrays iterate over the elements. They do not have an enclosing tag. + if (kind == reflect.Slice || kind == reflect.Array) && typ.Elem().Kind() != reflect.Uint8 { + for i, n := 0, val.Len(); i < n; i++ { + if err := p.marshalValue(val.Index(i), finfo, startTemplate); err != nil { + return err + } + } + return nil + } + + tinfo, err := getTypeInfo(typ) + if err != nil { + return err + } + + // Create start element. + // Precedence for the XML element name is: + // 0. startTemplate + // 1. XMLName field in underlying struct; + // 2. field name/tag in the struct field; and + // 3. type name + var start StartElement + + if startTemplate != nil { + start.Name = startTemplate.Name + start.Attr = append(start.Attr, startTemplate.Attr...) + } else if tinfo.xmlname != nil { + xmlname := tinfo.xmlname + if xmlname.name != "" { + start.Name.Space, start.Name.Local = xmlname.xmlns, xmlname.name + } else if v, ok := xmlname.value(val).Interface().(Name); ok && v.Local != "" { + start.Name = v + } + } + if start.Name.Local == "" && finfo != nil { + start.Name.Space, start.Name.Local = finfo.xmlns, finfo.name + } + if start.Name.Local == "" { + name := typ.Name() + if name == "" { + return &UnsupportedTypeError{typ} + } + start.Name.Local = name + } + + // Attributes + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + if finfo.flags&fAttr == 0 { + continue + } + fv := finfo.value(val) + name := Name{Space: finfo.xmlns, Local: finfo.name} + + if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { + continue + } + + if fv.Kind() == reflect.Interface && fv.IsNil() { + continue + } + + if fv.CanInterface() && fv.Type().Implements(marshalerAttrType) { + attr, err := fv.Interface().(MarshalerAttr).MarshalXMLAttr(name) + if err != nil { + return err + } + if attr.Name.Local != "" { + start.Attr = append(start.Attr, attr) + } + continue + } + + if fv.CanAddr() { + pv := fv.Addr() + if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { + attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) + if err != nil { + return err + } + if attr.Name.Local != "" { + start.Attr = append(start.Attr, attr) + } + continue + } + } + + if fv.CanInterface() && fv.Type().Implements(textMarshalerType) { + text, err := fv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + start.Attr = append(start.Attr, Attr{name, string(text)}) + continue + } + + if fv.CanAddr() { + pv := fv.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + start.Attr = append(start.Attr, Attr{name, string(text)}) + continue + } + } + + // Dereference or skip nil pointer, interface values. + switch fv.Kind() { + case reflect.Ptr, reflect.Interface: + if fv.IsNil() { + continue + } + fv = fv.Elem() + } + + s, b, err := p.marshalSimple(fv.Type(), fv) + if err != nil { + return err + } + if b != nil { + s = string(b) + } + start.Attr = append(start.Attr, Attr{name, s}) + } + +/* CLB, 11/30/13 + if err := p.writeStart(&start); err != nil { + return err + } +*/ + + // CLB, 11/30/13 + var noVal bool + + if val.Kind() == reflect.Struct { + if err := p.writeStart(&start); err != nil { + return err + } + err = p.marshalStruct(tinfo, val) + } else { + s, b, err1 := p.marshalSimple(typ, val) + // CLB, 11/30/13 + if s == "" && len(b) == 0 && useNullEndTag { + if err := p.writeStartDone(&start); err != nil { + return err + } + noVal = true + goto finishProc + } + if err := p.writeStart(&start); err != nil { + return err + } + if err1 != nil { + err = err1 + } else if b != nil { + EscapeText(p, b) + } else { + p.EscapeString(s) + } + } + if err != nil { + return err + } + +// CLB, 11/30/13 +finishProc: + + if noVal && useNullEndTag { + if err := p.writeEndDone(start.Name); err != nil { + return err + } + } else if err := p.writeEnd(start.Name); err != nil { + return err + } + + return p.cachedWriteError() +} + +// defaultStart returns the default start element to use, +// given the reflect type, field info, and start template. +func defaultStart(typ reflect.Type, finfo *fieldInfo, startTemplate *StartElement) StartElement { + var start StartElement + // Precedence for the XML element name is as above, + // except that we do not look inside structs for the first field. + if startTemplate != nil { + start.Name = startTemplate.Name + start.Attr = append(start.Attr, startTemplate.Attr...) + } else if finfo != nil && finfo.name != "" { + start.Name.Local = finfo.name + start.Name.Space = finfo.xmlns + } else if typ.Name() != "" { + start.Name.Local = typ.Name() + } else { + // Must be a pointer to a named type, + // since it has the Marshaler methods. + start.Name.Local = typ.Elem().Name() + } + return start +} + +// marshalInterface marshals a Marshaler interface value. +func (p *printer) marshalInterface(val Marshaler, start StartElement) error { + // Push a marker onto the tag stack so that MarshalXML + // cannot close the XML tags that it did not open. + p.tags = append(p.tags, Name{}) + n := len(p.tags) + + err := val.MarshalXML(p.encoder, start) + if err != nil { + return err + } + + // Make sure MarshalXML closed all its tags. p.tags[n-1] is the mark. + if len(p.tags) > n { + return fmt.Errorf("xml: %s.MarshalXML wrote invalid XML: <%s> not closed", receiverType(val), p.tags[len(p.tags)-1].Local) + } + p.tags = p.tags[:n-1] + return nil +} + +// marshalTextInterface marshals a TextMarshaler interface value. +func (p *printer) marshalTextInterface(val encoding.TextMarshaler, start StartElement) error { + if err := p.writeStart(&start); err != nil { + return err + } + text, err := val.MarshalText() + if err != nil { + return err + } + EscapeText(p, text) + return p.writeEnd(start.Name) +} + +// writeStart writes the given start element. +func (p *printer) writeStart(start *StartElement) error { + if start.Name.Local == "" { + return fmt.Errorf("xml: start tag with no name") + } + + p.tags = append(p.tags, start.Name) + p.markPrefix() + + p.writeIndent(1) + p.WriteByte('<') + p.WriteString(start.Name.Local) + + if start.Name.Space != "" { + p.WriteString(` xmlns="`) + p.EscapeString(start.Name.Space) + p.WriteByte('"') + } + + // Attributes + for _, attr := range start.Attr { + name := attr.Name + if name.Local == "" { + continue + } + p.WriteByte(' ') + if name.Space != "" { + p.WriteString(p.createAttrPrefix(name.Space)) + p.WriteByte(':') + } + p.WriteString(name.Local) + p.WriteString(`="`) + p.EscapeString(attr.Value) + p.WriteByte('"') + } + p.WriteByte('>') + return nil +} + +// CLB, 11/30/13 - +// writeStartDone writes the given start element. +func (p *printer) writeStartDone(start *StartElement) error { + if start.Name.Local == "" { + return fmt.Errorf("xml: start tag with no name") + } + + p.tags = append(p.tags, start.Name) + p.markPrefix() + + p.writeIndent(1) + p.WriteByte('<') + p.WriteString(start.Name.Local) + + if start.Name.Space != "" { + p.WriteString(` xmlns="`) + p.EscapeString(start.Name.Space) + p.WriteByte('"') + } + + // Attributes + for _, attr := range start.Attr { + name := attr.Name + if name.Local == "" { + continue + } + p.WriteByte(' ') + if name.Space != "" { + p.WriteString(p.createAttrPrefix(name.Space)) + p.WriteByte(':') + } + p.WriteString(name.Local) + p.WriteString(`="`) + p.EscapeString(attr.Value) + p.WriteByte('"') + } + // CLB, 11/30/13 + p.WriteString(`/>`) + return nil +} + +func (p *printer) writeEnd(name Name) error { + if name.Local == "" { + return fmt.Errorf("xml: end tag with no name") + } + if len(p.tags) == 0 || p.tags[len(p.tags)-1].Local == "" { + return fmt.Errorf("xml: end tag without start tag", name.Local) + } + if top := p.tags[len(p.tags)-1]; top != name { + if top.Local != name.Local { + return fmt.Errorf("xml: end tag does not match start tag <%s>", name.Local, top.Local) + } + return fmt.Errorf("xml: end tag in namespace %s does not match start tag <%s> in namespace %s", name.Local, name.Space, top.Local, top.Space) + } + p.tags = p.tags[:len(p.tags)-1] + + p.writeIndent(-1) + p.WriteByte('<') + p.WriteByte('/') + p.WriteString(name.Local) + p.WriteByte('>') + p.popPrefix() + return nil +} + +// CLB, 11/30/13 +func (p *printer) writeEndDone(name Name) error { + if name.Local == "" { + return fmt.Errorf("xml: end tag with no name") + } + if len(p.tags) == 0 || p.tags[len(p.tags)-1].Local == "" { + return fmt.Errorf("xml: end tag without start tag", name.Local) + } + if top := p.tags[len(p.tags)-1]; top != name { + if top.Local != name.Local { + return fmt.Errorf("xml: end tag does not match start tag <%s>", name.Local, top.Local) + } + return fmt.Errorf("xml: end tag in namespace %s does not match start tag <%s> in namespace %s", name.Local, name.Space, top.Local, top.Space) + } + p.tags = p.tags[:len(p.tags)-1] + +// CLB, 11/30/13 + p.writeIndent(-1) +/* + p.WriteByte('<') + p.WriteByte('/') + p.WriteString(name.Local) + p.WriteByte('>') +*/ + p.popPrefix() + return nil +} + +func (p *printer) marshalSimple(typ reflect.Type, val reflect.Value) (string, []byte, error) { + switch val.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(val.Int(), 10), nil, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return strconv.FormatUint(val.Uint(), 10), nil, nil + case reflect.Float32, reflect.Float64: + return strconv.FormatFloat(val.Float(), 'g', -1, val.Type().Bits()), nil, nil + case reflect.String: + return val.String(), nil, nil + case reflect.Bool: + return strconv.FormatBool(val.Bool()), nil, nil + case reflect.Array: + if typ.Elem().Kind() != reflect.Uint8 { + break + } + // [...]byte + var bytes []byte + if val.CanAddr() { + bytes = val.Slice(0, val.Len()).Bytes() + } else { + bytes = make([]byte, val.Len()) + reflect.Copy(reflect.ValueOf(bytes), val) + } + return "", bytes, nil + case reflect.Slice: + if typ.Elem().Kind() != reflect.Uint8 { + break + } + // []byte + return "", val.Bytes(), nil + } + return "", nil, &UnsupportedTypeError{typ} +} + +var ddBytes = []byte("--") + +func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { + s := parentStack{p: p} + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + if finfo.flags&fAttr != 0 { + continue + } + vf := finfo.value(val) + + // Dereference or skip nil pointer, interface values. + switch vf.Kind() { + case reflect.Ptr, reflect.Interface: + if !vf.IsNil() { + vf = vf.Elem() + } + } + + switch finfo.flags & fMode { + case fCharData: + if vf.CanInterface() && vf.Type().Implements(textMarshalerType) { + data, err := vf.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + Escape(p, data) + continue + } + if vf.CanAddr() { + pv := vf.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + data, err := pv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + Escape(p, data) + continue + } + } + var scratch [64]byte + switch vf.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + Escape(p, strconv.AppendInt(scratch[:0], vf.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + Escape(p, strconv.AppendUint(scratch[:0], vf.Uint(), 10)) + case reflect.Float32, reflect.Float64: + Escape(p, strconv.AppendFloat(scratch[:0], vf.Float(), 'g', -1, vf.Type().Bits())) + case reflect.Bool: + Escape(p, strconv.AppendBool(scratch[:0], vf.Bool())) + case reflect.String: + if err := EscapeText(p, []byte(vf.String())); err != nil { + return err + } + case reflect.Slice: + if elem, ok := vf.Interface().([]byte); ok { + if err := EscapeText(p, elem); err != nil { + return err + } + } + } + continue + + case fComment: + k := vf.Kind() + if !(k == reflect.String || k == reflect.Slice && vf.Type().Elem().Kind() == reflect.Uint8) { + return fmt.Errorf("xml: bad type for comment field of %s", val.Type()) + } + if vf.Len() == 0 { + continue + } + p.writeIndent(0) + p.WriteString("" is invalid grammar. Make it "- -->" + p.WriteByte(' ') + } + p.WriteString("-->") + continue + + case fInnerXml: + iface := vf.Interface() + switch raw := iface.(type) { + case []byte: + p.Write(raw) + continue + case string: + p.WriteString(raw) + continue + } + + case fElement, fElement | fAny: + if err := s.trim(finfo.parents); err != nil { + return err + } + if len(finfo.parents) > len(s.stack) { + if vf.Kind() != reflect.Ptr && vf.Kind() != reflect.Interface || !vf.IsNil() { + if err := s.push(finfo.parents[len(s.stack):]); err != nil { + return err + } + } + } + } + if err := p.marshalValue(vf, finfo, nil); err != nil { + return err + } + } + s.trim(nil) + return p.cachedWriteError() +} + +// return the bufio Writer's cached write error +func (p *printer) cachedWriteError() error { + _, err := p.Write(nil) + return err +} + +func (p *printer) writeIndent(depthDelta int) { + if len(p.prefix) == 0 && len(p.indent) == 0 { + return + } + if depthDelta < 0 { + p.depth-- + if p.indentedIn { + p.indentedIn = false + return + } + p.indentedIn = false + } + if p.putNewline { + p.WriteByte('\n') + } else { + p.putNewline = true + } + if len(p.prefix) > 0 { + p.WriteString(p.prefix) + } + if len(p.indent) > 0 { + for i := 0; i < p.depth; i++ { + p.WriteString(p.indent) + } + } + if depthDelta > 0 { + p.depth++ + p.indentedIn = true + } +} + +type parentStack struct { + p *printer + stack []string +} + +// trim updates the XML context to match the longest common prefix of the stack +// and the given parents. A closing tag will be written for every parent +// popped. Passing a zero slice or nil will close all the elements. +func (s *parentStack) trim(parents []string) error { + split := 0 + for ; split < len(parents) && split < len(s.stack); split++ { + if parents[split] != s.stack[split] { + break + } + } + for i := len(s.stack) - 1; i >= split; i-- { + if err := s.p.writeEnd(Name{Local: s.stack[i]}); err != nil { + return err + } + } + s.stack = parents[:split] + return nil +} + +// push adds parent elements to the stack and writes open tags. +func (s *parentStack) push(parents []string) error { + for i := 0; i < len(parents); i++ { + if err := s.p.writeStart(&StartElement{Name: Name{Local: parents[i]}}); err != nil { + return err + } + } + s.stack = append(s.stack, parents...) + return nil +} + +// A MarshalXMLError is returned when Marshal encounters a type +// that cannot be converted into XML. +type UnsupportedTypeError struct { + Type reflect.Type +} + +func (e *UnsupportedTypeError) Error() string { + return "xml: unsupported type: " + e.Type.String() +} + +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} diff --git a/Godeps/_workspace/src/github.com/clbanning/mxj/xml_test.go b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_test.go new file mode 100644 index 000000000..fc60d6516 --- /dev/null +++ b/Godeps/_workspace/src/github.com/clbanning/mxj/xml_test.go @@ -0,0 +1,207 @@ +package mxj + +import ( + "bytes" + "fmt" + "io" + "testing" +) + +func TestXmlHeader(t *testing.T) { + fmt.Println("\n---------------- xml_test.go ...\n") +} + +func TestNewMapXml(t *testing.T) { + x := []byte(`something more12`) + + mv, merr := NewMapXml(x) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + + fmt.Println("NewMapXml, x :", string(x)) + fmt.Println("NewMapXml, mv:", mv) +} + +func TestAttrHyphenFalse(t *testing.T) { + PrependAttrWithHyphen(false) + x := []byte(`something more12`) + + mv, merr := NewMapXml(x) + if merr != nil { + t.Fatal("merr:", merr.Error()) + } + + fmt.Println("AttrHyphenFalse, x :", string(x)) + fmt.Println("AttrHyphenFalse, mv:", mv) + PrependAttrWithHyphen(true) +} + +func TestNewMapXmlError(t *testing.T) { + x := []byte(`something more12`) + + m, merr := NewMapJson(x) + if merr == nil { + t.Fatal("NewMapXmlError, m:", m) + } + + fmt.Println("NewMapXmlError, x :", string(x)) + fmt.Println("NewMapXmlError, merr:", merr.Error()) + + x = []byte(`something more12`) + m, merr = NewMapJson(x) + if merr == nil { + t.Fatal("NewMapXmlError, m:", m) + } + + fmt.Println("NewMapXmlError, x :", string(x)) + fmt.Println("NewMapXmlError, merr:", merr.Error()) +} + +func TestNewMapXmlReader(t *testing.T) { + x := []byte(`is a testsomething more12`) + + r := bytes.NewReader(x) + + for { + m, raw, err := NewMapXmlReaderRaw(r) + if err != nil && err != io.EOF { + t.Fatal("err:", err.Error()) + } + if err == io.EOF && len(m) == 0 { + break + } + fmt.Println("NewMapXmlReader, raw:", string(raw)) + fmt.Println("NewMapXmlReader, m :", m) + } +} + +// --------------------- Xml() and XmlWriter() test cases ------------------- + +func TestXml_1(t *testing.T) { + mv := Map{"tag1": "some data", "tag2": "more data", "boolean": true, "float": 3.14159625, "null":nil} + + x, err := mv.Xml() + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("Xml_1, mv:", mv) + fmt.Println("Xml_1, x :", string(x)) +} + +func TestXml_2(t *testing.T) { + a := []interface{}{ "string", true, 36.4 } + mv := Map{"array": a } + + x, err := mv.Xml() + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("Xml_2, mv:", mv) + fmt.Println("Xml_2, x :", string(x)) +} + +func TestXml_3(t *testing.T) { + a := []interface{}{ "string", true, 36.4 } + mv := Map{"array": []interface{}{a, "string2"} } + + x, err := mv.Xml() + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("Xml_3, mv:", mv) + fmt.Println("Xml_3, x :", string(x)) +} + +func TestXml_4(t *testing.T) { + a := []interface{}{ "string", true, 36.4 } + mv := Map{"array": map[string]interface{}{ "innerkey": []interface{}{a, "string2"} } } + + x, err := mv.Xml() + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("Xml_4, mv:", mv) + fmt.Println("Xml_4, x :", string(x)) +} + +func TestXml_5(t *testing.T) { + a := []interface{}{ "string", true, 36.4 } + mv := Map{"array": []interface{}{ map[string]interface{}{ "innerkey": []interface{}{a, "string2"} }, map[string]interface{}{"some":"more"} } } + + x, err := mv.Xml() + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("Xml_5, mv:", mv) + fmt.Println("Xml_5, x :", string(x)) +} + +func TestXmlWriter(t *testing.T) { + mv := Map{"tag1": "some data", "tag2": "more data", "boolean": true, "float": 3.14159625} + w := new(bytes.Buffer) + + raw, err := mv.XmlWriterRaw(w, "myRootTag") + if err != nil { + t.Fatal("err:",err.Error()) + } + + b := make([]byte,w.Len()) + _, err = w.Read(b) + if err != nil { + t.Fatal("err:", err.Error()) + } + + fmt.Println("XmlWriter, raw:", string(raw)) + fmt.Println("XmlWriter, b :", string(b)) +} + +// -------------------------- XML Handler test cases ------------------------- + +/* tested in bulk_test.go ... +var xhdata = []byte(`is a testsomething more12`) + +func TestHandleXmlReader(t *testing.T) { + fmt.Println("HandleXmlReader:", string(xhdata)) + + rdr := bytes.NewReader(xhdata) + err := HandleXmlReader(rdr, xmhandler, xehandler) + if err != nil { + t.Fatal("err:", err.Error()) + } +} + +var xt *testing.T + +func xmhandler(m Map, raw []byte) bool { + x, xerr := m.Xml() + if xerr != nil { + xt.Fatal("... xmhandler:", xerr.Error()) + return false + } + + fmt.Println("... xmhandler, raw:", string(raw)) + fmt.Println("... xmhandler, x :", string(x)) + return true +} + +func xehandler(err error, raw []byte) bool { + if err == nil { + // shouldn't be here + xt.Fatal("... xehandler: ") + return false + } + if err == io.EOF { + return true + } + + fmt.Println("... xehandler raw:", string(raw)) + fmt.Println("... xehandler err:", err.Error()) + return true +} +*/ diff --git a/Godeps/_workspace/src/github.com/fatih/structs/.gitignore b/Godeps/_workspace/src/github.com/fatih/structs/.gitignore new file mode 100644 index 000000000..836562412 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/Godeps/_workspace/src/github.com/fatih/structs/.travis.yml b/Godeps/_workspace/src/github.com/fatih/structs/.travis.yml new file mode 100644 index 000000000..28381ef8e --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/.travis.yml @@ -0,0 +1,11 @@ +language: go +go: 1.3 +before_install: +- go get github.com/axw/gocov/gocov +- go get github.com/mattn/goveralls +- go get code.google.com/p/go.tools/cmd/cover +script: +- $HOME/gopath/bin/goveralls -repotoken $COVERALLS_TOKEN +env: + global: + - secure: hkc+92KPmMFqIH9n4yWdnH1JpZjahmOyDJwpTh8Yl0JieJNG0XEXpOqNao27eA0cLF+UHdyjFeGcPUJKNmgE46AoQjtovt+ICjCXKR2yF6S2kKJcUOz/Vd6boZF7qHV06jjxyxOebpID5iSoW6UfFr001bFxpd3jaSLFTzSHWRQ= diff --git a/Godeps/_workspace/src/github.com/fatih/structs/LICENSE b/Godeps/_workspace/src/github.com/fatih/structs/LICENSE new file mode 100644 index 000000000..34504e4b3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Fatih Arslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/fatih/structs/README.md b/Godeps/_workspace/src/github.com/fatih/structs/README.md new file mode 100644 index 000000000..87fccf24a --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/README.md @@ -0,0 +1,159 @@ +# Structs [![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/fatih/structs) [![Build Status](http://img.shields.io/travis/fatih/structs.svg?style=flat-square)](https://travis-ci.org/fatih/structs) [![Coverage Status](http://img.shields.io/coveralls/fatih/structs.svg?style=flat-square)](https://coveralls.io/r/fatih/structs) + +Structs contains various utilities to work with Go (Golang) structs. It was +initially used by me to convert a struct into a `map[string]interface{}`. With +time I've added other utilities for structs. It's basically a high level +package based on primitives from the reflect package. Feel free to add new +functions or improve the existing code. + +## Install + +```bash +go get github.com/fatih/structs +``` + +## Usage and Examples + +Just like the standard lib `strings`, `bytes` and co packages, `structs` has +many global functions to manipulate or organize your struct data. Lets define +and declare a struct: + +```go +type Server struct { + Name string `json:"name,omitempty"` + ID int + Enabled bool + users []string // not exported + http.Server // embedded +} + +server := &Server{ + Name: "gopher", + ID: 123456, + Enabled: true, +} +``` + +```go +// Convert a struct to a map[string]interface{} +// => {"Name":"gopher", "ID":123456, "Enabled":true} +m := structs.Map(server) + +// Convert the values of a struct to a []interface{} +// => ["gopher", 123456, true] +v := structs.Values(server) + +// Convert the values of a struct to a []*Field +// (see "Field methods" for more info about fields) +f := structs.Fields(server) + +// Return the struct name => "Server" +n := structs.Name(server) + +// Check if any field of a struct is initialized or not. +h := structs.HasZero(server) + +// Check if all fields of a struct is initialized or not. +z := structs.IsZero(server) + +// Check if server is a struct or a pointer to struct +i := structs.IsStruct(server) +``` + +### Struct methods + +The structs functions can be also used as independent methods by creating a new +`*structs.Struct`. This is handy if you want to have more control over the +structs (such as retrieving a single Field). + +```go +// Create a new struct type: +s := structs.New(server) + +m := s.Map() // Get a map[string]interface{} +v := s.Values() // Get a []interface{} +f := s.Fields() // Get a []*Field +f := s.Field(name) // Get a *Field based on the given field name +f, ok := s.FieldsOk(name) // Get a *Field based on the given field name +n := s.Name() // Get the struct name +h := s.HasZero() // Check if any field is initialized +z := s.IsZero() // Check if all fields are initialized +``` + +### Field methods + +We can easily examine a single Field for more detail. Below you can see how we +get and interact with various field methods: + + +```go +s := structs.New(server) + +// Get the Field struct for the "Name" field +name := s.Field("Name") + +// Get the underlying value, value => "gopher" +value := name.Value().(string) + +// Set the field's value +name.Set("another gopher") + +// Get the field's kind, kind => "string" +name.Kind() + +// Check if the field is exported or not +if name.IsExported() { + fmt.Println("Name field is exported") +} + +// Check if the value is a zero value, such as "" for string, 0 for int +if !name.IsZero() { + fmt.Println("Name is initialized") +} + +// Check if the field is an anonymous (embedded) field +if !name.IsEmbedded() { + fmt.Println("Name is not an embedded field") +} + +// Get the Field's tag value for tag name "json", tag value => "name,omitempty" +tagValue := name.Tag("json") +``` + +Nested structs are supported too: + +```go +addrField := s.Field("Server").Field("Addr") + +// Get the value for addr +a := addrField.Value().(string) + +// Or get all fields +httpServer := s.Field("Server").Fields() +``` + +We can also get a slice of Fields from the Struct type to iterate over all +fields. This is handy if you wish to examine all fields: + +```go +// Convert the fields of a struct to a []*Field +fields := s.Fields() + +for _, f := range fields { + fmt.Printf("field name: %+v\n", f.Name()) + + if f.IsExported() { + fmt.Printf("value : %+v\n", f.Value()) + fmt.Printf("is zero : %+v\n", f.IsZero()) + } +} +``` + +## Credits + + * [Fatih Arslan](https://github.com/fatih) + * [Cihangir Savas](https://github.com/cihangir) + +## License + +The MIT License (MIT) - see LICENSE.md for more details diff --git a/Godeps/_workspace/src/github.com/fatih/structs/field.go b/Godeps/_workspace/src/github.com/fatih/structs/field.go new file mode 100644 index 000000000..4b5e15b06 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/field.go @@ -0,0 +1,126 @@ +package structs + +import ( + "errors" + "fmt" + "reflect" +) + +var ( + errNotExported = errors.New("field is not exported") + errNotSettable = errors.New("field is not settable") +) + +// Field represents a single struct field that encapsulates high level +// functions around the field. +type Field struct { + value reflect.Value + field reflect.StructField + defaultTag string +} + +// Tag returns the value associated with key in the tag string. If there is no +// such key in the tag, Tag returns the empty string. +func (f *Field) Tag(key string) string { + return f.field.Tag.Get(key) +} + +// Value returns the underlying value of of the field. It panics if the field +// is not exported. +func (f *Field) Value() interface{} { + return f.value.Interface() +} + +// IsEmbedded returns true if the given field is an anonymous field (embedded) +func (f *Field) IsEmbedded() bool { + return f.field.Anonymous +} + +// IsExported returns true if the given field is exported. +func (f *Field) IsExported() bool { + return f.field.PkgPath == "" +} + +// IsZero returns true if the given field is not initalized (has a zero value). +// It panics if the field is not exported. +func (f *Field) IsZero() bool { + zero := reflect.Zero(f.value.Type()).Interface() + current := f.Value() + + return reflect.DeepEqual(current, zero) +} + +// Name returns the name of the given field +func (f *Field) Name() string { + return f.field.Name +} + +// Kind returns the fields kind, such as "string", "map", "bool", etc .. +func (f *Field) Kind() reflect.Kind { + return f.value.Kind() +} + +// Set sets the field to given value v. It retuns an error if the field is not +// settable (not addresable or not exported) or if the given value's type +// doesn't match the fields type. +func (f *Field) Set(val interface{}) error { + // we can't set unexported fields, so be sure this field is exported + if !f.IsExported() { + return errNotExported + } + + // do we get here? not sure... + if !f.value.CanSet() { + return errNotSettable + } + + given := reflect.ValueOf(val) + + if f.value.Kind() != given.Kind() { + return fmt.Errorf("wrong kind. got: %s want: %s", given.Kind(), f.value.Kind()) + } + + f.value.Set(given) + return nil +} + +// Fields returns a slice of Fields. This is particular handy to get the fields +// of a nested struct . A struct tag with the content of "-" ignores the +// checking of that particular field. Example: +// +// // Field is ignored by this package. +// Field *http.Request `structs:"-"` +// +// It panics if field is not exported or if field's kind is not struct +func (f *Field) Fields() []*Field { + return getFields(f.value, f.defaultTag) +} + +// Field returns the field from a nested struct. It panics if the nested struct +// is not exported or if the field was not found. +func (f *Field) Field(name string) *Field { + field, ok := f.FieldOk(name) + if !ok { + panic("field not found") + } + + return field +} + +// Field returns the field from a nested struct. The boolean returns true if +// the field was found. It panics if the nested struct is not exported or if +// the field was not found. +func (f *Field) FieldOk(name string) (*Field, bool) { + v := strctVal(f.value.Interface()) + t := v.Type() + + field, ok := t.FieldByName(name) + if !ok { + return nil, false + } + + return &Field{ + field: field, + value: v.FieldByName(name), + }, true +} diff --git a/Godeps/_workspace/src/github.com/fatih/structs/field_test.go b/Godeps/_workspace/src/github.com/fatih/structs/field_test.go new file mode 100644 index 000000000..46187d655 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/field_test.go @@ -0,0 +1,324 @@ +package structs + +import ( + "reflect" + "testing" +) + +// A test struct that defines all cases +type Foo struct { + A string + B int `structs:"y"` + C bool `json:"c"` + d string // not exported + E *Baz + x string `xml:"x"` // not exported, with tag + Y []string + Z map[string]interface{} + *Bar // embedded +} + +type Baz struct { + A string + B int +} + +type Bar struct { + E string + F int + g []string +} + +func newStruct() *Struct { + b := &Bar{ + E: "example", + F: 2, + g: []string{"zeynep", "fatih"}, + } + + // B and x is not initialized for testing + f := &Foo{ + A: "gopher", + C: true, + d: "small", + E: nil, + Y: []string{"example"}, + Z: nil, + } + f.Bar = b + + return New(f) +} + +func TestField_Set(t *testing.T) { + s := newStruct() + + f := s.Field("A") + err := f.Set("fatih") + if err != nil { + t.Error(err) + } + + if f.Value().(string) != "fatih" { + t.Errorf("Setted value is wrong: %s want: %s", f.Value().(string), "fatih") + } + + f = s.Field("Y") + err = f.Set([]string{"override", "with", "this"}) + if err != nil { + t.Error(err) + } + + sliceLen := len(f.Value().([]string)) + if sliceLen != 3 { + t.Errorf("Setted values slice length is wrong: %d, want: %d", sliceLen, 3) + } + + f = s.Field("C") + err = f.Set(false) + if err != nil { + t.Error(err) + } + + if f.Value().(bool) { + t.Errorf("Setted value is wrong: %s want: %s", f.Value().(bool), false) + } + + // let's pass a different type + f = s.Field("A") + err = f.Set(123) // Field A is of type string, but we are going to pass an integer + if err == nil { + t.Error("Setting a field's value with a different type than the field's type should return an error") + } + + // old value should be still there :) + if f.Value().(string) != "fatih" { + t.Errorf("Setted value is wrong: %s want: %s", f.Value().(string), "fatih") + } + + // let's access an unexported field, which should give an error + f = s.Field("d") + err = f.Set("large") + if err != errNotExported { + t.Error(err) + } + + // let's set a pointer to struct + b := &Bar{ + E: "gopher", + F: 2, + } + + f = s.Field("Bar") + err = f.Set(b) + if err != nil { + t.Error(err) + } + + baz := &Baz{ + A: "helloWorld", + B: 42, + } + + f = s.Field("E") + err = f.Set(baz) + if err != nil { + t.Error(err) + } + + ba := s.Field("E").Value().(*Baz) + + if ba.A != "helloWorld" { + t.Errorf("could not set baz. Got: %s Want: helloWorld", ba.A) + } +} + +func TestField(t *testing.T) { + s := newStruct() + + defer func() { + err := recover() + if err == nil { + t.Error("Retrieveing a non existing field from the struct should panic") + } + }() + + _ = s.Field("no-field") +} + +func TestField_Kind(t *testing.T) { + s := newStruct() + + f := s.Field("A") + if f.Kind() != reflect.String { + t.Errorf("Field A has wrong kind: %s want: %s", f.Kind(), reflect.String) + } + + f = s.Field("B") + if f.Kind() != reflect.Int { + t.Errorf("Field B has wrong kind: %s want: %s", f.Kind(), reflect.Int) + } + + // unexported + f = s.Field("d") + if f.Kind() != reflect.String { + t.Errorf("Field d has wrong kind: %s want: %s", f.Kind(), reflect.String) + } +} + +func TestField_Tag(t *testing.T) { + s := newStruct() + + v := s.Field("B").Tag("json") + if v != "" { + t.Errorf("Field's tag value of a non existing tag should return empty, got: %s", v) + } + + v = s.Field("C").Tag("json") + if v != "c" { + t.Errorf("Field's tag value of the existing field C should return 'c', got: %s", v) + } + + v = s.Field("d").Tag("json") + if v != "" { + t.Errorf("Field's tag value of a non exported field should return empty, got: %s", v) + } + + v = s.Field("x").Tag("xml") + if v != "x" { + t.Errorf("Field's tag value of a non exported field with a tag should return 'x', got: %s", v) + } + + v = s.Field("A").Tag("json") + if v != "" { + t.Errorf("Field's tag value of a existing field without a tag should return empty, got: %s", v) + } +} + +func TestField_Value(t *testing.T) { + s := newStruct() + + v := s.Field("A").Value() + val, ok := v.(string) + if !ok { + t.Errorf("Field's value of a A should be string") + } + + if val != "gopher" { + t.Errorf("Field's value of a existing tag should return 'gopher', got: %s", val) + } + + defer func() { + err := recover() + if err == nil { + t.Error("Value of a non exported field from the field should panic") + } + }() + + // should panic + _ = s.Field("d").Value() +} + +func TestField_IsEmbedded(t *testing.T) { + s := newStruct() + + if !s.Field("Bar").IsEmbedded() { + t.Errorf("Fields 'Bar' field is an embedded field") + } + + if s.Field("d").IsEmbedded() { + t.Errorf("Fields 'd' field is not an embedded field") + } +} + +func TestField_IsExported(t *testing.T) { + s := newStruct() + + if !s.Field("Bar").IsExported() { + t.Errorf("Fields 'Bar' field is an exported field") + } + + if !s.Field("A").IsExported() { + t.Errorf("Fields 'A' field is an exported field") + } + + if s.Field("d").IsExported() { + t.Errorf("Fields 'd' field is not an exported field") + } +} + +func TestField_IsZero(t *testing.T) { + s := newStruct() + + if s.Field("A").IsZero() { + t.Errorf("Fields 'A' field is an initialized field") + } + + if !s.Field("B").IsZero() { + t.Errorf("Fields 'B' field is not an initialized field") + } +} + +func TestField_Name(t *testing.T) { + s := newStruct() + + if s.Field("A").Name() != "A" { + t.Errorf("Fields 'A' field should have the name 'A'") + } +} + +func TestField_Field(t *testing.T) { + s := newStruct() + + e := s.Field("Bar").Field("E") + + val, ok := e.Value().(string) + if !ok { + t.Error("The value of the field 'e' inside 'Bar' struct should be string") + } + + if val != "example" { + t.Errorf("The value of 'e' should be 'example, got: %s", val) + } + + defer func() { + err := recover() + if err == nil { + t.Error("Field of a non existing nested struct should panic") + } + }() + + _ = s.Field("Bar").Field("e") +} + +func TestField_Fields(t *testing.T) { + s := newStruct() + fields := s.Field("Bar").Fields() + + if len(fields) != 3 { + t.Errorf("We expect 3 fields in embedded struct, was: %d", len(fields)) + } +} + +func TestField_FieldOk(t *testing.T) { + s := newStruct() + + b, ok := s.FieldOk("Bar") + if !ok { + t.Error("The field 'Bar' should exists.") + } + + e, ok := b.FieldOk("E") + if !ok { + t.Error("The field 'E' should exists.") + } + + val, ok := e.Value().(string) + if !ok { + t.Error("The value of the field 'e' inside 'Bar' struct should be string") + } + + if val != "example" { + t.Errorf("The value of 'e' should be 'example, got: %s", val) + } +} diff --git a/Godeps/_workspace/src/github.com/fatih/structs/structs.go b/Godeps/_workspace/src/github.com/fatih/structs/structs.go new file mode 100644 index 000000000..47bf35453 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/structs.go @@ -0,0 +1,422 @@ +// Package structs contains various utilities functions to work with structs. +package structs + +import "reflect" + +var ( + // DefaultTagName is the default tag name for struct fields which provides + // a more granular to tweak certain structs. Lookup the necessary functions + // for more info. + DefaultTagName = "structs" // struct's field default tag name +) + +// Struct encapsulates a struct type to provide several high level functions +// around the struct. +type Struct struct { + raw interface{} + value reflect.Value + TagName string +} + +// New returns a new *Struct with the struct s. It panics if the s's kind is +// not struct. +func New(s interface{}) *Struct { + return &Struct{ + raw: s, + value: strctVal(s), + TagName: DefaultTagName, + } +} + +// Map converts the given struct to a map[string]interface{}, where the keys +// of the map are the field names and the values of the map the associated +// values of the fields. The default key string is the struct field name but +// can be changed in the struct field's tag value. The "structs" key in the +// struct's field tag value is the key name. Example: +// +// // Field appears in map as key "myName". +// Name string `structs:"myName"` +// +// A tag value with the content of "-" ignores that particular field. Example: +// +// // Field is ignored by this package. +// Field bool `structs:"-"` +// +// A tag value with the option of "omitnested" stops iterating further if the type +// is a struct. Example: +// +// // Field is not processed further by this package. +// Field time.Time `structs:"myName,omitnested"` +// Field *http.Request `structs:",omitnested"` +// +// A tag value with the option of "omitempty" ignores that particular field if +// the field value is empty. Example: +// +// // Field appears in map as key "myName", but the field is +// // skipped if empty. +// Field string `structs:"myName,omitempty"` +// +// // Field appears in map as key "Field" (the default), but +// // the field is skipped if empty. +// Field string `structs:",omitempty"` +// +// Note that only exported fields of a struct can be accessed, non exported +// fields will be neglected. +func (s *Struct) Map() map[string]interface{} { + out := make(map[string]interface{}) + + fields := s.structFields() + + for _, field := range fields { + name := field.Name + val := s.value.FieldByName(name) + + var finalVal interface{} + + tagName, tagOpts := parseTag(field.Tag.Get(s.TagName)) + if tagName != "" { + name = tagName + } + + // if the value is a zero value and the field is marked as omitempty do + // not include + if tagOpts.Has("omitempty") { + zero := reflect.Zero(val.Type()).Interface() + current := val.Interface() + + if reflect.DeepEqual(current, zero) { + continue + } + } + + if IsStruct(val.Interface()) && !tagOpts.Has("omitnested") { + // look out for embedded structs, and convert them to a + // map[string]interface{} too + finalVal = Map(val.Interface()) + } else { + finalVal = val.Interface() + } + + out[name] = finalVal + } + + return out +} + +// Values converts the given s struct's field values to a []interface{}. A +// struct tag with the content of "-" ignores the that particular field. +// Example: +// +// // Field is ignored by this package. +// Field int `structs:"-"` +// +// A value with the option of "omitnested" stops iterating further if the type +// is a struct. Example: +// +// // Fields is not processed further by this package. +// Field time.Time `structs:",omitnested"` +// Field *http.Request `structs:",omitnested"` +// +// A tag value with the option of "omitempty" ignores that particular field and +// is not added to the values if the field value is empty. Example: +// +// // Field is skipped if empty +// Field string `structs:",omitempty"` +// +// Note that only exported fields of a struct can be accessed, non exported +// fields will be neglected. +func (s *Struct) Values() []interface{} { + fields := s.structFields() + + var t []interface{} + + for _, field := range fields { + val := s.value.FieldByName(field.Name) + + _, tagOpts := parseTag(field.Tag.Get(s.TagName)) + + // if the value is a zero value and the field is marked as omitempty do + // not include + if tagOpts.Has("omitempty") { + zero := reflect.Zero(val.Type()).Interface() + current := val.Interface() + + if reflect.DeepEqual(current, zero) { + continue + } + } + + if IsStruct(val.Interface()) && !tagOpts.Has("omitnested") { + // look out for embedded structs, and convert them to a + // []interface{} to be added to the final values slice + for _, embeddedVal := range Values(val.Interface()) { + t = append(t, embeddedVal) + } + } else { + t = append(t, val.Interface()) + } + } + + return t +} + +// Fields returns a slice of Fields. A struct tag with the content of "-" +// ignores the checking of that particular field. Example: +// +// // Field is ignored by this package. +// Field bool `structs:"-"` +// +// It panics if s's kind is not struct. +func (s *Struct) Fields() []*Field { + return getFields(s.value, s.TagName) +} + +func getFields(v reflect.Value, tagName string) []*Field { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + t := v.Type() + + var fields []*Field + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + + if tag := field.Tag.Get(tagName); tag == "-" { + continue + } + + f := &Field{ + field: field, + value: v.FieldByName(field.Name), + } + + fields = append(fields, f) + + } + + return fields +} + +// Field returns a new Field struct that provides several high level functions +// around a single struct field entity. It panics if the field is not found. +func (s *Struct) Field(name string) *Field { + f, ok := s.FieldOk(name) + if !ok { + panic("field not found") + } + + return f +} + +// Field returns a new Field struct that provides several high level functions +// around a single struct field entity. The boolean returns true if the field +// was found. +func (s *Struct) FieldOk(name string) (*Field, bool) { + t := s.value.Type() + + field, ok := t.FieldByName(name) + if !ok { + return nil, false + } + + return &Field{ + field: field, + value: s.value.FieldByName(name), + defaultTag: s.TagName, + }, true +} + +// IsZero returns true if all fields in a struct is a zero value (not +// initialized) A struct tag with the content of "-" ignores the checking of +// that particular field. Example: +// +// // Field is ignored by this package. +// Field bool `structs:"-"` +// +// A value with the option of "omitnested" stops iterating further if the type +// is a struct. Example: +// +// // Field is not processed further by this package. +// Field time.Time `structs:"myName,omitnested"` +// Field *http.Request `structs:",omitnested"` +// +// Note that only exported fields of a struct can be accessed, non exported +// fields will be neglected. It panics if s's kind is not struct. +func (s *Struct) IsZero() bool { + fields := s.structFields() + + for _, field := range fields { + val := s.value.FieldByName(field.Name) + + _, tagOpts := parseTag(field.Tag.Get(s.TagName)) + + if IsStruct(val.Interface()) && !tagOpts.Has("omitnested") { + ok := IsZero(val.Interface()) + if !ok { + return false + } + + continue + } + + // zero value of the given field, such as "" for string, 0 for int + zero := reflect.Zero(val.Type()).Interface() + + // current value of the given field + current := val.Interface() + + if !reflect.DeepEqual(current, zero) { + return false + } + } + + return true +} + +// HasZero returns true if a field in a struct is not initialized (zero value). +// A struct tag with the content of "-" ignores the checking of that particular +// field. Example: +// +// // Field is ignored by this package. +// Field bool `structs:"-"` +// +// A value with the option of "omitnested" stops iterating further if the type +// is a struct. Example: +// +// // Field is not processed further by this package. +// Field time.Time `structs:"myName,omitnested"` +// Field *http.Request `structs:",omitnested"` +// +// Note that only exported fields of a struct can be accessed, non exported +// fields will be neglected. It panics if s's kind is not struct. +func (s *Struct) HasZero() bool { + fields := s.structFields() + + for _, field := range fields { + val := s.value.FieldByName(field.Name) + + _, tagOpts := parseTag(field.Tag.Get(s.TagName)) + + if IsStruct(val.Interface()) && !tagOpts.Has("omitnested") { + ok := HasZero(val.Interface()) + if ok { + return true + } + + continue + } + + // zero value of the given field, such as "" for string, 0 for int + zero := reflect.Zero(val.Type()).Interface() + + // current value of the given field + current := val.Interface() + + if reflect.DeepEqual(current, zero) { + return true + } + } + + return false +} + +// Name returns the structs's type name within its package. For more info refer +// to Name() function. +func (s *Struct) Name() string { + return s.value.Type().Name() +} + +// structFields returns the exported struct fields for a given s struct. This +// is a convenient helper method to avoid duplicate code in some of the +// functions. +func (s *Struct) structFields() []reflect.StructField { + t := s.value.Type() + + var f []reflect.StructField + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + // we can't access the value of unexported fields + if field.PkgPath != "" { + continue + } + + // don't check if it's omitted + if tag := field.Tag.Get(s.TagName); tag == "-" { + continue + } + + f = append(f, field) + } + + return f +} + +func strctVal(s interface{}) reflect.Value { + v := reflect.ValueOf(s) + + // if pointer get the underlying element≤ + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + if v.Kind() != reflect.Struct { + panic("not struct") + } + + return v +} + +// Map converts the given struct to a map[string]interface{}. For more info +// refer to Struct types Map() method. It panics if s's kind is not struct. +func Map(s interface{}) map[string]interface{} { + return New(s).Map() +} + +// Values converts the given struct to a []interface{}. For more info refer to +// Struct types Values() method. It panics if s's kind is not struct. +func Values(s interface{}) []interface{} { + return New(s).Values() +} + +// Fields returns a slice of *Field. For more info refer to Struct types +// Fields() method. It panics if s's kind is not struct. +func Fields(s interface{}) []*Field { + return New(s).Fields() +} + +// IsZero returns true if all fields is equal to a zero value. For more info +// refer to Struct types IsZero() method. It panics if s's kind is not struct. +func IsZero(s interface{}) bool { + return New(s).IsZero() +} + +// HasZero returns true if any field is equal to a zero value. For more info +// refer to Struct types HasZero() method. It panics if s's kind is not struct. +func HasZero(s interface{}) bool { + return New(s).HasZero() +} + +// IsStruct returns true if the given variable is a struct or a pointer to +// struct. +func IsStruct(s interface{}) bool { + v := reflect.ValueOf(s) + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + // uninitialized zero value of a struct + if v.Kind() == reflect.Invalid { + return false + } + + return v.Kind() == reflect.Struct +} + +// Name returns the structs's type name within its package. It returns an +// empty string for unnamed types. It panics if s's kind is not struct. +func Name(s interface{}) string { + return New(s).Name() +} diff --git a/Godeps/_workspace/src/github.com/fatih/structs/structs_example_test.go b/Godeps/_workspace/src/github.com/fatih/structs/structs_example_test.go new file mode 100644 index 000000000..32bb82937 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/structs_example_test.go @@ -0,0 +1,351 @@ +package structs + +import ( + "fmt" + "time" +) + +func ExampleNew() { + type Server struct { + Name string + ID int32 + Enabled bool + } + + server := &Server{ + Name: "Arslan", + ID: 123456, + Enabled: true, + } + + s := New(server) + + fmt.Printf("Name : %v\n", s.Name()) + fmt.Printf("Values : %v\n", s.Values()) + fmt.Printf("Value of ID : %v\n", s.Field("ID").Value()) + // Output: + // Name : Server + // Values : [Arslan 123456 true] + // Value of ID : 123456 + +} + +func ExampleMap() { + type Server struct { + Name string + ID int32 + Enabled bool + } + + s := &Server{ + Name: "Arslan", + ID: 123456, + Enabled: true, + } + + m := Map(s) + + fmt.Printf("%#v\n", m["Name"]) + fmt.Printf("%#v\n", m["ID"]) + fmt.Printf("%#v\n", m["Enabled"]) + // Output: + // "Arslan" + // 123456 + // true + +} + +func ExampleMap_tags() { + // Custom tags can change the map keys instead of using the fields name + type Server struct { + Name string `structs:"server_name"` + ID int32 `structs:"server_id"` + Enabled bool `structs:"enabled"` + } + + s := &Server{ + Name: "Zeynep", + ID: 789012, + } + + m := Map(s) + + // access them by the custom tags defined above + fmt.Printf("%#v\n", m["server_name"]) + fmt.Printf("%#v\n", m["server_id"]) + fmt.Printf("%#v\n", m["enabled"]) + // Output: + // "Zeynep" + // 789012 + // false + +} + +func ExampleMap_nested() { + // By default field with struct types are processed too. We can stop + // processing them via "omitnested" tag option. + type Server struct { + Name string `structs:"server_name"` + ID int32 `structs:"server_id"` + Time time.Time `structs:"time,omitnested"` // do not convert to map[string]interface{} + } + + const shortForm = "2006-Jan-02" + t, _ := time.Parse("2006-Jan-02", "2013-Feb-03") + + s := &Server{ + Name: "Zeynep", + ID: 789012, + Time: t, + } + + m := Map(s) + + // access them by the custom tags defined above + fmt.Printf("%v\n", m["server_name"]) + fmt.Printf("%v\n", m["server_id"]) + fmt.Printf("%v\n", m["time"].(time.Time)) + // Output: + // Zeynep + // 789012 + // 2013-02-03 00:00:00 +0000 UTC +} + +func ExampleMap_omitEmpty() { + // By default field with struct types of zero values are processed too. We + // can stop processing them via "omitempty" tag option. + type Server struct { + Name string `structs:",omitempty"` + ID int32 `structs:"server_id,omitempty"` + Location string + } + + // Only add location + s := &Server{ + Location: "Tokyo", + } + + m := Map(s) + + // map contains only the Location field + fmt.Printf("%v\n", m) + // Output: + // map[Location:Tokyo] +} + +func ExampleValues() { + type Server struct { + Name string + ID int32 + Enabled bool + } + + s := &Server{ + Name: "Fatih", + ID: 135790, + Enabled: false, + } + + m := Values(s) + + fmt.Printf("Values: %+v\n", m) + // Output: + // Values: [Fatih 135790 false] +} + +func ExampleValues_omitEmpty() { + // By default field with struct types of zero values are processed too. We + // can stop processing them via "omitempty" tag option. + type Server struct { + Name string `structs:",omitempty"` + ID int32 `structs:"server_id,omitempty"` + Location string + } + + // Only add location + s := &Server{ + Location: "Ankara", + } + + m := Values(s) + + // values contains only the Location field + fmt.Printf("Values: %+v\n", m) + // Output: + // Values: [Ankara] +} + +func ExampleValues_tags() { + type Location struct { + City string + Country string + } + + type Server struct { + Name string + ID int32 + Enabled bool + Location Location `structs:"-"` // values from location are not included anymore + } + + s := &Server{ + Name: "Fatih", + ID: 135790, + Enabled: false, + Location: Location{City: "Ankara", Country: "Turkey"}, + } + + // Let get all values from the struct s. Note that we don't include values + // from the Location field + m := Values(s) + + fmt.Printf("Values: %+v\n", m) + // Output: + // Values: [Fatih 135790 false] +} + +func ExampleFields() { + type Access struct { + Name string + LastAccessed time.Time + Number int + } + + s := &Access{ + Name: "Fatih", + LastAccessed: time.Now(), + Number: 1234567, + } + + fields := Fields(s) + + for i, field := range fields { + fmt.Printf("[%d] %+v\n", i, field.Name()) + } + + // Output: + // [0] Name + // [1] LastAccessed + // [2] Number +} + +func ExampleFields_nested() { + type Person struct { + Name string + Number int + } + + type Access struct { + Person Person + HasPermission bool + LastAccessed time.Time + } + + s := &Access{ + Person: Person{Name: "fatih", Number: 1234567}, + LastAccessed: time.Now(), + HasPermission: true, + } + + // Let's get all fields from the struct s. + fields := Fields(s) + + for _, field := range fields { + if field.Name() == "Person" { + fmt.Printf("Access.Person.Name: %+v\n", field.Field("Name").Value()) + } + } + + // Output: + // Access.Person.Name: fatih +} + +func ExampleField() { + type Person struct { + Name string + Number int + } + + type Access struct { + Person Person + HasPermission bool + LastAccessed time.Time + } + + access := &Access{ + Person: Person{Name: "fatih", Number: 1234567}, + LastAccessed: time.Now(), + HasPermission: true, + } + + // Create a new Struct type + s := New(access) + + // Get the Field type for "Person" field + p := s.Field("Person") + + // Get the underlying "Name field" and print the value of it + name := p.Field("Name") + + fmt.Printf("Value of Person.Access.Name: %+v\n", name.Value()) + + // Output: + // Value of Person.Access.Name: fatih + +} + +func ExampleIsZero() { + type Server struct { + Name string + ID int32 + Enabled bool + } + + // Nothing is initalized + a := &Server{} + isZeroA := IsZero(a) + + // Name and Enabled is initialized, but not ID + b := &Server{ + Name: "Golang", + Enabled: true, + } + isZeroB := IsZero(b) + + fmt.Printf("%#v\n", isZeroA) + fmt.Printf("%#v\n", isZeroB) + // Output: + // true + // false +} + +func ExampleHasZero() { + // Let's define an Access struct. Note that the "Enabled" field is not + // going to be checked because we added the "structs" tag to the field. + type Access struct { + Name string + LastAccessed time.Time + Number int + Enabled bool `structs:"-"` + } + + // Name and Number is not initialized. + a := &Access{ + LastAccessed: time.Now(), + } + hasZeroA := HasZero(a) + + // Name and Number is initialized. + b := &Access{ + Name: "Fatih", + LastAccessed: time.Now(), + Number: 12345, + } + hasZeroB := HasZero(b) + + fmt.Printf("%#v\n", hasZeroA) + fmt.Printf("%#v\n", hasZeroB) + // Output: + // true + // false +} diff --git a/Godeps/_workspace/src/github.com/fatih/structs/structs_test.go b/Godeps/_workspace/src/github.com/fatih/structs/structs_test.go new file mode 100644 index 000000000..02788b105 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/structs_test.go @@ -0,0 +1,847 @@ +package structs + +import ( + "fmt" + "reflect" + "testing" + "time" +) + +func TestMapNonStruct(t *testing.T) { + foo := []string{"foo"} + + defer func() { + err := recover() + if err == nil { + t.Error("Passing a non struct into Map should panic") + } + }() + + // this should panic. We are going to recover and and test it + _ = Map(foo) +} + +func TestStructIndexes(t *testing.T) { + type C struct { + something int + Props map[string]interface{} + } + + defer func() { + err := recover() + if err != nil { + fmt.Printf("err %+v\n", err) + t.Error("Using mixed indexes should not panic") + } + }() + + // They should not panic + _ = Map(&C{}) + _ = Fields(&C{}) + _ = Values(&C{}) + _ = IsZero(&C{}) + _ = HasZero(&C{}) +} + +func TestMap(t *testing.T) { + var T = struct { + A string + B int + C bool + }{ + A: "a-value", + B: 2, + C: true, + } + + a := Map(T) + + if typ := reflect.TypeOf(a).Kind(); typ != reflect.Map { + t.Errorf("Map should return a map type, got: %v", typ) + } + + // we have three fields + if len(a) != 3 { + t.Errorf("Map should return a map of len 3, got: %d", len(a)) + } + + inMap := func(val interface{}) bool { + for _, v := range a { + if reflect.DeepEqual(v, val) { + return true + } + } + + return false + } + + for _, val := range []interface{}{"a-value", 2, true} { + if !inMap(val) { + t.Errorf("Map should have the value %v", val) + } + } + +} + +func TestMap_Tag(t *testing.T) { + var T = struct { + A string `structs:"x"` + B int `structs:"y"` + C bool `structs:"z"` + }{ + A: "a-value", + B: 2, + C: true, + } + + a := Map(T) + + inMap := func(key interface{}) bool { + for k := range a { + if reflect.DeepEqual(k, key) { + return true + } + } + return false + } + + for _, key := range []string{"x", "y", "z"} { + if !inMap(key) { + t.Errorf("Map should have the key %v", key) + } + } + +} + +func TestMap_CustomTag(t *testing.T) { + var T = struct { + A string `dd:"x"` + B int `dd:"y"` + C bool `dd:"z"` + }{ + A: "a-value", + B: 2, + C: true, + } + + s := New(T) + s.TagName = "dd" + + a := s.Map() + + inMap := func(key interface{}) bool { + for k := range a { + if reflect.DeepEqual(k, key) { + return true + } + } + return false + } + + for _, key := range []string{"x", "y", "z"} { + if !inMap(key) { + t.Errorf("Map should have the key %v", key) + } + } + +} + +func TestMap_MultipleCustomTag(t *testing.T) { + var A = struct { + X string `aa:"ax"` + }{"a_value"} + + aStruct := New(A) + aStruct.TagName = "aa" + + var B = struct { + X string `bb:"bx"` + }{"b_value"} + + bStruct := New(B) + bStruct.TagName = "bb" + + a, b := aStruct.Map(), bStruct.Map() + if !reflect.DeepEqual(a, map[string]interface{}{"ax": "a_value"}) { + t.Error("Map should have field ax with value a_value") + } + + if !reflect.DeepEqual(b, map[string]interface{}{"bx": "b_value"}) { + t.Error("Map should have field bx with value b_value") + } +} + +func TestMap_OmitEmpty(t *testing.T) { + type A struct { + Name string + Value string `structs:",omitempty"` + Time time.Time `structs:",omitempty"` + } + a := A{} + + m := Map(a) + + _, ok := m["Value"].(map[string]interface{}) + if ok { + t.Error("Map should not contain the Value field that is tagged as omitempty") + } + + _, ok = m["Time"].(map[string]interface{}) + if ok { + t.Error("Map should not contain the Time field that is tagged as omitempty") + } +} + +func TestMap_OmitNested(t *testing.T) { + type A struct { + Name string + Value string + Time time.Time `structs:",omitnested"` + } + a := A{Time: time.Now()} + + type B struct { + Desc string + A A + } + b := &B{A: a} + + m := Map(b) + + in, ok := m["A"].(map[string]interface{}) + if !ok { + t.Error("Map nested structs is not available in the map") + } + + // should not happen + if _, ok := in["Time"].(map[string]interface{}); ok { + t.Error("Map nested struct should omit recursiving parsing of Time") + } + + if _, ok := in["Time"].(time.Time); !ok { + t.Error("Map nested struct should stop parsing of Time at is current value") + } +} + +func TestMap_Nested(t *testing.T) { + type A struct { + Name string + } + a := &A{Name: "example"} + + type B struct { + A *A + } + b := &B{A: a} + + m := Map(b) + + if typ := reflect.TypeOf(m).Kind(); typ != reflect.Map { + t.Errorf("Map should return a map type, got: %v", typ) + } + + in, ok := m["A"].(map[string]interface{}) + if !ok { + t.Error("Map nested structs is not available in the map") + } + + if name := in["Name"].(string); name != "example" { + t.Errorf("Map nested struct's name field should give example, got: %s", name) + } +} + +func TestMap_Anonymous(t *testing.T) { + type A struct { + Name string + } + a := &A{Name: "example"} + + type B struct { + *A + } + b := &B{} + b.A = a + + m := Map(b) + + if typ := reflect.TypeOf(m).Kind(); typ != reflect.Map { + t.Errorf("Map should return a map type, got: %v", typ) + } + + in, ok := m["A"].(map[string]interface{}) + if !ok { + t.Error("Embedded structs is not available in the map") + } + + if name := in["Name"].(string); name != "example" { + t.Errorf("Embedded A struct's Name field should give example, got: %s", name) + } +} + +func TestStruct(t *testing.T) { + var T = struct{}{} + + if !IsStruct(T) { + t.Errorf("T should be a struct, got: %T", T) + } + + if !IsStruct(&T) { + t.Errorf("T should be a struct, got: %T", T) + } + +} + +func TestValues(t *testing.T) { + var T = struct { + A string + B int + C bool + }{ + A: "a-value", + B: 2, + C: true, + } + + s := Values(T) + + if typ := reflect.TypeOf(s).Kind(); typ != reflect.Slice { + t.Errorf("Values should return a slice type, got: %v", typ) + } + + inSlice := func(val interface{}) bool { + for _, v := range s { + if reflect.DeepEqual(v, val) { + return true + } + } + return false + } + + for _, val := range []interface{}{"a-value", 2, true} { + if !inSlice(val) { + t.Errorf("Values should have the value %v", val) + } + } +} + +func TestValues_OmitEmpty(t *testing.T) { + type A struct { + Name string + Value int `structs:",omitempty"` + } + + a := A{Name: "example"} + s := Values(a) + + if len(s) != 1 { + t.Errorf("Values of omitted empty fields should be not counted") + } + + if s[0].(string) != "example" { + t.Errorf("Values of omitted empty fields should left the value example") + } +} + +func TestValues_OmitNested(t *testing.T) { + type A struct { + Name string + Value int + } + + a := A{ + Name: "example", + Value: 123, + } + + type B struct { + A A `structs:",omitnested"` + C int + } + b := &B{A: a, C: 123} + + s := Values(b) + + if len(s) != 2 { + t.Errorf("Values of omitted nested struct should be not counted") + } + + inSlice := func(val interface{}) bool { + for _, v := range s { + if reflect.DeepEqual(v, val) { + return true + } + } + return false + } + + for _, val := range []interface{}{123, a} { + if !inSlice(val) { + t.Errorf("Values should have the value %v", val) + } + } +} + +func TestValues_Nested(t *testing.T) { + type A struct { + Name string + } + a := A{Name: "example"} + + type B struct { + A A + C int + } + b := &B{A: a, C: 123} + + s := Values(b) + + inSlice := func(val interface{}) bool { + for _, v := range s { + if reflect.DeepEqual(v, val) { + return true + } + } + return false + } + + for _, val := range []interface{}{"example", 123} { + if !inSlice(val) { + t.Errorf("Values should have the value %v", val) + } + } +} + +func TestValues_Anonymous(t *testing.T) { + type A struct { + Name string + } + a := A{Name: "example"} + + type B struct { + A + C int + } + b := &B{C: 123} + b.A = a + + s := Values(b) + + inSlice := func(val interface{}) bool { + for _, v := range s { + if reflect.DeepEqual(v, val) { + return true + } + } + return false + } + + for _, val := range []interface{}{"example", 123} { + if !inSlice(val) { + t.Errorf("Values should have the value %v", val) + } + } +} + +func TestFields(t *testing.T) { + var T = struct { + A string + B int + C bool + }{ + A: "a-value", + B: 2, + C: true, + } + + s := Fields(T) + + if len(s) != 3 { + t.Errorf("Fields should return a slice of len 3, got: %d", len(s)) + } + + inSlice := func(val string) bool { + for _, v := range s { + if reflect.DeepEqual(v.Name(), val) { + return true + } + } + return false + } + + for _, val := range []string{"A", "B", "C"} { + if !inSlice(val) { + t.Errorf("Fields should have the value %v", val) + } + } +} + +func TestFields_OmitNested(t *testing.T) { + type A struct { + Name string + Enabled bool + } + a := A{Name: "example"} + + type B struct { + A A + C int + Value string `structs:"-"` + Number int + } + b := &B{A: a, C: 123} + + s := Fields(b) + + if len(s) != 3 { + t.Errorf("Fields should omit nested struct. Expecting 2 got: %d", len(s)) + } + + inSlice := func(val interface{}) bool { + for _, v := range s { + if reflect.DeepEqual(v.Name(), val) { + return true + } + } + return false + } + + for _, val := range []interface{}{"A", "C"} { + if !inSlice(val) { + t.Errorf("Fields should have the value %v", val) + } + } +} + +func TestFields_Anonymous(t *testing.T) { + type A struct { + Name string + } + a := A{Name: "example"} + + type B struct { + A + C int + } + b := &B{C: 123} + b.A = a + + s := Fields(b) + + inSlice := func(val interface{}) bool { + for _, v := range s { + if reflect.DeepEqual(v.Name(), val) { + return true + } + } + return false + } + + for _, val := range []interface{}{"A", "C"} { + if !inSlice(val) { + t.Errorf("Fields should have the value %v", val) + } + } +} + +func TestIsZero(t *testing.T) { + var T = struct { + A string + B int + C bool `structs:"-"` + D []string + }{} + + ok := IsZero(T) + if !ok { + t.Error("IsZero should return true because none of the fields are initialized.") + } + + var X = struct { + A string + F *bool + }{ + A: "a-value", + } + + ok = IsZero(X) + if ok { + t.Error("IsZero should return false because A is initialized") + } + + var Y = struct { + A string + B int + }{ + A: "a-value", + B: 123, + } + + ok = IsZero(Y) + if ok { + t.Error("IsZero should return false because A and B is initialized") + } +} + +func TestIsZero_OmitNested(t *testing.T) { + type A struct { + Name string + D string + } + a := A{Name: "example"} + + type B struct { + A A `structs:",omitnested"` + C int + } + b := &B{A: a, C: 123} + + ok := IsZero(b) + if ok { + t.Error("IsZero should return false because A, B and C are initialized") + } + + aZero := A{} + bZero := &B{A: aZero} + + ok = IsZero(bZero) + if !ok { + t.Error("IsZero should return true because neither A nor B is initialized") + } + +} + +func TestIsZero_Nested(t *testing.T) { + type A struct { + Name string + D string + } + a := A{Name: "example"} + + type B struct { + A A + C int + } + b := &B{A: a, C: 123} + + ok := IsZero(b) + if ok { + t.Error("IsZero should return false because A, B and C are initialized") + } + + aZero := A{} + bZero := &B{A: aZero} + + ok = IsZero(bZero) + if !ok { + t.Error("IsZero should return true because neither A nor B is initialized") + } + +} + +func TestIsZero_Anonymous(t *testing.T) { + type A struct { + Name string + D string + } + a := A{Name: "example"} + + type B struct { + A + C int + } + b := &B{C: 123} + b.A = a + + ok := IsZero(b) + if ok { + t.Error("IsZero should return false because A, B and C are initialized") + } + + aZero := A{} + bZero := &B{} + bZero.A = aZero + + ok = IsZero(bZero) + if !ok { + t.Error("IsZero should return true because neither A nor B is initialized") + } +} + +func TestHasZero(t *testing.T) { + var T = struct { + A string + B int + C bool `structs:"-"` + D []string + }{ + A: "a-value", + B: 2, + } + + ok := HasZero(T) + if !ok { + t.Error("HasZero should return true because A and B are initialized.") + } + + var X = struct { + A string + F *bool + }{ + A: "a-value", + } + + ok = HasZero(X) + if !ok { + t.Error("HasZero should return true because A is initialized") + } + + var Y = struct { + A string + B int + }{ + A: "a-value", + B: 123, + } + + ok = HasZero(Y) + if ok { + t.Error("HasZero should return false because A and B is initialized") + } +} + +func TestHasZero_OmitNested(t *testing.T) { + type A struct { + Name string + D string + } + a := A{Name: "example"} + + type B struct { + A A `structs:",omitnested"` + C int + } + b := &B{A: a, C: 123} + + // Because the Field A inside B is omitted HasZero should return false + // because it will stop iterating deeper andnot going to lookup for D + ok := HasZero(b) + if ok { + t.Error("HasZero should return false because A and C are initialized") + } +} + +func TestHasZero_Nested(t *testing.T) { + type A struct { + Name string + D string + } + a := A{Name: "example"} + + type B struct { + A A + C int + } + b := &B{A: a, C: 123} + + ok := HasZero(b) + if !ok { + t.Error("HasZero should return true because D is not initialized") + } +} + +func TestHasZero_Anonymous(t *testing.T) { + type A struct { + Name string + D string + } + a := A{Name: "example"} + + type B struct { + A + C int + } + b := &B{C: 123} + b.A = a + + ok := HasZero(b) + if !ok { + t.Error("HasZero should return false because D is not initialized") + } +} + +func TestName(t *testing.T) { + type Foo struct { + A string + B bool + } + f := &Foo{} + + n := Name(f) + if n != "Foo" { + t.Errorf("Name should return Foo, got: %s", n) + } + + unnamed := struct{ Name string }{Name: "Cihangir"} + m := Name(unnamed) + if m != "" { + t.Errorf("Name should return empty string for unnamed struct, got: %s", n) + } + + defer func() { + err := recover() + if err == nil { + t.Error("Name should panic if a non struct is passed") + } + }() + + Name([]string{}) +} + +func TestNestedNilPointer(t *testing.T) { + type Collar struct { + Engraving string + } + + type Dog struct { + Name string + Collar *Collar + } + + type Person struct { + Name string + Dog *Dog + } + + person := &Person{ + Name: "John", + } + + personWithDog := &Person{ + Name: "Ron", + Dog: &Dog{ + Name: "Rover", + }, + } + + personWithDogWithCollar := &Person{ + Name: "Kon", + Dog: &Dog{ + Name: "Ruffles", + Collar: &Collar{ + Engraving: "If lost, call Kon", + }, + }, + } + + defer func() { + err := recover() + if err != nil { + fmt.Printf("err %+v\n", err) + t.Error("Internal nil pointer should not panic") + } + }() + + _ = Map(person) // Panics + _ = Map(personWithDog) // Panics + _ = Map(personWithDogWithCollar) // Doesn't panic +} diff --git a/Godeps/_workspace/src/github.com/fatih/structs/tags.go b/Godeps/_workspace/src/github.com/fatih/structs/tags.go new file mode 100644 index 000000000..8859341c1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/tags.go @@ -0,0 +1,32 @@ +package structs + +import "strings" + +// tagOptions contains a slice of tag options +type tagOptions []string + +// Has returns true if the given optiton is available in tagOptions +func (t tagOptions) Has(opt string) bool { + for _, tagOpt := range t { + if tagOpt == opt { + return true + } + } + + return false +} + +// parseTag splits a struct field's tag into its name and a list of options +// which comes after a name. A tag is in the form of: "name,option1,option2". +// The name can be neglectected. +func parseTag(tag string) (string, tagOptions) { + // tag is one of followings: + // "" + // "name" + // "name,opt" + // "name,opt,opt2" + // ",opt" + + res := strings.Split(tag, ",") + return res[0], res[1:] +} diff --git a/Godeps/_workspace/src/github.com/fatih/structs/tags_test.go b/Godeps/_workspace/src/github.com/fatih/structs/tags_test.go new file mode 100644 index 000000000..5d12724f1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/fatih/structs/tags_test.go @@ -0,0 +1,46 @@ +package structs + +import "testing" + +func TestParseTag_Name(t *testing.T) { + tags := []struct { + tag string + has bool + }{ + {"", false}, + {"name", true}, + {"name,opt", true}, + {"name , opt, opt2", false}, // has a single whitespace + {", opt, opt2", false}, + } + + for _, tag := range tags { + name, _ := parseTag(tag.tag) + + if (name != "name") && tag.has { + t.Errorf("Parse tag should return name: %#v", tag) + } + } +} + +func TestParseTag_Opts(t *testing.T) { + tags := []struct { + opts string + has bool + }{ + {"name", false}, + {"name,opt", true}, + {"name , opt, opt2", false}, // has a single whitespace + {",opt, opt2", true}, + {", opt3, opt4", false}, + } + + // search for "opt" + for _, tag := range tags { + _, opts := parseTag(tag.opts) + + if opts.Has("opt") != tag.has { + t.Errorf("Tag opts should have opt: %#v", tag) + } + } +}