diff --git a/go.mod b/go.mod index dc886b8..0e03df9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/spurge/goload go 1.15 require ( - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/google/uuid v1.1.1 github.com/kr/pretty v0.1.0 // indirect github.com/prometheus/client_golang v0.9.2 @@ -11,8 +10,7 @@ require ( github.com/prometheus/common v0.2.0 // indirect github.com/prometheus/procfs v0.0.0-20190209105433-f8d8b3f739bd // indirect github.com/sirupsen/logrus v1.2.0 - github.com/tidwall/gjson v1.1.5 - github.com/tidwall/match v1.0.1 // indirect + github.com/tidwall/gjson v1.9.3 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190204112747-618f46f3f0c8 gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index 5a6b19f..ded31db 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,12 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -26,36 +25,36 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190209105433-f8d8b3f739bd h1:pi7bGw6n4tfgHQtWDxJBBLYVdFr1GlfQEsDOyCDDFMM= github.com/prometheus/procfs v0.0.0-20190209105433-f8d8b3f739bd/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= -github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E= +github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -63,7 +62,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/vendor/github.com/tidwall/gjson/.travis.yml b/vendor/github.com/tidwall/gjson/.travis.yml deleted file mode 100644 index 4f2ee4d..0000000 --- a/vendor/github.com/tidwall/gjson/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index a7484c6..b7848de 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -3,13 +3,10 @@ src="logo.png" width="240" height="78" border="0" alt="GJSON">
-Build Status GoDoc GJSON Playground

- -

get json values quickly

GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. @@ -55,6 +52,9 @@ Prichard ## Path Syntax +Below is a quick overview of the path syntax, for more complete information please +check out [GJSON Syntax](SYNTAX.md). + A path is a series of keys separated by a dot. A key may contain special wildcard characters '\*' and '?'. To access an array value use the index as the key. @@ -68,9 +68,9 @@ The dot and wildcard characters can be escaped with '\\'. "children": ["Sara","Alex","Jack"], "fav.movie": "Deer Hunter", "friends": [ - {"first": "Dale", "last": "Murphy", "age": 44}, - {"first": "Roger", "last": "Craig", "age": 68}, - {"first": "Jane", "last": "Murphy", "age": 47} + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} ] } ``` @@ -87,46 +87,24 @@ The dot and wildcard characters can be escaped with '\\'. "friends.1.last" >> "Craig" ``` -You can also query an array for the first match by using `#[...]`, or find all matches with `#[...]#`. -Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators and the simple pattern matching `%` (like) and `!%` (not like) operators. - -``` -friends.#[last=="Murphy"].first >> "Dale" -friends.#[last=="Murphy"]#.first >> ["Dale","Jane"] -friends.#[age>45]#.last >> ["Craig","Murphy"] -friends.#[first%"D*"].last >> "Murphy" -friends.#[first!%"D*"].last >> "Craig" -``` - -## JSON Lines - -There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. - -For example: - -``` -{"name": "Gilbert", "age": 61} -{"name": "Alexa", "age": 34} -{"name": "May", "age": 57} -{"name": "Deloise", "age": 44} -``` +You can also query an array for the first match by using `#(...)`, or find all +matches with `#(...)#`. Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` +comparison operators and the simple pattern matching `%` (like) and `!%` +(not like) operators. ``` -..# >> 4 -..1 >> {"name": "Alexa", "age": 34} -..3 >> {"name": "Deloise", "age": 44} -..#.name >> ["Gilbert","Alexa","May","Deloise"] -..#[name="May"].age >> 57 +friends.#(last=="Murphy").first >> "Dale" +friends.#(last=="Murphy")#.first >> ["Dale","Jane"] +friends.#(age>45)#.last >> ["Craig","Murphy"] +friends.#(first%"D*").last >> "Murphy" +friends.#(first!%"D*").last >> "Craig" +friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"] ``` -The `ForEachLines` function will iterate through JSON lines. - -```go -gjson.ForEachLine(json, func(line gjson.Result) bool{ - println(line.String()) - return true -}) -``` +*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was +changed in v1.3.0 as to avoid confusion with the new +[multipath](SYNTAX.md#multipaths) syntax. For backwards compatibility, +`#[...]` will continue to work until the next major release.* ## Result Type @@ -145,11 +123,12 @@ nil, for JSON null To directly access the value: ```go -result.Type // can be String, Number, True, False, Null, or JSON -result.Str // holds the string -result.Num // holds the float64 number -result.Raw // holds the raw json -result.Index // index of raw value in original json, zero means index unknown +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Index // index of raw value in original json, zero means index unknown +result.Indexes // indexes of all the elements that match on a path containing the '#' query character. ``` There are a variety of handy functions that work on a result: @@ -172,10 +151,6 @@ result.Less(token Result, caseSensitive bool) bool The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: -The `result.Array()` function returns back an array of values. -If the result represents a non-existent value, then an empty array will be returned. -If the result is not a JSON array, the return value will be an array containing one result. - ```go boolean >> bool number >> float64 @@ -185,6 +160,10 @@ array >> []interface{} object >> map[string]interface{} ``` +The `result.Array()` function returns back an array of values. +If the result represents a non-existent value, then an empty array will be returned. +If the result is not a JSON array, the return value will be an array containing one result. + ### 64-bit integers The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers. @@ -194,6 +173,118 @@ result.Int() int64 // -9223372036854775808 to 9223372036854775807 result.Uint() int64 // 0 to 18446744073709551615 ``` +## Modifiers and path chaining + +New in version 1.2 is support for modifier functions and path chaining. + +A modifier is a path component that performs custom processing on the +json. + +Multiple paths can be "chained" together using the pipe character. +This is useful for getting results from a modified query. + +For example, using the built-in `@reverse` modifier on the above json document, +we'll get `children` array and reverse the order: + +``` +"children|@reverse" >> ["Jack","Alex","Sara"] +"children|@reverse|0" >> "Jack" +``` + +There are currently the following built-in modifiers: + +- `@reverse`: Reverse an array or the members of an object. +- `@ugly`: Remove all whitespace from a json document. +- `@pretty`: Make the json document more human readable. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. + +### Modifier arguments + +A modifier may accept an optional argument. The argument can be a valid JSON +document or just characters. + +For example, the `@pretty` modifier takes a json object as its argument. + +``` +@pretty:{"sortKeys":true} +``` + +Which makes the json pretty and orders all of its keys. + +```json +{ + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"age": 44, "first": "Dale", "last": "Murphy"}, + {"age": 68, "first": "Roger", "last": "Craig"}, + {"age": 47, "first": "Jane", "last": "Murphy"} + ], + "name": {"first": "Tom", "last": "Anderson"} +} +``` + +*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. +Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.* + +### Custom modifiers + +You can also add custom modifiers. + +For example, here we create a modifier that makes the entire json document upper +or lower case. + +```go +gjson.AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json +}) +``` + +``` +"children|@case:upper" >> ["SARA","ALEX","JACK"] +"children|@case:lower|@reverse" >> ["jack","alex","sara"] +``` + +## JSON Lines + +There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. + +For example: + +``` +{"name": "Gilbert", "age": 61} +{"name": "Alexa", "age": 34} +{"name": "May", "age": 57} +{"name": "Deloise", "age": 44} +``` + +``` +..# >> 4 +..1 >> {"name": "Alexa", "age": 34} +..3 >> {"name": "Deloise", "age": 44} +..#.name >> ["Gilbert","Alexa","May","Deloise"] +..#(name="May").age >> 57 +``` + +The `ForEachLines` function will iterate through JSON lines. + +```go +gjson.ForEachLine(json, func(line gjson.Result) bool{ + println(line.String()) + return true +}) +``` + ## Get nested array values Suppose you want all the last names from the following json: @@ -227,7 +318,7 @@ for _, name := range result.Array() { You can also query an object inside an array: ```go -name := gjson.Get(json, `programmers.#[lastName="Hunter"].firstName`) +name := gjson.Get(json, `programmers.#(lastName="Hunter").firstName`) println(name.String()) // prints "Elliotte" ``` @@ -383,7 +474,7 @@ JSON document used: } ``` -Each operation was rotated though one of the following search paths: +Each operation was rotated through one of the following search paths: ``` widget.window.name @@ -391,12 +482,4 @@ widget.image.hOffset widget.text.onMouseUp ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).* - - -## Contact -Josh Baker [@tidwall](http://twitter.com/tidwall) - -## License - -GJSON source code is available under the MIT [License](/LICENSE). +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md new file mode 100644 index 0000000..34fdccf --- /dev/null +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -0,0 +1,317 @@ +# GJSON Path Syntax + +A GJSON Path is a text string syntax that describes a search pattern for quickly retreiving values from a JSON payload. + +This document is designed to explain the structure of a GJSON Path through examples. + +- [Path structure](#path-structure) +- [Basic](#basic) +- [Wildcards](#wildcards) +- [Escape Character](#escape-character) +- [Arrays](#arrays) +- [Queries](#queries) +- [Dot vs Pipe](#dot-vs-pipe) +- [Modifiers](#modifiers) +- [Multipaths](#multipaths) + +The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson). +Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online. + + +## Path structure + +A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character. + +Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, and `?`. + +## Example + +Given this JSON + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ] +} +``` + +The following GJSON Paths evaluate to the accompanying values. + +### Basic + +In many cases you'll just want to retreive values by object name or array index. + +```go +name.last "Anderson" +name.first "Tom" +age 37 +children ["Sara","Alex","Jack"] +children.0 "Sara" +children.1 "Alex" +friends.1 {"first": "Roger", "last": "Craig", "age": 68} +friends.1.first "Roger" +``` + +### Wildcards + +A key may contain the special wildcard characters `*` and `?`. +The `*` will match on any zero+ characters, and `?` matches on any one character. + +```go +child*.2 "Jack" +c?ildren.0 "Sara" +``` + +### Escape character + +Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`. + +```go +fav\.movie "Deer Hunter" +``` + +You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in you source code. + +```go +// Go +val := gjson.Get(json, "fav\\.movie") // must escape the slash +val := gjson.Get(json, `fav\.movie`) // no need to escape the slash +``` + +```rust +// Rust +let val = gjson::get(json, "fav\\.movie") // must escape the slash +let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash +``` + + +### Arrays + +The `#` character allows for digging into JSON Arrays. + +To get the length of an array you'll just use the `#` all by itself. + +```go +friends.# 3 +friends.#.age [44,68,47] +``` + +### Queries + +You can also query an array for the first match by using `#(...)`, or find all matches with `#(...)#`. +Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators, +and the simple pattern matching `%` (like) and `!%` (not like) operators. + +```go +friends.#(last=="Murphy").first "Dale" +friends.#(last=="Murphy")#.first ["Dale","Jane"] +friends.#(age>45)#.last ["Craig","Murphy"] +friends.#(first%"D*").last "Murphy" +friends.#(first!%"D*").last "Craig" +``` + +To query for a non-object value in an array, you can forgo the string to the right of the operator. + +```go +children.#(!%"*a*") "Alex" +children.#(%"*a*")# ["Sara","Jack"] +``` + +Nested queries are allowed. + +```go +friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"] +``` + +*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was +changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths) +syntax. For backwards compatibility, `#[...]` will continue to work until the +next major release.* + +The `~` (tilde) operator will convert a value to a boolean before comparison. + +For example, using the following JSON: + +```json +{ + "vals": [ + { "a": 1, "b": true }, + { "a": 2, "b": true }, + { "a": 3, "b": false }, + { "a": 4, "b": "0" }, + { "a": 5, "b": 0 }, + { "a": 6, "b": "1" }, + { "a": 7, "b": 1 }, + { "a": 8, "b": "true" }, + { "a": 9, "b": false }, + { "a": 10, "b": null }, + { "a": 11 } + ] +} +``` + +You can now query for all true(ish) or false(ish) values: + +``` +vals.#(b==~true)#.a >> [1,2,6,7,8] +vals.#(b==~false)#.a >> [3,4,5,9,10,11] +``` + +The last value which was non-existent is treated as `false` + +### Dot vs Pipe + +The `.` is standard separator, but it's also possible to use a `|`. +In most cases they both end up returning the same results. +The cases where`|` differs from `.` is when it's used after the `#` for [Arrays](#arrays) and [Queries](#queries). + +Here are some examples + +```go +friends.0.first "Dale" +friends|0.first "Dale" +friends.0|first "Dale" +friends|0|first "Dale" +friends|# 3 +friends.# 3 +friends.#(last="Murphy")# [{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}] +friends.#(last="Murphy")#.first ["Dale","Jane"] +friends.#(last="Murphy")#|first +friends.#(last="Murphy")#.0 [] +friends.#(last="Murphy")#|0 {"first": "Dale", "last": "Murphy", "age": 44} +friends.#(last="Murphy")#.# [] +friends.#(last="Murphy")#|# 2 +``` + +Let's break down a few of these. + +The path `friends.#(last="Murphy")#` all by itself results in + +```json +[{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}] +``` + +The `.first` suffix will process the `first` path on each array element *before* returning the results. Which becomes + +```json +["Dale","Jane"] +``` + +But the `|first` suffix actually processes the `first` path *after* the previous result. +Since the previous result is an array, not an object, it's not possible to process +because `first` does not exist. + +Yet, `|0` suffix returns + +```json +{"first": "Dale", "last": "Murphy", "age": 44} +``` + +Because `0` is the first index of the previous result. + +### Modifiers + +A modifier is a path component that performs custom processing on the JSON. + +For example, using the built-in `@reverse` modifier on the above JSON payload will reverse the `children` array: + +```go +children.@reverse ["Jack","Alex","Sara"] +children.@reverse.0 "Jack" +``` + +There are currently the following built-in modifiers: + +- `@reverse`: Reverse an array or the members of an object. +- `@ugly`: Remove all whitespace from JSON. +- `@pretty`: Make the JSON more human readable. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. + +#### Modifier arguments + +A modifier may accept an optional argument. The argument can be a valid JSON payload or just characters. + +For example, the `@pretty` modifier takes a json object as its argument. + +``` +@pretty:{"sortKeys":true} +``` + +Which makes the json pretty and orders all of its keys. + +```json +{ + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"age": 44, "first": "Dale", "last": "Murphy"}, + {"age": 68, "first": "Roger", "last": "Craig"}, + {"age": 47, "first": "Jane", "last": "Murphy"} + ], + "name": {"first": "Tom", "last": "Anderson"} +} +``` + +*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. +Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.* + +#### Custom modifiers + +You can also add custom modifiers. + +For example, here we create a modifier which makes the entire JSON payload upper or lower case. + +```go +gjson.AddModifier("case", func(json, arg string) string { + if arg == "upper" { + return strings.ToUpper(json) + } + if arg == "lower" { + return strings.ToLower(json) + } + return json +}) +"children.@case:upper" ["SARA","ALEX","JACK"] +"children.@case:lower.@reverse" ["jack","alex","sara"] +``` + +*Note: Custom modifiers are not yet available in the Rust version* + +### Multipaths + +Starting with v1.3.0, GJSON added the ability to join multiple paths together +to form new documents. Wrapping comma-separated paths between `[...]` or +`{...}` will result in a new array or object, respectively. + +For example, using the given multipath + +``` +{name.first,age,"the_murphys":friends.#(last="Murphy")#.first} +``` + +Here we selected the first name, age, and the first name for friends with the +last name "Murphy". + +You'll notice that an optional key can be provided, in this case +"the_murphys", to force assign a key to a value. Otherwise, the name of the +actual field will be used, in this case "first". If a name cannot be +determined, then "_" is used. + +This results in + +``` +{"first":"Tom","age":37,"the_murphys":["Dale","Jane"]} +``` + + diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 2c9325f..95d210b 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -2,19 +2,16 @@ package gjson import ( - "encoding/base64" "encoding/json" - "errors" - "reflect" "strconv" "strings" - "sync" - "sync/atomic" "time" "unicode/utf16" "unicode/utf8" + "unsafe" "github.com/tidwall/match" + "github.com/tidwall/pretty" ) // Type is Result type @@ -67,6 +64,9 @@ type Result struct { Num float64 // Index of raw value in original json, zero means index unknown Index int + // Indexes of all the elements that match on a path containing the '#' + // query character. + Indexes []int } // String returns a string representation of the value. @@ -108,7 +108,8 @@ func (t Result) Bool() bool { case True: return true case String: - return t.Str != "" && t.Str != "0" && t.Str != "false" + b, _ := strconv.ParseBool(strings.ToLower(t.Str)) + return b case Number: return t.Num != 0 } @@ -126,16 +127,17 @@ func (t Result) Int() int64 { return n case Number: // try to directly convert the float64 to int64 - n, ok := floatToInt(t.Num) - if !ok { - // now try to parse the raw string - n, ok = parseInt(t.Raw) - if !ok { - // fallback to a standard conversion - return int64(t.Num) - } + i, ok := safeInt(t.Num) + if ok { + return i } - return n + // now try to parse the raw string + i, ok = parseInt(t.Raw) + if ok { + return i + } + // fallback to a standard conversion + return int64(t.Num) } } @@ -151,16 +153,17 @@ func (t Result) Uint() uint64 { return n case Number: // try to directly convert the float64 to uint64 - n, ok := floatToUint(t.Num) - if !ok { - // now try to parse the raw string - n, ok = parseUint(t.Raw) - if !ok { - // fallback to a standard conversion - return uint64(t.Num) - } + i, ok := safeInt(t.Num) + if ok && i >= 0 { + return uint64(i) } - return n + // now try to parse the raw string + u, ok := parseUint(t.Raw) + if ok { + return u + } + // fallback to a standard conversion + return uint64(t.Num) } } @@ -186,8 +189,9 @@ func (t Result) Time() time.Time { } // Array returns back an array of values. -// If the result represents a non-existent value, then an empty array will be returned. -// If the result is not a JSON array, the return value will be an array containing one result. +// If the result represents a non-existent value, then an empty array will be +// returned. If the result is not a JSON array, the return value will be an +// array containing one result. func (t Result) Array() []Result { if t.Type == Null { return []Result{} @@ -210,10 +214,11 @@ func (t Result) IsArray() bool { } // ForEach iterates through values. -// If the result represents a non-existent value, then no values will be iterated. -// If the result is an Object, the iterator will pass the key and value of each item. -// If the result is an Array, the iterator will only pass the value of each item. -// If the result is not a JSON array or object, the iterator will pass back one value equal to the result. +// If the result represents a non-existent value, then no values will be +// iterated. If the result is an Object, the iterator will pass the key and +// value of each item. If the result is an Array, the iterator will only pass +// the value of each item. If the result is not a JSON array or object, the +// iterator will pass back one value equal to the result. func (t Result) ForEach(iterator func(key, value Result) bool) { if !t.Exists() { return @@ -279,7 +284,8 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { } } -// Map returns back an map of values. The result should be a JSON array. +// Map returns back a map of values. The result should be a JSON object. +// If the result is not a JSON object, the return value will be an empty map. func (t Result) Map() map[string]Result { if t.Type != JSON { return map[string]Result{} @@ -461,11 +467,13 @@ func ParseBytes(json []byte) Result { } func squash(json string) string { - // expects that the lead character is a '[' or '{' + // expects that the lead character is a '[' or '{' or '(' or '"' // squash the value, ignoring all nested arrays and objects. - // the first '[' or '{' has already been read - depth := 1 - for i := 1; i < len(json); i++ { + var i, depth int + if json[0] != '"' { + i, depth = 1, 1 + } + for ; i < len(json); i++ { if json[i] >= '"' && json[i] <= '}' { switch json[i] { case '"': @@ -492,9 +500,15 @@ func squash(json string) string { break } } - case '{', '[': + if depth == 0 { + if i >= len(json) { + return json + } + return json[:i+1] + } + case '{', '[', '(': depth++ - case '}', ']': + case '}', ']', ')': depth-- if depth == 0 { return json[:i+1] @@ -574,7 +588,7 @@ func tostr(json string) (raw string, str string) { continue } } - break + return json[:i+1], unescape(json[1:i]) } } var ret string @@ -674,7 +688,8 @@ func parseNumber(json string, i int) (int, string) { var s = i i++ for ; i < len(json); i++ { - if json[i] <= ' ' || json[i] == ',' || json[i] == ']' || json[i] == '}' { + if json[i] <= ' ' || json[i] == ',' || json[i] == ']' || + json[i] == '}' { return i, json[s:i] } } @@ -695,25 +710,38 @@ func parseLiteral(json string, i int) (int, string) { type arrayPathResult struct { part string path string + pipe string + piped bool more bool alogok bool arrch bool alogkey string query struct { on bool + all bool path string op string value string - all bool } } func parseArrayPath(path string) (r arrayPathResult) { for i := 0; i < len(path); i++ { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } if path[i] == '.' { r.part = path[:i] - r.path = path[i+1:] - r.more = true + if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } return } if path[i] == '#' { @@ -723,100 +751,29 @@ func parseArrayPath(path string) (r arrayPathResult) { r.alogok = true r.alogkey = path[2:] r.path = path[:1] - } else if path[1] == '[' { - r.query.on = true + } else if path[1] == '[' || path[1] == '(' { // query - i += 2 - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - s := i - for ; i < len(path); i++ { - if path[i] <= ' ' || - path[i] == '!' || - path[i] == '=' || - path[i] == '<' || - path[i] == '>' || - path[i] == '%' || - path[i] == ']' { - break - } + r.query.on = true + qpath, op, value, _, fi, vesc, ok := + parseQuery(path[i:]) + if !ok { + // bad query, end now + break } - r.query.path = path[s:i] - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break + if len(value) > 2 && value[0] == '"' && + value[len(value)-1] == '"' { + value = value[1 : len(value)-1] + if vesc { + value = unescape(value) } } - if i < len(path) { - s = i - if path[i] == '!' { - if i < len(path)-1 && (path[i+1] == '=' || path[i+1] == '%') { - i++ - } - } else if path[i] == '<' || path[i] == '>' { - if i < len(path)-1 && path[i+1] == '=' { - i++ - } - } else if path[i] == '=' { - if i < len(path)-1 && path[i+1] == '=' { - s++ - i++ - } - } - i++ - r.query.op = path[s:i] - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - s = i - for ; i < len(path); i++ { - if path[i] == '"' { - i++ - s2 := i - for ; i < len(path); i++ { - if path[i] > '\\' { - continue - } - if path[i] == '"' { - // look for an escaped slash - if path[i-1] == '\\' { - n := 0 - for j := i - 2; j > s2-1; j-- { - if path[j] != '\\' { - break - } - n++ - } - if n%2 == 0 { - continue - } - } - break - } - } - } else if path[i] == ']' { - if i+1 < len(path) && path[i+1] == '#' { - r.query.all = true - } - break - } - } - if i > len(path) { - i = len(path) - } - v := path[s:i] - for len(v) > 0 && v[len(v)-1] <= ' ' { - v = v[:len(v)-1] - } - r.query.value = v + r.query.path = qpath + r.query.op = op + r.query.value = value + + i = fi - 1 + if i+1 < len(path) && path[i+1] == '#' { + r.query.all = true } } } @@ -828,19 +785,147 @@ func parseArrayPath(path string) (r arrayPathResult) { return } +// splitQuery takes a query and splits it into three parts: +// path, op, middle, and right. +// So for this query: +// #(first_name=="Murphy").last +// Becomes +// first_name # path +// =="Murphy" # middle +// .last # right +// Or, +// #(service_roles.#(=="one")).cap +// Becomes +// service_roles.#(=="one") # path +// # middle +// .cap # right +func parseQuery(query string) ( + path, op, value, remain string, i int, vesc, ok bool, +) { + if len(query) < 2 || query[0] != '#' || + (query[1] != '(' && query[1] != '[') { + return "", "", "", "", i, false, false + } + i = 2 + j := 0 // start of value part + depth := 1 + for ; i < len(query); i++ { + if depth == 1 && j == 0 { + switch query[i] { + case '!', '=', '<', '>', '%': + // start of the value part + j = i + continue + } + } + if query[i] == '\\' { + i++ + } else if query[i] == '[' || query[i] == '(' { + depth++ + } else if query[i] == ']' || query[i] == ')' { + depth-- + if depth == 0 { + break + } + } else if query[i] == '"' { + // inside selector string, balance quotes + i++ + for ; i < len(query); i++ { + if query[i] == '\\' { + vesc = true + i++ + } else if query[i] == '"' { + break + } + } + } + } + if depth > 0 { + return "", "", "", "", i, false, false + } + if j > 0 { + path = trim(query[2:j]) + value = trim(query[j:i]) + remain = query[i+1:] + // parse the compare op from the value + var opsz int + switch { + case len(value) == 1: + opsz = 1 + case value[0] == '!' && value[1] == '=': + opsz = 2 + case value[0] == '!' && value[1] == '%': + opsz = 2 + case value[0] == '<' && value[1] == '=': + opsz = 2 + case value[0] == '>' && value[1] == '=': + opsz = 2 + case value[0] == '=' && value[1] == '=': + value = value[1:] + opsz = 1 + case value[0] == '<': + opsz = 1 + case value[0] == '>': + opsz = 1 + case value[0] == '=': + opsz = 1 + case value[0] == '%': + opsz = 1 + } + op = value[:opsz] + value = trim(value[opsz:]) + } else { + path = trim(query[2:i]) + remain = query[i+1:] + } + return path, op, value, remain, i + 1, vesc, true +} + +func trim(s string) string { +left: + if len(s) > 0 && s[0] <= ' ' { + s = s[1:] + goto left + } +right: + if len(s) > 0 && s[len(s)-1] <= ' ' { + s = s[:len(s)-1] + goto right + } + return s +} + +// peek at the next byte and see if it's a '@', '[', or '{'. +func isDotPiperChar(c byte) bool { + return !DisableModifiers && (c == '@' || c == '[' || c == '{') +} + type objectPathResult struct { - part string - path string - wild bool - more bool + part string + path string + pipe string + piped bool + wild bool + more bool } func parseObjectPath(path string) (r objectPathResult) { for i := 0; i < len(path); i++ { + if path[i] == '|' { + r.part = path[:i] + r.pipe = path[i+1:] + r.piped = true + return + } if path[i] == '.' { r.part = path[:i] - r.path = path[i+1:] - r.more = true + if i < len(path)-1 && isDotPiperChar(path[i+1]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } return } if path[i] == '*' || path[i] == '?' { @@ -864,9 +949,19 @@ func parseObjectPath(path string) (r objectPathResult) { continue } else if path[i] == '.' { r.part = string(epart) - r.path = path[i+1:] + if i < len(path)-1 && isDotPiperChar(path[i+1]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + } r.more = true return + } else if path[i] == '|' { + r.part = string(epart) + r.pipe = path[i+1:] + r.piped = true + return } else if path[i] == '*' || path[i] == '?' { r.wild = true } @@ -883,9 +978,9 @@ func parseObjectPath(path string) (r objectPathResult) { } func parseSquash(json string, i int) (int, string) { - // expects that the lead character is a '[' or '{' + // expects that the lead character is a '[' or '{' or '(' // squash the value, ignoring all nested arrays and objects. - // the first '[' or '{' has already been read + // the first '[' or '{' or '(' has already been read s := i i++ depth := 1 @@ -916,9 +1011,9 @@ func parseSquash(json string, i int) (int, string) { break } } - case '{', '[': + case '{', '[', '(': depth++ - case '}', ']': + case '}', ']', ')': depth-- if depth == 0 { i++ @@ -934,6 +1029,10 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { var pmatch, kesc, vesc, ok, hit bool var key, val string rp := parseObjectPath(path) + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } for i < len(c.json) { for ; i < len(c.json); i++ { if c.json[i] == '"' { @@ -990,9 +1089,9 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } if rp.wild { if kesc { - pmatch = match.Match(unescape(key), rp.part) + pmatch = matchLimit(unescape(key), rp.part) } else { - pmatch = match.Match(key, rp.part) + pmatch = matchLimit(key, rp.part) } } else { if kesc { @@ -1077,10 +1176,35 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } return i, false } + +// matchLimit will limit the complexity of the match operation to avoid ReDos +// attacks from arbritary inputs. +// See the github.com/tidwall/match.MatchLimit function for more information. +func matchLimit(str, pattern string) bool { + matched, _ := match.MatchLimit(str, pattern, 10000) + return matched +} + func queryMatches(rp *arrayPathResult, value Result) bool { rpv := rp.query.value - if len(rpv) > 2 && rpv[0] == '"' && rpv[len(rpv)-1] == '"' { - rpv = rpv[1 : len(rpv)-1] + if len(rpv) > 0 && rpv[0] == '~' { + // convert to bool + rpv = rpv[1:] + if value.Bool() { + value = Result{Type: True} + } else { + value = Result{Type: False} + } + } + if !value.Exists() { + return false + } + if rp.query.op == "" { + // the query is only looking for existence, such as: + // friends.#(name) + // which makes sure that the array "friends" has an element of + // "name" that exists + return true } switch value.Type { case String: @@ -1098,9 +1222,9 @@ func queryMatches(rp *arrayPathResult, value Result) bool { case ">=": return value.Str >= rpv case "%": - return match.Match(value.Str, rpv) + return matchLimit(value.Str, rpv) case "!%": - return !match.Match(value.Str, rpv) + return !matchLimit(value.Str, rpv) } case Number: rpvn, _ := strconv.ParseFloat(rpv, 64) @@ -1150,6 +1274,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { var alog []int var partidx int var multires []byte + var queryIndexes []int rp := parseArrayPath(path) if !rp.arrch { n, ok := parseUint(rp.part) @@ -1159,6 +1284,61 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { partidx = int(n) } } + if !rp.more && rp.piped { + c.pipe = rp.pipe + c.piped = true + } + + procQuery := func(qval Result) bool { + if rp.query.all { + if len(multires) == 0 { + multires = append(multires, '[') + } + } + var tmp parseContext + tmp.value = qval + fillIndex(c.json, &tmp) + parentIndex := tmp.value.Index + var res Result + if qval.Type == JSON { + res = qval.Get(rp.query.path) + } else { + if rp.query.path != "" { + return false + } + res = qval + } + if queryMatches(&rp, res) { + if rp.more { + left, right, ok := splitPossiblePipe(rp.path) + if ok { + rp.path = left + c.pipe = right + c.piped = true + } + res = qval.Get(rp.path) + } else { + res = qval + } + if rp.query.all { + raw := res.Raw + if len(raw) == 0 { + raw = res.String() + } + if raw != "" { + if len(multires) > 1 { + multires = append(multires, ',') + } + multires = append(multires, raw...) + queryIndexes = append(queryIndexes, res.Index+parentIndex) + } + } else { + c.value = res + return true + } + } + return false + } for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h @@ -1186,7 +1366,19 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { if !ok { return i, false } - if hit { + if rp.query.on { + var qval Result + if vesc { + qval.Str = unescape(val[1 : len(val)-1]) + } else { + qval.Str = val[1 : len(val)-1] + } + qval.Raw = val + qval.Type = String + if procQuery(qval) { + return i, true + } + } else if hit { if rp.alogok { break } @@ -1211,24 +1403,8 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } else { i, val = parseSquash(c.json, i) if rp.query.on { - res := Get(val, rp.query.path) - if queryMatches(&rp, res) { - if rp.more { - res = Get(val, rp.path) - } else { - res = Result{Raw: val, Type: JSON} - } - if rp.query.all { - if len(multires) == 0 { - multires = append(multires, '[') - } else { - multires = append(multires, ',') - } - multires = append(multires, res.Raw...) - } else { - c.value = res - return i, true - } + if procQuery(Result{Raw: val, Type: JSON}) { + return i, true } } else if hit { if rp.alogok { @@ -1250,7 +1426,11 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } } else { i, val = parseSquash(c.json, i) - if hit { + if rp.query.on { + if procQuery(Result{Raw: val, Type: JSON}) { + return i, true + } + } else if hit { if rp.alogok { break } @@ -1261,7 +1441,15 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': i, val = parseNumber(c.json, i) - if hit { + if rp.query.on { + var qval Result + qval.Raw = val + qval.Type = Number + qval.Num, _ = strconv.ParseFloat(val, 64) + if procQuery(qval) { + return i, true + } + } else if hit { if rp.alogok { break } @@ -1273,7 +1461,19 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { case 't', 'f', 'n': vc := c.json[i] i, val = parseLiteral(c.json, i) - if hit { + if rp.query.on { + var qval Result + qval.Raw = val + switch vc { + case 't': + qval.Type = True + case 'f': + qval.Type = False + } + if procQuery(qval) { + return i, true + } + } else if hit { if rp.alogok { break } @@ -1289,40 +1489,74 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { case ']': if rp.arrch && rp.part == "#" { if rp.alogok { + left, right, ok := splitPossiblePipe(rp.alogkey) + if ok { + rp.alogkey = left + c.pipe = right + c.piped = true + } + var indexes = make([]int, 0, 64) var jsons = make([]byte, 0, 64) jsons = append(jsons, '[') - for j, k := 0, 0; j < len(alog); j++ { - _, res, ok := parseAny(c.json, alog[j], true) - if ok { - res := res.Get(rp.alogkey) - if res.Exists() { - if k > 0 { - jsons = append(jsons, ',') + idx := alog[j] + for idx < len(c.json) { + switch c.json[idx] { + case ' ', '\t', '\r', '\n': + idx++ + continue + } + break + } + if idx < len(c.json) && c.json[idx] != ']' { + _, res, ok := parseAny(c.json, idx, true) + parentIndex := res.Index + if ok { + res := res.Get(rp.alogkey) + if res.Exists() { + if k > 0 { + jsons = append(jsons, ',') + } + raw := res.Raw + if len(raw) == 0 { + raw = res.String() + } + jsons = append(jsons, []byte(raw)...) + indexes = append(indexes, + res.Index+parentIndex) + k++ } - jsons = append(jsons, []byte(res.Raw)...) - k++ } } } jsons = append(jsons, ']') c.value.Type = JSON c.value.Raw = string(jsons) + c.value.Indexes = indexes return i + 1, true } if rp.alogok { break } - c.value.Raw = "" + c.value.Type = Number c.value.Num = float64(h - 1) + c.value.Raw = strconv.Itoa(h - 1) c.calcd = true return i + 1, true } - if len(multires) > 0 && !c.value.Exists() { - c.value = Result{ - Raw: string(append(multires, ']')), - Type: JSON, + if !c.value.Exists() { + if len(multires) > 0 { + c.value = Result{ + Raw: string(append(multires, ']')), + Type: JSON, + Indexes: queryIndexes, + } + } else if rp.query.all { + c.value = Result{ + Raw: "[]", + Type: JSON, + } } } return i + 1, false @@ -1333,6 +1567,88 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { return i, false } +func splitPossiblePipe(path string) (left, right string, ok bool) { + // take a quick peek for the pipe character. If found we'll split the piped + // part of the path into the c.pipe field and shorten the rp. + var possible bool + for i := 0; i < len(path); i++ { + if path[i] == '|' { + possible = true + break + } + } + if !possible { + return + } + + if len(path) > 0 && path[0] == '{' { + squashed := squash(path[1:]) + if len(squashed) < len(path)-1 { + squashed = path[:len(squashed)+1] + remain := path[len(squashed):] + if remain[0] == '|' { + return squashed, remain[1:], true + } + } + return + } + + // split the left and right side of the path with the pipe character as + // the delimiter. This is a little tricky because we'll need to basically + // parse the entire path. + for i := 0; i < len(path); i++ { + if path[i] == '\\' { + i++ + } else if path[i] == '.' { + if i == len(path)-1 { + return + } + if path[i+1] == '#' { + i += 2 + if i == len(path) { + return + } + if path[i] == '[' || path[i] == '(' { + var start, end byte + if path[i] == '[' { + start, end = '[', ']' + } else { + start, end = '(', ')' + } + // inside selector, balance brackets + i++ + depth := 1 + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + } else if path[i] == start { + depth++ + } else if path[i] == end { + depth-- + if depth == 0 { + break + } + } else if path[i] == '"' { + // inside selector string, balance quotes + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + i++ + } else if path[i] == '"' { + break + } + } + } + } + } + } + } else if path[i] == '|' { + return path[:i], path[i+1:], true + } + } + return +} + // ForEachLine iterates through lines of JSON as specified by the JSON Lines // format (http://jsonlines.org/). // Each line is returned as a GJSON Result. @@ -1350,9 +1666,120 @@ func ForEachLine(json string, iterator func(line Result) bool) { } } +type subSelector struct { + name string + path string +} + +// parseSubSelectors returns the subselectors belonging to a '[path1,path2]' or +// '{"field1":path1,"field2":path2}' type subSelection. It's expected that the +// first character in path is either '[' or '{', and has already been checked +// prior to calling this function. +func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { + modifer := 0 + depth := 1 + colon := 0 + start := 1 + i := 1 + pushSel := func() { + var sel subSelector + if colon == 0 { + sel.path = path[start:i] + } else { + sel.name = path[start:colon] + sel.path = path[colon+1 : i] + } + sels = append(sels, sel) + colon = 0 + start = i + 1 + } + for ; i < len(path); i++ { + switch path[i] { + case '\\': + i++ + case '@': + if modifer == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { + modifer = i + } + case ':': + if modifer == 0 && colon == 0 && depth == 1 { + colon = i + } + case ',': + if depth == 1 { + pushSel() + } + case '"': + i++ + loop: + for ; i < len(path); i++ { + switch path[i] { + case '\\': + i++ + case '"': + break loop + } + } + case '[', '(', '{': + depth++ + case ']', ')', '}': + depth-- + if depth == 0 { + pushSel() + path = path[i+1:] + return sels, path, true + } + } + } + return +} + +// nameOfLast returns the name of the last component +func nameOfLast(path string) string { + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '|' || path[i] == '.' { + if i > 0 { + if path[i-1] == '\\' { + continue + } + } + return path[i+1:] + } + } + return path +} + +func isSimpleName(component string) bool { + for i := 0; i < len(component); i++ { + if component[i] < ' ' { + return false + } + switch component[i] { + case '[', ']', '{', '}', '(', ')', '#', '|': + return false + } + } + return true +} + +func appendJSONString(dst []byte, s string) []byte { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { + d, _ := json.Marshal(s) + return append(dst, string(d)...) + } + } + dst = append(dst, '"') + dst = append(dst, s...) + dst = append(dst, '"') + return dst +} + type parseContext struct { json string value Result + pipe string + piped bool calcd bool lines bool } @@ -1361,10 +1788,11 @@ type parseContext struct { // A path is in dot syntax, such as "name.last" or "age". // When the value is found it's returned immediately. // -// A path is a series of keys searated by a dot. +// A path is a series of keys separated by a dot. // A key may contain special wildcard characters '*' and '?'. // To access an array value use the index as the key. -// To get the number of elements in an array or to access a child path, use the '#' character. +// To get the number of elements in an array or to access a child path, use +// the '#' character. // The dot and wildcard character can be escaped with '\'. // // { @@ -1390,6 +1818,86 @@ type parseContext struct { // If you are consuming JSON from an unpredictable source then you may want to // use the Valid function first. func Get(json, path string) Result { + if len(path) > 1 { + if !DisableModifiers { + if path[0] == '@' { + // possible modifier + var ok bool + var npath string + var rjson string + npath, rjson, ok = execModifier(json, path) + if ok { + path = npath + if len(path) > 0 && (path[0] == '|' || path[0] == '.') { + res := Get(rjson, path[1:]) + res.Index = 0 + res.Indexes = nil + return res + } + return Parse(rjson) + } + } + } + if path[0] == '[' || path[0] == '{' { + // using a subselector path + kind := path[0] + var ok bool + var subs []subSelector + subs, path, ok = parseSubSelectors(path) + if ok { + if len(path) == 0 || (path[0] == '|' || path[0] == '.') { + var b []byte + b = append(b, kind) + var i int + for _, sub := range subs { + res := Get(json, sub.path) + if res.Exists() { + if i > 0 { + b = append(b, ',') + } + if kind == '{' { + if len(sub.name) > 0 { + if sub.name[0] == '"' && Valid(sub.name) { + b = append(b, sub.name...) + } else { + b = appendJSONString(b, sub.name) + } + } else { + last := nameOfLast(sub.path) + if isSimpleName(last) { + b = appendJSONString(b, last) + } else { + b = appendJSONString(b, "_") + } + } + b = append(b, ':') + } + var raw string + if len(res.Raw) == 0 { + raw = res.String() + if len(raw) == 0 { + raw = "null" + } + } else { + raw = res.Raw + } + b = append(b, raw...) + i++ + } + } + b = append(b, kind+2) + var res Result + res.Raw = string(b) + res.Type = JSON + if len(path) > 0 { + res = res.Get(path[1:]) + } + res.Index = 0 + return res + } + } + } + } var i int var c = &parseContext{json: json} if len(path) >= 2 && path[0] == '.' && path[1] == '.' { @@ -1409,6 +1917,11 @@ func Get(json, path string) Result { } } } + if c.piped { + res := c.value.Get(c.pipe) + res.Index = 0 + return res + } fillIndex(json, c) return c.value } @@ -1426,7 +1939,7 @@ func runeit(json string) rune { } // unescape unescapes a string -func unescape(json string) string { //, error) { +func unescape(json string) string { var str = make([]byte, 0, len(json)) for i := 0; i < len(json); i++ { switch { @@ -1466,7 +1979,8 @@ func unescape(json string) string { //, error) { i += 5 if utf16.IsSurrogate(r) { // need another code - if len(json[i:]) >= 6 && json[i] == '\\' && json[i+1] == 'u' { + if len(json[i:]) >= 6 && json[i] == '\\' && + json[i+1] == 'u' { // we expect it to be correct so just consume it r = utf16.DecodeRune(r, runeit(json[i+2:])) i += 6 @@ -1558,7 +2072,10 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { res.Raw = val res.Type = JSON } - return i, res, true + var tmp parseContext + tmp.value = res + fillIndex(json, &tmp) + return i, tmp.value, true } if json[i] <= ' ' { continue @@ -1608,11 +2125,6 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { return i, res, false } -var ( // used for testing - testWatchForFallback bool - testLastWasFallback bool -) - // GetMany searches json for the multiple paths. // The return value is a Result array where the number of items // will be equal to the number of input paths. @@ -1635,140 +2147,6 @@ func GetManyBytes(json []byte, path ...string) []Result { return res } -var fieldsmu sync.RWMutex -var fields = make(map[string]map[string]int) - -func assign(jsval Result, goval reflect.Value) { - if jsval.Type == Null { - return - } - switch goval.Kind() { - default: - case reflect.Ptr: - if !goval.IsNil() { - newval := reflect.New(goval.Elem().Type()) - assign(jsval, newval.Elem()) - goval.Elem().Set(newval.Elem()) - } else { - newval := reflect.New(goval.Type().Elem()) - assign(jsval, newval.Elem()) - goval.Set(newval) - } - case reflect.Struct: - fieldsmu.RLock() - sf := fields[goval.Type().String()] - fieldsmu.RUnlock() - if sf == nil { - fieldsmu.Lock() - sf = make(map[string]int) - for i := 0; i < goval.Type().NumField(); i++ { - f := goval.Type().Field(i) - tag := strings.Split(f.Tag.Get("json"), ",")[0] - if tag != "-" { - if tag != "" { - sf[tag] = i - sf[f.Name] = i - } else { - sf[f.Name] = i - } - } - } - fields[goval.Type().String()] = sf - fieldsmu.Unlock() - } - jsval.ForEach(func(key, value Result) bool { - if idx, ok := sf[key.Str]; ok { - f := goval.Field(idx) - if f.CanSet() { - assign(value, f) - } - } - return true - }) - case reflect.Slice: - if goval.Type().Elem().Kind() == reflect.Uint8 && jsval.Type == String { - data, _ := base64.StdEncoding.DecodeString(jsval.String()) - goval.Set(reflect.ValueOf(data)) - } else { - jsvals := jsval.Array() - slice := reflect.MakeSlice(goval.Type(), len(jsvals), len(jsvals)) - for i := 0; i < len(jsvals); i++ { - assign(jsvals[i], slice.Index(i)) - } - goval.Set(slice) - } - case reflect.Array: - i, n := 0, goval.Len() - jsval.ForEach(func(_, value Result) bool { - if i == n { - return false - } - assign(value, goval.Index(i)) - i++ - return true - }) - case reflect.Map: - if goval.Type().Key().Kind() == reflect.String && goval.Type().Elem().Kind() == reflect.Interface { - goval.Set(reflect.ValueOf(jsval.Value())) - } - case reflect.Interface: - goval.Set(reflect.ValueOf(jsval.Value())) - case reflect.Bool: - goval.SetBool(jsval.Bool()) - case reflect.Float32, reflect.Float64: - goval.SetFloat(jsval.Float()) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - goval.SetInt(jsval.Int()) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - goval.SetUint(jsval.Uint()) - case reflect.String: - goval.SetString(jsval.String()) - } - if len(goval.Type().PkgPath()) > 0 { - v := goval.Addr() - if v.Type().NumMethod() > 0 { - if u, ok := v.Interface().(json.Unmarshaler); ok { - u.UnmarshalJSON([]byte(jsval.Raw)) - } - } - } -} - -var validate uintptr = 1 - -// UnmarshalValidationEnabled provides the option to disable JSON validation -// during the Unmarshal routine. Validation is enabled by default. -// -// Deprecated: Use encoder/json.Unmarshal instead -func UnmarshalValidationEnabled(enabled bool) { - if enabled { - atomic.StoreUintptr(&validate, 1) - } else { - atomic.StoreUintptr(&validate, 0) - } -} - -// Unmarshal loads the JSON data into the value pointed to by v. -// -// This function works almost identically to json.Unmarshal except that -// gjson.Unmarshal will automatically attempt to convert JSON values to any Go -// type. For example, the JSON string "100" or the JSON number 100 can be equally -// assigned to Go string, int, byte, uint64, etc. This rule applies to all types. -// -// Deprecated: Use encoder/json.Unmarshal instead -func Unmarshal(data []byte, v interface{}) error { - if atomic.LoadUintptr(&validate) == 1 { - _, ok := validpayload(data, 0) - if !ok { - return errors.New("invalid json") - } - } - if v := reflect.ValueOf(v); v.Kind() == reflect.Ptr { - assign(ParseBytes(data), v) - } - return nil -} - func validpayload(data []byte, i int) (outi int, ok bool) { for ; i < len(data); i++ { switch data[i] { @@ -1947,6 +2325,12 @@ func validnumber(data []byte, i int) (outi int, ok bool) { // sign if data[i] == '-' { i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } } // int if i == len(data) { @@ -2012,19 +2396,22 @@ func validnumber(data []byte, i int) (outi int, ok bool) { } func validtrue(data []byte, i int) (outi int, ok bool) { - if i+3 <= len(data) && data[i] == 'r' && data[i+1] == 'u' && data[i+2] == 'e' { + if i+3 <= len(data) && data[i] == 'r' && data[i+1] == 'u' && + data[i+2] == 'e' { return i + 3, true } return i, false } func validfalse(data []byte, i int) (outi int, ok bool) { - if i+4 <= len(data) && data[i] == 'a' && data[i+1] == 'l' && data[i+2] == 's' && data[i+3] == 'e' { + if i+4 <= len(data) && data[i] == 'a' && data[i+1] == 'l' && + data[i+2] == 's' && data[i+3] == 'e' { return i + 4, true } return i, false } func validnull(data []byte, i int) (outi int, ok bool) { - if i+3 <= len(data) && data[i] == 'u' && data[i+1] == 'l' && data[i+2] == 'l' { + if i+3 <= len(data) && data[i] == 'u' && data[i+1] == 'l' && + data[i+2] == 'l' { return i + 3, true } return i, false @@ -2038,7 +2425,7 @@ func validnull(data []byte, i int) (outi int, ok bool) { // value := gjson.Get(json, "name.last") // func Valid(json string) bool { - _, ok := validpayload([]byte(json), 0) + _, ok := validpayload(stringBytes(json), 0) return ok } @@ -2049,7 +2436,7 @@ func Valid(json string) bool { // } // value := gjson.Get(json, "name.last") // -// If working with bytes, this method preferred over Valid(string(data)) +// If working with bytes, this method preferred over ValidBytes(string(data)) // func ValidBytes(json []byte) bool { _, ok := validpayload(json, 0) @@ -2094,23 +2481,404 @@ func parseInt(s string) (n int64, ok bool) { return n, true } -const minUint53 = 0 -const maxUint53 = 4503599627370495 -const minInt53 = -2251799813685248 -const maxInt53 = 2251799813685247 +// safeInt validates a given JSON number +// ensures it lies within the minimum and maximum representable JSON numbers +func safeInt(f float64) (n int64, ok bool) { + // https://tc39.es/ecma262/#sec-number.min_safe_integer + // https://tc39.es/ecma262/#sec-number.max_safe_integer + if f < -9007199254740991 || f > 9007199254740991 { + return 0, false + } + return int64(f), true +} + +// execModifier parses the path to find a matching modifier function. +// then input expects that the path already starts with a '@' +func execModifier(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + var hasArgs bool + for i := 1; i < len(path); i++ { + if path[i] == ':' { + pathOut = path[i+1:] + name = path[1:i] + hasArgs = len(pathOut) > 0 + break + } + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + if path[i] == '.' { + pathOut = path[i:] + name = path[1:i] + break + } + } + if fn, ok := modifiers[name]; ok { + var args string + if hasArgs { + var parsedArgs bool + switch pathOut[0] { + case '{', '[', '"': + res := Parse(pathOut) + if res.Exists() { + args = squash(pathOut) + pathOut = pathOut[len(args):] + parsedArgs = true + } + } + if !parsedArgs { + idx := strings.IndexByte(pathOut, '|') + if idx == -1 { + args = pathOut + pathOut = "" + } else { + args = pathOut[:idx] + pathOut = pathOut[idx:] + } + } + } + return pathOut, fn(json, args), true + } + return pathOut, res, false +} + +// unwrap removes the '[]' or '{}' characters around json +func unwrap(json string) string { + json = trim(json) + if len(json) >= 2 && (json[0] == '[' || json[0] == '{') { + json = json[1 : len(json)-1] + } + return json +} + +// DisableModifiers will disable the modifier syntax +var DisableModifiers = false + +var modifiers = map[string]func(json, arg string) string{ + "pretty": modPretty, + "ugly": modUgly, + "reverse": modReverse, + "this": modThis, + "flatten": modFlatten, + "join": modJoin, + "valid": modValid, +} + +// AddModifier binds a custom modifier command to the GJSON syntax. +// This operation is not thread safe and should be executed prior to +// using all other gjson function. +func AddModifier(name string, fn func(json, arg string) string) { + modifiers[name] = fn +} + +// ModifierExists returns true when the specified modifier exists. +func ModifierExists(name string, fn func(json, arg string) string) bool { + _, ok := modifiers[name] + return ok +} + +// cleanWS remove any non-whitespace from string +func cleanWS(s string) string { + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + continue + default: + var s2 []byte + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + s2 = append(s2, s[i]) + } + } + return string(s2) + } + } + return s +} + +// @pretty modifier makes the json look nice. +func modPretty(json, arg string) string { + if len(arg) > 0 { + opts := *pretty.DefaultOptions + Parse(arg).ForEach(func(key, value Result) bool { + switch key.String() { + case "sortKeys": + opts.SortKeys = value.Bool() + case "indent": + opts.Indent = cleanWS(value.String()) + case "prefix": + opts.Prefix = cleanWS(value.String()) + case "width": + opts.Width = int(value.Int()) + } + return true + }) + return bytesString(pretty.PrettyOptions(stringBytes(json), &opts)) + } + return bytesString(pretty.Pretty(stringBytes(json))) +} + +// @this returns the current element. Can be used to retrieve the root element. +func modThis(json, arg string) string { + return json +} + +// @ugly modifier removes all whitespace. +func modUgly(json, arg string) string { + return bytesString(pretty.Ugly(stringBytes(json))) +} + +// @reverse reverses array elements or root object members. +func modReverse(json, arg string) string { + res := Parse(json) + if res.IsArray() { + var values []Result + res.ForEach(func(_, value Result) bool { + values = append(values, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '[') + for i, j := len(values)-1, 0; i >= 0; i, j = i-1, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, values[i].Raw...) + } + out = append(out, ']') + return bytesString(out) + } + if res.IsObject() { + var keyValues []Result + res.ForEach(func(key, value Result) bool { + keyValues = append(keyValues, key, value) + return true + }) + out := make([]byte, 0, len(json)) + out = append(out, '{') + for i, j := len(keyValues)-2, 0; i >= 0; i, j = i-2, j+1 { + if j > 0 { + out = append(out, ',') + } + out = append(out, keyValues[i+0].Raw...) + out = append(out, ':') + out = append(out, keyValues[i+1].Raw...) + } + out = append(out, '}') + return bytesString(out) + } + return json +} + +// @flatten an array with child arrays. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]] +// The {"deep":true} arg can be provide for deep flattening. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7] +// The original json is returned when the json is not an array. +func modFlatten(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var deep bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "deep" { + deep = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '[') + var idx int + res.ForEach(func(_, value Result) bool { + var raw string + if value.IsArray() { + if deep { + raw = unwrap(modFlatten(value.Raw, arg)) + } else { + raw = unwrap(value.Raw) + } + } else { + raw = value.Raw + } + raw = strings.TrimSpace(raw) + if len(raw) > 0 { + if idx > 0 { + out = append(out, ',') + } + out = append(out, raw...) + idx++ + } + return true + }) + out = append(out, ']') + return bytesString(out) +} + +// @join multiple objects into a single object. +// [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} +// The arg can be "true" to specify that duplicate keys should be preserved. +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41} +// Without preserved keys: +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41} +// The original json is returned when the json is not an object. +func modJoin(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var preserve bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "preserve" { + preserve = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '{') + if preserve { + // Preserve duplicate keys. + var idx int + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + if idx > 0 { + out = append(out, ',') + } + out = append(out, unwrap(value.Raw)...) + idx++ + return true + }) + } else { + // Deduplicate keys and generate an object with stable ordering. + var keys []Result + kvals := make(map[string]Result) + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + value.ForEach(func(key, value Result) bool { + k := key.String() + if _, ok := kvals[k]; !ok { + keys = append(keys, key) + } + kvals[k] = value + return true + }) + return true + }) + for i := 0; i < len(keys); i++ { + if i > 0 { + out = append(out, ',') + } + out = append(out, keys[i].Raw...) + out = append(out, ':') + out = append(out, kvals[keys[i].String()].Raw...) + } + } + out = append(out, '}') + return bytesString(out) +} + +// @valid ensures that the json is valid before moving on. An empty string is +// returned when the json is not valid, otherwise it returns the original json. +func modValid(json, arg string) string { + if !Valid(json) { + return "" + } + return json +} -func floatToUint(f float64) (n uint64, ok bool) { - n = uint64(f) - if float64(n) == f && n >= minUint53 && n <= maxUint53 { - return n, true +// stringHeader instead of reflect.StringHeader +type stringHeader struct { + data unsafe.Pointer + len int +} + +// sliceHeader instead of reflect.SliceHeader +type sliceHeader struct { + data unsafe.Pointer + len int + cap int +} + +// getBytes casts the input json bytes to a string and safely returns the +// results as uniquely allocated data. This operation is intended to minimize +// copies and allocations for the large json string->[]byte. +func getBytes(json []byte, path string) Result { + var result Result + if json != nil { + // unsafe cast to string + result = Get(*(*string)(unsafe.Pointer(&json)), path) + // safely get the string headers + rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*stringHeader)(unsafe.Pointer(&result.Str)) + // create byte slice headers + rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len} + strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len} + if strh.data == nil { + // str is nil + if rawh.data == nil { + // raw is nil + result.Raw = "" + } else { + // raw has data, safely copy the slice header to a string + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + } + result.Str = "" + } else if rawh.data == nil { + // raw is nil + result.Raw = "" + // str has data, safely copy the slice header to a string + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } else if uintptr(strh.data) >= uintptr(rawh.data) && + uintptr(strh.data)+uintptr(strh.len) <= + uintptr(rawh.data)+uintptr(rawh.len) { + // Str is a substring of Raw. + start := uintptr(strh.data) - uintptr(rawh.data) + // safely copy the raw slice header + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + // substring the raw + result.Str = result.Raw[start : start+uintptr(strh.len)] + } else { + // safely copy both the raw and str slice headers to strings + result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) + result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) + } } - return 0, false + return result } -func floatToInt(f float64) (n int64, ok bool) { - n = int64(f) - if float64(n) == f && n >= minInt53 && n <= maxInt53 { - return n, true +// fillIndex finds the position of Raw data and assigns it to the Index field +// of the resulting value. If the position cannot be found then Index zero is +// used instead. +func fillIndex(json string, c *parseContext) { + if len(c.value.Raw) > 0 && !c.calcd { + jhdr := *(*stringHeader)(unsafe.Pointer(&json)) + rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data)) + if c.value.Index < 0 || c.value.Index >= len(json) { + c.value.Index = 0 + } } - return 0, false +} + +func stringBytes(s string) []byte { + return *(*[]byte)(unsafe.Pointer(&sliceHeader{ + data: (*stringHeader)(unsafe.Pointer(&s)).data, + len: len(s), + cap: len(s), + })) +} + +func bytesString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) } diff --git a/vendor/github.com/tidwall/gjson/gjson_gae.go b/vendor/github.com/tidwall/gjson/gjson_gae.go deleted file mode 100644 index 3a2b251..0000000 --- a/vendor/github.com/tidwall/gjson/gjson_gae.go +++ /dev/null @@ -1,10 +0,0 @@ -//+build appengine js - -package gjson - -func getBytes(json []byte, path string) Result { - return Get(string(json), path) -} -func fillIndex(json string, c *parseContext) { - // noop. Use zero for the Index value. -} diff --git a/vendor/github.com/tidwall/gjson/gjson_ngae.go b/vendor/github.com/tidwall/gjson/gjson_ngae.go deleted file mode 100644 index 59e944b..0000000 --- a/vendor/github.com/tidwall/gjson/gjson_ngae.go +++ /dev/null @@ -1,69 +0,0 @@ -//+build !appengine -//+build !js - -package gjson - -import ( - "reflect" - "unsafe" -) - -// getBytes casts the input json bytes to a string and safely returns the -// results as uniquely allocated data. This operation is intended to minimize -// copies and allocations for the large json string->[]byte. -func getBytes(json []byte, path string) Result { - var result Result - if json != nil { - // unsafe cast to string - result = Get(*(*string)(unsafe.Pointer(&json)), path) - // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) - // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { - // str is nil - if rawh.Data == 0 { - // raw is nil - result.Raw = "" - } else { - // raw has data, safely copy the slice header to a string - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - } - result.Str = "" - } else if rawh.Data == 0 { - // raw is nil - result.Raw = "" - // str has data, safely copy the slice header to a string - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { - // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) - // safely copy the raw slice header - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - // substring the raw - result.Str = result.Raw[start : start+strh.Len] - } else { - // safely copy both the raw and str slice headers to strings - result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) - result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } - } - return result -} - -// fillIndex finds the position of Raw data and assigns it to the Index field -// of the resulting value. If the position cannot be found then Index zero is -// used instead. -func fillIndex(json string, c *parseContext) { - if len(c.value.Raw) > 0 && !c.calcd { - jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) - c.value.Index = int(rhdr.Data - jhdr.Data) - if c.value.Index < 0 || c.value.Index >= len(json) { - c.value.Index = 0 - } - } -} diff --git a/vendor/github.com/tidwall/gjson/go.mod b/vendor/github.com/tidwall/gjson/go.mod new file mode 100644 index 0000000..6f64083 --- /dev/null +++ b/vendor/github.com/tidwall/gjson/go.mod @@ -0,0 +1,8 @@ +module github.com/tidwall/gjson + +go 1.12 + +require ( + github.com/tidwall/match v1.1.1 + github.com/tidwall/pretty v1.2.0 +) diff --git a/vendor/github.com/tidwall/gjson/go.sum b/vendor/github.com/tidwall/gjson/go.sum new file mode 100644 index 0000000..be39c8c --- /dev/null +++ b/vendor/github.com/tidwall/gjson/go.sum @@ -0,0 +1,4 @@ +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/vendor/github.com/tidwall/match/.travis.yml b/vendor/github.com/tidwall/match/.travis.yml deleted file mode 100644 index 4f2ee4d..0000000 --- a/vendor/github.com/tidwall/match/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/match/README.md b/vendor/github.com/tidwall/match/README.md index 2aa5bc3..5fdd4cf 100644 --- a/vendor/github.com/tidwall/match/README.md +++ b/vendor/github.com/tidwall/match/README.md @@ -1,20 +1,17 @@ -Match -===== -Build Status -GoDoc +# Match + +[![GoDoc](https://godoc.org/github.com/tidwall/match?status.svg)](https://godoc.org/github.com/tidwall/match) Match is a very simple pattern matcher where '*' matches on any number characters and '?' matches on any one character. -Installing ----------- +## Installing ``` go get -u github.com/tidwall/match ``` -Example -------- +## Example ```go match.Match("hello", "*llo") @@ -23,10 +20,10 @@ match.Match("hello", "h*o") ``` -Contact -------- +## Contact + Josh Baker [@tidwall](http://twitter.com/tidwall) -License -------- +## License + Redcon source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/match/go.mod b/vendor/github.com/tidwall/match/go.mod new file mode 100644 index 0000000..df19b5f --- /dev/null +++ b/vendor/github.com/tidwall/match/go.mod @@ -0,0 +1,3 @@ +module github.com/tidwall/match + +go 1.15 diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go index fcfe998..11da28f 100644 --- a/vendor/github.com/tidwall/match/match.go +++ b/vendor/github.com/tidwall/match/match.go @@ -1,12 +1,14 @@ -// Match provides a simple pattern matcher with unicode support. +// Package match provides a simple pattern matcher with unicode support. package match -import "unicode/utf8" +import ( + "unicode/utf8" +) // Match returns true if str matches pattern. This is a very // simple wildcard match where '*' matches on any number characters // and '?' matches on any one character. - +// // pattern: // { term } // term: @@ -19,113 +21,167 @@ func Match(str, pattern string) bool { if pattern == "*" { return true } - return deepMatch(str, pattern) + return match(str, pattern, 0, nil, -1) == rMatch } -func deepMatch(str, pattern string) bool { - for len(pattern) > 0 { - if pattern[0] > 0x7f { - return deepMatchRune(str, pattern) - } - switch pattern[0] { - default: - if len(str) == 0 { - return false - } - if str[0] > 0x7f { - return deepMatchRune(str, pattern) - } - if str[0] != pattern[0] { - return false - } - case '?': - if len(str) == 0 { - return false - } - case '*': - return deepMatch(str, pattern[1:]) || - (len(str) > 0 && deepMatch(str[1:], pattern)) - } - str = str[1:] - pattern = pattern[1:] + +// MatchLimit is the same as Match but will limit the complexity of the match +// operation. This is to avoid long running matches, specifically to avoid ReDos +// attacks from arbritary inputs. +// +// How it works: +// The underlying match routine is recursive and may call itself when it +// encounters a sandwiched wildcard pattern, such as: `user:*:name`. +// Everytime it calls itself a counter is incremented. +// The operation is stopped when counter > maxcomp*len(str). +func MatchLimit(str, pattern string, maxcomp int) (matched, stopped bool) { + if pattern == "*" { + return true, false + } + counter := 0 + r := match(str, pattern, len(str), &counter, maxcomp) + if r == rStop { + return false, true } - return len(str) == 0 && len(pattern) == 0 + return r == rMatch, false } -func deepMatchRune(str, pattern string) bool { - var sr, pr rune - var srsz, prsz int +type result int - // read the first rune ahead of time - if len(str) > 0 { - if str[0] > 0x7f { - sr, srsz = utf8.DecodeRuneInString(str) - } else { - sr, srsz = rune(str[0]), 1 +const ( + rNoMatch result = iota + rMatch + rStop +) + +func match(str, pat string, slen int, counter *int, maxcomp int) result { + // check complexity limit + if maxcomp > -1 { + if *counter > slen*maxcomp { + return rStop } - } else { - sr, srsz = utf8.RuneError, 0 + *counter++ } - if len(pattern) > 0 { - if pattern[0] > 0x7f { - pr, prsz = utf8.DecodeRuneInString(pattern) - } else { - pr, prsz = rune(pattern[0]), 1 + + for len(pat) > 0 { + var wild bool + pc, ps := rune(pat[0]), 1 + if pc > 0x7f { + pc, ps = utf8.DecodeRuneInString(pat) } - } else { - pr, prsz = utf8.RuneError, 0 - } - // done reading - for pr != utf8.RuneError { - switch pr { - default: - if srsz == utf8.RuneError { - return false - } - if sr != pr { - return false + var sc rune + var ss int + if len(str) > 0 { + sc, ss = rune(str[0]), 1 + if sc > 0x7f { + sc, ss = utf8.DecodeRuneInString(str) } + } + switch pc { case '?': - if srsz == utf8.RuneError { - return false + if ss == 0 { + return rNoMatch } case '*': - return deepMatchRune(str, pattern[prsz:]) || - (srsz > 0 && deepMatchRune(str[srsz:], pattern)) - } - str = str[srsz:] - pattern = pattern[prsz:] - // read the next runes - if len(str) > 0 { - if str[0] > 0x7f { - sr, srsz = utf8.DecodeRuneInString(str) - } else { - sr, srsz = rune(str[0]), 1 + // Ignore repeating stars. + for len(pat) > 1 && pat[1] == '*' { + pat = pat[1:] } - } else { - sr, srsz = utf8.RuneError, 0 - } - if len(pattern) > 0 { - if pattern[0] > 0x7f { - pr, prsz = utf8.DecodeRuneInString(pattern) - } else { - pr, prsz = rune(pattern[0]), 1 + + // If this star is the last character then it must be a match. + if len(pat) == 1 { + return rMatch } - } else { - pr, prsz = utf8.RuneError, 0 + + // Match and trim any non-wildcard suffix characters. + var ok bool + str, pat, ok = matchTrimSuffix(str, pat) + if !ok { + return rNoMatch + } + + // Check for single star again. + if len(pat) == 1 { + return rMatch + } + + // Perform recursive wildcard search. + r := match(str, pat[1:], slen, counter, maxcomp) + if r != rNoMatch { + return r + } + if len(str) == 0 { + return rNoMatch + } + wild = true + default: + if ss == 0 { + return rNoMatch + } + if pc == '\\' { + pat = pat[ps:] + pc, ps = utf8.DecodeRuneInString(pat) + if ps == 0 { + return rNoMatch + } + } + if sc != pc { + return rNoMatch + } + } + str = str[ss:] + if !wild { + pat = pat[ps:] } - // done reading } - - return srsz == 0 && prsz == 0 + if len(str) == 0 { + return rMatch + } + return rNoMatch } -var maxRuneBytes = func() []byte { - b := make([]byte, 4) - if utf8.EncodeRune(b, '\U0010FFFF') != 4 { - panic("invalid rune encoding") +// matchTrimSuffix matches and trims any non-wildcard suffix characters. +// Returns the trimed string and pattern. +// +// This is called because the pattern contains extra data after the wildcard +// star. Here we compare any suffix characters in the pattern to the suffix of +// the target string. Basically a reverse match that stops when a wildcard +// character is reached. This is a little trickier than a forward match because +// we need to evaluate an escaped character in reverse. +// +// Any matched characters will be trimmed from both the target +// string and the pattern. +func matchTrimSuffix(str, pat string) (string, string, bool) { + // It's expected that the pattern has at least two bytes and the first byte + // is a wildcard star '*' + match := true + for len(str) > 0 && len(pat) > 1 { + pc, ps := utf8.DecodeLastRuneInString(pat) + var esc bool + for i := 0; ; i++ { + if pat[len(pat)-ps-i-1] != '\\' { + if i&1 == 1 { + esc = true + ps++ + } + break + } + } + if pc == '*' && !esc { + match = true + break + } + sc, ss := utf8.DecodeLastRuneInString(str) + if !((pc == '?' && !esc) || pc == sc) { + match = false + break + } + str = str[:len(str)-ss] + pat = pat[:len(pat)-ps] } - return b -}() + return str, pat, match +} + +var maxRuneBytes = [...]byte{244, 143, 191, 191} // Allowable parses the pattern and determines the minimum and maximum allowable // values that the pattern can represent. @@ -146,7 +202,7 @@ func Allowable(pattern string) (min, max string) { } if pattern[i] == '?' { minb = append(minb, 0) - maxb = append(maxb, maxRuneBytes...) + maxb = append(maxb, maxRuneBytes[:]...) } else { minb = append(minb, pattern[i]) maxb = append(maxb, pattern[i]) diff --git a/vendor/github.com/tidwall/pretty/LICENSE b/vendor/github.com/tidwall/pretty/LICENSE new file mode 100644 index 0000000..993b83f --- /dev/null +++ b/vendor/github.com/tidwall/pretty/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2017 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md new file mode 100644 index 0000000..d3be5e5 --- /dev/null +++ b/vendor/github.com/tidwall/pretty/README.md @@ -0,0 +1,122 @@ +# Pretty + +[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) + +Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads. + +Getting Started +=============== + +## Installing + +To start using Pretty, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/pretty +``` + +This will retrieve the library. + +## Pretty + +Using this example: + +```json +{"name": {"first":"Tom","last":"Anderson"}, "age":37, +"children": ["Sara","Alex","Jack"], +"fav.movie": "Deer Hunter", "friends": [ + {"first": "Janet", "last": "Murphy", "age": 44} + ]} +``` + +The following code: +```go +result = pretty.Pretty(example) +``` + +Will format the json to: + +```json +{ + "name": { + "first": "Tom", + "last": "Anderson" + }, + "age": 37, + "children": ["Sara", "Alex", "Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + { + "first": "Janet", + "last": "Murphy", + "age": 44 + } + ] +} +``` + +## Color + +Color will colorize the json for outputing to the screen. + +```json +result = pretty.Color(json, nil) +``` + +Will add color to the result for printing to the terminal. +The second param is used for a customizing the style, and passing nil will use the default `pretty.TerminalStyle`. + +## Ugly + +The following code: +```go +result = pretty.Ugly(example) +``` + +Will format the json to: + +```json +{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}``` +``` + +## Customized output + +There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options: + +```go +type Options struct { + // Width is an max column width for single line arrays + // Default is 80 + Width int + // Prefix is a prefix for all lines + // Default is an empty string + Prefix string + // Indent is the nested indentation + // Default is two spaces + Indent string + // SortKeys will sort the keys alphabetically + // Default is false + SortKeys bool +} +``` +## Performance + +Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods. +``` +BenchmarkPretty-16 1000000 1034 ns/op 720 B/op 2 allocs/op +BenchmarkPrettySortKeys-16 586797 1983 ns/op 2848 B/op 14 allocs/op +BenchmarkUgly-16 4652365 254 ns/op 240 B/op 1 allocs/op +BenchmarkUglyInPlace-16 6481233 183 ns/op 0 B/op 0 allocs/op +BenchmarkJSONIndent-16 450654 2687 ns/op 1221 B/op 0 allocs/op +BenchmarkJSONCompact-16 685111 1699 ns/op 442 B/op 0 allocs/op +``` + +*These benchmarks were run on a MacBook Pro 2.4 GHz 8-Core Intel Core i9.* + +## Contact +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +Pretty source code is available under the MIT [License](/LICENSE). + diff --git a/vendor/github.com/tidwall/pretty/go.mod b/vendor/github.com/tidwall/pretty/go.mod new file mode 100644 index 0000000..6106735 --- /dev/null +++ b/vendor/github.com/tidwall/pretty/go.mod @@ -0,0 +1,3 @@ +module github.com/tidwall/pretty + +go 1.16 diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go new file mode 100644 index 0000000..f3f756a --- /dev/null +++ b/vendor/github.com/tidwall/pretty/pretty.go @@ -0,0 +1,674 @@ +package pretty + +import ( + "bytes" + "encoding/json" + "sort" + "strconv" +) + +// Options is Pretty options +type Options struct { + // Width is an max column width for single line arrays + // Default is 80 + Width int + // Prefix is a prefix for all lines + // Default is an empty string + Prefix string + // Indent is the nested indentation + // Default is two spaces + Indent string + // SortKeys will sort the keys alphabetically + // Default is false + SortKeys bool +} + +// DefaultOptions is the default options for pretty formats. +var DefaultOptions = &Options{Width: 80, Prefix: "", Indent: " ", SortKeys: false} + +// Pretty converts the input json into a more human readable format where each +// element is on it's own line with clear indentation. +func Pretty(json []byte) []byte { return PrettyOptions(json, nil) } + +// PrettyOptions is like Pretty but with customized options. +func PrettyOptions(json []byte, opts *Options) []byte { + if opts == nil { + opts = DefaultOptions + } + buf := make([]byte, 0, len(json)) + if len(opts.Prefix) != 0 { + buf = append(buf, opts.Prefix...) + } + buf, _, _, _ = appendPrettyAny(buf, json, 0, true, + opts.Width, opts.Prefix, opts.Indent, opts.SortKeys, + 0, 0, -1) + if len(buf) > 0 { + buf = append(buf, '\n') + } + return buf +} + +// Ugly removes insignificant space characters from the input json byte slice +// and returns the compacted result. +func Ugly(json []byte) []byte { + buf := make([]byte, 0, len(json)) + return ugly(buf, json) +} + +// UglyInPlace removes insignificant space characters from the input json +// byte slice and returns the compacted result. This method reuses the +// input json buffer to avoid allocations. Do not use the original bytes +// slice upon return. +func UglyInPlace(json []byte) []byte { return ugly(json, json) } + +func ugly(dst, src []byte) []byte { + dst = dst[:0] + for i := 0; i < len(src); i++ { + if src[i] > ' ' { + dst = append(dst, src[i]) + if src[i] == '"' { + for i = i + 1; i < len(src); i++ { + dst = append(dst, src[i]) + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + } + } + } + return dst +} + +func isNaNOrInf(src []byte) bool { + return src[0] == 'i' || //Inf + src[0] == 'I' || // inf + src[0] == '+' || // +Inf + src[0] == 'N' || // Nan + (src[0] == 'n' && len(src) > 1 && src[1] != 'u') // nan +} + +func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) { + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + if json[i] == '"' { + return appendPrettyString(buf, json, i, nl) + } + + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' || isNaNOrInf(json[i:]) { + return appendPrettyNumber(buf, json, i, nl) + } + if json[i] == '{' { + return appendPrettyObject(buf, json, i, '{', '}', pretty, width, prefix, indent, sortkeys, tabs, nl, max) + } + if json[i] == '[' { + return appendPrettyObject(buf, json, i, '[', ']', pretty, width, prefix, indent, sortkeys, tabs, nl, max) + } + switch json[i] { + case 't': + return append(buf, 't', 'r', 'u', 'e'), i + 4, nl, true + case 'f': + return append(buf, 'f', 'a', 'l', 's', 'e'), i + 5, nl, true + case 'n': + return append(buf, 'n', 'u', 'l', 'l'), i + 4, nl, true + } + } + return buf, i, nl, true +} + +type pair struct { + kstart, kend int + vstart, vend int +} + +type byKeyVal struct { + sorted bool + json []byte + buf []byte + pairs []pair +} + +func (arr *byKeyVal) Len() int { + return len(arr.pairs) +} +func (arr *byKeyVal) Less(i, j int) bool { + if arr.isLess(i, j, byKey) { + return true + } + if arr.isLess(j, i, byKey) { + return false + } + return arr.isLess(i, j, byVal) +} +func (arr *byKeyVal) Swap(i, j int) { + arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i] + arr.sorted = true +} + +type byKind int + +const ( + byKey byKind = 0 + byVal byKind = 1 +) + +type jtype int + +const ( + jnull jtype = iota + jfalse + jnumber + jstring + jtrue + jjson +) + +func getjtype(v []byte) jtype { + if len(v) == 0 { + return jnull + } + switch v[0] { + case '"': + return jstring + case 'f': + return jfalse + case 't': + return jtrue + case 'n': + return jnull + case '[', '{': + return jjson + default: + return jnumber + } +} + +func (arr *byKeyVal) isLess(i, j int, kind byKind) bool { + k1 := arr.json[arr.pairs[i].kstart:arr.pairs[i].kend] + k2 := arr.json[arr.pairs[j].kstart:arr.pairs[j].kend] + var v1, v2 []byte + if kind == byKey { + v1 = k1 + v2 = k2 + } else { + v1 = bytes.TrimSpace(arr.buf[arr.pairs[i].vstart:arr.pairs[i].vend]) + v2 = bytes.TrimSpace(arr.buf[arr.pairs[j].vstart:arr.pairs[j].vend]) + if len(v1) >= len(k1)+1 { + v1 = bytes.TrimSpace(v1[len(k1)+1:]) + } + if len(v2) >= len(k2)+1 { + v2 = bytes.TrimSpace(v2[len(k2)+1:]) + } + } + t1 := getjtype(v1) + t2 := getjtype(v2) + if t1 < t2 { + return true + } + if t1 > t2 { + return false + } + if t1 == jstring { + s1 := parsestr(v1) + s2 := parsestr(v2) + return string(s1) < string(s2) + } + if t1 == jnumber { + n1, _ := strconv.ParseFloat(string(v1), 64) + n2, _ := strconv.ParseFloat(string(v2), 64) + return n1 < n2 + } + return string(v1) < string(v2) + +} + +func parsestr(s []byte) []byte { + for i := 1; i < len(s); i++ { + if s[i] == '\\' { + var str string + json.Unmarshal(s, &str) + return []byte(str) + } + if s[i] == '"' { + return s[1:i] + } + } + return nil +} + +func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) { + var ok bool + if width > 0 { + if pretty && open == '[' && max == -1 { + // here we try to create a single line array + max := width - (len(buf) - nl) + if max > 3 { + s1, s2 := len(buf), i + buf, i, _, ok = appendPrettyObject(buf, json, i, '[', ']', false, width, prefix, "", sortkeys, 0, 0, max) + if ok && len(buf)-s1 <= max { + return buf, i, nl, true + } + buf = buf[:s1] + i = s2 + } + } else if max != -1 && open == '{' { + return buf, i, nl, false + } + } + buf = append(buf, open) + i++ + var pairs []pair + if open == '{' && sortkeys { + pairs = make([]pair, 0, 8) + } + var n int + for ; i < len(json); i++ { + if json[i] <= ' ' { + continue + } + if json[i] == close { + if pretty { + if open == '{' && sortkeys { + buf = sortPairs(json, buf, pairs) + } + if n > 0 { + nl = len(buf) + if buf[nl-1] == ' ' { + buf[nl-1] = '\n' + } else { + buf = append(buf, '\n') + } + } + if buf[len(buf)-1] != open { + buf = appendTabs(buf, prefix, indent, tabs) + } + } + buf = append(buf, close) + return buf, i + 1, nl, open != '{' + } + if open == '[' || json[i] == '"' { + if n > 0 { + buf = append(buf, ',') + if width != -1 && open == '[' { + buf = append(buf, ' ') + } + } + var p pair + if pretty { + nl = len(buf) + if buf[nl-1] == ' ' { + buf[nl-1] = '\n' + } else { + buf = append(buf, '\n') + } + if open == '{' && sortkeys { + p.kstart = i + p.vstart = len(buf) + } + buf = appendTabs(buf, prefix, indent, tabs+1) + } + if open == '{' { + buf, i, nl, _ = appendPrettyString(buf, json, i, nl) + if sortkeys { + p.kend = i + } + buf = append(buf, ':') + if pretty { + buf = append(buf, ' ') + } + } + buf, i, nl, ok = appendPrettyAny(buf, json, i, pretty, width, prefix, indent, sortkeys, tabs+1, nl, max) + if max != -1 && !ok { + return buf, i, nl, false + } + if pretty && open == '{' && sortkeys { + p.vend = len(buf) + if p.kstart > p.kend || p.vstart > p.vend { + // bad data. disable sorting + sortkeys = false + } else { + pairs = append(pairs, p) + } + } + i-- + n++ + } + } + return buf, i, nl, open != '{' +} +func sortPairs(json, buf []byte, pairs []pair) []byte { + if len(pairs) == 0 { + return buf + } + vstart := pairs[0].vstart + vend := pairs[len(pairs)-1].vend + arr := byKeyVal{false, json, buf, pairs} + sort.Stable(&arr) + if !arr.sorted { + return buf + } + nbuf := make([]byte, 0, vend-vstart) + for i, p := range pairs { + nbuf = append(nbuf, buf[p.vstart:p.vend]...) + if i < len(pairs)-1 { + nbuf = append(nbuf, ',') + nbuf = append(nbuf, '\n') + } + } + return append(buf[:vstart], nbuf...) +} + +func appendPrettyString(buf, json []byte, i, nl int) ([]byte, int, int, bool) { + s := i + i++ + for ; i < len(json); i++ { + if json[i] == '"' { + var sc int + for j := i - 1; j > s; j-- { + if json[j] == '\\' { + sc++ + } else { + break + } + } + if sc%2 == 1 { + continue + } + i++ + break + } + } + return append(buf, json[s:i]...), i, nl, true +} + +func appendPrettyNumber(buf, json []byte, i, nl int) ([]byte, int, int, bool) { + s := i + i++ + for ; i < len(json); i++ { + if json[i] <= ' ' || json[i] == ',' || json[i] == ':' || json[i] == ']' || json[i] == '}' { + break + } + } + return append(buf, json[s:i]...), i, nl, true +} + +func appendTabs(buf []byte, prefix, indent string, tabs int) []byte { + if len(prefix) != 0 { + buf = append(buf, prefix...) + } + if len(indent) == 2 && indent[0] == ' ' && indent[1] == ' ' { + for i := 0; i < tabs; i++ { + buf = append(buf, ' ', ' ') + } + } else { + for i := 0; i < tabs; i++ { + buf = append(buf, indent...) + } + } + return buf +} + +// Style is the color style +type Style struct { + Key, String, Number [2]string + True, False, Null [2]string + Escape [2]string + Append func(dst []byte, c byte) []byte +} + +func hexp(p byte) byte { + switch { + case p < 10: + return p + '0' + default: + return (p - 10) + 'a' + } +} + +// TerminalStyle is for terminals +var TerminalStyle *Style + +func init() { + TerminalStyle = &Style{ + Key: [2]string{"\x1B[94m", "\x1B[0m"}, + String: [2]string{"\x1B[92m", "\x1B[0m"}, + Number: [2]string{"\x1B[93m", "\x1B[0m"}, + True: [2]string{"\x1B[96m", "\x1B[0m"}, + False: [2]string{"\x1B[96m", "\x1B[0m"}, + Null: [2]string{"\x1B[91m", "\x1B[0m"}, + Escape: [2]string{"\x1B[35m", "\x1B[0m"}, + Append: func(dst []byte, c byte) []byte { + if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') { + dst = append(dst, "\\u00"...) + dst = append(dst, hexp((c>>4)&0xF)) + return append(dst, hexp((c)&0xF)) + } + return append(dst, c) + }, + } +} + +// Color will colorize the json. The style parma is used for customizing +// the colors. Passing nil to the style param will use the default +// TerminalStyle. +func Color(src []byte, style *Style) []byte { + if style == nil { + style = TerminalStyle + } + apnd := style.Append + if apnd == nil { + apnd = func(dst []byte, c byte) []byte { + return append(dst, c) + } + } + type stackt struct { + kind byte + key bool + } + var dst []byte + var stack []stackt + for i := 0; i < len(src); i++ { + if src[i] == '"' { + key := len(stack) > 0 && stack[len(stack)-1].key + if key { + dst = append(dst, style.Key[0]...) + } else { + dst = append(dst, style.String[0]...) + } + dst = apnd(dst, '"') + esc := false + uesc := 0 + for i = i + 1; i < len(src); i++ { + if src[i] == '\\' { + if key { + dst = append(dst, style.Key[1]...) + } else { + dst = append(dst, style.String[1]...) + } + dst = append(dst, style.Escape[0]...) + dst = apnd(dst, src[i]) + esc = true + if i+1 < len(src) && src[i+1] == 'u' { + uesc = 5 + } else { + uesc = 1 + } + } else if esc { + dst = apnd(dst, src[i]) + if uesc == 1 { + esc = false + dst = append(dst, style.Escape[1]...) + if key { + dst = append(dst, style.Key[0]...) + } else { + dst = append(dst, style.String[0]...) + } + } else { + uesc-- + } + } else { + dst = apnd(dst, src[i]) + } + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + if esc { + dst = append(dst, style.Escape[1]...) + } else if key { + dst = append(dst, style.Key[1]...) + } else { + dst = append(dst, style.String[1]...) + } + } else if src[i] == '{' || src[i] == '[' { + stack = append(stack, stackt{src[i], src[i] == '{'}) + dst = apnd(dst, src[i]) + } else if (src[i] == '}' || src[i] == ']') && len(stack) > 0 { + stack = stack[:len(stack)-1] + dst = apnd(dst, src[i]) + } else if (src[i] == ':' || src[i] == ',') && len(stack) > 0 && stack[len(stack)-1].kind == '{' { + stack[len(stack)-1].key = !stack[len(stack)-1].key + dst = apnd(dst, src[i]) + } else { + var kind byte + if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' || isNaNOrInf(src[i:]) { + kind = '0' + dst = append(dst, style.Number[0]...) + } else if src[i] == 't' { + kind = 't' + dst = append(dst, style.True[0]...) + } else if src[i] == 'f' { + kind = 'f' + dst = append(dst, style.False[0]...) + } else if src[i] == 'n' { + kind = 'n' + dst = append(dst, style.Null[0]...) + } else { + dst = apnd(dst, src[i]) + } + if kind != 0 { + for ; i < len(src); i++ { + if src[i] <= ' ' || src[i] == ',' || src[i] == ':' || src[i] == ']' || src[i] == '}' { + i-- + break + } + dst = apnd(dst, src[i]) + } + if kind == '0' { + dst = append(dst, style.Number[1]...) + } else if kind == 't' { + dst = append(dst, style.True[1]...) + } else if kind == 'f' { + dst = append(dst, style.False[1]...) + } else if kind == 'n' { + dst = append(dst, style.Null[1]...) + } + } + } + } + return dst +} + +// Spec strips out comments and trailing commas and convert the input to a +// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259 +// +// The resulting JSON will always be the same length as the input and it will +// include all of the same line breaks at matching offsets. This is to ensure +// the result can be later processed by a external parser and that that +// parser will report messages or errors with the correct offsets. +func Spec(src []byte) []byte { + return spec(src, nil) +} + +// SpecInPlace is the same as Spec, but this method reuses the input json +// buffer to avoid allocations. Do not use the original bytes slice upon return. +func SpecInPlace(src []byte) []byte { + return spec(src, src) +} + +func spec(src, dst []byte) []byte { + dst = dst[:0] + for i := 0; i < len(src); i++ { + if src[i] == '/' { + if i < len(src)-1 { + if src[i+1] == '/' { + dst = append(dst, ' ', ' ') + i += 2 + for ; i < len(src); i++ { + if src[i] == '\n' { + dst = append(dst, '\n') + break + } else if src[i] == '\t' || src[i] == '\r' { + dst = append(dst, src[i]) + } else { + dst = append(dst, ' ') + } + } + continue + } + if src[i+1] == '*' { + dst = append(dst, ' ', ' ') + i += 2 + for ; i < len(src)-1; i++ { + if src[i] == '*' && src[i+1] == '/' { + dst = append(dst, ' ', ' ') + i++ + break + } else if src[i] == '\n' || src[i] == '\t' || + src[i] == '\r' { + dst = append(dst, src[i]) + } else { + dst = append(dst, ' ') + } + } + continue + } + } + } + dst = append(dst, src[i]) + if src[i] == '"' { + for i = i + 1; i < len(src); i++ { + dst = append(dst, src[i]) + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + } else if src[i] == '}' || src[i] == ']' { + for j := len(dst) - 2; j >= 0; j-- { + if dst[j] <= ' ' { + continue + } + if dst[j] == ',' { + dst[j] = ' ' + } + break + } + } + } + return dst +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6a7bf4e..6299bdb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,7 +1,5 @@ # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 github.com/beorn7/perks/quantile -# github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b -## explicit # github.com/golang/protobuf v1.2.0 github.com/golang/protobuf/proto # github.com/google/uuid v1.1.1 @@ -35,12 +33,13 @@ github.com/prometheus/procfs/xfs # github.com/sirupsen/logrus v1.2.0 ## explicit github.com/sirupsen/logrus -# github.com/tidwall/gjson v1.1.5 +# github.com/tidwall/gjson v1.9.3 ## explicit github.com/tidwall/gjson -# github.com/tidwall/match v1.0.1 -## explicit +# github.com/tidwall/match v1.1.1 github.com/tidwall/match +# github.com/tidwall/pretty v1.2.0 +github.com/tidwall/pretty # golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 golang.org/x/crypto/ssh/terminal # golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5