From 4a48e9ce1fb67183bf697fabe0622fefc95922eb Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 6 Sep 2020 16:55:23 -0400 Subject: [PATCH] miller json reader prototype-complete; remove go-ordered-json dependency --- go/README.md | 9 +- go/experimental/jtry.go | 15 +- go/src/README.md | 9 +- go/src/localdeps/ordered/README.md | 17 - go/src/localdeps/ordered/ordered.go | 271 ------------- go/src/localdeps/ordered/ordered_test.go | 450 ---------------------- go/src/miller/input/record_reader_json.go | 101 ++--- go/src/miller/lib/mlrval_accessors.go | 7 + go/src/miller/lib/mlrval_json.go | 91 +++-- go/todo.txt | 4 +- 10 files changed, 118 insertions(+), 856 deletions(-) delete mode 100644 go/src/localdeps/ordered/README.md delete mode 100644 go/src/localdeps/ordered/ordered.go delete mode 100644 go/src/localdeps/ordered/ordered_test.go diff --git a/go/README.md b/go/README.md index 670c921be..ad5846c09 100644 --- a/go/README.md +++ b/go/README.md @@ -70,15 +70,12 @@ So, in broad overview, the key packages are: ### Dependencies -* Miller dependencies are all in the Go standard library, except a couple local ones: - * `src/localdeps/ordered` - * Insertion-ordered (order-preserving) maps from [gitlab.com/c0b/go-ordered-json](https://gitlab.com/c0b/go-ordered-json): - * If you have a JSON data record `{"x":3,"y":4,"z":5}` then the keys `x,y,z` should stay that way. This package makes that happen. +* Miller dependencies are all in the Go standard library, except a local one: * `src/github.com/goccmack` * GOCC lexer/parser code-generator from [github.com/goccmack/gocc](https://github.com/goccmack/gocc): * This package defines the grammar for Miller's domain-specific language (DSL) for the Miller `put` and `filter` verbs. And, GOCC is a joy to use. :) - * I didn't put GOCC into `src/localdeps` since `go get github.com/goccmack/gocc` uses this directory path, and is nice enough to also create `bin/gocc` for me -- so I thought I would just let it continue to do that. :) -* I kept these locally so I could source-control them with Miller and guarantee their stability. They are used on the terms of the open-source licenses within their respective directories. + * Note on the path: `go get github.com/goccmack/gocc` uses this directory path, and is nice enough to also create `bin/gocc` for me -- so I thought I would just let it continue to do that by using that local path. :) +* I kept this locally so I could source-control it along with Miller and guarantee its stability. It is used on the terms of its open-source license. ### Miller per se diff --git a/go/experimental/jtry.go b/go/experimental/jtry.go index 78065e9c3..226dc89ae 100644 --- a/go/experimental/jtry.go +++ b/go/experimental/jtry.go @@ -13,10 +13,15 @@ import ( func main() { decoder := json.NewDecoder(os.Stdin) - mlrval, err := lib.MlrvalDecodeFromJSON(decoder) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + for { + mlrval, eof, err := lib.MlrvalDecodeFromJSON(decoder) + if eof { + break + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(mlrval) } - fmt.Println(mlrval) } diff --git a/go/src/README.md b/go/src/README.md index 110a23225..c34e65beb 100644 --- a/go/src/README.md +++ b/go/src/README.md @@ -1,9 +1,6 @@ -* Miller dependencies are all in the Go standard library, except a couple local ones: - * `src/localdeps/ordered` - * Insertion-ordered (order-preserving) maps from [gitlab.com/c0b/go-ordered-json](https://gitlab.com/c0b/go-ordered-json): - * If you have a JSON data record `{"x":3,"y":4,"z":5}` then the keys `x,y,z` should stay that way. This package makes that happen. +* Miller dependencies are all in the Go standard library, except a local one: * `src/github.com/goccmack` * GOCC lexer/parser code-generator from [github.com/goccmack/gocc](https://github.com/goccmack/gocc): * This package defines the grammar for Miller's domain-specific language (DSL) for the Miller `put` and `filter` verbs. And, GOCC is a joy to use. :) - * I didn't put GOCC into `src/localdeps` since `go get github.com/goccmack/gocc` uses this directory path, and is nice enough to also create `bin/gocc` for me -- so I thought I would just let it continue to do that. :) -* I kept these locally so I could source-control them with Miller and guarantee their stability. They are used on the terms of the open-source licenses within their respective directories. + * Note on the path: `go get github.com/goccmack/gocc` uses this directory path, and is nice enough to also create `bin/gocc` for me -- so I thought I would just let it continue to do that by using that local path. :) +* I kept this locally so I could source-control it along with Miller and guarantee its stability. It is used on the terms of its open-source license. diff --git a/go/src/localdeps/ordered/README.md b/go/src/localdeps/ordered/README.md deleted file mode 100644 index fcb1e75a1..000000000 --- a/go/src/localdeps/ordered/README.md +++ /dev/null @@ -1,17 +0,0 @@ -JSON object parsing with preserving keys order -============================================= - -![coverage](https://gitlab.com/c0b/go-ordered-json/badges/master/coverage.svg) - -Refers - -1. JSON and Go https://blog.golang.org/json-and-go -2. Go-Ordered-JSON https://github.com/virtuald/go-ordered-json - from this thread [*Preserving key order in encoding/json*](https://groups.google.com/forum/#!topic/golang-dev/zBQwhm3VfvU) - and the [*Abandoned 7930: encoding/json: Optionally preserve the key order of JSON objects*](https://go-review.googlesource.com/c/go/+/7930) -3. Python OrderedDict https://github.com/python/cpython/blob/2.7/Lib/collections.py#L38 - the Python's OrderedDict uses a double linked list internally, maintain a consistent public interface with `dict` - -Disclaimer: - -same as Go's default [map](https://blog.golang.org/go-maps-in-action), this OrderedMap is not safe for concurrent use, if need atomic access, may use a sync.Mutex to synchronize. diff --git a/go/src/localdeps/ordered/ordered.go b/go/src/localdeps/ordered/ordered.go deleted file mode 100644 index 0a14523c6..000000000 --- a/go/src/localdeps/ordered/ordered.go +++ /dev/null @@ -1,271 +0,0 @@ -// ================================================================ -// FOUND AT https://gitlab.com/c0b/go-ordered-json -// ================================================================ - -// Package ordered provided a type OrderedMap for use in JSON handling -// although JSON spec says the keys order of an object should not matter -// but sometimes when working with particular third-party proprietary code -// which has incorrect using the keys order, we have to maintain the object keys -// in the same order of incoming JSON object, this package is useful for these cases. -// -// Disclaimer: -// same as Go's default [map](https://blog.golang.org/go-maps-in-action), -// this OrderedMap is not safe for concurrent use, if need atomic access, may use a sync.Mutex to synchronize. -package ordered - -// Refers -// JSON and Go https://blog.golang.org/json-and-go -// Go-Ordered-JSON https://github.com/virtuald/go-ordered-json -// Python OrderedDict https://github.com/python/cpython/blob/2.7/Lib/collections.py#L38 -// port OrderedDict https://github.com/cevaris/ordered_map - -import ( - "bytes" - "container/list" - "encoding/json" - "fmt" - "io" -) - -// the key-value pair type, for initializing from a list of key-value pairs, or for looping entries in the same order -type KVPair struct { - Key string - Value interface{} -} - -type m map[string]interface{} - -// the OrderedMap type, has similar operations as the default map, but maintained -// the keys order of inserted; similar to map, all single key operations (Get/Set/Delete) runs at O(1). -type OrderedMap struct { - m - l *list.List - keys map[string]*list.Element // the double linked list for delete and lookup to be O(1) -} - -// Create a new OrderedMap -func NewOrderedMap() *OrderedMap { - return &OrderedMap{ - m: make(map[string]interface{}), - l: list.New(), - keys: make(map[string]*list.Element), - } -} - -// Create a new OrderedMap and populate from a list of key-value pairs -func NewOrderedMapFromKVPairs(pairs []*KVPair) *OrderedMap { - om := NewOrderedMap() - for _, pair := range pairs { - om.Set(pair.Key, pair.Value) - } - return om -} - -// return all keys -// func (om *OrderedMap) Keys() []string { return om.keys } - -// set value for particular key, this will remember the order of keys inserted -// but if the key already exists, the order is not updated. -func (om *OrderedMap) Set(key string, value interface{}) { - if _, ok := om.m[key]; !ok { - om.keys[key] = om.l.PushBack(key) - } - om.m[key] = value -} - -// Check if value exists -func (om *OrderedMap) Has(key string) bool { - _, ok := om.m[key] - return ok -} - -// Get value for particular key, or nil if not exist; but don't rely on nil for non-exist; should check by Has or GetValue -func (om *OrderedMap) Get(key string) interface{} { - return om.m[key] -} - -// Get value and exists together -func (om *OrderedMap) GetValue(key string) (value interface{}, ok bool) { - value, ok = om.m[key] - return -} - -// deletes the element with the specified key (m[key]) from the map. If there is no such element, this is a no-op. -func (om *OrderedMap) Delete(key string) (value interface{}, ok bool) { - value, ok = om.m[key] - if ok { - om.l.Remove(om.keys[key]) - delete(om.keys, key) - delete(om.m, key) - } - return -} - -// Iterate all key/value pairs in the same order of object constructed -func (om *OrderedMap) EntriesIter() func() (*KVPair, bool) { - e := om.l.Front() - return func() (*KVPair, bool) { - if e != nil { - key := e.Value.(string) - e = e.Next() - return &KVPair{key, om.m[key]}, true - } - return nil, false - } -} - -// Iterate all key/value pairs in the reverse order of object constructed -func (om *OrderedMap) EntriesReverseIter() func() (*KVPair, bool) { - e := om.l.Back() - return func() (*KVPair, bool) { - if e != nil { - key := e.Value.(string) - e = e.Prev() - return &KVPair{key, om.m[key]}, true - } - return nil, false - } -} - -// this implements type json.Marshaler interface, so can be called in json.Marshal(om) -func (om *OrderedMap) MarshalJSON() (res []byte, err error) { - res = append(res, '{') - front, back := om.l.Front(), om.l.Back() - for e := front; e != nil; e = e.Next() { - k := e.Value.(string) - res = append(res, fmt.Sprintf("%q:", k)...) - var b []byte - b, err = json.Marshal(om.m[k]) - if err != nil { - return - } - res = append(res, b...) - if e != back { - res = append(res, ',') - } - } - res = append(res, '}') - // fmt.Printf("marshalled: %v: %#v\n", res, res) - return -} - -// this implements type json.Unmarshaler interface, so can be called in json.Unmarshal(data, om) -func (om *OrderedMap) UnmarshalJSON(data []byte) error { - dec := json.NewDecoder(bytes.NewReader(data)) - dec.UseNumber() - - // must open with a delim token '{' - t, err := dec.Token() - if err != nil { - return err - } - if delim, ok := t.(json.Delim); !ok || delim != '{' { - return fmt.Errorf("expect JSON object open with '{'") - } - - err = om.parseobject(dec) - if err != nil { - return err - } - - t, err = dec.Token() - if err != io.EOF { - return fmt.Errorf("expect end of JSON object but got more token: %T: %v or err: %v", t, t, err) - } - - return nil -} - -func (om *OrderedMap) parseobject(dec *json.Decoder) (err error) { - var t json.Token - for dec.More() { - t, err = dec.Token() - if err != nil { - return err - } - - key, ok := t.(string) - if !ok { - return fmt.Errorf("expecting JSON key should be always a string: %T: %v", t, t) - } - - t, err = dec.Token() - if err == io.EOF { - break - } else if err != nil { - return err - } - - var value interface{} - value, err = handledelim(t, dec) - if err != nil { - return err - } - - // om.keys = append(om.keys, key) - om.keys[key] = om.l.PushBack(key) - om.m[key] = value - } - - t, err = dec.Token() - if err != nil { - return err - } - if delim, ok := t.(json.Delim); !ok || delim != '}' { - return fmt.Errorf("expect JSON object close with '}'") - } - - return nil -} - -func parsearray(dec *json.Decoder) (arr []interface{}, err error) { - var t json.Token - arr = make([]interface{}, 0) - for dec.More() { - t, err = dec.Token() - if err != nil { - return - } - - var value interface{} - value, err = handledelim(t, dec) - if err != nil { - return - } - arr = append(arr, value) - } - t, err = dec.Token() - if err != nil { - return - } - if delim, ok := t.(json.Delim); !ok || delim != ']' { - err = fmt.Errorf("expect JSON array close with ']'") - return - } - - return -} - -func handledelim(t json.Token, dec *json.Decoder) (res interface{}, err error) { - if delim, ok := t.(json.Delim); ok { - switch delim { - case '{': - om2 := NewOrderedMap() - err = om2.parseobject(dec) - if err != nil { - return - } - return om2, nil - case '[': - var value []interface{} - value, err = parsearray(dec) - if err != nil { - return - } - return value, nil - default: - return nil, fmt.Errorf("Unexpected delimiter: %q", delim) - } - } - return t, nil -} diff --git a/go/src/localdeps/ordered/ordered_test.go b/go/src/localdeps/ordered/ordered_test.go deleted file mode 100644 index 6f7ba8dfd..000000000 --- a/go/src/localdeps/ordered/ordered_test.go +++ /dev/null @@ -1,450 +0,0 @@ -// ================================================================ -// FOUND AT https://gitlab.com/c0b/go-ordered-json -// ================================================================ - -package ordered - -import ( - "bytes" - "encoding/json" - "fmt" - "math" - "reflect" - "strings" - "testing" -) - -func TestMarshalOrderedMap(t *testing.T) { - om := NewOrderedMap() - om.Set("a", 34) - om.Set("b", []int{3, 4, 5}) - b, err := json.Marshal(om) - if err != nil { - t.Fatalf("Marshal OrderedMap: %v", err) - } - // fmt.Printf("%q\n", b) - const expected = "{\"a\":34,\"b\":[3,4,5]}" - if !bytes.Equal(b, []byte(expected)) { - t.Errorf("Marshal OrderedMap: %q not equal to expected %q", b, expected) - } -} - -func ExampleOrderedMap_UnmarshalJSON() { - const jsonStream = `{ - "country" : "United States", - "countryCode" : "US", - "region" : "CA", - "regionName" : "California", - "city" : "Mountain View", - "zip" : "94043", - "lat" : 37.4192, - "lon" : -122.0574, - "timezone" : "America/Los_Angeles", - "isp" : "Google Cloud", - "org" : "Google Cloud", - "as" : "AS15169 Google Inc.", - "mobile" : true, - "proxy" : false, - "query" : "35.192.xx.xxx" -}` - - // compare with if using a regular generic map, the unmarshalled result - // is a map with unpredictable order of keys - var m map[string]interface{} - err := json.Unmarshal([]byte(jsonStream), &m) - if err != nil { - fmt.Println("error:", err) - } - for key := range m { - // fmt.Printf("%-12s: %v\n", key, m[key]) - _ = key - } - - // use the OrderedMap to Unmarshal from JSON object - var om *OrderedMap = NewOrderedMap() - err = json.Unmarshal([]byte(jsonStream), om) - if err != nil { - fmt.Println("error:", err) - } - - // use an iterator func to loop over all key-value pairs, - // it is ok to call Set append-modify new key-value pairs, - // but not safe to call Delete during iteration. - iter := om.EntriesIter() - for { - pair, ok := iter() - if !ok { - break - } - fmt.Printf("%-12s: %v\n", pair.Key, pair.Value) - if pair.Key == "city" { - om.Set("mobile", false) - om.Set("extra", 42) - } - } - - // Output: - // country : United States - // countryCode : US - // region : CA - // regionName : California - // city : Mountain View - // zip : 94043 - // lat : 37.4192 - // lon : -122.0574 - // timezone : America/Los_Angeles - // isp : Google Cloud - // org : Google Cloud - // as : AS15169 Google Inc. - // mobile : false - // proxy : false - // query : 35.192.xx.xxx - // extra : 42 -} - -func TestUnmarshalOrderedMapFromInvalid(t *testing.T) { - om := NewOrderedMap() - - om.Set("m", math.NaN()) - b, err := json.Marshal(om) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", b, err) - } - // fmt.Println(om, b, err) - om.Delete("m") - - err = json.Unmarshal([]byte("[]"), om) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error") - } - - err = json.Unmarshal([]byte("["), om) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - - err = om.UnmarshalJSON([]byte(nil)) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - - err = om.UnmarshalJSON([]byte("{}3")) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - - err = om.UnmarshalJSON([]byte("{")) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - - err = om.UnmarshalJSON([]byte("{]")) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - - err = om.UnmarshalJSON([]byte(`{"a": 3, "b": [{`)) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - - err = om.UnmarshalJSON([]byte(`{"a": 3, "b": [}`)) - if err == nil { - t.Fatal("Unmarshal OrderedMap: expecting error:", om) - } - // fmt.Println("error:", om, err) -} - -func TestUnmarshalOrderedMap(t *testing.T) { - var ( - data = []byte(`{"as":"AS15169 Google Inc.","city":"Mountain View","country":"United States","countryCode":"US","isp":"Google Cloud","lat":37.4192,"lon":-122.0574,"org":"Google Cloud","query":"35.192.25.53","region":"CA","regionName":"California","status":"success","timezone":"America/Los_Angeles","zip":"94043"}`) - pairs = []*KVPair{ - {"as", "AS15169 Google Inc."}, - {"city", "Mountain View"}, - {"country", "United States"}, - {"countryCode", "US"}, - {"isp", "Google Cloud"}, - {"lat", 37.4192}, - {"lon", -122.0574}, - {"org", "Google Cloud"}, - {"query", "35.192.25.53"}, - {"region", "CA"}, - {"regionName", "California"}, - {"status", "success"}, - {"timezone", "America/Los_Angeles"}, - {"zip", "94043"}, - } - obj = NewOrderedMapFromKVPairs(pairs) - ) - - om := NewOrderedMap() - err := json.Unmarshal(data, om) - if err != nil { - t.Fatalf("Unmarshal OrderedMap: %v", err) - } - - // fix number type for deepequal test - for _, key := range []string{"lat", "lon"} { - numf, _ := om.Get(key).(json.Number).Float64() - om.Set(key, numf) - } - - // check by Has and GetValue - for _, kv := range pairs { - if !om.Has(kv.Key) { - t.Fatalf("expect key %q exists in Unmarshaled OrderedMap") - } - value, ok := om.GetValue(kv.Key) - if !ok || value != kv.Value { - t.Fatalf("expect for key %q: the value %v should equal to %v, in Unmarshaled OrderedMap", kv.Key, value, kv.Value) - } - } - - b, err := json.MarshalIndent(om, "", " ") - if err != nil { - t.Fatalf("Unmarshal OrderedMap: %v", err) - } - const expected = `{ - "as": "AS15169 Google Inc.", - "city": "Mountain View", - "country": "United States", - "countryCode": "US", - "isp": "Google Cloud", - "lat": 37.4192, - "lon": -122.0574, - "org": "Google Cloud", - "query": "35.192.25.53", - "region": "CA", - "regionName": "California", - "status": "success", - "timezone": "America/Los_Angeles", - "zip": "94043" -}` - if !bytes.Equal(b, []byte(expected)) { - t.Fatalf("Unmarshal OrderedMap marshal indent from %#v not equal to expected: %q\n", om, expected) - } - - if !reflect.DeepEqual(om, obj) { - t.Fatalf("Unmarshal OrderedMap not deeply equal: %#v %#v", om, obj) - } - - val, ok := om.Delete("org") - if !ok { - t.Fatalf("org should exist") - } - om.Set("org", val) - b, err = json.MarshalIndent(om, "", " ") - // fmt.Println("after delete", om, string(b), err) - if err != nil { - t.Fatalf("Unmarshal OrderedMap: %v", err) - } - const expected2 = `{ - "as": "AS15169 Google Inc.", - "city": "Mountain View", - "country": "United States", - "countryCode": "US", - "isp": "Google Cloud", - "lat": 37.4192, - "lon": -122.0574, - "query": "35.192.25.53", - "region": "CA", - "regionName": "California", - "status": "success", - "timezone": "America/Los_Angeles", - "zip": "94043", - "org": "Google Cloud" -}` - if !bytes.Equal(b, []byte(expected2)) { - t.Fatalf("Unmarshal OrderedMap marshal indent from %#v not equal to expected: %s\n", om, expected2) - } -} - -func TestUnmarshalNestedOrderedMap(t *testing.T) { - var ( - data = []byte(`{"a": true, "b": [3, 4, { "b": "3", "d": [] }]}`) - obj = NewOrderedMapFromKVPairs([]*KVPair{ - {"a", true}, - {"b", []interface{}{3, 4, NewOrderedMapFromKVPairs([]*KVPair{ - {"b", "3"}, - {"d", []interface{}{}}, - })}}, - }) - ) - - om := NewOrderedMap() - err := json.Unmarshal(data, om) - if err != nil { - t.Fatalf("Unmarshal OrderedMap: %v", err) - } - - // b, err := json.MarshalIndent(om, "", " ") - // fmt.Println(om, string(b), err, obj) - - // fix number type for deepequal test - elearr := om.Get("b").([]interface{}) - for i, v := range elearr { - if num, ok := v.(json.Number); ok { - numi, _ := num.Int64() - elearr[i] = int(numi) - } - } - - if !reflect.DeepEqual(om, obj) { - t.Fatalf("Unmarshal OrderedMap not deeply equal: %#v expected %#v", om, obj) - } -} - -func ExampleOrderedMap_EntriesReverseIter() { - // initialize from a list of key-value pairs - om := NewOrderedMapFromKVPairs([]*KVPair{ - {"country", "United States"}, - {"countryCode", "US"}, - {"region", "CA"}, - {"regionName", "California"}, - {"city", "Mountain View"}, - {"zip", "94043"}, - {"lat", 37.4192}, - {"lon", -122.0574}, - {"timezone", "America/Los_Angeles"}, - {"isp", "Google Cloud"}, - {"org", "Google Cloud"}, - {"as", "AS15169 Google Inc."}, - {"mobile", true}, - {"proxy", false}, - {"query", "35.192.xx.xxx"}, - }) - - iter := om.EntriesReverseIter() - for { - pair, ok := iter() - if !ok { - break - } - fmt.Printf("%-12s: %v\n", pair.Key, pair.Value) - } - - // Output: - // query : 35.192.xx.xxx - // proxy : false - // mobile : true - // as : AS15169 Google Inc. - // org : Google Cloud - // isp : Google Cloud - // timezone : America/Los_Angeles - // lon : -122.0574 - // lat : 37.4192 - // zip : 94043 - // city : Mountain View - // regionName : California - // region : CA - // countryCode : US - // country : United States -} - -var unmarshalTests = []struct { - in string - new func() interface{} - out interface{} - err error - useNumber bool - golden bool -}{ - {in: `true`, new: func() interface{} { return new(bool) }, out: true}, - {in: `1`, new: func() interface{} { return new(int) }, out: 1}, - {in: `1.2`, new: func() interface{} { return new(float64) }, out: 1.2}, - {in: `-5`, new: func() interface{} { return new(int16) }, out: int16(-5)}, - {in: `2`, new: func() interface{} { return new(json.Number) }, out: json.Number("2"), useNumber: true}, - {in: `2`, new: func() interface{} { return new(json.Number) }, out: json.Number("2")}, - {in: `2`, new: func() interface{} { return new(interface{}) }, out: float64(2.0)}, - {in: `2`, new: func() interface{} { return new(interface{}) }, out: json.Number("2"), useNumber: true}, - {in: `"a\u1234"`, new: func() interface{} { return new(string) }, out: "a\u1234"}, - {in: `"http:\/\/"`, new: func() interface{} { return new(string) }, out: "http://"}, - {in: `"g-clef: \uD834\uDD1E"`, new: func() interface{} { return new(string) }, out: "g-clef: \U0001D11E"}, - {in: `"invalid: \uD834x\uDD1E"`, new: func() interface{} { return new(string) }, out: "invalid: \uFFFDx\uFFFD"}, - {in: "null", new: func() interface{} { return new(interface{}) }, out: nil}, - {in: "{}", new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs([]*KVPair{})}, - {in: `{"a": 3}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( - []*KVPair{{"a", json.Number("3")}})}, - {in: `{"a": 3, "b": true}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( - []*KVPair{{"a", json.Number("3")}, {"b", true}})}, - {in: `{"a": 3, "b": true, "c": null}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( - []*KVPair{{"a", json.Number("3")}, {"b", true}, {"c", nil}})}, - {in: `{"a": 3, "c": null, "d": []}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( - []*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{}}})}, - {in: `{"a": 3, "c": null, "d": [3,4,true]}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( - []*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{ - json.Number("3"), json.Number("4"), true, - }}})}, - {in: `{"a": 3, "c": null, "d": [3,4,true, { "inner": "abc" }]}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( - []*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{ - json.Number("3"), json.Number("4"), true, NewOrderedMapFromKVPairs([]*KVPair{{"inner", "abc"}}), - }}})}, -} - -func TestUnmarshal(t *testing.T) { - for i, tt := range unmarshalTests { - in := []byte(tt.in) - if tt.new == nil { - continue - } - - // v = new(right-type) - v := tt.new() // reflect.New(reflect.TypeOf(tt.ptr).Elem()) - dec := json.NewDecoder(bytes.NewReader(in)) - if tt.useNumber { - dec.UseNumber() - } - if err := dec.Decode(v); !reflect.DeepEqual(err, tt.err) { - t.Errorf("#%d: %v, want %v", i, err, tt.err) - continue - } else if err != nil { - continue - } - if !reflect.DeepEqual(reflect.ValueOf(v).Elem().Interface(), tt.out) { - t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v, tt.out) - data, _ := json.Marshal(v) - println(string(data)) - data, _ = json.Marshal(tt.out) - println(string(data)) - continue - } - - // Check round trip also decodes correctly. - if tt.err == nil { - enc, err := json.Marshal(v) - if err != nil { - t.Errorf("#%d: error re-marshaling: %v", i, err) - continue - } - if tt.golden && !bytes.Equal(enc, in) { - t.Errorf("#%d: remarshal mismatch:\nhave: %s\nwant: %s", i, enc, in) - } - vv := tt.new() // reflect.New(reflect.TypeOf(tt.ptr).Elem()) - dec = json.NewDecoder(bytes.NewReader(enc)) - if tt.useNumber { - dec.UseNumber() - } - if err := dec.Decode(vv); err != nil { - t.Errorf("#%d: error re-unmarshaling %#q: %v", i, enc, err) - continue - } - if !reflect.DeepEqual(v, vv) { - t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v, vv) - t.Errorf(" In: %q", strings.Map(noSpace, string(in))) - t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc))) - continue - } - } - } -} - -func noSpace(c rune) rune { - if isSpace(byte(c)) { //only used for ascii - return -1 - } - return c -} - -func isSpace(c byte) bool { - return c == ' ' || c == '\t' || c == '\r' || c == '\n' -} diff --git a/go/src/miller/input/record_reader_json.go b/go/src/miller/input/record_reader_json.go index ef1929b5c..f6f9fd132 100644 --- a/go/src/miller/input/record_reader_json.go +++ b/go/src/miller/input/record_reader_json.go @@ -1,10 +1,10 @@ package input import ( - "encoding/json" + "errors" "os" - "localdeps/ordered" + "encoding/json" "miller/clitypes" "miller/lib" @@ -51,88 +51,35 @@ func (this *RecordReaderJSON) processHandle( echan chan error, ) { context.UpdateForStartOfFile(filename) + decoder := json.NewDecoder(handle) - jsonDecoder := json.NewDecoder(handle) - - // TODO: - // mlrval, err lib.MlrvalDecodeFromJSON(decoder) - // if err != nil { - // ... - // } - // if it's an object: ichan it - // else if it's an array: loop over & ichan each if each is an object - // else echan - - // // Read opening bracket - // t, err := jsonDecoder.Token() - // if err != nil { - // echan <- err - // return - // } - // fmt.Printf("%T: %v\n", t, t) - - // Ordered-map idea from: - // https://gitlab.com/c0b/go-ordered-json - // found via - // https://github.com/golang/go/issues/27179 - - for jsonDecoder.More() { - - record := lib.NewMlrmap() - - var om *ordered.OrderedMap = ordered.NewOrderedMap() - err := jsonDecoder.Decode(om) + for { + mlrval, eof, err := lib.MlrvalDecodeFromJSON(decoder) + if eof { + break + } if err != nil { echan <- err return } - // Use an iterator func to loop over all key-value pairs. It is OK to call Set - // append-modify new key-value pairs, but not safe to call Delete during - // iteration. - iter := om.EntriesIter() - for { - pair, ok := iter() - if !ok { - break - } - - key := pair.Key // copy - value := pair.Value - // TODO: handle object values - - //fmt.Println("value is a ", reflect.TypeOf(value)) - - // xxx make helper functions - sval, ok := value.(string) - if ok { - // If it's double-quoted, leave it as a string, even if it - // looks like something parseable as int or float. - mval := lib.MlrvalFromString(sval) - record.Put(&key, &mval) - } else { - nval, ok := value.(json.Number) - if ok { - sval = nval.String() - mval := lib.MlrvalFromInferredType(sval) - record.Put(&key, &mval) - } + // Find out what we got. + // * Map is an input record: deliver it. + // * Array is OK if it's array of input record: deliver them. + // * Non-collection types are valid but unmillerable JSON. + + if mlrval.IsMap() { + record := mlrval.GetMap() + if record == nil { + echan <- errors.New("Internal coding error detected in JSON record-reader") + return } + context.UpdateForInputRecord(record) + inrecsAndContexts <- lib.NewRecordAndContext( + record, + context, + ) + } else { } - - context.UpdateForInputRecord(record) - - inrecsAndContexts <- lib.NewRecordAndContext( - record, - context, - ) } - - // // Read closing bracket - // t, err = jsonDecoder.Token() - // if err != nil { - // echan <- err - // return - // } - // fmt.Printf("%T: %v\n", t, t) } diff --git a/go/src/miller/lib/mlrval_accessors.go b/go/src/miller/lib/mlrval_accessors.go index 03963050b..9b449e930 100644 --- a/go/src/miller/lib/mlrval_accessors.go +++ b/go/src/miller/lib/mlrval_accessors.go @@ -49,3 +49,10 @@ func (this *Mlrval) IsArray() bool { func (this *Mlrval) IsMap() bool { return this.mvtype == MT_MAP } +func (this *Mlrval) GetMap() *Mlrmap { + if this.mvtype == MT_MAP { + return this.mapval + } else { + return nil + } +} diff --git a/go/src/miller/lib/mlrval_json.go b/go/src/miller/lib/mlrval_json.go index d4fa54a5e..aa2eaaa53 100644 --- a/go/src/miller/lib/mlrval_json.go +++ b/go/src/miller/lib/mlrval_json.go @@ -57,12 +57,45 @@ import ( // * Number, for JSON numbers // * string, for JSON string literals // * nil, for JSON null +// +// ---------------------------------------------------------------- +// Note: we accept a sequence of valid JSON items, not just a JSON item. +// E.g. either +// +// { +// "a": 1, +// "b": 2 +// } +// { +// "a": 3, +// "b": 4 +// } +// +// or +// +// [ +// { +// "a": 1, +// "b": 2 +// }, +// { +// "a": 3, +// "b": 4 +// } +// ] +// +// This is so the Miller JSON record-reader can be streaming, not needing to +// ingest all records at once, and operable within a tail -f context. +// ================================================================ // ---------------------------------------------------------------- func (this *Mlrval) UnmarshalJSON(inputBytes []byte) error { *this = MlrvalFromPending() decoder := json.NewDecoder(bytes.NewReader(inputBytes)) - mlrval, err := MlrvalDecodeFromJSON(decoder) + mlrval, eof, err := MlrvalDecodeFromJSON(decoder) + if eof { + return errors.New("Miller JSON parser: unexpected premature EOF.") + } if err != nil { return err } @@ -71,45 +104,45 @@ func (this *Mlrval) UnmarshalJSON(inputBytes []byte) error { } // ---------------------------------------------------------------- -func MlrvalDecodeFromJSON(decoder *json.Decoder) (*Mlrval, error) { +func MlrvalDecodeFromJSON(decoder *json.Decoder) (mlrval *Mlrval, eof bool, err error) { // Causes the decoder to unmarshal a number into an interface{} as a Number // instead of as a float64. decoder.UseNumber() startToken, err := decoder.Token() if err == io.EOF { - return nil, errors.New("Miller JSON reader: Unexpected end of file") + return nil, true, nil } if err != nil { - return nil, err + return nil, false, err } delimiter, isDelim := startToken.(json.Delim) if !isDelim { if startToken == nil { mlrval := MlrvalFromVoid() - return &mlrval, nil + return &mlrval, false, nil } sval, ok := startToken.(string) if ok { mlrval := MlrvalFromString(sval) - return &mlrval, nil + return &mlrval, false, nil } bval, ok := startToken.(bool) if ok { mlrval := MlrvalFromBool(bval) - return &mlrval, nil + return &mlrval, false, nil } nval, ok := startToken.(json.Number) if ok { mlrval := MlrvalFromInferredType(nval.String()) - return &mlrval, nil + return &mlrval, false, nil } - return nil, errors.New( + return nil, false, errors.New( "Miller JSON reader: internal coding error: non-delimiter token unhandled", ) @@ -127,7 +160,7 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) (*Mlrval, error) { expectedClosingDelimiter = '}' collectionType = "JSON object`" } else { - return nil, errors.New( + return nil, false, errors.New( "Miller JSON reader: Unhandled opening delimiter \"" + string(delimiter) + "\"", ) } @@ -137,9 +170,13 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) (*Mlrval, error) { mlrval = MlrvalEmptyArray() for decoder.More() { - element, err := MlrvalDecodeFromJSON(decoder) + element, eof, err := MlrvalDecodeFromJSON(decoder) + if eof { + // xxx constify + return nil, false, errors.New("Miller JSON parser: unexpected premature EOF.") + } if err != nil { - return nil, err + return nil, false, err } mlrval.ArrayExtend(element) } @@ -148,20 +185,28 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) (*Mlrval, error) { mlrval = MlrvalEmptyMap() for decoder.More() { - key, err := MlrvalDecodeFromJSON(decoder) + key, eof, err := MlrvalDecodeFromJSON(decoder) + if eof { + // xxx constify + return nil, false, errors.New("Miller JSON parser: unexpected premature EOF.") + } if err != nil { - return nil, err + return nil, false, err } if !key.IsString() { - return nil, errors.New( + return nil, false, errors.New( // TODO: print out what was gotten "Miller JSON reader: obejct keys must be string-valued.", ) } - value, err := MlrvalDecodeFromJSON(decoder) + value, eof, err := MlrvalDecodeFromJSON(decoder) + if eof { + // xxx constify + return nil, false, errors.New("Miller JSON parser: unexpected premature EOF.") + } if err != nil { - return nil, err + return nil, false, err } // xxx check here string-valued key @@ -178,23 +223,23 @@ func MlrvalDecodeFromJSON(decoder *json.Decoder) (*Mlrval, error) { endToken, err := decoder.Token() if err != nil { - return nil, err + return nil, false, err } if endToken == nil { - return nil, imbalanceError + return nil, false, imbalanceError } dval, ok := endToken.(json.Delim) if !ok { - return nil, imbalanceError + return nil, false, imbalanceError } if rune(dval) != expectedClosingDelimiter { - return nil, imbalanceError + return nil, false, imbalanceError } - return &mlrval, nil + return &mlrval, false, nil } - return nil, errors.New("unimplemented") + return nil, false, errors.New("unimplemented") } // ================================================================ diff --git a/go/todo.txt b/go/todo.txt index c211195de..5ff43bd4f 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -4,12 +4,14 @@ TOP OF LIST: !! stdin read broken! put in checker. * echan -> errorChannel throughout * inrecsAndContexts -> inputChannel; likewise output +* fix panic in 'mlr --dkvp cat u/s.json' ! json unmarshal/marshal at mlrval * need nesting-aware json printer * thorough UT for json mlrval-parser * handle EOF better: '[1,2,3' - * how necessary is UnmarshalJSON + * how necessary is UnmarshalJSON -- UT only i think ... + * doc re no jlistwrap on input to get streaming input o support full JSON read/write of nested objects - only requirement is that top-level be sequence of string-valued objects ... - JSON-to-JSON cat-mapping should be identical