From da6830a7a9399cdce4d388ac02f10f92bf233db6 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 27 Aug 2020 14:33:57 -0400 Subject: [PATCH] json reader with ordered maps --- go/src/deps/ordered/README.md | 17 + go/src/deps/ordered/ordered.go | 267 +++++++++++++ go/src/deps/ordered/ordered_test.go | 446 ++++++++++++++++++++++ go/src/miller/input/record_reader_json.go | 45 ++- 4 files changed, 756 insertions(+), 19 deletions(-) create mode 100644 go/src/deps/ordered/README.md create mode 100644 go/src/deps/ordered/ordered.go create mode 100644 go/src/deps/ordered/ordered_test.go diff --git a/go/src/deps/ordered/README.md b/go/src/deps/ordered/README.md new file mode 100644 index 000000000..fcb1e75a1 --- /dev/null +++ b/go/src/deps/ordered/README.md @@ -0,0 +1,17 @@ +JSON object parsing with preserving keys order +============================================= + +![coverage](https://gitlab.com/c0b/go-ordered-json/badges/master/coverage.svg) + +Refers + +1. JSON and Go https://blog.golang.org/json-and-go +2. Go-Ordered-JSON https://github.com/virtuald/go-ordered-json + from this thread [*Preserving key order in encoding/json*](https://groups.google.com/forum/#!topic/golang-dev/zBQwhm3VfvU) + and the [*Abandoned 7930: encoding/json: Optionally preserve the key order of JSON objects*](https://go-review.googlesource.com/c/go/+/7930) +3. Python OrderedDict https://github.com/python/cpython/blob/2.7/Lib/collections.py#L38 + the Python's OrderedDict uses a double linked list internally, maintain a consistent public interface with `dict` + +Disclaimer: + +same as Go's default [map](https://blog.golang.org/go-maps-in-action), this OrderedMap is not safe for concurrent use, if need atomic access, may use a sync.Mutex to synchronize. diff --git a/go/src/deps/ordered/ordered.go b/go/src/deps/ordered/ordered.go new file mode 100644 index 000000000..550c0e4b9 --- /dev/null +++ b/go/src/deps/ordered/ordered.go @@ -0,0 +1,267 @@ +// Package ordered provided a type OrderedMap for use in JSON handling +// although JSON spec says the keys order of an object should not matter +// but sometimes when working with particular third-party proprietary code +// which has incorrect using the keys order, we have to maintain the object keys +// in the same order of incoming JSON object, this package is useful for these cases. +// +// Disclaimer: +// same as Go's default [map](https://blog.golang.org/go-maps-in-action), +// this OrderedMap is not safe for concurrent use, if need atomic access, may use a sync.Mutex to synchronize. +package ordered + +// Refers +// JSON and Go https://blog.golang.org/json-and-go +// Go-Ordered-JSON https://github.com/virtuald/go-ordered-json +// Python OrderedDict https://github.com/python/cpython/blob/2.7/Lib/collections.py#L38 +// port OrderedDict https://github.com/cevaris/ordered_map + +import ( + "bytes" + "container/list" + "encoding/json" + "fmt" + "io" +) + +// the key-value pair type, for initializing from a list of key-value pairs, or for looping entries in the same order +type KVPair struct { + Key string + Value interface{} +} + +type m map[string]interface{} + +// the OrderedMap type, has similar operations as the default map, but maintained +// the keys order of inserted; similar to map, all single key operations (Get/Set/Delete) runs at O(1). +type OrderedMap struct { + m + l *list.List + keys map[string]*list.Element // the double linked list for delete and lookup to be O(1) +} + +// Create a new OrderedMap +func NewOrderedMap() *OrderedMap { + return &OrderedMap{ + m: make(map[string]interface{}), + l: list.New(), + keys: make(map[string]*list.Element), + } +} + +// Create a new OrderedMap and populate from a list of key-value pairs +func NewOrderedMapFromKVPairs(pairs []*KVPair) *OrderedMap { + om := NewOrderedMap() + for _, pair := range pairs { + om.Set(pair.Key, pair.Value) + } + return om +} + +// return all keys +// func (om *OrderedMap) Keys() []string { return om.keys } + +// set value for particular key, this will remember the order of keys inserted +// but if the key already exists, the order is not updated. +func (om *OrderedMap) Set(key string, value interface{}) { + if _, ok := om.m[key]; !ok { + om.keys[key] = om.l.PushBack(key) + } + om.m[key] = value +} + +// Check if value exists +func (om *OrderedMap) Has(key string) bool { + _, ok := om.m[key] + return ok +} + +// Get value for particular key, or nil if not exist; but don't rely on nil for non-exist; should check by Has or GetValue +func (om *OrderedMap) Get(key string) interface{} { + return om.m[key] +} + +// Get value and exists together +func (om *OrderedMap) GetValue(key string) (value interface{}, ok bool) { + value, ok = om.m[key] + return +} + +// deletes the element with the specified key (m[key]) from the map. If there is no such element, this is a no-op. +func (om *OrderedMap) Delete(key string) (value interface{}, ok bool) { + value, ok = om.m[key] + if ok { + om.l.Remove(om.keys[key]) + delete(om.keys, key) + delete(om.m, key) + } + return +} + +// Iterate all key/value pairs in the same order of object constructed +func (om *OrderedMap) EntriesIter() func() (*KVPair, bool) { + e := om.l.Front() + return func() (*KVPair, bool) { + if e != nil { + key := e.Value.(string) + e = e.Next() + return &KVPair{key, om.m[key]}, true + } + return nil, false + } +} + +// Iterate all key/value pairs in the reverse order of object constructed +func (om *OrderedMap) EntriesReverseIter() func() (*KVPair, bool) { + e := om.l.Back() + return func() (*KVPair, bool) { + if e != nil { + key := e.Value.(string) + e = e.Prev() + return &KVPair{key, om.m[key]}, true + } + return nil, false + } +} + +// this implements type json.Marshaler interface, so can be called in json.Marshal(om) +func (om *OrderedMap) MarshalJSON() (res []byte, err error) { + res = append(res, '{') + front, back := om.l.Front(), om.l.Back() + for e := front; e != nil; e = e.Next() { + k := e.Value.(string) + res = append(res, fmt.Sprintf("%q:", k)...) + var b []byte + b, err = json.Marshal(om.m[k]) + if err != nil { + return + } + res = append(res, b...) + if e != back { + res = append(res, ',') + } + } + res = append(res, '}') + // fmt.Printf("marshalled: %v: %#v\n", res, res) + return +} + +// this implements type json.Unmarshaler interface, so can be called in json.Unmarshal(data, om) +func (om *OrderedMap) UnmarshalJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + + // must open with a delim token '{' + t, err := dec.Token() + if err != nil { + return err + } + if delim, ok := t.(json.Delim); !ok || delim != '{' { + return fmt.Errorf("expect JSON object open with '{'") + } + + err = om.parseobject(dec) + if err != nil { + return err + } + + t, err = dec.Token() + if err != io.EOF { + return fmt.Errorf("expect end of JSON object but got more token: %T: %v or err: %v", t, t, err) + } + + return nil +} + +func (om *OrderedMap) parseobject(dec *json.Decoder) (err error) { + var t json.Token + for dec.More() { + t, err = dec.Token() + if err != nil { + return err + } + + key, ok := t.(string) + if !ok { + return fmt.Errorf("expecting JSON key should be always a string: %T: %v", t, t) + } + + t, err = dec.Token() + if err == io.EOF { + break + } else if err != nil { + return err + } + + var value interface{} + value, err = handledelim(t, dec) + if err != nil { + return err + } + + // om.keys = append(om.keys, key) + om.keys[key] = om.l.PushBack(key) + om.m[key] = value + } + + t, err = dec.Token() + if err != nil { + return err + } + if delim, ok := t.(json.Delim); !ok || delim != '}' { + return fmt.Errorf("expect JSON object close with '}'") + } + + return nil +} + +func parsearray(dec *json.Decoder) (arr []interface{}, err error) { + var t json.Token + arr = make([]interface{}, 0) + for dec.More() { + t, err = dec.Token() + if err != nil { + return + } + + var value interface{} + value, err = handledelim(t, dec) + if err != nil { + return + } + arr = append(arr, value) + } + t, err = dec.Token() + if err != nil { + return + } + if delim, ok := t.(json.Delim); !ok || delim != ']' { + err = fmt.Errorf("expect JSON array close with ']'") + return + } + + return +} + +func handledelim(t json.Token, dec *json.Decoder) (res interface{}, err error) { + if delim, ok := t.(json.Delim); ok { + switch delim { + case '{': + om2 := NewOrderedMap() + err = om2.parseobject(dec) + if err != nil { + return + } + return om2, nil + case '[': + var value []interface{} + value, err = parsearray(dec) + if err != nil { + return + } + return value, nil + default: + return nil, fmt.Errorf("Unexpected delimiter: %q", delim) + } + } + return t, nil +} diff --git a/go/src/deps/ordered/ordered_test.go b/go/src/deps/ordered/ordered_test.go new file mode 100644 index 000000000..b716b0018 --- /dev/null +++ b/go/src/deps/ordered/ordered_test.go @@ -0,0 +1,446 @@ +package ordered + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "reflect" + "strings" + "testing" +) + +func TestMarshalOrderedMap(t *testing.T) { + om := NewOrderedMap() + om.Set("a", 34) + om.Set("b", []int{3, 4, 5}) + b, err := json.Marshal(om) + if err != nil { + t.Fatalf("Marshal OrderedMap: %v", err) + } + // fmt.Printf("%q\n", b) + const expected = "{\"a\":34,\"b\":[3,4,5]}" + if !bytes.Equal(b, []byte(expected)) { + t.Errorf("Marshal OrderedMap: %q not equal to expected %q", b, expected) + } +} + +func ExampleOrderedMap_UnmarshalJSON() { + const jsonStream = `{ + "country" : "United States", + "countryCode" : "US", + "region" : "CA", + "regionName" : "California", + "city" : "Mountain View", + "zip" : "94043", + "lat" : 37.4192, + "lon" : -122.0574, + "timezone" : "America/Los_Angeles", + "isp" : "Google Cloud", + "org" : "Google Cloud", + "as" : "AS15169 Google Inc.", + "mobile" : true, + "proxy" : false, + "query" : "35.192.xx.xxx" +}` + + // compare with if using a regular generic map, the unmarshalled result + // is a map with unpredictable order of keys + var m map[string]interface{} + err := json.Unmarshal([]byte(jsonStream), &m) + if err != nil { + fmt.Println("error:", err) + } + for key := range m { + // fmt.Printf("%-12s: %v\n", key, m[key]) + _ = key + } + + // use the OrderedMap to Unmarshal from JSON object + var om *OrderedMap = NewOrderedMap() + err = json.Unmarshal([]byte(jsonStream), om) + if err != nil { + fmt.Println("error:", err) + } + + // use an iterator func to loop over all key-value pairs, + // it is ok to call Set append-modify new key-value pairs, + // but not safe to call Delete during iteration. + iter := om.EntriesIter() + for { + pair, ok := iter() + if !ok { + break + } + fmt.Printf("%-12s: %v\n", pair.Key, pair.Value) + if pair.Key == "city" { + om.Set("mobile", false) + om.Set("extra", 42) + } + } + + // Output: + // country : United States + // countryCode : US + // region : CA + // regionName : California + // city : Mountain View + // zip : 94043 + // lat : 37.4192 + // lon : -122.0574 + // timezone : America/Los_Angeles + // isp : Google Cloud + // org : Google Cloud + // as : AS15169 Google Inc. + // mobile : false + // proxy : false + // query : 35.192.xx.xxx + // extra : 42 +} + +func TestUnmarshalOrderedMapFromInvalid(t *testing.T) { + om := NewOrderedMap() + + om.Set("m", math.NaN()) + b, err := json.Marshal(om) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", b, err) + } + // fmt.Println(om, b, err) + om.Delete("m") + + err = json.Unmarshal([]byte("[]"), om) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error") + } + + err = json.Unmarshal([]byte("["), om) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + + err = om.UnmarshalJSON([]byte(nil)) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + + err = om.UnmarshalJSON([]byte("{}3")) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + + err = om.UnmarshalJSON([]byte("{")) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + + err = om.UnmarshalJSON([]byte("{]")) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + + err = om.UnmarshalJSON([]byte(`{"a": 3, "b": [{`)) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + + err = om.UnmarshalJSON([]byte(`{"a": 3, "b": [}`)) + if err == nil { + t.Fatal("Unmarshal OrderedMap: expecting error:", om) + } + // fmt.Println("error:", om, err) +} + +func TestUnmarshalOrderedMap(t *testing.T) { + var ( + data = []byte(`{"as":"AS15169 Google Inc.","city":"Mountain View","country":"United States","countryCode":"US","isp":"Google Cloud","lat":37.4192,"lon":-122.0574,"org":"Google Cloud","query":"35.192.25.53","region":"CA","regionName":"California","status":"success","timezone":"America/Los_Angeles","zip":"94043"}`) + pairs = []*KVPair{ + {"as", "AS15169 Google Inc."}, + {"city", "Mountain View"}, + {"country", "United States"}, + {"countryCode", "US"}, + {"isp", "Google Cloud"}, + {"lat", 37.4192}, + {"lon", -122.0574}, + {"org", "Google Cloud"}, + {"query", "35.192.25.53"}, + {"region", "CA"}, + {"regionName", "California"}, + {"status", "success"}, + {"timezone", "America/Los_Angeles"}, + {"zip", "94043"}, + } + obj = NewOrderedMapFromKVPairs(pairs) + ) + + om := NewOrderedMap() + err := json.Unmarshal(data, om) + if err != nil { + t.Fatalf("Unmarshal OrderedMap: %v", err) + } + + // fix number type for deepequal test + for _, key := range []string{"lat", "lon"} { + numf, _ := om.Get(key).(json.Number).Float64() + om.Set(key, numf) + } + + // check by Has and GetValue + for _, kv := range pairs { + if !om.Has(kv.Key) { + t.Fatalf("expect key %q exists in Unmarshaled OrderedMap") + } + value, ok := om.GetValue(kv.Key) + if !ok || value != kv.Value { + t.Fatalf("expect for key %q: the value %v should equal to %v, in Unmarshaled OrderedMap", kv.Key, value, kv.Value) + } + } + + b, err := json.MarshalIndent(om, "", " ") + if err != nil { + t.Fatalf("Unmarshal OrderedMap: %v", err) + } + const expected = `{ + "as": "AS15169 Google Inc.", + "city": "Mountain View", + "country": "United States", + "countryCode": "US", + "isp": "Google Cloud", + "lat": 37.4192, + "lon": -122.0574, + "org": "Google Cloud", + "query": "35.192.25.53", + "region": "CA", + "regionName": "California", + "status": "success", + "timezone": "America/Los_Angeles", + "zip": "94043" +}` + if !bytes.Equal(b, []byte(expected)) { + t.Fatalf("Unmarshal OrderedMap marshal indent from %#v not equal to expected: %q\n", om, expected) + } + + if !reflect.DeepEqual(om, obj) { + t.Fatalf("Unmarshal OrderedMap not deeply equal: %#v %#v", om, obj) + } + + val, ok := om.Delete("org") + if !ok { + t.Fatalf("org should exist") + } + om.Set("org", val) + b, err = json.MarshalIndent(om, "", " ") + // fmt.Println("after delete", om, string(b), err) + if err != nil { + t.Fatalf("Unmarshal OrderedMap: %v", err) + } + const expected2 = `{ + "as": "AS15169 Google Inc.", + "city": "Mountain View", + "country": "United States", + "countryCode": "US", + "isp": "Google Cloud", + "lat": 37.4192, + "lon": -122.0574, + "query": "35.192.25.53", + "region": "CA", + "regionName": "California", + "status": "success", + "timezone": "America/Los_Angeles", + "zip": "94043", + "org": "Google Cloud" +}` + if !bytes.Equal(b, []byte(expected2)) { + t.Fatalf("Unmarshal OrderedMap marshal indent from %#v not equal to expected: %s\n", om, expected2) + } +} + +func TestUnmarshalNestedOrderedMap(t *testing.T) { + var ( + data = []byte(`{"a": true, "b": [3, 4, { "b": "3", "d": [] }]}`) + obj = NewOrderedMapFromKVPairs([]*KVPair{ + {"a", true}, + {"b", []interface{}{3, 4, NewOrderedMapFromKVPairs([]*KVPair{ + {"b", "3"}, + {"d", []interface{}{}}, + })}}, + }) + ) + + om := NewOrderedMap() + err := json.Unmarshal(data, om) + if err != nil { + t.Fatalf("Unmarshal OrderedMap: %v", err) + } + + // b, err := json.MarshalIndent(om, "", " ") + // fmt.Println(om, string(b), err, obj) + + // fix number type for deepequal test + elearr := om.Get("b").([]interface{}) + for i, v := range elearr { + if num, ok := v.(json.Number); ok { + numi, _ := num.Int64() + elearr[i] = int(numi) + } + } + + if !reflect.DeepEqual(om, obj) { + t.Fatalf("Unmarshal OrderedMap not deeply equal: %#v expected %#v", om, obj) + } +} + +func ExampleOrderedMap_EntriesReverseIter() { + // initialize from a list of key-value pairs + om := NewOrderedMapFromKVPairs([]*KVPair{ + {"country", "United States"}, + {"countryCode", "US"}, + {"region", "CA"}, + {"regionName", "California"}, + {"city", "Mountain View"}, + {"zip", "94043"}, + {"lat", 37.4192}, + {"lon", -122.0574}, + {"timezone", "America/Los_Angeles"}, + {"isp", "Google Cloud"}, + {"org", "Google Cloud"}, + {"as", "AS15169 Google Inc."}, + {"mobile", true}, + {"proxy", false}, + {"query", "35.192.xx.xxx"}, + }) + + iter := om.EntriesReverseIter() + for { + pair, ok := iter() + if !ok { + break + } + fmt.Printf("%-12s: %v\n", pair.Key, pair.Value) + } + + // Output: + // query : 35.192.xx.xxx + // proxy : false + // mobile : true + // as : AS15169 Google Inc. + // org : Google Cloud + // isp : Google Cloud + // timezone : America/Los_Angeles + // lon : -122.0574 + // lat : 37.4192 + // zip : 94043 + // city : Mountain View + // regionName : California + // region : CA + // countryCode : US + // country : United States +} + +var unmarshalTests = []struct { + in string + new func() interface{} + out interface{} + err error + useNumber bool + golden bool +}{ + {in: `true`, new: func() interface{} { return new(bool) }, out: true}, + {in: `1`, new: func() interface{} { return new(int) }, out: 1}, + {in: `1.2`, new: func() interface{} { return new(float64) }, out: 1.2}, + {in: `-5`, new: func() interface{} { return new(int16) }, out: int16(-5)}, + {in: `2`, new: func() interface{} { return new(json.Number) }, out: json.Number("2"), useNumber: true}, + {in: `2`, new: func() interface{} { return new(json.Number) }, out: json.Number("2")}, + {in: `2`, new: func() interface{} { return new(interface{}) }, out: float64(2.0)}, + {in: `2`, new: func() interface{} { return new(interface{}) }, out: json.Number("2"), useNumber: true}, + {in: `"a\u1234"`, new: func() interface{} { return new(string) }, out: "a\u1234"}, + {in: `"http:\/\/"`, new: func() interface{} { return new(string) }, out: "http://"}, + {in: `"g-clef: \uD834\uDD1E"`, new: func() interface{} { return new(string) }, out: "g-clef: \U0001D11E"}, + {in: `"invalid: \uD834x\uDD1E"`, new: func() interface{} { return new(string) }, out: "invalid: \uFFFDx\uFFFD"}, + {in: "null", new: func() interface{} { return new(interface{}) }, out: nil}, + {in: "{}", new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs([]*KVPair{})}, + {in: `{"a": 3}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( + []*KVPair{{"a", json.Number("3")}})}, + {in: `{"a": 3, "b": true}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( + []*KVPair{{"a", json.Number("3")}, {"b", true}})}, + {in: `{"a": 3, "b": true, "c": null}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( + []*KVPair{{"a", json.Number("3")}, {"b", true}, {"c", nil}})}, + {in: `{"a": 3, "c": null, "d": []}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( + []*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{}}})}, + {in: `{"a": 3, "c": null, "d": [3,4,true]}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( + []*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{ + json.Number("3"), json.Number("4"), true, + }}})}, + {in: `{"a": 3, "c": null, "d": [3,4,true, { "inner": "abc" }]}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs( + []*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{ + json.Number("3"), json.Number("4"), true, NewOrderedMapFromKVPairs([]*KVPair{{"inner", "abc"}}), + }}})}, +} + +func TestUnmarshal(t *testing.T) { + for i, tt := range unmarshalTests { + in := []byte(tt.in) + if tt.new == nil { + continue + } + + // v = new(right-type) + v := tt.new() // reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec := json.NewDecoder(bytes.NewReader(in)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(v); !reflect.DeepEqual(err, tt.err) { + t.Errorf("#%d: %v, want %v", i, err, tt.err) + continue + } else if err != nil { + continue + } + if !reflect.DeepEqual(reflect.ValueOf(v).Elem().Interface(), tt.out) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v, tt.out) + data, _ := json.Marshal(v) + println(string(data)) + data, _ = json.Marshal(tt.out) + println(string(data)) + continue + } + + // Check round trip also decodes correctly. + if tt.err == nil { + enc, err := json.Marshal(v) + if err != nil { + t.Errorf("#%d: error re-marshaling: %v", i, err) + continue + } + if tt.golden && !bytes.Equal(enc, in) { + t.Errorf("#%d: remarshal mismatch:\nhave: %s\nwant: %s", i, enc, in) + } + vv := tt.new() // reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec = json.NewDecoder(bytes.NewReader(enc)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(vv); err != nil { + t.Errorf("#%d: error re-unmarshaling %#q: %v", i, enc, err) + continue + } + if !reflect.DeepEqual(v, vv) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v, vv) + t.Errorf(" In: %q", strings.Map(noSpace, string(in))) + t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc))) + continue + } + } + } +} + +func noSpace(c rune) rune { + if isSpace(byte(c)) { //only used for ascii + return -1 + } + return c +} + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' +} diff --git a/go/src/miller/input/record_reader_json.go b/go/src/miller/input/record_reader_json.go index 815d2e9ed..a2abfc55d 100644 --- a/go/src/miller/input/record_reader_json.go +++ b/go/src/miller/input/record_reader_json.go @@ -7,8 +7,12 @@ import ( "os" //"reflect" "strconv" + // Miller: "miller/containers" + + // Local dependencies: + "deps/ordered" ) type RecordReaderJSON struct { @@ -43,47 +47,50 @@ func (this *RecordReaderJSON) Read( // } // fmt.Printf("%T: %v\n", t, t) + // Ordered-map idea from: + // https://gitlab.com/c0b/go-ordered-json + // found via + // https://github.com/golang/go/issues/27179 + for jsonDecoder.More() { - // decode an array value (Message) - var pairs map[string]interface{} + lrec := containers.LrecAlloc() - // oh no -- pairs are *not* order-preserved. :( - // https://github.com/golang/go/issues/27179 - - err = jsonDecoder.Decode(&pairs) + var om *ordered.OrderedMap = ordered.NewOrderedMap() + err = jsonDecoder.Decode(om) if err != nil { echan <- err return } - lrec := containers.LrecAlloc() + // Use an iterator func to loop over all key-value pairs. It is OK to call Set + // append-modify new key-value pairs, but not safe to call Delete during + // iteration. + iter := om.EntriesIter() + for { + pair, ok := iter() + if !ok { + break + } - //fmt.Printf("%v\n", pairs) - for key, value := range pairs { - //fmt.Printf("-- key: %v\n", key) - //fmt.Printf("-- value: %v\n", value) - foo := key // copy - // TODO: - // * handle int values - // * handle float values - // * handle object values + key := pair.Key // copy + value := pair.Value + // TODO: handle object values //fmt.Println("value is a ", reflect.TypeOf(value)) // xxx make helper functions sval, ok := value.(string) if ok { - lrec.Put(&foo, &sval) + lrec.Put(&key, &sval) } else { nval, ok := value.(float64) if ok { sval = strconv.FormatFloat(nval, 'g', -1, 64) - lrec.Put(&foo, &sval) + lrec.Put(&key, &sval) } } } - inrecs <- lrec }