mirror of
https://github.com/johnkerl/miller.git
synced 2026-01-23 10:15:36 +00:00
json reader with ordered maps
This commit is contained in:
John Kerl
parent
f1bfc32ad8
commit
da6830a7a9
4 changed files with 756 additions and 19 deletions
17
go/src/deps/ordered/README.md
Normal file
17
go/src/deps/ordered/README.md
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
JSON object parsing with preserving keys order
|
||||
=============================================
|
||||
|
||||
|
||||
|
||||
Refers
|
||||
|
||||
1. JSON and Go https://blog.golang.org/json-and-go
|
||||
2. Go-Ordered-JSON https://github.com/virtuald/go-ordered-json
|
||||
from this thread [*Preserving key order in encoding/json*](https://groups.google.com/forum/#!topic/golang-dev/zBQwhm3VfvU)
|
||||
and the [*Abandoned 7930: encoding/json: Optionally preserve the key order of JSON objects*](https://go-review.googlesource.com/c/go/+/7930)
|
||||
3. Python OrderedDict https://github.com/python/cpython/blob/2.7/Lib/collections.py#L38
|
||||
the Python's OrderedDict uses a double linked list internally, maintain a consistent public interface with `dict`
|
||||
|
||||
Disclaimer:
|
||||
|
||||
same as Go's default [map](https://blog.golang.org/go-maps-in-action), this OrderedMap is not safe for concurrent use, if need atomic access, may use a sync.Mutex to synchronize.
|
||||
267
go/src/deps/ordered/ordered.go
Normal file
267
go/src/deps/ordered/ordered.go
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
// Package ordered provided a type OrderedMap for use in JSON handling
|
||||
// although JSON spec says the keys order of an object should not matter
|
||||
// but sometimes when working with particular third-party proprietary code
|
||||
// which has incorrect using the keys order, we have to maintain the object keys
|
||||
// in the same order of incoming JSON object, this package is useful for these cases.
|
||||
//
|
||||
// Disclaimer:
|
||||
// same as Go's default [map](https://blog.golang.org/go-maps-in-action),
|
||||
// this OrderedMap is not safe for concurrent use, if need atomic access, may use a sync.Mutex to synchronize.
|
||||
package ordered
|
||||
|
||||
// Refers
|
||||
// JSON and Go https://blog.golang.org/json-and-go
|
||||
// Go-Ordered-JSON https://github.com/virtuald/go-ordered-json
|
||||
// Python OrderedDict https://github.com/python/cpython/blob/2.7/Lib/collections.py#L38
|
||||
// port OrderedDict https://github.com/cevaris/ordered_map
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// the key-value pair type, for initializing from a list of key-value pairs, or for looping entries in the same order
|
||||
type KVPair struct {
|
||||
Key string
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
type m map[string]interface{}
|
||||
|
||||
// the OrderedMap type, has similar operations as the default map, but maintained
|
||||
// the keys order of inserted; similar to map, all single key operations (Get/Set/Delete) runs at O(1).
|
||||
type OrderedMap struct {
|
||||
m
|
||||
l *list.List
|
||||
keys map[string]*list.Element // the double linked list for delete and lookup to be O(1)
|
||||
}
|
||||
|
||||
// Create a new OrderedMap
|
||||
func NewOrderedMap() *OrderedMap {
|
||||
return &OrderedMap{
|
||||
m: make(map[string]interface{}),
|
||||
l: list.New(),
|
||||
keys: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new OrderedMap and populate from a list of key-value pairs
|
||||
func NewOrderedMapFromKVPairs(pairs []*KVPair) *OrderedMap {
|
||||
om := NewOrderedMap()
|
||||
for _, pair := range pairs {
|
||||
om.Set(pair.Key, pair.Value)
|
||||
}
|
||||
return om
|
||||
}
|
||||
|
||||
// return all keys
|
||||
// func (om *OrderedMap) Keys() []string { return om.keys }
|
||||
|
||||
// set value for particular key, this will remember the order of keys inserted
|
||||
// but if the key already exists, the order is not updated.
|
||||
func (om *OrderedMap) Set(key string, value interface{}) {
|
||||
if _, ok := om.m[key]; !ok {
|
||||
om.keys[key] = om.l.PushBack(key)
|
||||
}
|
||||
om.m[key] = value
|
||||
}
|
||||
|
||||
// Check if value exists
|
||||
func (om *OrderedMap) Has(key string) bool {
|
||||
_, ok := om.m[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Get value for particular key, or nil if not exist; but don't rely on nil for non-exist; should check by Has or GetValue
|
||||
func (om *OrderedMap) Get(key string) interface{} {
|
||||
return om.m[key]
|
||||
}
|
||||
|
||||
// Get value and exists together
|
||||
func (om *OrderedMap) GetValue(key string) (value interface{}, ok bool) {
|
||||
value, ok = om.m[key]
|
||||
return
|
||||
}
|
||||
|
||||
// deletes the element with the specified key (m[key]) from the map. If there is no such element, this is a no-op.
|
||||
func (om *OrderedMap) Delete(key string) (value interface{}, ok bool) {
|
||||
value, ok = om.m[key]
|
||||
if ok {
|
||||
om.l.Remove(om.keys[key])
|
||||
delete(om.keys, key)
|
||||
delete(om.m, key)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Iterate all key/value pairs in the same order of object constructed
|
||||
func (om *OrderedMap) EntriesIter() func() (*KVPair, bool) {
|
||||
e := om.l.Front()
|
||||
return func() (*KVPair, bool) {
|
||||
if e != nil {
|
||||
key := e.Value.(string)
|
||||
e = e.Next()
|
||||
return &KVPair{key, om.m[key]}, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// Iterate all key/value pairs in the reverse order of object constructed
|
||||
func (om *OrderedMap) EntriesReverseIter() func() (*KVPair, bool) {
|
||||
e := om.l.Back()
|
||||
return func() (*KVPair, bool) {
|
||||
if e != nil {
|
||||
key := e.Value.(string)
|
||||
e = e.Prev()
|
||||
return &KVPair{key, om.m[key]}, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// this implements type json.Marshaler interface, so can be called in json.Marshal(om)
|
||||
func (om *OrderedMap) MarshalJSON() (res []byte, err error) {
|
||||
res = append(res, '{')
|
||||
front, back := om.l.Front(), om.l.Back()
|
||||
for e := front; e != nil; e = e.Next() {
|
||||
k := e.Value.(string)
|
||||
res = append(res, fmt.Sprintf("%q:", k)...)
|
||||
var b []byte
|
||||
b, err = json.Marshal(om.m[k])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
res = append(res, b...)
|
||||
if e != back {
|
||||
res = append(res, ',')
|
||||
}
|
||||
}
|
||||
res = append(res, '}')
|
||||
// fmt.Printf("marshalled: %v: %#v\n", res, res)
|
||||
return
|
||||
}
|
||||
|
||||
// this implements type json.Unmarshaler interface, so can be called in json.Unmarshal(data, om)
|
||||
func (om *OrderedMap) UnmarshalJSON(data []byte) error {
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
dec.UseNumber()
|
||||
|
||||
// must open with a delim token '{'
|
||||
t, err := dec.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if delim, ok := t.(json.Delim); !ok || delim != '{' {
|
||||
return fmt.Errorf("expect JSON object open with '{'")
|
||||
}
|
||||
|
||||
err = om.parseobject(dec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t, err = dec.Token()
|
||||
if err != io.EOF {
|
||||
return fmt.Errorf("expect end of JSON object but got more token: %T: %v or err: %v", t, t, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (om *OrderedMap) parseobject(dec *json.Decoder) (err error) {
|
||||
var t json.Token
|
||||
for dec.More() {
|
||||
t, err = dec.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
key, ok := t.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("expecting JSON key should be always a string: %T: %v", t, t)
|
||||
}
|
||||
|
||||
t, err = dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var value interface{}
|
||||
value, err = handledelim(t, dec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// om.keys = append(om.keys, key)
|
||||
om.keys[key] = om.l.PushBack(key)
|
||||
om.m[key] = value
|
||||
}
|
||||
|
||||
t, err = dec.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if delim, ok := t.(json.Delim); !ok || delim != '}' {
|
||||
return fmt.Errorf("expect JSON object close with '}'")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parsearray(dec *json.Decoder) (arr []interface{}, err error) {
|
||||
var t json.Token
|
||||
arr = make([]interface{}, 0)
|
||||
for dec.More() {
|
||||
t, err = dec.Token()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var value interface{}
|
||||
value, err = handledelim(t, dec)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
arr = append(arr, value)
|
||||
}
|
||||
t, err = dec.Token()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if delim, ok := t.(json.Delim); !ok || delim != ']' {
|
||||
err = fmt.Errorf("expect JSON array close with ']'")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func handledelim(t json.Token, dec *json.Decoder) (res interface{}, err error) {
|
||||
if delim, ok := t.(json.Delim); ok {
|
||||
switch delim {
|
||||
case '{':
|
||||
om2 := NewOrderedMap()
|
||||
err = om2.parseobject(dec)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return om2, nil
|
||||
case '[':
|
||||
var value []interface{}
|
||||
value, err = parsearray(dec)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return value, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected delimiter: %q", delim)
|
||||
}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
446
go/src/deps/ordered/ordered_test.go
Normal file
446
go/src/deps/ordered/ordered_test.go
Normal file
|
|
@ -0,0 +1,446 @@
|
|||
package ordered
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMarshalOrderedMap(t *testing.T) {
|
||||
om := NewOrderedMap()
|
||||
om.Set("a", 34)
|
||||
om.Set("b", []int{3, 4, 5})
|
||||
b, err := json.Marshal(om)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal OrderedMap: %v", err)
|
||||
}
|
||||
// fmt.Printf("%q\n", b)
|
||||
const expected = "{\"a\":34,\"b\":[3,4,5]}"
|
||||
if !bytes.Equal(b, []byte(expected)) {
|
||||
t.Errorf("Marshal OrderedMap: %q not equal to expected %q", b, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func ExampleOrderedMap_UnmarshalJSON() {
|
||||
const jsonStream = `{
|
||||
"country" : "United States",
|
||||
"countryCode" : "US",
|
||||
"region" : "CA",
|
||||
"regionName" : "California",
|
||||
"city" : "Mountain View",
|
||||
"zip" : "94043",
|
||||
"lat" : 37.4192,
|
||||
"lon" : -122.0574,
|
||||
"timezone" : "America/Los_Angeles",
|
||||
"isp" : "Google Cloud",
|
||||
"org" : "Google Cloud",
|
||||
"as" : "AS15169 Google Inc.",
|
||||
"mobile" : true,
|
||||
"proxy" : false,
|
||||
"query" : "35.192.xx.xxx"
|
||||
}`
|
||||
|
||||
// compare with if using a regular generic map, the unmarshalled result
|
||||
// is a map with unpredictable order of keys
|
||||
var m map[string]interface{}
|
||||
err := json.Unmarshal([]byte(jsonStream), &m)
|
||||
if err != nil {
|
||||
fmt.Println("error:", err)
|
||||
}
|
||||
for key := range m {
|
||||
// fmt.Printf("%-12s: %v\n", key, m[key])
|
||||
_ = key
|
||||
}
|
||||
|
||||
// use the OrderedMap to Unmarshal from JSON object
|
||||
var om *OrderedMap = NewOrderedMap()
|
||||
err = json.Unmarshal([]byte(jsonStream), om)
|
||||
if err != nil {
|
||||
fmt.Println("error:", err)
|
||||
}
|
||||
|
||||
// use an iterator func to loop over all key-value pairs,
|
||||
// it is ok to call Set append-modify new key-value pairs,
|
||||
// but not safe to call Delete during iteration.
|
||||
iter := om.EntriesIter()
|
||||
for {
|
||||
pair, ok := iter()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
fmt.Printf("%-12s: %v\n", pair.Key, pair.Value)
|
||||
if pair.Key == "city" {
|
||||
om.Set("mobile", false)
|
||||
om.Set("extra", 42)
|
||||
}
|
||||
}
|
||||
|
||||
// Output:
|
||||
// country : United States
|
||||
// countryCode : US
|
||||
// region : CA
|
||||
// regionName : California
|
||||
// city : Mountain View
|
||||
// zip : 94043
|
||||
// lat : 37.4192
|
||||
// lon : -122.0574
|
||||
// timezone : America/Los_Angeles
|
||||
// isp : Google Cloud
|
||||
// org : Google Cloud
|
||||
// as : AS15169 Google Inc.
|
||||
// mobile : false
|
||||
// proxy : false
|
||||
// query : 35.192.xx.xxx
|
||||
// extra : 42
|
||||
}
|
||||
|
||||
func TestUnmarshalOrderedMapFromInvalid(t *testing.T) {
|
||||
om := NewOrderedMap()
|
||||
|
||||
om.Set("m", math.NaN())
|
||||
b, err := json.Marshal(om)
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", b, err)
|
||||
}
|
||||
// fmt.Println(om, b, err)
|
||||
om.Delete("m")
|
||||
|
||||
err = json.Unmarshal([]byte("[]"), om)
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error")
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte("["), om)
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
|
||||
err = om.UnmarshalJSON([]byte(nil))
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
|
||||
err = om.UnmarshalJSON([]byte("{}3"))
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
|
||||
err = om.UnmarshalJSON([]byte("{"))
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
|
||||
err = om.UnmarshalJSON([]byte("{]"))
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
|
||||
err = om.UnmarshalJSON([]byte(`{"a": 3, "b": [{`))
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
|
||||
err = om.UnmarshalJSON([]byte(`{"a": 3, "b": [}`))
|
||||
if err == nil {
|
||||
t.Fatal("Unmarshal OrderedMap: expecting error:", om)
|
||||
}
|
||||
// fmt.Println("error:", om, err)
|
||||
}
|
||||
|
||||
func TestUnmarshalOrderedMap(t *testing.T) {
|
||||
var (
|
||||
data = []byte(`{"as":"AS15169 Google Inc.","city":"Mountain View","country":"United States","countryCode":"US","isp":"Google Cloud","lat":37.4192,"lon":-122.0574,"org":"Google Cloud","query":"35.192.25.53","region":"CA","regionName":"California","status":"success","timezone":"America/Los_Angeles","zip":"94043"}`)
|
||||
pairs = []*KVPair{
|
||||
{"as", "AS15169 Google Inc."},
|
||||
{"city", "Mountain View"},
|
||||
{"country", "United States"},
|
||||
{"countryCode", "US"},
|
||||
{"isp", "Google Cloud"},
|
||||
{"lat", 37.4192},
|
||||
{"lon", -122.0574},
|
||||
{"org", "Google Cloud"},
|
||||
{"query", "35.192.25.53"},
|
||||
{"region", "CA"},
|
||||
{"regionName", "California"},
|
||||
{"status", "success"},
|
||||
{"timezone", "America/Los_Angeles"},
|
||||
{"zip", "94043"},
|
||||
}
|
||||
obj = NewOrderedMapFromKVPairs(pairs)
|
||||
)
|
||||
|
||||
om := NewOrderedMap()
|
||||
err := json.Unmarshal(data, om)
|
||||
if err != nil {
|
||||
t.Fatalf("Unmarshal OrderedMap: %v", err)
|
||||
}
|
||||
|
||||
// fix number type for deepequal test
|
||||
for _, key := range []string{"lat", "lon"} {
|
||||
numf, _ := om.Get(key).(json.Number).Float64()
|
||||
om.Set(key, numf)
|
||||
}
|
||||
|
||||
// check by Has and GetValue
|
||||
for _, kv := range pairs {
|
||||
if !om.Has(kv.Key) {
|
||||
t.Fatalf("expect key %q exists in Unmarshaled OrderedMap")
|
||||
}
|
||||
value, ok := om.GetValue(kv.Key)
|
||||
if !ok || value != kv.Value {
|
||||
t.Fatalf("expect for key %q: the value %v should equal to %v, in Unmarshaled OrderedMap", kv.Key, value, kv.Value)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.MarshalIndent(om, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("Unmarshal OrderedMap: %v", err)
|
||||
}
|
||||
const expected = `{
|
||||
"as": "AS15169 Google Inc.",
|
||||
"city": "Mountain View",
|
||||
"country": "United States",
|
||||
"countryCode": "US",
|
||||
"isp": "Google Cloud",
|
||||
"lat": 37.4192,
|
||||
"lon": -122.0574,
|
||||
"org": "Google Cloud",
|
||||
"query": "35.192.25.53",
|
||||
"region": "CA",
|
||||
"regionName": "California",
|
||||
"status": "success",
|
||||
"timezone": "America/Los_Angeles",
|
||||
"zip": "94043"
|
||||
}`
|
||||
if !bytes.Equal(b, []byte(expected)) {
|
||||
t.Fatalf("Unmarshal OrderedMap marshal indent from %#v not equal to expected: %q\n", om, expected)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(om, obj) {
|
||||
t.Fatalf("Unmarshal OrderedMap not deeply equal: %#v %#v", om, obj)
|
||||
}
|
||||
|
||||
val, ok := om.Delete("org")
|
||||
if !ok {
|
||||
t.Fatalf("org should exist")
|
||||
}
|
||||
om.Set("org", val)
|
||||
b, err = json.MarshalIndent(om, "", " ")
|
||||
// fmt.Println("after delete", om, string(b), err)
|
||||
if err != nil {
|
||||
t.Fatalf("Unmarshal OrderedMap: %v", err)
|
||||
}
|
||||
const expected2 = `{
|
||||
"as": "AS15169 Google Inc.",
|
||||
"city": "Mountain View",
|
||||
"country": "United States",
|
||||
"countryCode": "US",
|
||||
"isp": "Google Cloud",
|
||||
"lat": 37.4192,
|
||||
"lon": -122.0574,
|
||||
"query": "35.192.25.53",
|
||||
"region": "CA",
|
||||
"regionName": "California",
|
||||
"status": "success",
|
||||
"timezone": "America/Los_Angeles",
|
||||
"zip": "94043",
|
||||
"org": "Google Cloud"
|
||||
}`
|
||||
if !bytes.Equal(b, []byte(expected2)) {
|
||||
t.Fatalf("Unmarshal OrderedMap marshal indent from %#v not equal to expected: %s\n", om, expected2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnmarshalNestedOrderedMap(t *testing.T) {
|
||||
var (
|
||||
data = []byte(`{"a": true, "b": [3, 4, { "b": "3", "d": [] }]}`)
|
||||
obj = NewOrderedMapFromKVPairs([]*KVPair{
|
||||
{"a", true},
|
||||
{"b", []interface{}{3, 4, NewOrderedMapFromKVPairs([]*KVPair{
|
||||
{"b", "3"},
|
||||
{"d", []interface{}{}},
|
||||
})}},
|
||||
})
|
||||
)
|
||||
|
||||
om := NewOrderedMap()
|
||||
err := json.Unmarshal(data, om)
|
||||
if err != nil {
|
||||
t.Fatalf("Unmarshal OrderedMap: %v", err)
|
||||
}
|
||||
|
||||
// b, err := json.MarshalIndent(om, "", " ")
|
||||
// fmt.Println(om, string(b), err, obj)
|
||||
|
||||
// fix number type for deepequal test
|
||||
elearr := om.Get("b").([]interface{})
|
||||
for i, v := range elearr {
|
||||
if num, ok := v.(json.Number); ok {
|
||||
numi, _ := num.Int64()
|
||||
elearr[i] = int(numi)
|
||||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(om, obj) {
|
||||
t.Fatalf("Unmarshal OrderedMap not deeply equal: %#v expected %#v", om, obj)
|
||||
}
|
||||
}
|
||||
|
||||
func ExampleOrderedMap_EntriesReverseIter() {
|
||||
// initialize from a list of key-value pairs
|
||||
om := NewOrderedMapFromKVPairs([]*KVPair{
|
||||
{"country", "United States"},
|
||||
{"countryCode", "US"},
|
||||
{"region", "CA"},
|
||||
{"regionName", "California"},
|
||||
{"city", "Mountain View"},
|
||||
{"zip", "94043"},
|
||||
{"lat", 37.4192},
|
||||
{"lon", -122.0574},
|
||||
{"timezone", "America/Los_Angeles"},
|
||||
{"isp", "Google Cloud"},
|
||||
{"org", "Google Cloud"},
|
||||
{"as", "AS15169 Google Inc."},
|
||||
{"mobile", true},
|
||||
{"proxy", false},
|
||||
{"query", "35.192.xx.xxx"},
|
||||
})
|
||||
|
||||
iter := om.EntriesReverseIter()
|
||||
for {
|
||||
pair, ok := iter()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
fmt.Printf("%-12s: %v\n", pair.Key, pair.Value)
|
||||
}
|
||||
|
||||
// Output:
|
||||
// query : 35.192.xx.xxx
|
||||
// proxy : false
|
||||
// mobile : true
|
||||
// as : AS15169 Google Inc.
|
||||
// org : Google Cloud
|
||||
// isp : Google Cloud
|
||||
// timezone : America/Los_Angeles
|
||||
// lon : -122.0574
|
||||
// lat : 37.4192
|
||||
// zip : 94043
|
||||
// city : Mountain View
|
||||
// regionName : California
|
||||
// region : CA
|
||||
// countryCode : US
|
||||
// country : United States
|
||||
}
|
||||
|
||||
var unmarshalTests = []struct {
|
||||
in string
|
||||
new func() interface{}
|
||||
out interface{}
|
||||
err error
|
||||
useNumber bool
|
||||
golden bool
|
||||
}{
|
||||
{in: `true`, new: func() interface{} { return new(bool) }, out: true},
|
||||
{in: `1`, new: func() interface{} { return new(int) }, out: 1},
|
||||
{in: `1.2`, new: func() interface{} { return new(float64) }, out: 1.2},
|
||||
{in: `-5`, new: func() interface{} { return new(int16) }, out: int16(-5)},
|
||||
{in: `2`, new: func() interface{} { return new(json.Number) }, out: json.Number("2"), useNumber: true},
|
||||
{in: `2`, new: func() interface{} { return new(json.Number) }, out: json.Number("2")},
|
||||
{in: `2`, new: func() interface{} { return new(interface{}) }, out: float64(2.0)},
|
||||
{in: `2`, new: func() interface{} { return new(interface{}) }, out: json.Number("2"), useNumber: true},
|
||||
{in: `"a\u1234"`, new: func() interface{} { return new(string) }, out: "a\u1234"},
|
||||
{in: `"http:\/\/"`, new: func() interface{} { return new(string) }, out: "http://"},
|
||||
{in: `"g-clef: \uD834\uDD1E"`, new: func() interface{} { return new(string) }, out: "g-clef: \U0001D11E"},
|
||||
{in: `"invalid: \uD834x\uDD1E"`, new: func() interface{} { return new(string) }, out: "invalid: \uFFFDx\uFFFD"},
|
||||
{in: "null", new: func() interface{} { return new(interface{}) }, out: nil},
|
||||
{in: "{}", new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs([]*KVPair{})},
|
||||
{in: `{"a": 3}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs(
|
||||
[]*KVPair{{"a", json.Number("3")}})},
|
||||
{in: `{"a": 3, "b": true}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs(
|
||||
[]*KVPair{{"a", json.Number("3")}, {"b", true}})},
|
||||
{in: `{"a": 3, "b": true, "c": null}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs(
|
||||
[]*KVPair{{"a", json.Number("3")}, {"b", true}, {"c", nil}})},
|
||||
{in: `{"a": 3, "c": null, "d": []}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs(
|
||||
[]*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{}}})},
|
||||
{in: `{"a": 3, "c": null, "d": [3,4,true]}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs(
|
||||
[]*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{
|
||||
json.Number("3"), json.Number("4"), true,
|
||||
}}})},
|
||||
{in: `{"a": 3, "c": null, "d": [3,4,true, { "inner": "abc" }]}`, new: func() interface{} { return NewOrderedMap() }, out: *NewOrderedMapFromKVPairs(
|
||||
[]*KVPair{{"a", json.Number("3")}, {"c", nil}, {"d", []interface{}{
|
||||
json.Number("3"), json.Number("4"), true, NewOrderedMapFromKVPairs([]*KVPair{{"inner", "abc"}}),
|
||||
}}})},
|
||||
}
|
||||
|
||||
func TestUnmarshal(t *testing.T) {
|
||||
for i, tt := range unmarshalTests {
|
||||
in := []byte(tt.in)
|
||||
if tt.new == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// v = new(right-type)
|
||||
v := tt.new() // reflect.New(reflect.TypeOf(tt.ptr).Elem())
|
||||
dec := json.NewDecoder(bytes.NewReader(in))
|
||||
if tt.useNumber {
|
||||
dec.UseNumber()
|
||||
}
|
||||
if err := dec.Decode(v); !reflect.DeepEqual(err, tt.err) {
|
||||
t.Errorf("#%d: %v, want %v", i, err, tt.err)
|
||||
continue
|
||||
} else if err != nil {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(reflect.ValueOf(v).Elem().Interface(), tt.out) {
|
||||
t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v, tt.out)
|
||||
data, _ := json.Marshal(v)
|
||||
println(string(data))
|
||||
data, _ = json.Marshal(tt.out)
|
||||
println(string(data))
|
||||
continue
|
||||
}
|
||||
|
||||
// Check round trip also decodes correctly.
|
||||
if tt.err == nil {
|
||||
enc, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Errorf("#%d: error re-marshaling: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if tt.golden && !bytes.Equal(enc, in) {
|
||||
t.Errorf("#%d: remarshal mismatch:\nhave: %s\nwant: %s", i, enc, in)
|
||||
}
|
||||
vv := tt.new() // reflect.New(reflect.TypeOf(tt.ptr).Elem())
|
||||
dec = json.NewDecoder(bytes.NewReader(enc))
|
||||
if tt.useNumber {
|
||||
dec.UseNumber()
|
||||
}
|
||||
if err := dec.Decode(vv); err != nil {
|
||||
t.Errorf("#%d: error re-unmarshaling %#q: %v", i, enc, err)
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(v, vv) {
|
||||
t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v, vv)
|
||||
t.Errorf(" In: %q", strings.Map(noSpace, string(in)))
|
||||
t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc)))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func noSpace(c rune) rune {
|
||||
if isSpace(byte(c)) { //only used for ascii
|
||||
return -1
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func isSpace(c byte) bool {
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n'
|
||||
}
|
||||
|
|
@ -7,8 +7,12 @@ import (
|
|||
"os"
|
||||
//"reflect"
|
||||
"strconv"
|
||||
|
||||
// Miller:
|
||||
"miller/containers"
|
||||
|
||||
// Local dependencies:
|
||||
"deps/ordered"
|
||||
)
|
||||
|
||||
type RecordReaderJSON struct {
|
||||
|
|
@ -43,47 +47,50 @@ func (this *RecordReaderJSON) Read(
|
|||
// }
|
||||
// fmt.Printf("%T: %v\n", t, t)
|
||||
|
||||
// Ordered-map idea from:
|
||||
// https://gitlab.com/c0b/go-ordered-json
|
||||
// found via
|
||||
// https://github.com/golang/go/issues/27179
|
||||
|
||||
for jsonDecoder.More() {
|
||||
|
||||
// decode an array value (Message)
|
||||
var pairs map[string]interface{}
|
||||
lrec := containers.LrecAlloc()
|
||||
|
||||
// oh no -- pairs are *not* order-preserved. :(
|
||||
// https://github.com/golang/go/issues/27179
|
||||
|
||||
err = jsonDecoder.Decode(&pairs)
|
||||
var om *ordered.OrderedMap = ordered.NewOrderedMap()
|
||||
err = jsonDecoder.Decode(om)
|
||||
if err != nil {
|
||||
echan <- err
|
||||
return
|
||||
}
|
||||
|
||||
lrec := containers.LrecAlloc()
|
||||
// Use an iterator func to loop over all key-value pairs. It is OK to call Set
|
||||
// append-modify new key-value pairs, but not safe to call Delete during
|
||||
// iteration.
|
||||
iter := om.EntriesIter()
|
||||
for {
|
||||
pair, ok := iter()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
//fmt.Printf("%v\n", pairs)
|
||||
for key, value := range pairs {
|
||||
//fmt.Printf("-- key: %v\n", key)
|
||||
//fmt.Printf("-- value: %v\n", value)
|
||||
foo := key // copy
|
||||
// TODO:
|
||||
// * handle int values
|
||||
// * handle float values
|
||||
// * handle object values
|
||||
key := pair.Key // copy
|
||||
value := pair.Value
|
||||
// TODO: handle object values
|
||||
|
||||
//fmt.Println("value is a ", reflect.TypeOf(value))
|
||||
|
||||
// xxx make helper functions
|
||||
sval, ok := value.(string)
|
||||
if ok {
|
||||
lrec.Put(&foo, &sval)
|
||||
lrec.Put(&key, &sval)
|
||||
} else {
|
||||
nval, ok := value.(float64)
|
||||
if ok {
|
||||
sval = strconv.FormatFloat(nval, 'g', -1, 64)
|
||||
lrec.Put(&foo, &sval)
|
||||
lrec.Put(&key, &sval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inrecs <- lrec
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue