diff --git a/go/check b/go/check index 6ceeb239a..b2b9bc50a 100755 --- a/go/check +++ b/go/check @@ -2,5 +2,6 @@ u/try-io > u/try-io.out u/try-chain > u/try-chain.out +u/try-verbs > u/try-verbs.out u/try-parse > u/try-parse.out u/try-cst > u/try-cst.out diff --git a/go/src/miller/cli/mlrcli_mappers.go b/go/src/miller/cli/mlrcli_mappers.go index 1a591238c..f249396c2 100644 --- a/go/src/miller/cli/mlrcli_mappers.go +++ b/go/src/miller/cli/mlrcli_mappers.go @@ -12,6 +12,7 @@ import ( // ---------------------------------------------------------------- var MAPPER_LOOKUP_TABLE = []mapping.MapperSetup{ mappers.CatSetup, + mappers.CutSetup, mappers.NothingSetup, mappers.PutSetup, mappers.TacSetup, diff --git a/go/src/miller/cli/mlrcli_parse.go b/go/src/miller/cli/mlrcli_parse.go index 72eadd0e1..ac35388b6 100644 --- a/go/src/miller/cli/mlrcli_parse.go +++ b/go/src/miller/cli/mlrcli_parse.go @@ -466,16 +466,16 @@ func parseReaderOptions(args []string, argc int, pargi *int, readerOptions *clit // readerOptions.allow_repeat_ifs = true; // argi += 1; // - // } else if args[argi] == "--mmap" { - // // No-op as of 5.6.3 (mmap is being abandoned) but don't break - // // the command-line user experience. - // argi += 1; - // - // } else if args[argi] == "--no-mmap" { - // // No-op as of 5.6.3 (mmap is being abandoned) but don't break - // // the command-line user experience. - // argi += 1; - // + } else if args[argi] == "--mmap" { + // No-op as of 5.6.3 (mmap is being abandoned) but don't break + // the command-line user experience. + argi += 1 + + } else if args[argi] == "--no-mmap" { + // No-op as of 5.6.3 (mmap is being abandoned) but don't break + // the command-line user experience. + argi += 1 + // } else if args[argi] == "--prepipe" { // checkArgCount(args, argi, argc, 2); // readerOptions.prepipe = args[argi+1]; diff --git a/go/src/miller/containers/lrec.go b/go/src/miller/containers/lrec.go index ebe59b407..ae02f2bc2 100644 --- a/go/src/miller/containers/lrec.go +++ b/go/src/miller/containers/lrec.go @@ -67,7 +67,7 @@ type lrecEntry struct { } // ---------------------------------------------------------------- -func LrecAlloc() *Lrec { +func NewLrec() *Lrec { return &Lrec{ 0, nil, @@ -76,7 +76,10 @@ func LrecAlloc() *Lrec { } // ---------------------------------------------------------------- -func (this *Lrec) Print(file *os.File) { +func (this *Lrec) Print() { + this.Fprint(os.Stdout) +} +func (this *Lrec) Fprint(file *os.File) { var buffer bytes.Buffer // 5x faster than fmt.Print() separately for pe := this.Head; pe != nil; pe = pe.Next { buffer.WriteString(*pe.Key) @@ -101,6 +104,11 @@ func lrecEntryAlloc(key *string, value *lib.Mlrval) *lrecEntry { } } +// ---------------------------------------------------------------- +func (this *Lrec) Has(key *string) bool { + return this.findEntry(key) != nil +} + // ---------------------------------------------------------------- func (this *Lrec) findEntry(key *string) *lrecEntry { for pe := this.Head; pe != nil; pe = pe.Next { @@ -171,14 +179,49 @@ func (this *Lrec) Clear() { this.Tail = nil } +// ---------------------------------------------------------------- func (this *Lrec) Copy() *Lrec { - that := LrecAlloc() + that := NewLrec() for pe := this.Head; pe != nil; pe = pe.Next { that.Put(pe.Key, pe.Value) } return that } +// ---------------------------------------------------------------- +// Returns true if it was found and removed +func (this *Lrec) Remove(key *string) bool { + pe := this.findEntry(key) + if pe == nil { + return false + } else { + this.unlink(pe) + return true + } +} + +// ---------------------------------------------------------------- +func (this *Lrec) unlink(pe *lrecEntry) { + if pe == this.Head { + if pe == this.Tail { + this.Head = nil + this.Tail = nil + } else { + this.Head = pe.Next + pe.Next.Prev = nil + } + } else { + pe.Prev.Next = pe.Next + if pe == this.Tail { + this.Tail = pe.Prev + } else { + pe.Next.Prev = pe.Prev + } + } + this.FieldCount-- +} + +// ---------------------------------------------------------------- //void lrec_prepend(Lrec* prec, char* key, char* value, char free_flags) { // lrecEntry* pe = lrec_find_entry(prec, key); // @@ -501,27 +544,6 @@ func (this *Lrec) Copy() *Lrec { // pe->free_flags &= ~FREE_ENTRY_VALUE; //} -//// ---------------------------------------------------------------- -//void lrec_unlink(Lrec* prec, lrecEntry* pe) { -// if (pe == prec->Head) { -// if (pe == prec->Tail) { -// prec->Head = NULL; -// prec->Tail = NULL; -// } else { -// prec->Head = pe->Next; -// pe->Next->Prev = NULL; -// } -// } else { -// pe->Prev->Next = pe->Next; -// if (pe == prec->Tail) { -// prec->Tail = pe->Prev; -// } else { -// pe->Next->Prev = pe->Prev; -// } -// } -// prec->field_count--; -//} - //// ---------------------------------------------------------------- //static void lrec_link_at_head(Lrec* prec, lrecEntry* pe) { // diff --git a/go/src/miller/dsl/cst/collections.go b/go/src/miller/dsl/cst/collections.go index ead5e0654..735135950 100644 --- a/go/src/miller/dsl/cst/collections.go +++ b/go/src/miller/dsl/cst/collections.go @@ -115,8 +115,7 @@ func BuildMapLiteralNode( // TODO - return &MapLiteralNode{ - }, nil + return &MapLiteralNode{}, nil } func (this *MapLiteralNode) Evaluate(state *State) lib.Mlrval { diff --git a/go/src/miller/input/record_reader_csv.go b/go/src/miller/input/record_reader_csv.go index e5dae47c8..bbb50c406 100644 --- a/go/src/miller/input/record_reader_csv.go +++ b/go/src/miller/input/record_reader_csv.go @@ -87,7 +87,7 @@ func (this *RecordReaderCSV) processHandle( return } - lrec := containers.LrecAlloc() + lrec := containers.NewLrec() // TODO: check for length mismatches n := len(header) diff --git a/go/src/miller/input/record_reader_dkvp.go b/go/src/miller/input/record_reader_dkvp.go index e36abc9d5..866d7280a 100644 --- a/go/src/miller/input/record_reader_dkvp.go +++ b/go/src/miller/input/record_reader_dkvp.go @@ -86,7 +86,7 @@ func lrecFromDKVPLine( ifs *string, ips *string, ) *containers.Lrec { - lrec := containers.LrecAlloc() + lrec := containers.NewLrec() pairs := strings.Split(*line, *ifs) for _, pair := range pairs { kv := strings.SplitN(pair, *ips, 2) diff --git a/go/src/miller/input/record_reader_json.go b/go/src/miller/input/record_reader_json.go index 40fb10b67..91f636c47 100644 --- a/go/src/miller/input/record_reader_json.go +++ b/go/src/miller/input/record_reader_json.go @@ -70,7 +70,7 @@ func (this *RecordReaderJSON) processHandle( for jsonDecoder.More() { - lrec := containers.LrecAlloc() + lrec := containers.NewLrec() var om *ordered.OrderedMap = ordered.NewOrderedMap() err := jsonDecoder.Decode(om) diff --git a/go/src/miller/input/record_reader_nidx.go b/go/src/miller/input/record_reader_nidx.go index 3b8437762..9de69eeeb 100644 --- a/go/src/miller/input/record_reader_nidx.go +++ b/go/src/miller/input/record_reader_nidx.go @@ -88,7 +88,7 @@ func (this *RecordReaderNIDX) processHandle( func lrecFromNIDXLine( line *string, ) *containers.Lrec { - lrec := containers.LrecAlloc() + lrec := containers.NewLrec() values := strings.Split(*line, " ") // TODO: repifs ... var i int64 = 0 for _, value := range values { diff --git a/perf/Makefile.no-autoconfig b/perf/Makefile.no-autoconfig index 053c9582b..377ecd4a9 100644 --- a/perf/Makefile.no-autoconfig +++ b/perf/Makefile.no-autoconfig @@ -1,4 +1,5 @@ -all: catc catc0 catm catd catrust catgo catnim cutd cutgo cutnim +all: catc catc0 catm catrust catgo catnim cutgo cutnim +# catd cutd allc: catc catc0 catm catc: catc.c @@ -7,16 +8,16 @@ catc0: catc0.c gcc -Wall catc0.c -o catc0 catm: catm.c gcc -Wall catm.c -o catm -catd: catd.d - dmd -O catd.d +#catd: catd.d +# dmd -O catd.d catrust: catrust.rs rustc catrust.rs catgo: catgo.go go build catgo.go catnim: catnim.nim nim compile catnim.nim -cutd: cutd.d - dmd -O cutd.d +#cutd: cutd.d +# dmd -O cutd.d cutgo: cutgo.go go build cutgo.go cutnim: cutnim.nim diff --git a/perf/catgo.go b/perf/catgo.go index 7b01385d1..6521c3a99 100644 --- a/perf/catgo.go +++ b/perf/catgo.go @@ -10,14 +10,13 @@ import ( // ---------------------------------------------------------------- func main() { args := os.Args[1:] - includeFields := []string {"a", "x"}; ok := true if len(args) == 0 { - ok = handle("-", includeFields) && ok + ok = handle("-") && ok } else { for _, arg := range args { - ok = handle(arg, includeFields) && ok + ok = handle(arg) && ok } } if ok { @@ -28,7 +27,7 @@ func main() { } // ---------------------------------------------------------------- -func handle(fileName string, includeFields []string) (ok bool) { +func handle(fileName string) (ok bool) { inputStream := os.Stdin if fileName != "-" { var err error diff --git a/perf/catgo2.go b/perf/catgo2.go index 5c4b3668d..db3f8f7f8 100644 --- a/perf/catgo2.go +++ b/perf/catgo2.go @@ -10,14 +10,13 @@ import ( // ---------------------------------------------------------------- func main() { args := os.Args[1:] - includeFields := []string {"a", "x"}; ok := true if len(args) == 0 { - ok = handle("-", includeFields) && ok + ok = handle("-") && ok } else { for _, arg := range args { - ok = handle(arg, includeFields) && ok + ok = handle(arg) && ok } } if ok { @@ -28,7 +27,7 @@ func main() { } // ---------------------------------------------------------------- -func handle(fileName string, includeFields []string) (ok bool) { +func handle(fileName string) (ok bool) { inputStream := os.Stdin if fileName != "-" { var err error diff --git a/perf/cutgo.go b/perf/cutgo.go index 56ec3be75..b5985c8db 100644 --- a/perf/cutgo.go +++ b/perf/cutgo.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "io" "log" "os" @@ -10,14 +11,14 @@ import ( // ---------------------------------------------------------------- func main() { - args := os.Args[1:] - includeFields := []string {"a", "x"}; + includeFields := strings.Split(os.Args[1], ",") + filenames := os.Args[2:] ok := true - if len(args) == 0 { + if len(filenames) == 0 { ok = handle("-", includeFields) && ok } else { - for _, arg := range args { + for _, arg := range filenames { ok = handle(arg, includeFields) && ok } } @@ -40,7 +41,6 @@ func handle(fileName string, includeFields []string) (ok bool) { } reader := bufio.NewReader(inputStream) - writer := bufio.NewWriter(os.Stdout) eof := false for !eof { @@ -56,59 +56,148 @@ func handle(fileName string, includeFields []string) (ok bool) { return false } else { - // 0.030s + // continue + // $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null + // TIME IN SECONDS 0.228 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.226 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.222 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.221 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.228 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.226 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.225 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.227 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.222 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 0.223 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // + // avg 0.2248 + // cumu 4% // Line to map mymap := make(map[string]string) - fields := strings.Split(line, ","); - for _, field := range(fields) { + fields := strings.Split(line, ",") + for _, field := range fields { kvps := strings.SplitN(field, "=", 2) mymap[kvps[0]] = kvps[1] } - // 0.220s - // delta 0.190s - // 27% + + // continue + // $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null + // TIME IN SECONDS 3.055 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.837 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.905 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.817 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.766 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.810 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.748 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.744 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.765 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.722 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // + // avg 2.8169 + // cumu 58% + // delta 54% // Map-to-map transform newmap := make(map[string]string) - for _, includeField := range(includeFields) { + for _, includeField := range includeFields { value, present := mymap[includeField] if present { newmap[includeField] = value } } - // 0.280s - // delta 0.060s - // 9% + + // continue + // $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null + // TIME IN SECONDS 3.101 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.028 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.992 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.005 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.991 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.986 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.992 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.992 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.989 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 2.991 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // + // avg 3.0067 + // cumu 62% + // delta 4% // Map to string - outs := make([]string, len(newmap)) + + // Faster to assemble in memory with single fmt.Println at the end, + // than multiple fmt.Print through the fields. + var buffer bytes.Buffer i := 0 - for k, v := range(newmap) { - outs[i] = k + "=" + v + for k, v := range newmap { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(k) + buffer.WriteString("=") + buffer.WriteString(v) i++ } - // 0.320s - // delta 0.040s - // 6% - out := strings.Join(outs, ",") - // 0.330s - // delta 0.010s - // 2% + // continue + // $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null + // TIME IN SECONDS 3.821 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.443 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.491 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.421 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.412 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.470 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.438 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.555 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.637 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.538 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // + // avg 3.5226 + // cumu 73% + // delta 11% - // Write to output stream - //fmt.Println("") - writer.WriteString(out) - // delta 0.390s - // 56% + buffer.WriteString("\n") + + // continue + // $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null + // TIME IN SECONDS 3.769 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.539 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.539 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.486 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.518 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.492 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.506 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.486 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.495 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 3.560 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // + // avg 3.539 + // cumu 73% + // delta 0% + + os.Stdout.WriteString(buffer.String()) + + // $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null + // TIME IN SECONDS 4.976 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.727 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.760 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.747 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.826 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.849 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.841 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.747 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.841 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // TIME IN SECONDS 4.765 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp + // + // avg 4.8079 + // cumu 100% + // delta 27% } } if fileName != "-" { inputStream.Close() } - writer.Flush() return true } diff --git a/perf/do-timings.rb b/perf/do-timings.rb index fe650b3ab..f5b20357b 100755 --- a/perf/do-timings.rb +++ b/perf/do-timings.rb @@ -28,10 +28,13 @@ run("catc0", "catc0 ../data/big.dkvp > /dev/null") run("catc", "catc < ../data/big.dkvp > /dev/null") run("catc", "catc ../data/big.dkvp > /dev/null") run("catm", "catm ../data/big.dkvp > /dev/null") +run("catgo", "catgo ../data/big.dkvp > /dev/null") run("catawk", "awk -F, '{print}' ../data/big.dkvp > /dev/null") run("catmawk", "mawk -F, '{print}' ../data/big.dkvp > /dev/null") -run("CATMLR", "mlr --no-mmap cat ../data/big.dkvp > /dev/null") -run("CATMLRM", "mlr --mmap cat ../data/big.dkvp > /dev/null") +run("CATMLR", "../c/mlr --no-mmap cat ../data/big.dkvp > /dev/null") +run("CATMLRM", "../c/mlr --mmap cat ../data/big.dkvp > /dev/null") +run("CATMLR", "../go/mlr --no-mmap cat ../data/big.dkvp > /dev/null") +run("CATMLRM", "../go/mlr --mmap cat ../data/big.dkvp > /dev/null") puts run("(catv)", "cat < ../data/big.csv > /dev/null") @@ -40,12 +43,13 @@ run("catc0v", "catc0 ../data/big.csv > /dev/null") run("catcv", "catc < ../data/big.csv > /dev/null") run("catcv", "catc ../data/big.csv > /dev/null") run("catmv", "catm ../data/big.csv > /dev/null") +run("catgov", "catgo ../data/big.csv > /dev/null") run("catawkv", "awk -F, '{print}' ../data/big.csv > /dev/null") run("catmawkv", "mawk -F, '{print}' ../data/big.csv > /dev/null") -run("CATMLRV", "mlr --no-mmap --csv --rs lf cat ../data/big.csv > /dev/null") -run("CATMLRVM", "mlr --mmap --csv --rs lf cat ../data/big.csv > /dev/null") -run("CATMLRV", "mlr --no-mmap --csvlite cat ../data/big.csv > /dev/null") -run("CATMLRVM", "mlr --mmap --csvlite cat ../data/big.csv > /dev/null") +run("CATMLRV", "../c/mlr --no-mmap --csv --rs lf cat ../data/big.csv > /dev/null") +run("CATMLRVM", "../c/mlr --mmap --csv --rs lf cat ../data/big.csv > /dev/null") +run("CATMLRV", "../c/mlr --no-mmap --csvlite cat ../data/big.csv > /dev/null") +run("CATMLRVM", "../c/mlr --mmap --csvlite cat ../data/big.csv > /dev/null") puts puts