Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-08-04 13:33:18 +00:00
Code Issues Projects Releases Wiki Activity

mlr cut; perf tests

Browse source
This commit is contained in:
John Kerl 2020-09-04 11:19:15 -04:00
parent 9d084a5522
commit b8fc6bb34d
14 changed files with 205 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

1
go/check
View file

@ -2,5 +2,6 @@
u/try-io > u/try-io.out
u/try-chain > u/try-chain.out
u/try-verbs > u/try-verbs.out
u/try-parse > u/try-parse.out
u/try-cst > u/try-cst.out

1
go/src/miller/cli/mlrcli_mappers.go
View file

@ -12,6 +12,7 @@ import (
// ----------------------------------------------------------------
var MAPPER_LOOKUP_TABLE = []mapping.MapperSetup{
mappers.CatSetup,
mappers.CutSetup,
mappers.NothingSetup,
mappers.PutSetup,
mappers.TacSetup,

20
go/src/miller/cli/mlrcli_parse.go
View file

@ -466,16 +466,16 @@ func parseReaderOptions(args []string, argc int, pargi *int, readerOptions *clit
// readerOptions.allow_repeat_ifs = true;
// argi += 1;
//
// } else if args[argi] == "--mmap" {
// // No-op as of 5.6.3 (mmap is being abandoned) but don't break
// // the command-line user experience.
// argi += 1;
//
// } else if args[argi] == "--no-mmap" {
// // No-op as of 5.6.3 (mmap is being abandoned) but don't break
// // the command-line user experience.
// argi += 1;
//
} else if args[argi] == "--mmap" {
// No-op as of 5.6.3 (mmap is being abandoned) but don't break
// the command-line user experience.
argi += 1
} else if args[argi] == "--no-mmap" {
// No-op as of 5.6.3 (mmap is being abandoned) but don't break
// the command-line user experience.
argi += 1
// } else if args[argi] == "--prepipe" {
// checkArgCount(args, argi, argc, 2);
// readerOptions.prepipe = args[argi+1];

70
go/src/miller/containers/lrec.go
View file

@ -67,7 +67,7 @@ type lrecEntry struct {
}
// ----------------------------------------------------------------
func LrecAlloc() *Lrec {
func NewLrec() *Lrec {
return &Lrec{
0,
nil,
@ -76,7 +76,10 @@ func LrecAlloc() *Lrec {
}
// ----------------------------------------------------------------
func (this *Lrec) Print(file *os.File) {
func (this *Lrec) Print() {
this.Fprint(os.Stdout)
}
func (this *Lrec) Fprint(file *os.File) {
var buffer bytes.Buffer // 5x faster than fmt.Print() separately
for pe := this.Head; pe != nil; pe = pe.Next {
buffer.WriteString(*pe.Key)
@ -101,6 +104,11 @@ func lrecEntryAlloc(key *string, value *lib.Mlrval) *lrecEntry {
}
}
// ----------------------------------------------------------------
func (this *Lrec) Has(key *string) bool {
return this.findEntry(key) != nil
}
// ----------------------------------------------------------------
func (this *Lrec) findEntry(key *string) *lrecEntry {
for pe := this.Head; pe != nil; pe = pe.Next {
@ -171,14 +179,49 @@ func (this *Lrec) Clear() {
this.Tail = nil
}
// ----------------------------------------------------------------
func (this *Lrec) Copy() *Lrec {
that := LrecAlloc()
that := NewLrec()
for pe := this.Head; pe != nil; pe = pe.Next {
that.Put(pe.Key, pe.Value)
}
return that
}
// ----------------------------------------------------------------
// Returns true if it was found and removed
func (this *Lrec) Remove(key *string) bool {
pe := this.findEntry(key)
if pe == nil {
return false
} else {
this.unlink(pe)
return true
}
}
// ----------------------------------------------------------------
func (this *Lrec) unlink(pe *lrecEntry) {
if pe == this.Head {
if pe == this.Tail {
this.Head = nil
this.Tail = nil
} else {
this.Head = pe.Next
pe.Next.Prev = nil
}
} else {
pe.Prev.Next = pe.Next
if pe == this.Tail {
this.Tail = pe.Prev
} else {
pe.Next.Prev = pe.Prev
}
}
this.FieldCount--
}
// ----------------------------------------------------------------
//void lrec_prepend(Lrec* prec, char* key, char* value, char free_flags) {
// lrecEntry* pe = lrec_find_entry(prec, key);
//
@ -501,27 +544,6 @@ func (this *Lrec) Copy() *Lrec {
// pe->free_flags &= ~FREE_ENTRY_VALUE;
//}
//// ----------------------------------------------------------------
//void lrec_unlink(Lrec* prec, lrecEntry* pe) {
// if (pe == prec->Head) {
// if (pe == prec->Tail) {
// prec->Head = NULL;
// prec->Tail = NULL;
// } else {
// prec->Head = pe->Next;
// pe->Next->Prev = NULL;
// }
// } else {
// pe->Prev->Next = pe->Next;
// if (pe == prec->Tail) {
// prec->Tail = pe->Prev;
// } else {
// pe->Next->Prev = pe->Prev;
// }
// }
// prec->field_count--;
//}
//// ----------------------------------------------------------------
//static void lrec_link_at_head(Lrec* prec, lrecEntry* pe) {
//

3
go/src/miller/dsl/cst/collections.go
View file

@ -115,8 +115,7 @@ func BuildMapLiteralNode(
// TODO
return &MapLiteralNode{
}, nil
return &MapLiteralNode{}, nil
}
func (this *MapLiteralNode) Evaluate(state *State) lib.Mlrval {

2
go/src/miller/input/record_reader_csv.go
View file

@ -87,7 +87,7 @@ func (this *RecordReaderCSV) processHandle(
return
}
lrec := containers.LrecAlloc()
lrec := containers.NewLrec()
// TODO: check for length mismatches
n := len(header)

2
go/src/miller/input/record_reader_dkvp.go
View file

@ -86,7 +86,7 @@ func lrecFromDKVPLine(
ifs *string,
ips *string,
) *containers.Lrec {
lrec := containers.LrecAlloc()
lrec := containers.NewLrec()
pairs := strings.Split(*line, *ifs)
for _, pair := range pairs {
kv := strings.SplitN(pair, *ips, 2)

2
go/src/miller/input/record_reader_json.go
View file

@ -70,7 +70,7 @@ func (this *RecordReaderJSON) processHandle(
for jsonDecoder.More() {
lrec := containers.LrecAlloc()
lrec := containers.NewLrec()
var om *ordered.OrderedMap = ordered.NewOrderedMap()
err := jsonDecoder.Decode(om)

2
go/src/miller/input/record_reader_nidx.go
View file

@ -88,7 +88,7 @@ func (this *RecordReaderNIDX) processHandle(
func lrecFromNIDXLine(
line *string,
) *containers.Lrec {
lrec := containers.LrecAlloc()
lrec := containers.NewLrec()
values := strings.Split(*line, " ") // TODO: repifs ...
var i int64 = 0
for _, value := range values {

11
perf/Makefile.no-autoconfig
View file

@ -1,4 +1,5 @@
all: catc catc0 catm catd catrust catgo catnim cutd cutgo cutnim
all: catc catc0 catm catrust catgo catnim cutgo cutnim
# catd cutd
allc: catc catc0 catm
catc: catc.c
@ -7,16 +8,16 @@ catc0: catc0.c
gcc -Wall catc0.c -o catc0
catm: catm.c
gcc -Wall catm.c -o catm
catd: catd.d
dmd -O catd.d
#catd: catd.d
# dmd -O catd.d
catrust: catrust.rs
rustc catrust.rs
catgo: catgo.go
go build catgo.go
catnim: catnim.nim
nim compile catnim.nim
cutd: cutd.d
dmd -O cutd.d
#cutd: cutd.d
# dmd -O cutd.d
cutgo: cutgo.go
go build cutgo.go
cutnim: cutnim.nim

7
perf/catgo.go
View file

@ -10,14 +10,13 @@ import (
// ----------------------------------------------------------------
func main() {
args := os.Args[1:]
includeFields := []string {"a", "x"};
ok := true
if len(args) == 0 {
ok = handle("-", includeFields) && ok
ok = handle("-") && ok
} else {
for _, arg := range args {
ok = handle(arg, includeFields) && ok
ok = handle(arg) && ok
}
}
if ok {
@ -28,7 +27,7 @@ func main() {
}
// ----------------------------------------------------------------
func handle(fileName string, includeFields []string) (ok bool) {
func handle(fileName string) (ok bool) {
inputStream := os.Stdin
if fileName != "-" {
var err error

7
perf/catgo2.go
View file

@ -10,14 +10,13 @@ import (
// ----------------------------------------------------------------
func main() {
args := os.Args[1:]
includeFields := []string {"a", "x"};
ok := true
if len(args) == 0 {
ok = handle("-", includeFields) && ok
ok = handle("-") && ok
} else {
for _, arg := range args {
ok = handle(arg, includeFields) && ok
ok = handle(arg) && ok
}
}
if ok {
@ -28,7 +27,7 @@ func main() {
}
// ----------------------------------------------------------------
func handle(fileName string, includeFields []string) (ok bool) {
func handle(fileName string) (ok bool) {
inputStream := os.Stdin
if fileName != "-" {
var err error

151
perf/cutgo.go
View file

@ -2,6 +2,7 @@ package main
import (
"bufio"
"bytes"
"io"
"log"
"os"
@ -10,14 +11,14 @@ import (
// ----------------------------------------------------------------
func main() {
args := os.Args[1:]
includeFields := []string {"a", "x"};
includeFields := strings.Split(os.Args[1], ",")
filenames := os.Args[2:]
ok := true
if len(args) == 0 {
if len(filenames) == 0 {
ok = handle("-", includeFields) && ok
} else {
for _, arg := range args {
for _, arg := range filenames {
ok = handle(arg, includeFields) && ok
}
}
@ -40,7 +41,6 @@ func handle(fileName string, includeFields []string) (ok bool) {
}
reader := bufio.NewReader(inputStream)
writer := bufio.NewWriter(os.Stdout)
eof := false
for !eof {
@ -56,59 +56,148 @@ func handle(fileName string, includeFields []string) (ok bool) {
return false
} else {
// 0.030s
// continue
// $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null
// TIME IN SECONDS 0.228 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.226 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.222 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.221 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.228 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.226 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.225 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.227 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.222 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 0.223 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
//
// avg 0.2248
// cumu 4%
// Line to map
mymap := make(map[string]string)
fields := strings.Split(line, ",");
for _, field := range(fields) {
fields := strings.Split(line, ",")
for _, field := range fields {
kvps := strings.SplitN(field, "=", 2)
mymap[kvps[0]] = kvps[1]
}
// 0.220s
// delta 0.190s
// 27%
// continue
// $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null
// TIME IN SECONDS 3.055 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.837 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.905 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.817 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.766 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.810 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.748 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.744 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.765 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.722 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
//
// avg 2.8169
// cumu 58%
// delta 54%
// Map-to-map transform
newmap := make(map[string]string)
for _, includeField := range(includeFields) {
for _, includeField := range includeFields {
value, present := mymap[includeField]
if present {
newmap[includeField] = value
}
}
// 0.280s
// delta 0.060s
// 9%
// continue
// $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null
// TIME IN SECONDS 3.101 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.028 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.992 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.005 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.991 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.986 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.992 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.992 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.989 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 2.991 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
//
// avg 3.0067
// cumu 62%
// delta 4%
// Map to string
outs := make([]string, len(newmap))
// Faster to assemble in memory with single fmt.Println at the end,
// than multiple fmt.Print through the fields.
var buffer bytes.Buffer
i := 0
for k, v := range(newmap) {
outs[i] = k + "=" + v
for k, v := range newmap {
if i > 0 {
buffer.WriteString(",")
}
buffer.WriteString(k)
buffer.WriteString("=")
buffer.WriteString(v)
i++
}
// 0.320s
// delta 0.040s
// 6%
out := strings.Join(outs, ",")
// 0.330s
// delta 0.010s
// 2%
// continue
// $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null
// TIME IN SECONDS 3.821 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.443 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.491 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.421 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.412 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.470 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.438 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.555 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.637 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.538 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
//
// avg 3.5226
// cumu 73%
// delta 11%
// Write to output stream
//fmt.Println("")
writer.WriteString(out)
// delta 0.390s
// 56%
buffer.WriteString("\n")
// continue
// $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null
// TIME IN SECONDS 3.769 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.539 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.539 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.486 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.518 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.492 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.506 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.486 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.495 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 3.560 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
//
// avg 3.539
// cumu 73%
// delta 0%
os.Stdout.WriteString(buffer.String())
// $ repeat 10 justtime cutgo ccode,milex,year,cinc ../c/nmc1.dkvp > /dev/null
// TIME IN SECONDS 4.976 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.727 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.760 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.747 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.826 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.849 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.841 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.747 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.841 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
// TIME IN SECONDS 4.765 -- cutgo ccode,milex,year,cinc ../c/nmc1.dkvp
//
// avg 4.8079
// cumu 100%
// delta 27%
}
}
if fileName != "-" {
inputStream.Close()
}
writer.Flush()
return true
}

16
perf/do-timings.rb
View file

@ -28,10 +28,13 @@ run("catc0", "catc0 ../data/big.dkvp > /dev/null")
run("catc", "catc < ../data/big.dkvp > /dev/null")
run("catc", "catc ../data/big.dkvp > /dev/null")
run("catm", "catm ../data/big.dkvp > /dev/null")
run("catgo", "catgo ../data/big.dkvp > /dev/null")
run("catawk", "awk -F, '{print}' ../data/big.dkvp > /dev/null")
run("catmawk", "mawk -F, '{print}' ../data/big.dkvp > /dev/null")
run("CATMLR", "mlr --no-mmap cat ../data/big.dkvp > /dev/null")
run("CATMLRM", "mlr --mmap cat ../data/big.dkvp > /dev/null")
run("CATMLR", "../c/mlr --no-mmap cat ../data/big.dkvp > /dev/null")
run("CATMLRM", "../c/mlr --mmap cat ../data/big.dkvp > /dev/null")
run("CATMLR", "../go/mlr --no-mmap cat ../data/big.dkvp > /dev/null")
run("CATMLRM", "../go/mlr --mmap cat ../data/big.dkvp > /dev/null")
puts
run("(catv)", "cat < ../data/big.csv > /dev/null")
@ -40,12 +43,13 @@ run("catc0v", "catc0 ../data/big.csv > /dev/null")
run("catcv", "catc < ../data/big.csv > /dev/null")
run("catcv", "catc ../data/big.csv > /dev/null")
run("catmv", "catm ../data/big.csv > /dev/null")
run("catgov", "catgo ../data/big.csv > /dev/null")
run("catawkv", "awk -F, '{print}' ../data/big.csv > /dev/null")
run("catmawkv", "mawk -F, '{print}' ../data/big.csv > /dev/null")
run("CATMLRV", "mlr --no-mmap --csv --rs lf cat ../data/big.csv > /dev/null")
run("CATMLRVM", "mlr --mmap --csv --rs lf cat ../data/big.csv > /dev/null")
run("CATMLRV", "mlr --no-mmap --csvlite cat ../data/big.csv > /dev/null")
run("CATMLRVM", "mlr --mmap --csvlite cat ../data/big.csv > /dev/null")
run("CATMLRV", "../c/mlr --no-mmap --csv --rs lf cat ../data/big.csv > /dev/null")
run("CATMLRVM", "../c/mlr --mmap --csv --rs lf cat ../data/big.csv > /dev/null")
run("CATMLRV", "../c/mlr --no-mmap --csvlite cat ../data/big.csv > /dev/null")
run("CATMLRVM", "../c/mlr --mmap --csvlite cat ../data/big.csv > /dev/null")
puts
puts