mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-25 08:53:55 +00:00
f233923351
* Rename inputChannel,outputChannel to readerChannel,writerChannel * Rename inputChannel,outputChannel to readerChannel,writerChannel (#772) * Start batched-reader API mods * Singleton-list step for reader-batching at input * CLI options for records-per-batch and hash-records * Push channelized-reader logic into DKVP reader * Push batching logic into chain-transformer, transformers, and channel-writer * foo * cmd/mprof and cmd/mprof2 * cmd/mprof3 and cmd/mprof4 * narrowed in on regexp-splitting on IFS/IPS as perf-hit * neaten * channelize nidx * cmd/mprof5 * channelize CSV reader * channelize NIDX reader * Dedupe DKVP-reader and NIDX-reader source files * channelize CSV-lite reader * channelize XTAB reader * batchify JSON reader * channelize GEN pseudo-reader * scripts for perf-testing on larger files * merge with main for #776 * Fix record-batching for join and repl * Fix comment-handling in channelized XTAB reader * Fix bug found in positional-rename
84 lines
2.2 KiB
Go
84 lines
2.2 KiB
Go
// Experiments in performance/profiling.
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"runtime"
|
|
"runtime/debug"
|
|
"runtime/pprof"
|
|
"strconv"
|
|
|
|
"github.com/pkg/profile" // for trace.out
|
|
|
|
"github.com/johnkerl/miller/internal/pkg/cli"
|
|
"github.com/johnkerl/miller/internal/pkg/stream"
|
|
"github.com/johnkerl/miller/internal/pkg/transformers"
|
|
"github.com/johnkerl/miller/internal/pkg/types"
|
|
)
|
|
|
|
func main() {
|
|
|
|
// Respect env $GOMAXPROCS, if provided, else set default.
|
|
haveSetGoMaxProcs := false
|
|
goMaxProcsString := os.Getenv("GOMAXPROCS")
|
|
if goMaxProcsString != "" {
|
|
goMaxProcs, err := strconv.Atoi(goMaxProcsString)
|
|
if err != nil {
|
|
runtime.GOMAXPROCS(goMaxProcs)
|
|
haveSetGoMaxProcs = true
|
|
}
|
|
}
|
|
if !haveSetGoMaxProcs {
|
|
// As of Go 1.16 this is the default anyway. For 1.15 and below we need
|
|
// to explicitly set this.
|
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
|
}
|
|
|
|
debug.SetGCPercent(500) // Empirical: See README-profiling.md
|
|
|
|
if os.Getenv("MPROF_PPROF") != "" {
|
|
// profiling with cpu.pprof and go tool pprof -http=:8080 cpu.pprof
|
|
profFilename := "cpu.pprof"
|
|
handle, err := os.Create(profFilename)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, os.Args[0], ": ", "Could not start CPU profile: ", err)
|
|
return
|
|
}
|
|
defer handle.Close()
|
|
|
|
if err := pprof.StartCPUProfile(handle); err != nil {
|
|
fmt.Fprintln(os.Stderr, os.Args[0], ": ", "Could not start CPU profile: ", err)
|
|
return
|
|
}
|
|
defer pprof.StopCPUProfile()
|
|
|
|
fmt.Fprintf(os.Stderr, "CPU profile started.\n")
|
|
fmt.Fprintf(os.Stderr, "go tool pprof -http=:8080 cpu.pprof\n")
|
|
defer fmt.Fprintf(os.Stderr, "CPU profile finished.\n")
|
|
}
|
|
|
|
if os.Getenv("MPROF_TRACE") != "" {
|
|
// tracing with trace.out and go tool trace trace.out
|
|
fmt.Fprintf(os.Stderr, "go tool trace trace.out\n")
|
|
defer profile.Start(profile.TraceProfile, profile.ProfilePath(".")).Stop()
|
|
}
|
|
|
|
options := cli.DefaultOptions()
|
|
types.SetInferrerStringOnly()
|
|
|
|
cat, err := transformers.NewTransformerCat(false, "", nil)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mprof5: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
xforms := []transformers.IRecordTransformer{cat}
|
|
|
|
filenames := os.Args[1:]
|
|
|
|
err = stream.Stream(filenames, options, xforms, os.Stdout, true)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mlr: %v.\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|