mlrcli iterate

c/cli/mlrcli.h

@@ -16,7 +16,7 @@
 #include <unistd.h>
 
 // ----------------------------------------------------------------
-typedef struct _genereator_opts_t {
+typedef struct _generator_opts_t {
 	char* field_name;
 	// xxx to do: convert to mv_t
 	long long start;

go/mlr.go

@@ -1,18 +1,12 @@
 package main
 
 import (
-	// System:
-	"flag"
 	"fmt"
 	"os"
 	"runtime"
-	"runtime/pprof"
-	// Miller:
+
+	"miller/cli"
 	"miller/stream"
-	// Temp:
-	"miller/dsl"
-	"miller/parsing/lexer"
-	"miller/parsing/parser"
 )
 
 // ----------------------------------------------------------------
@@ -22,7 +16,7 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "Usage: %s [options] {ifmt} {mapper} {ofmt} {filenames ...}\n",
 		os.Args[0])
 	fmt.Fprintf(os.Stderr, "If no file name is given, or if filename is \"-\", stdin is used.\n")
-	flag.PrintDefaults()
+	// stub
 	os.Exit(1)
 }
 
@@ -30,85 +24,15 @@ func usage() {
 func main() {
 	runtime.GOMAXPROCS(4) // Seems reasonable these days
 
-	cpuprofile := flag.String("cpuprofile", "", "Write CPU profile to `file`")
-
-	flag.Usage = usage
-	flag.Parse()
-	maybeProfile(cpuprofile)
-
-	args := flag.Args()
-
-	// xxx temp
-	if len(args) == 2 && args[0] == "parse" {
-		parse(args[1])
-		return
+	options, recordMappers, filenames, err := cli.ParseCommandLine(os.Args)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, os.Args[0], ": ", err)
+		os.Exit(1)
 	}
 
-	if len(args) < 3 {
-		usage()
-	}
-
-	var inputFormatName string
-	var mapperName string
-	var dslString string
-	var outputFormatName string
-	var filenames []string
-
-	// xxx temp
-	if args[1] == "put" {
-		if len(args) < 4 {
-			usage()
-		}
-		inputFormatName = args[0]
-		mapperName = args[1]
-		dslString = args[2]
-		outputFormatName = args[3]
-		filenames = args[4:]
-	} else {
-		inputFormatName = args[0]
-		mapperName = args[1]
-		dslString = ""
-		outputFormatName = args[2]
-		filenames = args[3:]
-	}
-
-	err := stream.Stream(filenames, inputFormatName, mapperName, dslString, outputFormatName)
+	err = stream.Stream(options, recordMappers, filenames)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, os.Args[0], ": ", err)
 		os.Exit(1)
 	}
 }
-
-// ----------------------------------------------------------------
-// xxx temp
-func parse(sourceString string) {
-	fmt.Printf("Input: %s\n", sourceString)
-	theLexer := lexer.NewLexer([]byte(sourceString))
-	theParser := parser.NewParser()
-	interfaceAST, err := theParser.Parse(theLexer)
-	if err == nil {
-		interfaceAST.(*dsl.AST).Print()
-	} else {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}
-
-// ----------------------------------------------------------------
-func maybeProfile(cpuprofile *string) {
-	// to do: move to method
-	// go tool pprof mlr foo.prof
-	//   top10
-	if *cpuprofile != "" {
-		f, err := os.Create(*cpuprofile)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, os.Args[0], ": ", "Could not start CPU profile: ", err)
-		}
-		defer f.Close()
-		if err := pprof.StartCPUProfile(f); err != nil {
-			fmt.Fprintln(os.Stderr, os.Args[0], ": ", "Could not start CPU profile: ", err)
-		}
-		defer pprof.StopCPUProfile()
-	}
-
-}

go/src/miller/input/record_reader_csv.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 	"miller/lib"
 	"miller/runtime"
@@ -16,7 +17,7 @@ type RecordReaderCSV struct {
 	//irs string
 }
 
-func NewRecordReaderCSV( /*ifs string, ips string*/ ) *RecordReaderCSV {
+func NewRecordReaderCSV(readerOptions *cli.TReaderOptions) *RecordReaderCSV {
 	return &RecordReaderCSV{
 		//ifs,
 		//irs,

go/src/miller/input/record_reader_dkvp.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 
+	"miller/cli"
 	"miller/containers"
 	"miller/lib"
 	"miller/runtime"
@@ -16,10 +17,10 @@ type RecordReaderDKVP struct {
 	ips string
 }
 
-func NewRecordReaderDKVP(ifs string, ips string) *RecordReaderDKVP {
+func NewRecordReaderDKVP(readerOptions *cli.TReaderOptions) *RecordReaderDKVP {
 	return &RecordReaderDKVP{
-		ifs,
-		ips,
+		ifs: readerOptions.IFS,
+		ips: readerOptions.IPS,
 	}
 }
 

go/src/miller/input/record_reader_factory.go

@@ -1,15 +1,19 @@
 package input
 
-func Create(inputFormatName string) IRecordReader {
-	switch inputFormatName {
+import (
+	"miller/cli"
+)
+
+func Create(readerOptions *cli.TReaderOptions) IRecordReader {
+	switch readerOptions.InputFileFormat {
 	case "csv":
-		return NewRecordReaderCSV() // TODO: parameterize
+		return NewRecordReaderCSV(readerOptions)
 	case "dkvp":
-		return NewRecordReaderDKVP(",", "=") // TODO: parameterize
+		return NewRecordReaderDKVP(readerOptions)
 	case "json":
-		return NewRecordReaderJSON()
+		return NewRecordReaderJSON(readerOptions)
 	case "nidx":
-		return NewRecordReaderNIDX() // TODO: parameterize
+		return NewRecordReaderNIDX(readerOptions)
 	default:
 		return nil
 	}

go/src/miller/input/record_reader_json.go

@@ -8,6 +8,7 @@ import (
 
 	"localdeps/ordered"
 
+	"miller/cli"
 	"miller/containers"
 	"miller/lib"
 	"miller/runtime"
@@ -16,7 +17,7 @@ import (
 type RecordReaderJSON struct {
 }
 
-func NewRecordReaderJSON() *RecordReaderJSON {
+func NewRecordReaderJSON(readerOptions *cli.TReaderOptions) *RecordReaderJSON {
 	return &RecordReaderJSON{}
 }
 

go/src/miller/input/record_reader_nidx.go

@@ -7,21 +7,22 @@ import (
 	"strconv"
 	"strings"
 
+	"miller/cli"
 	"miller/containers"
 	"miller/lib"
 	"miller/runtime"
 )
 
 type RecordReaderNIDX struct {
-	// TODO: parameterize
-	//ifs string
-	//ips string
+	// TODO: use the parameterization
+	ifs string
+	ips string
 }
 
-func NewRecordReaderNIDX() *RecordReaderNIDX {
+func NewRecordReaderNIDX(readerOptions *cli.TReaderOptions) *RecordReaderNIDX {
 	return &RecordReaderNIDX{
-		//ifs,
-		//ips,
+		ifs: readerOptions.IFS,
+		ips: readerOptions.IPS,
 	}
 }
 

go/src/miller/mapping/mapper_put.go

@@ -21,8 +21,8 @@ func NewMapperPut(dslString string) (*MapperPut, error) {
 		return nil, err
 	}
 	return &MapperPut{
-		ast,
-		dsl.NewInterpreter(),
+		ast: ast,
+		interpreter: dsl.NewInterpreter(),
 	}, nil
 }
 

go/src/miller/mapping/mapper_tac.go

@@ -13,7 +13,7 @@ type MapperTac struct {
 
 func NewMapperTac() (*MapperTac, error) {
 	return &MapperTac{
-		list.New(),
+		lrecs: list.New(),
 	}, nil
 }
 

go/src/miller/output/record_writer_csv.go

@@ -4,6 +4,7 @@ import (
 	"encoding/csv"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 )
 
@@ -13,10 +14,10 @@ type RecordWriterCSV struct {
 	csvWriter *csv.Writer
 }
 
-func NewRecordWriterCSV() *RecordWriterCSV {
+func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) *RecordWriterCSV {
 	return &RecordWriterCSV{
-		true,
-		csv.NewWriter(os.Stdout),
+		onFirst: true,
+		csvWriter: csv.NewWriter(os.Stdout),
 	}
 }
 

go/src/miller/output/record_writer_dkvp.go

@@ -4,21 +4,21 @@ import (
 	"bytes"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 )
 
-// ostream *os.File in constructors/factory
 type RecordWriterDKVP struct {
-	ifs string
-	ips string
+	ofs string
+	ops string
 	ors string
 }
 
-func NewRecordWriterDKVP(ifs string, ips string) *RecordWriterDKVP {
+func NewRecordWriterDKVP(writerOptions *cli.TWriterOptions) *RecordWriterDKVP {
 	return &RecordWriterDKVP{
-		ifs,
-		ips,
-		"\n", // TODO: parameterize
+		ofs: writerOptions.OFS,
+		ops: writerOptions.OPS,
+		ors: writerOptions.ORS,
 	}
 }
 
@@ -33,10 +33,10 @@ func (this *RecordWriterDKVP) Write(
 	var buffer bytes.Buffer // 5x faster than fmt.Print() separately
 	for pe := outrec.Head; pe != nil; pe = pe.Next {
 		buffer.WriteString(*pe.Key)
-		buffer.WriteString(this.ips)
+		buffer.WriteString(this.ops)
 		buffer.WriteString(pe.Value.String())
 		if pe.Next != nil {
-			buffer.WriteString(this.ifs)
+			buffer.WriteString(this.ofs)
 		}
 	}
 	buffer.WriteString(this.ors)

go/src/miller/output/record_writer_factory.go

@@ -1,19 +1,23 @@
 package output
 
-func Create(outputFormatName string) IRecordWriter {
-	switch outputFormatName {
+import (
+	"miller/cli"
+)
+
+func Create(writerOptions *cli.TWriterOptions) IRecordWriter {
+	switch writerOptions.OutputFileFormat {
 	case "csv":
-		return NewRecordWriterCSV() // TODO: parameterize
+		return NewRecordWriterCSV(writerOptions)
 	case "dkvp":
-		return NewRecordWriterDKVP(",", "=") // TODO: parameterize
+		return NewRecordWriterDKVP(writerOptions)
 	case "json":
-		return NewRecordWriterJSON() // TODO: parameterize
+		return NewRecordWriterJSON(writerOptions)
 	case "nidx":
-		return NewRecordWriterNIDX(",") // TODO: parameterize
+		return NewRecordWriterNIDX(writerOptions)
 	case "pprint":
-		return NewRecordWriterPPRINT() // TODO: parameterize
+		return NewRecordWriterPPRINT(writerOptions)
 	case "xtab":
-		return NewRecordWriterXTAB() // TODO: parameterize
+		return NewRecordWriterXTAB(writerOptions)
 	default:
 		return nil
 	}

go/src/miller/output/record_writer_json.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 )
 
@@ -12,9 +13,9 @@ type RecordWriterJSON struct {
 	onFirst bool
 }
 
-func NewRecordWriterJSON() *RecordWriterJSON {
+func NewRecordWriterJSON(writerOptions *cli.TWriterOptions) *RecordWriterJSON {
 	return &RecordWriterJSON{
-		true,
+		onFirst: true,
 	}
 }
 

go/src/miller/output/record_writer_nidx.go

@@ -4,19 +4,20 @@ import (
 	"bytes"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 )
 
 // ostream *os.File in constructors/factory
 type RecordWriterNIDX struct {
-	ifs string
+	ofs string
 	ors string
 }
 
-func NewRecordWriterNIDX(ifs string) *RecordWriterNIDX {
+func NewRecordWriterNIDX(writerOptions *cli.TWriterOptions) *RecordWriterNIDX {
 	return &RecordWriterNIDX{
-		ifs,
-		"\n", // TODO: parameterize
+		ofs: writerOptions.OFS,
+		ors: writerOptions.ORS,
 	}
 }
 
@@ -32,7 +33,7 @@ func (this *RecordWriterNIDX) Write(
 	for pe := outrec.Head; pe != nil; pe = pe.Next {
 		buffer.WriteString(pe.Value.String())
 		if pe.Next != nil {
-			buffer.WriteString(this.ifs)
+			buffer.WriteString(this.ofs)
 		}
 	}
 	buffer.WriteString(this.ors)

go/src/miller/output/record_writer_pprint.go

@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"fmt"
 
+	"miller/cli"
 	"miller/containers"
 )
 
@@ -12,9 +13,9 @@ type RecordWriterPPRINT struct {
 	lrecs *list.List
 }
 
-func NewRecordWriterPPRINT() *RecordWriterPPRINT {
+func NewRecordWriterPPRINT(writerOptions *cli.TWriterOptions) *RecordWriterPPRINT {
 	return &RecordWriterPPRINT{
-		list.New(),
+		lrecs: list.New(),
 	}
 }
 

go/src/miller/output/record_writer_xtab.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 )
 
@@ -12,9 +13,9 @@ type RecordWriterXTAB struct {
 	onFirst bool
 }
 
-func NewRecordWriterXTAB() *RecordWriterXTAB {
+func NewRecordWriterXTAB(writerOptions *cli.TWriterOptions) *RecordWriterXTAB {
 	return &RecordWriterXTAB{
-		true,
+		onFirst: true,
 	}
 }
 

go/src/miller/runtime/context.go

@@ -15,8 +15,8 @@ func NewLrecAndContext(
 	context *Context,
 ) *LrecAndContext {
 	return &LrecAndContext{
-		lrec,
-		*context,
+		Lrec: lrec,
+		Context: *context,
 	}
 }
 

go/src/miller/stream/stream.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 
+	"miller/cli"
 	"miller/containers"
 	"miller/input"
 	"miller/mapping"
@@ -14,31 +15,23 @@ import (
 
 // ----------------------------------------------------------------
 func Stream(
+	options cli.TOptions,
+	recordMappers []mapping.IRecordMapper,
 	filenames []string,
-	inputFormatName string,
-	mapperName string,
-	dslString string, // xxx temp
-	outputFormatName string,
 ) error {
 
 	initialContext := runtime.NewContext()
 
-	recordReader := input.Create(inputFormatName)
+	recordReader := input.Create(&options.ReaderOptions)
 	if recordReader == nil {
-		return errors.New("Input format not found: " + inputFormatName)
+		return errors.New("Input format not found: " + options.ReaderOptions.InputFileFormat)
 	}
 
-	recordMapper, err := mapping.Create(mapperName, dslString) // xxx temp
-	if err != nil {
-		return err
-	}
-	if recordMapper == nil {
-		return errors.New("Mapper not found: " + mapperName)
-	}
+	recordMapper := recordMappers[0] // xxx temp
 
-	recordWriter := output.Create(outputFormatName)
+	recordWriter := output.Create(&options.WriterOptions)
 	if recordWriter == nil {
-		return errors.New("Output format not found: " + outputFormatName)
+		return errors.New("Output format not found: " + options.WriterOptions.OutputFileFormat)
 	}
 
 	inrecs := make(chan *runtime.LrecAndContext, 10)