mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-28 10:13:59 +00:00
537 lines
17 KiB
Go
537 lines
17 KiB
Go
package cli
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"miller/src/cliutil"
|
|
"miller/src/dsl/cst"
|
|
"miller/src/lib"
|
|
"miller/src/transformers"
|
|
"miller/src/transforming"
|
|
"miller/src/version"
|
|
)
|
|
|
|
// ----------------------------------------------------------------
|
|
func ParseCommandLine(args []string) (
|
|
options cliutil.TOptions,
|
|
recordTransformers []transforming.IRecordTransformer,
|
|
err error,
|
|
) {
|
|
options = cliutil.DefaultOptions()
|
|
argc := len(args)
|
|
argi := 1
|
|
|
|
// Try .mlrrc overrides (then command-line on top of that).
|
|
// A --norc flag (if provided) must come before all other options.
|
|
// Or, they can set the environment variable MLRRC="__none__".
|
|
if argc >= 2 && args[argi] == "--norc" {
|
|
argi++
|
|
} else {
|
|
loadMlrrcOrDie(&options)
|
|
}
|
|
|
|
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
|
if args[argi][0] != '-' {
|
|
break // No more flag options to process
|
|
} else if args[argi] == "--cpuprofile" {
|
|
// Already handled in main(); ignore here.
|
|
cliutil.CheckArgCount(args, argi, argc, 1)
|
|
argi += 2
|
|
} else if parseTerminalUsage(args, argc, argi) {
|
|
os.Exit(0)
|
|
} else if cliutil.ParseReaderOptions(args, argc, &argi, &options.ReaderOptions) {
|
|
// handled
|
|
} else if cliutil.ParseWriterOptions(args, argc, &argi, &options.WriterOptions) {
|
|
// handled
|
|
} else if cliutil.ParseReaderWriterOptions(args, argc, &argi,
|
|
&options.ReaderOptions, &options.WriterOptions) {
|
|
// handled
|
|
} else if cliutil.ParseMiscOptions(args, argc, &argi, &options) {
|
|
// handled
|
|
} else {
|
|
// unhandled
|
|
usageUnrecognizedVerb(lib.MlrExeName(), args[argi])
|
|
}
|
|
}
|
|
|
|
// xxx to do
|
|
// cli_apply_defaults(popts);
|
|
|
|
// lhmss_t* default_rses = get_default_rses();
|
|
// lhmss_t* default_fses = get_default_fses();
|
|
// lhmss_t* default_pses = get_default_pses();
|
|
// lhmsll_t* default_repeat_ifses = get_default_repeat_ifses();
|
|
// lhmsll_t* default_repeat_ipses = get_default_repeat_ipses();
|
|
//
|
|
// if (options.ReaderOptions.IRS == nil)
|
|
// options.ReaderOptions.IRS = lhmss_get_or_die(default_rses, options.ReaderOptions.InputFileFormat);
|
|
// if (options.ReaderOptions.IFS == nil)
|
|
// options.ReaderOptions.IFS = lhmss_get_or_die(default_fses, options.ReaderOptions.InputFileFormat);
|
|
// if (options.ReaderOptions.ips == nil)
|
|
// options.ReaderOptions.ips = lhmss_get_or_die(default_pses, options.ReaderOptions.InputFileFormat);
|
|
//
|
|
// if (options.ReaderOptions.allow_repeat_ifs == NEITHER_TRUE_NOR_FALSE)
|
|
// options.ReaderOptions.allow_repeat_ifs = lhmsll_get_or_die(default_repeat_ifses, options.ReaderOptions.InputFileFormat);
|
|
// if (options.ReaderOptions.allow_repeat_ips == NEITHER_TRUE_NOR_FALSE)
|
|
// options.ReaderOptions.allow_repeat_ips = lhmsll_get_or_die(default_repeat_ipses, options.ReaderOptions.InputFileFormat);
|
|
//
|
|
// if (options.WriterOptions.ORS == nil)
|
|
// options.WriterOptions.ORS = lhmss_get_or_die(default_rses, options.WriterOptions.OutputFileFormat);
|
|
// if (options.WriterOptions.OFS == nil)
|
|
// options.WriterOptions.OFS = lhmss_get_or_die(default_fses, options.WriterOptions.OutputFileFormat);
|
|
// if (options.WriterOptions.ops == nil)
|
|
// options.WriterOptions.ops = lhmss_get_or_die(default_pses, options.WriterOptions.OutputFileFormat);
|
|
//
|
|
// if options.WriterOptions.OutputFileFormat == "pprint") && len(options.WriterOptions.OFS) != 1) {
|
|
// fmt.Fprintf(os.Stderr, "%s: OFS for PPRINT format must be single-character; got \"%s\".\n",
|
|
// lib.MlrExeName(), options.WriterOptions.OFS);
|
|
// return nil;
|
|
// }
|
|
|
|
// // Construct the transformer list for single use, e.g. the normal streaming case wherein the
|
|
// // transformers operate on all input files. Also retain information needed to construct them
|
|
// // for each input file, for in-place mode.
|
|
// options.transformer_argb = argi;
|
|
// options.original_argv = args;
|
|
// options.non_in_place_argv = copy_argv(args);
|
|
// options.argc = argc;
|
|
// *pptransformer_list = cli_parse_transformers(options.non_in_place_argv, &argi, argc, popts);
|
|
|
|
recordTransformers, ignoresInput, err := parseTransformers(args, &argi, argc, &options)
|
|
if err != nil {
|
|
return options, recordTransformers, err
|
|
}
|
|
if ignoresInput {
|
|
options.NoInput = true // e.g. then-chain begins with seqgen
|
|
}
|
|
|
|
if DecideFinalFlatten(&options) {
|
|
// E.g. '{"req": {"method": "GET", "path": "/api/check"}}' becomes
|
|
// req.method=GET,req.path=/api/check.
|
|
transformer, err := transformers.NewTransformerFlatten(options.WriterOptions.OFLATSEP, nil)
|
|
lib.InternalCodingErrorIf(err != nil)
|
|
lib.InternalCodingErrorIf(transformer == nil)
|
|
recordTransformers = append(recordTransformers, transformer)
|
|
}
|
|
|
|
if DecideFinalUnflatten(&options) {
|
|
// E.g. req.method=GET,req.path=/api/check becomes
|
|
// '{"req": {"method": "GET", "path": "/api/check"}}'
|
|
transformer, err := transformers.NewTransformerUnflatten(options.WriterOptions.OFLATSEP, nil)
|
|
lib.InternalCodingErrorIf(err != nil)
|
|
lib.InternalCodingErrorIf(transformer == nil)
|
|
recordTransformers = append(recordTransformers, transformer)
|
|
}
|
|
|
|
// There may already be one or more because of --from on the command line,
|
|
// so append.
|
|
for _, arg := range args[argi:] {
|
|
options.FileNames = append(options.FileNames, arg)
|
|
}
|
|
|
|
// E.g. mlr -n put -v '...'
|
|
if options.NoInput {
|
|
options.FileNames = nil
|
|
}
|
|
|
|
if options.DoInPlace && (options.FileNames == nil || len(options.FileNames) == 0) {
|
|
fmt.Fprintf(os.Stderr, "%s: -I option (in-place operation) requires input files.\n", lib.MlrExeName())
|
|
os.Exit(1)
|
|
}
|
|
|
|
if options.HaveRandSeed {
|
|
lib.SeedRandom(int64(options.RandSeed))
|
|
}
|
|
|
|
return options, recordTransformers, nil
|
|
}
|
|
|
|
// ================================================================
|
|
// Decide whether to insert a flatten or unflatten verb at the end of the
|
|
// chain. See also repl/verbs.go which handles the same issue in the REPL.
|
|
//
|
|
// ----------------------------------------------------------------
|
|
// PROBLEM TO BE SOLVED:
|
|
//
|
|
// JSON has nested structures and CSV et al. do not. For example:
|
|
// {
|
|
// "req" : {
|
|
// "method": "GET",
|
|
// "path": "api/check",
|
|
// }
|
|
// }
|
|
//
|
|
// For CSV we flatten this down to
|
|
//
|
|
// {
|
|
// "req.method": "GET",
|
|
// "req.path": "api/check"
|
|
// }
|
|
//
|
|
// ----------------------------------------------------------------
|
|
// APPROACH:
|
|
//
|
|
// Use the Principle of Least Surprise (POLS).
|
|
//
|
|
// * If input is JSON and output is JSON:
|
|
// o Records can be nested from record-read
|
|
// o They remain that way through the Miller record-processing stream
|
|
// o They are nested on record-write
|
|
// o No action needs to be taken
|
|
//
|
|
// * If input is JSON and output is non-JSON:
|
|
// o Records can be nested from record-read
|
|
// o They remain that way through the Miller record-processing stream
|
|
// o On record-write, nested structures will be converted to string (carriage
|
|
// returns and all) using json_stringify. People *might* want this but
|
|
// (using POLS) we will (by default) AUTO-FLATTEN for them. There is a
|
|
// --no-auto-unflatten CLI flag for those who want it.
|
|
//
|
|
// * If input is non-JSON and output is non-JSON:
|
|
// o If there is a "req.method" field, people should be able to do
|
|
// 'mlr sort -f req.method' with no surprises. (Again, POLS.) Therefore
|
|
// no auto-unflatten on input. People can insert an unflatten verb
|
|
// into their verb chain if they really want unflatten for non-JSON
|
|
// files.
|
|
// o The DSL can make nested data, so AUTO-FLATTEN at output.
|
|
//
|
|
// * If input is non-JSON and output is JSON:
|
|
// o Default is to auto-unflatten at output.
|
|
// o There is a --no-auto-unflatten for those who want it.
|
|
// ================================================================
|
|
|
|
func DecideFinalFlatten(options *cliutil.TOptions) bool {
|
|
ofmt := options.WriterOptions.OutputFileFormat
|
|
if options.WriterOptions.AutoFlatten {
|
|
if ofmt != "json" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func DecideFinalUnflatten(options *cliutil.TOptions) bool {
|
|
ifmt := options.ReaderOptions.InputFileFormat
|
|
ofmt := options.WriterOptions.OutputFileFormat
|
|
|
|
if options.WriterOptions.AutoUnflatten {
|
|
if ifmt != "json" {
|
|
if ofmt == "json" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// Returns a list of transformers, from the starting point in args given by *pargi.
|
|
// Bumps *pargi to point to remaining post-transformer-setup args, i.e. filenames.
|
|
|
|
func parseTransformers(
|
|
args []string,
|
|
pargi *int,
|
|
argc int,
|
|
options *cliutil.TOptions,
|
|
) (
|
|
transformerList []transforming.IRecordTransformer,
|
|
ignoresInput bool,
|
|
err error,
|
|
) {
|
|
|
|
transformerList = make([]transforming.IRecordTransformer, 0)
|
|
ignoresInput = false
|
|
|
|
argi := *pargi
|
|
|
|
// Allow then-chains to start with an initial 'then': 'mlr verb1 then verb2 then verb3' or
|
|
// 'mlr then verb1 then verb2 then verb3'. Particuarly useful in backslashy scripting contexts.
|
|
if (argc-argi) >= 1 && args[argi] == "then" {
|
|
argi++
|
|
}
|
|
|
|
if (argc - argi) < 1 {
|
|
fmt.Fprintf(os.Stderr, "%s: no verb supplied.\n", lib.MlrExeName())
|
|
mainUsageShort()
|
|
os.Exit(1)
|
|
}
|
|
|
|
onFirst := true
|
|
|
|
for {
|
|
cliutil.CheckArgCount(args, argi, argc, 1)
|
|
verb := args[argi]
|
|
|
|
transformerSetup := lookUpTransformerSetup(verb)
|
|
if transformerSetup == nil {
|
|
fmt.Fprintf(os.Stderr,
|
|
"%s: verb \"%s\" not found. Please use \"%s --help\" for a list.\n",
|
|
lib.MlrExeName(), verb, lib.MlrExeName())
|
|
os.Exit(1)
|
|
}
|
|
|
|
// E.g. then-chain begins with seqgen
|
|
if onFirst && transformerSetup.IgnoresInput {
|
|
ignoresInput = true
|
|
}
|
|
onFirst = false
|
|
|
|
// It's up to the parse func to print its usage, and exit 1, on
|
|
// CLI-parse failure. Also note: this assumes main reader/writer opts
|
|
// are all parsed *before* transformer parse-CLI methods are invoked.
|
|
transformer := transformerSetup.ParseCLIFunc(
|
|
&argi,
|
|
argc,
|
|
args,
|
|
&options.ReaderOptions,
|
|
&options.WriterOptions,
|
|
)
|
|
lib.InternalCodingErrorIf(transformer == nil)
|
|
|
|
transformerList = append(transformerList, transformer)
|
|
|
|
if argi >= argc || args[argi] != "then" {
|
|
break
|
|
}
|
|
argi++
|
|
}
|
|
|
|
*pargi = argi
|
|
return transformerList, ignoresInput, nil
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
func parseTerminalUsage(args []string, argc int, argi int) bool {
|
|
if args[argi] == "--version" {
|
|
fmt.Printf("Miller %s\n", version.STRING)
|
|
return true
|
|
} else if args[argi] == "-h" {
|
|
mainUsageLong(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
} else if args[argi] == "--help" {
|
|
mainUsageLong(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
// } else if args[argi] == "--print-type-arithmetic-info" {
|
|
// printTypeArithmeticInfo(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
//
|
|
} else if args[argi] == "--help-all-verbs" || args[argi] == "--usage-all-verbs" {
|
|
usageAllVerbs(lib.MlrExeName())
|
|
} else if args[argi] == "--list-all-verbs" || args[argi] == "-l" {
|
|
listAllVerbs(os.Stdout, " ")
|
|
return true
|
|
} else if args[argi] == "--list-all-verbs-raw" || args[argi] == "-L" {
|
|
listAllVerbsRaw(os.Stdout)
|
|
return true
|
|
|
|
} else if args[argi] == "--list-all-functions-raw" || args[argi] == "-F" {
|
|
cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionsRaw(os.Stdout)
|
|
return true
|
|
// } else if args[argi] == "--list-all-functions-as-table" {
|
|
// fmgr_t* pfmgr = fmgr_alloc();
|
|
// fmgr_list_all_functions_as_table(pfmgr, os.Stdout);
|
|
// fmgr_free(pfmgr, nil);
|
|
// return true;
|
|
} else if args[argi] == "--help-all-functions" || args[argi] == "-f" {
|
|
cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsages(os.Stdout)
|
|
return true
|
|
} else if args[argi] == "--help-function" || args[argi] == "--hf" {
|
|
cliutil.CheckArgCount(args, argi, argc, 2)
|
|
cst.BuiltinFunctionManagerInstance.ListBuiltinFunctionUsage(args[argi+1], os.Stdout)
|
|
argi++
|
|
return true
|
|
|
|
// } else if args[argi] == "--list-all-keywords-raw" || args[argi] == "-K" {
|
|
// mlr_dsl_list_all_keywords_raw(os.Stdout);
|
|
// return true;
|
|
// } else if args[argi] == "--help-all-keywords" || args[argi] == "-k" {
|
|
// mlr_dsl_keyword_usage(os.Stdout, nil);
|
|
// return true;
|
|
// } else if args[argi] == "--help-keyword" || args[argi] == "--hk" {
|
|
// cliutil.CheckArgCount(args, argi, argc, 2);
|
|
// mlr_dsl_keyword_usage(os.Stdout, args[argi+1]);
|
|
// return true;
|
|
//
|
|
// // main-usage subsections, individually accessible for the benefit of
|
|
// // the manpage-autogenerator
|
|
} else if args[argi] == "--usage-synopsis" {
|
|
mainUsageSynopsis(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
} else if args[argi] == "--usage-examples" {
|
|
mainUsageExamples(os.Stdout, lib.MlrExeName(), "")
|
|
return true
|
|
} else if args[argi] == "--usage-list-all-verbs" {
|
|
listAllVerbs(os.Stdout, "")
|
|
return true
|
|
} else if args[argi] == "--usage-help-options" {
|
|
mainUsageHelpOptions(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
// } else if args[argi] == "--usage-mlrrc" {
|
|
// mainUsageMlrrc(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
// } else if args[argi] == "--usage-functions" {
|
|
// mainUsageFunctions(os.Stdout, lib.MlrExeName(), "");
|
|
// return true;
|
|
// } else if args[argi] == "--usage-data-format-examples" {
|
|
mainUsageDataFormatExamples(os.Stdout, lib.MlrExeName())
|
|
// return true;
|
|
} else if args[argi] == "--usage-data-format-options" {
|
|
mainUsageDataFormatOptions(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
// } else if args[argi] == "--usage-comments-in-data" {
|
|
// mainUsageCommentsInData(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
} else if args[argi] == "--usage-format-conversion-keystroke-saver-options" {
|
|
mainUsageFormatConversionKeystrokeSaverOptions(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
// } else if args[argi] == "--usage-compressed-data-options" {
|
|
// mainUsageCompressedDataOptions(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
// } else if args[argi] == "--usage-separator-options" {
|
|
// mainUsageSeparatorOptions(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
// } else if args[argi] == "--usage-csv-options" {
|
|
// mainUsageCsvOptions(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
// } else if args[argi] == "--usage-double-quoting" {
|
|
// mainUsageDoubleQuoting(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
// } else if args[argi] == "--usage-numerical-formatting" {
|
|
// mainUsageNumericalFormatting(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
} else if args[argi] == "--usage-other-options" {
|
|
mainUsageOtherOptions(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
} else if args[argi] == "--usage-then-chaining" {
|
|
mainUsageThenChaining(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
// } else if args[argi] == "--usage-auxents" {
|
|
// mainUsageAuxents(os.Stdout, lib.MlrExeName());
|
|
// return true;
|
|
} else if args[argi] == "--usage-see-also" {
|
|
mainUsageSeeAlso(os.Stdout, lib.MlrExeName())
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ----------------------------------------------------------------
|
|
// * If $MLRRC is set, use it and only it.
|
|
// * Otherwise try first $HOME/.mlrrc and then ./.mlrrc but let them
|
|
// stack: e.g. $HOME/.mlrrc is lots of settings and maybe in one
|
|
// subdir you want to override just a setting or two.
|
|
|
|
// TODO: move to separate file?
|
|
func loadMlrrcOrDie(
|
|
options *cliutil.TOptions,
|
|
) {
|
|
env_mlrrc := os.Getenv("MLRRC")
|
|
|
|
if env_mlrrc != "" {
|
|
if env_mlrrc == "__none__" {
|
|
return
|
|
}
|
|
if tryLoadMlrrc(options, env_mlrrc) {
|
|
return
|
|
}
|
|
}
|
|
|
|
env_home := os.Getenv("HOME")
|
|
if env_home != "" {
|
|
path := env_home + "/.mlrrc"
|
|
tryLoadMlrrc(options, path)
|
|
}
|
|
|
|
tryLoadMlrrc(options, "./.mlrrc")
|
|
}
|
|
|
|
func tryLoadMlrrc(
|
|
options *cliutil.TOptions,
|
|
path string,
|
|
) bool {
|
|
handle, err := os.Open(path)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
defer handle.Close()
|
|
|
|
lineReader := bufio.NewReader(handle)
|
|
|
|
eof := false
|
|
lineno := 0
|
|
for !eof {
|
|
line, err := lineReader.ReadString('\n')
|
|
if err == io.EOF {
|
|
err = nil
|
|
eof = true
|
|
break
|
|
}
|
|
lineno++
|
|
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, lib.MlrExeName(), err)
|
|
os.Exit(1)
|
|
return false
|
|
}
|
|
|
|
// This is how to do a chomp:
|
|
// TODO: handle \r\n with libified solution.
|
|
line = strings.TrimRight(line, "\n")
|
|
|
|
if !handleMlrrcLine(options, line) {
|
|
fmt.Fprintf(os.Stderr, "%s: parse error at file \"%s\" line %d: %s\n",
|
|
lib.MlrExeName(), path, lineno, line,
|
|
)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func handleMlrrcLine(
|
|
options *cliutil.TOptions,
|
|
line string,
|
|
) bool {
|
|
|
|
// Comment-strip
|
|
re := regexp.MustCompile("#.*")
|
|
line = re.ReplaceAllString(line, "")
|
|
|
|
// Left-trim / right-trim
|
|
line = strings.TrimSpace(line)
|
|
|
|
if line == "" { // line was whitespace-only
|
|
return true
|
|
}
|
|
|
|
// Prepend initial "--" if it's not already there
|
|
if !strings.HasPrefix(line, "-") {
|
|
line = "--" + line
|
|
}
|
|
|
|
// Split line into args array
|
|
args := strings.Fields(line)
|
|
argi := 0
|
|
argc := len(args)
|
|
|
|
if cliutil.ParseReaderOptions(args, argc, &argi, &options.ReaderOptions) {
|
|
// handled
|
|
} else if cliutil.ParseWriterOptions(args, argc, &argi, &options.WriterOptions) {
|
|
// handled
|
|
} else if cliutil.ParseReaderWriterOptions(args, argc, &argi,
|
|
&options.ReaderOptions, &options.WriterOptions) {
|
|
// handled
|
|
} else if cliutil.ParseMiscOptions(args, argc, &argi, options) {
|
|
// handled
|
|
} else {
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|