Merge branch 'master' of git+ssh://github.com/johnkerl/miller

.gitignore

@@ -33,6 +33,7 @@ test_parse_trie
 test_peek_file_reader
 test_string_builder
 output-regtest
+output-reg-test
 termcvt
 a.out
 *.dSYM

c/dsl/function_manager.c

@@ -433,7 +433,7 @@ static function_lookup_t FUNCTION_LOOKUP_TABLE[] = {
 		"'haskey(mymap, mykey)'. Error if 1st argument is not a map."},
 	{FUNC_CLASS_MAPS, "joink",         2,0, "Makes string from map keys. E.g. 'joink($*, \",\")'."},
 	{FUNC_CLASS_MAPS, "joinkv",        3,0, "Makes string from map key-value pairs. E.g. 'joinkv(@v[2], \"=\", \",\")'"},
-	{FUNC_CLASS_MAPS, "joinv",         2,0, "Makes string from map keys. E.g. 'joinv(mymap, \",\")'."},
+	{FUNC_CLASS_MAPS, "joinv",         2,0, "Makes string from map values. E.g. 'joinv(mymap, \",\")'."},
 	{FUNC_CLASS_MAPS, "leafcount",     1,0, "Counts total number of terminal values in hashmap. For single-level maps,\n"
 		"same as length."},
 	{FUNC_CLASS_MAPS, "length",        1,0, "Counts number of top-level entries in hashmap. Scalars have length 1."},

c/mapping/mapper_head.c

@@ -65,7 +65,11 @@ static mapper_t* mapper_head_parse_cli(int* pargi, int argc, char** argv,
 }
 
 // ----------------------------------------------------------------
-static mapper_t* mapper_head_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, unsigned long long head_count) {
+static mapper_t* mapper_head_alloc(
+	ap_state_t* pargp,
+	slls_t* pgroup_by_field_names,
+	unsigned long long head_count
+) {
 	mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
 
 	mapper_head_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_head_state_t));

c/mapping/mapper_put_or_filter.c

@@ -161,7 +161,7 @@ static void mapper_filter_usage(FILE* o, char* argv0, char* verb) {
 	fprintf(o, "\n");
 	fprintf(o, "Examples:\n");
 	fprintf(o, "  %s %s 'log10($count) > 4.0'\n", argv0, verb);
-	fprintf(o, "  %s %s 'FNR == 2          (second record in each file)'\n", argv0, verb);
+	fprintf(o, "  %s %s 'FNR == 2'         (second record in each file)\n", argv0, verb);
 	fprintf(o, "  %s %s 'urand() < 0.001'  (subsampling)\n", argv0, verb);
 	fprintf(o, "  %s %s '$color != \"blue\" && $value > 4.2'\n", argv0, verb);
 	fprintf(o, "  %s %s '($x<.5 && $y<.5) || ($x>.5 && $y>.5)'\n", argv0, verb);

c/mapping/mapper_seqgen.c

@@ -139,6 +139,8 @@ static sllv_t* mapper_seqgen_process(lrec_t* pinrec, context_t* pctx, void* pvst
 	) {
 		lrec_t* poutrec = lrec_unbacked_alloc();
 		lrec_put(poutrec, pstate->field_name, mv_alloc_format_val(&counter), FREE_ENTRY_VALUE);
+		pctx->nr++;
+		pctx->fnr++;
 		sllv_append(poutrecs, poutrec);
 	}
 	sllv_append(poutrecs, NULL);

c/mapping/mapper_unsparsify.c

@@ -15,9 +15,13 @@ typedef struct _mapper_unsparsify_state_t {
 static void      mapper_unsparsify_usage(FILE* o, char* argv0, char* verb);
 static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv,
 	cli_reader_opts_t* _, cli_writer_opts_t* __);
-static mapper_t* mapper_unsparsify_alloc(ap_state_t* pargp, char* filler);
+static mapper_t* mapper_unsparsify_alloc(ap_state_t* pargp, slls_t* pspecified_field_name,
+	char* filler);
 static void      mapper_unsparsify_free(mapper_t* pmapper, context_t* _);
-static sllv_t*   mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void* pvstate);
+static sllv_t*   mapper_unsparsify_streaming_process(lrec_t* pinrec, context_t* pctx,
+	void* pvstate);
+static sllv_t*   mapper_unsparsify_non_streaming_process(lrec_t* pinrec, context_t* pctx,
+	void* pvstate);
 
 // ----------------------------------------------------------------
 mapper_setup_t mapper_unsparsify_setup = {
@@ -31,12 +35,14 @@ mapper_setup_t mapper_unsparsify_setup = {
 static void mapper_unsparsify_usage(FILE* o, char* argv0, char* verb) {
 	fprintf(o, "Usage: %s %s [options]\n", argv0, verb);
 	fprintf(o, "Prints records with the union of field names over all input records.\n");
-	fprintf(o, "For field names absent in a given record but present in others, fills in\n");
-	fprintf(o, "a value. This verb retains all input before producing any output.\n");
+	fprintf(o, "For field names absent in a given record but present in others, fills in a\n");
+	fprintf(o, "value. Without -f, this verb retains all input before producing any output.\n");
 	fprintf(o, "\n");
 	fprintf(o, "Options:\n");
 	fprintf(o, "--fill-with {filler string}  What to fill absent fields with. Defaults to\n");
 	fprintf(o, "                             the empty string.\n");
+	fprintf(o, "-f {a,b,c} Specify field names to be operated on. Any other fields won't be\n");
+	fprintf(o, "                             modified, and operation will be streaming.\n");
 	fprintf(o, "\n");
 	fprintf(o, "Example: if the input is two records, one being 'a=1,b=2' and the other\n");
 	fprintf(o, "being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and\n");
@@ -47,6 +53,7 @@ static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv,
 	cli_reader_opts_t* _, cli_writer_opts_t* __)
 {
 	char* filler = "";
+	slls_t* pspecified_field_names = slls_alloc();
 
 	if ((argc - *pargi) < 1) {
 		mapper_unsparsify_usage(stderr, argv[0], argv[*pargi]);
@@ -57,17 +64,23 @@ static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv,
 
 	ap_state_t* pargp = ap_alloc();
 	ap_define_string_flag(pargp, "--fill-with", &filler);
+	ap_define_string_list_flag(pargp, "-f", &pspecified_field_names);
+
 	if (!ap_parse(pargp, verb, pargi, argc, argv)) {
 		mapper_unsparsify_usage(stderr, argv[0], verb);
 		return NULL;
 	}
 
-	mapper_t* pmapper = mapper_unsparsify_alloc(pargp, filler);
+	mapper_t* pmapper = mapper_unsparsify_alloc(pargp, pspecified_field_names, filler);
 	return pmapper;
 }
 
 // ----------------------------------------------------------------
-static mapper_t* mapper_unsparsify_alloc(ap_state_t* pargp, char* filler) {
+static mapper_t* mapper_unsparsify_alloc(
+	ap_state_t* pargp,
+	slls_t* pspecified_field_names,
+	char* filler
+) {
 	mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
 
 	mapper_unsparsify_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_unsparsify_state_t));
@@ -77,8 +90,15 @@ static mapper_t* mapper_unsparsify_alloc(ap_state_t* pargp, char* filler) {
 	pstate->pargp = pargp;
 
 	pmapper->pvstate       = pstate;
-	pmapper->pprocess_func = mapper_unsparsify_process;
 	pmapper->pfree_func    = mapper_unsparsify_free;
+	if (pspecified_field_names->length == 0) {
+		pmapper->pprocess_func = mapper_unsparsify_non_streaming_process;
+	} else {
+		for (sllse_t* pe = pspecified_field_names->phead; pe != NULL; pe = pe ->pnext) {
+			lhmsi_put(pstate->key_names, mlr_strdup_or_die(pe->value), 1, NO_FREE);
+		}
+		pmapper->pprocess_func = mapper_unsparsify_streaming_process;
+	}
 
 	return pmapper;
 }
@@ -94,10 +114,13 @@ static void mapper_unsparsify_free(mapper_t* pmapper, context_t* _) {
 }
 
 // ----------------------------------------------------------------
-static sllv_t* mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void* pvstate) {
+static sllv_t* mapper_unsparsify_non_streaming_process(
+	lrec_t* pinrec,
+	context_t* pctx,
+	void* pvstate
+) {
 	mapper_unsparsify_state_t* pstate = pvstate;
-	if (pinrec != NULL) {
-		// Not end of stream.
+	if (pinrec != NULL) { // Not end of stream.
 		for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) {
 			if (!lhmsi_has_key(pstate->key_names, pe->key)) {
 				lhmsi_put(pstate->key_names, mlr_strdup_or_die(pe->key), 1, FREE_ENTRY_KEY);
@@ -107,8 +130,7 @@ static sllv_t* mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void*
 		sllv_append(pstate->records, pinrec);
 		return NULL;
 	}
-	else {
-		// End of stream.
+	else { // End of stream.
 		sllv_t* poutrecs = sllv_alloc();
 		for (sllve_t* pe = pstate->records->phead; pe != NULL; pe = pe->pnext) {
 			lrec_t* pinrec = pe->pvvalue;
@@ -132,3 +154,25 @@ static sllv_t* mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void*
 		return poutrecs;
 	}
 }
+
+// ----------------------------------------------------------------
+static sllv_t* mapper_unsparsify_streaming_process(
+	lrec_t* pinrec,
+	context_t* pctx,
+	void* pvstate
+) {
+	mapper_unsparsify_state_t* pstate = pvstate;
+	if (pinrec != NULL) { // Not end of stream.
+
+		for (lhmsie_t* pe = pstate->key_names->phead; pe != NULL; pe = pe->pnext) {
+			if (lrec_get(pinrec, pe->key) == NULL) {
+				lrec_put(pinrec, mlr_strdup_or_die(pe->key), pstate->filler, FREE_ENTRY_KEY);
+			}
+		}
+
+		return sllv_single(pinrec);
+	}
+	else { // End of stream.
+		return sllv_single(NULL);
+	}
+}

c/output/lrec_writer_pprint.c

@@ -10,10 +10,11 @@ typedef struct _lrec_writer_pprint_state_t {
 	sllv_t*    precords;
 	slls_t*    pprev_keys;
 	int        right_align;
+	int        barred;
+	int        headerless_output;
 	long long  num_blocks_written;
 	char*      ors;
 	char       ofs;
-	int        barred;
 } lrec_writer_pprint_state_t;
 
 static void lrec_writer_pprint_free(lrec_writer_t* pwriter, context_t* pctx);
@@ -21,12 +22,12 @@ static void lrec_writer_pprint_process(void* pvstate, FILE* output_stream, lrec_
 static void lrec_writer_pprint_process_auto_ors(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
 static void lrec_writer_pprint_process_nonauto_ors(void* pvstate, FILE* output_stream, lrec_t* prec, context_t* pctx);
 static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, char* ors, char ofs,
-	int right_align);
+	int right_align, int headerless_output);
 static void print_and_free_record_list_barred(sllv_t* precords, FILE* output_stream, char* ors, char ofs,
-	int right_align);
+	int right_align, int headerless_output);
 
 // ----------------------------------------------------------------
-lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int right_align, int barred) {
+lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int right_align, int barred, int headerless_output) {
 	lrec_writer_t* plrec_writer = mlr_malloc_or_die(sizeof(lrec_writer_t));
 
 	lrec_writer_pprint_state_t* pstate = mlr_malloc_or_die(sizeof(lrec_writer_pprint_state_t));
@@ -36,6 +37,7 @@ lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int right_align, in
 	pstate->ofs                = ofs;
 	pstate->right_align        = right_align;
 	pstate->barred             = barred;
+	pstate->headerless_output  = headerless_output;
 	pstate->num_blocks_written = 0LL;
 
 	plrec_writer->pvstate       = pstate;
@@ -89,10 +91,10 @@ static void lrec_writer_pprint_process(void* pvstate, FILE* output_stream, lrec_
 			fputs(ors, output_stream);
 		if (pstate->barred) {
 			print_and_free_record_list_barred(pstate->precords, output_stream, ors, pstate->ofs,
-				pstate->right_align);
+				pstate->right_align, pstate->headerless_output);
 		} else {
 			print_and_free_record_list(pstate->precords, output_stream, ors, pstate->ofs,
-				pstate->right_align);
+				pstate->right_align, pstate->headerless_output);
 		}
 		if (pstate->pprev_keys != NULL) {
 			slls_free(pstate->pprev_keys);
@@ -110,7 +112,7 @@ static void lrec_writer_pprint_process(void* pvstate, FILE* output_stream, lrec_
 
 // ----------------------------------------------------------------
 static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, char* ors, char ofs,
-	int right_align)
+	int right_align, int headerless_output)
 {
 	if (precords->length == 0) {
 		sllv_free(precords);
@@ -121,7 +123,11 @@ static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, ch
 	int* max_widths = mlr_malloc_or_die(sizeof(int) * prec1->field_count);
 	int j = 0;
 	for (lrece_t* pe = prec1->phead; pe != NULL; pe = pe->pnext, j++) {
-		max_widths[j] = strlen_for_utf8_display(pe->key);
+		if (headerless_output) {
+			max_widths[j] = 1;
+		} else {
+			max_widths[j] = strlen_for_utf8_display(pe->key);
+		}
 	}
 	for (sllve_t* pnode = precords->phead; pnode != NULL; pnode = pnode->pnext) {
 		lrec_t* prec = pnode->pvvalue;
@@ -137,7 +143,7 @@ static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, ch
 	for (sllve_t* pnode = precords->phead; pnode != NULL; pnode = pnode->pnext, onr++) {
 		lrec_t* prec = pnode->pvvalue;
 
-		if (onr == 0) {
+		if (onr == 0 && !headerless_output) {
 			j = 0;
 			for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext, j++) {
 				if (j > 0) {
@@ -198,7 +204,7 @@ static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, ch
 
 // ----------------------------------------------------------------
 static void print_and_free_record_list_barred(sllv_t* precords, FILE* output_stream, char* ors, char ofs,
-	int right_align)
+	int right_align, int headerless_output)
 {
 	if (precords->length == 0) {
 		sllv_free(precords);
@@ -209,7 +215,11 @@ static void print_and_free_record_list_barred(sllv_t* precords, FILE* output_str
 	int* max_widths = mlr_malloc_or_die(sizeof(int) * prec1->field_count);
 	int j = 0;
 	for (lrece_t* pe = prec1->phead; pe != NULL; pe = pe->pnext, j++) {
-		max_widths[j] = strlen_for_utf8_display(pe->key);
+		if (headerless_output) {
+			max_widths[j] = 1;
+		} else {
+			max_widths[j] = strlen_for_utf8_display(pe->key);
+		}
 	}
 	for (sllve_t* pnode = precords->phead; pnode != NULL; pnode = pnode->pnext) {
 		lrec_t* prec = pnode->pvvalue;
@@ -225,7 +235,7 @@ static void print_and_free_record_list_barred(sllv_t* precords, FILE* output_str
 	for (sllve_t* pnode = precords->phead; pnode != NULL; pnode = pnode->pnext, onr++) {
 		lrec_t* prec = pnode->pvvalue;
 
-		if (onr == 0) {
+		if (onr == 0 && !headerless_output) {
 
 			j = 0;
 			fputc('+', output_stream);

c/output/lrec_writers.c

@@ -35,7 +35,7 @@ lrec_writer_t*  lrec_writer_alloc(cli_writer_opts_t* popts) {
 			return NULL;
 		} else {
 			return lrec_writer_pprint_alloc(popts->ors, popts->ofs[0], popts->right_align_pprint,
-				popts->pprint_barred);
+				popts->pprint_barred, popts->headerless_csv_output);
 		}
 
 	} else {

c/output/lrec_writers.h

@@ -18,7 +18,7 @@ lrec_writer_t* lrec_writer_json_alloc(int stack_vertically, int wrap_json_output
 	int json_quote_int_keys, int json_quote_non_string_values,
 	char* output_json_flatten_separator, char* line_term);
 lrec_writer_t* lrec_writer_nidx_alloc(char* ors, char* ofs);
-lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int right_align, int barred);
+lrec_writer_t* lrec_writer_pprint_alloc(char* ors, char ofs, int right_align, int barred, int headerless_output);
 lrec_writer_t* lrec_writer_xtab_alloc(char* ofs, char* ops, int right_justify_value);
 
 // Pops and frees the lrecs in the argument list without sllv-freeing the list structure itself.

c/reg_test/expected/out

@@ -614,6 +614,135 @@ zee wye 8  0.5985540091064224  -                   -   -   -                  -
 -   -   9  0.03144187646093577 0.7495507603507059  hat wye -                  -   -
 pan wye 10 0.5026260055412137  0.9526183602969864  -   -   -                  -   -
 
+mlr --opprint unsparsify -f nonesuch ./reg_test/input/abixy-het
+a   b   i x                  y                  nonesuch
+pan pan 1 0.3467901443380824 0.7268028627434533 -
+eks pan 2 0.7586799647899636 0.5221511083334797 -
+
+aaa b   i x                   y                   nonesuch
+wye wye 3 0.20460330576630303 0.33831852551664776 -
+
+a   bbb i x                   y                   nonesuch
+eks wye 4 0.38139939387114097 0.13418874328430463 -
+
+a   b   i xxx                y                  nonesuch
+wye pan 5 0.5732889198020006 0.8636244699032729 -
+
+a   b   i x                  y                   nonesuch
+zee pan 6 0.5271261600918548 0.49322128674835697 -
+
+a   b   iii x                  y                  nonesuch
+eks zee 7   0.6117840605678454 0.1878849191181694 -
+
+a   b   i x                  yyy               nonesuch
+zee wye 8 0.5985540091064224 0.976181385699006 -
+
+aaa bbb i x                   y                  nonesuch
+hat wye 9 0.03144187646093577 0.7495507603507059 -
+
+a   b   i  x                  y                  nonesuch
+pan wye 10 0.5026260055412137 0.9526183602969864 -
+
+mlr --opprint unsparsify -f a,b,i,x,y ./reg_test/input/abixy-het
+a   b   i x                  y
+pan pan 1 0.3467901443380824 0.7268028627434533
+eks pan 2 0.7586799647899636 0.5221511083334797
+
+aaa b   i x                   y                   a
+wye wye 3 0.20460330576630303 0.33831852551664776 -
+
+a   bbb i x                   y                   b
+eks wye 4 0.38139939387114097 0.13418874328430463 -
+
+a   b   i xxx                y                  x
+wye pan 5 0.5732889198020006 0.8636244699032729 -
+
+a   b   i x                  y
+zee pan 6 0.5271261600918548 0.49322128674835697
+
+a   b   iii x                  y                  i
+eks zee 7   0.6117840605678454 0.1878849191181694 -
+
+a   b   i x                  yyy               y
+zee wye 8 0.5985540091064224 0.976181385699006 -
+
+aaa bbb i x                   y                  a b
+hat wye 9 0.03144187646093577 0.7495507603507059 - -
+
+a   b   i  x                  y
+pan wye 10 0.5026260055412137 0.9526183602969864
+
+mlr --opprint unsparsify -f aaa,bbb,xxx,iii,yyy ./reg_test/input/abixy-het
+a   b   i x                  y                  aaa bbb xxx iii yyy
+pan pan 1 0.3467901443380824 0.7268028627434533 -   -   -   -   -
+eks pan 2 0.7586799647899636 0.5221511083334797 -   -   -   -   -
+
+aaa b   i x                   y                   bbb xxx iii yyy
+wye wye 3 0.20460330576630303 0.33831852551664776 -   -   -   -
+
+a   bbb i x                   y                   aaa xxx iii yyy
+eks wye 4 0.38139939387114097 0.13418874328430463 -   -   -   -
+
+a   b   i xxx                y                  aaa bbb iii yyy
+wye pan 5 0.5732889198020006 0.8636244699032729 -   -   -   -
+
+a   b   i x                  y                   aaa bbb xxx iii yyy
+zee pan 6 0.5271261600918548 0.49322128674835697 -   -   -   -   -
+
+a   b   iii x                  y                  aaa bbb xxx yyy
+eks zee 7   0.6117840605678454 0.1878849191181694 -   -   -   -
+
+a   b   i x                  yyy               aaa bbb xxx iii
+zee wye 8 0.5985540091064224 0.976181385699006 -   -   -   -
+
+aaa bbb i x                   y                  xxx iii yyy
+hat wye 9 0.03144187646093577 0.7495507603507059 -   -   -
+
+a   b   i  x                  y                  aaa bbb xxx iii yyy
+pan wye 10 0.5026260055412137 0.9526183602969864 -   -   -   -   -
+
+mlr --opprint unsparsify -f a,b,i,x,y,aaa,bbb,xxx,iii,yyy ./reg_test/input/abixy-het
+a   b   i x                  y                  aaa bbb xxx iii yyy
+pan pan 1 0.3467901443380824 0.7268028627434533 -   -   -   -   -
+eks pan 2 0.7586799647899636 0.5221511083334797 -   -   -   -   -
+
+aaa b   i x                   y                   a bbb xxx iii yyy
+wye wye 3 0.20460330576630303 0.33831852551664776 - -   -   -   -
+
+a   bbb i x                   y                   b aaa xxx iii yyy
+eks wye 4 0.38139939387114097 0.13418874328430463 - -   -   -   -
+
+a   b   i xxx                y                  x aaa bbb iii yyy
+wye pan 5 0.5732889198020006 0.8636244699032729 - -   -   -   -
+
+a   b   i x                  y                   aaa bbb xxx iii yyy
+zee pan 6 0.5271261600918548 0.49322128674835697 -   -   -   -   -
+
+a   b   iii x                  y                  i aaa bbb xxx yyy
+eks zee 7   0.6117840605678454 0.1878849191181694 - -   -   -   -
+
+a   b   i x                  yyy               y aaa bbb xxx iii
+zee wye 8 0.5985540091064224 0.976181385699006 - -   -   -   -
+
+aaa bbb i x                   y                  a b xxx iii yyy
+hat wye 9 0.03144187646093577 0.7495507603507059 - - -   -   -
+
+a   b   i  x                  y                  aaa bbb xxx iii yyy
+pan wye 10 0.5026260055412137 0.9526183602969864 -   -   -   -   -
+
+mlr --opprint unsparsify -f a,b,i,x,y,aaa,bbb,xxx,iii,yyy then regularize ./reg_test/input/abixy-het
+a   b   i  x                   y                   aaa bbb xxx                iii yyy
+pan pan 1  0.3467901443380824  0.7268028627434533  -   -   -                  -   -
+eks pan 2  0.7586799647899636  0.5221511083334797  -   -   -                  -   -
+-   wye 3  0.20460330576630303 0.33831852551664776 wye -   -                  -   -
+eks -   4  0.38139939387114097 0.13418874328430463 -   wye -                  -   -
+wye pan 5  -                   0.8636244699032729  -   -   0.5732889198020006 -   -
+zee pan 6  0.5271261600918548  0.49322128674835697 -   -   -                  -   -
+eks zee -  0.6117840605678454  0.1878849191181694  -   -   -                  7   -
+zee wye 8  0.5985540091064224  -                   -   -   -                  -   0.976181385699006
+-   -   9  0.03144187646093577 0.7495507603507059  hat wye -                  -   -
+pan wye 10 0.5026260055412137  0.9526183602969864  -   -   -                  -   -
+
 mlr --csv skip-trivial-records ./reg_test/input/trivial-records.csv
 a,b,c
 1,2,3
@@ -52555,6 +52684,19 @@ y/,x/,i/,b/,a/
 0.803142013402256216,0.000047786161325772/,3/,pan/,zee/
 0.573903236805416328,0.676537984365847889/,4/,hat/,zee/
 
+mlr --icsv --otsv -N cat
+a	b	c
+1	2	3
+4	5	6
+
+mlr --ixtab --opprint -N cat
+1 2 3
+4 5 6
+
+mlr --icsv --pprint -N cat
+1,2,3
+4,5,6
+
 
 ================================================================
 HET-CSV INPUT

c/reg_test/run

@@ -12,13 +12,6 @@
 # output.
 # ================================================================
 
-# ----------------------------------------------------------------
-# Make sure the script fails if any commnand fails. Note that this
-# includes a mlr command which should not fail and does -- as well as
-# one which should fail and does not. (See the mlr_expect_fail
-# function below.)
-set -e
-
 # ----------------------------------------------------------------
 # Path to mlr executable, which for interactive development is
 # simply "./mlr".
@@ -80,6 +73,8 @@ expfile=$expdir/out
 # Functions
 
 num_completed=0
+num_passed=0
+num_failed=0
 
 announce() {
   echo >> $outfile
@@ -103,9 +98,14 @@ run_mlr() {
   echo mlr "$@" >> $outfile
   # Use path to mlr for invoking the command
   $path_to_mlr "$@" >> $outfile
-  echo >> $outfile
-  # since set -e
+  status=$?
   num_completed=`expr $num_completed + 1`
+  if [ $status -eq 0 ]; then
+    num_passed=`expr $num_passed + 1`
+  else
+    num_failed=`expr $num_failed + 1`
+  fi
+  echo >> $outfile
 }
 
 run_cat() {
@@ -120,8 +120,13 @@ run_mlr_for_auxents() {
   echo mlr "$@" >> $outfile
   # Use path to mlr for invoking the command
   $path_to_mlr_for_auxents "$@" >> $outfile
-  # since set -e
+  status=$?
   num_completed=`expr $num_completed + 1`
+  if [ $status -eq 0 ]; then
+    num_passed=`expr $num_passed + 1`
+  else
+    num_failed=`expr $num_failed + 1`
+  fi
 }
 
 run_mlr_for_auxents_no_output() {
@@ -129,8 +134,13 @@ run_mlr_for_auxents_no_output() {
   echo mlr "$@" >> $outfile
   # Use path to mlr for invoking the command
   $path_to_mlr_for_auxents "$@"
-  # since set -e
+  status=$?
   num_completed=`expr $num_completed + 1`
+  if [ $status -eq 0 ]; then
+    num_passed=`expr $num_passed + 1`
+  else
+    num_failed=`expr $num_failed + 1`
+  fi
 }
 
 # Just cats a file while also announcing that fact.
@@ -146,19 +156,20 @@ mlr_expect_fail() {
   echo mlr "$@"
   echo mlr "$@" >> $outfile
   # Use path to mlr for invoking the command
-  set +e
   stderr_capture=$( $path_to_mlr "$@" 3>&1 1>&2 2>&3 >>$outfile )
   status=$?
+  num_completed=`expr $num_completed + 1`
+  if [ $status -eq 1 ]; then
+    num_passed=`expr $num_passed + 1`
+  else
+    num_failed=`expr $num_failed + 1`
+  fi
   echo "${stderr_capture}" >> $outfile
   if [ $status -ne 1 ]; then
     echo "Exit status was $status; expected 1."
     echo "Exit status was $status; expected 1." >> $outfile
   fi
-  set -e
   echo >> $outfile
-  test $status -eq 1
-  # since set -e
-  num_completed=`expr $num_completed + 1`
 }
 
 # ================================================================
@@ -268,6 +279,12 @@ run_mlr tac /dev/null
 run_mlr --opprint unsparsify $indir/abixy
 run_mlr --opprint unsparsify $indir/abixy-het
 
+run_mlr --opprint unsparsify -f nonesuch $indir/abixy-het
+run_mlr --opprint unsparsify -f a,b,i,x,y $indir/abixy-het
+run_mlr --opprint unsparsify -f aaa,bbb,xxx,iii,yyy $indir/abixy-het
+run_mlr --opprint unsparsify -f a,b,i,x,y,aaa,bbb,xxx,iii,yyy $indir/abixy-het
+run_mlr --opprint unsparsify -f a,b,i,x,y,aaa,bbb,xxx,iii,yyy then regularize $indir/abixy-het
+
 run_mlr --csv skip-trivial-records $indir/trivial-records.csv
 run_mlr --csv remove-empty-columns $indir/remove-empty-columns.csv
 
@@ -6445,6 +6462,27 @@ run_mlr --implicit-csv-header --icsv --ifs /, --opprint cut -x -f 2  $indir/mult
 run_mlr --csv -N reorder -f 1,3,2,5,4 $indir/multi-sep.csv-crlf
 run_mlr --csv -N reorder -f 5,4,3,2,1 $indir/multi-sep.csv-crlf
 
+run_mlr --icsv --otsv -N cat <<EOF
+a,b,c
+1,2,3
+4,5,6
+EOF
+
+run_mlr --ixtab --opprint -N cat <<EOF
+a 1
+b 2
+c 3
+
+a 4
+b 5
+c 6
+EOF
+
+run_mlr --icsv --pprint -N cat <<EOF
+1,2,3
+4,5,6
+EOF
+
 # ----------------------------------------------------------------
 announce HET-CSV INPUT
 

docs/manpage.rst

@@ -140,7 +140,8 @@ This is simply a copy of what you should see on running **man mlr** at a command
     	group-by group-like having-fields head histogram join label least-frequent
     	merge-fields most-frequent nest nothing put regularize remove-empty-columns
     	rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle
-    	skip-trivial-records sort stats1 stats2 step tac tail tee top uniq unsparsify
+    	skip-trivial-records sort sort-within-records stats1 stats2 step tac tail tee
+    	top uniq unsparsify
     
        FUNCTION LIST
     	+ + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~ !=~
@@ -638,7 +639,7 @@ This is simply a copy of what you should see on running **man mlr** at a command
     
            Examples:
     	 mlr filter 'log10($count) > 4.0'
-    	 mlr filter 'FNR == 2	       (second record in each file)'
+    	 mlr filter 'FNR == 2'	       (second record in each file)
     	 mlr filter 'urand() < 0.001'  (subsampling)
     	 mlr filter '$color != "blue" && $value > 4.2'
     	 mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)'
@@ -1235,6 +1236,10 @@ This is simply a copy of what you should see on running **man mlr** at a command
            which is the same as:
     	 mlr sort -f a -f b -nr x -nr y -nr z
     
+       sort-within-records
+           Usage: mlr sort-within-records [no options]
+           Outputs records sorted lexically ascending by keys.
+    
        stats1
            Usage: mlr stats1 [options]
            Computes univariate statistics for one or more given fields, accumulated across
@@ -1408,12 +1413,14 @@ This is simply a copy of what you should see on running **man mlr** at a command
        unsparsify
            Usage: mlr unsparsify [options]
            Prints records with the union of field names over all input records.
-           For field names absent in a given record but present in others, fills in
-           a value. This verb retains all input before producing any output.
+           For field names absent in a given record but present in others, fills in a
+           value. Without -f, this verb retains all input before producing any output.
     
            Options:
            --fill-with {filler string}  What to fill absent fields with. Defaults to
     				    the empty string.
+           -f {a,b,c} Specify field names to be operated on. Any other fields won't be
+    				    modified, and operation will be streaming.
     
            Example: if the input is two records, one being 'a=1,b=2' and the other
            being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
@@ -1967,7 +1974,7 @@ This is simply a copy of what you should see on running **man mlr** at a command
            (class=maps #args=3): Makes string from map key-value pairs. E.g. 'joinkv(@v[2], "=", ",")'
     
        joinv
-           (class=maps #args=2): Makes string from map keys. E.g. 'joinv(mymap, ",")'.
+           (class=maps #args=2): Makes string from map values. E.g. 'joinv(mymap, ",")'.
     
        leafcount
            (class=maps #args=1): Counts total number of terminal values in hashmap. For single-level maps,
@@ -2370,4 +2377,4 @@ This is simply a copy of what you should see on running **man mlr** at a command
     
     
     
-    				  2020-10-26			     MILLER(1)
+    				  2020-11-15			     MILLER(1)

docs/manpage.txt

@@ -130,7 +130,8 @@ OPTIONS
 	group-by group-like having-fields head histogram join label least-frequent
 	merge-fields most-frequent nest nothing put regularize remove-empty-columns
 	rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle
-	skip-trivial-records sort stats1 stats2 step tac tail tee top uniq unsparsify
+	skip-trivial-records sort sort-within-records stats1 stats2 step tac tail tee
+	top uniq unsparsify
 
    FUNCTION LIST
 	+ + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~ !=~
@@ -628,7 +629,7 @@ VERBS
 
        Examples:
 	 mlr filter 'log10($count) > 4.0'
-	 mlr filter 'FNR == 2	       (second record in each file)'
+	 mlr filter 'FNR == 2'	       (second record in each file)
 	 mlr filter 'urand() < 0.001'  (subsampling)
 	 mlr filter '$color != "blue" && $value > 4.2'
 	 mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)'
@@ -1225,6 +1226,10 @@ VERBS
        which is the same as:
 	 mlr sort -f a -f b -nr x -nr y -nr z
 
+   sort-within-records
+       Usage: mlr sort-within-records [no options]
+       Outputs records sorted lexically ascending by keys.
+
    stats1
        Usage: mlr stats1 [options]
        Computes univariate statistics for one or more given fields, accumulated across
@@ -1398,12 +1403,14 @@ VERBS
    unsparsify
        Usage: mlr unsparsify [options]
        Prints records with the union of field names over all input records.
-       For field names absent in a given record but present in others, fills in
-       a value. This verb retains all input before producing any output.
+       For field names absent in a given record but present in others, fills in a
+       value. Without -f, this verb retains all input before producing any output.
 
        Options:
        --fill-with {filler string}  What to fill absent fields with. Defaults to
 				    the empty string.
+       -f {a,b,c} Specify field names to be operated on. Any other fields won't be
+				    modified, and operation will be streaming.
 
        Example: if the input is two records, one being 'a=1,b=2' and the other
        being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
@@ -1957,7 +1964,7 @@ FUNCTIONS FOR FILTER/PUT
        (class=maps #args=3): Makes string from map key-value pairs. E.g. 'joinkv(@v[2], "=", ",")'
 
    joinv
-       (class=maps #args=2): Makes string from map keys. E.g. 'joinv(mymap, ",")'.
+       (class=maps #args=2): Makes string from map values. E.g. 'joinv(mymap, ",")'.
 
    leafcount
        (class=maps #args=1): Counts total number of terminal values in hashmap. For single-level maps,
@@ -2360,4 +2367,4 @@ SEE ALSO
 
 
 
-				  2020-10-26			     MILLER(1)
+				  2020-11-15			     MILLER(1)

docs/mlr.1

@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2020-10-26
+.\"      Date: 2020-11-15
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2020-10-26" "\ \&" "\ \&"
+.TH "MILLER" "1" "2020-11-15" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -173,7 +173,8 @@ output separator to the given value.
  group-by group-like having-fields head histogram join label least-frequent
  merge-fields most-frequent nest nothing put regularize remove-empty-columns
  rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle
- skip-trivial-records sort stats1 stats2 step tac tail tee top uniq unsparsify
+ skip-trivial-records sort sort-within-records stats1 stats2 step tac tail tee
+ top uniq unsparsify
 .fi
 .if n \{\
 .RE
@@ -829,7 +830,7 @@ Use # to comment to end of line.
 
 Examples:
   mlr filter 'log10($count) > 4.0'
-  mlr filter 'FNR == 2          (second record in each file)'
+  mlr filter 'FNR == 2'         (second record in each file)
   mlr filter 'urand() < 0.001'  (subsampling)
   mlr filter '$color != "blue" && $value > 4.2'
   mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)'
@@ -1602,6 +1603,16 @@ which is the same as:
 .fi
 .if n \{\
 .RE
+.SS "sort-within-records"
+.if n \{\
+.RS 0
+.\}
+.nf
+Usage: mlr sort-within-records [no options]
+Outputs records sorted lexically ascending by keys.
+.fi
+.if n \{\
+.RE
 .SS "stats1"
 .if n \{\
 .RS 0
@@ -1827,12 +1838,14 @@ Options:
 .nf
 Usage: mlr unsparsify [options]
 Prints records with the union of field names over all input records.
-For field names absent in a given record but present in others, fills in
-a value. This verb retains all input before producing any output.
+For field names absent in a given record but present in others, fills in a
+value. Without -f, this verb retains all input before producing any output.
 
 Options:
 --fill-with {filler string}  What to fill absent fields with. Defaults to
                              the empty string.
+-f {a,b,c} Specify field names to be operated on. Any other fields won't be
+                             modified, and operation will be streaming.
 
 Example: if the input is two records, one being 'a=1,b=2' and the other
 being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
@@ -3318,7 +3331,7 @@ MT_STRING). For debug.
 .RS 0
 .\}
 .nf
-(class=maps #args=2): Makes string from map keys. E.g. 'joinv(mymap, ",")'.
+(class=maps #args=2): Makes string from map values. E.g. 'joinv(mymap, ",")'.
 .fi
 .if n \{\
 .RE

docs/reference-dsl.rst

@@ -4063,7 +4063,7 @@ joinv
 
 ::
 
-    joinv (class=maps #args=2): Makes string from map keys. E.g. 'joinv(mymap, ",")'.
+    joinv (class=maps #args=2): Makes string from map values. E.g. 'joinv(mymap, ",")'.
 
 
 

docs/reference-verbs.rst

@@ -793,7 +793,7 @@ filter
     
     Examples:
       mlr filter 'log10($count) > 4.0'
-      mlr filter 'FNR == 2          (second record in each file)'
+      mlr filter 'FNR == 2'         (second record in each file)
       mlr filter 'urand() < 0.001'  (subsampling)
       mlr filter '$color != "blue" && $value > 4.2'
       mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)'
@@ -3273,12 +3273,14 @@ unsparsify
     $ mlr unsparsify --help
     Usage: mlr unsparsify [options]
     Prints records with the union of field names over all input records.
-    For field names absent in a given record but present in others, fills in
-    a value. This verb retains all input before producing any output.
+    For field names absent in a given record but present in others, fills in a
+    value. Without -f, this verb retains all input before producing any output.
     
     Options:
     --fill-with {filler string}  What to fill absent fields with. Defaults to
                                  the empty string.
+    -f {a,b,c} Specify field names to be operated on. Any other fields won't be
+                                 modified, and operation will be streaming.
     
     Example: if the input is two records, one being 'a=1,b=2' and the other
     being 'b=3,c=4', then the output is the two records 'a=1,b=2,c=' and
@@ -3320,3 +3322,27 @@ Examples:
     1       missing 2       missing 3       missing
     missing missing 1       missing missing 2
 
+::
+
+    $ mlr --ijson --opprint unsparsify -f a,b,u data/sparse.json
+    a b v u
+    1 2 3 -
+    
+    u b a
+    1 2 -
+    
+    a v x b u
+    1 2 3 - -
+    
+    v w a b u
+    1 2 - - -
+
+::
+
+    $ mlr --ijson --opprint unsparsify -f a,b,u,v,w,x then regularize data/sparse.json
+    a b v u w x
+    1 2 3 - - -
+    - 2 - 1 - -
+    1 - 2 - - 3
+    - - 1 - 2 -
+

docs/reference-verbs.rst.in

@@ -1268,3 +1268,11 @@ POKI_RUN_COMMAND{{mlr --ijson --opprint unsparsify data/sparse.json}}HERE
 
 POKI_RUN_COMMAND{{mlr --ijson --opprint unsparsify --fill-with missing data/sparse.json}}HERE
 
+::
+
+POKI_RUN_COMMAND{{mlr --ijson --opprint unsparsify -f a,b,u data/sparse.json}}HERE
+
+::
+
+POKI_RUN_COMMAND{{mlr --ijson --opprint unsparsify -f a,b,u,v,w,x then regularize data/sparse.json}}HERE
+

docs/reference.rst

@@ -789,8 +789,8 @@ Examples:
        group-by group-like having-fields head histogram join label least-frequent
        merge-fields most-frequent nest nothing put regularize remove-empty-columns
        rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle
-       skip-trivial-records sort stats1 stats2 step tac tail tee top uniq
-       unsparsify
+       skip-trivial-records sort sort-within-records stats1 stats2 step tac tail
+       tee top uniq unsparsify
     
     Functions for the filter and put verbs:
        + + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~

go/.gitignore

@@ -6,3 +6,11 @@ mkv.ast
 s
 mlrgo
 gg
+
+parser-experiments/one/src/github.com
+parser-experiments/one/src/experimental
+parser-experiments/one/main
+
+parser-experiments/two/src/github.com
+parser-experiments/two/src/experimental
+parser-experiments/two/main

go/README.md

@@ -1,10 +1,19 @@
 # Status of the Go port
 
-* This will be a full Go port of [Miller](https://miller.readthedocs.io/). Things are very rough and very iterative and very incomplete. I don't have a firm timeline but I suspect it will take a few more months of late-evening/spare-time work.
+* This will be a full Go port of [Miller](https://miller.readthedocs.io/). Things are currently rough and iterative and incomplete. I don't have a firm timeline but I suspect it will take a few more months of late-evening/spare-time work.
 * The released Go port will become Miller 6.0. As noted below, this will be a win both at the source-code level, and for users of Miller.
 * I hope to retain backward compatibility at the command-line level as much as possible.
 * In the meantime I will still keep fixing bugs, doing some features, etc. in C on Miller 5.x -- in the near term, support for Miller's C implementation continues as before.
 
+# Port-completion criteria
+
+* `reg-test/run -g` completes -- either completing/fixing the C/Go source-code discrepancies, or accepting the changes as backward incomptabilities
+* Double-checking all Miller issues ever, in case I fixed/implemented something but didn't have reg-test coverage
+* All `TODO`/`xxx` comments in Go, BNF source code, and case-files are resolved
+* Release notes including Go-only features, and C/Go backward-incompatibilities
+* Docs updated at https://miller.readthedocs.io/ (`../docs`)
+* Equivalent of `./configure`, whatever that turns out to be
+
 # Trying out the Go port
 
 * Caveat: *lots* of things present in the C implementation are currently missing in the Go implementation. So if something doesn't work, it's almost certainly because it doesn't work *yet*.
@@ -24,8 +33,8 @@
 # Benefits of porting to Go
 
 * The [lack of a streaming (record-by-record) JSON reader](http://johnkerl.org/miller/doc/file-formats.html#JSON_non-streaming) in the C implementation ([issue 99](https://github.com/johnkerl/miller/issues/99)) is immediately solved in the Go implementation.
-* Previously [arrays were not supported in the DSL](http://johnkerl.org/miller/doc/file-formats.html#Arrays); now they will be.
-* [Flattening nested map structures to output records](http://johnkerl.org/miller/doc/file-formats.html#Formatting_JSON_options) was clumsy. Now, Miller will be a full JSON-to-JSON processor, if your inputs and outputs are both JSON; JSON input and output will be idiomatic.
+* In the C implementation, [arrays were not supported in the DSL](http://johnkerl.org/miller/doc/file-formats.html#Arrays); in the Go implementation they are.
+* [Flattening nested map structures to output records](http://johnkerl.org/miller/doc/file-formats.html#Formatting_JSON_options) was clumsy. Now, Miller will be a JSON-to-JSON processor, if your inputs and outputs are both JSON; JSON input and output will be idiomatic.
 * The quoted-DKVP feature from [issue 266](https://github.com/johnkerl/miller/issues/266) will be easily addressed.
 * String/number-formatting issues in [issue 211](https://github.com/johnkerl/miller/issues/211), [issue 178](https://github.com/johnkerl/miller/issues/178), [issue 151](https://github.com/johnkerl/miller/issues/151), and [issue 259](https://github.com/johnkerl/miller/issues/259) will be fixed during the Go port.
 * I think some DST/timezone issues such as [issue 359](https://github.com/johnkerl/miller/issues/359) will be easier to fix using the Go datetime library than using the C datetime library

go/accept-case

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# See ./reg-test/run
+for arg; do
+  src=output-reg-test/$arg.out
+  if [ -f $src ]; then
+    cp $src reg-test/expected
+  elif [ -f $arg ]; then
+    src=output-reg-test/$(basename $arg).out
+    cp $src reg-test/expected
+  else
+    echo "Cannot find source $arg" 1>&2
+    exit 1
+  fi
+done

go/build

@@ -10,4 +10,4 @@ set -euo pipefail
 
 ./build-go
 ./tester
-./check $verbose
+./reg-test/run -n $verbose

go/build-dsl

@@ -33,4 +33,7 @@ fi
 #   src/miller/parsing/token/
 #   src/miller/parsing/util/
 
+# Override GOCC codegen with customized error handling
+cp src/miller/parsing/errors.go.template src/miller/parsing/errors/errors.go
+
 for x in src/miller/parsing/*/*.go; do gofmt -w $x; done

go/build-dsl-foo

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+us=$(basename $0)
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 {.bnf file}" 1>&2
+  exit 1
+fi
+bnf="$1"
+dir=./parserfoo
+verbose="true"
+#verbose="false"
+
+mkdir -p $dir
+
+export GOPATH=$(pwd)
+
+# Build the bin/gocc executable:
+go get github.com/goccmack/gocc
+
+rm -f $dir/*.txt
+if [ "$verbose" = "true" ]; then
+  lr1="$dir/LR1_conflicts.txt"
+  bin/gocc -v -o $dir $bnf || expand -2 $lr1
+else
+  bin/gocc -o $dir $bnf
+fi
+
+# Code-gen directories:
+#   $dir/errors/
+#   $dir/lexer/
+#   $dir/parser/
+#   $dir/token/
+#   $dir/util/

go/build-go

@@ -3,6 +3,10 @@
 set -euo pipefail
 
 export GOPATH=$(pwd)
+
+# Override GOCC codegen with customized error handling
+cp src/miller/parsing/errors.go.template src/miller/parsing/errors/errors.go
+
 go build mlr.go
 cp mlr mlrgo
 echo "Compile OK"

go/case-diff

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# See ./reg-test/run
+diff_context_flag=""
+for arg; do
+  if [ "$arg" = "-C5" ]; then
+    diff_context_flag="$arg"
+  else
+    expfile=./reg-test/expected/$arg.out
+    outfile=./output-reg-test/$arg.out
+    diff -a -I '^mlr' -I '^Miller:' -I '^cat' $expfile $outfile
+  fi
+done

go/cases-to-do.txt

@@ -0,0 +1,240 @@
+================================================================
+BUG:
+
+rrv ./reg-test/cases/case-c-dsl-context-specific-validation.sh
+* subr and more ... also needs check for filter statement within filter commands
+
+rrv ./reg-test/cases/case-c-dsl-forbind-typedecl.sh
+rrv ./reg-test/cases/case-c-dsl-localvar-typedecl.sh
+rrv ./reg-test/cases/case-c-dsl-local-map-variable-typedecl.sh
+* str s = "a"; s = 3 doesn't trigger -- needs gate-retention ...
+
+rrv ./reg-test/cases/case-c-dsl-for-map-depth.sh
+* Miller: looped-over item is not a map or array.
+  Needs to treat non-mappable as zero-pass case -- ?
+
+rrv ./reg-test/cases/case-c-dsl-null-empty-handling.sh
+* "conditional expression did not evaluate to boolean" -- versus @nonesuch {} P/A in C ...
+
+rrv ./reg-test/cases/case-c-dsl-triple-for-loops.sh
+* Needs to do this:
+  "Continuation statements are split into the final boolean, and the statements before (if any)."
+
+rrv ./reg-test/cases/case-c-label.sh
+* needs uniqueness-checker
+
+================================================================
+MULTIPLE ISSUES:
+
+rrv ./reg-test/cases/case-c-dsl-functional-tests.sh
+* bug mlr filter $nosuchfield>.3 ./reg-test/input/abixy
+* ofmt
+* env
+-> split up this big case-file
+
+* typeof prints different things -- regularize ...
+
+* 'x=' -> MT_EMPTY vs MT_STRING
+
+================================================================
+LF/CRLF ETC.
+
+rrv ./reg-test/cases/case-c-multi-character-ixs.sh
+rrv ./reg-test/cases/case-c-implicit-header-csv-input.sh
+rrv ./reg-test/cases/case-c-missing-final-lf.sh
+
+================================================================
+ENV
+
+rrv ./reg-test/cases/case-c-dsl-functional-tests.sh
+
+================================================================
+PUT -S / -F / -A
+
+rrv ./reg-test/cases/case-c-dsl-type-inference.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-for-srec-loops.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-type-inference.sh
+rrv ./reg-test/cases/case-c-dsl-pythonic-division.sh
+
+rrv -C20 ./reg-test/cases/case-c-dsl-local-map-variable-write-read.sh
+rrv -C20 ./reg-test/cases/case-c-dsl-local-map-variable-write-read.sh
+* put -a
+
+================================================================
+ARRAY/MAP
+
+rrv -C5 ./reg-test/cases/case-c-dsl-for-oosvar-loops.sh
+rrv ./reg-test/cases/case-c-dsl-mapvar-assignments.sh
+
+rrv ./reg-test/cases/case-c-dsl-split-join.sh
+rrv -C5 ./reg-test/cases/case-c-dsl-asserting.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-length-depth-leafcount.sh
+* int map keys
+
+rrv -C1 ./reg-test/cases/case-c-dsl-mapvar-for-loops.sh
+* int map keys
+
+rrv ./reg-test/cases/case-c-dsl-haskey.sh
+* haskey({3:4}, 3)
+
+rrv ./reg-test/cases/case-c-dsl-extended-typeof.sh
+
+rrv ./reg-test/cases/case-c-dsl-is-predicates.sh
+* {1:2} is empty-map in Go; should step up (populate) or step down (error)
+
+  func (this *Mlrval) MapPut(key *Mlrval, value *Mlrval) {
+      if this.mvtype != MT_MAP {
+          // TODO: need to be careful about semantics here.
+          // Silent no-ops are not good UX ...
+          return
+      }
+      if key.mvtype != MT_STRING {
+          // TODO: need to be careful about semantics here.
+          // Silent no-ops are not good UX ...
+          return
+      }
+      this.mapval.PutCopy(&key.printrep, value)
+  }
+
+================================================================
+SUBR
+
+rrv -C1 ./reg-test/cases/case-c-dsl-context-specific-validation.sh
+* subr and more ... also needs check for filter statement within filter commands
+
+rrv ./reg-test/cases/case-c-dsl-user-defined-functions-and-subroutines.sh
+rrv ./reg-test/cases/case-c-dsl-user-defined-subroutines.sh
+rrv ./reg-test/cases/case-c-dsl-trailing-commas.sh
+rrv ./reg-test/cases/case-c-dsl-argpass-typedecl.sh
+rrv ./reg-test/cases/case-c-dsl-multipart-scripts.sh
+
+================================================================
+FORMATTING?
+
+rrv ./reg-test/cases/case-c-dsl-forbind-typedecl.sh
+rrv ./reg-test/cases/case-c-dsl-indirect-srec-assignments.sh
+rrv ./reg-test/cases/case-c-dsl-operator-associativity.sh
+rrv ./reg-test/cases/case-c-dsl-operator-precedence.sh
+rrv ./reg-test/cases/case-c-dsl-typed-overlay.sh
+rrv ./reg-test/cases/case-c-localvar-unsets.sh
+
+================================================================
+PARTIALLY IMPLEMENTED:
+
+rrv ./reg-test/cases/case-c-clean-whitespace.sh
+rrv ./reg-test/cases/case-c-cut.sh
+rrv ./reg-test/cases/case-c-dsl-indirect-oosvar-assignments.sh
+rrv ./reg-test/cases/case-c-dsl-scientific-notation.sh
+rrv ./reg-test/cases/case-c-format-values.sh
+rrv ./reg-test/cases/case-c-implicit-header-csv-input.sh
+rrv ./reg-test/cases/case-c-localvar-unsets.sh
+rrv ./reg-test/cases/case-c-rfc-csv.sh
+rrv ./reg-test/cases/case-c-space-padding.sh
+rrv ./reg-test/cases/case-c-utf8-alignment.sh
+
+================================================================
+PPRINT --BARRED --RIGHT
+
+rrv ./reg-test/cases/case-c-barred-pprint.sh
+
+================================================================
+BACKWARD INCOMPATIBILITIES:
+
+rrv ./reg-test/cases/case-c-dsl-print.sh
+rrv ./reg-test/cases/case-c-dsl-substr.sh
+
+rrv -C20 ./reg-test/cases/case-c-dsl-user-defined-functions.sh
+* array 1-up
+
+================================================================
+NEED EMITP/MULTI-EMIT
+
+rrv ./reg-test/cases/case-c-dsl-absent-empty.sh
+rrv ./reg-test/cases/case-c-dsl-break-continue.sh
+rrv ./reg-test/cases/case-c-dsl-lashed-emitp-singles.sh
+rrv ./reg-test/cases/case-c-dsl-mapvar-emit.sh
+rrv ./reg-test/cases/case-c-dsl-oosvar-assignments.sh
+
+rrv -C20 ./reg-test/cases/case-c-dsl-type-predicates.sh
+* multi-emit
+
+================================================================
+NOT IMPLEMENTED YET:
+
+rrv ./reg-test/cases/case-c-format-conversion-keystroke-savers.sh
+* needs --ipprint
+
+rrv ./reg-test/cases/case-c-dsl-filter-pattern-action.sh
+* regex
+
+rrv -C1 ./reg-test/cases/case-c-dsl-filter-pattern-action.sh
+* regexi
+
+rrv ./reg-test/cases/case-c-het-pprint-input.sh
+* new pprint reader not done yet
+
+rrv ./reg-test/cases/case-c-dsl-oosvars.sh
+* uses mlr step
+
+rrv ./reg-test/cases/case-c-auxents.sh
+rrv ./reg-test/cases/case-c-bar.sh
+rrv ./reg-test/cases/case-c-compressed-input.sh
+rrv ./reg-test/cases/case-c-count-distinct.sh
+rrv ./reg-test/cases/case-c-csv-rs-environment-defaults.sh
+rrv ./reg-test/cases/case-c-dsl-gmt-date-time-functions.sh
+rrv ./reg-test/cases/case-c-dsl-local-date-time-functions.sh
+rrv ./reg-test/cases/case-c-dsl-mapvars-udfs-subroutines.sh
+rrv ./reg-test/cases/case-c-dsl-parameterized-emit.sh
+rrv ./reg-test/cases/case-c-dsl-positional-indexing.sh
+rrv ./reg-test/cases/case-c-dsl-prefixed-unprefixed-emit.sh
+rrv ./reg-test/cases/case-c-dsl-redirects.sh
+rrv ./reg-test/cases/case-c-dsl-regex-captures.sh
+rrv ./reg-test/cases/case-c-dsl-regex-matching.sh
+rrv ./reg-test/cases/case-c-dsl-stack-allocation.sh
+rrv ./reg-test/cases/case-c-format-values.sh
+rrv ./reg-test/cases/case-c-fraction.sh
+rrv ./reg-test/cases/case-c-having-fields.sh
+rrv ./reg-test/cases/case-c-head-early-out.sh
+rrv ./reg-test/cases/case-c-histogram.sh
+rrv ./reg-test/cases/case-c-in-place-processing.sh
+rrv ./reg-test/cases/case-c-int-float-stats1-step1.sh
+rrv ./reg-test/cases/case-c-join-mixed-format.sh
+rrv ./reg-test/cases/case-c-join-prepipe.sh
+rrv ./reg-test/cases/case-c-join.sh
+rrv ./reg-test/cases/case-c-json-io.sh
+rrv ./reg-test/cases/case-c-lf-crlf-and-autodetect.sh
+rrv ./reg-test/cases/case-c-markdown-output.sh
+rrv ./reg-test/cases/case-c-merge-fields.sh
+rrv ./reg-test/cases/case-c-most-least-frequent.sh
+rrv ./reg-test/cases/case-c-nest.sh
+rrv ./reg-test/cases/case-c-null-field-input.sh
+rrv ./reg-test/cases/case-c-number-formatting.sh
+rrv ./reg-test/cases/case-c-rename-regex.sh
+rrv ./reg-test/cases/case-c-repeat.sh
+rrv ./reg-test/cases/case-c-reshape.sh
+rrv ./reg-test/cases/case-c-seqgen.sh
+rrv ./reg-test/cases/case-c-skip-pass-comments.sh
+rrv ./reg-test/cases/case-c-stats1-percentile-interpolation.sh
+rrv ./reg-test/cases/case-c-stats1-regexed-field-names.sh
+rrv ./reg-test/cases/case-c-stats1.sh
+rrv ./reg-test/cases/case-c-stats2.sh
+rrv ./reg-test/cases/case-c-step.sh
+rrv ./reg-test/cases/case-c-top.sh
+rrv ./reg-test/cases/case-c-uniq.sh
+rrv ./reg-test/cases/case-c-utf8-bom.sh
+
+================================================================
+JUST FORMATTING NOW??
+
+rrv ./reg-test/cases/case-c-dsl-type-inference.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-for-srec-loops.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-type-inference.sh
+rrv ./reg-test/cases/case-c-dsl-pythonic-division.sh
+
+rrv -C20 ./reg-test/cases/case-c-dsl-local-map-variable-write-read.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-local-map-variable-write-read.sh
+
+================================================================
+rrv -C20 ./reg-test/cases/case-c-dsl-empty-statements.sh
+rrv -C1 ./reg-test/cases/case-c-dsl-empty-statements.sh
+================================================================

go/check

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# ================================================================
-# COMMAND-LINE TESTS
-# ================================================================
-
-runner="bash"
-if [ "$1" = "-v" ]; then
-  set -x
-  runner="bash -x"
-fi
-
-set -euo pipefail
-
-$runner u/try-io    > u/try-io.out
-$runner u/try-chain > u/try-chain.out
-$runner u/try-verbs > u/try-verbs.out
-$runner u/try-help  > u/try-help.out
-$runner u/try-parse > u/try-parse.out
-$runner u/try-cst   > u/try-cst.out
-
-git diff \
-  u/try-io.out \
-  u/try-chain.out \
-  u/try-verbs.out \
-  u/try-parse.out \
-  u/try-cst.out
-
-echo "Test OK"

go/list-todo-cases

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./reg-test/run -g -o | grep '^FAIL ' | tee ooo

go/mlr.go

@@ -46,17 +46,11 @@ func main() {
 
 		fmt.Fprintf(os.Stderr, "CPU profile started.\n")
 		defer fmt.Fprintf(os.Stderr, "CPU profile finished.\n")
-		//	c := make(chan os.Signal)
-		//	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-		//	go func() {
-		//		<-c
-		//		pprof.StopCPUProfile()
-		//		os.Exit(0)
-		//	}()
 	}
-	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
+	//  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 	// Start of Miller main per se
+
 	options, recordMappers, err := cli.ParseCommandLine(os.Args)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, os.Args[0], ": ", err)

go/parser-experiments/.vimrc

@@ -0,0 +1 @@
+map \f :w<C-m>:!clear;echo Building ...; echo; build; echo; main<C-m>

go/parser-experiments/README.md

@@ -0,0 +1,2 @@
+Running the GOCC parser-generator for the full Miller grammar takes a few minutes. That's
+a bit painful for experimentation; hence this.

go/parser-experiments/one/build

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# ----------------------------------------------------------------
+# Setup
+us=$(basename $0)
+set -euo pipefail
+
+verbose="false"
+
+if [ $# -eq 1 ]; then
+  bnf="$1"
+elif [ $# -eq 2 ]; then
+  if [ "$1" = "-v" ]; then
+    verbose="true"
+  else
+    echo "Usage: $0 {.bnf file}" 1>&2
+    exit 1
+  fi
+  bnf="$2"
+else
+  echo "Usage: $0 {.bnf file}" 1>&2
+  exit 1
+fi
+
+dir=src/experimental
+mkdir -p $dir
+
+# ----------------------------------------------------------------
+# Run the parser-generator
+export GOPATH=$(pwd)
+
+# Build the bin/gocc executable:
+go get github.com/goccmack/gocc
+
+rm -f $dir/*.txt
+if [ "$verbose" = "true" ]; then
+  lr1="$dir/LR1_conflicts.txt"
+  # The -o specifies the package name within the autogen
+  bin/gocc -v -o $dir $bnf || expand -2 $lr1
+else
+  bin/gocc -o $dir $bnf
+fi
+
+echo "Parser-autogen OK"
+
+# Code-gen directories:
+#   $dir/errors/
+#   $dir/lexer/
+#   $dir/parser/
+#   $dir/token/
+#   $dir/util/
+
+# ----------------------------------------------------------------
+# Compile the main and the parser-autogen
+
+go build main.go
+echo "Compile OK"

go/parser-experiments/one/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"experimental/lexer"
+	"experimental/parser"
+)
+
+func parseOne(input string) {
+	theLexer := lexer.NewLexer([]byte(input))
+	theParser := parser.NewParser()
+	_, err := theParser.Parse(theLexer)
+
+	green := "\033[32;01m"
+	red := "\033[31;01m"
+	textdefault := "\033[0m"
+
+	if err != nil {
+		//fmt.Println(err)
+		fmt.Printf("%sFail%s %s\n", red, textdefault, input)
+	} else {
+		fmt.Printf("%sOK%s   %s\n", green, textdefault, input)
+	}
+}
+
+func main() {
+	if len(os.Args) == 1 {
+
+		fmt.Println("EXPECT OK")
+		goods := []string{
+			"",
+			";",
+			";;",
+			"x",
+			"x;x",
+			"x;x;x",
+			"x;x;x;x",
+			"x;",
+			"x;;",
+			";x",
+			";;x",
+			"x ; {}",
+			"{} ; x",
+			"{} x",
+			"{ x }",
+			"{ x; x }",
+			"x; { x; x }",
+			"{ x; x } x",
+			"{ x; x } ; x",
+			"{};{}",
+			"{} {}",
+		}
+		for _, input := range goods {
+			parseOne(input)
+		}
+
+		fmt.Println()
+		fmt.Println("EXPECT FAIL")
+		bads := []string{
+			"x x",
+			"x {}",
+		}
+		for _, input := range bads {
+			parseOne(input)
+		}
+
+	} else {
+		for _, arg := range os.Args[1:] {
+			parseOne(arg)
+		}
+	}
+}

go/parser-experiments/one/run

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 {.bnf file} [expressions]" 1>&2
+  exit 1
+fi
+bnf="$1"
+shift
+./build $bnf && echo && main "$@"

go/parser-experiments/one/semi1.bnf

@@ -0,0 +1,29 @@
+!whitespace : ' ' | '\t' | '\n' | '\r' ;
+
+Root
+  : StatementBlock
+;
+
+StatementBlock
+  : ";"
+  | BracelessStatement
+  | ";" BracelessStatement
+  | BracefulStatement
+  | BracelessStatement ";" NonEmptyStatementBlock
+  | BracefulStatement      NonEmptyStatementBlock
+;
+
+NonEmptyStatementBlock
+  : BracelessStatement
+  | BracelessStatement ";" StatementBlock
+  | BracefulStatement
+  | BracefulStatement      StatementBlock
+;
+
+BracelessStatement
+  : "x"
+;
+
+BracefulStatement
+  : "{" StatementBlock "}"
+;

go/parser-experiments/one/semi2.bnf

@@ -0,0 +1,25 @@
+!whitespace : ' ' | '\t' | '\n' | '\r' ;
+
+Root
+  : StatementBlock
+;
+
+StatementBlock
+  : empty
+  | ";" StatementBlock
+  | BracelessStatement
+  | BracefulStatement
+  | BracelessStatement ";" StatementBlock
+  | BracefulStatement  ";" BracefulStatement
+  | BracefulStatement      BracefulStatement
+  | BracefulStatement  ";" BracelessStatement
+  | BracefulStatement      BracelessStatement
+;
+
+BracelessStatement
+  : "x"
+;
+
+BracefulStatement
+  : "{" StatementBlock "}"
+;

go/parser-experiments/two/build

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# ----------------------------------------------------------------
+# Setup
+us=$(basename $0)
+set -euo pipefail
+
+verbose="false"
+
+if [ $# -eq 1 ]; then
+  bnf="$1"
+elif [ $# -eq 2 ]; then
+  if [ "$1" = "-v" ]; then
+    verbose="true"
+  else
+    echo "Usage: $0 {.bnf file}" 1>&2
+    exit 1
+  fi
+  bnf="$2"
+else
+  echo "Usage: $0 {.bnf file}" 1>&2
+  exit 1
+fi
+
+dir=src/experimental
+mkdir -p $dir
+
+# ----------------------------------------------------------------
+# Run the parser-generator
+export GOPATH=$(pwd)
+
+# Build the bin/gocc executable:
+go get github.com/goccmack/gocc
+
+rm -f $dir/*.txt
+if [ "$verbose" = "true" ]; then
+  lr1="$dir/LR1_conflicts.txt"
+  # The -o specifies the package name within the autogen
+  bin/gocc -v -o $dir $bnf || expand -2 $lr1
+else
+  bin/gocc -o $dir $bnf
+fi
+
+echo "Parser-autogen OK"
+
+# Code-gen directories:
+#   $dir/errors/
+#   $dir/lexer/
+#   $dir/parser/
+#   $dir/token/
+#   $dir/util/
+
+# ----------------------------------------------------------------
+# Compile the main and the parser-autogen
+
+go build main.go
+echo "Compile OK"

go/parser-experiments/two/main.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"dsl"
+	"experimental/lexer"
+	"experimental/parser"
+)
+
+const GREEN = "\033[32;01m"
+const RED = "\033[31;01m"
+const TEXTDEFAULT = "\033[0m"
+
+func parseOne(input string) bool {
+	theLexer := lexer.NewLexer([]byte(input))
+	theParser := parser.NewParser()
+	iast, err := theParser.Parse(theLexer)
+	if err == nil {
+		fmt.Printf("%sOK%s   %s\n", GREEN, TEXTDEFAULT, input)
+		iast.(*dsl.AST).Print()
+		fmt.Println()
+		return true
+	} else {
+		//fmt.Println(err)
+		fmt.Printf("%sFail%s %s\n", RED, TEXTDEFAULT, input)
+		fmt.Println()
+		return false
+	}
+}
+
+func main() {
+	if len(os.Args) == 1 {
+		ok := true
+
+		fmt.Println("----------------------------------------------------------------")
+		fmt.Println("EXPECT OK")
+		goods := []string{
+			"",
+			";",
+			";;",
+			";;;",
+			"x",
+			"x;x",
+			"x;x;x",
+			"x;x;x;x",
+			"x;",
+			"x;;",
+			";x",
+			";;x",
+			"x ; {}",
+			"{} ; x",
+			"{} x",
+			"{ x }",
+			"{ x; x }",
+			"x; { x; x }",
+			"{ x; x } x",
+			"{ x; x } ; x",
+			"{};{}",
+			"{} {}",
+			"{} {};",
+			"{};{};",
+			"{} {} {}",
+			"{};{}; {}",
+			"{} ; ; {}",
+			"x; x;",
+			"{x; x;}",
+			"{x; x;} x",
+			"{x; x;} x;",
+			"{x; x;} {};",
+			"{} x; {}",
+			"{} x; {};",
+			"{} x; x; {}",
+			"{} x; x; {};",
+			"{} x; x; x; {}",
+			"{} x; x; x; {};",
+			"{} {} ;;; {;;} x; x; x; x; x; {}",
+		}
+		for _, input := range goods {
+			if parseOne(input) == false {
+				ok = false
+			}
+		}
+
+		fmt.Println()
+		fmt.Println("----------------------------------------------------------------")
+		fmt.Println("EXPECT FAIL")
+		bads := []string{
+			"x x",
+			"x {}",
+		}
+		for _, input := range bads {
+			if parseOne(input) == true {
+				ok = false
+			}
+		}
+
+		fmt.Println()
+		fmt.Println("----------------------------------------------------------------")
+		if ok {
+			fmt.Printf("%sALL AS EXPECTED%s\n", GREEN, TEXTDEFAULT)
+		} else {
+			fmt.Printf("%sNOT ALL AS EXPECTED%s\n", RED, TEXTDEFAULT)
+			os.Exit(1)
+		}
+
+	} else {
+		for _, arg := range os.Args[1:] {
+			parseOne(arg)
+		}
+	}
+}

go/parser-experiments/two/mktags

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# ~/.ctags needs to look like
+# --langdef=Go
+# --langmap=Go:.go
+# --regex-Go=/func([ \t]+\([^)]+\))?[ \t]+([a-zA-Z0-9_]+)/\2/d,func/
+# --regex-Go=/var[ \t]+([a-zA-Z_][a-zA-Z0-9_]+)/\1/d,var/
+# --regex-Go=/type[ \t]+([a-zA-Z_][a-zA-Z0-9_]+)/\1/d,type/
+#
+# See also https://stackoverflow.com/questions/8204367/ctag-database-for-go
+
+ctags -f gosource.tags -R `pwd`
+mv gosource.tags tags

go/parser-experiments/two/run

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 {.bnf file} [expressions]" 1>&2
+  exit 1
+fi
+bnf="$1"
+shift
+./build $bnf && echo && main "$@"

go/parser-experiments/two/semi3.bnf

@@ -0,0 +1,93 @@
+// ================================================================
+// LEXER
+
+!whitespace : ' ' | '\t' | '\n' | '\r' ;
+
+// ================================================================
+// IMPORT
+
+<< import "dsl" >>
+
+// ================================================================
+// PARSER
+
+// ----------------------------------------------------------------
+Root
+  : StatementBlock
+   << dsl.NewAST($0) >>
+;
+
+// ================================================================
+StatementBlock
+
+  // Empty statement. This allows for 'mlr put ""', as well as repeated semicolons.
+  : empty
+    << dsl.NewASTNodeZary(nil, dsl.NodeTypeStatementBlock) >>
+
+  | NonEmptyStatementBlock
+    << dsl.Wrap($0) >>
+;
+
+// ----------------------------------------------------------------
+NonEmptyStatementBlock
+  // ---------------------- Terminal rules
+
+  // Things not ending in a curly brace, like assignments -- and also do-while.
+  : BracelessStatement
+    << dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStatementBlock) >>
+
+  // Things ending in a curly brace, like for/do/while, begin/end, and pattern-action blocks
+  | BracefulStatement
+    << dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStatementBlock) >>
+
+  // ---------------------- Recursive rules
+
+  // So statements can start with a semicolon
+  | ";" StatementBlock
+    << dsl.Wrap($1) >>
+
+  // Normal case for sequential statements like '$x=1; $y=2'
+  | BracelessStatement ";" StatementBlock
+    <<dsl.PrependChild($2, $0) >>
+
+  // For 'begin {...} ; $x=1'
+  | BracefulStatement ";" StatementBlock
+    <<dsl.PrependChild($2, $0) >>
+
+  // These are for things like 'begin {...} begin {...} ...' -- where people
+  // shouldn't have to put semicolons after the closing curly braces.
+  //
+  // We get LR-1 conflicts with the following, so we need a pair of more
+  // explicit lookahead-by-more production rules instead. (By using ternaries
+  // we are effectively getting lookahead-by-two.)
+  //
+  // | BracefulStatement StatementBlock
+  //    <<dsl.PrependChild($1, $0) >>
+
+  // E.g. 'begin {...} begin {...} $x=1'
+  | BracefulStatement BracefulStatement StatementBlock
+    <<dsl.PrependTwoChildren($2, $0, $1) >>
+
+  // E.g. 'begin {...} $x=1'
+  | BracefulStatement BracelessStatement
+    << dsl.NewASTNodeBinary(nil, $0, $1, dsl.NodeTypeStatementBlock) >>
+
+  // E.g. 'begin {...} $x=1 ;'
+  | BracefulStatement BracelessStatement ";"
+    << dsl.NewASTNodeBinary(nil, $0, $1, dsl.NodeTypeStatementBlock) >>
+
+  | BracefulStatement BracelessStatement ";" NonEmptyStatementBlock
+    <<dsl.PrependTwoChildren($3, $0, $1) >>
+;
+
+// ----------------------------------------------------------------
+BracelessStatement
+  : "x"
+    << dsl.NewASTNodeZary($0, dsl.NodeTypeStatement) >>
+;
+
+// ----------------------------------------------------------------
+BracefulStatement
+  : "{" StatementBlock "}"
+    << dsl.Wrap($1) >>
+;

go/parser-experiments/two/src/dsl/ast.go

@@ -0,0 +1,178 @@
+package dsl
+
+import (
+	"fmt"
+
+	"experimental/token"
+)
+
+type TNodeType string
+
+const (
+	NodeTypeEmptyStatement TNodeType = "empty statement"
+
+	NodeTypeStatementBlock = "statement block"
+	NodeTypeStatement      = "statement"
+)
+
+// ----------------------------------------------------------------
+type AST struct {
+	RootNode *ASTNode
+}
+
+// This is for the GOCC/BNF parser, which produces an AST
+func NewAST(iroot interface{}) (*AST, error) {
+	return &AST{
+		RootNode: iroot.(*ASTNode),
+	}, nil
+}
+
+func (this *AST) Print() {
+	this.RootNode.Print()
+}
+
+// ----------------------------------------------------------------
+type ASTNode struct {
+	Token    *token.Token // Nil for tokenless/structural nodes
+	Type     TNodeType
+	Children []*ASTNode
+}
+
+func (this *ASTNode) Print() {
+	this.PrintAux(0)
+}
+func (this *ASTNode) PrintAux(depth int) {
+	for i := 0; i < depth; i++ {
+		fmt.Print("    ")
+	}
+	tok := this.Token
+	fmt.Print("* " + this.Type)
+
+	if tok != nil {
+		//fmt.Printf(" \"%s\" \"%s\"",
+		//	token.TokMap.Id(tok.Type), string(tok.Lit))
+		fmt.Printf(" \"%s\"", string(tok.Lit))
+	}
+	fmt.Println()
+	if this.Children != nil {
+		for _, child := range this.Children {
+			child.PrintAux(depth + 1)
+		}
+	}
+}
+
+// For handling empty expressions.
+func NewASTNodeEmptyNestable(nodeType TNodeType) *ASTNode {
+	return &ASTNode{
+		Token:    nil,
+		Type:     nodeType,
+		Children: nil,
+	}
+}
+
+// xxx comment why grammar use
+func NewASTNodeNestable(itok interface{}, nodeType TNodeType) *ASTNode {
+	var tok *token.Token = nil
+	if itok != nil {
+		tok = itok.(*token.Token)
+	}
+	return &ASTNode{
+		Token:    tok,
+		Type:     nodeType,
+		Children: nil,
+	}
+}
+
+// Signature: Token Type
+func NewASTNodeZary(itok interface{}, nodeType TNodeType) (*ASTNode, error) {
+	parent := NewASTNodeNestable(itok, nodeType)
+	convertToZary(parent)
+	return parent, nil
+}
+
+// Signature: Token Node Type
+func NewASTNodeUnary(itok, childA interface{}, nodeType TNodeType) (*ASTNode, error) {
+	parent := NewASTNodeNestable(itok, nodeType)
+	convertToUnary(parent, childA)
+	return parent, nil
+}
+
+// Signature: Token Node Node Type
+func NewASTNodeBinaryNestable(itok, childA, childB interface{}, nodeType TNodeType) *ASTNode {
+	parent := NewASTNodeNestable(itok, nodeType)
+	convertToBinary(parent, childA, childB)
+	return parent
+}
+
+// Signature: Token Node Node Type
+func NewASTNodeBinary(
+	itok, childA, childB interface{}, nodeType TNodeType,
+) (*ASTNode, error) {
+	return NewASTNodeBinaryNestable(itok, childA, childB, nodeType), nil
+}
+
+func NewASTNodeTernary(itok, childA, childB, childC interface{}, nodeType TNodeType) (*ASTNode, error) {
+	parent := NewASTNodeNestable(itok, nodeType)
+	convertToTernary(parent, childA, childB, childC)
+	return parent, nil
+}
+
+func convertToZary(iparent interface{}) {
+	parent := iparent.(*ASTNode)
+	children := make([]*ASTNode, 0)
+	parent.Children = children
+}
+
+// xxx inline this. can be a one-liner.
+func convertToUnary(iparent interface{}, childA interface{}) {
+	parent := iparent.(*ASTNode)
+	children := make([]*ASTNode, 1)
+	children[0] = childA.(*ASTNode)
+	parent.Children = children
+}
+
+func convertToBinary(iparent interface{}, childA, childB interface{}) {
+	parent := iparent.(*ASTNode)
+	children := make([]*ASTNode, 2)
+	children[0] = childA.(*ASTNode)
+	children[1] = childB.(*ASTNode)
+	parent.Children = children
+}
+
+func convertToTernary(iparent interface{}, childA, childB, childC interface{}) {
+	parent := iparent.(*ASTNode)
+	children := make([]*ASTNode, 3)
+	children[0] = childA.(*ASTNode)
+	children[1] = childB.(*ASTNode)
+	children[2] = childC.(*ASTNode)
+	parent.Children = children
+}
+
+func PrependChild(iparent interface{}, ichild interface{}) (*ASTNode, error) {
+	parent := iparent.(*ASTNode)
+	child := ichild.(*ASTNode)
+	if parent.Children == nil {
+		convertToUnary(iparent, ichild)
+	} else {
+		parent.Children = append([]*ASTNode{child}, parent.Children...)
+	}
+	return parent, nil
+}
+
+func PrependTwoChildren(iparent interface{}, ichildA, ichildB interface{}) (*ASTNode, error) {
+	parent := iparent.(*ASTNode)
+	childA := ichildA.(*ASTNode)
+	childB := ichildB.(*ASTNode)
+	if parent.Children == nil {
+		convertToBinary(iparent, ichildA, ichildB)
+	} else {
+		parent.Children = append([]*ASTNode{childA, childB}, parent.Children...)
+	}
+	return parent, nil
+}
+
+// TODO: comment
+func Wrap(inode interface{}) (*ASTNode, error) {
+	node := inode.(*ASTNode)
+	return node, nil
+}

go/reg-test/cases/case-altkv.sh

@@ -0,0 +1,15 @@
+run_mlr altkv <<EOF
+a,b,c,d,e,f
+EOF
+
+run_mlr altkv <<EOF
+a,b,c,d,e,f,g
+EOF
+
+run_mlr --inidx --ifs comma altkv <<EOF
+a,b,c,d,e,f
+EOF
+
+run_mlr --inidx --ifs comma altkv <<EOF
+a,b,c,d,e,f,g
+EOF

go/reg-test/cases/case-bootstrap.sh

@@ -0,0 +1,3 @@
+run_mlr --seed 12345 bootstrap       $indir/abixy-het
+run_mlr --seed 12345 bootstrap -n  2 $indir/abixy-het
+run_mlr --seed 12345 bootstrap -n 20 $indir/abixy-het

go/reg-test/cases/case-c-auxents.sh

@@ -0,0 +1,17 @@
+run_mlr lecat --mono < $indir/line-ending-cr.bin
+run_mlr lecat --mono < $indir/line-ending-lf.bin
+run_mlr lecat --mono < $indir/line-ending-crlf.bin
+
+run_mlr_no_output termcvt --cr2lf   < $indir/line-ending-cr.bin   > $outdir/line-ending-temp-1.bin
+run_mlr_no_output termcvt --cr2crlf < $indir/line-ending-cr.bin   > $outdir/line-ending-temp-2.bin
+run_mlr_no_output termcvt --lf2cr   < $indir/line-ending-lf.bin   > $outdir/line-ending-temp-3.bin
+run_mlr_no_output termcvt --lf2crlf < $indir/line-ending-lf.bin   > $outdir/line-ending-temp-4.bin
+run_mlr_no_output termcvt --crlf2cr < $indir/line-ending-crlf.bin > $outdir/line-ending-temp-5.bin
+run_mlr_no_output termcvt --crlf2lf < $indir/line-ending-crlf.bin > $outdir/line-ending-temp-6.bin
+
+run_mlr hex < $outdir/line-ending-temp-1.bin
+run_mlr hex < $outdir/line-ending-temp-2.bin
+run_mlr hex < $outdir/line-ending-temp-3.bin
+run_mlr hex < $outdir/line-ending-temp-4.bin
+run_mlr hex < $outdir/line-ending-temp-5.bin
+run_mlr hex < $outdir/line-ending-temp-6.bin

go/reg-test/cases/case-c-awkish-conds.sh

@@ -0,0 +1,7 @@
+
+run_mlr put -v 'begin{@a=1}; $e=2; $f==$g||$h==$i {};               $x=6; end{@z=9}' /dev/null
+run_mlr put -v 'begin{@a=1}; $e=2; $f==$g||$h==$i {$s=1};           $x=6; end{@z=9}' /dev/null
+run_mlr put -v 'begin{@a=1}; $e=2; $f==$g||$h==$i {$s=1;$t=2};      $x=6; end{@z=9}' /dev/null
+run_mlr put -v 'begin{@a=1}; $e=2; $f==$g||$h==$i {$s=1;$t=2;$u=3}; $x=6; end{@z=9}' /dev/null
+run_mlr put -v 'begin{@a=1}; $e=2; $f==$g||$h==$i {$s=1;@t["u".$5]=2;emit @v;emit @w; dump}; $x=6; end{@z=9}' /dev/null
+run_mlr put -v 'begin{true{@x=1}}; true{@x=2}; end{true{@x=3}}' /dev/null

go/reg-test/cases/case-c-bar.sh

@@ -0,0 +1,3 @@
+run_mlr --opprint bar -f x,y --lo 0 --hi 1 $indir/abixy
+run_mlr --opprint bar -f x,y -c c -x x -b b --lo 0.1 --hi 0.9 -w 20 $indir/abixy
+run_mlr --opprint bar --auto -f x,y -w 10 $indir/abixy

go/reg-test/cases/case-c-barred-pprint.sh

@@ -0,0 +1,5 @@
+run_mlr --opprint --barred cat $indir/abixy
+run_mlr --opprint --barred --right cat $indir/abixy
+
+run_mlr --opprint --barred cat $indir/abixy-het
+run_mlr --opprint --barred --right cat $indir/abixy-het

go/reg-test/cases/case-c-chaining.sh

@@ -0,0 +1,20 @@
+run_mlr cat then cat $indir/short
+run_mlr cat then tac $indir/short
+run_mlr tac then cat $indir/short
+run_mlr tac then tac $indir/short
+
+run_mlr cat then cat then cat $indir/short
+run_mlr cat then cat then tac $indir/short
+run_mlr cat then tac then cat $indir/short
+run_mlr cat then tac then tac $indir/short
+run_mlr tac then cat then cat $indir/short
+run_mlr tac then cat then tac $indir/short
+run_mlr tac then tac then cat $indir/short
+run_mlr tac then tac then tac $indir/short
+
+# Test allowing then-chains to start with an initial 'then'
+run_mlr \
+    then cat \
+    then head -n 2 -g a,b \
+    then tac \
+    $indir/abixy-het

go/reg-test/cases/case-c-clean-whitespace.sh

@@ -0,0 +1,10 @@
+run_mlr --icsv --ojson cat                                $indir/clean-whitespace.csv
+run_mlr --icsv --ojson put '$a = lstrip($a)'              $indir/clean-whitespace.csv
+run_mlr --icsv --ojson put '$a = rstrip($a)'              $indir/clean-whitespace.csv
+run_mlr --icsv --ojson put '$a = strip($a)'               $indir/clean-whitespace.csv
+run_mlr --icsv --ojson put '$a = collapse_whitespace($a)' $indir/clean-whitespace.csv
+run_mlr --icsv --ojson put '$a = clean_whitespace($a)'    $indir/clean-whitespace.csv
+
+run_mlr --icsv --ojson clean-whitespace -k $indir/clean-whitespace.csv
+run_mlr --icsv --ojson clean-whitespace -v $indir/clean-whitespace.csv
+run_mlr --icsv --ojson clean-whitespace    $indir/clean-whitespace.csv

go/reg-test/cases/case-c-compressed-input.sh

@@ -0,0 +1,4 @@
+run_mlr --csv  --prepipe 'cat'   cat   $indir/rfc-csv/simple.csv-crlf
+run_mlr --dkvp --prepipe 'cat'   cat   $indir/abixy
+run_mlr --csv  --prepipe 'cat'   cat < $indir/rfc-csv/simple.csv-crlf
+run_mlr --dkvp --prepipe 'cat'   cat < $indir/abixy

go/reg-test/cases/case-c-count-distinct.sh

@@ -0,0 +1,12 @@
+run_mlr count-distinct -f a      $indir/small $indir/abixy
+run_mlr count-distinct -f a,b    $indir/small $indir/abixy
+run_mlr count-distinct -f a,b -u $indir/small $indir/abixy
+
+run_mlr count-distinct -f a   -n $indir/small $indir/abixy
+run_mlr count-distinct -f a,b -n $indir/small $indir/abixy
+
+run_mlr count-distinct -f a   -o foo $indir/small $indir/abixy
+run_mlr count-distinct -f a,b -o foo $indir/small $indir/abixy
+
+run_mlr count-distinct -f a   -n -o foo $indir/small $indir/abixy
+run_mlr count-distinct -f a,b -n -o foo $indir/small $indir/abixy

go/reg-test/cases/case-c-count-similar.sh

@@ -0,0 +1,3 @@
+run_mlr --opprint --from $indir/abixy count-similar -g a
+run_mlr --opprint --from $indir/abixy count-similar -g b,i
+run_mlr --opprint --from $indir/abixy count-similar -g a,b,i -o other_name_for_counter

go/reg-test/cases/case-c-csv-rs-environment-defaults.sh

@@ -0,0 +1,6 @@
+# ----------------------------------------------------------------
+announce CSV/RS ENVIRONMENT DEFAULTS
+
+run_mlr --csv cut -f a $indir/rfc-csv/simple.csv-crlf
+run_mlr --csv --rs crlf cut -f a $indir/rfc-csv/simple.csv-crlf
+mlr_expect_fail --csv --rs lf cut -f a $indir/rfc-csv/simple.csv-crlf

go/reg-test/cases/case-c-cut.sh

@@ -0,0 +1,15 @@
+# Intended to be invoked by "." from reg-test/run
+
+run_mlr cut -f a,x $indir/abixy
+run_mlr cut --complement -f a,x $indir/abixy
+
+run_mlr cut -r    -f c,e         $indir/having-fields-regex.dkvp
+run_mlr cut -r    -f '"C","E"'   $indir/having-fields-regex.dkvp
+run_mlr cut -r    -f '"c"i,"e"'  $indir/having-fields-regex.dkvp
+run_mlr cut -r    -f '"C"i,"E"'  $indir/having-fields-regex.dkvp
+run_mlr cut -r -x -f c,e         $indir/having-fields-regex.dkvp
+run_mlr cut -r -x -f '"C","E"'   $indir/having-fields-regex.dkvp
+run_mlr cut -r -x -f '"C","E"i'  $indir/having-fields-regex.dkvp
+run_mlr cut -r -x -f '"c","e"i'  $indir/having-fields-regex.dkvp
+
+run_mlr --csvlite cut -r -f '^Name$,^Date_[0-9].*$' $indir/date1.csv $indir/date2.csv

go/reg-test/cases/case-c-decimate.sh

@@ -0,0 +1,6 @@
+run_mlr decimate         -n 4 $indir/abixy
+run_mlr decimate      -b -n 4 $indir/abixy
+run_mlr decimate      -e -n 4 $indir/abixy
+run_mlr decimate -g a    -n 2 $indir/abixy
+run_mlr decimate -g a -b -n 2 $indir/abixy
+run_mlr decimate -g a -e -n 2 $indir/abixy

go/reg-test/cases/case-c-dsl-absent-empty.sh

@@ -0,0 +1,79 @@
+run_mlr put -q '@sum     += $x; end{emitp @sum}'      $indir/abixy
+run_mlr put -q '@sum[$a] += $x; end{emitp @sum, "a"}' $indir/abixy
+run_mlr put    '$nonesuch = @nonesuch' $indir/abixy
+
+run_mlr put -q '@sum     += $x; end{emitp @sum}'      $indir/abixy-het
+run_mlr put -q '@sum[$a] += $x; end{emitp @sum, "a"}' $indir/abixy-het
+run_mlr put    '$nonesuch = @nonesuch' $indir/abixy-het
+
+run_mlr put -q '@sum += $x; @sumtype = typeof(@sum); @xtype = typeof($x); emitf @sumtype, @xtype, @sum; end{emitp @sum}' $indir/abixy
+run_mlr put -q '@sum += $x; @sumtype = typeof(@sum); @xtype = typeof($x); emitf @sumtype, @xtype, @sum; end{emitp @sum}' $indir/abixy-het
+
+run_mlr put '$z = $x + $y' $indir/typeof.dkvp
+run_mlr put '$z = $x + $u' $indir/typeof.dkvp
+
+run_mlr put '@s = @s + $y; emitp @s' $indir/typeof.dkvp
+run_mlr put '@s = @s + $u; emitp @s' $indir/typeof.dkvp
+
+run_mlr put '$z = $x + $y; $x=typeof($x);$y=typeof($y);$z=typeof($z)' $indir/typeof.dkvp
+run_mlr put '$z = $x + $u; $x=typeof($x);$y=typeof($y);$z=typeof($z)' $indir/typeof.dkvp
+
+run_mlr put '@s = @s + $y; $x=typeof($x);$y=typeof($y);$z=typeof($z);$s=typeof(@s)' $indir/typeof.dkvp
+run_mlr put '@s = @s + $u; $x=typeof($x);$y=typeof($y);$z=typeof($z);$s=typeof(@s)' $indir/typeof.dkvp
+
+run_mlr cat << EOF
+x=1
+x=
+x=7
+EOF
+
+run_mlr --ofs tab put '$osum=@sum; $ostype=typeof(@sum);$xtype=typeof($x);@sum+=$x; $nstype=typeof(@sum);$nsum=@sum; end { emit @sum }' <<EOF
+x=1
+x=
+x=7
+EOF
+
+run_mlr --ofs tab put '$osum=@sum; $ostype=typeof(@sum);$xtype=typeof($x);is_present($x){@sum+=$x}; $nstype=typeof(@sum);$nsum=@sum; end { emit @sum }' <<EOF
+x=1
+x=
+x=7
+EOF
+
+run_mlr cat << EOF
+x=1
+xxx=
+x=7
+EOF
+
+run_mlr --ofs tab put '$osum=@sum; $ostype=typeof(@sum);$xtype=typeof($x);@sum+=$x; $nstype=typeof(@sum);$nsum=@sum; end { emit @sum }' <<EOF
+x=1
+xxx=
+x=7
+EOF
+
+run_mlr --ofs tab put '$osum=@sum; $ostype=typeof(@sum);$xtype=typeof($x);is_present($x){@sum+=$x}; $nstype=typeof(@sum);$nsum=@sum; end { emit @sum }' <<EOF
+x=1
+xxx=
+x=7
+EOF
+
+run_mlr cat << EOF
+x=1
+x=
+y=
+x=7
+EOF
+
+run_mlr --ofs tab put '$xtype=typeof($x);$sum = $x + 10; $stype=typeof($sum)' <<EOF
+x=1
+x=
+y=
+x=7
+EOF
+
+run_mlr --ofs tab put '$xtype=typeof($x);$sum = is_present($x) ? $x + 10 : 999; $stype=typeof($sum)' <<EOF
+x=1
+x=
+y=
+x=7
+EOF

go/reg-test/cases/case-c-dsl-argpass-typedecl.sh

@@ -0,0 +1,58 @@
+run_mlr --from $indir/abixy put '
+  func f(int i) {
+    return i+3;
+  }
+  $c = f($i);
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(int i) {
+    return i+3;
+  }
+  $c = f($x);
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(num i): int {
+    return i+3.45;
+  }
+  $c = f($x);
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(num i): int {
+    return i+3.45;
+  }
+  $c = f($i);
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(num i): int {
+    i = "a";
+    return 2;
+  }
+  $c = f($x);
+'
+
+
+run_mlr --from $indir/abixy put '
+  subr s(int i) {
+    print i+3;
+  }
+  call s($i);
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  subr s(int i) {
+    print i+3;
+  }
+  call s($x);
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  subr s(num i) {
+    i = "a";
+    print 2;
+  }
+  call s($x);
+'

go/reg-test/cases/case-c-dsl-asserting.sh

@@ -0,0 +1,112 @@
+
+run_mlr         --opprint put '$f=asserting_absent($nosuch)'                     $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_absent(@nosuch)'                     $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_absent($x)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_absent($y)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_absent($z)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '@somesuch=1;$f=asserting_absent(@somesuch)'       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put 'foo=asserting_absent($*)'                         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put 'foo=asserting_absent({1:2})'                      $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_empty($z)'                           $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty($x)'                           $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty($y)'                           $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty($nosuch)'                      $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty(@nosuch)'                      $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty({1:2})'                        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '@somesuch=1;$f=asserting_empty(@somesuch)'        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put 'foo=asserting_empty($*)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put 'foo=asserting_empty({1:2})'                       $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_empty_map({})'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map($*)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map($x)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map($y)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map($z)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map($nosuch)'                  $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map(@nosuch)'                  $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_empty_map({1:2})'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '@somesuch=1;$f=asserting_empty_map(@somesuch)'    $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_map($*)'                             $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_map({1:2})'                          $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_map({})'                             $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_map($x)'                             $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_map($y)'                             $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_map($z)'                             $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_map($nosuch)'                        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_map(@nosuch)'                        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '@somesuch=1;$f=asserting_map(@somesuch)'          $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_nonempty_map($*)'                    $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_nonempty_map({1:2})'                 $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_nonempty_map($x)'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_nonempty_map($y)'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_nonempty_map($z)'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_nonempty_map($nosuch)'               $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_nonempty_map(@nosuch)'               $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_nonempty_map({})'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '@somesuch=1;$f=asserting_nonempty_map(@somesuch)' $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$*=asserting_not_empty($*)'                       $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_empty($nosuch)'                  $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_empty(@nosuch)'                  $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_empty({1:2})'                    $indir/nullvals.dkvp
+run_mlr         --opprint put '$nosuch=asserting_not_empty($nosuch)'             $indir/nullvals.dkvp
+run_mlr         --opprint put '@somesuch=1;$f=asserting_not_empty(@somesuch)'    $indir/nullvals.dkvp
+run_mlr         --opprint put '$*=asserting_not_empty($*)'                       $indir/nullvals.dkvp
+run_mlr         --opprint put '$*=asserting_not_empty({1:2})'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_empty($x)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_empty($y)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_empty($z)'                       $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_not_map($x)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_map($y)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_map($z)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_map($nosuch)'                    $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_not_map(@nosuch)'                    $indir/nullvals.dkvp
+run_mlr         --opprint put '@somesuch=1;$f=asserting_not_map(@somesuch)'      $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_map($*)'                         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_map({1:2})'                      $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_map({})'                         $indir/nullvals.dkvp
+
+run_mlr         --opprint put '@somesuch=1;$f=asserting_not_null(@somesuch)'     $indir/nullvals.dkvp
+run_mlr         --opprint put '$*=asserting_not_null($*)'                        $indir/nullvals.dkvp
+run_mlr         --opprint put '$*=asserting_not_null({1:2})'                     $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_null($x)'                        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_null($y)'                        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_null($z)'                        $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_null($nosuch)'                   $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_not_null(@nosuch)'                   $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_null($z)'                            $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_null($nosuch)'                       $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_null(@nosuch)'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_null($x)'                            $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_null($y)'                            $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '@somesuch=1;$f=asserting_null(@somesuch)'         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$*=asserting_null($*)'                            $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$*=asserting_null({1:2})'                         $indir/nullvals.dkvp
+
+mlr_expect_fail --opprint put '$f=asserting_numeric($x)'                         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_numeric($y)'                         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_numeric($z)'                         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$*=asserting_numeric($*)'                         $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$*=asserting_numeric({1:2})'                      $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_numeric($nosuch)'                    $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_present($x)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_present($y)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '$f=asserting_present($z)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '@somesuch=1;$f=asserting_present(@somesuch)'      $indir/nullvals.dkvp
+run_mlr         --opprint put '$*=asserting_present($*)'                         $indir/nullvals.dkvp
+run_mlr         --opprint put '$*=asserting_present({1:2})'                      $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_present($nosuch)'                    $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_present(@nosuch)'                    $indir/nullvals.dkvp
+
+run_mlr         --opprint put '$f=asserting_string($z)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$*=asserting_string($*)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$*=asserting_string({1:2})'                       $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_string($x)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_string($y)'                          $indir/nullvals.dkvp
+mlr_expect_fail --opprint put '$f=asserting_string($nosuch)'                     $indir/nullvals.dkvp

go/reg-test/cases/case-c-dsl-break-continue.sh

@@ -0,0 +1,962 @@
+# ----------------------------------------------------------------
+announce DSL BREAK/CONTINUE IN SINGLE WHILE/DO-WHILE
+
+run_mlr --opprint --from $indir/abixy put '
+  while ($i < 5) {
+    $i += 1;
+    break;
+    $a = "ERROR";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  while ($i < 5) {
+    $i += 1;
+    continue;
+    $a = "ERROR";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  do {
+    $i += 1;
+    break;
+    $a = "ERROR";
+  } while ($i < 5);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  do {
+    $i += 1;
+    continue;
+    $a = "ERROR";
+  } while ($i < 5);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $NR = NR;
+  while ($i < 5) {
+    $i += 1;
+    if (NR == 2) {
+      break;
+    }
+    $a = "reached";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $NR = NR;
+  while ($i < 5) {
+    $i += 1;
+    if (NR == 2) {
+      continue;
+    }
+    $a = "reached";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+$NR = NR;
+  do {
+    $i += 1;
+    if (NR == 2) {
+      break;
+    }
+    $a = "reached";
+  } while ($i < 5);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $NR = NR;
+  do {
+    $i += 1;
+    if (NR == 2) {
+      continue;
+    }
+    $a = "reached";
+  } while ($i < 5);
+'
+
+# ----------------------------------------------------------------
+announce DSL BREAK/CONTINUE IN NESTED WHILE/DO-WHILE
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    break;
+    while ($k < 7) {
+      $k += 1
+    }
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    continue;
+    while ($k < 7) {
+      $k += 1
+    }
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    while ($k < 7) {
+      $k += 1;
+      break;
+      $k += 10000;
+    }
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    while ($k < 7) {
+      $k += 1;
+      continue;
+      $k += 10000;
+    }
+  }
+'
+
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    if (NR == 2 || NR == 8) {
+      break;
+    }
+    while ($k < 7) {
+      $k += 1
+    }
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    if (NR == 2 || NR == 8) {
+      continue;
+    }
+    while ($k < 7) {
+      $k += 1
+    }
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    while ($k < 7) {
+      $k += 1;
+      if (NR == 2 || NR == 8) {
+        break;
+      }
+      $k += 10000;
+    }
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  while ($j < 4) {
+    $k = NR;
+    $j += 1;
+    while ($k < 7) {
+      $k += 1;
+      if (NR == 2 || NR == 8) {
+        continue;
+      }
+      $k += 10000;
+    }
+  }
+'
+
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    break;
+    do {
+      $k += 1
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    continue;
+    do {
+      $k += 1
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    do {
+      $k += 1;
+      break;
+      $k += 10000;
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    do {
+      $k += 1;
+      continue;
+      $k += 10000;
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    if (NR == 2 || NR == 8) {
+      break;
+    }
+    do {
+      $k += 1
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    if (NR == 2 || NR == 8) {
+      continue;
+    }
+    do {
+      $k += 1
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    do {
+      $k += 1;
+      if (NR == 2 || NR == 8) {
+        break;
+      }
+      $k += 10000;
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+run_mlr --opprint --from $indir/abixy put '
+  $j = NR;
+  do {
+    $k = NR;
+    $j += 1;
+    do {
+      $k += 1;
+      if (NR == 2 || NR == 8) {
+        continue;
+      }
+      $k += 10000;
+    } while ($k < 7);
+  } while ($j < 4);
+'
+
+# ----------------------------------------------------------------
+announce DSL BREAK/CONTINUE IN SINGLE FOR-SREC
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k,v in $*) {
+      @logging1[NR][k] = v;
+      if (k == "x") {
+          break;
+      }
+  }
+  end {
+    emitp @logging1, "NR", "k";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k,v in $*) {
+      if (k == "x") {
+          break;
+      }
+      @logging2[NR][k] = v;
+  }
+  end {
+    emitp @logging2, "NR", "k";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k,v in $*) {
+      @logging3[NR][k] = v;
+      if (k == "x") {
+          continue;
+      }
+  }
+  end {
+    emitp @logging3, "NR", "k";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k,v in $*) {
+      if (k == "x") {
+          continue;
+      }
+      @logging4[NR][k] = v;
+  }
+  end {
+    emitp @logging4, "NR", "k"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k,v in $*) {
+      @logging1[NR][k] = v;
+      if (k == "x") {
+          break;
+      }
+  }
+
+  for (k,v in $*) {
+      if (k == "x") {
+          break;
+      }
+      @logging2[NR][k] = v;
+  }
+
+  for (k,v in $*) {
+      @logging3[NR][k] = v;
+      if (k == "x") {
+          continue;
+      }
+  }
+
+  for (k,v in $*) {
+      if (k == "x") {
+          continue;
+      }
+      @logging4[NR][k] = v;
+  }
+
+  end {
+    emitp @logging1, "NR", "k";
+    emitp @logging2, "NR", "k";
+    emitp @logging3, "NR", "k";
+    emitp @logging4, "NR", "k"
+  }
+'
+
+# ----------------------------------------------------------------
+announce DSL BREAK/CONTINUE IN NESTED FOR-SREC
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    break;
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    continue;
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      break;
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      continue;
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    break;
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      break;
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    continue;
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      break;
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    break;
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      continue;
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    continue;
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      continue;
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    if (k1 == "b") {
+        break
+    }
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    if (k1 == "b") {
+        continue
+    }
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      if (k2 == "a") {
+          break
+      }
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      if (k2 == "b") {
+          continue
+      }
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    if (k1 == "b") {
+        break
+    }
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      if (k2 == "a") {
+          break
+      }
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    if (k1 == "b") {
+        continue
+    }
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      if (k2 == "a") {
+          break
+      }
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    if (k1 == "b") {
+        break
+    }
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      if (k2 == "a") {
+          continue
+      }
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  for (k1, v1 in $*) {
+    @output1[NR][k1] = "before";
+    if (k1 == "b") {
+        continue
+    }
+    @output1[NR][k1] = v1;
+    for (k2, v2 in $*) {
+      @output2[NR][k1."_".k2] = "before";
+      if (k2 == "a") {
+          continue
+      }
+      @output2[NR][k1."_".k2] = v2;
+    }
+  }
+  end {
+    emit @output1, "NR", "name";
+    emit @output2, "NR", "names";
+  }
+'
+# ----------------------------------------------------------------
+
+announce DSL BREAK/CONTINUE IN SINGLE FOR-OOSVAR
+
+mention single-key tests, direct break/continue
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        break;
+        @output[k1] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        @output[k1] = v;
+        break;
+        @output[k1] = "ERROR";
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        continue;
+        @output[k1] = v
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        @output[k1] = v;
+        continue;
+        @output[k1] = "ERROR";
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+mention single-key tests, indirect break/continue
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        if (k1 == "i") {
+          break;
+        }
+        @output[k1] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        @output[k1] = v;
+        if (k1 == "i") {
+          break;
+        }
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        if (k1 == "i") {
+          continue;
+        }
+        @output[k1] = v
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for (k1, v in @logging[2]) {
+        @output[k1] = v;
+        if (k1 == "i") {
+          continue;
+        }
+        @output[k1] = "reached";
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+mention multiple-key tests, direct break/continue
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        break;
+        @output[k1][k2] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        @output[k1][k2] = v;
+        break;
+        @output[k1][k2] = "ERROR"
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        continue;
+        @output[k1][k2] = v
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        @output[k1][k2] = v;
+        continue;
+        @output[k1][k2] = "ERROR";
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+mention multiple-key tests, indirect break/continue
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        if (k1 == 5) {
+          break;
+        }
+        @output[k1][k2] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        if (k2 == "i") {
+          break;
+        }
+        @output[k1][k2] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        @output[k1][k2] = v;
+        if (k1 == 5) {
+          break;
+        }
+    }
+    emit @output, "NR", "name"
+  }
+'
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        @output[k1][k2] = v;
+        if (k2 == "i") {
+          break;
+        }
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        if (k1 == 5) {
+          continue;
+        }
+        @output[k1][k2] = v
+    }
+    emit @output, "NR", "name"
+  }
+'
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        if (k2 == "i") {
+          continue;
+        }
+        @output[k1][k2] = v
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        @output[k1][k2] = "before";
+        if (k1 == 5) {
+          continue;
+        }
+        @output[k1][k2] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        @output[k1][k2] = "before";
+        if (k2 == "i") {
+          continue;
+        }
+        @output[k1][k2] = v;
+    }
+    emit @output, "NR", "name"
+  }
+'
+
+# ----------------------------------------------------------------
+announce DSL BREAK/CONTINUE IN NESTED FOR-OOSVAR
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @logging[NR] = $*;
+  end {
+    for ((k1, k2), v in @logging) {
+        if (k1 != 2) {
+          continue
+        }
+        for ((k3, k4), v in @logging) {
+          if (k3 != 4) {
+            continue
+          }
+          @output[k1][k2][k3][k4] = v;
+        }
+    }
+    emit @output, "NR1", "name1", "NR2", "name2"
+  }
+'

go/reg-test/cases/case-c-dsl-comments.sh

@@ -0,0 +1,20 @@
+# ----------------------------------------------------------------
+announce DSL COMMENTS
+
+run_mlr --from $indir/abixy put '
+  $s = 1;
+  #$t = 2;
+  $u = 3;
+'
+
+run_mlr --from $indir/abixy filter '
+  NR == 1 ||
+  #NR == 2 ||
+  NR == 3
+'
+
+run_mlr --from $indir/abixy put '
+  $s = "here is a pound#sign"; # but this is a comment
+  #$t = 2;
+  $u = 3;
+'

go/reg-test/cases/case-c-dsl-context-specific-validation.sh

@@ -0,0 +1,67 @@
+
+mention non-top-level begin/end
+mlr_expect_fail put -v 'begin{begin{@x=1}}'
+mlr_expect_fail put -v 'true{begin{@x=1}}'
+mlr_expect_fail put -v 'end{end{@x=1}}'
+mlr_expect_fail put -v 'true{end{@x=1}}'
+
+mention srecs in begin/end
+mlr_expect_fail put -v 'begin{$x=1}'
+mlr_expect_fail put -v 'begin{@x=$y}'
+mlr_expect_fail put -v 'end{$x=1}'
+mlr_expect_fail put -v 'end{@x=$y}'
+mlr_expect_fail put -v 'begin{@v=$*}'
+mlr_expect_fail put -v 'end{$*=@v}'
+
+mlr_expect_fail put -v 'begin{unset $x}'
+mlr_expect_fail put -v 'end{unset $x}'
+mlr_expect_fail put -v 'begin{unset $*}'
+mlr_expect_fail put -v 'end{unset $*}'
+
+mention break/continue outside loop
+mlr_expect_fail put -v 'break'
+mlr_expect_fail put -v 'continue'
+
+mention oosvars etc. in mlr filter
+mlr_expect_fail filter -v 'break'
+mlr_expect_fail filter -v 'continue'
+
+mention expanded filter
+
+run_mlr --from $indir/abixy filter '
+  begin {
+    @avoid = 3
+  }
+  NR != @avoid
+'
+
+run_mlr --from $indir/abixy filter -x '
+  begin {
+    @avoid = 3
+  }
+  NR != @avoid
+'
+
+run_mlr --from $indir/abixy filter '
+  func f(n) {
+    return n - 1
+  }
+  f(NR) == 5
+'
+
+run_mlr --from $indir/abixy filter '
+  subr s(n) {
+    print "NR is ".n
+  }
+  call s(NR);
+  false
+'
+
+run_mlr --from $indir/abixy filter '
+  int a = 5;
+  int b = 7;
+  a <= NR && NR <= b
+'
+
+mlr_expect_fail --from $indir/abixy filter 'filter false'
+mlr_expect_fail --from $indir/abixy filter 'filter false; true'

go/reg-test/cases/case-c-dsl-do-while-loops.sh

@@ -0,0 +1,2 @@
+run_mlr put -v 'while($i < 5) { $i += 1}' $indir/abixy
+run_mlr put -v 'do {$i += 1} while($i < 5)' $indir/abixy

go/reg-test/cases/case-c-dsl-empty-statements.sh

@@ -0,0 +1,32 @@
+run_mlr -n put -v ''
+mlr_expect_fail -n filter -v ''
+
+run_mlr -n put -v 'begin {}'
+run_mlr -n put -v 'begin {;}'
+run_mlr -n put -v 'begin {;;}'
+run_mlr -n put -v 'begin {;;;}'
+run_mlr -n put -v 'begin {@x=1}'
+run_mlr -n put -v 'begin {@x=1;}'
+run_mlr -n put -v 'begin {;@x=1}'
+run_mlr -n put -v 'begin {@x=1;@y=2}'
+run_mlr -n put -v 'begin {@x=1;;@y=2}'
+
+run_mlr -n put -v 'true {}'
+run_mlr -n put -v 'true {;}'
+run_mlr -n put -v 'true {;;}'
+run_mlr -n put -v 'true {;;;}'
+run_mlr -n put -v 'true {@x=1}'
+run_mlr -n put -v 'true {@x=1;}'
+run_mlr -n put -v 'true {;@x=1}'
+run_mlr -n put -v 'true {@x=1;@y=2}'
+run_mlr -n put -v 'true {@x=1;;@y=2}'
+
+run_mlr -n put -v 'end {}'
+run_mlr -n put -v 'end {;}'
+run_mlr -n put -v 'end {;;}'
+run_mlr -n put -v 'end {;;;}'
+run_mlr -n put -v 'end {@x=1}'
+run_mlr -n put -v 'end {@x=1;}'
+run_mlr -n put -v 'end {;@x=1}'
+run_mlr -n put -v 'end {@x=1;@y=2}'
+run_mlr -n put -v 'end {@x=1;;@y=2}'

go/reg-test/cases/case-c-dsl-extended-typeof.sh

@@ -0,0 +1,29 @@
+run_mlr --from $indir/xyz2 --oxtab put '
+  a = {1:2, 3:4};
+  b = {1: {2:3, 4:5}};
+  @c = {1:2, 3:4};
+  @d = {1: {2:3, 4:5}};
+
+  $ta  = typeof(a);
+  $ta1 = typeof(a[1]);
+  $ta7 = typeof(a[7]);
+
+  $tb  = typeof(b);
+  $tb1 = typeof(b[1]);
+  $tb7 = typeof(b[7]);
+
+  $tc  = typeof(@c);
+  $tc1 = typeof(@c[1]);
+  $tc7 = typeof(@c[7]);
+
+  $td  = typeof(@d);
+  $td1 = typeof(@d[1]);
+  $td7 = typeof(@d[7]);
+
+  $te1 = typeof(6);
+  $te2 = typeof({6:4});
+
+  $tsa = typeof($a);
+  $tsx = typeof($x);
+  $ts  = typeof($*);
+'

go/reg-test/cases/case-c-dsl-filter-pattern-action.sh

@@ -0,0 +1,100 @@
+run_mlr --opprint put '         $x > 0.5;  $z = "flag"'  $indir/abixy
+run_mlr --opprint put '       !($x > 0.5); $z = "flag"'  $indir/abixy
+run_mlr --opprint put 'filter   $x > 0.5;  $z = "flag"'  $indir/abixy
+run_mlr --opprint put '         $x > 0.5  {$z = "flag"}' $indir/abixy
+run_mlr --opprint put 'filter !($x > 0.5); $z = "flag"'  $indir/abixy
+run_mlr --opprint put '       !($x > 0.5) {$z = "flag"}' $indir/abixy
+ 
+# ----------------------------------------------------------------
+announce DSL SUB/GSUB/REGEX_EXTRACT
+
+run_mlr --opprint put '$y = sub($x, "e.*l",        "")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "e.*l"i,       "")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "e.*"."l",     "")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "e.*l",        "y123y")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "e.*l"i,       "y123y")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "e.*"."l",     "y123y")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "([hg])e.*l(.)", "y\1y123\2y")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "([hg])e.*l.",   "y\1y123\2y")' $indir/sub.dat
+run_mlr --opprint put '$y = sub($x, "([hg])e.*l(.)", "y\1y123.y")'  $indir/sub.dat
+
+run_mlr --opprint put '$y = sub($x,  "a",    "aa")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "a",    "aa")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "A",    "Aa")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "a"i,   "Aa")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "A"i,   "Aa")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "a(.)", "aa\1\1\1")' $indir/gsub.dat
+
+run_mlr --opprint put '$y = sub($x,  "a",    "")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "a",    "")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "A",    "")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "a"i,   "")'   $indir/gsub.dat
+run_mlr --opprint put '$y = gsub($x, "A"i,   "")'   $indir/gsub.dat
+
+run_mlr --oxtab cat                       $indir/subtab.dkvp
+run_mlr --oxtab put -f $indir/subtab1.mlr $indir/subtab.dkvp
+run_mlr --oxtab put -f $indir/subtab2.mlr $indir/subtab.dkvp
+run_mlr --oxtab put -f $indir/subtab3.mlr $indir/subtab.dkvp
+run_mlr --oxtab put -f $indir/subtab4.mlr $indir/subtab.dkvp
+
+run_mlr --opprint put '$y = ssub($x, "HE",       "")'           $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "HE",       "HE")'         $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "HE",       "12345")'      $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LL",       "1")'          $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LL",       "12")'         $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LL",       "12345")'      $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LLO",      "")'           $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LLO",      "12")'         $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LLO",      "123")'        $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "LLO",      "123456")'     $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "HELLO",    "")'           $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "HELLO",    "1234")'       $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "HELLO",    "12345")'      $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "HELLO",    "1234678")'    $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "nonesuch", "")'           $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "nonesuch", "1234")'       $indir/sub.dat
+run_mlr --opprint put '$y = ssub($x, "nonesuch", "1234567890")' $indir/sub.dat
+
+run_mlr --oxtab put '$y = regextract($x, "[A-Z]+")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[A-Z]*")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[a-z]+")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[a-z]*")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[0-9]+")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[0-9]*")' $indir/sub.dat
+
+run_mlr --oxtab put '$y = regextract($x, "[ef]+")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[ef]*")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[hi]+")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[hi]*")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[op]+")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract($x, "[op]*")' $indir/sub.dat
+
+run_mlr --oxtab put '$y = regextract_or_else($x, "[A-Z]+", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[A-Z]*", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[a-z]+", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[a-z]*", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[0-9]+", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[0-9]*", "DEFAULT")' $indir/sub.dat
+
+run_mlr --oxtab put '$y = regextract_or_else($x, "[ef]+", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[ef]*", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[hi]+", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[hi]*", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[op]+", "DEFAULT")' $indir/sub.dat
+run_mlr --oxtab put '$y = regextract_or_else($x, "[op]*", "DEFAULT")' $indir/sub.dat
+
+run_mlr --nidx put '$1 = sub($1, "ab(.)d(..)g",  "ab<<\1>>d<<\2>>g")' <<EOF
+abcdefg
+EOF
+
+run_mlr --nidx put '$1 = sub($1, "ab(c)?d(..)g", "ab<<\1>>d<<\2>>g")' <<EOF
+abcdefg
+EOF
+
+run_mlr --nidx put '$1 = sub($1, "ab(c)?d(..)g", "ab<<\1>>d<<\2>>g")' <<EOF
+abXdefg
+EOF
+
+run_mlr --nidx put '$1 = sub($1, "ab(c)?d(..)g", "ab<<\1>>d<<\2>>g")' <<EOF
+abdefg
+EOF

go/reg-test/cases/case-c-dsl-for-bind-loops-for-valgrind.sh

@@ -0,0 +1,127 @@
+mention key-only fors
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  map o = {};
+  o[ab] = xy;
+  for (k in o) {
+    print "k is " . k;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  unset @o;
+  @o[ab] = xy;
+  for (k in @o) {
+    print "k is " . k;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  for (k in {ab : xy}) {
+    print "k is " . k;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  func f(a, b, x, y): map {
+    ab = $a . "_" . $b;
+    xy = $x . "_" . $y;
+    return {ab : xy};
+  }
+  for (k in f($a, $b, $x, $y)) {
+    print "k is " . k;
+  }
+'
+
+mention key-value fors with scalar values
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  map o = {};
+  o[ab] = xy;
+  for (k, v in o) {
+    print "k is " . k . "  v is " . v;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  unset @o;
+  @o[ab] = xy;
+  for (k, v in @o) {
+    print "k is " . k . "  v is " . v;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  for (k, v in {ab : xy}) {
+    print "k is " . k . "  v is " . v;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  func f(a, b, x, y): map {
+    ab = $a . "_" . $b;
+    xy = $x . "_" . $y;
+    return {ab : xy};
+  }
+  for (k, v in f($a, $b, $x, $y)) {
+    print "k is " . k . "  v is " . v;
+  }
+'
+
+
+mention key-value fors with map values
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  map o = {};
+  o[ab] = {"foo": xy};
+  for (k, v in o) {
+    print "k is " . k . "  v is ";
+    dump v;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  unset @o;
+  @o[ab]["foo"] = xy;
+  for (k, v in @o) {
+    print "k is " . k . "  v is ";
+    dump v;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  ab = $a . "_" . $b;
+  xy = $x . "_" . $y;
+  for (k, v in {ab : {"foo": xy}}) {
+    print "k is " . k . "  v is ";
+    dump v;
+  }
+'
+
+run_mlr --from $indir/abixy-het put -q '
+  func f(a, b, x, y): map {
+    ab = $a . "_" . $b;
+    xy = $x . "_" . $y;
+    return {ab : {"foo" : xy}};
+  }
+  for (k, v in f($a, $b, $x, $y)) {
+    print "k is " . k . "  v is ";
+    dump v;
+  }
+'

go/reg-test/cases/case-c-dsl-for-map-depth.sh

@@ -0,0 +1,19 @@
+mention 'for full oosvar'
+run_mlr --from $indir/abixy put '@o[1][2] = 7; for(k1,v in @*) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put '@o[1][2] = 7; for((k1,k2),v in @*) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put '@o[1][2] = 7; for((k1,k2,k3),v in @*) {$x+=10;$y+=100}'
+
+mention 'for oosvar submap'
+run_mlr --from $indir/abixy put '@o[1][2][3] = 7; for(k1,v in @o[1][2]) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put '@o[1][2][3] = 7; for((k1,k2),v in @o[1][2]) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put '@o[1][2][3] = 7; for((k1,k2,k3),v in @o[1][2]) {$x+=10;$y+=100}'
+
+mention 'for local'
+run_mlr --from $indir/abixy put 'o[1][2] = 7; for(k1,v in o) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put 'o[1][2] = 7; for((k1,k2),v in o) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put 'o[1][2] = 7; for((k1,k2,k3),v in o) {$x+=10;$y+=100}'
+
+mention 'for map-literal'
+run_mlr --from $indir/abixy put 'for(k1,v in {1:{2:7}}) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put 'for((k1,k2),v in {1:{2:7}}) {$x+=10;$y+=100}'
+run_mlr --from $indir/abixy put 'for((k1,k2,k3),v in {1:{2:7}}) {$x+=10;$y+=100}'

go/reg-test/cases/case-c-dsl-for-oosvar-loops.sh

@@ -0,0 +1,69 @@
+run_mlr --opprint -n put -v '
+  @o[1][1]["text1"][NR] = $a;
+  @o[1][2]["text2"][NR] = $b;
+  @o[1][2][$a][$i*100] = $x;
+  for((k1,k2),v in @o[1][2]) {
+    @n[3][4][k2][k1] = v;
+  }
+  end {
+    emit @n, "a", "b", "c", "d"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @o[1][1]["text1"][NR] = $a;
+  @o[1][2]["text2"][NR] = $b;
+  @o[1][2][$a][$i*100] = $x;
+  for((k1,k2),v in @o[1][2]) {
+    @n[3][4][k2][k1] = v;
+  }
+  end {
+    emit @n, "a", "b", "c", "d"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @sum[$a][$b] += $x;
+  @count[$a][$b] += 1;
+  end {
+    for ((k1, k2), v in @sum) {
+      @mean[k1][k2] = @sum[k1][k2] / @count[k1][k2]
+    }
+    emitp @sum, "a", "b";
+    emitp @count, "a", "b";
+    emitp @mean, "a", "b"
+  }
+'
+
+run_mlr --opprint --from $indir/abixy-wide put -q '
+  @value["sum"][$a][$b] += $x;
+  @value["count"][$a][$b] += 1;
+  end {
+    for ((k1, k2), v in @value["sum"]) {
+      @value["mean"][k1][k2] = @value["sum"][k1][k2] / @value["count"][k1][k2]
+    }
+    emitp @value, "stat", "a", "b";
+  }
+'
+
+mlr_expect_fail -n put -v 'for (k, k in $*) {}'
+
+mlr_expect_fail -n put -v 'for (k, k in @*) {}'
+
+mlr_expect_fail -n put -v 'for ((a,a), c in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b), a in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b), b in @*) {}'
+
+mlr_expect_fail -n put -v 'for ((a,a,c), d in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b,a), d in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b,c), a in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b,b), d in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b,c), b in @*) {}'
+mlr_expect_fail -n put -v 'for ((a,b,c), c in @*) {}'
+
+run_mlr --from $indir/xyz2 put -q 'func f() { return {"a"."b":"c"."d",3:4}}; for (k,v in f()){print "k=".k.",v=".v}'
+run_mlr --from $indir/xyz2 put -q 'for (k,v in {"a"."b":"c"."d",3:"c"}) {print "k=".k.",v=".v}'
+run_mlr --from $indir/xyz2 put -q 'o["a"."b"]="c"."d"; for (k,v in o) {print "k=".k.",v=".v}'
+run_mlr --from $indir/xyz2 put -q '@o["a"."b"]="c"."d"; for (k,v in @o) {print "k=".k.",v=".v}'
+run_mlr --from $indir/xyz2 put 'for (k in $*) { print k}'
+run_mlr --from $indir/xyz2 put 'm=$*; for (k in m) { print k}'

go/reg-test/cases/case-c-dsl-for-srec-loops.sh

@@ -0,0 +1,27 @@
+mention empty for-srec
+run_mlr --from $indir/abixy put -v 'for(k,v in $*) { }'
+
+mention for-srec without boundvars
+run_mlr --from $indir/abixy put -v 'for(k,v in $*) {$nr= NR}'
+
+mention for-srec modifying the srec
+run_mlr --from $indir/abixy put -v 'for(k,v in $*) {unset $[k]}; $j = NR'
+run_mlr --from $indir/abixy put -v 'for(k,v in $*) {if (k != "x") {unset $[k]}}; $j = NR'
+run_mlr --from $indir/abixy --opprint put -S -v 'for(k,v in $*) {$[k."_orig"]=v; $[k] = "other"}'
+run_mlr --from $indir/abixy put -S -v 'for(k,v in $*) {$[v]=k}'
+
+run_mlr --from $indir/abixy put -v '
+  $sum = 0;
+  for(k,v in $*) {
+    if (k =~ "^[xy]$") {
+      $sum += $[k]
+    }
+  }'
+
+run_mlr --from $indir/abixy put -S -v '
+  $sum = float(0);
+  for(k,v in $*) {
+    if (k =~ "^[xy]$") {
+      $sum += float($[k])
+    }
+  }'

go/reg-test/cases/case-c-dsl-forbind-typedecl.sh

@@ -0,0 +1,39 @@
+run_mlr --from $indir/abixy put '
+  for (int i = 0; i < $i; i += 1) {
+    $c = i * 10;
+  }
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  for (float i = 0; i < $i; i += 1) {
+    $c = i * 10;
+  }
+'
+
+run_mlr --from $indir/abixy put '
+  for (int i = 0; i < $i; i += 1) {
+    i += 2;
+    $c = i;
+  }
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  for (int i = 0; i < $i; i += 1) {
+    i += 1.5;
+    $c = i;
+  }
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  for (int i = 0; i < $i; i += 1) {
+    i += 1.0;
+    $c = i;
+  }
+'
+
+run_mlr --from $indir/abixy put '
+  for (num i = 0; i < $i; i += 1) {
+    i += 1.0;
+    $c = i;
+  }
+'

go/reg-test/cases/case-c-dsl-from-file.sh

@@ -0,0 +1,9 @@
+
+run_mlr put    -f $indir/put-example.dsl $indir/abixy
+run_mlr filter -f $indir/filter-example.dsl $indir/abixy
+
+run_mlr --from $indir/abixy put    -f $indir/put-example.dsl
+run_mlr --from $indir/abixy filter -f $indir/filter-example.dsl
+
+run_mlr --from $indir/abixy --from $indir/abixy-het put    -f $indir/put-example.dsl
+run_mlr --from $indir/abixy --from $indir/abixy-het filter -f $indir/filter-example.dsl

go/reg-test/cases/case-c-dsl-functional-tests.sh

@@ -0,0 +1,161 @@
+
+run_mlr filter '$x>.3'    $indir/abixy
+run_mlr filter '$x>.3;'   $indir/abixy
+run_mlr filter '$x>0.3'   $indir/abixy
+run_mlr filter '$x>0.3 && $y>0.3'   $indir/abixy
+run_mlr filter '$x>0.3 || $y>0.3'   $indir/abixy
+run_mlr filter 'NR>=4 && NR <= 7'   $indir/abixy
+
+run_mlr filter -x '$x>.3'    $indir/abixy
+run_mlr filter -x '$x>0.3'   $indir/abixy
+run_mlr filter -x '$x>0.3 && $y>0.3'   $indir/abixy
+run_mlr filter -x '$x>0.3 || $y>0.3'   $indir/abixy
+run_mlr filter -x 'NR>=4 && NR <= 7'   $indir/abixy
+
+run_mlr filter '$nosuchfield>.3'    $indir/abixy
+
+run_mlr put '$x2 = $x**2'  $indir/abixy
+run_mlr put '$x2 = $x**2;' $indir/abixy
+run_mlr put '$z = -0.024*$x+0.13' $indir/abixy
+run_mlr put '$c = $a . $b' $indir/abixy
+run_mlr put '$ii = $i + $i' $indir/abixy
+run_mlr put '$emptytest = $i + $nosuch' $indir/abixy
+
+run_mlr --opprint put '$nr=NR;$fnr=FNR;$nf=NF;$filenum=FILENUM' $indir/abixy $indir/abixy
+
+run_mlr --opprint put '$y=madd($x,10,37)' then put '$z=msub($x,10,37)' $indir/modarith.dat
+run_mlr --opprint put '$y=mexp($x,35,37)' then put '$z=mmul($x,$y,37)' $indir/modarith.dat
+
+run_mlr put '$z=min($x, $y)' $indir/minmax.dkvp
+run_mlr put '$z=max($x, $y)' $indir/minmax.dkvp
+
+run_mlr put '$o=min()' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=max()' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=min($x)' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=max($x)' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=min($x,$y)' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=max($x,$y)' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=min($x,$y,$z)' <<EOF
+x=1,y=2,z=3
+EOF
+
+run_mlr put '$o=max($x,$y,$z)' <<EOF
+x=1,y=2,z=3
+EOF
+
+
+run_mlr put '$u=min($x,$y);$v=max($x,$y)' <<EOF
+x=1,y=b
+EOF
+
+run_mlr put '$u=min($x,$y);$v=max($x,$y)' <<EOF
+x=a,y=2
+EOF
+
+run_mlr put '$u=min($x,$y);$v=max($x,$y)' <<EOF
+x=a,y=b
+EOF
+
+
+run_mlr --icsvlite --oxtab put '${x+y} = ${name.x} + ${name.y}; ${x*y} = ${name.x} * ${name.y}' $indir/braced.csv
+run_mlr --icsvlite --oxtab filter '${name.y} < ${z}' $indir/braced.csv
+
+run_mlr --opprint put '$z = $x < 0.5 ? 0 : 1' $indir/abixy
+
+run_mlr --csvlite filter 'true  && true'  $indir/b.csv
+run_mlr --csvlite filter 'true  && false' $indir/b.csv
+run_mlr --csvlite filter 'false && true'  $indir/b.csv
+run_mlr --csvlite filter 'false && false' $indir/b.csv
+
+run_mlr --csvlite filter 'true  || true'  $indir/b.csv
+run_mlr --csvlite filter 'true  || false' $indir/b.csv
+run_mlr --csvlite filter 'false || true'  $indir/b.csv
+run_mlr --csvlite filter 'false || false' $indir/b.csv
+
+run_mlr --csvlite filter 'true  ^^ true'  $indir/b.csv
+run_mlr --csvlite filter 'true  ^^ false' $indir/b.csv
+run_mlr --csvlite filter 'false ^^ true'  $indir/b.csv
+run_mlr --csvlite filter 'false ^^ false' $indir/b.csv
+
+# This tests boolean short-circuiting
+run_mlr put '$x==2 && $a =~ "....." { $y=4 }'  $indir/short-circuit.dkvp
+
+export X=97
+export Y=98
+run_mlr put '$x = ENV["X"]; $y = ENV[$name]' $indir/env-var.dkvp
+export X=
+export Y=
+run_mlr put '$x = ENV["X"]; $y = ENV[$name]' $indir/env-var.dkvp
+
+run_mlr -n put 'begin{ENV["HOME"]="foobar"} end{print ENV["HOME"]}'
+
+run_mlr put '$y = toupper($x)' <<EOF
+x=hello
+EOF
+
+run_mlr put '$y = toupper($x)' <<EOF
+x=HELLO
+EOF
+
+run_mlr put '$y = toupper($x)' <<EOF
+x=
+EOF
+
+run_mlr put '$y = toupper($z)' <<EOF
+x=hello
+EOF
+
+
+run_mlr put '$y = tolower($x)' <<EOF
+x=hello
+EOF
+
+run_mlr put '$y = tolower($x)' <<EOF
+x=HELLO
+EOF
+
+run_mlr put '$y = tolower($x)' <<EOF
+x=
+EOF
+
+run_mlr put '$y = tolower($z)' <<EOF
+x=hello
+EOF
+
+
+run_mlr put '$y = capitalize($x)' <<EOF
+x=hello
+EOF
+
+run_mlr put '$y = capitalize($x)' <<EOF
+x=HELLO
+EOF
+
+run_mlr put '$y = capitalize($x)' <<EOF
+x=
+EOF
+
+run_mlr put '$y = capitalize($z)' <<EOF
+x=hello
+EOF
+
+mention LHS value on first record should result in ZYX for process creation
+export indir; run_mlr --from $indir/abixy put -q 'ENV["ZYX"]="CBA".NR; print | ENV["indir"]."/env-assign.sh" , "a is " . $a'

go/reg-test/cases/case-c-dsl-gmt-date-time-functions.sh

@@ -0,0 +1,23 @@
+
+run_mlr --csvlite put '$gmt = sec2gmt($sec)' $indir/sec2gmt
+run_mlr --csvlite put '$gmt = sec2gmt($sec,1)' $indir/sec2gmt
+run_mlr --csvlite put '$gmt = sec2gmt($sec,3)' $indir/sec2gmt
+run_mlr --csvlite put '$gmt = sec2gmt($sec,6)' $indir/sec2gmt
+run_mlr --csvlite put '$sec = gmt2sec($gmt)' $indir/gmt2sec
+run_mlr --csvlite put '$gmtdate = sec2gmtdate($sec)' $indir/sec2gmt
+
+run_mlr --icsv --opprint put '$gmt = strftime($sec, "%Y-%m-%dT%H:%M:%SZ")'  $indir/sec2gmt
+run_mlr --icsv --opprint put '$gmt = strftime($sec, "%Y-%m-%dT%H:%M:%1SZ")' $indir/sec2gmt
+run_mlr --icsv --opprint put '$gmt = strftime($sec, "%Y-%m-%dT%H:%M:%3SZ")' $indir/sec2gmt
+run_mlr --icsv --opprint put '$gmt = strftime($sec, "%Y-%m-%dT%H:%M:%6SZ")' $indir/sec2gmt
+run_mlr --icsv --opprint put '$sec = strptime($gmt, "%Y-%m-%dT%H:%M:%SZ")'  $indir/gmt2sec
+
+run_mlr --csvlite sec2gmt sec $indir/sec2gmt
+
+run_mlr --opprint put '$hms=sec2hms($sec);   $resec=hms2sec($hms);   $diff=$resec-$sec' $indir/sec2xhms
+run_mlr --opprint put '$hms=fsec2hms($sec);  $resec=hms2fsec($hms);  $diff=$resec-$sec' $indir/fsec2xhms
+run_mlr --opprint put '$hms=sec2dhms($sec);  $resec=dhms2sec($hms);  $diff=$resec-$sec' $indir/sec2xhms
+run_mlr --opprint put '$hms=fsec2dhms($sec); $resec=dhms2fsec($hms); $diff=$resec-$sec' $indir/fsec2xhms
+
+run_mlr --csvlite sec2gmt     sec $indir/sec2gmt
+run_mlr --csvlite sec2gmtdate sec $indir/sec2gmt

go/reg-test/cases/case-c-dsl-haskey.sh

@@ -0,0 +1,18 @@
+run_mlr --from $indir/abixy-het put '$haskeya = haskey($*, "a")'
+run_mlr --from $indir/abixy-het put '$haskey3 = haskey($*, 3)'
+
+run_mlr --from $indir/xyz2 put '$haskeya = haskey({3:4}, "a")'
+run_mlr --from $indir/xyz2 put '$haskey3 = haskey({3:4}, 3)'
+run_mlr --from $indir/xyz2 put '$haskey3 = haskey({3:4}, 4)'
+
+run_mlr --from $indir/xyz2 put 'o = {3:4}; $haskeya = haskey(o, "a")'
+run_mlr --from $indir/xyz2 put 'o = {3:4}; $haskey3 = haskey(o, 3)'
+run_mlr --from $indir/xyz2 put 'o = {3:4}; $haskey3 = haskey(o, 4)'
+
+run_mlr --from $indir/xyz2 put '@o = {3:4}; $haskeya = haskey(@o, "a")'
+run_mlr --from $indir/xyz2 put '@o = {3:4}; $haskey3 = haskey(@o, 3)'
+run_mlr --from $indir/xyz2 put '@o = {3:4}; $haskey3 = haskey(@o, 4)'
+
+run_mlr --from $indir/xyz2 put 'o = "3:4"; $haskeya = haskey(@o, "a")'
+run_mlr --from $indir/xyz2 put 'o = "3:4"; $haskey3 = haskey(@o, 3)'
+run_mlr --from $indir/xyz2 put 'o = "3:4"; $haskey3 = haskey(@o, 4)'

go/reg-test/cases/case-c-dsl-if-chaining.sh

@@ -0,0 +1,103 @@
+mention IF/ELIF WITH ELSE
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR==1) {
+    $x = 2;
+    $y = 3
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 4) {
+    $x = 5;
+    $y = 6
+  } else {
+    $x = 1007;
+    $y = 1008
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 9) {
+    $x = 10;
+    $y = 11
+  } elif (NR == 12) {
+    $x = 13;
+    $y = 14
+  } else {
+    $x = 1015;
+    $y = 1016
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 17) {
+    $x = 18;
+    $y = 19
+  } elif (NR == 20) {
+    $x = 21;
+    $y = 22
+  } elif (NR == 23) {
+    $x = 24;
+    $y = 25
+  } else {
+    $x = 1026;
+    $y = 1027
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 28) {
+    $x = 29;
+    $y = 30
+  } elif (NR == 31) {
+    $x = 32;
+    $y = 33
+  } elif (NR == 34) {
+    $x = 35;
+    $y = 36
+  } elif (NR == 37) {
+    $x = 38;
+    $y = 39
+  } else {
+    $x = 1040;
+    $y = 1041
+  }'
+
+mention IF/ELIF WITHOUT ELSE
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 1) {
+    $x = 2;
+    $y = 3
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 4) {
+    $x = 5;
+    $y = 6
+  } elif (NR == 7) {
+    $x = 8;
+    $y = 9
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 10) {
+    $x = 11;
+    $y = 12
+  } elif (NR == 13) {
+    $x = 14;
+    $y = 15
+  } elif (NR == 16) {
+    $x = 17;
+    $y = 18
+  }'
+
+run_mlr --from $indir/xy40.dkvp put -v '
+  if (NR == 19) {
+    $x = 20;
+    $y = 21
+  } elif (NR == 22) {
+    $x = 23;
+    $y = 24
+  } elif (NR == 25) {
+    $x = 26;
+    $y = 37
+  } elif (NR == 28) {
+    $x = 29;
+    $y = 30
+  }'

go/reg-test/cases/case-c-dsl-indirect-oosvar-assignments.sh

@@ -0,0 +1,11 @@
+run_mlr --opprint put -v '@s = NR; $t = @s; $u=@["s"]; $v = $t - $u' $indir/abixy
+
+run_mlr put -v '@t["u"] = NR; $tu = @["t"]["u"]; emitp all' $indir/abixy
+run_mlr put -v '@t["u"] = NR; $tu = @["t"]["u"]; emitp @*' $indir/abixy
+
+run_mlr put -v '@["s"] = $x; emitp all' $indir/abixy
+
+run_mlr put -v '@["t"]["u"] = $y; emitp all' $indir/abixy
+
+# xxx @* on the right
+# xxx @* on the left

go/reg-test/cases/case-c-dsl-indirect-srec-assignments.sh

@@ -0,0 +1,2 @@
+run_mlr put -v '$["a"] = $["b"]; $["x"] = 10 * $["y"]' $indir/abixy
+run_mlr --from $indir/abixy put 'while (NF < 256) { $["k".string(NF+1)] = "v".string(NF) }'

go/reg-test/cases/case-c-dsl-is-predicates.sh

@@ -0,0 +1,132 @@
+
+run_mlr --opprint put '$f=is_absent($x)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_absent($y)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_absent($z)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_absent($nosuch)'                     $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_absent(@nosuch)'                     $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_absent(@somesuch)'       $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_bool($x>1)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_bool($y>1)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_bool($z>1)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_bool($nosuch>1)'                     $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_bool(@nosuch>1)'                     $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_bool(@somesuch>1)'       $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_boolean($x>1)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_boolean($y>1)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_boolean($z>1)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_boolean($nosuch>1)'                  $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_boolean(@nosuch>1)'                  $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_boolean(@somesuch>1)'    $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_empty($x)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty($y)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty($z)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty($nosuch)'                      $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty(@nosuch)'                      $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty($*)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty({1:2})'                        $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_empty(@somesuch)'        $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_empty_map($x)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map($y)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map($z)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map($nosuch)'                  $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map(@nosuch)'                  $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map($*)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map({1:2})'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_empty_map({})'                       $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_empty_map(@somesuch)'    $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_float($x)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_float($y)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_float($z)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_float($nosuch)'                      $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_float(@nosuch)'                      $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_float($*)'                           $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_float({1:2})'                        $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_float(@somesuch)'        $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_int($x)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_int($y)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_int($z)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_int($nosuch)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_int(@nosuch)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_int($*)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_int({1:2})'                          $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_int(@somesuch)'          $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_map($x)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map($y)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map($z)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map($nosuch)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map(@nosuch)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map($*)'                             $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map({1:2})'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_map({})'                             $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_map(@somesuch)'          $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_nonempty_map($x)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map($y)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map($z)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map($nosuch)'               $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map(@nosuch)'               $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map($*)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map({1:2})'                 $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_nonempty_map({})'                    $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_nonempty_map(@somesuch)' $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_not_empty($x)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_empty($y)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_empty($z)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_empty($nosuch)'                  $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_empty(@nosuch)'                  $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_empty($*)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_empty({1:2})'                    $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_not_empty(@somesuch)'    $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_not_map($x)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map($y)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map($z)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map($nosuch)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map(@nosuch)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map($*)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map({1:2})'                      $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_map({})'                         $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_not_map(@somesuch)'      $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_not_null($x)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_null($y)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_null($z)'                        $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_null($nosuch)'                   $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_not_null(@nosuch)'                   $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_not_null(@somesuch)'     $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_null($x)'                            $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_null($y)'                            $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_null($z)'                            $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_null($nosuch)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_null(@nosuch)'                       $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_null(@somesuch)'         $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_numeric($x)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_numeric($y)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_numeric($z)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_numeric($nosuch)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_numeric(@nosuch)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_numeric(@somesuch)'      $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_present($x)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_present($y)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_present($z)'                         $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_present($nosuch)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_present(@nosuch)'                    $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_present(@somesuch)'      $indir/nullvals.dkvp
+
+run_mlr --opprint put '$f=is_string($x)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_string($y)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_string($z)'                          $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_string($nosuch)'                     $indir/nullvals.dkvp
+run_mlr --opprint put '$f=is_string(@nosuch)'                     $indir/nullvals.dkvp
+run_mlr --opprint put '@somesuch=1;$f=is_string(@somesuch)'       $indir/nullvals.dkvp

go/reg-test/cases/case-c-dsl-lashed-emitp-singles.sh

@@ -0,0 +1,428 @@
+run_mlr -n put 'end {
+  @a = 111;
+  emitp @a
+}'
+run_mlr -n put 'end {
+  @a = 111;
+  emitp (@a)
+}'
+
+run_mlr -n put 'end {
+  @a[111] = 222;
+  emitp @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111] = 222;
+  emitp (@a), "s"
+}'
+
+run_mlr -n put 'end {
+  @a[111] = 222;
+  @a[333] = 444;
+  emitp @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111] = 222;
+  @a[333] = 444;
+  emitp (@a), "s"
+}'
+
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emitp @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emitp (@a), "s"
+}'
+
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  @a[444][555] = 666;
+  emitp @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  @a[444][555] = 666;
+  emitp (@a), "s"
+}'
+
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emitp @a, "s", "t"
+}'
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emitp (@a), "s", "t"
+}'
+
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emitp @a[111], "t"
+}'
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emitp (@a[111]), "t"
+}'
+
+# ----------------------------------------------------------------
+announce LASHED EMIT SINGLES
+
+run_mlr -n put 'end {
+  @a = 111;
+  emit @a
+}'
+run_mlr -n put 'end {
+  @a = 111;
+  emit (@a)
+}'
+
+run_mlr -n put 'end {
+  @a[111] = 222;
+  emit @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111] = 222;
+  emit (@a), "s"
+}'
+
+run_mlr -n put 'end {
+  @a[111] = 222;
+  @a[333] = 444;
+  emit @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111] = 222;
+  @a[333] = 444;
+  emit (@a), "s"
+}'
+
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emit @a, "s"
+}'
+run_mlr -n put 'end {
+  @a[111][222] = 333;
+  emit (@a), "s"
+}'
+
+# ----------------------------------------------------------------
+announce LASHED EMITP PAIRS
+
+run_mlr -n put 'end {
+  @a = 111;
+  @b = 222;
+  emitp (@a, @b)
+}'
+
+run_mlr -n put 'end {
+  @a[1] = 111;
+  @b[1] = 222;
+  emitp (@a[1], @b[1])
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[1][2][3] = 8;
+  emitp (@a, @b), "s", "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emitp (@a[1], @b[5]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][6][3] = 8;
+  emitp (@a[1][2], @b[5][6]), "u"
+}'
+
+# ----------------------------------------------------------------
+announce LASHED EMIT PAIRS
+
+run_mlr -n put 'end {
+  @a = 111;
+  @b = 222;
+  emit (@a, @b)
+}'
+
+run_mlr -n put 'end {
+  @a[1] = 111;
+  @b[1] = 222;
+  emit (@a[1], @b[1])
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[1][2][3] = 8;
+  emit (@a, @b), "s", "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emit (@a[1], @b[5]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][6][3] = 8;
+  emit (@a[1][2], @b[5][6]), "u"
+}'
+
+# ----------------------------------------------------------------
+announce LASHED EMIT WITH VARYING DEPTH
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emit (@a[1], @b[2]), "t"
+}'
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emit (@a, @b), "s", "t"
+}'
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @a[3] = 10;
+  @a[4] = 11;
+  @a[5][6][7] = 12;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emit (@a, @b), "s", "t"
+}'
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @a[3] = 10;
+  @a[4] = 11;
+  @a[5][6][7] = 12;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emit (@b, @a), "s", "t"
+}'
+
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emit (@a[1], @b[3]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emit (@a[1][2], @b[5][9]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emit (@a[1][2], @b[9][2]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emit (@a[9], @b[5]), "t", "u"
+}'
+
+# ----------------------------------------------------------------
+announce LASHED EMITP WITH VARYING DEPTH
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emitp (@a[1], @b[2]), "t"
+}'
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emitp (@a, @b), "s", "t"
+}'
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @a[3] = 10;
+  @a[4] = 11;
+  @a[5][6][7] = 12;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emitp (@a, @b), "s", "t"
+}'
+
+run_mlr -n put 'end {
+  @a[1][1] = 1;
+  @a[1][2] = 2;
+  @a[2][1] = 3;
+  @a[2][2] = 4;
+  @a[3] = 10;
+  @a[4] = 11;
+  @a[5][6][7] = 12;
+  @b[1][1] = 5;
+  @b[1][2] = 6;
+  @b[2][1] = 7;
+  @b[2][2] = 8;
+  emitp (@b, @a), "s", "t"
+}'
+
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emitp (@a[1], @b[3]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emitp (@a[1][2], @b[5][9]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emitp (@a[1][2], @b[9][2]), "t", "u"
+}'
+
+run_mlr -n put 'end {
+  @a[1][2][3] = 4;
+  @b[5][2][3] = 8;
+  emitp (@a[9], @b[5]), "t", "u"
+}'
+
+# ----------------------------------------------------------------
+announce CANONICAL LASHED EMIT
+
+run_mlr --from $indir/abixy-wide --opprint put -q '
+  @count[$a] += 1;
+  @sum[$a] += $x;
+  end {
+      for (a, c in @count) {
+          @mean[a] = @sum[a] / @count[a]
+      }
+      emit (@sum, @count, @mean), "a"
+  }
+'
+
+run_mlr --from $indir/abixy-wide --opprint put -q '
+  @count[$a][$b] += 1;
+  @sum[$a][$b] += $x;
+  end {
+      for ((a, b), c in @count) {
+          @mean[a][b] = @sum[a][b] / @count[a][b]
+      }
+      emit (@sum, @count, @mean), "a", "b"
+  }
+'
+
+# ----------------------------------------------------------------
+announce MAP-VARIANT EMITS
+
+run_mlr         --from $indir/abixy-het --opprint put -q 'o=$a.$b; emit o'
+run_mlr         --from $indir/abixy-het --opprint put -q 'o={"ab":$a.$b}; emit o'
+
+run_mlr         --from $indir/abixy-het --opprint put -q '@o=$a.$b; emit @o'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o={"ab":$a.$b}; emit @o'
+
+run_mlr         --from $indir/abixy-het --opprint put -q '@o=$a.$b; emit @*'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o={"ab":$a.$b}; emit @*'
+
+mlr_expect_fail --from $indir/abixy-het --opprint put -q 'emit $a.$b'
+run_mlr         --from $indir/abixy-het --opprint put -q 'emit {"ab":$a.$b}'
+
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a,b) { return a.b } o = f($a, $b); emit o'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a,b) { return a.b } emit f($a, $b)'
+
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a,b) { return {"ab": a.b} } o = f($a, $b); emit o'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a,b) { return {"ab": a.b} } emit f($a, $b)'
+
+# ----------------------------------------------------------------
+announce MAP-VARIANT LASHED EMITS
+
+# scalar emits aren't reasonable ... but they shouldn't segv
+
+mention scalar lashed emits
+mlr_expect_fail --from $indir/abixy-het --opprint put -q 'emit ($a . "_" . $b, $x . "_" . $y)'
+run_mlr         --from $indir/abixy-het --opprint put -q ' o = $a . "_" . $b;  p = $x . "_" . $y; emit  (o,  p)'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o = $a . "_" . $b; @p = $x . "_" . $y; emit (@o, @p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return a . "_" . b }  o = f($a, $b);  p = f($x, $y); emit  (o,  p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return a . "_" . b } @o = f($a, $b); @p = f($x, $y); emit (@o, @p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return a . "_" . b } emit (f($a, $b), f($x, $y))'
+
+mention non-scalar non-keyed lashed emits
+run_mlr         --from $indir/abixy-het --opprint put -q 'emit ({"ab": $a . "_" . $b}, {"ab": $x . "_" . $y})'
+run_mlr         --from $indir/abixy-het --opprint put -q ' o = {"ab": $a . "_" . $b};  p = {"ab": $x . "_" . $y}; emit  (o, p)'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o = {"ab": $a . "_" . $b}; @p = {"ab": $x . "_" . $y}; emit (@o, @p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} }  o = f($a, $b);  p = f($x, $y); emit  (o, p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } @o = f($a, $b); @p = f($x, $y); emit (@o, @p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } emit (f($a, $b), f($x, $y))'
+
+mention non-scalar non-keyed lashed emits
+run_mlr         --from $indir/abixy-het --opprint put -q 'emitp ({"ab": $a . "_" . $b}, {"ab": $x . "_" . $y})'
+run_mlr         --from $indir/abixy-het --opprint put -q ' o = {"ab": $a . "_" . $b};  p = {"ab": $x . "_" . $y}; emitp  (o, p)'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o = {"ab": $a . "_" . $b}; @p = {"ab": $x . "_" . $y}; emitp (@o, @p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} }  o = f($a, $b);  p = f($x, $y); emitp  (o, p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } @o = f($a, $b); @p = f($x, $y); emitp (@o, @p)'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } emit (f($a, $b), f($x, $y))'
+
+mention non-scalar keyed lashed emits
+run_mlr         --from $indir/abixy-het --opprint put -q 'emit ({"ab": $a . "_" . $b}, {"ab": $x . "_" . $y}), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q ' o = {"ab": $a . "_" . $b};  p = {"ab": $x . "_" . $y}; emit  (o, p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o = {"ab": $a . "_" . $b}; @p = {"ab": $x . "_" . $y}; emit (@o, @p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} }  o = f($a, $b);  p = f($x, $y); emit  (o, p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } @o = f($a, $b); @p = f($x, $y); emit (@o, @p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } emit (f($a, $b), f($x, $y)), "ab"'
+
+mention non-scalar keyed lashed emits
+run_mlr         --from $indir/abixy-het --opprint put -q 'emitp ({"ab": $a . "_" . $b}, {"ab": $x . "_" . $y}), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q ' o = {"ab": $a . "_" . $b};  p = {"ab": $x . "_" . $y}; emitp  (o, p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q '@o = {"ab": $a . "_" . $b}; @p = {"ab": $x . "_" . $y}; emitp (@o, @p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} }  o = f($a, $b);  p = f($x, $y); emitp  (o, p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } @o = f($a, $b); @p = f($x, $y); emitp (@o, @p), "ab"'
+run_mlr         --from $indir/abixy-het --opprint put -q 'func f(a, b) { return {"ab": a . "_" . b} } emitp (f($a, $b), f($x, $y)), "ab"'

go/reg-test/cases/case-c-dsl-length-depth-leafcount.sh

@@ -0,0 +1,20 @@
+run_mlr --from $indir/abixy-het put '$length = length($a)'
+run_mlr --from $indir/abixy-het put '$length = length($*)'
+run_mlr --from $indir/xyz2 put '$length= length({3:4, 5:{6:7}, 8:{9:{10:11}}})'
+run_mlr --from $indir/xyz2 put 'o = {3:4, 5:{6:7}, 8:{9:{10:11}}}; $length = length(o)'
+run_mlr --from $indir/xyz2 put '@o = {3:4, 5:{6:7}, 8:{9:{10:11}}}; $length = length(@o)'
+
+run_mlr --from $indir/abixy-het put '$depth = depth($a)'
+run_mlr --from $indir/abixy-het put '$depth = depth($*)'
+run_mlr --from $indir/xyz2 put '$depth= depth({3:4, 5:{6:7}, 8:{9:{10:11}}})'
+run_mlr --from $indir/xyz2 put 'o = {3:4, 5:{6:7}, 8:{9:{10:11}}}; $depth = depth(o)'
+run_mlr --from $indir/xyz2 put '@o = {3:4, 5:{6:7}, 8:{9:{10:11}}}; $depth = depth(@o)'
+
+run_mlr --from $indir/abixy-het put '$leafcount = leafcount($a)'
+run_mlr --from $indir/abixy-het put '$leafcount = leafcount($*)'
+run_mlr --from $indir/xyz2 put '$leafcount= leafcount({3:4, 5:{6:7}, 8:{9:{10:11}}})'
+run_mlr --from $indir/xyz2 put 'o = {3:4, 5:{6:7}, 8:{9:{10:11}}}; $leafcount = leafcount(o)'
+run_mlr --from $indir/xyz2 put '@o = {3:4, 5:{6:7}, 8:{9:{10:11}}}; $leafcount = leafcount(@o)'
+
+# xxx ternary operator (and most RHS expressions in the grammar) doesn't handle maps.
+# run_mlr --from $indir/abixy-het put -q 'o = haskey(NR==4 ? {"a": NF} : {"b": NF}, "a"); print "NR=".NR.",haskeya=".o'

go/reg-test/cases/case-c-dsl-local-date-time-functions.sh

@@ -0,0 +1,73 @@
+
+# See also the system date command:
+# export TZ=America/Sao_Paulo; date -j -f "%Y-%m-%d %H:%M:%S %Z" "2017-02-19 00:30:00" +%s
+# export TZ=America/Sao_Paulo; date -r  86400 +"%Y-%m-%d %H:%M:%S %Z"
+
+export TZ=America/Sao_Paulo
+#echo TZ=$TZ
+run_mlr --opprint put '$b=localtime2sec($a); $c=sec2localtime($b); $d=sec2localdate($b)' <<_EOF
+a=2017-02-18 23:00:00
+a=2017-02-18 23:59:59
+a=2017-02-19 00:00:00
+a=2017-02-19 00:30:00
+a=2017-02-19 01:00:00
+a=2017-10-14 23:00:00
+a=2017-10-14 23:59:59
+a=2017-10-15 00:00:00
+a=2017-10-15 00:30:00
+a=2017-10-15 01:00:00
+_EOF
+export TZ=
+
+export TZ=America/Sao_Paulo
+#echo TZ=$TZ
+run_mlr --opprint put '$b=localtime2sec($a); $c=sec2localtime($b); $d=sec2localdate($b)' <<_EOF
+a=2017-02-14 00:00:00
+a=2017-02-15 00:00:00
+a=2017-02-16 00:00:00
+a=2017-02-17 00:00:00
+a=2017-02-18 00:00:00
+a=2017-02-19 00:00:00
+a=2017-02-20 00:00:00
+a=2017-10-12 00:00:00
+a=2017-10-13 00:00:00
+a=2017-10-14 00:00:00
+a=2017-10-15 00:00:00
+a=2017-10-16 00:00:00
+a=2017-10-17 00:00:00
+a=2017-10-18 00:00:00
+a=2017-10-19 00:00:00
+_EOF
+export TZ=
+
+export TZ=America/Sao_Paulo
+#echo TZ=$TZ
+run_mlr --opprint put '$b=strptime_local($a, "%Y-%m-%d %H:%M:%S"); $c=strftime_local($b, "%Y-%m-%d %H:%M:%S")' <<_EOF
+a=2017-02-18 23:00:00
+a=2017-02-18 23:59:59
+a=2017-02-19 00:00:00
+a=2017-02-19 00:30:00
+a=2017-02-19 01:00:00
+a=2017-10-14 23:00:00
+a=2017-10-14 23:59:59
+a=2017-10-15 00:00:00
+a=2017-10-15 00:30:00
+a=2017-10-15 01:00:00
+_EOF
+export TZ=
+
+export TZ=America/Sao_Paulo
+#echo TZ=$TZ
+run_mlr --opprint put '$b=strptime_local($a, "%Y-%m-%d %H:%M:%S"); $c=strftime_local($b, "%Y-%m-%d %H:%M:%S")' <<_EOF
+a=2017-02-18 23:00:00
+a=2017-02-18 23:59:59
+a=2017-02-19 00:00:00
+a=2017-02-19 00:30:00
+a=2017-02-19 01:00:00
+a=2017-10-14 23:00:00
+a=2017-10-14 23:59:59
+a=2017-10-15 00:00:00
+a=2017-10-15 00:30:00
+a=2017-10-15 01:00:00
+_EOF
+export TZ=

go/reg-test/cases/case-c-dsl-local-map-variable-typedecl.sh

@@ -0,0 +1,22 @@
+
+run_mlr --from $indir/xyz345 put '
+    map a = {};
+'
+
+run_mlr --from $indir/xyz345 put '
+    map a = {};
+    a[1]=2;
+    a[3][4]=5;
+'
+
+mlr_expect_fail --from $indir/xyz345 put '
+    map a = {};
+    a=2;
+    a[3][4]=5;
+'
+
+mlr_expect_fail --from $indir/xyz345 put '
+    map a = {};
+    a[3][4]=5;
+    a=2;
+'

go/reg-test/cases/case-c-dsl-local-map-variable-write-read.sh

@@ -0,0 +1,50 @@
+run_mlr --from $indir/xyz345 put -v -a '
+    $s = a;
+    $t = b[$x];
+    $u = c[$x][$y];
+    $v = d[$x][$y][$z];
+'
+
+run_mlr --from $indir/xyz345 put -v -a '
+    d[$x][$y][$z] = 9;
+    $d = d[$x][$y][$z];
+'
+
+run_mlr --from $indir/xyz345 put -v -a '
+    a = 6;
+    b[$x] = 7;
+    c[$x][$y] = 8;
+    d[$x][$y][$z] = 9;
+
+    $a = a;
+    $b = b[$x];
+    $c = c[$x][$y];
+    $d = d[$x][$y][$z];
+'
+
+run_mlr --from $indir/xyz345 put -v -a '
+    a = 6;
+    b[$x] = 7;
+    c[$x][$y] = 8;
+    d[$x][$y][$z] = 9;
+
+    $a0 = a;
+    $a1 = a[$x];
+    $a2 = a[$x][$y];
+    $a3 = a[$x][$y][$z];
+
+    $b0 = b;
+    $b1 = b[$x];
+    $b2 = b[$x][$y];
+    $b3 = b[$x][$y][$z];
+
+    $c0 = c;
+    $c1 = c[$x];
+    $c2 = c[$x][$y];
+    $c3 = c[$x][$y][$z];
+
+    $d0 = d;
+    $d1 = d[$x];
+    $d2 = d[$x][$y];
+    $d3 = d[$x][$y][$z];
+'

go/reg-test/cases/case-c-dsl-localvar-scope-and-extent.sh

@@ -0,0 +1,54 @@
+run_mlr --opprint --from $indir/abixy put '
+  func f() {
+    var a = 1;
+    if (NR > 5) {
+      a = 2;
+    }
+    return a;
+  }
+  func g() {
+    var b = 1;
+    if (NR > 5) {
+      var b = 2;
+    }
+    return b;
+  }
+  func h() {
+    var a = 1;
+    if (NR > 5) {
+      a = 2;
+      return a;
+    }
+    return a;
+  }
+  func i() {
+    var b = 1;
+    if (NR > 5) {
+      var b = 2;
+      return b;
+    }
+    return b;
+  }
+  $of = f();
+  $og = g();
+  $oh = h();
+  $oi = i();
+ '
+
+# test fencing at function entry
+run_mlr --opprint --from $indir/abixy put '
+  func f() {
+    var a = 2;
+    var b = 2;
+    if (NR > 5) {
+      var a = 3;
+      b = 3;
+    }
+    return 1000 * a + b;
+  }
+  a = 1;
+  b = 1;
+  $ab = f();
+  $oa = a;
+  $ob = b;
+ '

go/reg-test/cases/case-c-dsl-localvar-typedecl.sh

@@ -0,0 +1,23 @@
+run_mlr --from $indir/abixy put '
+  str a = $a;
+  a = "a:".NR;
+  $c = a;
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  str a = $a;
+  a = NR;
+  $c = a;
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  int a = $a;
+  a = NR;
+  $c = a;
+'
+
+run_mlr --from $indir/abixy put '
+  int i = $i;
+  i = NR;
+  $c = a;
+'

go/reg-test/cases/case-c-dsl-map-variant-dumps.sh

@@ -0,0 +1,5 @@
+# ----------------------------------------------------------------
+announce MAP-VARIANT DUMPS
+
+run_mlr --from $indir/abixy-het put -q 'dump {"a"."b":$a.$b}'
+run_mlr --from $indir/abixy-het put -q 'func f(a, b) { return {"a"."b":a.b} } dump f($a, $b)'

go/reg-test/cases/case-c-dsl-mapsum-mapdiff-mapexcept.sh

@@ -0,0 +1,33 @@
+run_mlr --from $indir/abixy-het put -q 'o = mapsum(); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff(); dump o'
+mlr_expect_fail --from $indir/abixy-het put -q 'o = mapexcept(); dump o'
+mlr_expect_fail --from $indir/abixy-het put -q 'o = mapselect(); dump o'
+
+run_mlr --from $indir/abixy-het put -q 'o = mapsum($*); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapsum({"a":999}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff($*); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff({"a":999}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapexcept($*); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapexcept({"a":999}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect($*); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect({"a":999}); dump o'
+
+run_mlr --from $indir/abixy-het put -q 'o = mapsum($*, {"a":999}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapsum({"a":999}, $*); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff($*, {"a":999}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff({"a":999}, $*); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapexcept($*, "a"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapexcept({"a":999}, "a"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapexcept({"a":999}, "nonesuch"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect($*, "a"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect({"a":999}, "a"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect({"a":999}, "nonesuch"); dump o'
+
+run_mlr --from $indir/abixy-het put -q 'o = mapsum($*, {"a":999}, {"b": 444}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapsum({"a":999}, $*, {"b": 444}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff($*, {"a":999}, {"b": 444}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapdiff({"a":999}, $*, {"b": 444}); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapexcept($*, "a", "b"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect({"a":999}, "b", "nonesuch"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect($*, "a", "b"); dump o'
+run_mlr --from $indir/abixy-het put -q 'o = mapselect({"a":999}, "b", "nonesuch"); dump o'

go/reg-test/cases/case-c-dsl-mapvar-assignments.sh

@@ -0,0 +1,139 @@
+
+# ----------------------------------------------------------------
+announce MAPVAR ASSIGNMENTS TO FULL SREC
+
+run_mlr --from $indir/xyz2 put '$* = {"a":1, "b": {"x":8,"y":9}, "c":3}'
+run_mlr --from $indir/xyz2 put '$* = $*'
+run_mlr --from $indir/xyz2 put '@a = 1; $* = @a'
+run_mlr --from $indir/xyz2 put '@b[1] = 2; $* = @b'
+run_mlr --from $indir/xyz2 put '@c[1][2] = 3; $* = @c'
+run_mlr --from $indir/xyz2 put '@a = 1; $* = @*'
+run_mlr --from $indir/xyz2 put '@b[1] = 2; $* = @*'
+run_mlr --from $indir/xyz2 put '@c[1][2] = 3; $* = @*'
+run_mlr --from $indir/xyz2 put 'a = 1; $* = a'
+run_mlr --from $indir/xyz2 put 'b[1] = 2; $* = b'
+run_mlr --from $indir/xyz2 put 'c[1][2] = 3; $* = c'
+run_mlr --from $indir/xyz2 put '$* = 3'
+
+run_mlr --from $indir/xyz2 put '
+  func map_valued_func() {
+    return {"a":1,"b":2}
+  }
+  map m = map_valued_func();
+  $* = m;
+'
+
+run_mlr --from $indir/xyz2 put '
+  func map_valued_func() {
+    return {"a":1,"b":2}
+  }
+  $* = map_valued_func();
+'
+
+# ----------------------------------------------------------------
+announce MAPVAR ASSIGNMENTS TO FULL OOSVAR
+
+run_mlr --from $indir/xyz2 put '@* = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump'
+run_mlr --from $indir/xyz2 put '@* = $*; dump'
+run_mlr --from $indir/xyz2 put '@* = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump; @* = @*; dump'
+run_mlr --from $indir/xyz2 put '@a = 1; @* = @a; dump'
+run_mlr --from $indir/xyz2 put '@b[1] = 2; @* = @b; dump'
+run_mlr --from $indir/xyz2 put '@c[1][2] = 3; @* = @c; dump'
+run_mlr --from $indir/xyz2 put '@a = 1; @* = @*; dump'
+run_mlr --from $indir/xyz2 put '@b[1] = 2; @* = @*; dump'
+run_mlr --from $indir/xyz2 put '@c[1][2] = 3; @* = @*; dump'
+run_mlr --from $indir/xyz2 put 'a = 1; @* = a; dump'
+run_mlr --from $indir/xyz2 put 'b[1] = 2; @* = b; dump'
+run_mlr --from $indir/xyz2 put 'c[1][2] = 3; @* = c; dump'
+run_mlr --from $indir/xyz2 put '@* = 3'
+
+run_mlr --from $indir/xyz2 put '
+  func map_valued_func() {
+    return {"a":1,"b":2}
+  }
+  map m = map_valued_func();
+  @* = m;
+  dump
+'
+
+run_mlr --from $indir/xyz2 put '
+  func map_valued_func() {
+    return {"a":1,"b":2}
+  }
+  @* = map_valued_func();
+  dump
+'
+
+# ----------------------------------------------------------------
+announce MAPVAR ASSIGNMENTS TO OOSVAR
+
+run_mlr --from $indir/xyz2 put -q '@o = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump @o'
+run_mlr --from $indir/xyz2 put -q '@o = @o; dump @o'
+run_mlr --from $indir/xyz2 put -q '@o = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump; @o = @*; dump'
+run_mlr --from $indir/xyz2 put -q '@o = $*; dump @o'
+run_mlr --from $indir/xyz2 put -q '@o = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump @o; @o = @o; dump @o'
+run_mlr --from $indir/xyz2 put -q '@o = {"a":1, "b": {"x":8,"y":9}, "c":3}; @o = @o; dump @o'
+run_mlr --from $indir/xyz2 put -q '@a = 1; @o = @a; dump @o'
+run_mlr --from $indir/xyz2 put -q '@b[1] = 2; @o = @b; dump @o'
+run_mlr --from $indir/xyz2 put -q '@c[1][2] = 3; @o = @c; dump @o'
+run_mlr --from $indir/xyz2 put -q '@a = 1; @o = @*; dump @o'
+run_mlr --from $indir/xyz2 put -q '@b[1] = 2; @o = @*; dump @o'
+run_mlr --from $indir/xyz2 put -q '@c[1][2] = 3; @o = @*; dump @o'
+run_mlr --from $indir/xyz2 put -q 'a = 1; @o = a; dump @o'
+run_mlr --from $indir/xyz2 put -q 'b[1] = 2; @o = b; dump @o'
+run_mlr --from $indir/xyz2 put -q 'c[1][2] = 3; @o = c; dump @o'
+run_mlr --from $indir/xyz2 put -q 'func map_valued_func() { return {"a":1,"b":2}} @o = map_valued_func(); dump @o'
+
+# ----------------------------------------------------------------
+announce MAPVAR ASSIGNMENTS TO MAP-TYPED LOCAL
+
+run_mlr         --from $indir/xyz2 put -q 'map o = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump o'
+mlr_expect_fail --from $indir/xyz2 put -q 'map o = @o; dump o'
+run_mlr         --from $indir/xyz2 put -q 'map o = $*; dump o'
+run_mlr         --from $indir/xyz2 put -q 'map o = {"a":1, "b": {"x":8,"y":9}, "c":3}; o = o; dump o'
+mlr_expect_fail --from $indir/xyz2 put -q '@a = 1; map o = @a; dump o'
+run_mlr         --from $indir/xyz2 put -q '@b[1] = 2; map o = @b; dump o'
+run_mlr         --from $indir/xyz2 put -q '@c[1][2] = 3; map o = @c; dump o'
+run_mlr         --from $indir/xyz2 put -q '@a = 1; map o = @*; dump o'
+run_mlr         --from $indir/xyz2 put -q '@b[1] = 2; map o = @*; dump o'
+run_mlr         --from $indir/xyz2 put -q '@c[1][2] = 3; map o = @*; dump o'
+mlr_expect_fail --from $indir/xyz2 put -q 'a = 1; map o = a; dump o'
+run_mlr         --from $indir/xyz2 put -q 'b[1] = 2; map o = b; dump o'
+run_mlr         --from $indir/xyz2 put -q 'c[1][2] = 3; map o = c; dump o'
+run_mlr --from $indir/xyz2 put -q 'func map_valued_func() { return {"a":1,"b":2}} map o = map_valued_func(); dump o'
+
+# ----------------------------------------------------------------
+announce MAPVAR ASSIGNMENTS TO VAR-TYPED LOCAL
+
+run_mlr --from $indir/xyz2 put -q 'var o = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump o'
+run_mlr --from $indir/xyz2 put -q 'var o = @o; dump o'
+run_mlr --from $indir/xyz2 put -q 'var o = $*; dump o'
+run_mlr --from $indir/xyz2 put -q 'var o = {"a":1, "b": {"x":8,"y":9}, "c":3}; o = o; dump o'
+run_mlr --from $indir/xyz2 put -q '@a = 1; var o = @a; dump o'
+run_mlr --from $indir/xyz2 put -q '@b[1] = 2; var o = @b; dump o'
+run_mlr --from $indir/xyz2 put -q '@c[1][2] = 3; var o = @c; dump o'
+run_mlr --from $indir/xyz2 put -q '@a = 1; var o = @*; dump o'
+run_mlr --from $indir/xyz2 put -q '@b[1] = 2; var o = @*; dump o'
+run_mlr --from $indir/xyz2 put -q '@c[1][2] = 3; var o = @*; dump o'
+run_mlr --from $indir/xyz2 put -q 'a = 1; var o = a; dump o'
+run_mlr --from $indir/xyz2 put -q 'b[1] = 2; var o = b; dump o'
+run_mlr --from $indir/xyz2 put -q 'c[1][2] = 3; var o = c; dump o'
+run_mlr --from $indir/xyz2 put -q 'func map_valued_func() { return {"a":1,"b":2}} var o = map_valued_func(); dump o'
+
+# ----------------------------------------------------------------
+announce MAPVAR ASSIGNMENTS TO UNTYPED LOCAL
+
+run_mlr --from $indir/xyz2 put -q 'o = {"a":1, "b": {"x":8,"y":9}, "c":3}; dump o'
+run_mlr --from $indir/xyz2 put -q 'o = @o; dump o'
+run_mlr --from $indir/xyz2 put -q 'o = $*; dump o'
+run_mlr --from $indir/xyz2 put -q 'o = {"a":1, "b": {"x":8,"y":9}, "c":3}; o = o; dump o'
+run_mlr --from $indir/xyz2 put -q '@a = 1; o = @a; dump o'
+run_mlr --from $indir/xyz2 put -q '@b[1] = 2; o = @b; dump o'
+run_mlr --from $indir/xyz2 put -q '@c[1][2] = 3; o = @c; dump o'
+run_mlr --from $indir/xyz2 put -q '@a = 1; o = @*; dump o'
+run_mlr --from $indir/xyz2 put -q '@b[1] = 2; o = @*; dump o'
+run_mlr --from $indir/xyz2 put -q '@c[1][2] = 3; o = @*; dump o'
+run_mlr --from $indir/xyz2 put -q 'a = 1; o = a; dump o'
+run_mlr --from $indir/xyz2 put -q 'b[1] = 2; o = b; dump o'
+run_mlr --from $indir/xyz2 put -q 'c[1][2] = 3; o = c; dump o'
+run_mlr --from $indir/xyz2 put -q 'func map_valued_func() { return {"a":1,"b":2}} o = map_valued_func(); dump o'

go/reg-test/cases/case-c-dsl-mapvar-emit.sh

@@ -0,0 +1,28 @@
+# ----------------------------------------------------------------
+announce MAPVAR EMIT
+
+run_mlr -n put 'end {
+  @a[2][3] = 4;
+  b[2][3] = 8;
+  emit  (@a, b, {2:{3:12}});
+  emitp (@a, b, {2:{3:12}});
+  emit {};
+  emit  (@a, b, {2:{3:12}}), "t";
+  emitp (@a, b, {2:{3:12}}), "t";
+  emit {};
+  emit  (@a, b, {2:{3:12}}), "t", "u";
+  emitp (@a, b, {2:{3:12}}), "t", "u";
+}'
+
+run_mlr -n put 'end {
+  @a[2][3] = 4;
+  b[2][3] = 8;
+  emitp (@a, b, {2:{3:12}});
+  emit  (@a, b, {2:{3:12}});
+  emit {};
+  emitp (@a, b, {2:{3:12}}), "t";
+  emit  (@a, b, {2:{3:12}}), "t";
+  emit {};
+  emitp (@a, b, {2:{3:12}}), "t", "u";
+  emit  (@a, b, {2:{3:12}}), "t", "u";
+}'

go/reg-test/cases/case-c-dsl-mapvar-for-loops.sh

@@ -0,0 +1,14 @@
+run_mlr --from $indir/abixy put -q '
+  func f() {
+    return {
+      "a:".$a: $i,
+      "b:".$b: $y,
+      NR: NF,
+    }
+  }
+  for (k,v in f()) {
+    print "k=".k;
+    print "v=".v;
+  }
+  print
+'

go/reg-test/cases/case-c-dsl-mapvars-in-scalar-function-call-contexts.sh

@@ -0,0 +1,6 @@
+# ----------------------------------------------------------------
+announce MAPVARS IN SCALAR FUNCTION-CALL CONTEXTS
+
+run_mlr --from $indir/abixy put '$z=strlen($*)'
+run_mlr --from $indir/abixy put '$z=strlen({})'
+run_mlr --from $indir/abixy put 'a={}; $z=strlen(a)'

go/reg-test/cases/case-c-dsl-mapvars-udfs-subroutines.sh

@@ -0,0 +1,105 @@
+run_mlr --from $indir/abixy put '
+  func f(m) {
+    dump m;
+    sum = 0;
+    for (k, v in m) {
+      sum += k
+    }
+    return sum
+  }
+  @v[$i] = $a;
+  $y = f(@v)
+'
+
+run_mlr --from $indir/abixy put '
+  subr s(m) {
+    dump m;
+    sum = 0;
+    for (k, v in m) {
+      sum += k
+    }
+    @sum = sum;
+  }
+  @v[$i] = $a;
+  call s(@v);
+  $y = @sum;
+'
+
+run_mlr --from $indir/abixy-het put    'func f(x) {return {"a":x,"b":x**2}}; map o = f($x); $* = o'
+run_mlr --from $indir/abixy-het put -q 'func f(x) {return x**2}; var z = f($x); dump z'
+run_mlr --from $indir/abixy-het put -q 'func f(x) {map m = {NR:x};return m}; z = f($y); dump z'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(int x): map {
+    if (NR==2) {
+      return 2
+    } else {
+      return {}
+    }
+  }
+  $y=f($x)
+'
+
+run_mlr --from $indir/abixy put '
+  func f(int x): map {
+    if (NR==200) {
+      return 2
+    } else {
+      return {}
+    }
+  }
+  $y=f($i)
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(int x): map {
+    if (NR==200) {
+      return 2
+    } else {
+      return {}
+    }
+  }
+  $y=f($x)
+'
+
+run_mlr --from $indir/abixy put '
+  func f(int x): var {
+    if (NR==2) {
+      return 2
+    } else {
+      return {}
+    }
+  }
+  $y=f($i)
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(int x): var {
+    if (NR==2) {
+      return 2
+    } else {
+      return {}
+    }
+  }
+  $y=f($x)
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  func f(x): int {
+    # fall-through return value is absent-null
+  }
+  $y=f($x)
+'
+
+mlr_expect_fail --from $indir/abixy put '
+  int a = 1;
+  var b = a[2]; # cannot index localvar declared non-map
+'
+
+# This one is intended to particularly look at freeing, e.g.  with './reg-test/run --valgrind'.
+run_mlr --oxtab --from $indir/abixy-het put '
+ $* = mapdiff(
+   mapsum($*, {"a": "newval"}),
+   {"b": "nonesuch"},
+ )
+'

go/reg-test/cases/case-c-dsl-multipart-scripts.sh

@@ -0,0 +1,12 @@
+
+run_mlr --opprint --from $indir/abixy put -f $indir/put-script-piece-1
+run_mlr --opprint --from $indir/abixy put -f $indir/put-script-piece-1 -f $indir/put-script-piece-2
+run_mlr --opprint --from $indir/abixy put -f $indir/put-script-piece-1 -f $indir/put-script-piece-2 -f $indir/put-script-piece-3
+
+run_mlr --opprint --from $indir/abixy put -e '$xy = $x**2 + $y**2'
+run_mlr --opprint --from $indir/abixy filter -e 'NR == 7'
+
+run_mlr --opprint --from $indir/abixy put -e 'print "PRE";' -f $indir/put-script-piece-1 -f $indir/put-script-piece-2 -f $indir/put-script-piece-3 -e 'print "POST"'
+
+run_mlr --opprint --from $indir/abixy filter -f $indir/filter-script-piece-1
+run_mlr --opprint --from $indir/abixy filter -f $indir/filter-script-piece-1 -f $indir/filter-script-piece-2

go/reg-test/cases/case-c-dsl-null-empty-handling.sh

@@ -0,0 +1,75 @@
+
+run_mlr put '$z = $s . $s'     $indir/null-vs-empty.dkvp
+run_mlr put '$z = $s == ""'    $indir/null-vs-empty.dkvp
+run_mlr put '$z = $s == $s'    $indir/null-vs-empty.dkvp
+run_mlr put '$z = is_empty($s)' $indir/null-vs-empty.dkvp
+
+run_mlr put '$z = $x + $y'      $indir/null-vs-empty.dkvp
+run_mlr put '$z = $y + $y'      $indir/null-vs-empty.dkvp
+run_mlr put '$z = $x + $nosuch' $indir/null-vs-empty.dkvp
+run_mlr put '$t = sub($s,       "ell", "X")' $indir/null-vs-empty.dkvp
+run_mlr put '$t = sub($s,       "ell", "")'  $indir/null-vs-empty.dkvp
+run_mlr put '$t = sub($nosuch,  "ell", "X")' $indir/null-vs-empty.dkvp
+run_mlr put '$t = gsub($s,      "l",   "X")' $indir/null-vs-empty.dkvp
+run_mlr put '$t = gsub($s,      "l",   "")'  $indir/null-vs-empty.dkvp
+run_mlr put '$t = gsub($nosuch, "l",   "X")' $indir/null-vs-empty.dkvp
+
+mention EMIT
+run_mlr put -q '@v=1; @nonesuch       {emit @v}' $indir/abixy
+run_mlr put -q '@v=1; @nonesuch==true {emit @v}' $indir/abixy
+run_mlr put -q '@v=1; $nonesuch       {emit @v}' $indir/abixy
+run_mlr put -q '@v=1; $nonesuch==true {emit @v}' $indir/abixy
+
+mention PLUS
+run_mlr --ofs tab put 'begin{};          $xy = $x + $y; $sy = @s + $y; $xt = $x + @t; $st = @s + @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x + $y; $sy = @s + $y; $xt = $x + @t; $st = @s + @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x + $y; $sy = @s + $y; $xt = $x + @t; $st = @s + @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x + $y; $sy = @s + $y; $xt = $x + @t; $st = @s + @t' $indir/typeof.dkvp
+
+mention MINUS
+run_mlr --ofs tab put 'begin{};          $xy = $x - $y; $sy = @s - $y; $xt = $x - @t; $st = @s - @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x - $y; $sy = @s - $y; $xt = $x - @t; $st = @s - @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x - $y; $sy = @s - $y; $xt = $x - @t; $st = @s - @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x - $y; $sy = @s - $y; $xt = $x - @t; $st = @s - @t' $indir/typeof.dkvp
+
+mention TIMES
+run_mlr --ofs tab put 'begin{};          $xy = $x * $y; $sy = @s * $y; $xt = $x * @t; $st = @s * @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x * $y; $sy = @s * $y; $xt = $x * @t; $st = @s * @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x * $y; $sy = @s * $y; $xt = $x * @t; $st = @s * @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x * $y; $sy = @s * $y; $xt = $x * @t; $st = @s * @t' $indir/typeof.dkvp
+
+mention DIVIDE
+run_mlr --ofs tab put 'begin{};          $xy = $x / $y; $sy = @s / $y; $xt = $x / @t; $st = @s / @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x / $y; $sy = @s / $y; $xt = $x / @t; $st = @s / @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x / $y; $sy = @s / $y; $xt = $x / @t; $st = @s / @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x / $y; $sy = @s / $y; $xt = $x / @t; $st = @s / @t' $indir/typeof.dkvp
+
+mention INTEGER DIVIDE
+run_mlr --ofs tab put 'begin{};          $xy = $x // $y; $sy = @s // $y; $xt = $x // @t; $st = @s // @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x // $y; $sy = @s // $y; $xt = $x // @t; $st = @s // @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x // $y; $sy = @s // $y; $xt = $x // @t; $st = @s // @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x // $y; $sy = @s // $y; $xt = $x // @t; $st = @s // @t' $indir/typeof.dkvp
+
+mention REMAINDER
+run_mlr --ofs tab put 'begin{};          $xy = $x % $y; $sy = @s % $y; $xt = $x % @t; $st = @s % @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x % $y; $sy = @s % $y; $xt = $x % @t; $st = @s % @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x % $y; $sy = @s % $y; $xt = $x % @t; $st = @s % @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x % $y; $sy = @s % $y; $xt = $x % @t; $st = @s % @t' $indir/typeof.dkvp
+
+mention BITWISE AND
+run_mlr --ofs tab put 'begin{};          $xy = $x & $y; $sy = @s & $y; $xt = $x & @t; $st = @s & @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x & $y; $sy = @s & $y; $xt = $x & @t; $st = @s & @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x & $y; $sy = @s & $y; $xt = $x & @t; $st = @s & @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x & $y; $sy = @s & $y; $xt = $x & @t; $st = @s & @t' $indir/typeof.dkvp
+
+mention BITWISE OR
+run_mlr --ofs tab put 'begin{};          $xy = $x | $y; $sy = @s | $y; $xt = $x | @t; $st = @s | @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x | $y; $sy = @s | $y; $xt = $x | @t; $st = @s | @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x | $y; $sy = @s | $y; $xt = $x | @t; $st = @s | @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x | $y; $sy = @s | $y; $xt = $x | @t; $st = @s | @t' $indir/typeof.dkvp
+
+mention BITWISE XOR
+run_mlr --ofs tab put 'begin{};          $xy = $x ^ $y; $sy = @s ^ $y; $xt = $x ^ @t; $st = @s ^ @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3};      $xy = $x ^ $y; $sy = @s ^ $y; $xt = $x ^ @t; $st = @s ^ @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@t=4};      $xy = $x ^ $y; $sy = @s ^ $y; $xt = $x ^ @t; $st = @s ^ @t' $indir/typeof.dkvp
+run_mlr --ofs tab put 'begin{@s=3;@t=4}; $xy = $x ^ $y; $sy = @s ^ $y; $xt = $x ^ @t; $st = @s ^ @t' $indir/typeof.dkvp

go/reg-test/cases/case-c-dsl-oosvar-assignments.sh

@@ -0,0 +1,76 @@
+# ----------------------------------------------------------------
+announce OOSVAR-FROM-SREC ASSIGNMENT
+
+run_mlr put -v '@v     = $*' /dev/null
+run_mlr put -v '@v[1]  = $*' /dev/null
+run_mlr put -v '@v[$2] = $*' /dev/null
+run_mlr put -v 'NR == 3 {@v     = $*}' /dev/null
+run_mlr put -v 'NR == 3 {@v[1]  = $*}' /dev/null
+run_mlr put -v 'NR == 3 {@v[$2] = $*}' /dev/null
+
+run_mlr --oxtab put -q '@v = $*; end {emitp @v }' $indir/abixy-het
+
+run_mlr --oxtab put -q '@v[$a] = $*; end {emitp @v      }' $indir/abixy-het
+run_mlr --oxtab put -q '@v[$a] = $*; end {emitp @v, "a" }' $indir/abixy-het
+
+run_mlr --oxtab put -q '@v[$a][$b] = $*; end {emitp @v          }' $indir/abixy-het
+run_mlr --oxtab put -q '@v[$a][$b] = $*; end {emitp @v, "a"     }' $indir/abixy-het
+run_mlr --oxtab put -q '@v[$a][$b] = $*; end {emitp @v, "a", "b"}' $indir/abixy-het
+
+# ----------------------------------------------------------------
+announce SREC-FROM-OOSVAR ASSIGNMENT
+
+run_mlr put -v '$* = @v    ' /dev/null
+run_mlr put -v '$* = @v[1] ' /dev/null
+run_mlr put -v '$* = @v[$2]' /dev/null
+run_mlr put -v 'NR == 3 {$* = @v    }' /dev/null
+run_mlr put -v 'NR == 3 {$* = @v[1] }' /dev/null
+run_mlr put -v 'NR == 3 {$* = @v[$2]}' /dev/null
+
+run_mlr put '@v[NR] = $a; NR == 7 { @v = $*} ; $* = @v' $indir/abixy-het
+
+# ----------------------------------------------------------------
+announce OOSVAR-FROM-OOSVAR ASSIGNMENT
+
+run_mlr put -v '@u    = @v'    /dev/null
+run_mlr put -v '@u    = @v[1]' /dev/null
+run_mlr put -v '@u[2] = @v'    /dev/null
+run_mlr put -v '@u[2] = @v[1]' /dev/null
+
+run_mlr put -v 'begin { @u    = @v }'    /dev/null
+run_mlr put -v 'begin { @u    = @v[1] }' /dev/null
+run_mlr put -v 'begin { @u[2] = @v }'    /dev/null
+run_mlr put -v 'begin { @u[2] = @v[1] }' /dev/null
+
+run_mlr put -v 'NR == 3 { @u    = @v }'    /dev/null
+run_mlr put -v 'NR == 3 { @u    = @v[1] }' /dev/null
+run_mlr put -v 'NR == 3 { @u[2] = @v }'    /dev/null
+run_mlr put -v 'NR == 3 { @u[2] = @v[1] }' /dev/null
+
+run_mlr put -v 'end { @u    = @v }'    /dev/null
+run_mlr put -v 'end { @u    = @v[1] }' /dev/null
+run_mlr put -v 'end { @u[2] = @v }'    /dev/null
+run_mlr put -v 'end { @u[2] = @v[1] }' /dev/null
+
+
+run_mlr put -q '@s    += $i; @t=@s;             end{dump; emitp@s; emitp @t}' $indir/abixy
+
+run_mlr put -q '@s[1] += $i; @t=@s;             end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1] += $i; @t=@s[1];          end{dump; emitp@s; emitp @t}' $indir/abixy
+
+run_mlr put -q '@s[1] += $i; @t[3]=@s;          end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1] += $i; @t[3]=@s[1];       end{dump; emitp@s; emitp @t}' $indir/abixy
+
+run_mlr put -q '@s[1][2] += $i; @t=@s;             end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1][2] += $i; @t=@s[1];          end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1][2] += $i; @t=@s[1][2];       end{dump; emitp@s; emitp @t}' $indir/abixy
+
+run_mlr put -q '@s[1][2] += $i; @t[3]=@s;          end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1][2] += $i; @t[3]=@s[1];       end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1][2] += $i; @t[3]=@s[1][2];    end{dump; emitp@s; emitp @t}' $indir/abixy
+
+run_mlr put -q '@s[1][2] += $i; @t[3][4]=@s;       end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1][2] += $i; @t[3][4]=@s[1];    end{dump; emitp@s; emitp @t}' $indir/abixy
+run_mlr put -q '@s[1][2] += $i; @t[3][4]=@s[1][2]; end{dump; emitp@s; emitp @t}' $indir/abixy
+
+run_mlr --opprint put -q '@s[NR][NR] = $i/100; @t[NR*10]=@s; end{emitp@s,"A","B"; emitp @t,"C","D","E"}' $indir/abixy

go/reg-test/cases/case-c-dsl-oosvars.sh

@@ -0,0 +1,66 @@
+
+run_mlr --opprint put -v 'begin{@ox=0}; $d=$x-@ox; @ox=$x' $indir/abixy
+run_mlr --opprint put -v 'begin{@ox="no"}; $d=@ox == "no" ? 1.0 : $x/@ox; @ox=$x' then step -a ratio -f x $indir/abixy
+run_mlr --opprint put -v '$d=$x/@ox; @ox=$x' then step -a ratio -f x $indir/abixy
+run_mlr --opprint put -v 'begin{@ox="no"}; $d=@ox == "no" ? 1.0 : $x/@ox; @ox=$x' then step -a ratio -f x $indir/abixy
+run_mlr --opprint put -v 'begin{@rsum = 0}; @rsum = @rsum + $x; $rsum = @rsum' $indir/abixy
+run_mlr --opprint put -v 'begin{@a=0; @b=0; @c=0}; $za=@a; $zb=@b; $zc=@c; $d=@a+@b+@c; @a=@b; @b=@c; @c=$i' $indir/abixy
+run_mlr --opprint put -v 'begin {@a=0; @b=0; @c=0}; $za=@a; $zb=@b; $zc=@c; $d=@a+@b+@c; @a=@b; @b=@c; @c=$i' $indir/abixy
+run_mlr --opprint put -v 'begin{@ox=0}; $d=$x-@ox; @ox=$x' $indir/abixy
+
+run_mlr put -v '@a=$a; @b=$b; @c=$x; end {emitf @a; emitf @b; emitf @c}' $indir/abixy
+run_mlr put -v '@a=$a; @b=$b; @c=$x; end{emitf @a, @b, @c}' $indir/abixy
+
+run_mlr --opprint put -v 'begin {@count=0; @sum=0.0}; @count=@count+1; @sum=@sum+$x; end{@mean=@sum/@count; emitf @mean}' $indir/abixy
+run_mlr --opprint put -v 'end{@mean=@sum/@count; emitf @mean}; begin {@count=0; @sum=0.0}; @count=@count+1; @sum=@sum+$x' $indir/abixy
+
+run_mlr put -v 'begin{ @a = @b[1] }; $c = @d; @e[$i][2+$j][3] = $4; end{@f[@g[5][@h]] = 6}' /dev/null
+
+run_mlr put '@y[$a]=$y; end{dump}' $indir/abixy
+
+run_mlr stats1 -a sum -f y -g a $indir/abixy
+run_mlr put '@y_sum[$a] = $y; end{dump}' $indir/abixy
+
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @s      ; dump}' $indir/unset1.dkvp
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @t      ; dump}' $indir/unset1.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @t[1]   ; dump}' $indir/unset1.dkvp
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @u      ; dump}' $indir/unset1.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @u[1]   ; dump}' $indir/unset1.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @u[1][2]; dump}' $indir/unset1.dkvp
+
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @s      ; dump}' $indir/unset4.dkvp
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @t      ; dump}' $indir/unset4.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @t[1]   ; dump}' $indir/unset4.dkvp
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @u      ; dump}' $indir/unset4.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @u[1]   ; dump}' $indir/unset4.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @u[1][2]; dump}' $indir/unset4.dkvp
+
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset all;      dump}' $indir/unset4.dkvp
+run_mlr put -q '@s=$x; @t[$a]=$x; @u[$a][$b]=$x; end{dump; unset @*;       dump}' $indir/unset4.dkvp
+
+run_mlr put 'unset $x' $indir/unset4.dkvp
+run_mlr put 'unset $*; $aaa = 999' $indir/unset4.dkvp
+run_mlr --from $indir/abixy put 'x = 1; print "OX=".x; unset x; print "NX=".x'
+
+run_mlr put -q '@{variable.name} += $x; end{emit @{variable.name}}' $indir/abixy
+run_mlr put -q '@{variable.name}[$a] += $x; end{emit @{variable.name},"a"}' $indir/abixy
+
+run_mlr put 'for (k,v in $*) { if (k == "i") {unset $[k]}}' $indir/abixy
+
+run_mlr --opprint --from $indir/abixy put -q '
+  @output[NR] = $*;
+  end {
+    for ((nr, k), v in @output) {
+      if (nr == 4 || k == "i") {
+        unset @output[nr][k]
+      }
+    }
+    emitp @output, "NR", "k"
+  }
+'

go/reg-test/cases/case-c-dsl-operator-associativity.sh

@@ -0,0 +1,146 @@
+# Note: filter -v and put -v print the AST.
+
+run_mlr put    -v '$x = 1 || 2 || 3'   /dev/null
+run_mlr filter -v '     1 || 2 || 3'   /dev/null
+run_mlr put    -v '$x = 1 ^^ 2 ^^ 3'   /dev/null
+run_mlr filter -v '     1 ^^ 2 ^^ 3'   /dev/null
+run_mlr put    -v '$x = 1 && 2 && 3'   /dev/null
+run_mlr filter -v '     1 && 2 && 3'   /dev/null
+
+run_mlr put    -v '$x = 1  == 2  == 3' /dev/null
+run_mlr filter -v '     1  == 2  == 3' /dev/null
+run_mlr put    -v '$x = 1  != 2  != 3' /dev/null
+run_mlr filter -v '     1  != 2  != 3' /dev/null
+run_mlr put    -v '$x = 1  =~ 2  =~ 3' /dev/null
+run_mlr filter -v '     1  =~ 2  =~ 3' /dev/null
+run_mlr put    -v '$x = 1 !=~ 2 !=~ 3' /dev/null
+run_mlr filter -v '     1 !=~ 2 !=~ 3' /dev/null
+run_mlr put    -v '$x = 1  == 2  != 3' /dev/null
+run_mlr filter -v '     1  == 2  != 3' /dev/null
+run_mlr put    -v '$x = 1  != 2  == 3' /dev/null
+run_mlr filter -v '     1  != 2  == 3' /dev/null
+
+run_mlr put    -v '$x = 1  <  2  <  3' /dev/null
+run_mlr filter -v '     1  <  2  <  3' /dev/null
+run_mlr put    -v '$x = 1  <= 2  <= 3' /dev/null
+run_mlr filter -v '     1  <= 2  <= 3' /dev/null
+run_mlr put    -v '$x = 1  >  2  >  3' /dev/null
+run_mlr filter -v '     1  >  2  >  3' /dev/null
+run_mlr put    -v '$x = 1  >= 2  >= 3' /dev/null
+run_mlr filter -v '     1  >= 2  >= 3' /dev/null
+run_mlr put    -v '$x = 1  <  2  <= 3' /dev/null
+run_mlr filter -v '     1  <  2  <= 3' /dev/null
+run_mlr put    -v '$x = 1  <= 2  <  3' /dev/null
+run_mlr filter -v '     1  <= 2  <  3' /dev/null
+
+run_mlr put    -v '$x = 1 |  2 |  3'   /dev/null
+run_mlr filter -v '     1 |  2 |  3'   /dev/null
+run_mlr put    -v '$x = 1 ^  2 ^  3'   /dev/null
+run_mlr filter -v '     1 ^  2 ^  3'   /dev/null
+run_mlr put    -v '$x = 1 &  2 &  3'   /dev/null
+run_mlr filter -v '     1 &  2 &  3'   /dev/null
+run_mlr put    -v '$x = 1 |  2 &  3'   /dev/null
+run_mlr filter -v '     1 |  2 &  3'   /dev/null
+run_mlr put    -v '$x = 1 |  2 ^  3'   /dev/null
+run_mlr filter -v '     1 |  2 ^  3'   /dev/null
+run_mlr put    -v '$x = 1 ^  2 |  3'   /dev/null
+run_mlr filter -v '     1 ^  2 |  3'   /dev/null
+run_mlr put    -v '$x = 1 ^  2 &  3'   /dev/null
+run_mlr filter -v '     1 ^  2 &  3'   /dev/null
+run_mlr put    -v '$x = 1 &  2 |  3'   /dev/null
+run_mlr filter -v '     1 &  2 |  3'   /dev/null
+run_mlr put    -v '$x = 1 &  2 ^  3'   /dev/null
+run_mlr filter -v '     1 &  2 ^  3'   /dev/null
+
+run_mlr put    -v '$x = 1  << 2  << 3' /dev/null
+run_mlr filter -v '     1  << 2  << 3' /dev/null
+run_mlr put    -v '$x = 1  >> 2  >> 3' /dev/null
+run_mlr filter -v '     1  >> 2  >> 3' /dev/null
+run_mlr put    -v '$x = 1  << 2  >> 3' /dev/null
+run_mlr filter -v '     1  << 2  >> 3' /dev/null
+run_mlr put    -v '$x = 1  >> 2  << 3' /dev/null
+run_mlr filter -v '     1  >> 2  << 3' /dev/null
+
+run_mlr put    -v '$x = 1 + 2 + 3'   /dev/null
+run_mlr filter -v '     1 + 2 + 3'   /dev/null
+run_mlr put    -v '$x = 1 - 2 - 3'   /dev/null
+run_mlr filter -v '     1 - 2 - 3'   /dev/null
+run_mlr put    -v '$x = 1 + 2 - 3'   /dev/null
+run_mlr filter -v '     1 + 2 - 3'   /dev/null
+run_mlr put    -v '$x = 1 - 2 + 3'   /dev/null
+run_mlr filter -v '     1 - 2 + 3'   /dev/null
+run_mlr put    -v '$x = 1 . 2 . 3'   /dev/null
+run_mlr filter -v '     1 . 2 . 3'   /dev/null
+
+run_mlr put    -v '$x = 1 * 2 * 3'   /dev/null
+run_mlr filter -v '     1 * 2 * 3'   /dev/null
+run_mlr put    -v '$x = 1 / 2 / 3'   /dev/null
+run_mlr filter -v '     1 / 2 / 3'   /dev/null
+run_mlr put    -v '$x = 1 // 2 // 3' /dev/null
+run_mlr filter -v '     1 // 2 // 3' /dev/null
+run_mlr put    -v '$x = 1 % 2 % 3'   /dev/null
+run_mlr filter -v '     1 % 2 % 3'   /dev/null
+run_mlr put    -v '$x = 1 ** 2 ** 3' /dev/null
+run_mlr filter -v '     1 ** 2 ** 3' /dev/null
+
+
+run_mlr put    -v '$x = 1 *  2 /  3'   /dev/null
+run_mlr filter -v '     1 *  2 /  3'   /dev/null
+run_mlr put    -v '$x = 1 *  2 // 3'   /dev/null
+run_mlr filter -v '     1 *  2 // 3'   /dev/null
+run_mlr put    -v '$x = 1 *  2 %  3'   /dev/null
+run_mlr filter -v '     1 *  2 %  3'   /dev/null
+run_mlr put    -v '$x = 1 *  2 ** 3'   /dev/null
+run_mlr filter -v '     1 *  2 ** 3'   /dev/null
+
+run_mlr put    -v '$x = 1 /  2 *  3'   /dev/null
+run_mlr filter -v '     1 /  2 *  3'   /dev/null
+run_mlr put    -v '$x = 1 /  2 // 3'   /dev/null
+run_mlr filter -v '     1 /  2 // 3'   /dev/null
+run_mlr put    -v '$x = 1 /  2 %  3'   /dev/null
+run_mlr filter -v '     1 /  2 %  3'   /dev/null
+run_mlr put    -v '$x = 1 /  2 ** 3'   /dev/null
+run_mlr filter -v '     1 /  2 ** 3'   /dev/null
+
+run_mlr put    -v '$x = 1 // 2 *  3'   /dev/null
+run_mlr filter -v '     1 // 2 *  3'   /dev/null
+run_mlr put    -v '$x = 1 // 2 /  3'   /dev/null
+run_mlr filter -v '     1 // 2 /  3'   /dev/null
+run_mlr put    -v '$x = 1 // 2 %  3'   /dev/null
+run_mlr filter -v '     1 // 2 %  3'   /dev/null
+run_mlr put    -v '$x = 1 // 2 ** 3'   /dev/null
+run_mlr filter -v '     1 // 2 ** 3'   /dev/null
+
+run_mlr put    -v '$x = 1 %  2 *  3'   /dev/null
+run_mlr filter -v '     1 %  2 *  3'   /dev/null
+run_mlr put    -v '$x = 1 %  2 /  3'   /dev/null
+run_mlr filter -v '     1 %  2 /  3'   /dev/null
+run_mlr put    -v '$x = 1 %  2 // 3'   /dev/null
+run_mlr filter -v '     1 %  2 // 3'   /dev/null
+run_mlr put    -v '$x = 1 %  2 ** 3'   /dev/null
+run_mlr filter -v '     1 %  2 ** 3'   /dev/null
+
+run_mlr put    -v '$x = 1 ** 2 *  3'   /dev/null
+run_mlr filter -v '     1 ** 2 *  3'   /dev/null
+run_mlr put    -v '$x = 1 ** 2 /  3'   /dev/null
+run_mlr filter -v '     1 ** 2 /  3'   /dev/null
+run_mlr put    -v '$x = 1 ** 2 // 3'   /dev/null
+run_mlr filter -v '     1 ** 2 // 3'   /dev/null
+run_mlr put    -v '$x = 1 ** 2 %  3'   /dev/null
+run_mlr filter -v '     1 ** 2 %  3'   /dev/null
+
+run_mlr put    -v '$x = ++1'   /dev/null
+run_mlr filter -v '     ++1'   /dev/null
+run_mlr put    -v '$x = --1'   /dev/null
+run_mlr filter -v '     --1'   /dev/null
+run_mlr put    -v '$x = !!1'   /dev/null
+run_mlr filter -v '     !!1'   /dev/null
+run_mlr put    -v '$x = ~~1'   /dev/null
+run_mlr filter -v '     ~~1'   /dev/null
+
+run_mlr put    -v '$x = 1 ? 2 : 3'         /dev/null
+run_mlr filter -v '     1 ? 2 : 3'         /dev/null
+run_mlr put    -v '$x = 1 ? 2 ? 3 : 4 : 5' /dev/null
+run_mlr filter -v '     1 ? 2 ? 3 : 4 : 5' /dev/null
+run_mlr put    -v '$x = 1 ? 2 : 3 ? 4 : 5' /dev/null
+run_mlr filter -v '     1 ? 2 : 3 ? 4 : 5' /dev/null

go/reg-test/cases/case-c-dsl-operator-precedence.sh

@@ -0,0 +1,153 @@
+# Note: filter -v and put -v print the AST.
+
+run_mlr put    -v '$x = 1 || 2 ^^ 3'   /dev/null
+run_mlr filter -v '     1 || 2 ^^ 3'   /dev/null
+run_mlr put    -v '$x = 1 || 2 && 3'   /dev/null
+run_mlr filter -v '     1 || 2 && 3'   /dev/null
+
+run_mlr put    -v '$x = 1 ^^ 2 || 3'   /dev/null
+run_mlr filter -v '     1 ^^ 2 || 3'   /dev/null
+run_mlr put    -v '$x = 1 ^^ 2 && 3'   /dev/null
+run_mlr filter -v '     1 ^^ 2 && 3'   /dev/null
+
+run_mlr put    -v '$x = 1 && 2 || 3'   /dev/null
+run_mlr filter -v '     1 && 2 || 3'   /dev/null
+run_mlr put    -v '$x = 1 && 2 ^^ 3'   /dev/null
+run_mlr filter -v '     1 && 2 ^^ 3'   /dev/null
+
+run_mlr put    -v '$x =  1 == 2 <= 3'  /dev/null
+run_mlr filter -v '      1 == 2 <= 3'  /dev/null
+run_mlr put    -v '$x =  1 <= 2 == 3'  /dev/null
+run_mlr filter -v '      1 <= 2 == 3'  /dev/null
+
+run_mlr put    -v '$x =  1 <= 2 |  3'  /dev/null
+run_mlr filter -v '      1 <= 2 |  3'  /dev/null
+run_mlr put    -v '$x =  1 |  2 <= 3'  /dev/null
+run_mlr filter -v '      1 |  2 <= 3'  /dev/null
+
+run_mlr put    -v '$x =  1 |  2 ^  3'  /dev/null
+run_mlr filter -v '      1 |  2 ^  3'  /dev/null
+run_mlr put    -v '$x =  1 ^  2 |  3'  /dev/null
+run_mlr filter -v '      1 ^  2 |  3'  /dev/null
+
+run_mlr put    -v '$x =  1 ^  2 &  3'  /dev/null
+run_mlr filter -v '      1 ^  2 &  3'  /dev/null
+run_mlr put    -v '$x =  1 &  2 ^  3'  /dev/null
+run_mlr filter -v '      1 &  2 ^  3'  /dev/null
+
+run_mlr put    -v '$x =  1 &  2 << 3'  /dev/null
+run_mlr filter -v '      1 &  2 << 3'  /dev/null
+run_mlr put    -v '$x =  1 << 2 &  3'  /dev/null
+run_mlr filter -v '      1 << 2 &  3'  /dev/null
+
+run_mlr put    -v '$x =  1 +  2 * 3'   /dev/null
+run_mlr filter -v '      1 +  2 * 3'   /dev/null
+run_mlr put    -v '$x =  1 *  2 + 3'   /dev/null
+run_mlr filter -v '      1 *  2 + 3'   /dev/null
+run_mlr put    -v '$x =  1 + (2 * 3)'  /dev/null
+run_mlr filter -v '      1 + (2 * 3)'  /dev/null
+run_mlr put    -v '$x =  1 * (2 + 3)'  /dev/null
+run_mlr filter -v '      1 * (2 + 3)'  /dev/null
+run_mlr put    -v '$x = (1 + 2) * 3'   /dev/null
+run_mlr filter -v '     (1 + 2) * 3'   /dev/null
+run_mlr put    -v '$x = (1 * 2) + 3'   /dev/null
+run_mlr filter -v '     (1 * 2) + 3'   /dev/null
+
+run_mlr put    -v '$x =  1 +   2 ** 3'  /dev/null
+run_mlr filter -v '      1 +   2 ** 3'  /dev/null
+run_mlr put    -v '$x =  1 **  2 +  3'  /dev/null
+run_mlr filter -v '      1 **  2 +  3'  /dev/null
+run_mlr put    -v '$x =  1 +  (2 ** 3)' /dev/null
+run_mlr filter -v '      1 +  (2 ** 3)' /dev/null
+run_mlr put    -v '$x =  1 ** (2 +  3)' /dev/null
+run_mlr filter -v '      1 ** (2 +  3)' /dev/null
+run_mlr put    -v '$x = (1 +  2) ** 3'  /dev/null
+run_mlr filter -v '     (1 +  2) ** 3'  /dev/null
+run_mlr put    -v '$x = (1 ** 2) +  3'  /dev/null
+run_mlr filter -v '     (1 ** 2) +  3'  /dev/null
+
+run_mlr put    -v '$x =  1 *   2 ** 3'  /dev/null
+run_mlr filter -v '      1 *   2 ** 3'  /dev/null
+run_mlr put    -v '$x =  1 **  2 *  3'  /dev/null
+run_mlr filter -v '      1 **  2 *  3'  /dev/null
+run_mlr put    -v '$x =  1 *  (2 ** 3)' /dev/null
+run_mlr filter -v '      1 *  (2 ** 3)' /dev/null
+run_mlr put    -v '$x =  1 ** (2 *  3)' /dev/null
+run_mlr filter -v '      1 ** (2 *  3)' /dev/null
+run_mlr put    -v '$x = (1 *  2) ** 3'  /dev/null
+run_mlr filter -v '     (1 *  2) ** 3'  /dev/null
+run_mlr put    -v '$x = (1 ** 2) *  3'  /dev/null
+run_mlr filter -v '     (1 ** 2) *  3'  /dev/null
+
+run_mlr put    -v '$x = -1 +  2 *  3'  /dev/null
+run_mlr filter -v '     -1 +  2 *  3'  /dev/null
+run_mlr put    -v '$x = -1 *  2 +  3'  /dev/null
+run_mlr filter -v '     -1 *  2 +  3'  /dev/null
+run_mlr put    -v '$x =  1 + -2 *  3'  /dev/null
+run_mlr filter -v '      1 + -2 *  3'  /dev/null
+run_mlr put    -v '$x =  1 * -2 +  3'  /dev/null
+run_mlr filter -v '      1 * -2 +  3'  /dev/null
+run_mlr put    -v '$x =  1 +  2 * -3'  /dev/null
+run_mlr filter -v '      1 +  2 * -3'  /dev/null
+run_mlr put    -v '$x =  1 *  2 + -3'  /dev/null
+run_mlr filter -v '      1 *  2 + -3'  /dev/null
+
+run_mlr put    -v '$x = ~1 |  2 &  3'  /dev/null
+run_mlr filter -v '     ~1 |  2 &  3'  /dev/null
+run_mlr put    -v '$x = ~1 &  2 |  3'  /dev/null
+run_mlr filter -v '     ~1 &  2 |  3'  /dev/null
+run_mlr put    -v '$x =  1 | ~2 &  3'  /dev/null
+run_mlr filter -v '      1 | ~2 &  3'  /dev/null
+run_mlr put    -v '$x =  1 & ~2 |  3'  /dev/null
+run_mlr filter -v '      1 & ~2 |  3'  /dev/null
+run_mlr put    -v '$x =  1 |  2 & ~3'  /dev/null
+run_mlr filter -v '      1 |  2 & ~3'  /dev/null
+run_mlr put    -v '$x =  1 &  2 | ~3'  /dev/null
+run_mlr filter -v '      1 &  2 | ~3'  /dev/null
+
+run_mlr put    -v '$x = $a==1 && $b == 1 && $c == 1' /dev/null
+run_mlr filter -v '     $a==1 && $b == 1 && $c == 1' /dev/null
+run_mlr put    -v '$x = $a==1 || $b == 1 && $c == 1' /dev/null
+run_mlr filter -v '     $a==1 || $b == 1 && $c == 1' /dev/null
+run_mlr put    -v '$x = $a==1 || $b == 1 || $c == 1' /dev/null
+run_mlr filter -v '     $a==1 || $b == 1 || $c == 1' /dev/null
+run_mlr put    -v '$x = $a==1 && $b == 1 || $c == 1' /dev/null
+run_mlr filter -v '     $a==1 && $b == 1 || $c == 1' /dev/null
+
+run_mlr put    -v '$x = $a==1 ? $b == 2 : $c == 3' /dev/null
+run_mlr filter -v '     $a==1 ? $b == 2 : $c == 3' /dev/null
+
+run_mlr put    -v '$x = true' /dev/null
+run_mlr filter -v '     true' /dev/null
+
+run_mlr put    -v 'true || 1==0; $x = 3' /dev/null
+run_mlr filter -v '        true || 1==0' /dev/null
+
+run_mlr put    -v '1==0 || false; $x = 3' /dev/null
+run_mlr filter -v '        1==0 || false' /dev/null
+
+run_mlr put    -v 'true && false; $x = 3' /dev/null
+run_mlr filter -v '        true && false' /dev/null
+
+run_mlr put    -v 'true && false && true; $x = 3' /dev/null
+run_mlr filter -v '        true && false && true' /dev/null
+
+run_mlr put    -v '$y += $x + 3'  /dev/null
+run_mlr put    -v '$y += $x * 3'  /dev/null
+
+run_mlr put -v '$y ||= $x' /dev/null
+run_mlr put -v '$y ^^= $x' /dev/null
+run_mlr put -v '$y &&= $x' /dev/null
+run_mlr put -v '$y |=  $x' /dev/null
+run_mlr put -v '$y ^=  $x' /dev/null
+run_mlr put -v '$y &=  $x' /dev/null
+run_mlr put -v '$y <<= $x' /dev/null
+run_mlr put -v '$y >>= $x' /dev/null
+run_mlr put -v '$y +=  $x' /dev/null
+run_mlr put -v '$y -=  $x' /dev/null
+run_mlr put -v '$y .=  $x' /dev/null
+run_mlr put -v '$y *=  $x' /dev/null
+run_mlr put -v '$y /=  $x' /dev/null
+run_mlr put -v '$y //= $x' /dev/null
+run_mlr put -v '$y %=  $x' /dev/null
+run_mlr put -v '$y **= $x' /dev/null