diff --git a/c/mapping/mlr_dsl_cst.c b/c/mapping/mlr_dsl_cst.c index 468e9d018..50d404541 100644 --- a/c/mapping/mlr_dsl_cst.c +++ b/c/mapping/mlr_dsl_cst.c @@ -1748,6 +1748,9 @@ static void handle_for_srec( // Copy, not pointer-reference, in case of srec-unset in loop body: mv_t mvkey = mv_from_string_no_free(pe->key); + // xxx something like rval_evaluator_alloc_from_indirect_field_name (extract-method?) + // for typed eval here. + mv_t* poverlay = lhmsv_get(pvars->ptyped_overlay, pe->key); mv_t mvval = (poverlay != NULL) ? mv_copy(poverlay) diff --git a/c/mapping/mlr_dsl_cst.h b/c/mapping/mlr_dsl_cst.h index 884abbce6..c6115a8a1 100644 --- a/c/mapping/mlr_dsl_cst.h +++ b/c/mapping/mlr_dsl_cst.h @@ -123,7 +123,7 @@ typedef struct _mlr_dsl_cst_statement_t { // if-elif-elif-else: sllv_t* pif_chain_statements; - // for-srec: + // for-srec / for-oosvar: char* for_srec_k_name; slls_t* pfor_oosvar_k_names; char* for_v_name; diff --git a/c/todo.txt b/c/todo.txt index 6f6013d76..e2f218e1d 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -6,6 +6,7 @@ BUGFIXES ================================================================ TOP OF LIST: +! typing for v in for-srec ! * re-do valgrinds before release COOKBOOK: @@ -45,10 +46,10 @@ UT: HYGIENE: ! lhmsv->lhmsmv @ typed overlay -? sllv->slln @ ast ! chk copy/reference semantics for mlrvals esp. w/r/t evaluators ! chk lhmsmv mem mgmt @ enlarge as in mlhmmv -* lemon note/message-wording re resolvable conflicts +! lemon note/message-wording re resolvable conflicts +* release poki mod 4.1.0 release notes: * new forx2, while/do-while, if-chains diff --git a/doc/content-for-reference.html b/doc/content-for-reference.html index 4c9866b20..a7fa78659 100644 --- a/doc/content-for-reference.html +++ b/doc/content-for-reference.html @@ -709,28 +709,6 @@ stdout.

Features of out-of-stream variables, and examples of their use, will be presented in the following sections. -

Pattern-action blocks for put

- -

These are reminiscent of awk syntax. They can be used to allow -assignments to be done only when appropriate — e.g. for math-function -domain restrictions, regex-matching, and so on: - -POKI_RUN_COMMAND{{mlr cat data/put-gating-example-1.dkvp}}HERE -POKI_RUN_COMMAND{{mlr put '$x > 0.0 { $y = log10($x); $z = sqrt($y) }' data/put-gating-example-1.dkvp}}HERE -POKI_RUN_COMMAND{{mlr cat data/put-gating-example-2.dkvp}}HERE -POKI_RUN_COMMAND{{mlr put '$a =~ "([a-z]+)_([0-9]+)" { $b = "left_\1"; $c = "right_\2" }' data/put-gating-example-2.dkvp}}HERE - -

This produces heteregenous output which Miller, of course, has no problems -with (see POKI_PUT_LINK_FOR_PAGE(record-heterogeneity.html)HERE). But -if homogeneous output is desired, the curly braces can be omitted. This -causes put to evaluate the boolean expression (along with any side -effects, namely, regex-captures \1, \2, etc.) but -doesn’t use it as a criterion for whether subsequent assignments should -be executed. Instead, subsequent assignments are done unconditionally: - -POKI_RUN_COMMAND{{mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp}}HERE -POKI_RUN_COMMAND{{mlr put '$a =~ "([a-z]+)_([0-9]+)"; $b = "left_\1"; $c = "right_\2"' data/put-gating-example-2.dkvp}}HERE -

Begin/end blocks for put

Miller supports an awk-like begin/end syntax. The @@ -904,6 +882,123 @@ stream: POKI_RUN_COMMAND{{cat data/small}}HERE POKI_RUN_COMMAND{{mlr --opprint put -q 'isnull(@xmax) || $x > @xmax {@xmax=$x; @recmax=$*}; end {emit @recmax}' data/small}}HERE +

Semicolons and curly braces for put

+ +

Miller uses semicolons as statement separators, not statement terminators. This means you can write: + +POKI_CARDIFY(mlr put 'x=1')HERE +POKI_CARDIFY(mlr put 'x=1;$y=2')HERE +POKI_CARDIFY(mlr put 'x=1;$y=2;')HERE +POKI_CARDIFY(mlr put 'x=1;;;;$y=2;')HERE + +

Semicolons are required between statements even if those statements are on separate lines. + +

Semicolons are optional after closing curly braces (which close conditionals and loops as discussed below). + +POKI_RUN_COMMAND{{echo x=1,y=2 | mlr put 'while (NF < 10) { $[NF+1] = ""} $foo = "bar"'}}HERE + +

Bodies for all compound statements must be enclosed in curly braces, even if the body is a single statement: +POKI_CARDIFY{{mlr put 'if ($x == 1) $y = 2 # Syntax error}}HERE + +

Bodies for compound statements may be empty: + +POKI_CARDIFY{{mlr put 'if ($x == 1) { } # This no-op is syntactically acceptable}}HERE + +

Pattern-action blocks for put

+ +

These are reminiscent of awk syntax. They can be used to allow +assignments to be done only when appropriate — e.g. for math-function +domain restrictions, regex-matching, and so on: + +POKI_RUN_COMMAND{{mlr cat data/put-gating-example-1.dkvp}}HERE +POKI_RUN_COMMAND{{mlr put '$x > 0.0 { $y = log10($x); $z = sqrt($y) }' data/put-gating-example-1.dkvp}}HERE +POKI_RUN_COMMAND{{mlr cat data/put-gating-example-2.dkvp}}HERE +POKI_RUN_COMMAND{{mlr put '$a =~ "([a-z]+)_([0-9]+)" { $b = "left_\1"; $c = "right_\2" }' data/put-gating-example-2.dkvp}}HERE + +

This produces heteregenous output which Miller, of course, has no problems +with (see POKI_PUT_LINK_FOR_PAGE(record-heterogeneity.html)HERE). But if +homogeneous output is desired, the curly braces can be replaced with a +semicolon between the expression and the body statements. This causes +put to evaluate the boolean expression (along with any side effects, +namely, regex-captures \1, \2, etc.) but doesn’t use it +as a criterion for whether subsequent assignments should be executed. Instead, +subsequent assignments are done unconditionally: + +POKI_RUN_COMMAND{{mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp}}HERE +POKI_RUN_COMMAND{{mlr put '$a =~ "([a-z]+)_([0-9]+)"; $b = "left_\1"; $c = "right_\2"' data/put-gating-example-2.dkvp}}HERE + +

If-statements for put

+ +

These are again reminiscent of awk. Pattern-action blocks are a special case of if with no +elif or else blocks, no if keyword, and parentheses optional around the boolean expression: + +POKI_CARDIFY{{mlr put 'NR == 4 {$foo = "bar"}'}}HERE +POKI_CARDIFY{{mlr put 'if (NR == 4) {$foo = "bar"}'}}HERE + +

Compound statements use elif (rather than elsif or else if): + +POKI_INCLUDE_ESCAPED(data/if-chain.sh)HERE + +

While-loop and do-while-loop statements for put

+ +

Miller’s while and do-while are unsurprising in +comparison to various languages, as are break and continue: + +POKI_INCLUDE_AND_RUN_ESCAPED(data/while-example-1.sh)HERE +POKI_INCLUDE_AND_RUN_ESCAPED(data/while-example-2.sh)HERE + +

A break or continue within nested conditional blocks or if-statements will, of course, +propagate to the loop enclosing them, if any. A break or continue outside a loop is a syntax +error that will be flagged as soon as the expression is parsed. + +

The existence of while, do-while, and for loops +in Miller’s DSL means that you can create infinite-loop scenarios +inadvertently. In particular, please recall that DSL statements are executed +once if in begin or end blocks, and once per record +otherwise. For example, while (NR < 10) will never terminate as +NR is only incremented between records. + +

For-loop statements for put

+ +

While Miller’s while and do-while statements are +much as in many other languages, for loops are more idiosyncratic to +Miller. They are loops over key-value pairs, whether in stream records or +out-of-stream variables: more reminiscent of foreach, as in (for +example) PHP. + +

There are a few variants, as follows. In each case the in keyword +specifies the hashmap being iterated over, and the variable names between +for and in are bound to the keys and values, respectively, of +the hashmap's key-value pairs on each loop iteration. Also, as with +while and do-while, a break or continue +within nested control structures will propagate to the loop enclosing them, if +any, and a break or continue outside a loop is a syntax error +that will be flagged as soon as the expression is parsed. + +

For-loop over the current stream record: + +POKI_RUN_COMMAND{{cat data/for-srec-example.tbl}}HERE +POKI_INCLUDE_AND_RUN_ESCAPED(data/for-srec-example-1.sh)HERE + +

Note that the value of the current field in the for-loop can be gotten either using the bound +variable value, or through an indirect field name using $[key]. + +

No triple-for: As of Miller 4.1.0 there is no C-style triple-for of the form + +POKI_CARDIFY{{for (i = 1; i <= 10; i++) { ... } # No such}}HERE + +but this can be synthesized using out-of-stream variables and while: + +POKI_CARDIFY{{@i = 1; while (@i <= 10) {...; @i += 1}}}HERE + +

For-loop over out-of-stream variable: This is similar to looping over the current stream +record except for additional degrees of freedom: you can start iterating on sub-hashmaps of +an out-of-stream variable; you + +

xxx finish typing me up + +POKI_INCLUDE_AND_RUN_ESCAPED(data/for-oosvar-example-1.sh)HERE +

regularize

diff --git a/doc/data/for-oosvar-example-1.sh b/doc/data/for-oosvar-example-1.sh new file mode 100644 index 000000000..1cb5a773d --- /dev/null +++ b/doc/data/for-oosvar-example-1.sh @@ -0,0 +1,32 @@ +mlr --opprint put -q ' + @xes[NR] = $x; + @all_fields[NR] = $*; + end { + dump; + + # Simple loop over key-value pairs in an out-of-stream variable: + for (key, value in @xes) { + @item1[key] = value + } + emitp @item1, "key"; + + # Simple loop over key-value pairs in a subhashmap of an out-of-stream variable: + for (key, value in @all_fields[4]) { + @item2[key] = value + } + emitp @item2, "key"; + + # Loop over multilevel key-value pairs: + for ((nr, key), value in @all_fields) { + @item3[nr][key] = value + } + emitp @item3, "NR", "key"; + + # Loop over all out-of-stream variables with depth two + for ((k1, k2), v in @*) { + @item4[string(k1)."_".string(k2)] = v; + } + emitp @item4, "key", "value" + + } +' data/small diff --git a/doc/data/for-srec-example-1.sh b/doc/data/for-srec-example-1.sh new file mode 100644 index 000000000..2bd081339 --- /dev/null +++ b/doc/data/for-srec-example-1.sh @@ -0,0 +1,12 @@ +mlr --pprint put ' + $sum1 = $f1 + $f2 + $f3; + $sum2 = 0; + $sum3 = 0; + for (key, value in $*) { + if (key =~ "^f[0-9]+") { + # xxx fix me + $sum2 += float(value); + $sum3 += $[key]; + } + } +' data/for-srec-example.tbl diff --git a/doc/data/for-srec-example.tbl b/doc/data/for-srec-example.tbl new file mode 100644 index 000000000..64791c613 --- /dev/null +++ b/doc/data/for-srec-example.tbl @@ -0,0 +1,4 @@ +label1 label2 f1 f2 f3 +blue green 100 240 350 +red green 120 11 195 +yellow blue 140 0 240 diff --git a/doc/data/if-chain.sh b/doc/data/if-chain.sh new file mode 100644 index 000000000..ebe2de795 --- /dev/null +++ b/doc/data/if-chain.sh @@ -0,0 +1,11 @@ +mlr put ' + if (NR == 2) { + ... + } elif (NR ==4) { + ... + } elif (NR ==6) { + ... + } else { + ... + } +' diff --git a/doc/data/while-example-1.sh b/doc/data/while-example-1.sh new file mode 100644 index 000000000..16575ef86 --- /dev/null +++ b/doc/data/while-example-1.sh @@ -0,0 +1,6 @@ +echo x=1,y=2 | mlr put ' + while (NF < 10) { + $[NF+1] = "" + } + $foo = "bar" +' diff --git a/doc/data/while-example-2.sh b/doc/data/while-example-2.sh new file mode 100644 index 000000000..ffbb78d9e --- /dev/null +++ b/doc/data/while-example-2.sh @@ -0,0 +1,9 @@ +echo x=1,y=2 | mlr put ' + do { + $[NF+1] = ""; + if (NF == 5) { + break + } + } while (NF < 10); + $foo = "bar" +' diff --git a/doc/mlr.1 b/doc/mlr.1 index 10c5b73d1..77285c3cd 100644 --- a/doc/mlr.1 +++ b/doc/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2016-05-27 +.\" Date: 2016-05-29 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2016-05-27" "\ \&" "\ \&" +.TH "MILLER" "1" "2016-05-29" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/reference.html b/doc/reference.html index 084cf30ca..6538aaa84 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -173,13 +173,17 @@ Miller commands were run with pretty-print-tabular output format.         • Built-in variables for put
        • Expression formatting for put
        • Out-of-stream variables for put
-        • Pattern-action blocks for put
        • Begin/end blocks for put
        • Indexed out-of-stream variables for put
        • Emit statements for put
        • Emit-all statements for put
        • Unset statements for put
        • More variable assignments for put
+        • Semicolons and curly braces for put
+        • Pattern-action blocks for put
+        • If-statements for put
+        • While-loop and do-while-loop statements for put
+        • For-loop statements for put
    • regularize
    • rename
    • reorder
@@ -2318,88 +2322,6 @@ stdout.

Features of out-of-stream variables, and examples of their use, will be presented in the following sections. -

Pattern-action blocks for put

- -

These are reminiscent of awk syntax. They can be used to allow -assignments to be done only when appropriate — e.g. for math-function -domain restrictions, regex-matching, and so on: - -

-

-
-$ mlr cat data/put-gating-example-1.dkvp
-x=-1
-x=0
-x=1
-x=2
-x=3
-
-
-

-

-

-
-$ mlr put '$x > 0.0 { $y = log10($x); $z = sqrt($y) }' data/put-gating-example-1.dkvp
-x=-1
-x=0
-x=1,y=0.000000,z=0.000000
-x=2,y=0.301030,z=0.548662
-x=3,y=0.477121,z=0.690740
-
-
-

-

-

-
-$ mlr cat data/put-gating-example-2.dkvp
-a=abc_123
-a=some other name
-a=xyz_789
-
-
-

-

-

-
-$ mlr put '$a =~ "([a-z]+)_([0-9]+)" { $b = "left_\1"; $c = "right_\2" }' data/put-gating-example-2.dkvp
-a=abc_123,b=left_abc,c=right_123
-a=some other name
-a=xyz_789,b=left_xyz,c=right_789
-
-
-

- -

This produces heteregenous output which Miller, of course, has no problems -with (see Record-heterogeneity). But -if homogeneous output is desired, the curly braces can be omitted. This -causes put to evaluate the boolean expression (along with any side -effects, namely, regex-captures \1, \2, etc.) but -doesn’t use it as a criterion for whether subsequent assignments should -be executed. Instead, subsequent assignments are done unconditionally: - -

-

-
-$ mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp
-x=-1,y=nan,z=nan
-x=0,y=-inf,z=nan
-x=1,y=0.000000,z=0.000000
-x=2,y=0.301030,z=0.548662
-x=3,y=0.477121,z=0.690740
-
-
-

-

-

-
-$ mlr put '$a =~ "([a-z]+)_([0-9]+)"; $b = "left_\1"; $c = "right_\2"' data/put-gating-example-2.dkvp
-a=abc_123,b=left_abc,c=right_123
-a=some other name,b=left_,c=right_
-a=xyz_789,b=left_xyz,c=right_789
-
-
-

-

Begin/end blocks for put

Miller supports an awk-like begin/end syntax. The @@ -3132,6 +3054,464 @@ eks pan 2 0.7586799647899636 0.5221511083334797

+

Semicolons and curly braces for put

+ +

Miller uses semicolons as statement separators, not statement terminators. This means you can write: + +

+

+
+mlr put 'x=1'
+
+
+

+

+

+
+mlr put 'x=1;$y=2'
+
+
+

+

+

+
+mlr put 'x=1;$y=2;'
+
+
+

+

+

+
+mlr put 'x=1;;;;$y=2;'
+
+
+

+ +

Semicolons are required between statements even if those statements are on separate lines. + +

Semicolons are optional after closing curly braces (which close conditionals and loops as discussed below). + +

+

+
+$ echo x=1,y=2 | mlr put 'while (NF < 10) { $[NF+1] = ""} $foo = "bar"'
+x=1,y=2,3=,4=,5=,6=,7=,8=,9=,10=,foo=bar
+
+
+

+ +

Bodies for all compound statements must be enclosed in curly braces, even if the body is a single statement: +

+

+
+mlr put 'if ($x == 1) $y = 2 # Syntax error
+
+
+

+ +

Bodies for compound statements may be empty: + +

+

+
+mlr put 'if ($x == 1) { } # This no-op is syntactically acceptable
+
+
+

+ +

Pattern-action blocks for put

+ +

These are reminiscent of awk syntax. They can be used to allow +assignments to be done only when appropriate — e.g. for math-function +domain restrictions, regex-matching, and so on: + +

+

+
+$ mlr cat data/put-gating-example-1.dkvp
+x=-1
+x=0
+x=1
+x=2
+x=3
+
+
+

+

+

+
+$ mlr put '$x > 0.0 { $y = log10($x); $z = sqrt($y) }' data/put-gating-example-1.dkvp
+x=-1
+x=0
+x=1,y=0.000000,z=0.000000
+x=2,y=0.301030,z=0.548662
+x=3,y=0.477121,z=0.690740
+
+
+

+

+

+
+$ mlr cat data/put-gating-example-2.dkvp
+a=abc_123
+a=some other name
+a=xyz_789
+
+
+

+

+

+
+$ mlr put '$a =~ "([a-z]+)_([0-9]+)" { $b = "left_\1"; $c = "right_\2" }' data/put-gating-example-2.dkvp
+a=abc_123,b=left_abc,c=right_123
+a=some other name
+a=xyz_789,b=left_xyz,c=right_789
+
+
+

+ +

This produces heteregenous output which Miller, of course, has no problems +with (see Record-heterogeneity). But if +homogeneous output is desired, the curly braces can be replaced with a +semicolon between the expression and the body statements. This causes +put to evaluate the boolean expression (along with any side effects, +namely, regex-captures \1, \2, etc.) but doesn’t use it +as a criterion for whether subsequent assignments should be executed. Instead, +subsequent assignments are done unconditionally: + +

+

+
+$ mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp
+x=-1,y=nan,z=nan
+x=0,y=-inf,z=nan
+x=1,y=0.000000,z=0.000000
+x=2,y=0.301030,z=0.548662
+x=3,y=0.477121,z=0.690740
+
+
+

+

+

+
+$ mlr put '$a =~ "([a-z]+)_([0-9]+)"; $b = "left_\1"; $c = "right_\2"' data/put-gating-example-2.dkvp
+a=abc_123,b=left_abc,c=right_123
+a=some other name,b=left_,c=right_
+a=xyz_789,b=left_xyz,c=right_789
+
+
+

+ +

If-statements for put

+ +

These are again reminiscent of awk. Pattern-action blocks are a special case of if with no +elif or else blocks, no if keyword, and parentheses optional around the boolean expression: + +

+

+
+mlr put 'NR == 4 {$foo = "bar"}'
+
+
+

+

+

+
+mlr put 'if (NR == 4) {$foo = "bar"}'
+
+
+

+ +

Compound statements use elif (rather than elsif or else if): + +

+

+
+mlr put '
+  if (NR == 2) {
+    ...
+  } elif (NR ==4) {
+    ...
+  } elif (NR ==6) {
+    ...
+  } else {
+    ...
+  }
+'
+
+
+

+ +

While-loop and do-while-loop statements for put

+ +

Miller’s while and do-while are unsurprising in +comparison to various languages, as are break and continue: + +

+

+
+$ echo x=1,y=2 | mlr put '
+  while (NF < 10) {
+    $[NF+1] = ""
+  }
+  $foo = "bar"
+'
+x=1,y=2,3=,4=,5=,6=,7=,8=,9=,10=,foo=bar
+
+
+

+

+

+
+$ echo x=1,y=2 | mlr put '
+  do {
+    $[NF+1] = "";
+    if (NF == 5) {
+      break
+    }
+  } while (NF < 10);
+  $foo = "bar"
+'
+x=1,y=2,3=,4=,5=,foo=bar
+
+
+

+ +

A break or continue within nested conditional blocks or if-statements will, of course, +propagate to the loop enclosing them, if any. A break or continue outside a loop is a syntax +error that will be flagged as soon as the expression is parsed. + +

The existence of while, do-while, and for loops +in Miller’s DSL means that you can create infinite-loop scenarios +inadvertently. In particular, please recall that DSL statements are executed +once if in begin or end blocks, and once per record +otherwise. For example, while (NR < 10) will never terminate as +NR is only incremented between records. + +

For-loop statements for put

+ +

While Miller’s while and do-while statements are +much as in many other languages, for loops are more idiosyncratic to +Miller. They are loops over key-value pairs, whether in stream records or +out-of-stream variables: more reminiscent of foreach, as in (for +example) PHP. + +

There are a few variants, as follows. In each case the in keyword +specifies the hashmap being iterated over, and the variable names between +for and in are bound to the keys and values, respectively, of +the hashmap's key-value pairs on each loop iteration. Also, as with +while and do-while, a break or continue +within nested control structures will propagate to the loop enclosing them, if +any, and a break or continue outside a loop is a syntax error +that will be flagged as soon as the expression is parsed. + +

For-loop over the current stream record: + +

+

+
+$ cat data/for-srec-example.tbl
+label1 label2 f1  f2  f3
+blue   green  100 240 350
+red    green  120 11  195
+yellow blue   140 0   240
+
+
+

+

+

+
+$ mlr --pprint put '
+  $sum1 = $f1 + $f2 + $f3;
+  $sum2 = 0;
+  $sum3 = 0;
+  for (key, value in $*) {
+    if (key =~ "^f[0-9]+") {
+      # xxx fix me
+      $sum2 += float(value);
+      $sum3 += $[key];
+    }
+  }
+' data/for-srec-example.tbl
+label1 label2 f1  f2  f3  sum1 sum2       sum3
+blue   green  100 240 350 690  690.000000 690
+red    green  120 11  195 326  326.000000 326
+yellow blue   140 0   240 380  380.000000 380
+
+
+

+ +

Note that the value of the current field in the for-loop can be gotten either using the bound +variable value, or through an indirect field name using $[key]. + +

No triple-for: As of Miller 4.1.0 there is no C-style triple-for of the form + +

+

+
+for (i = 1; i <= 10; i++) { ... } # No such
+
+
+

+ +but this can be synthesized using out-of-stream variables and while: + +

+

+
+@i = 1; while (@i <= 10) {...; @i += 1}
+
+
+

+ +

For-loop over out-of-stream variable: This is similar to looping over the current stream +record except for additional degrees of freedom: you can start iterating on sub-hashmaps of +an out-of-stream variable; you + +

xxx finish typing me up + +

+

+
+$ mlr --opprint put -q '
+  @xes[NR] = $x;
+  @all_fields[NR] = $*;
+  end {
+    dump;
+
+    # Simple loop over key-value pairs in an out-of-stream variable:
+    for (key, value in @xes) {
+      @item1[key] = value
+    }
+    emitp @item1, "key";
+
+    # Simple loop over key-value pairs in a subhashmap of an out-of-stream variable:
+    for (key, value in @all_fields[4]) {
+      @item2[key] = value
+    }
+    emitp @item2, "key";
+
+    # Loop over multilevel key-value pairs:
+    for ((nr, key), value in @all_fields) {
+      @item3[nr][key] = value
+    }
+    emitp @item3, "NR", "key";
+
+    # Loop over all out-of-stream variables with depth two
+    for ((k1, k2), v in @*) {
+      @item4[string(k1)."_".string(k2)] = v;
+    }
+    emitp @item4, "key", "value"
+
+  }
+' data/small
+{
+  "xes": {
+    "1": 0.346790,
+    "2": 0.758680,
+    "3": 0.204603,
+    "4": 0.381399,
+    "5": 0.573289
+  },
+  "all_fields": {
+    "1": {
+      "a": "pan",
+      "b": "pan",
+      "i": 1,
+      "x": 0.3467901443380824,
+      "y": 0.7268028627434533
+    },
+    "2": {
+      "a": "eks",
+      "b": "pan",
+      "i": 2,
+      "x": 0.7586799647899636,
+      "y": 0.5221511083334797
+    },
+    "3": {
+      "a": "wye",
+      "b": "wye",
+      "i": 3,
+      "x": 0.20460330576630303,
+      "y": 0.33831852551664776
+    },
+    "4": {
+      "a": "eks",
+      "b": "wye",
+      "i": 4,
+      "x": 0.38139939387114097,
+      "y": 0.13418874328430463
+    },
+    "5": {
+      "a": "wye",
+      "b": "pan",
+      "i": 5,
+      "x": 0.5732889198020006,
+      "y": 0.8636244699032729
+    }
+  }
+}
+key item1
+1   0.346790
+2   0.758680
+3   0.204603
+4   0.381399
+5   0.573289
+
+key item2
+a   eks
+b   wye
+i   4
+x   0.38139939387114097
+y   0.13418874328430463
+
+NR key item3
+1  a   pan
+1  b   pan
+1  i   1
+1  x   0.3467901443380824
+1  y   0.7268028627434533
+2  a   eks
+2  b   pan
+2  i   2
+2  x   0.7586799647899636
+2  y   0.5221511083334797
+3  a   wye
+3  b   wye
+3  i   3
+3  x   0.20460330576630303
+3  y   0.33831852551664776
+4  a   eks
+4  b   wye
+4  i   4
+4  x   0.38139939387114097
+4  y   0.13418874328430463
+5  a   wye
+5  b   pan
+5  i   5
+5  x   0.5732889198020006
+5  y   0.8636244699032729
+
+key     item4
+xes_1   0.346790
+xes_2   0.758680
+xes_3   0.204603
+xes_4   0.381399
+xes_5   0.573289
+item1_1 0.346790
+item1_2 0.758680
+item1_3 0.204603
+item1_4 0.381399
+item1_5 0.573289
+item2_a eks
+item2_b wye
+item2_i 4
+item2_x 0.38139939387114097
+item2_y 0.13418874328430463
+
+
+

+

regularize