diff --git a/c/todo.txt b/c/todo.txt
index 340cf66df..ff4f8fe16 100644
--- a/c/todo.txt
+++ b/c/todo.txt
@@ -54,6 +54,11 @@ MAPVAR CHECKLIST:
-> string keys before/after int?
-> how to incorporate numeric & lexical flags
+* cookbook x 3:
+ - simple stuff like headerless csv
+ - verbs vs dsl list
+ - weird stuff
+
* debt-reduction
! naming conventions to clarify transfer/copy semantics
- copy/ref of keys vs. values in mlhmmv, local_stack, and callees -- ?!?
diff --git a/doc/content-for-reference-dsl.html b/doc/content-for-reference-dsl.html
index d4caf0f7f..2417a0352 100644
--- a/doc/content-for-reference-dsl.html
+++ b/doc/content-for-reference-dsl.html
@@ -228,6 +228,63 @@ stream are passed through to output (written to standard output) unless fields
are removed with cut, or records are excluded with filter or
put -q, etc. Simply assign a value to a field and it will be output.
+
+Out-of-stream variables
+
+ These are prefixed with an at-sign, e.g. @sum. Furthermore,
+unlike built-in variables and stream-record fields, they are maintained in an
+arbitrarily nested hashmap: you can do @sum += $quanity, or
+@sum[$color] += $quanity, or @sum[$color][$shape] +=
+$quanity. The keys for the multi-level hashmap can be any expression which
+evaluates to string or integer: e.g. @sum[NR] = $a + $b,
+@sum[$a."-".$b] = $x, etc.
+
+ Their names and their values are entirely under your control; they change
+only when you assign to them.
+
+ Just as for field names in stream records, if you want to define out-of-stream variables
+with special characters such as . then you can use braces, e.g. '@{variable.name}["index"]'.
+
+You may use a computed key in square brackets, e.g.
+
+POKI_RUN_COMMAND{{echo s=green,t=blue,a=3,b=4 | mlr put -q '@[$s."_".$t] = $a * $b; emit all'}}HERE
+
+ Out-of-stream variables are scoped to the put command in
+which they appear. In particular, if you have two or more put
+commands separated by then, each put will have its own set of
+out-of-stream variables:
+
+POKI_RUN_COMMAND{{cat data/a.dkvp}}HERE
+POKI_RUN_COMMAND{{mlr put '@sum += $a; end {emit @sum}' then put 'ispresent($a) {$a=10*$a; @sum += $a}; end {emit @sum}' data/a.dkvp}}HERE
+
+ Out-of-stream variables’ extent is from the start to the end of the record stream,
+i.e. every time the put or filter statement referring to them is executed.
+
+ Out-of-stream variables are read-write: you can do $sum=@sum, @sum=$sum,
+etc.
+
+
+Indexed out-of-stream variables
+
+Using an index on the @count and @sum variables, we get the benefit of the
+-g (group-by) option which mlr stats1 and various other Miller commands have:
+
+POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-6.sh)HERE
+POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-7.sh)HERE
+
+Indices can be arbitrarily deep — here there are two or more of them:
+
+POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-6a.sh)HERE
+
+The idea is that stats1, and other Miller commands, encapsulate
+frequently-used patterns with a minimum of keystroking (and run a little
+faster), whereas using out-of-stream variables you have more flexibility and
+control in what you do.
+
+Begin/end blocks can be mixed with pattern/action blocks. For example:
+
+POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-8.sh)HERE
+
Local variables
@@ -475,60 +532,6 @@ when the function is called:
-
-Out-of-stream variables
-
- These are prefixed with an at-sign, e.g. @sum. Furthermore,
-unlike built-in variables and stream-record fields, they are maintained in an
-arbitrarily nested hashmap: you can do @sum += $quanity, or
-@sum[$color] += $quanity, or @sum[$color][$shape] +=
-$quanity. The keys for the multi-level hashmap can be any expression which
-evaluates to string or integer: e.g. @sum[NR] = $a + $b,
-@sum[$a."-".$b] = $x, etc.
-
- Their names and their values are entirely under your control; they change
-only when you assign to them.
-
- Just as for field names in stream records, if you want to define out-of-stream variables
-with special characters such as . then you can use braces, e.g. '@{variable.name}["index"]'.
-
-You may use a computed key in square brackets, e.g.
-
-POKI_RUN_COMMAND{{echo s=green,t=blue,a=3,b=4 | mlr put -q '@[$s."_".$t] = $a * $b; emit all'}}HERE
-
- Out-of-stream variables are scoped to the put command in
-which they appear. In particular, if you have two or more put
-commands separated by then, each put will have its own set of
-out-of-stream variables:
-
-POKI_RUN_COMMAND{{cat data/a.dkvp}}HERE
-POKI_RUN_COMMAND{{mlr put '@sum += $a; end {emit @sum}' then put 'ispresent($a) {$a=10*$a; @sum += $a}; end {emit @sum}' data/a.dkvp}}HERE
-
- Out-of-stream variables are read-write: you can do $sum=@sum, @sum=$sum,
-etc.
-
-
-Indexed out-of-stream variables
-
-Using an index on the @count and @sum variables, we get the benefit of the
--g (group-by) option which mlr stats1 and various other Miller commands have:
-
-POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-6.sh)HERE
-POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-7.sh)HERE
-
-Indices can be arbitrarily deep — here there are two or more of them:
-
-POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-6a.sh)HERE
-
-The idea is that stats1, and other Miller commands, encapsulate
-frequently-used patterns with a minimum of keystroking (and run a little
-faster), whereas using out-of-stream variables you have more flexibility and
-control in what you do.
-
-Begin/end blocks can be mixed with pattern/action blocks. For example:
-
-POKI_INCLUDE_AND_RUN_ESCAPED(data/begin-end-example-8.sh)HERE
-
Aggregate variable assignments
diff --git a/doc/cookbook.html b/doc/cookbook.html
index 4a7164819..df21db1dd 100644
--- a/doc/cookbook.html
+++ b/doc/cookbook.html
@@ -607,33 +607,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
i o fcount seconds_delta
1 1 1 0
-2 2 3 0.000028133
-3 3 5 0.000013828
-4 5 9 0.000017166
-5 8 15 0.000022888
-6 13 25 0.000035048
-7 21 41 0.000047922
-8 34 67 0.000075102
-9 55 109 0.000117064
-10 89 177 0.000185966
-11 144 287 0.000295877
-12 233 465 0.000476122
-13 377 753 0.001272917
-14 610 1219 0.001202106
-15 987 1973 0.001879930
-16 1597 3193 0.003035069
-17 2584 5167 0.005100012
-18 4181 8361 0.009974003
-19 6765 13529 0.012595892
-20 10946 21891 0.019485950
-21 17711 35421 0.031538010
-22 28657 57313 0.050611019
-23 46368 92735 0.084169149
-24 75025 150049 0.131798983
-25 121393 242785 0.211344004
-26 196418 392835 0.345608950
-27 317811 635621 0.557806015
-28 514229 1028457 0.899939060
+2 2 3 0.000025988
+3 3 5 0.000013113
+4 5 9 0.000015974
+5 8 15 0.000021935
+6 13 25 0.000031948
+7 21 41 0.000046015
+8 34 67 0.000072002
+9 55 109 0.000109911
+10 89 177 0.000177145
+11 144 287 0.000280857
+12 233 465 0.000450134
+13 377 753 0.000726938
+14 610 1219 0.001173019
+15 987 1973 0.001877069
+16 1597 3193 0.003116846
+17 2584 5167 0.004981041
+18 4181 8361 0.007852077
+19 6765 13529 0.012398958
+20 10946 21891 0.020217896
+21 17711 35421 0.039513111
+22 28657 57313 0.053396940
+23 46368 92735 0.089246035
+24 75025 150049 0.142396927
+25 121393 242785 0.235119104
+26 196418 392835 0.355266094
+27 317811 635621 0.571523905
+28 514229 1028457 0.946005106
@@ -665,33 +665,33 @@ $ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
i o fcount seconds_delta
1 1 1 0
-2 2 3 0.000028133
-3 3 3 0.000012875
-4 5 3 0.000010967
+2 2 3 0.000069141
+3 3 3 0.000022888
+4 5 3 0.000012159
5 8 3 0.000010967
-6 13 3 0.000010014
+6 13 3 0.000011921
7 21 3 0.000010014
8 34 3 0.000010967
9 55 3 0.000010014
-10 89 3 0.000010014
+10 89 3 0.000010967
11 144 3 0.000010014
-12 233 3 0.000014067
-13 377 3 0.000010014
+12 233 3 0.000015020
+13 377 3 0.000011921
14 610 3 0.000010014
15 987 3 0.000010967
16 1597 3 0.000010014
17 2584 3 0.000010014
-18 4181 3 0.000010014
+18 4181 3 0.000010967
19 6765 3 0.000010014
-20 10946 3 0.000009060
-21 17711 3 0.000011921
+20 10946 3 0.000010014
+21 17711 3 0.000010014
22 28657 3 0.000010014
-23 46368 3 0.000011921
+23 46368 3 0.000013113
24 75025 3 0.000010967
-25 121393 3 0.000010014
-26 196418 3 0.000010014
-27 317811 3 0.000010967
-28 514229 3 0.000010014
+25 121393 3 0.000011921
+26 196418 3 0.000010967
+27 317811 3 0.000010014
+28 514229 3 0.000010967
diff --git a/doc/reference-dsl.html b/doc/reference-dsl.html
index 51bbb8d27..2397f9306 100644
--- a/doc/reference-dsl.html
+++ b/doc/reference-dsl.html
@@ -151,12 +151,12 @@ Miller commands were run with pretty-print-tabular output format.
• Variables
• Built-in variables
• Field names
+ • Out-of-stream variables
+ • Indexed out-of-stream variables
• Local variables
• Type-checking
• Type-test and type-assertion expressions
• Type-declarations for local variables, function parameter, and function return values
- • Out-of-stream variables
- • Indexed out-of-stream variables
• Aggregate variable assignments
• Keywords for filter and put
• Operator precedence
@@ -672,6 +672,184 @@ stream are passed through to output (written to standard output) unless fields
are removed with cut, or records are excluded with filter or
put -q, etc. Simply assign a value to a field and it will be output.
+
+Out-of-stream variables
+
+ These are prefixed with an at-sign, e.g. @sum. Furthermore,
+unlike built-in variables and stream-record fields, they are maintained in an
+arbitrarily nested hashmap: you can do @sum += $quanity, or
+@sum[$color] += $quanity, or @sum[$color][$shape] +=
+$quanity. The keys for the multi-level hashmap can be any expression which
+evaluates to string or integer: e.g. @sum[NR] = $a + $b,
+@sum[$a."-".$b] = $x, etc.
+
+ Their names and their values are entirely under your control; they change
+only when you assign to them.
+
+ Just as for field names in stream records, if you want to define out-of-stream variables
+with special characters such as . then you can use braces, e.g. '@{variable.name}["index"]'.
+
+You may use a computed key in square brackets, e.g.
+
+
+
+
+$ echo s=green,t=blue,a=3,b=4 | mlr put -q '@[$s."_".$t] = $a * $b; emit all'
+green_blue=12
+
+
+
+$ cat data/a.dkvp
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
+
+$ mlr put '@sum += $a; end {emit @sum}' then put 'ispresent($a) {$a=10*$a; @sum += $a}; end {emit @sum}' data/a.dkvp
+a=10,b=2,c=3
+a=40,b=5,c=6
+sum=5
+sum=50
+
+
Indexed out-of-stream variables
+
+Using an index on the @count and @sum variables, we get the benefit of the
+-g (group-by) option which mlr stats1 and various other Miller commands have:
+
+
+
+
+$ mlr put -q '
+ @x_count[$a] += 1;
+ @x_sum[$a] += $x;
+ end {
+ emit @x_count, "a";
+ emit @x_sum, "a";
+ }
+' ../data/small
+a=pan,x_count=2
+a=eks,x_count=3
+a=wye,x_count=2
+a=zee,x_count=2
+a=hat,x_count=1
+a=pan,x_sum=0.849416
+a=eks,x_sum=1.751863
+a=wye,x_sum=0.777892
+a=zee,x_sum=1.125680
+a=hat,x_sum=0.031442
+
+
+
+$ mlr stats1 -a count,sum -f x -g a ../data/small
+a=pan,x_count=2,x_sum=0.849416
+a=eks,x_count=3,x_sum=1.751863
+a=wye,x_count=2,x_sum=0.777892
+a=zee,x_count=2,x_sum=1.125680
+a=hat,x_count=1,x_sum=0.031442
+
+
+
+$ mlr --from data/medium put -q '
+ @x_count[$a][$b] += 1;
+ @x_sum[$a][$b] += $x;
+ end {
+ emit (@x_count, @x_sum), "a", "b";
+ }
+'
+a=pan,b=pan,x_count=427,x_sum=219.185129
+a=pan,b=wye,x_count=395,x_sum=198.432931
+a=pan,b=eks,x_count=429,x_sum=216.075228
+a=pan,b=hat,x_count=417,x_sum=205.222776
+a=pan,b=zee,x_count=413,x_sum=205.097518
+a=eks,b=pan,x_count=371,x_sum=179.963030
+a=eks,b=wye,x_count=407,x_sum=196.945286
+a=eks,b=zee,x_count=357,x_sum=176.880365
+a=eks,b=eks,x_count=413,x_sum=215.916097
+a=eks,b=hat,x_count=417,x_sum=208.783171
+a=wye,b=wye,x_count=377,x_sum=185.295850
+a=wye,b=pan,x_count=392,x_sum=195.847900
+a=wye,b=hat,x_count=426,x_sum=212.033183
+a=wye,b=zee,x_count=385,x_sum=194.774048
+a=wye,b=eks,x_count=386,x_sum=204.812961
+a=zee,b=pan,x_count=389,x_sum=202.213804
+a=zee,b=wye,x_count=455,x_sum=233.991394
+a=zee,b=eks,x_count=391,x_sum=190.961778
+a=zee,b=zee,x_count=403,x_sum=206.640635
+a=zee,b=hat,x_count=409,x_sum=191.300006
+a=hat,b=wye,x_count=423,x_sum=208.883010
+a=hat,b=zee,x_count=385,x_sum=196.349450
+a=hat,b=eks,x_count=389,x_sum=189.006793
+a=hat,b=hat,x_count=381,x_sum=182.853532
+a=hat,b=pan,x_count=363,x_sum=168.553807
+
+
+
+$ mlr put '
+ begin {
+ @num_total = 0;
+ @num_positive = 0;
+ };
+ @num_total += 1;
+ $x > 0.0 {
+ @num_positive += 1;
+ $y = log10($x); $z = sqrt($y)
+ };
+ end {
+ emitf @num_total, @num_positive
+ }
+' data/put-gating-example-1.dkvp
+x=-1
+x=0
+x=1,y=0.000000,z=0.000000
+x=2,y=0.301030,z=0.548662
+x=3,y=0.477121,z=0.690740
+num_total=5,num_positive=3
+
+
Local variables
@@ -1169,181 +1347,6 @@ when the function is called:
-
-Out-of-stream variables
-
- These are prefixed with an at-sign, e.g. @sum. Furthermore,
-unlike built-in variables and stream-record fields, they are maintained in an
-arbitrarily nested hashmap: you can do @sum += $quanity, or
-@sum[$color] += $quanity, or @sum[$color][$shape] +=
-$quanity. The keys for the multi-level hashmap can be any expression which
-evaluates to string or integer: e.g. @sum[NR] = $a + $b,
-@sum[$a."-".$b] = $x, etc.
-
- Their names and their values are entirely under your control; they change
-only when you assign to them.
-
- Just as for field names in stream records, if you want to define out-of-stream variables
-with special characters such as . then you can use braces, e.g. '@{variable.name}["index"]'.
-
-You may use a computed key in square brackets, e.g.
-
-
-
-
-$ echo s=green,t=blue,a=3,b=4 | mlr put -q '@[$s."_".$t] = $a * $b; emit all'
-green_blue=12
-
-
-
-$ cat data/a.dkvp
-a=1,b=2,c=3
-a=4,b=5,c=6
-
-
-
-$ mlr put '@sum += $a; end {emit @sum}' then put 'ispresent($a) {$a=10*$a; @sum += $a}; end {emit @sum}' data/a.dkvp
-a=10,b=2,c=3
-a=40,b=5,c=6
-sum=5
-sum=50
-
-
Indexed out-of-stream variables
-
-Using an index on the @count and @sum variables, we get the benefit of the
--g (group-by) option which mlr stats1 and various other Miller commands have:
-
-
-
-
-$ mlr put -q '
- @x_count[$a] += 1;
- @x_sum[$a] += $x;
- end {
- emit @x_count, "a";
- emit @x_sum, "a";
- }
-' ../data/small
-a=pan,x_count=2
-a=eks,x_count=3
-a=wye,x_count=2
-a=zee,x_count=2
-a=hat,x_count=1
-a=pan,x_sum=0.849416
-a=eks,x_sum=1.751863
-a=wye,x_sum=0.777892
-a=zee,x_sum=1.125680
-a=hat,x_sum=0.031442
-
-
-
-$ mlr stats1 -a count,sum -f x -g a ../data/small
-a=pan,x_count=2,x_sum=0.849416
-a=eks,x_count=3,x_sum=1.751863
-a=wye,x_count=2,x_sum=0.777892
-a=zee,x_count=2,x_sum=1.125680
-a=hat,x_count=1,x_sum=0.031442
-
-
-
-$ mlr --from data/medium put -q '
- @x_count[$a][$b] += 1;
- @x_sum[$a][$b] += $x;
- end {
- emit (@x_count, @x_sum), "a", "b";
- }
-'
-a=pan,b=pan,x_count=427,x_sum=219.185129
-a=pan,b=wye,x_count=395,x_sum=198.432931
-a=pan,b=eks,x_count=429,x_sum=216.075228
-a=pan,b=hat,x_count=417,x_sum=205.222776
-a=pan,b=zee,x_count=413,x_sum=205.097518
-a=eks,b=pan,x_count=371,x_sum=179.963030
-a=eks,b=wye,x_count=407,x_sum=196.945286
-a=eks,b=zee,x_count=357,x_sum=176.880365
-a=eks,b=eks,x_count=413,x_sum=215.916097
-a=eks,b=hat,x_count=417,x_sum=208.783171
-a=wye,b=wye,x_count=377,x_sum=185.295850
-a=wye,b=pan,x_count=392,x_sum=195.847900
-a=wye,b=hat,x_count=426,x_sum=212.033183
-a=wye,b=zee,x_count=385,x_sum=194.774048
-a=wye,b=eks,x_count=386,x_sum=204.812961
-a=zee,b=pan,x_count=389,x_sum=202.213804
-a=zee,b=wye,x_count=455,x_sum=233.991394
-a=zee,b=eks,x_count=391,x_sum=190.961778
-a=zee,b=zee,x_count=403,x_sum=206.640635
-a=zee,b=hat,x_count=409,x_sum=191.300006
-a=hat,b=wye,x_count=423,x_sum=208.883010
-a=hat,b=zee,x_count=385,x_sum=196.349450
-a=hat,b=eks,x_count=389,x_sum=189.006793
-a=hat,b=hat,x_count=381,x_sum=182.853532
-a=hat,b=pan,x_count=363,x_sum=168.553807
-
-
-
-$ mlr put '
- begin {
- @num_total = 0;
- @num_positive = 0;
- };
- @num_total += 1;
- $x > 0.0 {
- @num_positive += 1;
- $y = log10($x); $z = sqrt($y)
- };
- end {
- emitf @num_total, @num_positive
- }
-' data/put-gating-example-1.dkvp
-x=-1
-x=0
-x=1,y=0.000000,z=0.000000
-x=2,y=0.301030,z=0.548662
-x=3,y=0.477121,z=0.690740
-num_total=5,num_positive=3
-
-
Aggregate variable assignments