New concat DSL function for arrays (#868)

* New concat DSL function for arrays

* Documentation artifacts for previous commit

* Regression-test cases

docs/src/manpage.md

@@ -205,9 +205,9 @@ FUNCTION LIST
        asserting_map asserting_nonempty_map asserting_not_array asserting_not_empty
        asserting_not_map asserting_not_null asserting_null asserting_numeric
        asserting_present asserting_string atan atan2 atanh bitcount boolean
-       capitalize cbrt ceil clean_whitespace collapse_whitespace cos cosh depth
-       dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum fold
-       fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
+       capitalize cbrt ceil clean_whitespace collapse_whitespace concat cos cosh
+       depth dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum
+       fold fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
        hexfmt hms2fsec hms2sec hostname int invqnorm is_absent is_array is_bool
        is_boolean is_empty is_empty_map is_error is_float is_int is_map
        is_nonempty_map is_not_array is_not_empty is_not_map is_not_null is_null
@@ -2077,6 +2077,13 @@ FUNCTIONS FOR FILTER/PUT
    collapse_whitespace
         (class=string #args=1) Strip repeated whitespace from string.
 
+   concat
+        (class=collections #args=variadic) Returns the array concatenation of the arguments. Non-array arguments are treated as single-element arrays.
+       Examples:
+       concat(1,2,3) is [1,2,3]
+       concat([1,2],3) is [1,2,3]
+       concat([1,2],[3]) is [1,2,3]
+
    cos
         (class=math #args=1) Trigonometric cosine.
 
@@ -3054,5 +3061,5 @@ SEE ALSO
 
 
 
-                                  2022-01-11                         MILLER(1)
+                                  2022-01-13                         MILLER(1)
 </pre>

docs/src/manpage.txt

@@ -184,9 +184,9 @@ FUNCTION LIST
        asserting_map asserting_nonempty_map asserting_not_array asserting_not_empty
        asserting_not_map asserting_not_null asserting_null asserting_numeric
        asserting_present asserting_string atan atan2 atanh bitcount boolean
-       capitalize cbrt ceil clean_whitespace collapse_whitespace cos cosh depth
-       dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum fold
-       fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
+       capitalize cbrt ceil clean_whitespace collapse_whitespace concat cos cosh
+       depth dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum
+       fold fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
        hexfmt hms2fsec hms2sec hostname int invqnorm is_absent is_array is_bool
        is_boolean is_empty is_empty_map is_error is_float is_int is_map
        is_nonempty_map is_not_array is_not_empty is_not_map is_not_null is_null
@@ -2056,6 +2056,13 @@ FUNCTIONS FOR FILTER/PUT
    collapse_whitespace
         (class=string #args=1) Strip repeated whitespace from string.
 
+   concat
+        (class=collections #args=variadic) Returns the array concatenation of the arguments. Non-array arguments are treated as single-element arrays.
+       Examples:
+       concat(1,2,3) is [1,2,3]
+       concat([1,2],3) is [1,2,3]
+       concat([1,2],[3]) is [1,2,3]
+
    cos
         (class=math #args=1) Trigonometric cosine.
 
@@ -3033,4 +3040,4 @@ SEE ALSO
 
 
 
-                                  2022-01-11                         MILLER(1)
+                                  2022-01-13                         MILLER(1)

docs/src/reference-dsl-builtin-functions.md

@@ -69,7 +69,7 @@ is 2. Unary operators such as `!` and `~` show argument-count of 1; the ternary
 
 * [**Arithmetic functions**](#arithmetic-functions):  [bitcount](#bitcount),  [madd](#madd),  [mexp](#mexp),  [mmul](#mmul),  [msub](#msub),  [pow](#pow),  [%](#percent),  [&](#bitwise-and),  [\*](#times),  [\**](#exponentiation),  [\+](#plus),  [\-](#minus),  [\.\*](#dot-times),  [\.\+](#dot-plus),  [\.\-](#dot-minus),  [\./](#dot-slash),  [/](#slash),  [//](#slash-slash),  [<<](#lsh),  [>>](#srsh),  [>>>](#ursh),  [^](#bitwise-xor),  [\|](#bitwise-or),  [~](#bitwise-not).
 * [**Boolean functions**](#boolean-functions):  [\!](#exclamation-point),  [\!=](#exclamation-point-equals),  [!=~](#regnotmatch),  [&&](#logical-and),  [<](#less-than),  [<=](#less-than-or-equals),  [<=>](#<=>),  [==](#double-equals),  [=~](#regmatch),  [>](#greater-than),  [>=](#greater-than-or-equals),  [?:](#question-mark-colon),  [??](#absent-coalesce),  [???](#absent-empty-coalesce),  [^^](#logical-xor),  [\|\|](#logical-or).
-* [**Collections functions**](#collections-functions):  [append](#append),  [arrayify](#arrayify),  [depth](#depth),  [flatten](#flatten),  [get_keys](#get_keys),  [get_values](#get_values),  [haskey](#haskey),  [json_parse](#json_parse),  [json_stringify](#json_stringify),  [leafcount](#leafcount),  [length](#length),  [mapdiff](#mapdiff),  [mapexcept](#mapexcept),  [mapselect](#mapselect),  [mapsum](#mapsum),  [unflatten](#unflatten).
+* [**Collections functions**](#collections-functions):  [append](#append),  [arrayify](#arrayify),  [concat](#concat),  [depth](#depth),  [flatten](#flatten),  [get_keys](#get_keys),  [get_values](#get_values),  [haskey](#haskey),  [json_parse](#json_parse),  [json_stringify](#json_stringify),  [leafcount](#leafcount),  [length](#length),  [mapdiff](#mapdiff),  [mapexcept](#mapexcept),  [mapselect](#mapselect),  [mapsum](#mapsum),  [unflatten](#unflatten).
 * [**Conversion functions**](#conversion-functions):  [boolean](#boolean),  [float](#float),  [fmtnum](#fmtnum),  [hexfmt](#hexfmt),  [int](#int),  [joink](#joink),  [joinkv](#joinkv),  [joinv](#joinv),  [splita](#splita),  [splitax](#splitax),  [splitkv](#splitkv),  [splitkvx](#splitkvx),  [splitnv](#splitnv),  [splitnvx](#splitnvx),  [string](#string).
 * [**Hashing functions**](#hashing-functions):  [md5](#md5),  [sha1](#sha1),  [sha256](#sha256),  [sha512](#sha512).
 * [**Higher-order-functions functions**](#higher-order-functions-functions):  [any](#any),  [apply](#apply),  [every](#every),  [fold](#fold),  [reduce](#reduce),  [select](#select),  [sort](#sort).
@@ -371,6 +371,16 @@ arrayify  (class=collections #args=1) Walks through a nested map/array, converti
 </pre>
 
 
+### concat
+<pre class="pre-non-highlight-non-pair">
+concat  (class=collections #args=variadic) Returns the array concatenation of the arguments. Non-array arguments are treated as single-element arrays.
+Examples:
+concat(1,2,3) is [1,2,3]
+concat([1,2],3) is [1,2,3]
+concat([1,2],[3]) is [1,2,3]
+</pre>
+
+
 ### depth
 <pre class="pre-non-highlight-non-pair">
 depth  (class=collections #args=1) Prints maximum depth of map/array. Scalars have depth 0.

docs/src/reference-dsl-differences.md

@@ -64,7 +64,7 @@ error.
 </pre>
 <pre class="pre-non-highlight-in-pair">
 mlr: cannot parse DSL expression.
-Parse error on token "$y" at line 6 columnn 3.
+Parse error on token "$y" at line 6 column 3.
 Please check for missing semicolon.
 Expected one of:
   $ ; > >> | ? || ^^ && ?? ??? =~ !=~ == != <=> >= < <= ^ & << >>> + - .+

docs/src/reference-dsl-higher-order-functions.md

@@ -809,7 +809,7 @@ As of September 2021, immediately invoked function expressions (IIFEs) are not p
 </pre>
 <pre class="pre-non-highlight-in-pair">
 mlr: cannot parse DSL expression.
-Parse error on token "(" at line 4 columnn 35.
+Parse error on token "(" at line 4 column 35.
 Please check for missing semicolon.
 Expected one of:
   ; } > >> | ? || ^^ && ?? ??? =~ !=~ == != <=> >= < <= ^ & << >>> + - .+

internal/pkg/bifs/collections.go

@@ -187,6 +187,24 @@ func BIF_haskey(input1, input2 *mlrval.Mlrval) *mlrval.Mlrval {
 	}
 }
 
+// ================================================================
+func BIF_concat(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval {
+	output := mlrval.FromEmptyArray()
+
+	for _, arg := range mlrvals {
+		argArray := arg.GetArray()
+		if argArray == nil { // not an array
+			output.ArrayAppend(arg.Copy())
+		} else {
+			for i := range argArray {
+				output.ArrayAppend(argArray[i].Copy())
+			}
+		}
+	}
+
+	return output
+}
+
 // ================================================================
 func BIF_mapselect(mlrvals []*mlrval.Mlrval) *mlrval.Mlrval {
 	if len(mlrvals) < 1 {

internal/pkg/dsl/cst/builtin_function_manager.go

@@ -1494,6 +1494,19 @@ strftime_local.`,
 			unaryFunc: bifs.BIF_arrayify,
 		},
 
+		{
+			name:  "concat",
+			class: FUNC_CLASS_COLLECTIONS,
+			help: `Returns the array concatenation of the arguments. Non-array arguments are treated as
+single-element arrays.`,
+			examples: []string{
+				`concat(1,2,3) is [1,2,3]`,
+				`concat([1,2],3) is [1,2,3]`,
+				`concat([1,2],[3]) is [1,2,3]`,
+			},
+			variadicFunc: bifs.BIF_concat,
+		},
+
 		{
 			name:      "depth",
 			class:     FUNC_CLASS_COLLECTIONS,

man/manpage.txt

@@ -184,9 +184,9 @@ FUNCTION LIST
        asserting_map asserting_nonempty_map asserting_not_array asserting_not_empty
        asserting_not_map asserting_not_null asserting_null asserting_numeric
        asserting_present asserting_string atan atan2 atanh bitcount boolean
-       capitalize cbrt ceil clean_whitespace collapse_whitespace cos cosh depth
-       dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum fold
-       fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
+       capitalize cbrt ceil clean_whitespace collapse_whitespace concat cos cosh
+       depth dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum
+       fold fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
        hexfmt hms2fsec hms2sec hostname int invqnorm is_absent is_array is_bool
        is_boolean is_empty is_empty_map is_error is_float is_int is_map
        is_nonempty_map is_not_array is_not_empty is_not_map is_not_null is_null
@@ -2056,6 +2056,13 @@ FUNCTIONS FOR FILTER/PUT
    collapse_whitespace
         (class=string #args=1) Strip repeated whitespace from string.
 
+   concat
+        (class=collections #args=variadic) Returns the array concatenation of the arguments. Non-array arguments are treated as single-element arrays.
+       Examples:
+       concat(1,2,3) is [1,2,3]
+       concat([1,2],3) is [1,2,3]
+       concat([1,2],[3]) is [1,2,3]
+
    cos
         (class=math #args=1) Trigonometric cosine.
 
@@ -3033,4 +3040,4 @@ SEE ALSO
 
 
 
-                                  2022-01-11                         MILLER(1)
+                                  2022-01-13                         MILLER(1)

man/mlr.1

@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2022-01-11
+.\"      Date: 2022-01-13
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2022-01-11" "\ \&" "\ \&"
+.TH "MILLER" "1" "2022-01-13" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -231,9 +231,9 @@ asserting_empty_map asserting_error asserting_float asserting_int
 asserting_map asserting_nonempty_map asserting_not_array asserting_not_empty
 asserting_not_map asserting_not_null asserting_null asserting_numeric
 asserting_present asserting_string atan atan2 atanh bitcount boolean
-capitalize cbrt ceil clean_whitespace collapse_whitespace cos cosh depth
-dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum fold
-fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
+capitalize cbrt ceil clean_whitespace collapse_whitespace concat cos cosh
+depth dhms2fsec dhms2sec erf erfc every exp expm1 flatten float floor fmtnum
+fold fsec2dhms fsec2hms get_keys get_values gmt2localtime gmt2sec gsub haskey
 hexfmt hms2fsec hms2sec hostname int invqnorm is_absent is_array is_bool
 is_boolean is_empty is_empty_map is_error is_float is_int is_map
 is_nonempty_map is_not_array is_not_empty is_not_map is_not_null is_null
@@ -2805,6 +2805,19 @@ Map example: apply({"a":1, "b":3, "c":5}, func(k,v) {return {toupper(k): v ** 2}
 .fi
 .if n \{\
 .RE
+.SS "concat"
+.if n \{\
+.RS 0
+.\}
+.nf
+ (class=collections #args=variadic) Returns the array concatenation of the arguments. Non-array arguments are treated as single-element arrays.
+Examples:
+concat(1,2,3) is [1,2,3]
+concat([1,2],3) is [1,2,3]
+concat([1,2],[3]) is [1,2,3]
+.fi
+.if n \{\
+.RE
 .SS "cos"
 .if n \{\
 .RS 0

test/cases/dsl-concat/0001/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0001/expout

@@ -0,0 +1 @@
+[]

test/cases/dsl-concat/0001/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat()
+}

test/cases/dsl-concat/0002/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0002/expout

@@ -0,0 +1 @@
+[1]

test/cases/dsl-concat/0002/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat(1)
+}

test/cases/dsl-concat/0003/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0003/expout

@@ -0,0 +1 @@
+[]

test/cases/dsl-concat/0003/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat([])
+}

test/cases/dsl-concat/0004/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0004/expout

@@ -0,0 +1 @@
+[1, 2]

test/cases/dsl-concat/0004/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat([1,2])
+}

test/cases/dsl-concat/0005/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0005/expout

@@ -0,0 +1 @@
+[1, 2, 3]

test/cases/dsl-concat/0005/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat(1,2,3)
+}

test/cases/dsl-concat/0006/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0006/expout

@@ -0,0 +1 @@
+[1, 2, 3]

test/cases/dsl-concat/0006/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat([1,2],3)
+}

test/cases/dsl-concat/0007/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0007/expout

@@ -0,0 +1 @@
+[1, 2, 3]

test/cases/dsl-concat/0007/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat([1,2],[3])
+}

test/cases/dsl-concat/0008/cmd

@@ -0,0 +1 @@
+mlr -n put -f ${CASEDIR}/mlr

test/cases/dsl-concat/0008/expout

@@ -0,0 +1,9 @@
+[
+  1,
+  2,
+  [3, 4],
+  {
+    "a": 5,
+    "b": 6
+  }
+]

test/cases/dsl-concat/0008/mlr

@@ -0,0 +1,3 @@
+end {
+  print concat([1,2], [ [3,4] ], {"a":5, "b":6})
+}

todo.txt

@@ -36,6 +36,7 @@ UX
 ? trace-mode ?
 * strptime w/ ...00.Z -> error
 * main-level (verb-level?) flag for "," -> X in verbs -- in case commas in field names
+* bnf fix for '[[' ']]' etc -- make it a nesting of singles. since otherwise no '[[3,4]]' literals :(
 
 ================================================================
 DOC