From 59fa1f482fceb720ef3514b67530a444cc8dae41 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 23 Feb 2021 22:31:04 -0500 Subject: [PATCH] split out substr0 and substr1, and make substr==substr0 the default for backward compatibility with Miller <= 5 --- go/reg-test/expected/case-dsl-slice.sh.out | 6 +- go/reg-test/expected/case-dsl-substr.sh.out | 14 ++--- go/src/dsl/cst/builtin_function_manager.go | 21 +++---- go/src/dsl/cst/collections.go | 2 +- go/src/types/mlrval_functions_strings.go | 62 ++++++++++++++++++++- 5 files changed, 82 insertions(+), 23 deletions(-) diff --git a/go/reg-test/expected/case-dsl-slice.sh.out b/go/reg-test/expected/case-dsl-slice.sh.out index b092586f2..4ad100451 100644 --- a/go/reg-test/expected/case-dsl-slice.sh.out +++ b/go/reg-test/expected/case-dsl-slice.sh.out @@ -62,9 +62,9 @@ mlr -n --ojson put "s93": "" } { - "u35": "cde", - "u37": "cdefg", - "u17": "abcdefg", + "u35": "def", + "u37": "", + "u17": "", "u07": "", "u39": "", "u53": (error), diff --git a/go/reg-test/expected/case-dsl-substr.sh.out b/go/reg-test/expected/case-dsl-substr.sh.out index 332406b7b..cba7d7a32 100644 --- a/go/reg-test/expected/case-dsl-substr.sh.out +++ b/go/reg-test/expected/case-dsl-substr.sh.out @@ -1,28 +1,28 @@ mlr put $y = substr($x, 0, 0) -x=abcdefg,y= +x=abcdefg,y=a mlr put $y = substr($x, 0, 7) x=abcdefg,y= mlr put $y = substr($x, 1, 7) -x=abcdefg,y=abcdefg +x=abcdefg,y= mlr put $y = substr($x, 1, 6) -x=abcdefg,y=abcdef +x=abcdefg,y=bcdefg mlr put $y = substr($x, 2, 5) -x=abcdefg,y=bcde +x=abcdefg,y=cdef mlr put $y = substr($x, 2, 3) -x=abcdefg,y=bc +x=abcdefg,y=cd mlr put $y = substr($x, 3, 3) -x=abcdefg,y=c +x=abcdefg,y=d mlr put $y = substr($x, 4, 3) x=abcdefg,y=(error) mlr put $y = substr($x, 2, 5) -x=1234567,y=2345 +x=1234567,y=3456 diff --git a/go/src/dsl/cst/builtin_function_manager.go b/go/src/dsl/cst/builtin_function_manager.go index 718924230..693928f70 100644 --- a/go/src/dsl/cst/builtin_function_manager.go +++ b/go/src/dsl/cst/builtin_function_manager.go @@ -428,27 +428,28 @@ regex-match operator: try '$y = ~$x'.`, ternaryFunc: types.MlrvalSub, }, - { - name: "substr", - class: FUNC_CLASS_STRING, - help: `substr(s,m,n) gives substring of s from 1-up position m to n xxxxxx fix me -inclusive. Negative indices -len .. -1 alias to 1 .. len.`, - ternaryFunc: types.MlrvalSubstr0Up, - }, { name: "substr0", class: FUNC_CLASS_STRING, - help: `substr(s,m,n) gives substring of s from 1-up position m to n xxxxxx fix me -inclusive. Negative indices -len .. -1 alias to 1 .. len.`, + help: `substr0(s,m,n) gives substring of s from 0-up position m to n +inclusive. Negative indices -len .. -1 alias to 0 .. len-1.`, ternaryFunc: types.MlrvalSubstr0Up, }, { name: "substr1", class: FUNC_CLASS_STRING, - help: `substr(s,m,n) gives substring of s from 1-up position m to n xxxxxx fix me + help: `substr1(s,m,n) gives substring of s from 1-up position m to n inclusive. Negative indices -len .. -1 alias to 1 .. len.`, ternaryFunc: types.MlrvalSubstr1Up, }, + { + name: "substr", + class: FUNC_CLASS_STRING, + help: `substr is an alias for substr0. See also substr1. Miller is generally 1-up +with all array indices, but, this is a backward-compatibility issue with Miller 5 and below. +Arrays are new in Miller 6; the substr function is older.`, + ternaryFunc: types.MlrvalSubstr0Up, + }, { name: "tolower", diff --git a/go/src/dsl/cst/collections.go b/go/src/dsl/cst/collections.go index 982ee54f7..9c986637b 100644 --- a/go/src/dsl/cst/collections.go +++ b/go/src/dsl/cst/collections.go @@ -150,7 +150,7 @@ func (this *ArraySliceAccessNode) Evaluate( return types.MLRVAL_ABSENT } if baseMlrval.IsString() { - return types.MlrvalSubstr(baseMlrval, lowerIndexMlrval, upperIndexMlrval) + return types.MlrvalSubstr1Up(baseMlrval, lowerIndexMlrval, upperIndexMlrval) } array := baseMlrval.GetArray() if array == nil { diff --git a/go/src/types/mlrval_functions_strings.go b/go/src/types/mlrval_functions_strings.go index 98ff92ed9..bb3d3ff1a 100644 --- a/go/src/types/mlrval_functions_strings.go +++ b/go/src/types/mlrval_functions_strings.go @@ -60,10 +60,10 @@ func MlrvalDot(input1, input2 *Mlrval) *Mlrval { } // ================================================================ -// substr(s,m,n) gives substring of s from 1-up position m to n inclusive. +// substr1(s,m,n) gives substring of s from 1-up position m to n inclusive. // Negative indices -len .. -1 alias to 0 .. len-1. -func MlrvalSubstr(input1, input2, input3 *Mlrval) *Mlrval { +func MlrvalSubstr1Up(input1, input2, input3 *Mlrval) *Mlrval { if !input1.IsStringOrVoid() { if input1.IsNumeric() { // JIT-stringify, if not already (e.g. intval scanned from string @@ -109,6 +109,64 @@ func MlrvalSubstr(input1, input2, input3 *Mlrval) *Mlrval { } } +// ================================================================ +// substr0(s,m,n) gives substring of s from 0-up position m to n inclusive. +// Negative indices -len .. -1 alias to 0 .. len-1. + +func MlrvalSubstr0Up(input1, input2, input3 *Mlrval) *Mlrval { + if !input1.IsStringOrVoid() { + if input1.IsNumeric() { + // JIT-stringify, if not already (e.g. intval scanned from string + // in input-file data) + input1.setPrintRep() + } else { + return MLRVAL_ERROR + } + } + // TODO: fix this with regard to UTF-8 and runes. + strlen := int(len(input1.printrep)) + + // For array slices like s[1:2], s[:2], s[1:], when the lower index is + // empty in the DSL expression it comes in here as a 1. But when the upper + // index is empty in the DSL expression it comes in here as "". + if !input2.IsInt() { + return MLRVAL_ERROR + } + lowerMindex := input2.intval + if lowerMindex >= 0 { + // Make 1-up + lowerMindex += 1 + } + + upperMindex := strlen + if input3.IsEmpty() { + // Keep strlen + } else if !input3.IsInt() { + return MLRVAL_ERROR + } else { + upperMindex = input3.intval + if upperMindex >= 0 { + // Make 1-up + upperMindex += 1 + } + } + + // Convert from negative-aliased 1-up to positive-only 0-up + m, mok := UnaliasArrayLengthIndex(strlen, lowerMindex) + n, nok := UnaliasArrayLengthIndex(strlen, upperMindex) + + if !mok || !nok { + return MlrvalPointerFromString("") + } else if m > n { + return MLRVAL_ERROR + } else { + // Note Golang slice indices are 0-up, and the 1st index is inclusive + // while the 2nd is exclusive. For Miller, indices are 1-up and both + // are inclusive. + return MlrvalPointerFromString(input1.printrep[m : n+1]) + } +} + // ================================================================ func MlrvalTruncate(input1, input2 *Mlrval) *Mlrval { if input1.IsErrorOrAbsent() {