From aff21172d53cd680fa24d647dd36962498bf1ce7 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 26 Oct 2020 17:14:39 -0400 Subject: [PATCH] unset iterate; truncate DSL function --- c/dsl/function_manager.c | 2 + c/dsl/rval_evaluators.h | 5 ++ c/dsl/rval_func_evaluators.c | 46 ++++++++++ c/lib/mvfuncs.c | 17 ++++ c/lib/mvfuncs.h | 2 + c/reg_test/expected/out | 98 ++++++++++++++------- c/reg_test/run | 10 +++ docs/manpage.rst | 38 ++++---- docs/manpage.txt | 38 ++++---- docs/mlr.1 | 46 ++++++---- docs/reference-dsl.rst | 13 +++ docs/reference.rst | 2 +- go/src/miller/dsl/cst/lvalues.go | 101 +++++++++++++++++----- go/src/miller/types/mlrmap_accessors.go | 7 ++ go/src/miller/types/mlrval_collections.go | 3 +- go/todo.txt | 2 + 16 files changed, 322 insertions(+), 108 deletions(-) diff --git a/c/dsl/function_manager.c b/c/dsl/function_manager.c index 67251df01..88ce59584 100644 --- a/c/dsl/function_manager.c +++ b/c/dsl/function_manager.c @@ -241,6 +241,7 @@ static function_lookup_t FUNCTION_LOOKUP_TABLE[] = { "inclusive. Negative indices -len .. -1 alias to 0 .. len-1."}, {FUNC_CLASS_STRING, "tolower", 1,0, "Convert string to lowercase."}, {FUNC_CLASS_STRING, "toupper", 1,0, "Convert string to uppercase."}, + {FUNC_CLASS_STRING, "truncate", 2,0, "Truncates string first argument to max length of int second argument."}, {FUNC_CLASS_STRING, "capitalize", 1,0, "Convert string's first character to uppercase."}, {FUNC_CLASS_STRING, "lstrip", 1,0, "Strip leading whitespace from string."}, {FUNC_CLASS_STRING, "rstrip", 1,0, "Strip trailing whitespace from string."}, @@ -1288,6 +1289,7 @@ static rval_evaluator_t* fmgr_alloc_evaluator_from_binary_func_name(char* fnnm, } else if (streq(fnnm, "strptime")) { return rval_evaluator_alloc_from_x_ss_func(i_ss_strptime_func, parg1, parg2); } else if (streq(fnnm, "strptime_local")) { return rval_evaluator_alloc_from_x_ss_func(i_ss_strptime_local_func, parg1, parg2); } else if (streq(fnnm, "urandrange")) { return rval_evaluator_alloc_from_f_ff_func(f_ff_urandrange_func, parg1, parg2); + } else if (streq(fnnm, "truncate")) { return rval_evaluator_alloc_from_s_si_func(s_si_truncate_func, parg1, parg2); } else { return NULL; } } diff --git a/c/dsl/rval_evaluators.h b/c/dsl/rval_evaluators.h index a03913f1d..b25c78f6d 100644 --- a/c/dsl/rval_evaluators.h +++ b/c/dsl/rval_evaluators.h @@ -136,6 +136,11 @@ rval_evaluator_t* rval_evaluator_alloc_from_f_s_func(mv_unary_func_t* pfunc, rva rval_evaluator_t* rval_evaluator_alloc_from_i_s_func(mv_unary_func_t* pfunc, rval_evaluator_t* parg1); rval_evaluator_t* rval_evaluator_alloc_from_x_x_func(mv_unary_func_t* pfunc, rval_evaluator_t* parg1); +rval_evaluator_t* rval_evaluator_alloc_from_s_si_func( + mv_binary_func_t* pfunc, + rval_evaluator_t* parg1, + rval_evaluator_t* parg2); + rval_evaluator_t* rval_evaluator_alloc_from_x_xi_func( mv_binary_func_t* pfunc, rval_evaluator_t* parg1, diff --git a/c/dsl/rval_func_evaluators.c b/c/dsl/rval_func_evaluators.c index 836b07673..1c1b5432e 100644 --- a/c/dsl/rval_func_evaluators.c +++ b/c/dsl/rval_func_evaluators.c @@ -978,6 +978,52 @@ rval_evaluator_t* rval_evaluator_alloc_from_x_xi_func(mv_binary_func_t* pfunc, return pevaluator; } +// ---------------------------------------------------------------- +typedef struct _rval_evaluator_s_si_state_t { + mv_binary_func_t* pfunc; + rval_evaluator_t* parg1; + rval_evaluator_t* parg2; +} rval_evaluator_s_si_state_t; + +static mv_t rval_evaluator_s_si_func(void* pvstate, variables_t* pvars) { + rval_evaluator_s_si_state_t* pstate = pvstate; + mv_t val1 = pstate->parg1->pprocess_func(pstate->parg1->pvstate, pvars); + mv_t val2 = pstate->parg2->pprocess_func(pstate->parg2->pvstate, pvars); + + if (val1.type != MT_EMPTY && val1.type != MT_STRING) { + return mv_error(); + } + if (val2.type != MT_INT) { + return mv_error(); + } + + // nullity of 1st argument handled by full disposition matrices + return pstate->pfunc(&val1, &val2); +} +static void rval_evaluator_s_si_free(rval_evaluator_t* pevaluator) { + rval_evaluator_s_si_state_t* pstate = pevaluator->pvstate; + pstate->parg1->pfree_func(pstate->parg1); + pstate->parg2->pfree_func(pstate->parg2); + free(pstate); + free(pevaluator); +} + +rval_evaluator_t* rval_evaluator_alloc_from_s_si_func(mv_binary_func_t* pfunc, + rval_evaluator_t* parg1, rval_evaluator_t* parg2) +{ + rval_evaluator_s_si_state_t* pstate = mlr_malloc_or_die(sizeof(rval_evaluator_s_si_state_t)); + pstate->pfunc = pfunc; + pstate->parg1 = parg1; + pstate->parg2 = parg2; + + rval_evaluator_t* pevaluator = mlr_malloc_or_die(sizeof(rval_evaluator_t)); + pevaluator->pvstate = pstate; + pevaluator->pprocess_func = rval_evaluator_s_si_func; + pevaluator->pfree_func = rval_evaluator_s_si_free; + + return pevaluator; +} + // ---------------------------------------------------------------- typedef struct _rval_evaluator_x_ns_state_t { mv_binary_func_t* pfunc; diff --git a/c/lib/mvfuncs.c b/c/lib/mvfuncs.c index fcbfb972c..86c3fb178 100644 --- a/c/lib/mvfuncs.c +++ b/c/lib/mvfuncs.c @@ -490,6 +490,23 @@ mv_t s_s_system_func(mv_t* pval1) { return retval; } +mv_t s_si_truncate_func(mv_t* pval1, mv_t* pval2) { + char* string = pval1->u.strv; + int len = strlen(string); + int maxlen = pval2->u.intv; + + if (len <= maxlen) { + return *pval1; + } else { + char* buf = mlr_malloc_or_die(maxlen+1); + strncpy(buf, string, maxlen); + buf[len] = 0; + mv_free(pval1); + pval1->u.strv = NULL; + return mv_from_string_with_free(buf); + } +} + // ---------------------------------------------------------------- mv_t s_s_lstrip_func(mv_t* pval1) { if (!isspace(pval1->u.strv[0])) { diff --git a/c/lib/mvfuncs.h b/c/lib/mvfuncs.h index 9d937c2b0..e1ffcf112 100644 --- a/c/lib/mvfuncs.h +++ b/c/lib/mvfuncs.h @@ -215,6 +215,8 @@ mv_t s_s_clean_whitespace_func(mv_t* pval1); mv_t s_s_system_func(mv_t* pval1); +mv_t s_si_truncate_func(mv_t* pval1, mv_t* pval2); + mv_t s_xx_dot_func(mv_t* pval1, mv_t* pval2); mv_t sub_no_precomp_func(mv_t* pval1, mv_t* pval2, mv_t* pval3); diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index d496c444d..09bb11852 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -10619,7 +10619,7 @@ x_p100 10 ================================================================ IN-PLACE PROCESSING -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 @@ -10631,7 +10631,7 @@ a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 @@ -10643,14 +10643,14 @@ a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 -mlr -I --opprint head -n 2 /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 +mlr -I --opprint head -n 2 /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 a b i x y pan pan 1 0.3467901443380824 0.7268028627434533 eks pan 2 0.7586799647899636 0.5221511083334797 -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 a b i x y pan pan 1 0.3467901443380824 0.7268028627434533 eks pan 2 0.7586799647899636 0.5221511083334797 @@ -10658,10 +10658,10 @@ eks pan 2 0.7586799647899636 0.5221511083334797 mlr -I --opprint head -n 2 mlr: -I option (in-place operation) requires input files. -mlr -I --opprint -n head -n 2 /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 +mlr -I --opprint -n head -n 2 /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 mlr: -I option (in-place operation) requires input files. -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 @@ -10673,7 +10673,7 @@ a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797 a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 @@ -10685,9 +10685,9 @@ a=zee,b=wye,i=8,x=0.5985540091064224,y=0.976181385699006 a=hat,b=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864 -mlr -I --opprint rename a,AYE,b,BEE /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 +mlr -I --opprint rename a,AYE,b,BEE /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp1 AYE BEE i x y pan pan 1 0.3467901443380824 0.7268028627434533 eks pan 2 0.7586799647899636 0.5221511083334797 @@ -10700,7 +10700,7 @@ zee wye 8 0.5985540091064224 0.976181385699006 hat wye 9 0.03144187646093577 0.7495507603507059 pan wye 10 0.5026260055412137 0.9526183602969864 -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/abixy.temp2 AYE BEE i x y pan pan 1 0.3467901443380824 0.7268028627434533 eks pan 2 0.7586799647899636 0.5221511083334797 @@ -22947,6 +22947,46 @@ s3 10 s4 9 +================================================================ +TRUNCATE + +mlr put $y=truncate($x, 0) +x=abcd,y= + +mlr put $y=truncate($x, 1) +x=abcd,y=a + +mlr put $y=truncate($x, 2) +x=abcd,y=ab + +mlr put $y=truncate($x, 3) +x=abcd,y=abc + +mlr put $y=truncate($x, 4) +x=abcd,y=abcd + +mlr put $y=truncate($x, 5) +x=abcd,y=abcd + +mlr put $y=truncate($x, 0) +x=abcdefgh,y= + +mlr put $y=truncate($x, 1) +x=abcdefgh,y=a + +mlr put $y=truncate($x, 2) +x=abcdefgh,y=ab + +mlr put $y=truncate($x, 3) +x=abcdefgh,y=abc + +mlr put $y=truncate($x, 4) +x=abcdefgh,y=abcd + +mlr put $y=truncate($x, 5) +x=abcdefgh,y=abcde + + ================================================================ STATS1/STEP INT/FLOAT @@ -53430,7 +53470,7 @@ c 6 mlr termcvt --lf2crlf ---------------------------------------------------------------- input comments1-crlf.dkvp -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp # hello world 1 a=1,b=2,c=3 a=4,b=5,c=6 @@ -53446,7 +53486,7 @@ a 4 b 5 c 6 -mlr --skip-comments --idkvp --oxtab cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp +mlr --skip-comments --idkvp --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp a 1 b 2 c 3 @@ -53467,7 +53507,7 @@ a 4 b 5 c 6 -mlr --pass-comments --idkvp --oxtab cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp +mlr --pass-comments --idkvp --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp # hello world 1 a 1 b 2 @@ -53782,7 +53822,7 @@ mlr --pass-comments --inidx --oxtab cat ./reg_test/input/comments/comments3.nidx mlr termcvt --lf2crlf ---------------------------------------------------------------- input comments1-crlf.nidx -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx # hello world 1 aX1 bX2 cX3 aX4 bX5 cX6 @@ -53798,7 +53838,7 @@ mlr --skip-comments --inidx --oxtab cat 2 bX5 3 cX6 -mlr --skip-comments --inidx --oxtab cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx +mlr --skip-comments --inidx --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx 1 aX1 2 bX2 3 cX3 @@ -53819,7 +53859,7 @@ mlr --pass-comments --inidx --oxtab cat 2 bX5 3 cX6 -mlr --pass-comments --inidx --oxtab cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx +mlr --pass-comments --inidx --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx # hello world 1 1 aX1 2 bX2 @@ -53923,7 +53963,7 @@ a=4,b=5,c=6 mlr termcvt --lf2crlf ---------------------------------------------------------------- input comments1-crlf.json -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json # hello world 1 { "a":1, "b":2, "c":3 } { "a":4, "b":5, "c":6 } @@ -53934,7 +53974,7 @@ mlr --skip-comments --ijson --odkvp cat a=1,b=2,c=3 a=4,b=5,c=6 -mlr --skip-comments --ijson --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json +mlr --skip-comments --ijson --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json a=1,b=2,c=3 a=4,b=5,c=6 @@ -53945,7 +53985,7 @@ mlr --pass-comments --ijson --odkvp cat a=1,b=2,c=3 a=4,b=5,c=6 -mlr --pass-comments --ijson --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json +mlr --pass-comments --ijson --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json # hello world 1 a=1,b=2,c=3 a=4,b=5,c=6 @@ -54042,7 +54082,7 @@ x=2,z=5 mlr termcvt --lf2crlf ---------------------------------------------------------------- input comments1-crlf.xtab -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab x 1 #hello world3 z 3 @@ -54058,7 +54098,7 @@ mlr --skip-comments --ixtab --odkvp cat x=1,z=3 x=2,z=5 -mlr --skip-comments --ixtab --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab +mlr --skip-comments --ixtab --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab x=1,z=3 x=2,z=5 @@ -54070,7 +54110,7 @@ x=1,z=3 #hello world4 x=2,z=5 -mlr --pass-comments --ixtab --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab +mlr --pass-comments --ixtab --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab #hello world3 x=1,z=3 #hello world4 @@ -54152,7 +54192,7 @@ a=4,b=5,c=6 mlr termcvt --lf2crlf ---------------------------------------------------------------- input comments1-crlf.csv -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv # hello a,b,c 1,2,3 @@ -54164,7 +54204,7 @@ mlr --skip-comments --icsvlite --odkvp cat a=1,b=2,c=3 a=4,b=5,c=6 -mlr --skip-comments --icsvlite --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv +mlr --skip-comments --icsvlite --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv a=1,b=2,c=3 a=4,b=5,c=6 @@ -54175,7 +54215,7 @@ mlr --pass-comments --icsvlite --odkvp cat a=1,b=2,c=3 a=4,b=5,c=6 -mlr --pass-comments --icsvlite --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv +mlr --pass-comments --icsvlite --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv # hello a=1,b=2,c=3 a=4,b=5,c=6 @@ -54256,7 +54296,7 @@ a=4,b=5,c=6 mlr termcvt --lf2crlf ---------------------------------------------------------------- input comments1-crlf.csv -cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv +cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv # hello a,b,c 1,2,3 @@ -54268,7 +54308,7 @@ mlr --skip-comments --icsv --odkvp cat a=1,b=2,c=3 a=4,b=5,c=6 -mlr --skip-comments --icsv --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv +mlr --skip-comments --icsv --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv a=1,b=2,c=3 a=4,b=5,c=6 @@ -54279,7 +54319,7 @@ mlr --pass-comments --icsv --odkvp cat a=1,b=2,c=3 a=4,b=5,c=6 -mlr --pass-comments --icsv --odkvp cat /Users/johnkerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv +mlr --pass-comments --icsv --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv # hello a=1,b=2,c=3 a=4,b=5,c=6 diff --git a/c/reg_test/run b/c/reg_test/run index 3d504a80d..205f86eb6 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -1948,6 +1948,16 @@ announce UTF-8 STRLEN run_mlr --inidx --ifs comma --oxtab put '$s1=strlen($1);$s2=strlen($2);$s3=strlen($3);$s4=strlen($4)' $indir/utf8-align.dkvp +# ---------------------------------------------------------------- +announce TRUNCATE + +for n in 0 1 2 3 4 5; do + echo "x=abcd" | run_mlr put '$y=truncate($x, '$n')' +done +for n in 0 1 2 3 4 5; do + echo "x=abcdefgh" | run_mlr put '$y=truncate($x, '$n')' +done + # ---------------------------------------------------------------- announce STATS1/STEP INT/FLOAT diff --git a/docs/manpage.rst b/docs/manpage.rst index ad5b425c0..bca886b89 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -145,22 +145,23 @@ This is simply a copy of what you should see on running **man mlr** at a command FUNCTION LIST + + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub regextract regextract_or_else strlen sub ssub - substr tolower toupper capitalize lstrip rstrip strip collapse_whitespace - clean_whitespace system abs acos acosh asin asinh atan atan2 atanh cbrt ceil - cos cosh erf erfc exp expm1 floor invqnorm log log10 log1p logifit madd max - mexp min mmul msub pow qnorm round roundm sgn sin sinh sqrt tan tanh urand - urandrange urand32 urandint dhms2fsec dhms2sec fsec2dhms fsec2hms gmt2sec - localtime2sec hms2fsec hms2sec sec2dhms sec2gmt sec2gmt sec2gmtdate - sec2localtime sec2localtime sec2localdate sec2hms strftime strftime_local - strptime strptime_local systime is_absent is_bool is_boolean is_empty - is_empty_map is_float is_int is_map is_nonempty_map is_not_empty is_not_map - is_not_null is_null is_numeric is_present is_string asserting_absent - asserting_bool asserting_boolean asserting_empty asserting_empty_map - asserting_float asserting_int asserting_map asserting_nonempty_map - asserting_not_empty asserting_not_map asserting_not_null asserting_null - asserting_numeric asserting_present asserting_string boolean float fmtnum - hexfmt int string typeof depth haskey joink joinkv joinv leafcount length - mapdiff mapexcept mapselect mapsum splitkv splitkvx splitnv splitnvx + substr tolower toupper truncate capitalize lstrip rstrip strip + collapse_whitespace clean_whitespace system abs acos acosh asin asinh atan + atan2 atanh cbrt ceil cos cosh erf erfc exp expm1 floor invqnorm log log10 + log1p logifit madd max mexp min mmul msub pow qnorm round roundm sgn sin sinh + sqrt tan tanh urand urandrange urand32 urandint dhms2fsec dhms2sec fsec2dhms + fsec2hms gmt2sec localtime2sec hms2fsec hms2sec sec2dhms sec2gmt sec2gmt + sec2gmtdate sec2localtime sec2localtime sec2localdate sec2hms strftime + strftime_local strptime strptime_local systime is_absent is_bool is_boolean + is_empty is_empty_map is_float is_int is_map is_nonempty_map is_not_empty + is_not_map is_not_null is_null is_numeric is_present is_string + asserting_absent asserting_bool asserting_boolean asserting_empty + asserting_empty_map asserting_float asserting_int asserting_map + asserting_nonempty_map asserting_not_empty asserting_not_map + asserting_not_null asserting_null asserting_numeric asserting_present + asserting_string boolean float fmtnum hexfmt int string typeof depth haskey + joink joinkv joinv leafcount length mapdiff mapexcept mapselect mapsum splitkv + splitkvx splitnv splitnvx Please use "mlr --help-function {function name}" for function-specific help. @@ -1568,6 +1569,9 @@ This is simply a copy of what you should see on running **man mlr** at a command toupper (class=string #args=1): Convert string to uppercase. + truncate + (class=string #args=2): Truncates string first argument to max length of int second argument. + capitalize (class=string #args=1): Convert string's first character to uppercase. @@ -2366,4 +2370,4 @@ This is simply a copy of what you should see on running **man mlr** at a command - 2020-10-07 MILLER(1) + 2020-10-26 MILLER(1) diff --git a/docs/manpage.txt b/docs/manpage.txt index ca6878635..cce47109b 100644 --- a/docs/manpage.txt +++ b/docs/manpage.txt @@ -135,22 +135,23 @@ OPTIONS FUNCTION LIST + + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub regextract regextract_or_else strlen sub ssub - substr tolower toupper capitalize lstrip rstrip strip collapse_whitespace - clean_whitespace system abs acos acosh asin asinh atan atan2 atanh cbrt ceil - cos cosh erf erfc exp expm1 floor invqnorm log log10 log1p logifit madd max - mexp min mmul msub pow qnorm round roundm sgn sin sinh sqrt tan tanh urand - urandrange urand32 urandint dhms2fsec dhms2sec fsec2dhms fsec2hms gmt2sec - localtime2sec hms2fsec hms2sec sec2dhms sec2gmt sec2gmt sec2gmtdate - sec2localtime sec2localtime sec2localdate sec2hms strftime strftime_local - strptime strptime_local systime is_absent is_bool is_boolean is_empty - is_empty_map is_float is_int is_map is_nonempty_map is_not_empty is_not_map - is_not_null is_null is_numeric is_present is_string asserting_absent - asserting_bool asserting_boolean asserting_empty asserting_empty_map - asserting_float asserting_int asserting_map asserting_nonempty_map - asserting_not_empty asserting_not_map asserting_not_null asserting_null - asserting_numeric asserting_present asserting_string boolean float fmtnum - hexfmt int string typeof depth haskey joink joinkv joinv leafcount length - mapdiff mapexcept mapselect mapsum splitkv splitkvx splitnv splitnvx + substr tolower toupper truncate capitalize lstrip rstrip strip + collapse_whitespace clean_whitespace system abs acos acosh asin asinh atan + atan2 atanh cbrt ceil cos cosh erf erfc exp expm1 floor invqnorm log log10 + log1p logifit madd max mexp min mmul msub pow qnorm round roundm sgn sin sinh + sqrt tan tanh urand urandrange urand32 urandint dhms2fsec dhms2sec fsec2dhms + fsec2hms gmt2sec localtime2sec hms2fsec hms2sec sec2dhms sec2gmt sec2gmt + sec2gmtdate sec2localtime sec2localtime sec2localdate sec2hms strftime + strftime_local strptime strptime_local systime is_absent is_bool is_boolean + is_empty is_empty_map is_float is_int is_map is_nonempty_map is_not_empty + is_not_map is_not_null is_null is_numeric is_present is_string + asserting_absent asserting_bool asserting_boolean asserting_empty + asserting_empty_map asserting_float asserting_int asserting_map + asserting_nonempty_map asserting_not_empty asserting_not_map + asserting_not_null asserting_null asserting_numeric asserting_present + asserting_string boolean float fmtnum hexfmt int string typeof depth haskey + joink joinkv joinv leafcount length mapdiff mapexcept mapselect mapsum splitkv + splitkvx splitnv splitnvx Please use "mlr --help-function {function name}" for function-specific help. @@ -1558,6 +1559,9 @@ FUNCTIONS FOR FILTER/PUT toupper (class=string #args=1): Convert string to uppercase. + truncate + (class=string #args=2): Truncates string first argument to max length of int second argument. + capitalize (class=string #args=1): Convert string's first character to uppercase. @@ -2356,4 +2360,4 @@ SEE ALSO - 2020-10-07 MILLER(1) + 2020-10-26 MILLER(1) diff --git a/docs/mlr.1 b/docs/mlr.1 index 40910a17f..a2bbce476 100644 --- a/docs/mlr.1 +++ b/docs/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2020-10-07 +.\" Date: 2020-10-26 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2020-10-07" "\ \&" "\ \&" +.TH "MILLER" "1" "2020-10-26" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -184,22 +184,23 @@ output separator to the given value. .nf + + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub regextract regextract_or_else strlen sub ssub - substr tolower toupper capitalize lstrip rstrip strip collapse_whitespace - clean_whitespace system abs acos acosh asin asinh atan atan2 atanh cbrt ceil - cos cosh erf erfc exp expm1 floor invqnorm log log10 log1p logifit madd max - mexp min mmul msub pow qnorm round roundm sgn sin sinh sqrt tan tanh urand - urandrange urand32 urandint dhms2fsec dhms2sec fsec2dhms fsec2hms gmt2sec - localtime2sec hms2fsec hms2sec sec2dhms sec2gmt sec2gmt sec2gmtdate - sec2localtime sec2localtime sec2localdate sec2hms strftime strftime_local - strptime strptime_local systime is_absent is_bool is_boolean is_empty - is_empty_map is_float is_int is_map is_nonempty_map is_not_empty is_not_map - is_not_null is_null is_numeric is_present is_string asserting_absent - asserting_bool asserting_boolean asserting_empty asserting_empty_map - asserting_float asserting_int asserting_map asserting_nonempty_map - asserting_not_empty asserting_not_map asserting_not_null asserting_null - asserting_numeric asserting_present asserting_string boolean float fmtnum - hexfmt int string typeof depth haskey joink joinkv joinv leafcount length - mapdiff mapexcept mapselect mapsum splitkv splitkvx splitnv splitnvx + substr tolower toupper truncate capitalize lstrip rstrip strip + collapse_whitespace clean_whitespace system abs acos acosh asin asinh atan + atan2 atanh cbrt ceil cos cosh erf erfc exp expm1 floor invqnorm log log10 + log1p logifit madd max mexp min mmul msub pow qnorm round roundm sgn sin sinh + sqrt tan tanh urand urandrange urand32 urandint dhms2fsec dhms2sec fsec2dhms + fsec2hms gmt2sec localtime2sec hms2fsec hms2sec sec2dhms sec2gmt sec2gmt + sec2gmtdate sec2localtime sec2localtime sec2localdate sec2hms strftime + strftime_local strptime strptime_local systime is_absent is_bool is_boolean + is_empty is_empty_map is_float is_int is_map is_nonempty_map is_not_empty + is_not_map is_not_null is_null is_numeric is_present is_string + asserting_absent asserting_bool asserting_boolean asserting_empty + asserting_empty_map asserting_float asserting_int asserting_map + asserting_nonempty_map asserting_not_empty asserting_not_map + asserting_not_null asserting_null asserting_numeric asserting_present + asserting_string boolean float fmtnum hexfmt int string typeof depth haskey + joink joinkv joinv leafcount length mapdiff mapexcept mapselect mapsum splitkv + splitkvx splitnv splitnvx Please use "mlr --help-function {function name}" for function-specific help. .fi @@ -2243,6 +2244,15 @@ inclusive. Negative indices -len .. -1 alias to 0 .. len-1. .fi .if n \{\ .RE +.SS "truncate" +.if n \{\ +.RS 0 +.\} +.nf +(class=string #args=2): Truncates string first argument to max length of int second argument. +.fi +.if n \{\ +.RE .SS "capitalize" .if n \{\ .RS 0 diff --git a/docs/reference-dsl.rst b/docs/reference-dsl.rst index 698b2d2bb..72e1a32fd 100644 --- a/docs/reference-dsl.rst +++ b/docs/reference-dsl.rst @@ -2620,6 +2620,8 @@ Built-in functions for filter and put, summary +----------------------------+------------+----------+ | ``toupper`` | string | 1 | +----------------------------+------------+----------+ +| ``truncate`` | string | 2 | ++----------------------------+------------+----------+ | ``capitalize`` | string | 1 | +----------------------------+------------+----------+ | ``lstrip`` | string | 1 | @@ -4716,6 +4718,17 @@ toupper +.. _reference-dsl-truncate: + +truncate +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + truncate (class=string #args=2): Truncates string first argument to max length of int second argument. + + + .. _reference-dsl-typeof: typeof diff --git a/docs/reference.rst b/docs/reference.rst index 6952c6700..e2e8270b3 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -795,7 +795,7 @@ Examples: Functions for the filter and put verbs: + + - - * / // .+ .+ .- .- .* ./ .// % ** | ^ & ~ << >> bitcount == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub regextract regextract_or_else strlen sub - ssub substr tolower toupper capitalize lstrip rstrip strip + ssub substr tolower toupper truncate capitalize lstrip rstrip strip collapse_whitespace clean_whitespace system abs acos acosh asin asinh atan atan2 atanh cbrt ceil cos cosh erf erfc exp expm1 floor invqnorm log log10 log1p logifit madd max mexp min mmul msub pow qnorm round roundm sgn sin diff --git a/go/src/miller/dsl/cst/lvalues.go b/go/src/miller/dsl/cst/lvalues.go index 4afc505a2..a900955d3 100644 --- a/go/src/miller/dsl/cst/lvalues.go +++ b/go/src/miller/dsl/cst/lvalues.go @@ -102,19 +102,22 @@ func (this *DirectFieldValueLvalueNode) AssignIndexed( func (this *DirectFieldValueLvalueNode) Unset( state *State, ) { - lib.InternalCodingErrorIf(!this.lhsFieldName.IsString()) - name := this.lhsFieldName.String() - state.Inrec.Remove(&name) + this.UnsetIndexed(nil, state) } func (this *DirectFieldValueLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO: indexed - //lib.InternalCodingErrorIf(!this.lhsFieldName.IsString()) - //name := this.lhsFieldName.String() - //state.Inrec.Remove(&name) + if indices == nil { + lib.InternalCodingErrorIf(!this.lhsFieldName.IsString()) + name := this.lhsFieldName.String() + state.Inrec.Remove(&name) + } else { + state.Inrec.UnsetIndexed( + append([]*types.Mlrval{this.lhsFieldName}, indices...), + ) + } } // ---------------------------------------------------------------- @@ -177,16 +180,22 @@ func (this *IndirectFieldValueLvalueNode) AssignIndexed( func (this *IndirectFieldValueLvalueNode) Unset( state *State, ) { - lhsFieldName := this.lhsFieldNameExpression.Evaluate(state) - name := lhsFieldName.String() - state.Inrec.Remove(&name) + this.UnsetIndexed(nil, state) } func (this *IndirectFieldValueLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO + lhsFieldName := this.lhsFieldNameExpression.Evaluate(state) + if indices == nil { + name := lhsFieldName.String() + state.Inrec.Remove(&name) + } else { + state.Inrec.UnsetIndexed( + append([]*types.Mlrval{&lhsFieldName}, indices...), + ) + } } // ---------------------------------------------------------------- @@ -231,14 +240,18 @@ func (this *FullSrecLvalueNode) AssignIndexed( func (this *FullSrecLvalueNode) Unset( state *State, ) { - state.Inrec.Clear() + this.UnsetIndexed(nil, state) } func (this *FullSrecLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO + if indices == nil { + state.Inrec.Clear() + } else { + state.Inrec.UnsetIndexed(indices) + } } // ---------------------------------------------------------------- @@ -290,15 +303,22 @@ func (this *DirectOosvarValueLvalueNode) AssignIndexed( func (this *DirectOosvarValueLvalueNode) Unset( state *State, ) { - name := this.lhsOosvarName.String() - state.Oosvars.Remove(&name) + this.UnsetIndexed(nil, state) } func (this *DirectOosvarValueLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO + if indices == nil { + name := this.lhsOosvarName.String() + state.Oosvars.Remove(&name) + } else { + // TODO + //state.Inrec.UnsetIndexed( + //append([]*types.Mlrval{&lhsFieldName}, indices...), + //) + } } // ---------------------------------------------------------------- @@ -364,8 +384,7 @@ func (this *IndirectOosvarValueLvalueNode) AssignIndexed( func (this *IndirectOosvarValueLvalueNode) Unset( state *State, ) { - name := this.lhsOosvarNameExpression.Evaluate(state).String() - state.Oosvars.Remove(&name) + this.UnsetIndexed(nil, state) } func (this *IndirectOosvarValueLvalueNode) UnsetIndexed( @@ -373,6 +392,18 @@ func (this *IndirectOosvarValueLvalueNode) UnsetIndexed( state *State, ) { // TODO + //name := this.lhsOosvarNameExpression.Evaluate(state).String() + //state.Oosvars.Remove(&name) + +// lhsFieldName := this.lhsFieldNameExpression.Evaluate(state) +// if indices == nil { +// name := lhsFieldName.String() +// state.Inrec.Remove(&name) +// } else { +// state.Inrec.UnsetIndexed( +// append([]*types.Mlrval{&lhsFieldName}, indices...), +// ) +// } } // ---------------------------------------------------------------- @@ -417,14 +448,21 @@ func (this *FullOosvarLvalueNode) AssignIndexed( func (this *FullOosvarLvalueNode) Unset( state *State, ) { - state.Oosvars.Clear() + this.UnsetIndexed(nil, state) } func (this *FullOosvarLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO + if indices == nil { + state.Oosvars.Clear() + } else { + // TODO + //state.Inrec.UnsetIndexed( + //append([]*types.Mlrval{&lhsFieldName}, indices...), + //) + } } // ---------------------------------------------------------------- @@ -493,14 +531,21 @@ func (this *LocalVariableLvalueNode) AssignIndexed( func (this *LocalVariableLvalueNode) Unset( state *State, ) { - state.stack.UnsetVariable(this.typeGatedMlrvalName.Name) + this.UnsetIndexed(nil, state) } func (this *LocalVariableLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO + if indices == nil { + state.stack.UnsetVariable(this.typeGatedMlrvalName.Name) + } else { + // TODO + //state.Inrec.UnsetIndexed( + //append([]*types.Mlrval{&lhsFieldName}, indices...), + //) + } } // ---------------------------------------------------------------- @@ -603,12 +648,20 @@ func (this *IndexedLvalueNode) AssignIndexed( func (this *IndexedLvalueNode) Unset( state *State, ) { - // TODO + indices := make([]*types.Mlrval, len(this.indexEvaluables)) + + for i, indexEvaluable := range this.indexEvaluables { + index := indexEvaluable.Evaluate(state) + indices[i] = &index + } + + this.baseLvalue.UnsetIndexed(indices, state) } func (this *IndexedLvalueNode) UnsetIndexed( indices []*types.Mlrval, state *State, ) { - // TODO + // We are the delegator, not the delegatee + lib.InternalCodingErrorIf(true) } diff --git a/go/src/miller/types/mlrmap_accessors.go b/go/src/miller/types/mlrmap_accessors.go index c9a918ef2..a7de7917f 100644 --- a/go/src/miller/types/mlrmap_accessors.go +++ b/go/src/miller/types/mlrmap_accessors.go @@ -166,6 +166,13 @@ func (this *Mlrmap) GetWithMlrvalIndex(index *Mlrval) (*Mlrval, error) { } } +func (this *Mlrmap) RemoveWithPositionalIndex(position int64) { + mapEntry := this.findEntryByPositionalIndex(position) + if mapEntry != nil { + this.unlink(mapEntry) + } +} + // ---------------------------------------------------------------- func (this *Mlrmap) Clear() { this.FieldCount = 0 diff --git a/go/src/miller/types/mlrval_collections.go b/go/src/miller/types/mlrval_collections.go index 753097e40..505c42351 100644 --- a/go/src/miller/types/mlrval_collections.go +++ b/go/src/miller/types/mlrval_collections.go @@ -430,8 +430,7 @@ func unsetIndexedOnMap(baseMap *Mlrmap, indices []*Mlrval) error { baseMap.Remove(&baseIndex.printrep) return nil } else if baseIndex.mvtype == MT_INT { - // TODO: - // * RemoveByPositionalIndex + baseMap.RemoveWithPositionalIndex(baseIndex.intval) return nil } else { return errors.New( diff --git a/go/todo.txt b/go/todo.txt index 6e9fe6462..0707ed893 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -23,6 +23,7 @@ TOP OF LIST: * UDS ! parse errors need some context somehow +? ;; is parse error; also {...}; * more copy-on-retain for concurrent pointer-mods ! o make a thorough audit, and warn everywhere @@ -194,3 +195,4 @@ NITS/NON-IMMEDIATE: * good example of wording for why/when to make a breaking release: https://webpack.js.org/blog/2020-10-10-webpack-5-release/ * unset, unassign, remove -- too many different names. also assign/put ... maybe stick w/ 2? +* make truncate() UTF8-friendly