From cce2a4f700158fe3809dd76cf5084609d9c50901 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 28 Feb 2021 10:14:32 -0500 Subject: [PATCH] Port fraction verb from C to Go --- go/cases-to-do.txt | 1 - go/gg | 35 +- go/reg-test/cases/case-c-verb-fraction.sh | 15 - go/reg-test/cases/case-verb-fraction.sh | 15 + .../expected/case-c-verb-fraction.sh.out | 145 --- go/reg-test/expected/case-cli-help.sh.out | 115 ++- .../expected/case-verb-fraction.sh.out | 909 ++++++++++++++++++ go/src/cli/mlrcli_transformers.go | 7 +- go/src/transformers/fraction.go | 278 ++++++ go/src/types/mlrval_accessors.go | 4 + go/src/types/mlrval_functions_booleans.go | 10 +- go/todo.txt | 1 + 12 files changed, 1288 insertions(+), 247 deletions(-) delete mode 100644 go/reg-test/cases/case-c-verb-fraction.sh create mode 100644 go/reg-test/cases/case-verb-fraction.sh delete mode 100644 go/reg-test/expected/case-c-verb-fraction.sh.out create mode 100644 go/reg-test/expected/case-verb-fraction.sh.out create mode 100644 go/src/transformers/fraction.go diff --git a/go/cases-to-do.txt b/go/cases-to-do.txt index a06c32294..b0b92f1cd 100644 --- a/go/cases-to-do.txt +++ b/go/cases-to-do.txt @@ -92,7 +92,6 @@ rrv ./reg-test/cases/case-c-format-values.sh rrv ./reg-test/cases/case-c-fraction.sh rrv ./reg-test/cases/case-c-having-fields.sh rrv ./reg-test/cases/case-c-head-early-out.sh -rrv ./reg-test/cases/case-c-histogram.sh rrv ./reg-test/cases/case-c-in-place-processing.sh rrv ./reg-test/cases/case-c-int-float-stats1-step1.sh rrv ./reg-test/cases/case-c-join-prepipe.sh diff --git a/go/gg b/go/gg index 1e5e18b0c..93db9424f 100755 --- a/go/gg +++ b/go/gg @@ -52,39 +52,6 @@ mention() { # ================================================================ -run_mlr --from r put -q ' - a = 1; - b = 2; - print; - print "OUT1a ". a; - print "OUT1b ". b; - if (NR == 1) { - a = 3; - c = 4; - print "IN1a ". a; - print "IN1c ". c; - } else { - b = 5; - d = 6; - print "IN2a ". b; - print "IN2c ". d; - } - print "OUT2a ". a; - print "OUT2a ". b; -' - -# OUT1a 1 -# OUT1b 2 -# IN1a 3 -# IN1c 4 -# OUT2a 3 -# OUT2a 2 -# -# OUT1a 1 -# OUT1b 2 -# IN2a 5 -# IN2c 6 -# OUT2a 1 -# OUT2a 5 +run_mlr --oxtab fraction -f x,y -g a -p -c ./reg-test/input/abixy-het # ================================================================ diff --git a/go/reg-test/cases/case-c-verb-fraction.sh b/go/reg-test/cases/case-c-verb-fraction.sh deleted file mode 100644 index 686d4bc1a..000000000 --- a/go/reg-test/cases/case-c-verb-fraction.sh +++ /dev/null @@ -1,15 +0,0 @@ -run_mlr fraction -f x,y $indir/abixy-het -run_mlr fraction -f x,y -g a $indir/abixy-het -run_mlr fraction -f x,y -g a,b $indir/abixy-het - -run_mlr fraction -f x,y -p $indir/abixy-het -run_mlr fraction -f x,y -g a -p $indir/abixy-het -run_mlr fraction -f x,y -g a,b -p $indir/abixy-het - -run_mlr fraction -f x,y -c $indir/abixy-het -run_mlr fraction -f x,y -g a -c $indir/abixy-het -run_mlr fraction -f x,y -g a,b -c $indir/abixy-het - -run_mlr fraction -f x,y -p -c $indir/abixy-het -run_mlr fraction -f x,y -g a -p -c $indir/abixy-het -run_mlr fraction -f x,y -g a,b -p -c $indir/abixy-het diff --git a/go/reg-test/cases/case-verb-fraction.sh b/go/reg-test/cases/case-verb-fraction.sh new file mode 100644 index 000000000..12274bc43 --- /dev/null +++ b/go/reg-test/cases/case-verb-fraction.sh @@ -0,0 +1,15 @@ +run_mlr --oxtab fraction -f x,y $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a,b $indir/abixy-het + +run_mlr --oxtab fraction -f x,y -p $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a -p $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a,b -p $indir/abixy-het + +run_mlr --oxtab fraction -f x,y -c $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a -c $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a,b -c $indir/abixy-het + +run_mlr --oxtab fraction -f x,y -p -c $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a -p -c $indir/abixy-het +run_mlr --oxtab fraction -f x,y -g a,b -p -c $indir/abixy-het diff --git a/go/reg-test/expected/case-c-verb-fraction.sh.out b/go/reg-test/expected/case-c-verb-fraction.sh.out deleted file mode 100644 index e3367ae59..000000000 --- a/go/reg-test/expected/case-c-verb-fraction.sh.out +++ /dev/null @@ -1,145 +0,0 @@ - -mlr fraction -f x,y ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_fraction=0.087507,y_fraction=0.146286 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_fraction=0.191441,y_fraction=0.105095 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,x_fraction=0.051628,y_fraction=0.068095 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_fraction=0.096240,y_fraction=0.027009 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_fraction=0.173825 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_fraction=0.133012,y_fraction=0.099272 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_fraction=0.154374,y_fraction=0.037816 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_fraction=0.151035 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059,x_fraction=0.007934,y_fraction=0.150865 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_fraction=0.126830,y_fraction=0.191737 - -mlr fraction -f x,y -g a ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_fraction=0.408269,y_fraction=0.432770 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_fraction=0.433070,y_fraction=0.618498 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_fraction=0.217711,y_fraction=0.158949 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_fraction=1.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_fraction=0.468273,y_fraction=1.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_fraction=0.349219,y_fraction=0.222553 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_fraction=0.531727 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_fraction=0.591731,y_fraction=0.567230 - -mlr fraction -f x,y -g a,b ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_fraction=1.000000,y_fraction=1.000000 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_fraction=1.000000,y_fraction=1.000000 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_fraction=1.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_fraction=1.000000,y_fraction=1.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_fraction=1.000000,y_fraction=1.000000 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_fraction=1.000000 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_fraction=1.000000,y_fraction=1.000000 - -mlr fraction -f x,y -p ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_percent=8.750687,y_percent=14.628624 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_percent=19.144058,y_percent=10.509524 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,x_percent=5.162832,y_percent=6.809459 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_percent=9.623995,y_percent=2.700865 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_percent=17.382482 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_percent=13.301173,y_percent=9.927243 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_percent=15.437378,y_percent=3.781628 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_percent=15.103539 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059,x_percent=0.793385,y_percent=15.086479 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_percent=12.682952,y_percent=19.173694 - -mlr fraction -f x,y -g a -p ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_percent=40.826884,y_percent=43.276985 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_percent=43.307027,y_percent=61.849773 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_percent=21.771069,y_percent=15.894907 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_percent=100.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_percent=46.827347,y_percent=100.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_percent=34.921904,y_percent=22.255319 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_percent=53.172653 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_percent=59.173116,y_percent=56.723015 - -mlr fraction -f x,y -g a,b -p ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_percent=100.000000,y_percent=100.000000 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_percent=100.000000,y_percent=100.000000 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_percent=100.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_percent=100.000000,y_percent=100.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_percent=100.000000,y_percent=100.000000 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_percent=100.000000 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_percent=100.000000,y_percent=100.000000 - -mlr fraction -f x,y -c ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_cumulative_fraction=0.087507,y_cumulative_fraction=0.146286 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_cumulative_fraction=0.278947,y_cumulative_fraction=0.251381 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,x_cumulative_fraction=0.330576,y_cumulative_fraction=0.319476 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_cumulative_fraction=0.426816,y_cumulative_fraction=0.346485 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_cumulative_fraction=0.520310 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_cumulative_fraction=0.559827,y_cumulative_fraction=0.619582 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_cumulative_fraction=0.714201,y_cumulative_fraction=0.657398 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_cumulative_fraction=0.865237 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059,x_cumulative_fraction=0.873170,y_cumulative_fraction=0.808263 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 - -mlr fraction -f x,y -g a -c ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_cumulative_fraction=0.408269,y_cumulative_fraction=0.432770 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_cumulative_fraction=0.433070,y_cumulative_fraction=0.618498 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_cumulative_fraction=0.650781,y_cumulative_fraction=0.777447 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_cumulative_fraction=1.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_cumulative_fraction=0.468273,y_cumulative_fraction=1.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_cumulative_fraction=1.000000 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 - -mlr fraction -f x,y -g a,b -c ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_cumulative_fraction=1.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_cumulative_fraction=1.000000 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_cumulative_fraction=1.000000,y_cumulative_fraction=1.000000 - -mlr fraction -f x,y -p -c ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_cumulative_percent=8.750687,y_cumulative_percent=14.628624 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_cumulative_percent=27.894745,y_cumulative_percent=25.138148 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776,x_cumulative_percent=33.057577,y_cumulative_percent=31.947608 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_cumulative_percent=42.681572,y_cumulative_percent=34.648473 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_cumulative_percent=52.030955 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_cumulative_percent=55.982746,y_cumulative_percent=61.958199 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_cumulative_percent=71.420124,y_cumulative_percent=65.739826 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_cumulative_percent=86.523663 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059,x_cumulative_percent=87.317048,y_cumulative_percent=80.826306 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 - -mlr fraction -f x,y -g a -p -c ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_cumulative_percent=40.826884,y_cumulative_percent=43.276985 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_cumulative_percent=43.307027,y_cumulative_percent=61.849773 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463,x_cumulative_percent=65.078096,y_cumulative_percent=77.744681 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_cumulative_percent=100.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_cumulative_percent=46.827347,y_cumulative_percent=100.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_cumulative_percent=100.000000 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 - -mlr fraction -f x,y -g a,b -p -c ./reg-test/input/abixy-het -a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 -a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 -aaa=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776 -a=eks,bbb=wye,i=4,x=0.38139939387114097,y=0.13418874328430463 -a=wye,b=pan,i=5,xxx=0.5732889198020006,y=0.8636244699032729,y_cumulative_percent=100.000000 -a=zee,b=pan,i=6,x=0.5271261600918548,y=0.49322128674835697,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 -a=eks,b=zee,iii=7,x=0.6117840605678454,y=0.1878849191181694,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 -a=zee,b=wye,i=8,x=0.5985540091064224,yyy=0.976181385699006,x_cumulative_percent=100.000000 -aaa=hat,bbb=wye,i=9,x=0.03144187646093577,y=0.7495507603507059 -a=pan,b=wye,i=10,x=0.5026260055412137,y=0.9526183602969864,x_cumulative_percent=100.000000,y_cumulative_percent=100.000000 - diff --git a/go/reg-test/expected/case-cli-help.sh.out b/go/reg-test/expected/case-cli-help.sh.out index 59d742668..cbce6e990 100644 --- a/go/reg-test/expected/case-cli-help.sh.out +++ b/go/reg-test/expected/case-cli-help.sh.out @@ -149,6 +149,27 @@ Options: -s Separator, defaulting to mlr --oflatsep value. -h|--help Show this message. ================================================================ +Usage: mlr fraction [options] +For each record's value in specified fields, computes the ratio of that +value to the sum of values in that field over all input records. +E.g. with input records x=1 x=2 x=3 and x=4, emits output records +x=1,x_fraction=0.1 x=2,x_fraction=0.2 x=3,x_fraction=0.3 and x=4,x_fraction=0.4 + +Note: this is internally a two-pass algorithm: on the first pass it retains +input records and accumulates sums; on the second pass it computes quotients +and emits output records. This means it produces no output until all input is read. + +Options: +-f {a,b,c} Field name(s) for fraction calculation +-g {d,e,f} Optional group-by-field name(s) for fraction counts +-p Produce percents [0..100], not fractions [0..1]. Output field names + end with "_percent" rather than "_fraction" +-c Produce cumulative distributions, i.e. running sums: each output + value folds in the sum of the previous for the specified group + E.g. with input records x=1 x=2 x=3 and x=4, emits output records + x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3 + x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0 +================================================================ Usage: mlr gap [options] Emits an empty record every n records, or when certain values change. Options: @@ -218,6 +239,20 @@ Usage: mlr histogram [options] -o {prefix} Prefix for output field name. Default: no prefix. -h|--help Show this message. ================================================================ +Usage: mlr json-parse [options] +Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3]. +Options: +-f {...} Comma-separated list of field names to json-parse (default all). +-h|--help Show this message. +================================================================ +Usage: mlr json-stringify [options] +Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]". +Options: +-f {...} Comma-separated list of field names to json-parse (default all). +--jvstack Produce multi-line JSON output. +--no-jvstack Produce single-line JSON output per record (default). +-h|--help Show this message. +================================================================ Usage: mlr join [options] Joins records from specified left file name with records from all file names at the end of the Miller argument list. @@ -261,20 +296,6 @@ Please use "mlr --usage-separator-options" for information on specifying separat Please see https://miller.readthedocs.io/en/latest/reference-verbs.html#join for more information including examples. ================================================================ -Usage: mlr json-parse [options] -Tries to convert string field values to parsed JSON, e.g. "[1,2,3]" -> [1,2,3]. -Options: --f {...} Comma-separated list of field names to json-parse (default all). --h|--help Show this message. -================================================================ -Usage: mlr json-stringify [options] -Produces string field values from field-value data, e.g. [1,2,3] -> "[1,2,3]". -Options: --f {...} Comma-separated list of field names to json-parse (default all). ---jvstack Produce multi-line JSON output. ---no-jvstack Produce single-line JSON output per record (default). --h|--help Show this message. -================================================================ Usage: mlr label [options] {new1,new2,new3,...} Given n comma-separated names, renames the first n fields of each record to have the respective name. (Fields past the nth are left with their original @@ -330,32 +351,6 @@ semicolons to separate expressions.) -h|--help Show this message. ================================================================ -Usage: mlr repeat [options] -Copies input records to output records multiple times. -Options must be exactly one of the following: --n {repeat count} Repeat each input record this many times. --f {field name} Same, but take the repeat count from the specified - field name of each input record. --h|--help Show this message. -Example: - echo x=0 | mlr repeat -n 4 then put '$x=urand()' -produces: - x=0.488189 - x=0.484973 - x=0.704983 - x=0.147311 -Example: - echo a=1,b=2,c=3 | mlr repeat -f b -produces: - a=1,b=2,c=3 - a=1,b=2,c=3 -Example: - echo a=1,b=2,c=3 | mlr repeat -f c -produces: - a=1,b=2,c=3 - a=1,b=2,c=3 - a=1,b=2,c=3 -================================================================ Usage: mlr regularize [options] Outputs records sorted lexically ascending by keys.Options: -h|--help Show this message. @@ -398,6 +393,32 @@ Examples: mlr reorder -f a,b sends input record "d=4,b=2,a=1,c=3" to "a=1,b=2,d=4,c=3". mlr reorder -e -f a,b sends input record "d=4,b=2,a=1,c=3" to "d=4,c=3,a=1,b=2". ================================================================ +Usage: mlr repeat [options] +Copies input records to output records multiple times. +Options must be exactly one of the following: +-n {repeat count} Repeat each input record this many times. +-f {field name} Same, but take the repeat count from the specified + field name of each input record. +-h|--help Show this message. +Example: + echo x=0 | mlr repeat -n 4 then put '$x=urand()' +produces: + x=0.488189 + x=0.484973 + x=0.704983 + x=0.147311 +Example: + echo a=1,b=2,c=3 | mlr repeat -f b +produces: + a=1,b=2,c=3 + a=1,b=2,c=3 +Example: + echo a=1,b=2,c=3 | mlr repeat -f c +produces: + a=1,b=2,c=3 + a=1,b=2,c=3 + a=1,b=2,c=3 +================================================================ Usage: mlr sample [options] Reservoir sampling (subsampling without replacement), optionally by category. See also mlr bootstrap and mlr shuffle. @@ -565,6 +586,13 @@ Prints records in reverse order from the order in which they were encountered. Options: -h|--help Show this message. ================================================================ +Usage: mlr tail [options] +Passes through the last n records, optionally by category. +Options: +-g {a,b,c} Optional group-by-field names for head counts, e.g. a,b,c. +-n {n} Head-count to print. Default 10. +-h|--help Show this message. +================================================================ Usage: mlr tee [options] {filename} Options: -a Append to existing file, if any, rather than overwriting. @@ -574,13 +602,6 @@ Any of the output-format command-line flags (see mlr -h). Example: using the input is CSV, the output is pretty-print tabular, but the tee-file output is written in JSON format. --h|--help Show this message. -================================================================ -Usage: mlr tail [options] -Passes through the last n records, optionally by category. -Options: --g {a,b,c} Optional group-by-field names for head counts, e.g. a,b,c. --n {n} Head-count to print. Default 10. -h|--help Show this message. ================================================================ Usage: mlr unflatten [options] diff --git a/go/reg-test/expected/case-verb-fraction.sh.out b/go/reg-test/expected/case-verb-fraction.sh.out new file mode 100644 index 000000000..b3b68f0ee --- /dev/null +++ b/go/reg-test/expected/case-verb-fraction.sh.out @@ -0,0 +1,909 @@ + +mlr --oxtab fraction -f x,y ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_fraction 0.08750686695876585 +y_fraction 0.1462862415661702 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_fraction 0.19144058107497372 +y_fraction 0.10509524257979809 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 +x_fraction 0.051628324937518816 +y_fraction 0.06809459357827034 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_fraction 0.09623994961373192 +y_fraction 0.027008653820448102 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_fraction 0.17382482142385294 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_fraction 0.13301173494905932 +y_fraction 0.09927243272886704 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_fraction 0.1543737827318787 +y_fraction 0.037816277389185964 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_fraction 0.15103539387627238 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 +x_fraction 0.00793384744440362 +y_fraction 0.15086479321349863 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_fraction 0.1268295184133955 +y_fraction 0.19173694369990862 + +mlr --oxtab fraction -f x,y -g a ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_fraction 0.4082688378215578 +y_fraction 0.4327698452134853 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_fraction 0.4330702704688488 +y_fraction 0.6184977347659367 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_fraction 0.2177106900485466 +y_fraction 0.15894907130872915 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_fraction 1 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_fraction 0.46827347102266204 +y_fraction 1 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_fraction 0.34921903948260463 +y_fraction 0.22255319392533407 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_fraction 0.531726528977338 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_fraction 0.5917311621784421 +y_fraction 0.5672301547865147 + +mlr --oxtab fraction -f x,y -g a,b ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_fraction 1 +y_fraction 1 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_fraction 1 +y_fraction 1 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_fraction 1 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_fraction 1 +y_fraction 1 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_fraction 1 +y_fraction 1 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_fraction 1 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_fraction 1 +y_fraction 1 + +mlr --oxtab fraction -f x,y -p ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_percent 8.750686695876585 +y_percent 14.62862415661702 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_percent 19.144058107497372 +y_percent 10.509524257979809 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 +x_percent 5.162832493751882 +y_percent 6.809459357827034 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_percent 9.623994961373192 +y_percent 2.7008653820448103 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_percent 17.382482142385296 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_percent 13.301173494905932 +y_percent 9.927243272886704 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_percent 15.437378273187871 +y_percent 3.7816277389185964 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_percent 15.103539387627238 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 +x_percent 0.7933847444403621 +y_percent 15.086479321349863 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_percent 12.682951841339548 +y_percent 19.173694369990862 + +mlr --oxtab fraction -f x,y -g a -p ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_percent 40.82688378215578 +y_percent 43.27698452134853 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_percent 43.307027046884876 +y_percent 61.849773476593676 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_percent 21.77106900485466 +y_percent 15.894907130872916 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_percent 100 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_percent 46.8273471022662 +y_percent 100 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_percent 34.921903948260464 +y_percent 22.25531939253341 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_percent 53.1726528977338 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_percent 59.17311621784421 +y_percent 56.723015478651476 + +mlr --oxtab fraction -f x,y -g a,b -p ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_percent 100 +y_percent 100 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_percent 100 +y_percent 100 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_percent 100 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_percent 100 +y_percent 100 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_percent 100 +y_percent 100 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_percent 100 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_percent 100 +y_percent 100 + +mlr --oxtab fraction -f x,y -c ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_cumulative_fraction 0.08750686695876585 +y_cumulative_fraction 0.1462862415661702 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_cumulative_fraction 0.2789474480337396 +y_cumulative_fraction 0.2513814841459683 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 +x_cumulative_fraction 0.3305757729712584 +y_cumulative_fraction 0.3194760777242386 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_cumulative_fraction 0.42681572258499034 +y_cumulative_fraction 0.34648473154468673 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_cumulative_fraction 0.5203095529685396 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_cumulative_fraction 0.5598274575340497 +y_cumulative_fraction 0.6195819856974067 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_cumulative_fraction 0.7142012402659285 +y_cumulative_fraction 0.6573982630865928 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_cumulative_fraction 0.8652366341422009 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 +x_cumulative_fraction 0.8731704815866045 +y_cumulative_fraction 0.8082630563000914 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +mlr --oxtab fraction -f x,y -g a -c ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_cumulative_fraction 0.4082688378215578 +y_cumulative_fraction 0.4327698452134853 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_cumulative_fraction 0.4330702704688488 +y_cumulative_fraction 0.6184977347659367 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_cumulative_fraction 0.6507809605173953 +y_cumulative_fraction 0.7774468060746659 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_cumulative_fraction 1 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_cumulative_fraction 0.46827347102266204 +y_cumulative_fraction 1 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_cumulative_fraction 1 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +mlr --oxtab fraction -f x,y -g a,b -c ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_cumulative_fraction 1 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_cumulative_fraction 1 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_cumulative_fraction 1 +y_cumulative_fraction 1 + +mlr --oxtab fraction -f x,y -p -c ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_cumulative_percent 8.750686695876585 +y_cumulative_percent 14.62862415661702 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_cumulative_percent 27.89474480337396 +y_cumulative_percent 25.138148414596827 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 +x_cumulative_percent 33.05757729712584 +y_cumulative_percent 31.947607772423858 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_cumulative_percent 42.68157225849903 +y_cumulative_percent 34.648473154468675 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_cumulative_percent 52.03095529685397 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_cumulative_percent 55.98274575340497 +y_cumulative_percent 61.958198569740674 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_cumulative_percent 71.42012402659284 +y_cumulative_percent 65.73982630865927 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_cumulative_percent 86.52366341422008 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 +x_cumulative_percent 87.31704815866046 +y_cumulative_percent 80.82630563000914 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_cumulative_percent 100 +y_cumulative_percent 100 + +mlr --oxtab fraction -f x,y -g a -p -c ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_cumulative_percent 40.82688378215578 +y_cumulative_percent 43.27698452134853 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_cumulative_percent 43.307027046884876 +y_cumulative_percent 61.849773476593676 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 +x_cumulative_percent 65.07809605173954 +y_cumulative_percent 77.74468060746659 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_cumulative_percent 100 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_cumulative_percent 46.8273471022662 +y_cumulative_percent 100 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_cumulative_percent 100 +y_cumulative_percent 100 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_cumulative_percent 100 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_cumulative_percent 100 +y_cumulative_percent 100 + +mlr --oxtab fraction -f x,y -g a,b -p -c ./reg-test/input/abixy-het +a pan +b pan +i 1 +x 0.3467901443380824 +y 0.7268028627434533 +x_cumulative_percent 100 +y_cumulative_percent 100 + +a eks +b pan +i 2 +x 0.7586799647899636 +y 0.5221511083334797 +x_cumulative_percent 100 +y_cumulative_percent 100 + +aaa wye +b wye +i 3 +x 0.20460330576630303 +y 0.33831852551664776 + +a eks +bbb wye +i 4 +x 0.38139939387114097 +y 0.13418874328430463 + +a wye +b pan +i 5 +xxx 0.5732889198020006 +y 0.8636244699032729 +y_cumulative_percent 100 + +a zee +b pan +i 6 +x 0.5271261600918548 +y 0.49322128674835697 +x_cumulative_percent 100 +y_cumulative_percent 100 + +a eks +b zee +iii 7 +x 0.6117840605678454 +y 0.1878849191181694 +x_cumulative_percent 100 +y_cumulative_percent 100 + +a zee +b wye +i 8 +x 0.5985540091064224 +yyy 0.976181385699006 +x_cumulative_percent 100 + +aaa hat +bbb wye +i 9 +x 0.03144187646093577 +y 0.7495507603507059 + +a pan +b wye +i 10 +x 0.5026260055412137 +y 0.9526183602969864 +x_cumulative_percent 100 +y_cumulative_percent 100 + diff --git a/go/src/cli/mlrcli_transformers.go b/go/src/cli/mlrcli_transformers.go index 0637e0a31..6dfdfc264 100644 --- a/go/src/cli/mlrcli_transformers.go +++ b/go/src/cli/mlrcli_transformers.go @@ -23,6 +23,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.FillDownSetup, transformers.FilterSetup, transformers.FlattenSetup, + transformers.FractionSetup, transformers.GapSetup, transformers.GrepSetup, transformers.GroupBySetup, @@ -30,17 +31,17 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.HavingFieldsSetup, transformers.HeadSetup, transformers.HistogramSetup, - transformers.JoinSetup, transformers.JSONParseSetup, transformers.JSONStringifySetup, + transformers.JoinSetup, transformers.LabelSetup, transformers.NothingSetup, transformers.PutSetup, - transformers.RepeatSetup, transformers.RegularizeSetup, transformers.RemoveEmptyColumnsSetup, transformers.RenameSetup, transformers.ReorderSetup, + transformers.RepeatSetup, transformers.SampleSetup, transformers.Sec2GMTSetup, transformers.SeqgenSetup, @@ -51,8 +52,8 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{ transformers.Stats1Setup, transformers.StepSetup, transformers.TacSetup, - transformers.TeeSetup, transformers.TailSetup, + transformers.TeeSetup, transformers.UnflattenSetup, transformers.UnsparsifySetup, } diff --git a/go/src/transformers/fraction.go b/go/src/transformers/fraction.go new file mode 100644 index 000000000..16ec23c40 --- /dev/null +++ b/go/src/transformers/fraction.go @@ -0,0 +1,278 @@ +package transformers + +import ( + "container/list" + "fmt" + "os" + "strings" + + "miller/src/cliutil" + "miller/src/lib" + "miller/src/transforming" + "miller/src/types" +) + +// ---------------------------------------------------------------- +const verbNameFraction = "fraction" + +var FractionSetup = transforming.TransformerSetup{ + Verb: verbNameFraction, + UsageFunc: transformerFractionUsage, + ParseCLIFunc: transformerFractionParseCLI, + IgnoresInput: false, +} + +func transformerFractionUsage( + o *os.File, + doExit bool, + exitCode int, +) { + argv0 := lib.MlrExeName() + verb := verbNameFraction + fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb) + fmt.Fprintf(o, "For each record's value in specified fields, computes the ratio of that\n") + fmt.Fprintf(o, "value to the sum of values in that field over all input records.\n") + fmt.Fprintf(o, "E.g. with input records x=1 x=2 x=3 and x=4, emits output records\n") + fmt.Fprintf(o, "x=1,x_fraction=0.1 x=2,x_fraction=0.2 x=3,x_fraction=0.3 and x=4,x_fraction=0.4\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "Note: this is internally a two-pass algorithm: on the first pass it retains\n") + fmt.Fprintf(o, "input records and accumulates sums; on the second pass it computes quotients\n") + fmt.Fprintf(o, "and emits output records. This means it produces no output until all input is read.\n") + fmt.Fprintf(o, "\n") + fmt.Fprintf(o, "Options:\n") + fmt.Fprintf(o, "-f {a,b,c} Field name(s) for fraction calculation\n") + fmt.Fprintf(o, "-g {d,e,f} Optional group-by-field name(s) for fraction counts\n") + fmt.Fprintf(o, "-p Produce percents [0..100], not fractions [0..1]. Output field names\n") + fmt.Fprintf(o, " end with \"_percent\" rather than \"_fraction\"\n") + fmt.Fprintf(o, "-c Produce cumulative distributions, i.e. running sums: each output\n") + fmt.Fprintf(o, " value folds in the sum of the previous for the specified group\n") + fmt.Fprintf(o, " E.g. with input records x=1 x=2 x=3 and x=4, emits output records\n") + fmt.Fprintf(o, " x=1,x_cumulative_fraction=0.1 x=2,x_cumulative_fraction=0.3\n") + fmt.Fprintf(o, " x=3,x_cumulative_fraction=0.6 and x=4,x_cumulative_fraction=1.0\n") + + if doExit { + os.Exit(exitCode) + } +} + +func transformerFractionParseCLI( + pargi *int, + argc int, + args []string, + _ *cliutil.TReaderOptions, + __ *cliutil.TWriterOptions, +) transforming.IRecordTransformer { + + // Skip the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + // Parse local flags + var fractionFieldNames []string = nil + var groupByFieldNames []string = nil + doPercents := false + doCumu := false + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + opt := args[argi] + if !strings.HasPrefix(opt, "-") { + break // No more flag options to process + } + argi++ + + if opt == "-h" || opt == "--help" { + transformerFractionUsage(os.Stdout, true, 0) + + } else if opt == "-f" { + fractionFieldNames = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-g" { + groupByFieldNames = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-p" { + doPercents = true + + } else if opt == "-c" { + doCumu = true + + } else { + transformerFractionUsage(os.Stderr, true, 1) + } + } + + if fractionFieldNames == nil { + transformerFractionUsage(os.Stderr, true, 1) + } + + transformer, _ := NewTransformerFraction( + fractionFieldNames, + groupByFieldNames, + doPercents, + doCumu, + ) + + *pargi = argi + return transformer +} + +// ---------------------------------------------------------------- +type TransformerFraction struct { + fractionFieldNames []string + groupByFieldNames []string + doCumu bool + + recordsAndContexts *list.List + // Two-level map: Group-by field names are the first keyset; + // fraction field names are keys into the second. + sums map[string]map[string]*types.Mlrval + cumus map[string]map[string]*types.Mlrval + + outputFieldNameSuffix string // "_fraction" or "_percent" + multiplier *types.Mlrval // 1.0 for fraction or 100.0 for percent + zero *types.Mlrval +} + +// ---------------------------------------------------------------- +func NewTransformerFraction( + fractionFieldNames []string, + groupByFieldNames []string, + doPercents bool, + doCumu bool, +) (*TransformerFraction, error) { + + recordsAndContexts := list.New() + sums := make(map[string]map[string]*types.Mlrval) + cumus := make(map[string]map[string]*types.Mlrval) + + var multiplier *types.Mlrval + var outputFieldNameSuffix string + if doPercents { + multiplier = types.MlrvalPointerFromInt(100) + if doCumu { + outputFieldNameSuffix = "_cumulative_percent" + } else { + outputFieldNameSuffix = "_percent" + } + } else { + multiplier = types.MlrvalPointerFromInt(1) + if doCumu { + outputFieldNameSuffix = "_cumulative_fraction" + } else { + outputFieldNameSuffix = "_fraction" + } + } + + zero := types.MlrvalPointerFromInt(0) + + return &TransformerFraction{ + fractionFieldNames: fractionFieldNames, + groupByFieldNames: groupByFieldNames, + doCumu: doCumu, + recordsAndContexts: recordsAndContexts, + sums: sums, + cumus: cumus, + outputFieldNameSuffix: outputFieldNameSuffix, + multiplier: multiplier, + zero: zero, + }, nil +} + +// ---------------------------------------------------------------- +func (this *TransformerFraction) Transform( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { // Not end of stream; pass 1 + inrec := inrecAndContext.Record + + // Append records into a single output list (so that this verb is order-preserving). + this.recordsAndContexts.PushBack(inrecAndContext) + + // Accumulate sums of fraction-field values grouped by group-by field names + groupingKey, hasAll := inrec.GetSelectedValuesJoined(this.groupByFieldNames) + + if hasAll { + sumsForGroup := this.sums[groupingKey] + var cumusForGroup map[string]*types.Mlrval = nil + if sumsForGroup == nil { + sumsForGroup = make(map[string]*types.Mlrval) + this.sums[groupingKey] = sumsForGroup + cumusForGroup = make(map[string]*types.Mlrval) + this.cumus[groupingKey] = cumusForGroup + } + for _, fractionFieldName := range this.fractionFieldNames { + value := inrec.Get(fractionFieldName) + if value != nil { + value.AssertNumeric() // may fatal the process + sum := sumsForGroup[fractionFieldName] + if sum == nil { // First value for group + sumsForGroup[fractionFieldName] = value.Copy() + cumusForGroup[fractionFieldName] = this.zero + } else { + sumsForGroup[fractionFieldName] = types.MlrvalBinaryPlus(sum, value) + } + } + } + } + + } else { // End of stream; pass 2 + // Iterate over the retained records, decorating them with fraction fields. + endOfStreamContext := inrecAndContext.Context + + for { + element := this.recordsAndContexts.Front() + if element == nil { + break + } + this.recordsAndContexts.Remove(element) + recordAndContext := element.Value.(*types.RecordAndContext) + outrec := recordAndContext.Record + + groupingKey, hasAll := outrec.GetSelectedValuesJoined(this.groupByFieldNames) + if hasAll { + sumsForGroup := this.sums[groupingKey] + cumusForGroup := this.cumus[groupingKey] + lib.InternalCodingErrorIf(sumsForGroup == nil) // should have been populated on pass 1 + + for _, fractionFieldName := range this.fractionFieldNames { + value := outrec.Get(fractionFieldName) + if value != nil { + value.AssertNumeric() // may fatal the process + + var numerator *types.Mlrval = nil + var cumu *types.Mlrval = nil + var outputValue *types.Mlrval = nil + + if this.doCumu { + cumu = cumusForGroup[fractionFieldName] + numerator = types.MlrvalBinaryPlus(value, cumu) + } else { + numerator = value + } + + denominator := sumsForGroup[fractionFieldName] + if !types.MlrvalEqualsAsBool(value, this.zero) { + outputValue = types.MlrvalDivide(numerator, denominator) + outputValue = types.MlrvalTimes(outputValue, this.multiplier) + } else { + outputValue = types.MLRVAL_ERROR + } + + outrec.PutCopy( + fractionFieldName+this.outputFieldNameSuffix, + outputValue, + ) + + if this.doCumu { + cumusForGroup[fractionFieldName] = types.MlrvalBinaryPlus(cumu, value) + } + } + } + } + + outputChannel <- types.NewRecordAndContext(outrec, &endOfStreamContext) + } + outputChannel <- inrecAndContext // end-of-stream marker + } +} diff --git a/go/src/types/mlrval_accessors.go b/go/src/types/mlrval_accessors.go index 18839faa2..1a8561679 100644 --- a/go/src/types/mlrval_accessors.go +++ b/go/src/types/mlrval_accessors.go @@ -147,6 +147,10 @@ func (this *Mlrval) GetNumericToFloatValueOrDie() (floatValue float64) { return floatValue } +func (this *Mlrval) AssertNumeric() { + _ = this.GetNumericToFloatValueOrDie() +} + func (this *Mlrval) GetBoolValue() (boolValue bool, isBool bool) { if this.mvtype == MT_BOOL { return this.boolval, true diff --git a/go/src/types/mlrval_functions_booleans.go b/go/src/types/mlrval_functions_booleans.go index 7589fa38c..279537ae8 100644 --- a/go/src/types/mlrval_functions_booleans.go +++ b/go/src/types/mlrval_functions_booleans.go @@ -224,7 +224,6 @@ var eq_dispositions = [MT_DIM][MT_DIM]BinaryFunc{} func init() { eq_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ - // . ERROR ABSENT VOID STRING INT FLOAT BOOL ARRAY MAP /*ERROR */ {_erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro, _erro}, /*ABSENT */ {_erro, _absn, _absn, _absn, _absn, _absn, _absn, _absn, _absn}, @@ -236,7 +235,6 @@ func init() { /*ARRAY */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, eq_b_aa, _fals}, /*MAP */ {_erro, _absn, _fals, _fals, _fals, _fals, _fals, _fals, eq_b_mm}, } - } var ne_dispositions = [MT_DIM][MT_DIM]BinaryFunc{ @@ -333,6 +331,14 @@ func MlrvalLessThanForSort(input1, input2 *Mlrval) bool { return retval } +// Convenience wrapper for non-DSL callsites that just want a bool +func MlrvalEqualsAsBool(input1, input2 *Mlrval) bool { + mretval := eq_dispositions[input1.mvtype][input2.mvtype](input1, input2) + retval, ok := mretval.GetBoolValue() + lib.InternalCodingErrorIf(!ok) + return retval +} + // ---------------------------------------------------------------- func MlrvalLogicalAND(input1, input2 *Mlrval) *Mlrval { if input1.mvtype == MT_BOOL && input2.mvtype == MT_BOOL { diff --git a/go/todo.txt b/go/todo.txt index d73f3aca3..2d7282a5f 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -523,3 +523,4 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision * more copy-on-retain for concurrent pointer-mods o make a thorough audit, and warn everywhere o either do copy for all retainers, or treat inrecs as immutable ... + o 'this.recordsAndContexts.PushBack(inrecAndContext)' idiom needs copy everywhere ...