diff --git a/docs6/data-diving-examples.rst b/docs6/data-diving-examples.rst index e26805fcd..83ec92760 100644 --- a/docs6/data-diving-examples.rst +++ b/docs6/data-diving-examples.rst @@ -45,8 +45,8 @@ Categorization of total insured value: :emphasize-lines: 1-1 mlr --from data/flins.csv --icsv --opprint stats1 -a min,mean,max -f tiv_2012 - tiv_2012_min tiv_2012_mean tiv_2012_max - 19757.91 1.0615314637499999e+06 2.78555163e+06 + tiv_2012_min tiv_2012_mean tiv_2012_max + 19757.91 1061531.4637499999 2785551.63 .. code-block:: none :emphasize-lines: 1-2 @@ -143,7 +143,7 @@ Here it looks reasonable that ``u`` is unit-uniform; something's up with ``v`` b flag_mean 0.39888866838658465 flag_max 1 - u_min 4.3912454007477564e-05 + u_min 0.000043912454007477564 u_mean 0.4983263438118866 u_max 0.9999687954968421 @@ -179,13 +179,13 @@ Look at univariate stats by color and shape. In particular, color-dependent flag mlr --opprint stats1 -a min,mean,max -f flag,u,v -g color \ then sort -f color \ data/colored-shapes.dkvp - color flag_min flag_mean flag_max u_min u_mean u_max v_min v_mean v_max - blue 0 0.5843537414965987 1 4.3912454007477564e-05 0.517717155039078 0.9999687954968421 0.0014886830387470518 0.49105642841387653 0.9995761761685742 - green 0 0.20919747520288548 1 0.00048750676198217047 0.5048610622924616 0.9999361779701204 0.0005012669003675585 0.49908475928072205 0.9996764373885353 - orange 0 0.5214521452145214 1 0.00123537823160913 0.49053241689014415 0.9988853487546249 0.0024486660337188493 0.4877637745987629 0.998475130432018 - purple 0 0.09019264448336252 1 0.0002655214518428872 0.4940049543793683 0.9996465731736793 0.0003641137096487279 0.497050699948439 0.9999751864255598 - red 0 0.3031674208144796 1 0.0006711367180041172 0.49255964831571375 0.9998822102016469 -0.09270905318501277 0.4965350959465078 1.0724998185026013 - yellow 0 0.8924274593064402 1 0.001300228762057487 0.49712912165196765 0.99992313390574 0.0007109695568577878 0.510626599360317 0.9999189897724752 + color flag_min flag_mean flag_max u_min u_mean u_max v_min v_mean v_max + blue 0 0.5843537414965987 1 0.000043912454007477564 0.517717155039078 0.9999687954968421 0.0014886830387470518 0.49105642841387653 0.9995761761685742 + green 0 0.20919747520288548 1 0.00048750676198217047 0.5048610622924616 0.9999361779701204 0.0005012669003675585 0.49908475928072205 0.9996764373885353 + orange 0 0.5214521452145214 1 0.00123537823160913 0.49053241689014415 0.9988853487546249 0.0024486660337188493 0.4877637745987629 0.998475130432018 + purple 0 0.09019264448336252 1 0.0002655214518428872 0.4940049543793683 0.9996465731736793 0.0003641137096487279 0.497050699948439 0.9999751864255598 + red 0 0.3031674208144796 1 0.0006711367180041172 0.49255964831571375 0.9998822102016469 -0.09270905318501277 0.4965350959465078 1.0724998185026013 + yellow 0 0.8924274593064402 1 0.001300228762057487 0.49712912165196765 0.99992313390574 0.0007109695568577878 0.510626599360317 0.9999189897724752 .. code-block:: none :emphasize-lines: 1-3 @@ -193,10 +193,10 @@ Look at univariate stats by color and shape. In particular, color-dependent flag mlr --opprint stats1 -a min,mean,max -f flag,u,v -g shape \ then sort -f shape \ data/colored-shapes.dkvp - shape flag_min flag_mean flag_max u_min u_mean u_max v_min v_mean v_max - circle 0 0.3998456194519491 1 4.3912454007477564e-05 0.49855450951394115 0.99992313390574 -0.09270905318501277 0.49552415740048406 1.0724998185026013 - square 0 0.39611178614823817 1 0.0001881939925673093 0.499385458061097 0.9999687954968421 8.930277299445954e-05 0.49653825501903986 0.9999751864255598 - triangle 0 0.4015421115065243 1 0.000881025170573424 0.4968585405884252 0.9996614910922645 0.000716883409890845 0.501049532862137 0.9999946837499262 + shape flag_min flag_mean flag_max u_min u_mean u_max v_min v_mean v_max + circle 0 0.3998456194519491 1 0.000043912454007477564 0.49855450951394115 0.99992313390574 -0.09270905318501277 0.49552415740048406 1.0724998185026013 + square 0 0.39611178614823817 1 0.0001881939925673093 0.499385458061097 0.9999687954968421 0.00008930277299445954 0.49653825501903986 0.9999751864255598 + triangle 0 0.4015421115065243 1 0.000881025170573424 0.4968585405884252 0.9996614910922645 0.000716883409890845 0.501049532862137 0.9999946837499262 Look at bivariate stats by color and shape. In particular, ``u,v`` pairwise correlation for red circles pops out: diff --git a/docs6/dates-and-times.rst b/docs6/dates-and-times.rst index 94e877eb4..3c4caa90e 100644 --- a/docs6/dates-and-times.rst +++ b/docs6/dates-and-times.rst @@ -67,16 +67,16 @@ Since there are 1372 lines in the data file, some automation is called for. To f then put '$datestamp = strptime($date, "%Y-%m-%d")' \ then step -a delta -f datestamp \ | head - n=1,date=2012-03-05,qoh=10055,datestamp=1.3309056e+09,datestamp_delta=0 - n=2,date=2012-03-06,qoh=10486,datestamp=1.330992e+09,datestamp_delta=86400 - n=3,date=2012-03-07,qoh=10430,datestamp=1.3310784e+09,datestamp_delta=86400 - n=4,date=2012-03-08,qoh=10674,datestamp=1.3311648e+09,datestamp_delta=86400 - n=5,date=2012-03-09,qoh=10880,datestamp=1.3312512e+09,datestamp_delta=86400 - n=6,date=2012-03-10,qoh=10718,datestamp=1.3313376e+09,datestamp_delta=86400 - n=7,date=2012-03-11,qoh=10795,datestamp=1.331424e+09,datestamp_delta=86400 - n=8,date=2012-03-12,qoh=11043,datestamp=1.3315104e+09,datestamp_delta=86400 - n=9,date=2012-03-13,qoh=11177,datestamp=1.3315968e+09,datestamp_delta=86400 - n=10,date=2012-03-14,qoh=11498,datestamp=1.3316832e+09,datestamp_delta=86400 + n=1,date=2012-03-05,qoh=10055,datestamp=1330905600,datestamp_delta=0 + n=2,date=2012-03-06,qoh=10486,datestamp=1330992000,datestamp_delta=86400 + n=3,date=2012-03-07,qoh=10430,datestamp=1331078400,datestamp_delta=86400 + n=4,date=2012-03-08,qoh=10674,datestamp=1331164800,datestamp_delta=86400 + n=5,date=2012-03-09,qoh=10880,datestamp=1331251200,datestamp_delta=86400 + n=6,date=2012-03-10,qoh=10718,datestamp=1331337600,datestamp_delta=86400 + n=7,date=2012-03-11,qoh=10795,datestamp=1331424000,datestamp_delta=86400 + n=8,date=2012-03-12,qoh=11043,datestamp=1331510400,datestamp_delta=86400 + n=9,date=2012-03-13,qoh=11177,datestamp=1331596800,datestamp_delta=86400 + n=10,date=2012-03-14,qoh=11498,datestamp=1331683200,datestamp_delta=86400 Then, filter for adjacent difference not being 86400 (the number of seconds in a day): @@ -88,8 +88,8 @@ Then, filter for adjacent difference not being 86400 (the number of seconds in a then put '$datestamp = strptime($date, "%Y-%m-%d")' \ then step -a delta -f datestamp \ then filter '$datestamp_delta != 86400 && $n != 1' - n=774,date=2014-04-19,qoh=130140,datestamp=1.3978656e+09,datestamp_delta=259200 - n=1119,date=2015-03-31,qoh=181625,datestamp=1.42776e+09,datestamp_delta=172800 + n=774,date=2014-04-19,qoh=130140,datestamp=1397865600,datestamp_delta=259200 + n=1119,date=2015-03-31,qoh=181625,datestamp=1427760000,datestamp_delta=172800 Given this, it's now easy to see where the gaps are: diff --git a/docs6/reference-verbs.rst b/docs6/reference-verbs.rst index 8d0c400c2..6d6ce6485 100644 --- a/docs6/reference-verbs.rst +++ b/docs6/reference-verbs.rst @@ -2796,7 +2796,7 @@ optionally categorized by one or more other fields. mlr --oxtab stats1 -a count,sum,min,p10,p50,mean,p90,max -f x,y data/medium x_count 10000 x_sum 4986.019681679581 - x_min 4.509679127584487e-05 + x_min 0.00004509679127584487 x_p10 0.09332217805283527 x_p50 0.5011592202840128 x_mean 0.49860196816795804 @@ -2804,7 +2804,7 @@ optionally categorized by one or more other fields. x_max 0.999952670371898 y_count 10000 y_sum 5062.057444929905 - y_min 8.818962627266114e-05 + y_min 0.00008818962627266114 y_p10 0.10213207378968225 y_p50 0.5060212582772865 y_mean 0.5062057444929905 @@ -2899,7 +2899,7 @@ fields, optionally categorized by one or more fields. mlr --oxtab put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' \ then stats2 -a cov,corr -f x,y,y,y,x2,xy,x2,y2 \ data/medium - x_y_cov 4.2574820827444476e-05 + x_y_cov 0.000042574820827444476 x_y_corr 0.0005042001844467462 y_y_cov 0.08461122467974003 y_y_corr 1 @@ -2914,12 +2914,12 @@ fields, optionally categorized by one or more fields. mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' \ then stats2 -a linreg-ols,r2 -f x,y,y,y,xy,y2 -g a \ data/medium - a x_y_ols_m x_y_ols_b x_y_ols_n x_y_r2 y_y_ols_m y_y_ols_b y_y_ols_n y_y_r2 xy_y2_ols_m xy_y2_ols_b xy_y2_ols_n xy_y2_r2 - pan 0.01702551273681908 0.5004028922897639 2081 0.00028691820445814767 1 0 2081 1 0.8781320866715662 0.11908230147563566 2081 0.41749827377311266 - eks 0.0407804923685586 0.48140207967651016 1965 0.0016461239223448587 1 0 1965 1 0.8978728611690183 0.10734054433612333 1965 0.45563223864254526 - wye -0.03915349075204814 0.5255096523974456 1966 0.0015051268704373607 1 0 1966 1 0.8538317334220835 0.1267454301662969 1966 0.38991721818599295 - zee 0.0027812364960399147 0.5043070448033061 2047 7.751652858786137e-06 1 0 2047 1 0.8524439912011013 0.12401684308018937 2047 0.39356598090006495 - hat -0.018620577041095078 0.5179005397264935 1941 0.0003520036646055585 1 0 1941 1 0.8412305086345014 0.13557328318623216 1941 0.3687944261732265 + a x_y_ols_m x_y_ols_b x_y_ols_n x_y_r2 y_y_ols_m y_y_ols_b y_y_ols_n y_y_r2 xy_y2_ols_m xy_y2_ols_b xy_y2_ols_n xy_y2_r2 + pan 0.01702551273681908 0.5004028922897639 2081 0.00028691820445814767 1 0 2081 1 0.8781320866715662 0.11908230147563566 2081 0.41749827377311266 + eks 0.0407804923685586 0.48140207967651016 1965 0.0016461239223448587 1 0 1965 1 0.8978728611690183 0.10734054433612333 1965 0.45563223864254526 + wye -0.03915349075204814 0.5255096523974456 1966 0.0015051268704373607 1 0 1966 1 0.8538317334220835 0.1267454301662969 1966 0.38991721818599295 + zee 0.0027812364960399147 0.5043070448033061 2047 0.000007751652858786137 1 0 2047 1 0.8524439912011013 0.12401684308018937 2047 0.39356598090006495 + hat -0.018620577041095078 0.5179005397264935 1941 0.0003520036646055585 1 0 1941 1 0.8412305086345014 0.13557328318623216 1941 0.3687944261732265 Here's an example simple line-fit. The ``x`` and ``y`` fields of the ``data/medium`` dataset are just independent uniformly @@ -3262,6 +3262,29 @@ tee -h|--help Show this message. +.. _reference-verbs-template: + +template +---------------------------------------------------------------- + +.. code-block:: none + :emphasize-lines: 1-1 + + mlr template --help + Usage: mlr template [options] + Places input-record fields in the order specified by list of column names. + If the input record is missing a specified field, it will be filled with the fill-with. + If the input record possesses an unspecified field, it will be discarded. + Options: + -f {a,b,c} Comma-separated field names for template, e.g. a,b,c. + -t {filename} CSV file whose header line will be used for template. + --fill-with {filler string} What to fill absent fields with. Defaults to the empty string. + -h|--help Show this message. + Example: + * Specified fields are a,b,c. + * Input record is c=3,a=1,f=6. + * Output record is a=1,b=,c=3. + .. _reference-verbs-top: top diff --git a/docs6/reference-verbs.rst.in b/docs6/reference-verbs.rst.in index 50b51aea6..c3ce7e52b 100644 --- a/docs6/reference-verbs.rst.in +++ b/docs6/reference-verbs.rst.in @@ -1188,6 +1188,15 @@ GENRST_RUN_COMMAND mlr tee --help GENRST_EOF +.. _reference-verbs-template: + +template +---------------------------------------------------------------- + +GENRST_RUN_COMMAND +mlr template --help +GENRST_EOF + .. _reference-verbs-top: top diff --git a/docs6/two-pass-algorithms.rst b/docs6/two-pass-algorithms.rst index 67f41e1f2..ea9e441d4 100644 --- a/docs6/two-pass-algorithms.rst +++ b/docs6/two-pass-algorithms.rst @@ -587,7 +587,7 @@ Min/max without/with oosvars :emphasize-lines: 1-1 mlr --oxtab stats1 -a min,max -f x data/medium - x_min 4.509679127584487e-05 + x_min 0.00004509679127584487 x_max 0.999952670371898 .. code-block:: none @@ -598,7 +598,7 @@ Min/max without/with oosvars @x_max = max(@x_max, $x); end{emitf @x_min, @x_max} ' data/medium - x_min 4.509679127584487e-05 + x_min 0.00004509679127584487 x_max 0.999952670371898 Keyed min/max without/with oosvars @@ -613,7 +613,7 @@ Keyed min/max without/with oosvars eks 0.0006917972627396018 0.9988110946859143 wye 0.0001874794831505655 0.9998228522652893 zee 0.0005486114815762555 0.9994904324789629 - hat 4.509679127584487e-05 0.999952670371898 + hat 0.00004509679127584487 0.999952670371898 .. code-block:: none :emphasize-lines: 1-7 @@ -630,7 +630,7 @@ Keyed min/max without/with oosvars eks 0.0006917972627396018 0.9988110946859143 wye 0.0001874794831505655 0.9998228522652893 zee 0.0005486114815762555 0.9994904324789629 - hat 4.509679127584487e-05 0.999952670371898 + hat 0.00004509679127584487 0.999952670371898 Delta without/with oosvars ---------------------------------------------------------------- diff --git a/go/regtest/cases/cli-help/0001/expout b/go/regtest/cases/cli-help/0001/expout index 1d950e5b5..232e9a855 100644 --- a/go/regtest/cases/cli-help/0001/expout +++ b/go/regtest/cases/cli-help/0001/expout @@ -966,6 +966,21 @@ is written in JSON format. -h|--help Show this message. +================================================================ +Usage: mlr template [options] +Places input-record fields in the order specified by list of column names. +If the input record is missing a specified field, it will be filled with the fill-with. +If the input record possesses an unspecified field, it will be discarded. +Options: + -f {a,b,c} Comma-separated field names for template, e.g. a,b,c. + -t {filename} CSV file whose header line will be used for template. +--fill-with {filler string} What to fill absent fields with. Defaults to the empty string. +-h|--help Show this message. +Example: +* Specified fields are a,b,c. +* Input record is c=3,a=1,f=6. +* Output record is a=1,b=,c=3. + ================================================================ Usage: mlr top [options] -f {a,b,c} Value-field names for top counts. diff --git a/go/regtest/cases/verb-template/0001/cmd b/go/regtest/cases/verb-template/0001/cmd new file mode 100644 index 000000000..81dbdcd2b --- /dev/null +++ b/go/regtest/cases/verb-template/0001/cmd @@ -0,0 +1 @@ +mlr --opprint template -f x,a,b regtest/input/abixy diff --git a/go/regtest/cases/verb-template/0001/experr b/go/regtest/cases/verb-template/0001/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-template/0001/expout b/go/regtest/cases/verb-template/0001/expout new file mode 100644 index 000000000..d1ceb205f --- /dev/null +++ b/go/regtest/cases/verb-template/0001/expout @@ -0,0 +1,11 @@ +x a b +0.3467901443380824 pan pan +0.7586799647899636 eks pan +0.20460330576630303 wye wye +0.38139939387114097 eks wye +0.5732889198020006 wye pan +0.5271261600918548 zee pan +0.6117840605678454 eks zee +0.5985540091064224 zee wye +0.03144187646093577 hat wye +0.5026260055412137 pan wye diff --git a/go/regtest/cases/verb-template/0002/cmd b/go/regtest/cases/verb-template/0002/cmd new file mode 100644 index 000000000..19f42bb62 --- /dev/null +++ b/go/regtest/cases/verb-template/0002/cmd @@ -0,0 +1 @@ +mlr --opprint template -f x,z,a,b regtest/input/abixy-het diff --git a/go/regtest/cases/verb-template/0002/experr b/go/regtest/cases/verb-template/0002/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-template/0002/expout b/go/regtest/cases/verb-template/0002/expout new file mode 100644 index 000000000..a05ea7079 --- /dev/null +++ b/go/regtest/cases/verb-template/0002/expout @@ -0,0 +1,11 @@ +x z a b +0.3467901443380824 - pan pan +0.7586799647899636 - eks pan +0.20460330576630303 - - wye +0.38139939387114097 - eks - +- - wye pan +0.5271261600918548 - zee pan +0.6117840605678454 - eks zee +0.5985540091064224 - zee wye +0.03144187646093577 - - - +0.5026260055412137 - pan wye diff --git a/go/regtest/cases/verb-template/0003/cmd b/go/regtest/cases/verb-template/0003/cmd new file mode 100644 index 000000000..84436b5f0 --- /dev/null +++ b/go/regtest/cases/verb-template/0003/cmd @@ -0,0 +1 @@ +mlr --opprint template --fill-with N/A -f x,z,a,b regtest/input/abixy-het diff --git a/go/regtest/cases/verb-template/0003/experr b/go/regtest/cases/verb-template/0003/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-template/0003/expout b/go/regtest/cases/verb-template/0003/expout new file mode 100644 index 000000000..e0bea3266 --- /dev/null +++ b/go/regtest/cases/verb-template/0003/expout @@ -0,0 +1,11 @@ +x z a b +0.3467901443380824 N/A pan pan +0.7586799647899636 N/A eks pan +0.20460330576630303 N/A N/A wye +0.38139939387114097 N/A eks N/A +N/A N/A wye pan +0.5271261600918548 N/A zee pan +0.6117840605678454 N/A eks zee +0.5985540091064224 N/A zee wye +0.03144187646093577 N/A N/A N/A +0.5026260055412137 N/A pan wye diff --git a/go/regtest/cases/verb-template/0004/cmd b/go/regtest/cases/verb-template/0004/cmd new file mode 100644 index 000000000..21caaf80f --- /dev/null +++ b/go/regtest/cases/verb-template/0004/cmd @@ -0,0 +1 @@ +mlr --opprint template -f none,at,all regtest/input/abixy diff --git a/go/regtest/cases/verb-template/0004/experr b/go/regtest/cases/verb-template/0004/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-template/0004/expout b/go/regtest/cases/verb-template/0004/expout new file mode 100644 index 000000000..7ae07cc1e --- /dev/null +++ b/go/regtest/cases/verb-template/0004/expout @@ -0,0 +1,11 @@ +none at all +- - - +- - - +- - - +- - - +- - - +- - - +- - - +- - - +- - - +- - - diff --git a/go/regtest/cases/verb-template/0005/cmd b/go/regtest/cases/verb-template/0005/cmd new file mode 100644 index 000000000..955d603fd --- /dev/null +++ b/go/regtest/cases/verb-template/0005/cmd @@ -0,0 +1 @@ +mlr --opprint template -t regtest/input/template-1.csv regtest/input/abixy diff --git a/go/regtest/cases/verb-template/0005/experr b/go/regtest/cases/verb-template/0005/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-template/0005/expout b/go/regtest/cases/verb-template/0005/expout new file mode 100644 index 000000000..9f6155be2 --- /dev/null +++ b/go/regtest/cases/verb-template/0005/expout @@ -0,0 +1,11 @@ +x y a +0.3467901443380824 0.7268028627434533 pan +0.7586799647899636 0.5221511083334797 eks +0.20460330576630303 0.33831852551664776 wye +0.38139939387114097 0.13418874328430463 eks +0.5732889198020006 0.8636244699032729 wye +0.5271261600918548 0.49322128674835697 zee +0.6117840605678454 0.1878849191181694 eks +0.5985540091064224 0.976181385699006 zee +0.03144187646093577 0.7495507603507059 hat +0.5026260055412137 0.9526183602969864 pan diff --git a/go/regtest/cases/verb-template/0006/cmd b/go/regtest/cases/verb-template/0006/cmd new file mode 100644 index 000000000..955d603fd --- /dev/null +++ b/go/regtest/cases/verb-template/0006/cmd @@ -0,0 +1 @@ +mlr --opprint template -t regtest/input/template-1.csv regtest/input/abixy diff --git a/go/regtest/cases/verb-template/0006/experr b/go/regtest/cases/verb-template/0006/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/verb-template/0006/expout b/go/regtest/cases/verb-template/0006/expout new file mode 100644 index 000000000..9f6155be2 --- /dev/null +++ b/go/regtest/cases/verb-template/0006/expout @@ -0,0 +1,11 @@ +x y a +0.3467901443380824 0.7268028627434533 pan +0.7586799647899636 0.5221511083334797 eks +0.20460330576630303 0.33831852551664776 wye +0.38139939387114097 0.13418874328430463 eks +0.5732889198020006 0.8636244699032729 wye +0.5271261600918548 0.49322128674835697 zee +0.6117840605678454 0.1878849191181694 eks +0.5985540091064224 0.976181385699006 zee +0.03144187646093577 0.7495507603507059 hat +0.5026260055412137 0.9526183602969864 pan diff --git a/go/regtest/input/template-1.csv b/go/regtest/input/template-1.csv new file mode 100644 index 000000000..d73c62bcf --- /dev/null +++ b/go/regtest/input/template-1.csv @@ -0,0 +1,2 @@ +x,y,a +4,5,6 diff --git a/go/regtest/input/template-2.csv b/go/regtest/input/template-2.csv new file mode 100644 index 000000000..748166eb4 --- /dev/null +++ b/go/regtest/input/template-2.csv @@ -0,0 +1,2 @@ +"x","y","a" +4,5,6 diff --git a/go/src/lib/readfiles.go b/go/src/lib/readfiles.go index 6c72e8882..878c5fd4f 100644 --- a/go/src/lib/readfiles.go +++ b/go/src/lib/readfiles.go @@ -6,6 +6,7 @@ package lib import ( + "encoding/csv" "io/ioutil" "os" "strings" @@ -72,3 +73,17 @@ func LoadStringsFromDir(dirname string, extension string) ([]string, error) { return dslStrings, nil } + +func ReadCSVHeader(filename string) ([]string, error) { + handle, err := os.Open(filename) + if err != nil { + return nil, err + } + defer handle.Close() + csvReader := csv.NewReader(handle) + header, err := csvReader.Read() + if err != nil { + return nil, err + } + return header, nil +} diff --git a/go/src/transformers/aaa_transformer_table.go b/go/src/transformers/aaa_transformer_table.go index 5b946ebb8..ad69510af 100644 --- a/go/src/transformers/aaa_transformer_table.go +++ b/go/src/transformers/aaa_transformer_table.go @@ -63,6 +63,7 @@ var TRANSFORMER_LOOKUP_TABLE = []TransformerSetup{ TacSetup, TailSetup, TeeSetup, + TemplateSetup, TopSetup, UnflattenSetup, UniqSetup, diff --git a/go/src/transformers/template.go b/go/src/transformers/template.go new file mode 100644 index 000000000..5c4244fc0 --- /dev/null +++ b/go/src/transformers/template.go @@ -0,0 +1,148 @@ +package transformers + +import ( + "fmt" + "os" + "strings" + + "miller/src/cliutil" + "miller/src/lib" + "miller/src/types" +) + +// ---------------------------------------------------------------- +const verbNameTemplate = "template" + +var TemplateSetup = TransformerSetup{ + Verb: verbNameTemplate, + UsageFunc: transformerTemplateUsage, + ParseCLIFunc: transformerTemplateParseCLI, + IgnoresInput: false, +} + +func transformerTemplateUsage( + o *os.File, + doExit bool, + exitCode int, +) { + fmt.Fprintf(o, "Usage: %s %s [options]\n", "mlr", verbNameTemplate) + fmt.Fprintf(o, "Places input-record fields in the order specified by list of column names.\n") + fmt.Fprintf(o, "If the input record is missing a specified field, it will be filled with the fill-with.\n") + fmt.Fprintf(o, "If the input record possesses an unspecified field, it will be discarded.\n") + fmt.Fprintf(o, "Options:\n") + fmt.Fprintf(o, " -f {a,b,c} Comma-separated field names for template, e.g. a,b,c.\n") + fmt.Fprintf(o, " -t {filename} CSV file whose header line will be used for template.\n") + fmt.Fprintf(o, "--fill-with {filler string} What to fill absent fields with. Defaults to the empty string.\n") + fmt.Fprintf(o, "-h|--help Show this message.\n") + fmt.Fprintf(o, "Example:\n") + fmt.Fprintf(o, "* Specified fields are a,b,c.\n") + fmt.Fprintf(o, "* Input record is c=3,a=1,f=6.\n") + fmt.Fprintf(o, "* Output record is a=1,b=,c=3.\n") + + if doExit { + os.Exit(exitCode) + } +} + +func transformerTemplateParseCLI( + pargi *int, + argc int, + args []string, + _ *cliutil.TOptions, +) IRecordTransformer { + + // Skip the verb name from the current spot in the mlr command line + argi := *pargi + verb := args[argi] + argi++ + + var fieldNames []string = nil + fillWith := "" + + for argi < argc /* variable increment: 1 or 2 depending on flag */ { + opt := args[argi] + if !strings.HasPrefix(opt, "-") { + break // No more flag options to process + } + argi++ + + if opt == "-h" || opt == "--help" { + transformerTemplateUsage(os.Stdout, true, 0) + + } else if opt == "-f" { + fieldNames = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc) + + } else if opt == "-t" { + templateFileName := cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + temp, err := lib.ReadCSVHeader(templateFileName) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fieldNames = temp + + } else if opt == "--fill-with" { + fillWith = cliutil.VerbGetStringArgOrDie(verb, opt, args, &argi, argc) + + } else { + transformerTemplateUsage(os.Stderr, true, 1) + } + } + + if fieldNames == nil { + transformerTemplateUsage(os.Stderr, true, 1) + } + + transformer, err := NewTransformerTemplate( + fieldNames, + fillWith, + ) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + *pargi = argi + return transformer +} + +// ---------------------------------------------------------------- +type TransformerTemplate struct { + fieldNameList []string + fieldNameSet map[string]bool + fillWith *types.Mlrval +} + +func NewTransformerTemplate( + fieldNames []string, + fillWith string, +) (*TransformerTemplate, error) { + return &TransformerTemplate{ + fieldNameList: fieldNames, + fieldNameSet: lib.StringListToSet(fieldNames), + fillWith: types.MlrvalPointerFromString(fillWith), + }, nil +} + +// ---------------------------------------------------------------- +func (tr *TransformerTemplate) Transform( + inrecAndContext *types.RecordAndContext, + outputChannel chan<- *types.RecordAndContext, +) { + if !inrecAndContext.EndOfStream { + inrec := inrecAndContext.Record + outrec := types.NewMlrmap() + for _, fieldName := range tr.fieldNameList { + value := inrec.Get(fieldName) + if value != nil { + outrec.PutReference(fieldName, value) // inrec will be GC'ed + } else { + outrec.PutCopy(fieldName, tr.fillWith) + } + } + outrecAndContext := types.NewRecordAndContext(outrec, &inrecAndContext.Context) + outputChannel <- outrecAndContext + } else { + outputChannel <- inrecAndContext + } +} diff --git a/go/todo.txt b/go/todo.txt index 420214b2e..ce5b79063 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -7,6 +7,8 @@ TOP OF LIST: * survey email followups * template verb +* quicklinks -- ? + * windows ./mlr hack * windows &&/|| debug @@ -29,6 +31,10 @@ DOC6 o no out-of-core sort yet ... o older versions +* number-formatting page + +* check <-> manpage/olh sections & doc6 sections + ! fill out: o new-in-miller-6: flatten/unflatten o JSON flatten/unflatten section @@ -48,8 +54,6 @@ DOC6 o stats1 --fr cookbook2.rst.in o file-formats: colon -> dot - o unset multiple lvalues in the BNF - faq.rst.in o list at end of todo.txt o file-formats.rst.in & olh: dkvp not first anymore @@ -132,6 +136,8 @@ w survey * headerful/headerless mix -- ? TOptions as list, not single -- ? +* uniqify-field-names in record-readers -- which issue? + * non-blocker: read-from-URL support -- ? * non-blocker: _ variable feature?