From e2bfd015db664e1e06f838054b1af3cd14eccc2e Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 23 Feb 2017 21:45:10 -0800 Subject: [PATCH] mlr unsparsify --- c/cli/mlrcli.c | 1 + c/mapping/Makefile.am | 1 + c/mapping/mapper_unsparsify.c | 109 +++++++++++++++++++++++++++ c/mapping/mappers.h | 1 + c/reg_test/expected/out | 28 ++++++- c/reg_test/run | 3 + c/todo.txt | 2 + doc/content-for-reference-verbs.html | 12 +++ doc/cookbook.html | 1 - doc/manpage.html | 10 ++- doc/manpage.txt | 10 ++- doc/mlr.1 | 18 ++++- doc/reference-verbs.html | 54 +++++++++++++ doc/reference.html | 2 +- 14 files changed, 242 insertions(+), 10 deletions(-) create mode 100644 c/mapping/mapper_unsparsify.c diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index ed1389440..1df4aeda0 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -74,6 +74,7 @@ static mapper_setup_t* mapper_lookup_table[] = { &mapper_tee_setup, &mapper_top_setup, &mapper_uniq_setup, + &mapper_unsparsify_setup, }; static int mapper_lookup_table_length = sizeof(mapper_lookup_table) / sizeof(mapper_lookup_table[0]); diff --git a/c/mapping/Makefile.am b/c/mapping/Makefile.am index a97aac796..9b12bbd37 100644 --- a/c/mapping/Makefile.am +++ b/c/mapping/Makefile.am @@ -38,6 +38,7 @@ libmapping_la_SOURCES= \ mapper_tee.c \ mapper_top.c \ mapper_uniq.c \ + mapper_unsparsify.c \ mappers.c \ mappers.h \ stats1_accumulators.c \ diff --git a/c/mapping/mapper_unsparsify.c b/c/mapping/mapper_unsparsify.c new file mode 100644 index 000000000..627b78db3 --- /dev/null +++ b/c/mapping/mapper_unsparsify.c @@ -0,0 +1,109 @@ +#include +#include "lib/mlrutil.h" +#include "containers/lhmsi.h" +#include "containers/sllv.h" +#include "mapping/mappers.h" + +typedef struct _mapper_unsparsify_state_t { + lhmsi_t* key_names; + sllv_t* records; +} mapper_unsparsify_state_t; + +static void mapper_unsparsify_usage(FILE* o, char* argv0, char* verb); +static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv, + cli_reader_opts_t* _, cli_writer_opts_t* __); +static mapper_t* mapper_unsparsify_alloc(); +static void mapper_unsparsify_free(mapper_t* pmapper, context_t* _); +static sllv_t* mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void* pvstate); + +// ---------------------------------------------------------------- +mapper_setup_t mapper_unsparsify_setup = { + .verb = "unsparsify", + .pusage_func = mapper_unsparsify_usage, + .pparse_func = mapper_unsparsify_parse_cli, + .ignores_input = FALSE, +}; + +// ---------------------------------------------------------------- +static void mapper_unsparsify_usage(FILE* o, char* argv0, char* verb) { + fprintf(o, "Usage: %s %s\n", argv0, verb); + fprintf(o, "Prints records with the union of field names over all input records.\n"); + fprintf(o, "For field names absent in a given record but present in others, fills value with\n"); + fprintf(o, "empty string. This verb retains all input before producing any output.\n"); +} + +static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv, + cli_reader_opts_t* _, cli_writer_opts_t* __) +{ + if ((argc - *pargi) < 1) { + mapper_unsparsify_usage(stderr, argv[0], argv[*pargi]); + return NULL; + } + mapper_t* pmapper = mapper_unsparsify_alloc(); + *pargi += 1; + return pmapper; +} + +// ---------------------------------------------------------------- +static mapper_t* mapper_unsparsify_alloc() { + mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); + + mapper_unsparsify_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_unsparsify_state_t)); + pstate->records = sllv_alloc(); + pstate->key_names = lhmsi_alloc(); + + pmapper->pvstate = pstate; + pmapper->pprocess_func = mapper_unsparsify_process; + pmapper->pfree_func = mapper_unsparsify_free; + + return pmapper; +} + +static void mapper_unsparsify_free(mapper_t* pmapper, context_t* _) { + mapper_unsparsify_state_t* pstate = pmapper->pvstate; + // Free the container + sllv_free(pstate->records); + lhmsi_free(pstate->key_names); + free(pstate); + free(pmapper); +} + +// ---------------------------------------------------------------- +static sllv_t* mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { + mapper_unsparsify_state_t* pstate = pvstate; + if (pinrec != NULL) { + // Not end of stream. + for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) { + if (!lhmsi_has_key(pstate->key_names, pe->key)) { + lhmsi_put(pstate->key_names, mlr_strdup_or_die(pe->key), 1, FREE_ENTRY_KEY); + } + } + // The caller will free the outrecs + sllv_append(pstate->records, pinrec); + return NULL; + } + else { + // End of stream. + sllv_t* poutrecs = sllv_alloc(); + for (sllve_t* pe = pstate->records->phead; pe != NULL; pe = pe->pnext) { + lrec_t* pinrec = pe->pvvalue; + lrec_t* poutrec = lrec_unbacked_alloc(); + for (lhmsie_t* pf = pstate->key_names->phead; pf != NULL; pf = pf->pnext) { + char* key = pf->key; + char* value = lrec_get(pinrec, key); + if (value == NULL) { + lrec_put(poutrec, mlr_strdup_or_die(key), "", FREE_ENTRY_KEY); + } else { + lrec_put(poutrec, mlr_strdup_or_die(key), mlr_strdup_or_die(value), + FREE_ENTRY_KEY|FREE_ENTRY_VALUE); + } + } + sllv_append(poutrecs, poutrec); + // Free the void-star payload + lrec_free(pinrec); + } + + sllv_append(poutrecs, NULL); + return poutrecs; + } +} diff --git a/c/mapping/mappers.h b/c/mapping/mappers.h index da2d72a48..8ef76174b 100644 --- a/c/mapping/mappers.h +++ b/c/mapping/mappers.h @@ -45,6 +45,7 @@ extern mapper_setup_t mapper_tail_setup; extern mapper_setup_t mapper_tee_setup; extern mapper_setup_t mapper_top_setup; extern mapper_setup_t mapper_uniq_setup; +extern mapper_setup_t mapper_unsparsify_setup; // Construction is in mlrcli.c. void mapper_chain_free(sllv_t* pmapper_chain, context_t* pctx); diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index 3dc5cc61a..eb1db9ba0 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -524,6 +524,32 @@ a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533 mlr tac /dev/null +mlr --opprint unsparsify ./reg_test/input/abixy +a b i x y +pan pan 1 0.3467901443380824 0.7268028627434533 +eks pan 2 0.7586799647899636 0.5221511083334797 +wye wye 3 0.20460330576630303 0.33831852551664776 +eks wye 4 0.38139939387114097 0.13418874328430463 +wye pan 5 0.5732889198020006 0.8636244699032729 +zee pan 6 0.5271261600918548 0.49322128674835697 +eks zee 7 0.6117840605678454 0.1878849191181694 +zee wye 8 0.5985540091064224 0.976181385699006 +hat wye 9 0.03144187646093577 0.7495507603507059 +pan wye 10 0.5026260055412137 0.9526183602969864 + +mlr --opprint unsparsify ./reg_test/input/abixy-het +a b i x y aaa bbb xxx iii yyy +pan pan 1 0.3467901443380824 0.7268028627434533 - - - - - +eks pan 2 0.7586799647899636 0.5221511083334797 - - - - - +- wye 3 0.20460330576630303 0.33831852551664776 wye - - - - +eks - 4 0.38139939387114097 0.13418874328430463 - wye - - - +wye pan 5 - 0.8636244699032729 - - 0.5732889198020006 - - +zee pan 6 0.5271261600918548 0.49322128674835697 - - - - - +eks zee - 0.6117840605678454 0.1878849191181694 - - - 7 - +zee wye 8 0.5985540091064224 - - - - - 0.976181385699006 +- - 9 0.03144187646093577 0.7495507603507059 hat wye - - - +pan wye 10 0.5026260055412137 0.9526183602969864 - - - - - + ================================================================ HEAD/TAIL/ETC. @@ -50353,7 +50379,7 @@ mlr -n put -q -f ./reg_test/input/sieve.mlr 89 97 -mlr -n put -q -f ./reg_test/input/mand.mlr +mlr -n put -q -f ./reg_test/input/mand.mlr -e begin {@verbose = true} RCORN = -2.000000 ICORN = -2.000000 SIDE = 4.000000 diff --git a/c/reg_test/run b/c/reg_test/run index e2f507b04..89fddbb46 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -189,6 +189,9 @@ run_mlr group-like $indir/het.dkvp run_mlr tac $indir/abixy run_mlr tac /dev/null +run_mlr --opprint unsparsify $indir/abixy +run_mlr --opprint unsparsify $indir/abixy-het + # ---------------------------------------------------------------- announce HEAD/TAIL/ETC. diff --git a/c/todo.txt b/c/todo.txt index 07a6df083..dcaf12ed2 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -15,6 +15,8 @@ BUGFIXES * re-valgrinds after UT expansion * release notes +* mlr unsparse + ! for (k, m in nested) { ... } -- this is confusing when m is non-terminal. * mlr hex -> auxes. and make an aux-list. and doc: olh as well as webdoc? diff --git a/doc/content-for-reference-verbs.html b/doc/content-for-reference-verbs.html index b13e833ee..46e09388a 100644 --- a/doc/content-for-reference-verbs.html +++ b/doc/content-for-reference-verbs.html @@ -795,3 +795,15 @@ POKI_RUN_COMMAND{{mlr --opprint uniq -n -g color,shape data/colored-shapes.dkvp} + +

unsparsify

+ +
+ +POKI_RUN_COMMAND{{mlr unsparsify --help}}HERE + +POKI_RUN_COMMAND{{cat data/sparse.json}}HERE +POKI_RUN_COMMAND{{mlr --json unsparsify data/sparse.json}}HERE +POKI_RUN_COMMAND{{mlr --ijson --opprint unsparsify data/sparse.json}}HERE + +
diff --git a/doc/cookbook.html b/doc/cookbook.html index 18be765cf..928e887ea 100644 --- a/doc/cookbook.html +++ b/doc/cookbook.html @@ -1191,7 +1191,6 @@ end {

- 

diff --git a/doc/manpage.html b/doc/manpage.html
index 859998ea4..14b0f238d 100644
--- a/doc/manpage.html
+++ b/doc/manpage.html
@@ -316,7 +316,7 @@ OPTIONS
 	group-like having-fields head histogram join label least-frequent merge-fields
 	most-frequent nest nothing put regularize rename reorder repeat reshape sample
 	sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step tac tail tee top
-	uniq
+	uniq unsparsify
 
    FUNCTION LIST
 	+ + - - * / // % ** | ^ & ~ << >> == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub
@@ -1291,6 +1291,12 @@ VERBS
        Prints distinct values for specified field names. With -c, same as
        count-distinct. For uniq, -f is a synonym for -g.
 
+   unsparsify
+       Usage: mlr unsparsify
+       Prints records with the union of field names over all input records.
+       For field names absent in a given record but present in others, fills value with
+       empty string. This verb retains all input before producing any output.
+
 FUNCTIONS FOR FILTER/PUT
    +
        (class=arithmetic #args=2): Addition.
@@ -2055,7 +2061,7 @@ SEE ALSO
 
 
 
-				  2017-02-21			     MILLER(1)
+				  2017-02-24			     MILLER(1)

diff --git a/doc/manpage.txt b/doc/manpage.txt index 8a74cc89e..2e424a711 100644 --- a/doc/manpage.txt +++ b/doc/manpage.txt @@ -124,7 +124,7 @@ OPTIONS group-like having-fields head histogram join label least-frequent merge-fields most-frequent nest nothing put regularize rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step tac tail tee top - uniq + uniq unsparsify FUNCTION LIST + + - - * / // % ** | ^ & ~ << >> == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub @@ -1099,6 +1099,12 @@ VERBS Prints distinct values for specified field names. With -c, same as count-distinct. For uniq, -f is a synonym for -g. + unsparsify + Usage: mlr unsparsify + Prints records with the union of field names over all input records. + For field names absent in a given record but present in others, fills value with + empty string. This verb retains all input before producing any output. + FUNCTIONS FOR FILTER/PUT + (class=arithmetic #args=2): Addition. @@ -1863,4 +1869,4 @@ SEE ALSO - 2017-02-21 MILLER(1) + 2017-02-24 MILLER(1) diff --git a/doc/mlr.1 b/doc/mlr.1 index dccb4a6a6..a2dcb32fa 100644 --- a/doc/mlr.1 +++ b/doc/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2017-02-21 +.\" Date: 2017-02-24 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2017-02-21" "\ \&" "\ \&" +.TH "MILLER" "1" "2017-02-24" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -167,7 +167,7 @@ output separator to the given value. group-like having-fields head histogram join label least-frequent merge-fields most-frequent nest nothing put regularize rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step tac tail tee top - uniq + uniq unsparsify .fi .if n \{\ .RE @@ -1452,6 +1452,18 @@ count-distinct. For uniq, -f is a synonym for -g. .fi .if n \{\ .RE +.SS "unsparsify" +.if n \{\ +.RS 0 +.\} +.nf +Usage: mlr unsparsify +Prints records with the union of field names over all input records. +For field names absent in a given record but present in others, fills value with +empty string. This verb retains all input before producing any output. +.fi +.if n \{\ +.RE .SH "FUNCTIONS FOR FILTER/PUT" .sp diff --git a/doc/reference-verbs.html b/doc/reference-verbs.html index 86ffd1284..d68100e4d 100644 --- a/doc/reference-verbs.html +++ b/doc/reference-verbs.html @@ -229,6 +229,7 @@ Miller commands were run with pretty-print-tabular output format. • tee
• top
• uniq
+• unsparsify

@@ -3379,6 +3380,59 @@ count

+ + +

unsparsify

+ +
+ +

+

+
+$ mlr unsparsify --help
+Usage: mlr unsparsify
+Prints records with the union of field names over all input records.
+For field names absent in a given record but present in others, fills value with
+empty string. This verb retains all input before producing any output.
+
+
+

+ +

+

+
+$ cat data/sparse.json
+{"a":1,"b":2,"v":3}
+{"u":1,"b":2}
+{"a":1,"v":2,"x":3}
+{"v":1,"w":2}
+
+
+

+

+

+
+$ mlr --json unsparsify data/sparse.json
+{ "a": 1, "b": 2, "v": 3, "u": "", "x": "", "w": "" }
+{ "a": "", "b": 2, "v": "", "u": 1, "x": "", "w": "" }
+{ "a": 1, "b": "", "v": 2, "u": "", "x": 3, "w": "" }
+{ "a": "", "b": "", "v": 1, "u": "", "x": "", "w": 2 }
+
+
+

+

+

+
+$ mlr --ijson --opprint unsparsify data/sparse.json
+a b v u x w
+1 2 3 - - -
+- 2 - 1 - -
+1 - 2 - 3 -
+- - 1 - - 2
+
+
+

+

diff --git a/doc/reference.html b/doc/reference.html index 4a5191d85..5d01c8b04 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -1289,7 +1289,7 @@ Verbs: group-like having-fields head histogram join label least-frequent merge-fields most-frequent nest nothing put regularize rename reorder repeat reshape sample sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step - tac tail tee top uniq + tac tail tee top uniq unsparsify Functions for the filter and put verbs: + + - - * / // % ** | ^ & ~ << >> == != =~ !=~ > >= < <= && || ^^ ! ? : .