mlr unsparsify

c/cli/mlrcli.c

@@ -74,6 +74,7 @@ static mapper_setup_t* mapper_lookup_table[] = {
 	&mapper_tee_setup,
 	&mapper_top_setup,
 	&mapper_uniq_setup,
+	&mapper_unsparsify_setup,
 };
 static int mapper_lookup_table_length = sizeof(mapper_lookup_table) / sizeof(mapper_lookup_table[0]);
 

c/mapping/Makefile.am

@@ -38,6 +38,7 @@ libmapping_la_SOURCES=	\
 			mapper_tee.c \
 			mapper_top.c \
 			mapper_uniq.c \
+			mapper_unsparsify.c \
 			mappers.c \
 			mappers.h \
 			stats1_accumulators.c \

c/mapping/mapper_unsparsify.c

@@ -0,0 +1,109 @@
+#include <stdio.h>
+#include "lib/mlrutil.h"
+#include "containers/lhmsi.h"
+#include "containers/sllv.h"
+#include "mapping/mappers.h"
+
+typedef struct _mapper_unsparsify_state_t {
+	lhmsi_t* key_names;
+	sllv_t* records;
+} mapper_unsparsify_state_t;
+
+static void      mapper_unsparsify_usage(FILE* o, char* argv0, char* verb);
+static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv,
+	cli_reader_opts_t* _, cli_writer_opts_t* __);
+static mapper_t* mapper_unsparsify_alloc();
+static void      mapper_unsparsify_free(mapper_t* pmapper, context_t* _);
+static sllv_t*   mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void* pvstate);
+
+// ----------------------------------------------------------------
+mapper_setup_t mapper_unsparsify_setup = {
+	.verb = "unsparsify",
+	.pusage_func = mapper_unsparsify_usage,
+	.pparse_func = mapper_unsparsify_parse_cli,
+	.ignores_input = FALSE,
+};
+
+// ----------------------------------------------------------------
+static void mapper_unsparsify_usage(FILE* o, char* argv0, char* verb) {
+	fprintf(o, "Usage: %s %s\n", argv0, verb);
+	fprintf(o, "Prints records with the union of field names over all input records.\n");
+	fprintf(o, "For field names absent in a given record but present in others, fills value with\n");
+	fprintf(o, "empty string. This verb retains all input before producing any output.\n");
+}
+
+static mapper_t* mapper_unsparsify_parse_cli(int* pargi, int argc, char** argv,
+	cli_reader_opts_t* _, cli_writer_opts_t* __)
+{
+	if ((argc - *pargi) < 1) {
+		mapper_unsparsify_usage(stderr, argv[0], argv[*pargi]);
+		return NULL;
+	}
+	mapper_t* pmapper = mapper_unsparsify_alloc();
+	*pargi += 1;
+	return pmapper;
+}
+
+// ----------------------------------------------------------------
+static mapper_t* mapper_unsparsify_alloc() {
+	mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
+
+	mapper_unsparsify_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_unsparsify_state_t));
+	pstate->records = sllv_alloc();
+	pstate->key_names = lhmsi_alloc();
+
+	pmapper->pvstate       = pstate;
+	pmapper->pprocess_func = mapper_unsparsify_process;
+	pmapper->pfree_func    = mapper_unsparsify_free;
+
+	return pmapper;
+}
+
+static void mapper_unsparsify_free(mapper_t* pmapper, context_t* _) {
+	mapper_unsparsify_state_t* pstate = pmapper->pvstate;
+	// Free the container
+	sllv_free(pstate->records);
+	lhmsi_free(pstate->key_names);
+	free(pstate);
+	free(pmapper);
+}
+
+// ----------------------------------------------------------------
+static sllv_t* mapper_unsparsify_process(lrec_t* pinrec, context_t* pctx, void* pvstate) {
+	mapper_unsparsify_state_t* pstate = pvstate;
+	if (pinrec != NULL) {
+		// Not end of stream.
+		for (lrece_t* pe = pinrec->phead; pe != NULL; pe = pe->pnext) {
+			if (!lhmsi_has_key(pstate->key_names, pe->key)) {
+				lhmsi_put(pstate->key_names, mlr_strdup_or_die(pe->key), 1, FREE_ENTRY_KEY);
+			}
+		}
+		// The caller will free the outrecs
+		sllv_append(pstate->records, pinrec);
+		return NULL;
+	}
+	else {
+		// End of stream.
+		sllv_t* poutrecs = sllv_alloc();
+		for (sllve_t* pe = pstate->records->phead; pe != NULL; pe = pe->pnext) {
+			lrec_t* pinrec = pe->pvvalue;
+			lrec_t* poutrec = lrec_unbacked_alloc();
+			for (lhmsie_t* pf = pstate->key_names->phead; pf != NULL; pf = pf->pnext) {
+				char* key = pf->key;
+				char* value = lrec_get(pinrec, key);
+				if (value == NULL) {
+					lrec_put(poutrec, mlr_strdup_or_die(key), "", FREE_ENTRY_KEY);
+				} else {
+					lrec_put(poutrec, mlr_strdup_or_die(key), mlr_strdup_or_die(value),
+						FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
+				}
+			}
+			sllv_append(poutrecs, poutrec);
+			// Free the void-star payload
+			lrec_free(pinrec);
+		}
+
+		sllv_append(poutrecs, NULL);
+		return poutrecs;
+	}
+}

c/mapping/mappers.h

@@ -45,6 +45,7 @@ extern mapper_setup_t mapper_tail_setup;
 extern mapper_setup_t mapper_tee_setup;
 extern mapper_setup_t mapper_top_setup;
 extern mapper_setup_t mapper_uniq_setup;
+extern mapper_setup_t mapper_unsparsify_setup;
 
 // Construction is in mlrcli.c.
 void mapper_chain_free(sllv_t* pmapper_chain, context_t* pctx);

c/reg_test/expected/out

@@ -524,6 +524,32 @@ a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
 
 mlr tac /dev/null
 
+mlr --opprint unsparsify ./reg_test/input/abixy
+a   b   i  x                   y
+pan pan 1  0.3467901443380824  0.7268028627434533
+eks pan 2  0.7586799647899636  0.5221511083334797
+wye wye 3  0.20460330576630303 0.33831852551664776
+eks wye 4  0.38139939387114097 0.13418874328430463
+wye pan 5  0.5732889198020006  0.8636244699032729
+zee pan 6  0.5271261600918548  0.49322128674835697
+eks zee 7  0.6117840605678454  0.1878849191181694
+zee wye 8  0.5985540091064224  0.976181385699006
+hat wye 9  0.03144187646093577 0.7495507603507059
+pan wye 10 0.5026260055412137  0.9526183602969864
+
+mlr --opprint unsparsify ./reg_test/input/abixy-het
+a   b   i  x                   y                   aaa bbb xxx                iii yyy
+pan pan 1  0.3467901443380824  0.7268028627434533  -   -   -                  -   -
+eks pan 2  0.7586799647899636  0.5221511083334797  -   -   -                  -   -
+-   wye 3  0.20460330576630303 0.33831852551664776 wye -   -                  -   -
+eks -   4  0.38139939387114097 0.13418874328430463 -   wye -                  -   -
+wye pan 5  -                   0.8636244699032729  -   -   0.5732889198020006 -   -
+zee pan 6  0.5271261600918548  0.49322128674835697 -   -   -                  -   -
+eks zee -  0.6117840605678454  0.1878849191181694  -   -   -                  7   -
+zee wye 8  0.5985540091064224  -                   -   -   -                  -   0.976181385699006
+-   -   9  0.03144187646093577 0.7495507603507059  hat wye -                  -   -
+pan wye 10 0.5026260055412137  0.9526183602969864  -   -   -                  -   -
+
 
 ================================================================
 HEAD/TAIL/ETC.
@@ -50353,7 +50379,7 @@ mlr -n put -q -f ./reg_test/input/sieve.mlr
 89
 97
 
-mlr -n put -q -f ./reg_test/input/mand.mlr
+mlr -n put -q -f ./reg_test/input/mand.mlr -e begin {@verbose = true}
 RCORN     = -2.000000
 ICORN     = -2.000000
 SIDE      = 4.000000

c/reg_test/run

@@ -189,6 +189,9 @@ run_mlr group-like $indir/het.dkvp
 run_mlr tac $indir/abixy
 run_mlr tac /dev/null
 
+run_mlr --opprint unsparsify $indir/abixy
+run_mlr --opprint unsparsify $indir/abixy-het
+
 # ----------------------------------------------------------------
 announce HEAD/TAIL/ETC.
 

c/todo.txt

@@ -15,6 +15,8 @@ BUGFIXES
 * re-valgrinds after UT expansion
 * release notes
 
+* mlr unsparse
+
 ! for (k, m in nested) { ... } -- this is confusing when m is non-terminal.
 
 * mlr hex -> auxes. and make an aux-list. and doc: olh as well as webdoc?

doc/content-for-reference-verbs.html

@@ -795,3 +795,15 @@ POKI_RUN_COMMAND{{mlr --opprint uniq -n -g color,shape data/colored-shapes.dkvp}
 </td></tr></table>
 
 </div>
+<!-- ================================================================ -->
+<h1>unsparsify</h1>
+<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_unsparsify');" href="javascript:;">Toggle section visibility</button>
+<div id="section_toggle_unsparsify" style="display: block">
+
+POKI_RUN_COMMAND{{mlr unsparsify --help}}HERE
+
+POKI_RUN_COMMAND{{cat data/sparse.json}}HERE
+POKI_RUN_COMMAND{{mlr --json unsparsify data/sparse.json}}HERE
+POKI_RUN_COMMAND{{mlr --ijson --opprint unsparsify data/sparse.json}}HERE
+
+</div>

doc/cookbook.html

@@ -1191,7 +1191,6 @@ end {
 </pre>
 </div>
 <p/>
-
 <p/>
 <div class="pokipanel">
 <pre>

doc/manpage.html

@@ -316,7 +316,7 @@ OPTIONS
 	group-like having-fields head histogram join label least-frequent merge-fields
 	most-frequent nest nothing put regularize rename reorder repeat reshape sample
 	sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step tac tail tee top
-	uniq
+	uniq unsparsify
 
    FUNCTION LIST
 	+ + - - * / // % ** | ^ & ~ << >> == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub
@@ -1291,6 +1291,12 @@ VERBS
        Prints distinct values for specified field names. With -c, same as
        count-distinct. For uniq, -f is a synonym for -g.
 
+   unsparsify
+       Usage: mlr unsparsify
+       Prints records with the union of field names over all input records.
+       For field names absent in a given record but present in others, fills value with
+       empty string. This verb retains all input before producing any output.
+
 FUNCTIONS FOR FILTER/PUT
    +
        (class=arithmetic #args=2): Addition.
@@ -2055,7 +2061,7 @@ SEE ALSO
 
 
 
-				  2017-02-21			     MILLER(1)
+				  2017-02-24			     MILLER(1)
 </pre>
 </div>
 <p/>

doc/manpage.txt

@@ -124,7 +124,7 @@ OPTIONS
 	group-like having-fields head histogram join label least-frequent merge-fields
 	most-frequent nest nothing put regularize rename reorder repeat reshape sample
 	sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step tac tail tee top
-	uniq
+	uniq unsparsify
 
    FUNCTION LIST
 	+ + - - * / // % ** | ^ & ~ << >> == != =~ !=~ > >= < <= && || ^^ ! ? : . gsub
@@ -1099,6 +1099,12 @@ VERBS
        Prints distinct values for specified field names. With -c, same as
        count-distinct. For uniq, -f is a synonym for -g.
 
+   unsparsify
+       Usage: mlr unsparsify
+       Prints records with the union of field names over all input records.
+       For field names absent in a given record but present in others, fills value with
+       empty string. This verb retains all input before producing any output.
+
 FUNCTIONS FOR FILTER/PUT
    +
        (class=arithmetic #args=2): Addition.
@@ -1863,4 +1869,4 @@ SEE ALSO
 
 
 
-				  2017-02-21			     MILLER(1)
+				  2017-02-24			     MILLER(1)

doc/mlr.1

@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2017-02-21
+.\"      Date: 2017-02-24
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2017-02-21" "\ \&" "\ \&"
+.TH "MILLER" "1" "2017-02-24" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -167,7 +167,7 @@ output separator to the given value.
  group-like having-fields head histogram join label least-frequent merge-fields
  most-frequent nest nothing put regularize rename reorder repeat reshape sample
  sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step tac tail tee top
- uniq
+ uniq unsparsify
 .fi
 .if n \{\
 .RE
@@ -1452,6 +1452,18 @@ count-distinct. For uniq, -f is a synonym for -g.
 .fi
 .if n \{\
 .RE
+.SS "unsparsify"
+.if n \{\
+.RS 0
+.\}
+.nf
+Usage: mlr unsparsify
+Prints records with the union of field names over all input records.
+For field names absent in a given record but present in others, fills value with
+empty string. This verb retains all input before producing any output.
+.fi
+.if n \{\
+.RE
 .SH "FUNCTIONS FOR FILTER/PUT"
 .sp
 

doc/reference-verbs.html

@@ -229,6 +229,7 @@ Miller commands were run with pretty-print-tabular output format.
 &bull;&nbsp;<a href="#tee">tee</a><br/>
 &bull;&nbsp;<a href="#top">top</a><br/>
 &bull;&nbsp;<a href="#uniq">uniq</a><br/>
+&bull;&nbsp;<a href="#unsparsify">unsparsify</a><br/>
 </div>
 <p/>
 
@@ -3379,6 +3380,59 @@ count
 <p/>
 </td></tr></table>
 
+</div>
+<!-- ================================================================ -->
+<a id="unsparsify"/><h1>unsparsify</h1>
+<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_unsparsify');" href="javascript:;">Toggle section visibility</button>
+<div id="section_toggle_unsparsify" style="display: block">
+
+<p/>
+<div class="pokipanel">
+<pre>
+$ mlr unsparsify --help
+Usage: mlr unsparsify
+Prints records with the union of field names over all input records.
+For field names absent in a given record but present in others, fills value with
+empty string. This verb retains all input before producing any output.
+</pre>
+</div>
+<p/>
+
+<p/>
+<div class="pokipanel">
+<pre>
+$ cat data/sparse.json
+{"a":1,"b":2,"v":3}
+{"u":1,"b":2}
+{"a":1,"v":2,"x":3}
+{"v":1,"w":2}
+</pre>
+</div>
+<p/>
+<p/>
+<div class="pokipanel">
+<pre>
+$ mlr --json unsparsify data/sparse.json
+{ "a": 1, "b": 2, "v": 3, "u": "", "x": "", "w": "" }
+{ "a": "", "b": 2, "v": "", "u": 1, "x": "", "w": "" }
+{ "a": 1, "b": "", "v": 2, "u": "", "x": 3, "w": "" }
+{ "a": "", "b": "", "v": 1, "u": "", "x": "", "w": 2 }
+</pre>
+</div>
+<p/>
+<p/>
+<div class="pokipanel">
+<pre>
+$ mlr --ijson --opprint unsparsify data/sparse.json
+a b v u x w
+1 2 3 - - -
+- 2 - 1 - -
+1 - 2 - 3 -
+- - 1 - - 2
+</pre>
+</div>
+<p/>
+
 </div>
     </div>
   </td>

doc/reference.html

@@ -1289,7 +1289,7 @@ Verbs:
    group-like having-fields head histogram join label least-frequent
    merge-fields most-frequent nest nothing put regularize rename reorder repeat
    reshape sample sec2gmt sec2gmtdate seqgen shuffle sort stats1 stats2 step
-   tac tail tee top uniq
+   tac tail tee top uniq unsparsify
 
 Functions for the filter and put verbs:
    + + - - * / // % ** | ^ & ~ << >> == != =~ !=~ > >= < <= && || ^^ ! ? : .