shuffle iterate

c/cli/mlrcli.c

@@ -47,6 +47,7 @@ static mapper_setup_t* mapper_lookup_table[] = {
 	&mapper_reorder_setup,
 	&mapper_reshape_setup,
 	&mapper_sample_setup,
+	&mapper_shuffle_setup,
 	&mapper_sec2gmt_setup,
 	&mapper_sort_setup,
 	&mapper_stats1_setup,

c/containers/mlrval.h

@@ -293,6 +293,7 @@ static inline mv_t f_fff_logifit_func(mv_t* pval1, mv_t* pval2, mv_t* pval3) {
 	return mv_from_float(1.0 / (1.0 + exp(-m*x-b)));
 }
 
+// xxx libify
 static inline mv_t i_ii_urandint_func(mv_t* pval1, mv_t* pval2) {
 	long long a = pval1->u.intv;
 	long long b = pval2->u.intv;

c/mapping/Makefile.am

@@ -27,6 +27,7 @@ libmapping_la_SOURCES=	\
 			mapper_reshape.c \
 			mapper_sample.c \
 			mapper_sec2gmt.c \
+			mapper_shuffle.c \
 			mapper_sort.c \
 			mapper_stats1.c \
 			mapper_stats2.c \

c/mapping/mapper_nest.c

@@ -39,6 +39,12 @@
 typedef struct _mapper_nest_state_t {
 	ap_state_t* pargp;
 
+	// -e/-i/--explode/--implode: required: -- via process func
+	// --values/--pairs: required:
+	// --fields/--records: streaming & non
+	// --nested-fs: required
+	// --nested-fp: required
+
 	// for wide-to-long:
 	slls_t* input_field_names;
 	sllv_t* input_field_regexes;

c/mapping/mapper_shuffle.c

@@ -0,0 +1,134 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "lib/mlrutil.h"
+#include "lib/mtrand.h"
+#include "containers/sllv.h"
+#include "containers/slls.h"
+#include "containers/lhmslv.h"
+#include "containers/lhmsv.h"
+#include "containers/mixutil.h"
+#include "mapping/mappers.h"
+#include "cli/argparse.h"
+
+// ----------------------------------------------------------------
+typedef struct _mapper_shuffle_state_t {
+	ap_state_t* pargp;
+	sllv_t*     precs;
+} mapper_shuffle_state_t;
+
+static void      mapper_shuffle_usage(FILE* o, char* argv0, char* verb);
+static mapper_t* mapper_shuffle_parse_cli(int* pargi, int argc, char** argv);
+static mapper_t* mapper_shuffle_alloc(ap_state_t* pargp);
+static void      mapper_shuffle_free(mapper_t* pmapper);
+static sllv_t*   mapper_shuffle_process(lrec_t* pinrec, context_t* pctx, void* pvstate);
+
+// ----------------------------------------------------------------
+mapper_setup_t mapper_shuffle_setup = {
+	.verb = "shuffle",
+	.pusage_func = mapper_shuffle_usage,
+	.pparse_func = mapper_shuffle_parse_cli
+};
+
+// ----------------------------------------------------------------
+static void mapper_shuffle_usage(FILE* o, char* argv0, char* verb) {
+	fprintf(o, "Usage: %s %s [options]\n", argv0, verb);
+	fprintf(o, "-- xxx under construction --\n");
+}
+
+static mapper_t* mapper_shuffle_parse_cli(int* pargi, int argc, char** argv) {
+	char* verb = argv[(*pargi)++];
+
+	ap_state_t* pstate = ap_alloc();
+
+	if (!ap_parse(pstate, verb, pargi, argc, argv)) {
+		mapper_shuffle_usage(stderr, argv[0], verb);
+		return NULL;
+	}
+
+	return mapper_shuffle_alloc(pstate);
+}
+
+// ----------------------------------------------------------------
+static mapper_t* mapper_shuffle_alloc(ap_state_t* pargp) {
+	mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
+
+	mapper_shuffle_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_shuffle_state_t));
+
+	pstate->pargp = pargp;
+	pstate->precs = sllv_alloc();
+
+	pmapper->pvstate       = pstate;
+	pmapper->pprocess_func = mapper_shuffle_process;
+	pmapper->pfree_func    = mapper_shuffle_free;
+
+	return pmapper;
+}
+
+static void mapper_shuffle_free(mapper_t* pmapper) {
+	mapper_shuffle_state_t* pstate = pmapper->pvstate;
+	// Records will have been freed by the emitter; here, free the list structure.
+	sllv_free(pstate->precs);
+	ap_free(pstate->pargp);
+	free(pstate);
+	free(pmapper);
+}
+
+// ----------------------------------------------------------------
+static sllv_t* mapper_shuffle_process(lrec_t* pinrec, context_t* pctx, void* pvstate) {
+	mapper_shuffle_state_t* pstate = pvstate;
+
+	// Not end of input stream: retain the record, and emit nothing until end of stream.
+	if (pinrec != NULL) {
+		sllv_append(pstate->precs, pinrec);
+		return NULL;
+	}
+
+	sllv_t* poutrecs = sllv_alloc();
+
+	// Knuth shuffle:
+	// * Initial permutation is identity.
+	// * Make a pseudorandom permutation using pseudorandom swaps in the image map.
+	int n = pstate->precs->length;
+	int* images = mlr_malloc_or_die(n * sizeof(int));
+	for (int i = 0; i < n; i++)
+		images[i] = i;
+
+	int unused_start = 0;
+	int num_unused   = n;
+	for (int i = 0; i < n; i++) {
+		// Select a pseudorandom element from the pool of unused images.
+		int u = unused_start + num_unused * get_mtrand_double();
+		int temp  = images[u];
+		images[u] = images[i];
+		images[i] = temp;
+
+		// Decrease the size of the pool by 1.  (Yes, unused_start and k always have the same value.
+		// Using two variables wastes neglible memory and makes the code easier to understand.)
+		unused_start++;
+		num_unused--;
+	}
+
+	// Make an array of pointers into the input list.
+	lrec_t** record_array = mlr_malloc_or_die(n * sizeof(lrec_t**));
+	sllve_t* pe = pstate->precs->phead;
+	for (int i = 0; i < n; i++, pe = pe->pnext) {
+		record_array[i] = pe->pvvalue;
+	}
+
+	// Transfer from input array to output list. Because permutations are one-to-one maps,
+	// all input records have ownership transferred exactly once. So, there are no
+	// records to copy here, or free here.
+	for (int i = 0; i < n; i++) {
+		sllv_append(poutrecs, record_array[images[i]]);
+	}
+
+	free(record_array);
+	free(images);
+
+	// Null-terminate the output list to signify end of stream.
+	sllv_append(poutrecs, NULL);
+	return poutrecs;
+}
+

c/mapping/mappers.h

@@ -29,6 +29,7 @@ extern mapper_setup_t mapper_reorder_setup;
 extern mapper_setup_t mapper_reshape_setup;
 extern mapper_setup_t mapper_sample_setup;
 extern mapper_setup_t mapper_sec2gmt_setup;
+extern mapper_setup_t mapper_shuffle_setup;
 extern mapper_setup_t mapper_sort_setup;
 extern mapper_setup_t mapper_stats1_setup;
 extern mapper_setup_t mapper_stats2_setup;

c/todo.txt

@@ -10,7 +10,8 @@ TOP-OF-LIST SUMMARY
 
 ! nest --explode/--implode
 ? json arrays -> nested w/ some delimiter? only if all array elements are terminals.
-? shuffle? similar to bootstrap & sample. just use knuth algo.
+* shuffle: similar to bootstrap & sample. just use knuth algo.
+! 64-bit lengths for containers. test with 5-billion-integer-seq data.
 
 * tri-split null
 * parameterized emit: needs spec & impl. prereq: tri-split null.
@@ -29,17 +30,23 @@ TOP-OF-LIST SUMMARY
   - filter/gate/bare-boolean -- pending pattern-action
   - pattern-action -- still under development
 
+================================================================
+SHUFFLE
+
+seq -f %f 1 100000 | mlr --inidx shuffle then count-distinct -f 1 then stats1 -a min,max -f count
+count_min=1,count_max=1
+
 ================================================================
 EXPLODE/UNNEST/ETC:
 
 ----------------------------------------------------------------
 explode across records by value (forward):
-implode across records by value (reverse):
   mlr nest
-    --explode / --implode
+    --explode
     --values
-    --records
+    --across-records
     --nested-fs semicolon
+    -f x
 
   x=a;b;c,y=d
 
@@ -47,24 +54,51 @@ implode across records by value (reverse):
   x=b,y=d
   x=c,y=d
 
-explode across fields by value (forward):
-implode across fields by value (reverse):
+implode across records by value (reverse):
   mlr nest
-    --explode / --implode
+    --implode
     --values
-    --fields
+    --across-records
     --nested-fs semicolon
+    -f x
+
+  x=a,y=d
+  x=b,y=d
+  x=c,y=d
+
+  x=a;b;c,y=d
+
+----------------------------------------------------------------
+explode across fields by value (forward):
+  mlr nest
+    --explode
+    --values
+    --across-fields
+    --nested-fs semicolon
+    -f x
 
   x=a;b;c,y=d
 
   x_1=a,x_2=b,x_3=cy=d
 
-explode across records by key-value (forward):
-implode across records by key-value (reverse):
+implode across fields by value (reverse):
   mlr nest
-    --explode / --implode
+    --implode
+    --values
+    --across-fields
+    --nested-fs semicolon
+    -f x
+
+  x_1=a,x_2=b,x_3=cy=d
+
+  x=a;b;c,y=d
+
+----------------------------------------------------------------
+explode across records by key-value (forward):
+  mlr nest
+    --explode
     --pairs
-    --records
+    --across-records
     --nested-ps colon
     --nested-fs semicolon
 
@@ -74,12 +108,26 @@ implode across records by key-value (reverse):
   b=2,y=d
   c=3,y=d
 
-explode across fields by key-value (forward):
-implode across fields by key-value (reverse):
+implode across records by key-value (reverse):
   mlr nest
-    --explode / --implode
+    --implode
     --pairs
-    --fields
+    --across-records
+    --nested-ps colon
+    --nested-fs semicolon
+
+  a=1,y=d
+  b=2,y=d
+  c=3,y=d
+
+  x=a:1;b:2;c:3,y=d
+
+----------------------------------------------------------------
+explode across fields by key-value (forward):
+  mlr nest
+    --explode
+    --pairs
+    --across-fields
     --nested-ps colon
     --nested-fs semicolon
 
@@ -87,6 +135,18 @@ implode across fields by key-value (reverse):
 
   a=1,b=2,c=3,y=d
 
+implode across fields by key-value (reverse):
+  mlr nest
+    --implode
+    --pairs
+    --across-fields
+    --nested-ps colon
+    --nested-fs semicolon
+
+  a=1,b=2,c=3,y=d
+
+  x=a:1;b:2;c:3,y=d
+
 ----------------------------------------------------------------
   TSV: