From 383548bb93ccdb424b60943709ee1d885034ebb0 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 15 Feb 2016 10:41:12 -0500 Subject: [PATCH] shuffle iterate --- c/cli/mlrcli.c | 1 + c/containers/mlrval.h | 1 + c/mapping/Makefile.am | 1 + c/mapping/mapper_nest.c | 6 ++ c/mapping/mapper_shuffle.c | 134 +++++++++++++++++++++++++++++++++++++ c/mapping/mappers.h | 1 + c/todo.txt | 92 ++++++++++++++++++++----- 7 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 c/mapping/mapper_shuffle.c diff --git a/c/cli/mlrcli.c b/c/cli/mlrcli.c index a7c780560..bd1eca317 100644 --- a/c/cli/mlrcli.c +++ b/c/cli/mlrcli.c @@ -47,6 +47,7 @@ static mapper_setup_t* mapper_lookup_table[] = { &mapper_reorder_setup, &mapper_reshape_setup, &mapper_sample_setup, + &mapper_shuffle_setup, &mapper_sec2gmt_setup, &mapper_sort_setup, &mapper_stats1_setup, diff --git a/c/containers/mlrval.h b/c/containers/mlrval.h index 993714072..1f7aaf98c 100644 --- a/c/containers/mlrval.h +++ b/c/containers/mlrval.h @@ -293,6 +293,7 @@ static inline mv_t f_fff_logifit_func(mv_t* pval1, mv_t* pval2, mv_t* pval3) { return mv_from_float(1.0 / (1.0 + exp(-m*x-b))); } +// xxx libify static inline mv_t i_ii_urandint_func(mv_t* pval1, mv_t* pval2) { long long a = pval1->u.intv; long long b = pval2->u.intv; diff --git a/c/mapping/Makefile.am b/c/mapping/Makefile.am index 4cf2d3728..ee517f32e 100644 --- a/c/mapping/Makefile.am +++ b/c/mapping/Makefile.am @@ -27,6 +27,7 @@ libmapping_la_SOURCES= \ mapper_reshape.c \ mapper_sample.c \ mapper_sec2gmt.c \ + mapper_shuffle.c \ mapper_sort.c \ mapper_stats1.c \ mapper_stats2.c \ diff --git a/c/mapping/mapper_nest.c b/c/mapping/mapper_nest.c index fb8705e02..3cf266d14 100644 --- a/c/mapping/mapper_nest.c +++ b/c/mapping/mapper_nest.c @@ -39,6 +39,12 @@ typedef struct _mapper_nest_state_t { ap_state_t* pargp; + // -e/-i/--explode/--implode: required: -- via process func + // --values/--pairs: required: + // --fields/--records: streaming & non + // --nested-fs: required + // --nested-fp: required + // for wide-to-long: slls_t* input_field_names; sllv_t* input_field_regexes; diff --git a/c/mapping/mapper_shuffle.c b/c/mapping/mapper_shuffle.c new file mode 100644 index 000000000..2db1f14c8 --- /dev/null +++ b/c/mapping/mapper_shuffle.c @@ -0,0 +1,134 @@ +#include +#include +#include +#include +#include "lib/mlrutil.h" +#include "lib/mtrand.h" +#include "containers/sllv.h" +#include "containers/slls.h" +#include "containers/lhmslv.h" +#include "containers/lhmsv.h" +#include "containers/mixutil.h" +#include "mapping/mappers.h" +#include "cli/argparse.h" + +// ---------------------------------------------------------------- +typedef struct _mapper_shuffle_state_t { + ap_state_t* pargp; + sllv_t* precs; +} mapper_shuffle_state_t; + +static void mapper_shuffle_usage(FILE* o, char* argv0, char* verb); +static mapper_t* mapper_shuffle_parse_cli(int* pargi, int argc, char** argv); +static mapper_t* mapper_shuffle_alloc(ap_state_t* pargp); +static void mapper_shuffle_free(mapper_t* pmapper); +static sllv_t* mapper_shuffle_process(lrec_t* pinrec, context_t* pctx, void* pvstate); + +// ---------------------------------------------------------------- +mapper_setup_t mapper_shuffle_setup = { + .verb = "shuffle", + .pusage_func = mapper_shuffle_usage, + .pparse_func = mapper_shuffle_parse_cli +}; + +// ---------------------------------------------------------------- +static void mapper_shuffle_usage(FILE* o, char* argv0, char* verb) { + fprintf(o, "Usage: %s %s [options]\n", argv0, verb); + fprintf(o, "-- xxx under construction --\n"); +} + +static mapper_t* mapper_shuffle_parse_cli(int* pargi, int argc, char** argv) { + char* verb = argv[(*pargi)++]; + + ap_state_t* pstate = ap_alloc(); + + if (!ap_parse(pstate, verb, pargi, argc, argv)) { + mapper_shuffle_usage(stderr, argv[0], verb); + return NULL; + } + + return mapper_shuffle_alloc(pstate); +} + +// ---------------------------------------------------------------- +static mapper_t* mapper_shuffle_alloc(ap_state_t* pargp) { + mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); + + mapper_shuffle_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_shuffle_state_t)); + + pstate->pargp = pargp; + pstate->precs = sllv_alloc(); + + pmapper->pvstate = pstate; + pmapper->pprocess_func = mapper_shuffle_process; + pmapper->pfree_func = mapper_shuffle_free; + + return pmapper; +} + +static void mapper_shuffle_free(mapper_t* pmapper) { + mapper_shuffle_state_t* pstate = pmapper->pvstate; + // Records will have been freed by the emitter; here, free the list structure. + sllv_free(pstate->precs); + ap_free(pstate->pargp); + free(pstate); + free(pmapper); +} + +// ---------------------------------------------------------------- +static sllv_t* mapper_shuffle_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { + mapper_shuffle_state_t* pstate = pvstate; + + // Not end of input stream: retain the record, and emit nothing until end of stream. + if (pinrec != NULL) { + sllv_append(pstate->precs, pinrec); + return NULL; + } + + sllv_t* poutrecs = sllv_alloc(); + + // Knuth shuffle: + // * Initial permutation is identity. + // * Make a pseudorandom permutation using pseudorandom swaps in the image map. + int n = pstate->precs->length; + int* images = mlr_malloc_or_die(n * sizeof(int)); + for (int i = 0; i < n; i++) + images[i] = i; + + int unused_start = 0; + int num_unused = n; + for (int i = 0; i < n; i++) { + // Select a pseudorandom element from the pool of unused images. + int u = unused_start + num_unused * get_mtrand_double(); + int temp = images[u]; + images[u] = images[i]; + images[i] = temp; + + // Decrease the size of the pool by 1. (Yes, unused_start and k always have the same value. + // Using two variables wastes neglible memory and makes the code easier to understand.) + unused_start++; + num_unused--; + } + + // Make an array of pointers into the input list. + lrec_t** record_array = mlr_malloc_or_die(n * sizeof(lrec_t**)); + sllve_t* pe = pstate->precs->phead; + for (int i = 0; i < n; i++, pe = pe->pnext) { + record_array[i] = pe->pvvalue; + } + + // Transfer from input array to output list. Because permutations are one-to-one maps, + // all input records have ownership transferred exactly once. So, there are no + // records to copy here, or free here. + for (int i = 0; i < n; i++) { + sllv_append(poutrecs, record_array[images[i]]); + } + + free(record_array); + free(images); + + // Null-terminate the output list to signify end of stream. + sllv_append(poutrecs, NULL); + return poutrecs; +} + diff --git a/c/mapping/mappers.h b/c/mapping/mappers.h index 39a05b544..935cdc19d 100644 --- a/c/mapping/mappers.h +++ b/c/mapping/mappers.h @@ -29,6 +29,7 @@ extern mapper_setup_t mapper_reorder_setup; extern mapper_setup_t mapper_reshape_setup; extern mapper_setup_t mapper_sample_setup; extern mapper_setup_t mapper_sec2gmt_setup; +extern mapper_setup_t mapper_shuffle_setup; extern mapper_setup_t mapper_sort_setup; extern mapper_setup_t mapper_stats1_setup; extern mapper_setup_t mapper_stats2_setup; diff --git a/c/todo.txt b/c/todo.txt index 81d687931..5e233704d 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -10,7 +10,8 @@ TOP-OF-LIST SUMMARY ! nest --explode/--implode ? json arrays -> nested w/ some delimiter? only if all array elements are terminals. -? shuffle? similar to bootstrap & sample. just use knuth algo. +* shuffle: similar to bootstrap & sample. just use knuth algo. +! 64-bit lengths for containers. test with 5-billion-integer-seq data. * tri-split null * parameterized emit: needs spec & impl. prereq: tri-split null. @@ -29,17 +30,23 @@ TOP-OF-LIST SUMMARY - filter/gate/bare-boolean -- pending pattern-action - pattern-action -- still under development +================================================================ +SHUFFLE + +seq -f %f 1 100000 | mlr --inidx shuffle then count-distinct -f 1 then stats1 -a min,max -f count +count_min=1,count_max=1 + ================================================================ EXPLODE/UNNEST/ETC: ---------------------------------------------------------------- explode across records by value (forward): -implode across records by value (reverse): mlr nest - --explode / --implode + --explode --values - --records + --across-records --nested-fs semicolon + -f x x=a;b;c,y=d @@ -47,24 +54,51 @@ implode across records by value (reverse): x=b,y=d x=c,y=d -explode across fields by value (forward): -implode across fields by value (reverse): +implode across records by value (reverse): mlr nest - --explode / --implode + --implode --values - --fields + --across-records --nested-fs semicolon + -f x + + x=a,y=d + x=b,y=d + x=c,y=d + + x=a;b;c,y=d + +---------------------------------------------------------------- +explode across fields by value (forward): + mlr nest + --explode + --values + --across-fields + --nested-fs semicolon + -f x x=a;b;c,y=d x_1=a,x_2=b,x_3=cy=d -explode across records by key-value (forward): -implode across records by key-value (reverse): +implode across fields by value (reverse): mlr nest - --explode / --implode + --implode + --values + --across-fields + --nested-fs semicolon + -f x + + x_1=a,x_2=b,x_3=cy=d + + x=a;b;c,y=d + +---------------------------------------------------------------- +explode across records by key-value (forward): + mlr nest + --explode --pairs - --records + --across-records --nested-ps colon --nested-fs semicolon @@ -74,12 +108,26 @@ implode across records by key-value (reverse): b=2,y=d c=3,y=d -explode across fields by key-value (forward): -implode across fields by key-value (reverse): +implode across records by key-value (reverse): mlr nest - --explode / --implode + --implode --pairs - --fields + --across-records + --nested-ps colon + --nested-fs semicolon + + a=1,y=d + b=2,y=d + c=3,y=d + + x=a:1;b:2;c:3,y=d + +---------------------------------------------------------------- +explode across fields by key-value (forward): + mlr nest + --explode + --pairs + --across-fields --nested-ps colon --nested-fs semicolon @@ -87,6 +135,18 @@ implode across fields by key-value (reverse): a=1,b=2,c=3,y=d +implode across fields by key-value (reverse): + mlr nest + --implode + --pairs + --across-fields + --nested-ps colon + --nested-fs semicolon + + a=1,b=2,c=3,y=d + + x=a:1;b:2;c:3,y=d + ---------------------------------------------------------------- TSV: