From a64e2b5532001f50cab80d3ea2e413334eca3aa8 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 9 May 2015 11:38:02 -0700 Subject: [PATCH] support numeric and/or descending sorts --- c/containers/lhmslv.h | 2 +- c/mapping/mapper_sort.c | 248 ++++++++++++++++++++++++++-------------- c/t1.rb | 2 +- c/todo.txt | 2 + doc/reference.html | 22 +++- 5 files changed, 181 insertions(+), 95 deletions(-) diff --git a/c/containers/lhmslv.h b/c/containers/lhmslv.h index 99af16afc..206ff560c 100644 --- a/c/containers/lhmslv.h +++ b/c/containers/lhmslv.h @@ -22,7 +22,7 @@ typedef struct _lhmslve_t { int ideal_index; slls_t* key; - void* value; + void* value; // xxx rename pvvalue struct _lhmslve_t *pprev; struct _lhmslve_t *pnext; } lhmslve_t; diff --git a/c/mapping/mapper_sort.c b/c/mapping/mapper_sort.c index 85f764d80..0991fd906 100644 --- a/c/mapping/mapper_sort.c +++ b/c/mapping/mapper_sort.c @@ -8,146 +8,203 @@ #include "containers/mixutil.h" #include "mapping/mappers.h" -#define SORT_NUMERIC 0x40 -#define SORT_DESCENDING 0x80 - -typedef struct _mapper_sort_state_t { - slls_t* pkey_field_names; - int* psort_params; - // map from list of string to list of record - lhmslv_t* precords_by_key_field_names; - int do_sort; -} mapper_sort_state_t; - -static int cmp_for_qsort(const void* pva, const void* pvb); - -// state for bucket choices: -// o slls of key-field values -// n slls/array of key_field_names -// config for the sort: -// * compare two arrays of u.string/double by array of num/lex asc/dec - -// need num/lex & +/- flags for the sort -- static context for the non-reentrant qsort -// 'v' payload should be a pair of: -// * union of double & char*: here do the sscanf of the double if doing numeric sort on that field || abend -// * record-list as now -// at the end make an array of the pairs and qsort them on the union part -// then emit the record-lists as now +// ================================================================ +// OVERVIEW // -// before doing that, do the num/lex & the +/- at the CLI interface & handle (inefficiently) -// via repeated sscanf in the comparator: -// -rn a,b -fn c,d,e -r f,g -f h,i,j,k -- OR-- -// -nr a,b -nf c,d,e -r f,g -f h,i,j,k +// * Suppose we are sorting records lexically ascending on field "a" and then +// numerically descending on field "x". // -// data structure down there: sllv of sort info which is triple of char* field_name, char do_num, char do_rev. +// * CLI syntax is "mlr sort -f a -nr x". +// +// * We first consume all input records and for each extract the string values +// of fields a and x. For each uniq combination of a-value (e.g. "red", +// "green", "blue") and x-value (e.g. "1", "1.0", "2.4") -- e.g. +// pairs ["red","1"], and so on -- we keep a linked list of all the records +// having those sort-key values, in the order encountered. +// +// * For each of those unique sort-key-value combinations, we also parse the +// numerical fields at this point into an array of union-string-double. +// E.g. the list ["red", "1.0"] maps to the array ["red", 1.0]. +// +// * The pairing of parsed-value array the linked list of same-key-value records +// is called a *bucket*. E.g the records +// {"a":"red","b":"circle","x":"1.0","y":"3.9"} +// {"a":"red","b":"square","x":"1.0","z":"5.7", "q":"even"} +// would both land in the ["red","1.0"] bucket. +// +// * Buckets are retained in a hash map: the key is the string-list of the form +// ["red","1.0"] and the value is the pairing of parsed-value array ["red",1.0] +// ane linked list of records. +// +// * Once all the input records are ingested into this hash map, we copy the +// bucket-pointers into an array and sort it: this being the pairing of +// parsed-value array and linked list of records. The comparator callback for +// the sort walks through the parsed-value arrays one slot at a time, +// looking at the first difference, e.g. if one has "a"="red" and the other +// has "a"="blue". If the first field matches then the sort moves to the +// second field, and so on. +// +// * Recall in particular that string keys ["a":"red","x":"1"] and +// ["a":"red","x":"1.0"] map to different buckets, but will sort equally. +// +// ================================================================ -// xxx cmt about re-entrancy -static int* pcmp_sort_params = NULL; -static int cmp_for_qsort(const void* pva, const void* pvb) { - const lhmslve_t* pea = pva; - const lhmslve_t* peb = pvb; - slls_t* pa = pea->key; - slls_t* pb = peb->key; - if (pa->length != pb->length) - return pa->length - pb->length; - sllse_t* pe = pa->phead; - sllse_t* pf = pb->phead; - for (int i = 0; pe != NULL && pf != NULL; pe = pe->pnext, pf = pf->pnext, i++) { - if (pcmp_sort_params[i] & SORT_NUMERIC) { - double e, f; - if (sscanf(pe->value, "%lf", &e) != 1) { - fprintf(stderr, "xxx b04k!\n"); - exit(1); - } - if (sscanf(pf->value, "%lf", &f) != 1) { - fprintf(stderr, "xxx b04k!\n"); - exit(1); - } - double d = e - f; - int s = (d < 0) ? -1 : 1; +// ---------------------------------------------------------------- +#define SORT_NUMERIC 0x80 +#define SORT_DESCENDING 0x40 + +// Each sort key is string or number; use union to save space. +typedef struct _typed_sort_key_t { + union { + char* s; + double d; + } u; +} typed_sort_key_t; + +typedef struct _bucket_t { + typed_sort_key_t* typed_sort_keys; + sllv_t* precords; +} bucket_t; + +// qsort is non-reentrant but qsort_r isn't portable. But since Miller is +// single-threaded, even if we've got one sort chained to another, only one is +// active at a time. We adopt the convention that we set the sort params +// right before the sort. +static int* pcmp_sort_params = NULL; +static int cmp_params_length = 0; +static int pbucket_comparator(const void* pva, const void* pvb) { + // We are sorting an array of bucket_t*. + const bucket_t** pba = (const bucket_t**)pva; + const bucket_t** pbb = (const bucket_t**)pvb; + typed_sort_key_t* akeys = (*pba)->typed_sort_keys; + typed_sort_key_t* bkeys = (*pbb)->typed_sort_keys; + for (int i = 0; i < cmp_params_length; i++) { + int sort_param = pcmp_sort_params[i]; + if (sort_param & SORT_NUMERIC) { + double d = akeys[i].u.d - bkeys[i].u.d; + int s = (d < 0) ? -1 : (d > 0) ? 1 : 0; if (s != 0) - return (pcmp_sort_params[i] & SORT_DESCENDING) ? -s : s; + return (sort_param & SORT_DESCENDING) ? -s : s; } else { - int s = strcmp(pe->value, pf->value); + int s = strcmp(akeys[i].u.s, bkeys[i].u.s); if (s != 0) - return (pcmp_sort_params[i] & SORT_DESCENDING) ? -s : s; + return (sort_param & SORT_DESCENDING) ? -s : s; } } return 0; } +// E.g. parse the list ["red","1.0"] into the array ["red",1.0]. +static typed_sort_key_t* parse_sort_keys(slls_t* pkey_field_values, int* sort_params) { + typed_sort_key_t* typed_sort_keys = mlr_malloc_or_die(pkey_field_values->length * sizeof(typed_sort_key_t)); + int i = 0; + for (sllse_t* pe = pkey_field_values->phead; pe != NULL; pe = pe->pnext, i++) { + if (sort_params[i] & SORT_NUMERIC) { + if (sscanf(pe->value, "%lf", &typed_sort_keys[i].u.d) != 1) { + // xxx to do: print some more context here, e.g. file name & line number + fprintf(stderr, "Couldn't parse \"%s\" as number.\n", pe->value); + exit(1); + } + } else { + typed_sort_keys[i].u.s = pe->value; + } + } + return typed_sort_keys; +} + +// ---------------------------------------------------------------- +typedef struct _mapper_sort_state_t { + // Input parameters + slls_t* pkey_field_names; // Fields to sort on + int* sort_params; // Lexical/numeric; ascending/descending + int do_sort; // If false, just do group-by + // Sort state: buckets of like records. + lhmslv_t* pbuckets_by_key_field_names; +} mapper_sort_state_t; + // ---------------------------------------------------------------- sllv_t* mapper_sort_func(lrec_t* pinrec, context_t* pctx, void* pvstate) { mapper_sort_state_t* pstate = pvstate; if (pinrec != NULL) { + // Consume another input record. slls_t* pkey_field_values = mlr_selected_values_from_record(pinrec, pstate->pkey_field_names); - sllv_t* plist = lhmslv_get(pstate->precords_by_key_field_names, pkey_field_values); - if (plist == NULL) { - plist = sllv_alloc(); - sllv_add(plist, pinrec); - lhmslv_put(pstate->precords_by_key_field_names, slls_copy(pkey_field_values), plist); - } else { - sllv_add(plist, pinrec); + bucket_t* pbucket = lhmslv_get(pstate->pbuckets_by_key_field_names, pkey_field_values); + if (pbucket == NULL) { // New key-field-value: new bucket and hash-map entry + slls_t* pkey_field_values_copy = slls_copy(pkey_field_values); + bucket_t* pbucket = mlr_malloc_or_die(sizeof(bucket_t)); // xxx free in the free func + pbucket->typed_sort_keys = parse_sort_keys(pkey_field_values_copy, pstate->sort_params); + pbucket->precords = sllv_alloc(); + sllv_add(pbucket->precords, pinrec); + lhmslv_put(pstate->pbuckets_by_key_field_names, pkey_field_values_copy, pbucket); + } else { // Previously seen key-field-value: append record to bucket + sllv_add(pbucket->precords, pinrec); } return NULL; } else if (!pstate->do_sort) { + // Group-by sllv_t* poutput = sllv_alloc(); - for (lhmslve_t* pe = pstate->precords_by_key_field_names->phead; pe != NULL; pe = pe->pnext) { - sllv_t* plist = pe->value; - for (sllve_t* pf = plist->phead; pf != NULL; pf = pf->pnext) { + for (lhmslve_t* pe = pstate->pbuckets_by_key_field_names->phead; pe != NULL; pe = pe->pnext) { + bucket_t* pbucket = pe->value; + for (sllve_t* pf = pbucket->precords->phead; pf != NULL; pf = pf->pnext) { sllv_add(poutput, pf->pvdata); } } sllv_add(poutput, NULL); return poutput; } else { - int num_lists = pstate->precords_by_key_field_names->num_occupied; - lhmslve_t* pairs = mlr_malloc_or_die(num_lists * sizeof(lhmslve_t)); + // End of input stream: sort bucket labels + int num_buckets = pstate->pbuckets_by_key_field_names->num_occupied; + bucket_t** pbucket_array = mlr_malloc_or_die(num_buckets * sizeof(bucket_t*)); + // Copy bucket-pointers to an array for qsort int i = 0; - for (lhmslve_t* pe = pstate->precords_by_key_field_names->phead; pe != NULL; pe = pe->pnext, i++) { - pairs[i].key = pe->key; - pairs[i].value = pe->value; + for (lhmslve_t* pe = pstate->pbuckets_by_key_field_names->phead; pe != NULL; pe = pe->pnext, i++) { + pbucket_array[i] = pe->value; } - pcmp_sort_params = pstate->psort_params; - qsort(pairs, num_lists, sizeof(pairs[0]), cmp_for_qsort); - pcmp_sort_params = NULL; + pcmp_sort_params = pstate->sort_params; + cmp_params_length = pstate->pkey_field_names->length; + qsort(pbucket_array, num_buckets, sizeof(bucket_t*), pbucket_comparator); + pcmp_sort_params = NULL; + cmp_params_length = 0; + // Emit each bucket's record sllv_t* poutput = sllv_alloc(); - for (i = 0; i < num_lists; i++) { - sllv_t* plist = pairs[i].value; + for (i = 0; i < num_buckets; i++) { + sllv_t* plist = pbucket_array[i]->precords; for (sllve_t* pf = plist->phead; pf != NULL; pf = pf->pnext) { sllv_add(poutput, pf->pvdata); } } - free(pairs); - sllv_add(poutput, NULL); + free(pbucket_array); + sllv_add(poutput, NULL); // Signal end of output-record stream. return poutput; } } // ---------------------------------------------------------------- +// xxx to do: at end of sort, free and nullify the things populated within +// mapper_sort_func. here, free the things allocated as setup time. static void mapper_sort_free(void* pvstate) { mapper_sort_state_t* pstate = pvstate; if (pstate->pkey_field_names != NULL) slls_free(pstate->pkey_field_names); - if (pstate->precords_by_key_field_names != NULL) + if (pstate->pbuckets_by_key_field_names != NULL) // xxx free void-star payloads 1st - lhmslv_free(pstate->precords_by_key_field_names); - if (pstate->psort_params != NULL) - free(pstate->psort_params); + lhmslv_free(pstate->pbuckets_by_key_field_names); + if (pstate->sort_params != NULL) + free(pstate->sort_params); } -mapper_t* mapper_sort_alloc(slls_t* pkey_field_names, int* psort_params, int do_sort) { +mapper_t* mapper_sort_alloc(slls_t* pkey_field_names, int* sort_params, int do_sort) { mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); mapper_sort_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_sort_state_t)); pstate->pkey_field_names = pkey_field_names; - pstate->psort_params = psort_params; - pstate->precords_by_key_field_names = lhmslv_alloc(); + pstate->sort_params = sort_params; + pstate->pbuckets_by_key_field_names = lhmslv_alloc(); pstate->do_sort = do_sort; // xxx put this line by the next x all mapper_*_allocs @@ -169,9 +226,12 @@ mapper_t* mapper_group_by_parse_cli(int* pargi, int argc, char** argv) { } slls_t* pnames = slls_from_line(argv[*pargi+1], ',', FALSE); + int* opt_array = mlr_malloc_or_die(pnames->length * sizeof(int)); + for (int i = 0; i < pnames->length; i++) + opt_array[i] = 0; *pargi += 2; - return mapper_sort_alloc(pnames, NULL, FALSE); + return mapper_sort_alloc(pnames, opt_array, FALSE); } // ---------------------------------------------------------------- @@ -183,8 +243,15 @@ mapper_setup_t mapper_group_by_setup = { // ---------------------------------------------------------------- void mapper_sort_usage(char* argv0, char* verb) { - fprintf(stdout, "Usage: %s %s {xxx}\n", argv0, verb); - fprintf(stdout, " xxx info goes here\n"); + fprintf(stdout, "Usage: %s %s {flags}\n", argv0, verb); + fprintf(stdout, "Flags:\n"); + fprintf(stdout, " -f {comma-separated field names} Lexical ascending\n"); + fprintf(stdout, " -n {comma-separated field names} Numerical ascending\n"); + fprintf(stdout, " -nf {comma-separated field names} Numerical ascending\n"); + fprintf(stdout, " -r {comma-separated field names} Lexical descending\n"); + fprintf(stdout, " -nr {comma-separated field names} Numerical descending\n"); + fprintf(stdout, "Example:\n"); + fprintf(stdout, " %s %s -f a,b -nr x,y,z\n", argv0, verb); } mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv) { if ((argc - *pargi) < 3) { @@ -212,6 +279,7 @@ mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv) { mapper_sort_usage(argv[0], verb); } slls_t* pnames_for_flag = slls_from_line(value, ',', FALSE); + // E.g. with "-nr a,b,c", replicate the "-nr" flag three times. for (sllse_t* pe = pnames_for_flag->phead; pe != NULL; pe = pe->pnext) { slls_add_no_free(pnames, pe->value); slls_add_no_free(pflags, flag); @@ -222,6 +290,8 @@ mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv) { if (pnames->length < 1) mapper_sort_usage(argv[0], verb); + // Convert the list such as ["-nf","-nf","-r","-r","-r"] into an array of + // bit-flags, one per sort-key field. int* opt_array = mlr_malloc_or_die(pnames->length * sizeof(int)); sllse_t* pe; int di; diff --git a/c/t1.rb b/c/t1.rb index facf1ff1f..9ead58959 100755 --- a/c/t1.rb +++ b/c/t1.rb @@ -81,7 +81,7 @@ run("sortsort1", "sort -t, -k 1,2 < data/big > /dev/null") run("SORTMLR1", "mlr sort -f a,b < data/big > /dev/null") puts -run("sortsort2", "sort -t, -n -k 4,5 < data/big > /dev/null") +run("sortsort2", "sort -t, -k 4,5 < data/big > /dev/null") run("SORTMLR2", "mlr sort -n x,y < data/big > /dev/null") puts puts diff --git a/c/todo.txt b/c/todo.txt index 6fbf59021..d71af8e5a 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -8,6 +8,8 @@ FEATURES !! sort -nr !! -> put uniq & count-distinct into the same file. then rm extraneous extern @ .h + -> check 1 vs. 1.0 etc in sort -n + -> cmp heap/merge/quick/... !! quantiles !! !! reminder pgr legend is broken !! http://en.wikipedia.org/wiki/Order_statistic_tree diff --git a/doc/reference.html b/doc/reference.html index aa2bfbda6..17d88bc96 100644 --- a/doc/reference.html +++ b/doc/reference.html @@ -234,8 +234,15 @@ I/O options: 
 $ mlr sort --help
-Usage: mlr sort {xxx}
-       xxx info goes here
+Usage: mlr sort {flags}
+Flags:
+  -f  {comma-separated field names}  Lexical ascending
+  -n  {comma-separated field names}  Numerical ascending
+  -nf {comma-separated field names}  Numerical ascending
+  -r  {comma-separated field names}  Lexical descending
+  -nr {comma-separated field names}  Numerical descending
+Example:
+  mlr sort -f a,b -nr x,y,z

@@ -1099,8 +1106,15 @@ wye 0.5732889198020006 0.8636244699032729 5 pan 

 $ mlr sort --help
-Usage: mlr sort {xxx}
-       xxx info goes here
+Usage: mlr sort {flags}
+Flags:
+  -f  {comma-separated field names}  Lexical ascending
+  -n  {comma-separated field names}  Numerical ascending
+  -nf {comma-separated field names}  Numerical ascending
+  -r  {comma-separated field names}  Lexical descending
+  -nr {comma-separated field names}  Numerical descending
+Example:
+  mlr sort -f a,b -nr x,y,z