Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

support numeric and/or descending sorts

Browse source
This commit is contained in:
John Kerl 2015-05-09 11:38:02 -07:00
parent 819de81857
commit a64e2b5532
5 changed files with 181 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

2
c/containers/lhmslv.h
View file

@ -22,7 +22,7 @@
typedef struct _lhmslve_t {
int ideal_index;
slls_t* key;
void* value;
void* value; // xxx rename pvvalue
struct _lhmslve_t *pprev;
struct _lhmslve_t *pnext;
} lhmslve_t;

248
c/mapping/mapper_sort.c
View file

@ -8,146 +8,203 @@
#include "containers/mixutil.h"
#include "mapping/mappers.h"
#define SORT_NUMERIC 0x40
#define SORT_DESCENDING 0x80
typedef struct _mapper_sort_state_t {
slls_t* pkey_field_names;
int* psort_params;
// map from list of string to list of record
lhmslv_t* precords_by_key_field_names;
int do_sort;
} mapper_sort_state_t;
static int cmp_for_qsort(const void* pva, const void* pvb);
// state for bucket choices:
// o slls of key-field values
// n slls/array of key_field_names
// config for the sort:
// * compare two arrays of u.string/double by array of num/lex asc/dec
// need num/lex & +/- flags for the sort -- static context for the non-reentrant qsort
// 'v' payload should be a pair of:
// * union of double & char*: here do the sscanf of the double if doing numeric sort on that field || abend
// * record-list as now
// at the end make an array of the pairs and qsort them on the union part
// then emit the record-lists as now
// ================================================================
// OVERVIEW
//
// before doing that, do the num/lex & the +/- at the CLI interface & handle (inefficiently)
// via repeated sscanf in the comparator:
// -rn a,b -fn c,d,e -r f,g -f h,i,j,k -- OR--
// -nr a,b -nf c,d,e -r f,g -f h,i,j,k
// * Suppose we are sorting records lexically ascending on field "a" and then
// numerically descending on field "x".
//
// data structure down there: sllv of sort info which is triple of char* field_name, char do_num, char do_rev.
// * CLI syntax is "mlr sort -f a -nr x".
//
// * We first consume all input records and for each extract the string values
// of fields a and x. For each uniq combination of a-value (e.g. "red",
// "green", "blue") and x-value (e.g. "1", "1.0", "2.4") -- e.g.
// pairs ["red","1"], and so on -- we keep a linked list of all the records
// having those sort-key values, in the order encountered.
//
// * For each of those unique sort-key-value combinations, we also parse the
// numerical fields at this point into an array of union-string-double.
// E.g. the list ["red", "1.0"] maps to the array ["red", 1.0].
//
// * The pairing of parsed-value array the linked list of same-key-value records
// is called a *bucket*. E.g the records
// {"a":"red","b":"circle","x":"1.0","y":"3.9"}
// {"a":"red","b":"square","x":"1.0","z":"5.7", "q":"even"}
// would both land in the ["red","1.0"] bucket.
//
// * Buckets are retained in a hash map: the key is the string-list of the form
// ["red","1.0"] and the value is the pairing of parsed-value array ["red",1.0]
// ane linked list of records.
//
// * Once all the input records are ingested into this hash map, we copy the
// bucket-pointers into an array and sort it: this being the pairing of
// parsed-value array and linked list of records. The comparator callback for
// the sort walks through the parsed-value arrays one slot at a time,
// looking at the first difference, e.g. if one has "a"="red" and the other
// has "a"="blue". If the first field matches then the sort moves to the
// second field, and so on.
//
// * Recall in particular that string keys ["a":"red","x":"1"] and
// ["a":"red","x":"1.0"] map to different buckets, but will sort equally.
//
// ================================================================
// xxx cmt about re-entrancy
static int* pcmp_sort_params = NULL;
static int cmp_for_qsort(const void* pva, const void* pvb) {
const lhmslve_t* pea = pva;
const lhmslve_t* peb = pvb;
slls_t* pa = pea->key;
slls_t* pb = peb->key;
if (pa->length != pb->length)
return pa->length - pb->length;
sllse_t* pe = pa->phead;
sllse_t* pf = pb->phead;
for (int i = 0; pe != NULL && pf != NULL; pe = pe->pnext, pf = pf->pnext, i++) {
if (pcmp_sort_params[i] & SORT_NUMERIC) {
double e, f;
if (sscanf(pe->value, "%lf", &e) != 1) {
fprintf(stderr, "xxx b04k!\n");
exit(1);
}
if (sscanf(pf->value, "%lf", &f) != 1) {
fprintf(stderr, "xxx b04k!\n");
exit(1);
}
double d = e - f;
int s = (d < 0) ? -1 : 1;
// ----------------------------------------------------------------
#define SORT_NUMERIC 0x80
#define SORT_DESCENDING 0x40
// Each sort key is string or number; use union to save space.
typedef struct _typed_sort_key_t {
union {
char* s;
double d;
} u;
} typed_sort_key_t;
typedef struct _bucket_t {
typed_sort_key_t* typed_sort_keys;
sllv_t* precords;
} bucket_t;
// qsort is non-reentrant but qsort_r isn't portable. But since Miller is
// single-threaded, even if we've got one sort chained to another, only one is
// active at a time. We adopt the convention that we set the sort params
// right before the sort.
static int* pcmp_sort_params = NULL;
static int cmp_params_length = 0;
static int pbucket_comparator(const void* pva, const void* pvb) {
// We are sorting an array of bucket_t*.
const bucket_t** pba = (const bucket_t**)pva;
const bucket_t** pbb = (const bucket_t**)pvb;
typed_sort_key_t* akeys = (*pba)->typed_sort_keys;
typed_sort_key_t* bkeys = (*pbb)->typed_sort_keys;
for (int i = 0; i < cmp_params_length; i++) {
int sort_param = pcmp_sort_params[i];
if (sort_param & SORT_NUMERIC) {
double d = akeys[i].u.d - bkeys[i].u.d;
int s = (d < 0) ? -1 : (d > 0) ? 1 : 0;
if (s != 0)
return (pcmp_sort_params[i] & SORT_DESCENDING) ? -s : s;
return (sort_param & SORT_DESCENDING) ? -s : s;
} else {
int s = strcmp(pe->value, pf->value);
int s = strcmp(akeys[i].u.s, bkeys[i].u.s);
if (s != 0)
return (pcmp_sort_params[i] & SORT_DESCENDING) ? -s : s;
return (sort_param & SORT_DESCENDING) ? -s : s;
}
}
return 0;
}
// E.g. parse the list ["red","1.0"] into the array ["red",1.0].
static typed_sort_key_t* parse_sort_keys(slls_t* pkey_field_values, int* sort_params) {
typed_sort_key_t* typed_sort_keys = mlr_malloc_or_die(pkey_field_values->length * sizeof(typed_sort_key_t));
int i = 0;
for (sllse_t* pe = pkey_field_values->phead; pe != NULL; pe = pe->pnext, i++) {
if (sort_params[i] & SORT_NUMERIC) {
if (sscanf(pe->value, "%lf", &typed_sort_keys[i].u.d) != 1) {
// xxx to do: print some more context here, e.g. file name & line number
fprintf(stderr, "Couldn't parse \"%s\" as number.\n", pe->value);
exit(1);
}
} else {
typed_sort_keys[i].u.s = pe->value;
}
}
return typed_sort_keys;
}
// ----------------------------------------------------------------
typedef struct _mapper_sort_state_t {
// Input parameters
slls_t* pkey_field_names; // Fields to sort on
int* sort_params; // Lexical/numeric; ascending/descending
int do_sort; // If false, just do group-by
// Sort state: buckets of like records.
lhmslv_t* pbuckets_by_key_field_names;
} mapper_sort_state_t;
// ----------------------------------------------------------------
sllv_t* mapper_sort_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
mapper_sort_state_t* pstate = pvstate;
if (pinrec != NULL) {
// Consume another input record.
slls_t* pkey_field_values = mlr_selected_values_from_record(pinrec, pstate->pkey_field_names);
sllv_t* plist = lhmslv_get(pstate->precords_by_key_field_names, pkey_field_values);
if (plist == NULL) {
plist = sllv_alloc();
sllv_add(plist, pinrec);
lhmslv_put(pstate->precords_by_key_field_names, slls_copy(pkey_field_values), plist);
} else {
sllv_add(plist, pinrec);
bucket_t* pbucket = lhmslv_get(pstate->pbuckets_by_key_field_names, pkey_field_values);
if (pbucket == NULL) { // New key-field-value: new bucket and hash-map entry
slls_t* pkey_field_values_copy = slls_copy(pkey_field_values);
bucket_t* pbucket = mlr_malloc_or_die(sizeof(bucket_t)); // xxx free in the free func
pbucket->typed_sort_keys = parse_sort_keys(pkey_field_values_copy, pstate->sort_params);
pbucket->precords = sllv_alloc();
sllv_add(pbucket->precords, pinrec);
lhmslv_put(pstate->pbuckets_by_key_field_names, pkey_field_values_copy, pbucket);
} else { // Previously seen key-field-value: append record to bucket
sllv_add(pbucket->precords, pinrec);
}
return NULL;
}
else if (!pstate->do_sort) {
// Group-by
sllv_t* poutput = sllv_alloc();
for (lhmslve_t* pe = pstate->precords_by_key_field_names->phead; pe != NULL; pe = pe->pnext) {
sllv_t* plist = pe->value;
for (sllve_t* pf = plist->phead; pf != NULL; pf = pf->pnext) {
for (lhmslve_t* pe = pstate->pbuckets_by_key_field_names->phead; pe != NULL; pe = pe->pnext) {
bucket_t* pbucket = pe->value;
for (sllve_t* pf = pbucket->precords->phead; pf != NULL; pf = pf->pnext) {
sllv_add(poutput, pf->pvdata);
}
}
sllv_add(poutput, NULL);
return poutput;
} else {
int num_lists = pstate->precords_by_key_field_names->num_occupied;
lhmslve_t* pairs = mlr_malloc_or_die(num_lists * sizeof(lhmslve_t));
// End of input stream: sort bucket labels
int num_buckets = pstate->pbuckets_by_key_field_names->num_occupied;
bucket_t** pbucket_array = mlr_malloc_or_die(num_buckets * sizeof(bucket_t*));
// Copy bucket-pointers to an array for qsort
int i = 0;
for (lhmslve_t* pe = pstate->precords_by_key_field_names->phead; pe != NULL; pe = pe->pnext, i++) {
pairs[i].key = pe->key;
pairs[i].value = pe->value;
for (lhmslve_t* pe = pstate->pbuckets_by_key_field_names->phead; pe != NULL; pe = pe->pnext, i++) {
pbucket_array[i] = pe->value;
}
pcmp_sort_params = pstate->psort_params;
qsort(pairs, num_lists, sizeof(pairs[0]), cmp_for_qsort);
pcmp_sort_params = NULL;
pcmp_sort_params = pstate->sort_params;
cmp_params_length = pstate->pkey_field_names->length;
qsort(pbucket_array, num_buckets, sizeof(bucket_t*), pbucket_comparator);
pcmp_sort_params = NULL;
cmp_params_length = 0;
// Emit each bucket's record
sllv_t* poutput = sllv_alloc();
for (i = 0; i < num_lists; i++) {
sllv_t* plist = pairs[i].value;
for (i = 0; i < num_buckets; i++) {
sllv_t* plist = pbucket_array[i]->precords;
for (sllve_t* pf = plist->phead; pf != NULL; pf = pf->pnext) {
sllv_add(poutput, pf->pvdata);
}
}
free(pairs);
sllv_add(poutput, NULL);
free(pbucket_array);
sllv_add(poutput, NULL); // Signal end of output-record stream.
return poutput;
}
}
// ----------------------------------------------------------------
// xxx to do: at end of sort, free and nullify the things populated within
// mapper_sort_func. here, free the things allocated as setup time.
static void mapper_sort_free(void* pvstate) {
mapper_sort_state_t* pstate = pvstate;
if (pstate->pkey_field_names != NULL)
slls_free(pstate->pkey_field_names);
if (pstate->precords_by_key_field_names != NULL)
if (pstate->pbuckets_by_key_field_names != NULL)
// xxx free void-star payloads 1st
lhmslv_free(pstate->precords_by_key_field_names);
if (pstate->psort_params != NULL)
free(pstate->psort_params);
lhmslv_free(pstate->pbuckets_by_key_field_names);
if (pstate->sort_params != NULL)
free(pstate->sort_params);
}
mapper_t* mapper_sort_alloc(slls_t* pkey_field_names, int* psort_params, int do_sort) {
mapper_t* mapper_sort_alloc(slls_t* pkey_field_names, int* sort_params, int do_sort) {
mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
mapper_sort_state_t* pstate = mlr_malloc_or_die(sizeof(mapper_sort_state_t));
pstate->pkey_field_names = pkey_field_names;
pstate->psort_params = psort_params;
pstate->precords_by_key_field_names = lhmslv_alloc();
pstate->sort_params = sort_params;
pstate->pbuckets_by_key_field_names = lhmslv_alloc();
pstate->do_sort = do_sort;
// xxx put this line by the next x all mapper_*_allocs
@ -169,9 +226,12 @@ mapper_t* mapper_group_by_parse_cli(int* pargi, int argc, char** argv) {
}
slls_t* pnames = slls_from_line(argv[*pargi+1], ',', FALSE);
int* opt_array = mlr_malloc_or_die(pnames->length * sizeof(int));
for (int i = 0; i < pnames->length; i++)
opt_array[i] = 0;
*pargi += 2;
return mapper_sort_alloc(pnames, NULL, FALSE);
return mapper_sort_alloc(pnames, opt_array, FALSE);
}
// ----------------------------------------------------------------
@ -183,8 +243,15 @@ mapper_setup_t mapper_group_by_setup = {
// ----------------------------------------------------------------
void mapper_sort_usage(char* argv0, char* verb) {
fprintf(stdout, "Usage: %s %s {xxx}\n", argv0, verb);
fprintf(stdout, " xxx info goes here\n");
fprintf(stdout, "Usage: %s %s {flags}\n", argv0, verb);
fprintf(stdout, "Flags:\n");
fprintf(stdout, " -f {comma-separated field names} Lexical ascending\n");
fprintf(stdout, " -n {comma-separated field names} Numerical ascending\n");
fprintf(stdout, " -nf {comma-separated field names} Numerical ascending\n");
fprintf(stdout, " -r {comma-separated field names} Lexical descending\n");
fprintf(stdout, " -nr {comma-separated field names} Numerical descending\n");
fprintf(stdout, "Example:\n");
fprintf(stdout, " %s %s -f a,b -nr x,y,z\n", argv0, verb);
}
mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv) {
if ((argc - *pargi) < 3) {
@ -212,6 +279,7 @@ mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv) {
mapper_sort_usage(argv[0], verb);
}
slls_t* pnames_for_flag = slls_from_line(value, ',', FALSE);
// E.g. with "-nr a,b,c", replicate the "-nr" flag three times.
for (sllse_t* pe = pnames_for_flag->phead; pe != NULL; pe = pe->pnext) {
slls_add_no_free(pnames, pe->value);
slls_add_no_free(pflags, flag);
@ -222,6 +290,8 @@ mapper_t* mapper_sort_parse_cli(int* pargi, int argc, char** argv) {
if (pnames->length < 1)
mapper_sort_usage(argv[0], verb);
// Convert the list such as ["-nf","-nf","-r","-r","-r"] into an array of
// bit-flags, one per sort-key field.
int* opt_array = mlr_malloc_or_die(pnames->length * sizeof(int));
sllse_t* pe;
int di;

2
c/t1.rb
View file

@ -81,7 +81,7 @@ run("sortsort1", "sort -t, -k 1,2 < data/big > /dev/null")
run("SORTMLR1", "mlr sort -f a,b < data/big > /dev/null")
puts
run("sortsort2", "sort -t, -n -k 4,5 < data/big > /dev/null")
run("sortsort2", "sort -t, -k 4,5 < data/big > /dev/null")
run("SORTMLR2", "mlr sort -n x,y < data/big > /dev/null")
puts
puts

2
c/todo.txt
View file

@ -8,6 +8,8 @@
FEATURES
!! sort -nr !!
-> put uniq & count-distinct into the same file. then rm extraneous extern @ .h
-> check 1 vs. 1.0 etc in sort -n
-> cmp heap/merge/quick/...
!! quantiles !!
!! reminder pgr legend is broken !!
http://en.wikipedia.org/wiki/Order_statistic_tree

22
doc/reference.html
View file

@ -234,8 +234,15 @@ I/O options:
<div class="pokipanel">
<pre>
$ mlr sort --help
Usage: mlr sort {xxx}
xxx info goes here
Usage: mlr sort {flags}
Flags:
-f {comma-separated field names} Lexical ascending
-n {comma-separated field names} Numerical ascending
-nf {comma-separated field names} Numerical ascending
-r {comma-separated field names} Lexical descending
-nr {comma-separated field names} Numerical descending
Example:
mlr sort -f a,b -nr x,y,z
</pre>
</div>
<p/>
@ -1099,8 +1106,15 @@ wye 0.5732889198020006 0.8636244699032729 5 pan
<div class="pokipanel">
<pre>
$ mlr sort --help
Usage: mlr sort {xxx}
xxx info goes here
Usage: mlr sort {flags}
Flags:
-f {comma-separated field names} Lexical ascending
-n {comma-separated field names} Numerical ascending
-nf {comma-separated field names} Numerical ascending
-r {comma-separated field names} Lexical descending
-nr {comma-separated field names} Numerical descending
Example:
mlr sort -f a,b -nr x,y,z
</pre>
</div>
<p/>