diff --git a/c/containers/lhmsll.c b/c/containers/lhmsll.c index 90103e965..c32b4d671 100644 --- a/c/containers/lhmsll.c +++ b/c/containers/lhmsll.c @@ -210,6 +210,22 @@ int lhmsll_test_and_get(lhmsll_t* pmap, char* key, long long* pval) { } } +int lhmsll_test_and_increment(lhmsll_t* pmap, char* key) { + int ideal_index = 0; + int index = lhmsll_find_index_for_key(pmap, key, &ideal_index); + lhmslle_t* pe = &pmap->entries[index]; + + if (pmap->states[index] == OCCUPIED) { + pe->value++; + return TRUE; + } else if (pmap->states[index] == EMPTY) { + return FALSE; + } else { + fprintf(stderr, "%s: lhmsll_find_index_for_key did not find end of chain.\n", MLR_GLOBALS.bargv0); + exit(1); + } +} + lhmslle_t* lhmsll_get_entry(lhmsll_t* pmap, char* key) { int ideal_index = 0; int index = lhmsll_find_index_for_key(pmap, key, &ideal_index); diff --git a/c/containers/lhmsll.h b/c/containers/lhmsll.h index 9f9ab8177..1abea23d0 100644 --- a/c/containers/lhmsll.h +++ b/c/containers/lhmsll.h @@ -45,6 +45,7 @@ void lhmsll_free(lhmsll_t* pmap); void lhmsll_put(lhmsll_t* pmap, char* key, int value, char free_flags); long long lhmsll_get(lhmsll_t* pmap, char* key); // caller must do lhmsll_has_key to check validity int lhmsll_test_and_get(lhmsll_t* pmap, char* key, long long* pval); // *pval undefined if return is FALSE +int lhmsll_test_and_increment(lhmsll_t* pmap, char* key); // increments value only if mapping exists lhmslle_t* lhmsll_get_entry(lhmsll_t* pmap, char* key); int lhmsll_has_key(lhmsll_t* pmap, char* key); diff --git a/c/draft-release-notes.md b/c/draft-release-notes.md index 2ed3b8b32..781c50ec0 100644 --- a/c/draft-release-notes.md +++ b/c/draft-release-notes.md @@ -6,6 +6,8 @@ This is a relatively minor release, containing feature requests. * xxx min/max functions and stats1/merge-fields min/max/percentile mix int and string. esp. string-only order statistics. doclink for mixed case. interpolation obv nonsensical. +* xxx count-distinct -u + * xxx `./configure` vs. `autoreconf -fiv` 1st, and which issue is resolved by this. * xxx UTF-8 BOM strip for CSV files; resolves xxx diff --git a/c/mapping/mapper_uniq.c b/c/mapping/mapper_uniq.c index f984f26f2..133697aac 100644 --- a/c/mapping/mapper_uniq.c +++ b/c/mapping/mapper_uniq.c @@ -5,6 +5,7 @@ #include "lib/mlrutil.h" #include "containers/sllv.h" #include "containers/lhmslv.h" +#include "containers/lhmsv.h" #include "containers/lhmsll.h" #include "containers/mixutil.h" #include "mapping/mappers.h" @@ -18,7 +19,7 @@ typedef struct _mapper_uniq_state_t { int show_counts; int show_num_distinct_only; lhmslv_t* pcounts_by_group; - lhmsll_t* pcounts_unlashed; + lhmsv_t* pcounts_unlashed; // string field name -> string field value -> long long count char* output_field_name; } mapper_uniq_state_t; @@ -28,10 +29,11 @@ static mapper_t* mapper_uniq_parse_cli(int* pargi, int argc, char** argv, static void mapper_count_distinct_usage(FILE* o, char* argv0, char* verb); static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** argv, cli_reader_opts_t* _, cli_writer_opts_t* __); -static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, +static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, int do_lashed, int show_counts, int show_num_distinct_only, char* output_field_name); static void mapper_uniq_free(mapper_t* pmapper, context_t* _); +static sllv_t* mapper_uniq_process_unlashed(lrec_t* pinrec, context_t* pctx, void* pvstate); static sllv_t* mapper_uniq_process_num_distinct_only(lrec_t* pinrec, context_t* pctx, void* pvstate); static sllv_t* mapper_uniq_process_with_counts(lrec_t* pinrec, context_t* pctx, void* pvstate); static sllv_t* mapper_uniq_process_no_counts(lrec_t* pinrec, context_t* pctx, void* pvstate); @@ -55,8 +57,14 @@ mapper_setup_t mapper_uniq_setup = { static void mapper_count_distinct_usage(FILE* o, char* argv0, char* verb) { fprintf(o, "Usage: %s %s [options]\n", argv0, verb); fprintf(o, "-f {a,b,c} Field names for distinct count.\n"); - fprintf(o, "-n Show only the number of distinct values.\n"); + fprintf(o, "-n Show only the number of distinct values. Not compatible with -u.\n"); fprintf(o, "-o {name} Field name for output count. Default \"%s\".\n", DEFAULT_OUTPUT_FIELD_NAME); + fprintf(o, " Ignored with -u.\n"); + fprintf(o, "-u Do unlashed counts for multiple field names. With -f a,b and\n"); + fprintf(o, " without -u, computes counts for distinct combinations of a\n"); + fprintf(o, " and b field values. With -f a,b and with -u, computes counts\n"); + fprintf(o, " for distinct a field values and counts for distinct b field\n"); + fprintf(o, " values separately.\n"); fprintf(o, "Prints number of records having distinct values for specified field names.\n"); fprintf(o, "Same as uniq -c.\n"); } @@ -68,6 +76,7 @@ static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** ar slls_t* pfield_names = NULL; int show_num_distinct_only = FALSE; char* output_field_name = DEFAULT_OUTPUT_FIELD_NAME; + int do_lashed = TRUE; char* verb = argv[(*pargi)++]; @@ -75,6 +84,7 @@ static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** ar ap_define_string_list_flag(pstate, "-f", &pfield_names); ap_define_true_flag(pstate, "-n", &show_num_distinct_only); ap_define_string_flag(pstate, "-o", &output_field_name); + ap_define_false_flag(pstate, "-u", &do_lashed); if (!ap_parse(pstate, verb, pargi, argc, argv)) { mapper_count_distinct_usage(stderr, argv[0], verb); @@ -85,8 +95,12 @@ static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** ar mapper_count_distinct_usage(stderr, argv[0], verb); return NULL; } + if (!do_lashed && show_num_distinct_only) { + mapper_count_distinct_usage(stderr, argv[0], verb); + return NULL; + } - return mapper_uniq_alloc(pstate, pfield_names, TRUE, show_num_distinct_only, + return mapper_uniq_alloc(pstate, pfield_names, do_lashed, TRUE, show_num_distinct_only, output_field_name); } @@ -108,6 +122,7 @@ static mapper_t* mapper_uniq_parse_cli(int* pargi, int argc, char** argv, int show_counts = FALSE; int show_num_distinct_only = FALSE; char* output_field_name = DEFAULT_OUTPUT_FIELD_NAME; + int do_lashed = TRUE; char* verb = argv[(*pargi)++]; @@ -128,12 +143,12 @@ static mapper_t* mapper_uniq_parse_cli(int* pargi, int argc, char** argv, return NULL; } - return mapper_uniq_alloc(pstate, pgroup_by_field_names, show_counts, show_num_distinct_only, + return mapper_uniq_alloc(pstate, pgroup_by_field_names, do_lashed, show_counts, show_num_distinct_only, output_field_name); } // ---------------------------------------------------------------- -static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, +static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, int do_lashed, int show_counts, int show_num_distinct_only, char* output_field_name) { mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); @@ -145,11 +160,13 @@ static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_na pstate->show_counts = show_counts; pstate->show_num_distinct_only = show_num_distinct_only; pstate->pcounts_by_group = lhmslv_alloc(); - pstate->pcounts_unlashed = lhmsll_alloc(); + pstate->pcounts_unlashed = lhmsv_alloc(); pstate->output_field_name = output_field_name; pmapper->pvstate = pstate; - if (show_num_distinct_only) + if (!do_lashed) + pmapper->pprocess_func = mapper_uniq_process_unlashed; + else if (show_num_distinct_only) pmapper->pprocess_func = mapper_uniq_process_num_distinct_only; else if (show_counts) pmapper->pprocess_func = mapper_uniq_process_with_counts; @@ -169,7 +186,11 @@ static void mapper_uniq_free(mapper_t* pmapper, context_t* _) { free(pcount); } lhmslv_free(pstate->pcounts_by_group); - lhmsll_free(pstate->pcounts_unlashed); + for (lhmsve_t* pb = pstate->pcounts_unlashed->phead; pb != NULL; pb = pb->pnext) { + lhmsll_t* pmap = pb->pvvalue; + lhmsll_free(pmap); + } + lhmsv_free(pstate->pcounts_unlashed); pstate->pgroup_by_field_names = NULL; pstate->pcounts_by_group = NULL; pstate->pcounts_unlashed = NULL; @@ -179,10 +200,50 @@ static void mapper_uniq_free(mapper_t* pmapper, context_t* _) { } // ---------------------------------------------------------------- +static sllv_t* mapper_uniq_process_unlashed(lrec_t* pinrec, context_t* pctx, void* pvstate) { + mapper_uniq_state_t* pstate = pvstate; + if (pinrec != NULL) { + for (sllse_t* pe = pstate->pgroup_by_field_names->phead; pe != NULL; pe = pe->pnext) { + char* field_name = pe->value; + lhmsll_t* pcounts_for_field_name = lhmsv_get(pstate->pcounts_unlashed, field_name); + if (pcounts_for_field_name == NULL) { + pcounts_for_field_name = lhmsll_alloc(); + lhmsv_put(pstate->pcounts_unlashed, field_name, pcounts_for_field_name, NO_FREE); + } + char* field_value = lrec_get(pinrec, field_name); + if (field_value != NULL) { + if (!lhmsll_test_and_increment(pcounts_for_field_name, field_value)) { + lhmsll_put(pcounts_for_field_name, mlr_strdup_or_die(field_value), 1LL, FREE_ENTRY_KEY); + } + } + } + lrec_free(pinrec); + return NULL; + } + else { + sllv_t* poutrecs = sllv_alloc(); + for (lhmsve_t* pe = pstate->pcounts_unlashed->phead; pe != NULL; pe = pe->pnext) { + lrec_t* poutrec = lrec_unbacked_alloc(); + char* field_name= pe->key; + lhmsll_t* pcounts_for_field_name = pe->pvvalue; + lrec_put(poutrec, "field", field_name, NO_FREE); + for (lhmslle_t* pf = pcounts_for_field_name->phead; pf != NULL; pf = pf->pnext) { + char* field_value = pf->key; + lrec_put(poutrec, mlr_paste_2_strings(field_value, "_count"), mlr_alloc_string_from_ll(pf->value), + FREE_ENTRY_KEY|FREE_ENTRY_VALUE); + } + sllv_append(poutrecs, poutrec); + } + sllv_append(poutrecs, NULL); + return poutrecs; + } +} + static sllv_t* mapper_uniq_process_num_distinct_only(lrec_t* pinrec, context_t* pctx, void* pvstate) { mapper_uniq_state_t* pstate = pvstate; if (pinrec != NULL) { - slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names); + slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, + pstate->pgroup_by_field_names); if (pgroup_by_field_values != NULL) { unsigned long long* pcount = lhmslv_get(pstate->pcounts_by_group, pgroup_by_field_values); if (pcount == NULL) { @@ -213,7 +274,8 @@ static sllv_t* mapper_uniq_process_num_distinct_only(lrec_t* pinrec, context_t* static sllv_t* mapper_uniq_process_with_counts(lrec_t* pinrec, context_t* pctx, void* pvstate) { mapper_uniq_state_t* pstate = pvstate; if (pinrec != NULL) { - slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names); + slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, + pstate->pgroup_by_field_names); if (pgroup_by_field_values != NULL) { unsigned long long* pcount = lhmslv_get(pstate->pcounts_by_group, pgroup_by_field_values); if (pcount == NULL) { @@ -259,7 +321,8 @@ static sllv_t* mapper_uniq_process_no_counts(lrec_t* pinrec, context_t* pctx, vo return sllv_single(NULL); } - slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names); + slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, + pstate->pgroup_by_field_names); if (pgroup_by_field_values == NULL) { lrec_free(pinrec); return NULL; diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out index 963d8c6c4..de4569394 100644 --- a/c/reg_test/expected/out +++ b/c/reg_test/expected/out @@ -918,6 +918,10 @@ a=zee,b=wye,count=2 a=hat,b=wye,count=2 a=pan,b=wye,count=2 +mlr count-distinct -f a,b -u ./reg_test/input/small ./reg_test/input/abixy +field=a,pan_count=4,eks_count=6,wye_count=4,zee_count=4,hat_count=2 +field=b,pan_count=8,wye_count=10,zee_count=2 + mlr count-distinct -f a -n ./reg_test/input/small ./reg_test/input/abixy count=5 diff --git a/c/reg_test/run b/c/reg_test/run index 1df8b008d..5304818da 100755 --- a/c/reg_test/run +++ b/c/reg_test/run @@ -261,8 +261,9 @@ run_mlr uniq -f a,b $indir/abixy-het run_mlr uniq -c -g a $indir/abixy-het run_mlr uniq -c -g a,b $indir/abixy-het -run_mlr count-distinct -f a $indir/small $indir/abixy -run_mlr count-distinct -f a,b $indir/small $indir/abixy +run_mlr count-distinct -f a $indir/small $indir/abixy +run_mlr count-distinct -f a,b $indir/small $indir/abixy +run_mlr count-distinct -f a,b -u $indir/small $indir/abixy run_mlr count-distinct -f a -n $indir/small $indir/abixy run_mlr count-distinct -f a,b -n $indir/small $indir/abixy diff --git a/c/todo.txt b/c/todo.txt index a32426a2d..61ce82d74 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -22,8 +22,6 @@ FUNDAM: ---------------------------------------------------------------- airable: -! count-distinct --unlashed - ! !autoreconf doc note w/ as-of-5.2.0 caveat * reg_test/run --mlrexec flag diff --git a/doc/manpage.html b/doc/manpage.html index 6a1ddac8d..c91841f76 100644 --- a/doc/manpage.html +++ b/doc/manpage.html @@ -566,8 +566,14 @@ VERBS count-distinct Usage: mlr count-distinct [options] -f {a,b,c} Field names for distinct count. - -n Show only the number of distinct values. + -n Show only the number of distinct values. Not compatible with -u. -o {name} Field name for output count. Default "count". + Ignored with -u. + -u Do unlashed counts for multiple field names. With -f a,b and + without -u, computes counts for distinct combinations of a + and b field values. With -f a,b and with -u, computes counts + for distinct a field values and counts for distinct b field + values separately. Prints number of records having distinct values for specified field names. Same as uniq -c. @@ -2214,7 +2220,7 @@ SEE ALSO - 2017-04-27 MILLER(1) + 2017-05-05 MILLER(1)
diff --git a/doc/manpage.txt b/doc/manpage.txt index 3b5f6d492..40ffbb450 100644 --- a/doc/manpage.txt +++ b/doc/manpage.txt @@ -373,8 +373,14 @@ VERBS count-distinct Usage: mlr count-distinct [options] -f {a,b,c} Field names for distinct count. - -n Show only the number of distinct values. + -n Show only the number of distinct values. Not compatible with -u. -o {name} Field name for output count. Default "count". + Ignored with -u. + -u Do unlashed counts for multiple field names. With -f a,b and + without -u, computes counts for distinct combinations of a + and b field values. With -f a,b and with -u, computes counts + for distinct a field values and counts for distinct b field + values separately. Prints number of records having distinct values for specified field names. Same as uniq -c. @@ -2021,4 +2027,4 @@ SEE ALSO - 2017-04-27 MILLER(1) + 2017-05-05 MILLER(1) diff --git a/doc/mlr.1 b/doc/mlr.1 index bcefdf016..8bdefe0d7 100644 --- a/doc/mlr.1 +++ b/doc/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2017-04-27 +.\" Date: 2017-05-05 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2017-04-27" "\ \&" "\ \&" +.TH "MILLER" "1" "2017-05-05" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -508,8 +508,14 @@ Useful for doing a well-formatted check on input data. .nf Usage: mlr count-distinct [options] -f {a,b,c} Field names for distinct count. --n Show only the number of distinct values. +-n Show only the number of distinct values. Not compatible with -u. -o {name} Field name for output count. Default "count". + Ignored with -u. +-u Do unlashed counts for multiple field names. With -f a,b and + without -u, computes counts for distinct combinations of a + and b field values. With -f a,b and with -u, computes counts + for distinct a field values and counts for distinct b field + values separately. Prints number of records having distinct values for specified field names. Same as uniq -c. .fi diff --git a/doc/reference-dsl.html b/doc/reference-dsl.html index 0f8acbf8b..2e6aa073a 100644 --- a/doc/reference-dsl.html +++ b/doc/reference-dsl.html @@ -1419,6 +1419,16 @@ $ mlr --from ../c/s put ' } $* = f({"a": $a, "x": $x}); ' +a=pan,x=69.358029 +a=eks,x=151.735993 +a=wye,x=40.920661 +a=eks,x=76.279879 +a=wye,x=114.657784 +a=zee,x=105.425232 +a=eks,x=122.356812 +a=zee,x=119.710802 +a=hat,x=6.288375 +a=pan,x=100.525201 diff --git a/doc/reference-verbs.html b/doc/reference-verbs.html index 4fd787dfb..15e48c454 100644 --- a/doc/reference-verbs.html +++ b/doc/reference-verbs.html @@ -604,8 +604,14 @@ Useful for doing a well-formatted check on input data. $ mlr count-distinct --help Usage: mlr count-distinct [options] -f {a,b,c} Field names for distinct count. --n Show only the number of distinct values. +-n Show only the number of distinct values. Not compatible with -u. -o {name} Field name for output count. Default "count". + Ignored with -u. +-u Do unlashed counts for multiple field names. With -f a,b and + without -u, computes counts for distinct combinations of a + and b field values. With -f a,b and with -u, computes counts + for distinct a field values and counts for distinct b field + values separately. Prints number of records having distinct values for specified field names. Same as uniq -c.