Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-27 17:53:57 +00:00
Code Issues Projects Releases Wiki Activity

count-distinct --unlashed

Browse source
This commit is contained in:
John Kerl 2017-05-05 15:40:08 -07:00
parent 4795137160
commit 4246088e53
12 changed files with 143 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

16
c/containers/lhmsll.c
View file

@ -210,6 +210,22 @@ int lhmsll_test_and_get(lhmsll_t* pmap, char* key, long long* pval) {
}
}
int lhmsll_test_and_increment(lhmsll_t* pmap, char* key) {
int ideal_index = 0;
int index = lhmsll_find_index_for_key(pmap, key, &ideal_index);
lhmslle_t* pe = &pmap->entries[index];
if (pmap->states[index] == OCCUPIED) {
pe->value++;
return TRUE;
} else if (pmap->states[index] == EMPTY) {
return FALSE;
} else {
fprintf(stderr, "%s: lhmsll_find_index_for_key did not find end of chain.\n", MLR_GLOBALS.bargv0);
exit(1);
}
}
lhmslle_t* lhmsll_get_entry(lhmsll_t* pmap, char* key) {
int ideal_index = 0;
int index = lhmsll_find_index_for_key(pmap, key, &ideal_index);

1
c/containers/lhmsll.h
View file

@ -45,6 +45,7 @@ void lhmsll_free(lhmsll_t* pmap);
void lhmsll_put(lhmsll_t* pmap, char* key, int value, char free_flags);
long long lhmsll_get(lhmsll_t* pmap, char* key); // caller must do lhmsll_has_key to check validity
int lhmsll_test_and_get(lhmsll_t* pmap, char* key, long long* pval); // *pval undefined if return is FALSE
int lhmsll_test_and_increment(lhmsll_t* pmap, char* key); // increments value only if mapping exists
lhmslle_t* lhmsll_get_entry(lhmsll_t* pmap, char* key);
int lhmsll_has_key(lhmsll_t* pmap, char* key);

2
c/draft-release-notes.md
View file

@ -6,6 +6,8 @@ This is a relatively minor release, containing feature requests.
* xxx min/max functions and stats1/merge-fields min/max/percentile mix int and string. esp. string-only order statistics. doclink for mixed case. interpolation obv nonsensical.
* xxx count-distinct -u
* xxx `./configure` vs. `autoreconf -fiv` 1st, and which issue is resolved by this.
* xxx UTF-8 BOM strip for CSV files; resolves xxx

87
c/mapping/mapper_uniq.c
View file

@ -5,6 +5,7 @@
#include "lib/mlrutil.h"
#include "containers/sllv.h"
#include "containers/lhmslv.h"
#include "containers/lhmsv.h"
#include "containers/lhmsll.h"
#include "containers/mixutil.h"
#include "mapping/mappers.h"
@ -18,7 +19,7 @@ typedef struct _mapper_uniq_state_t {
int show_counts;
int show_num_distinct_only;
lhmslv_t* pcounts_by_group;
lhmsll_t* pcounts_unlashed;
lhmsv_t* pcounts_unlashed; // string field name -> string field value -> long long count
char* output_field_name;
} mapper_uniq_state_t;
@ -28,10 +29,11 @@ static mapper_t* mapper_uniq_parse_cli(int* pargi, int argc, char** argv,
static void mapper_count_distinct_usage(FILE* o, char* argv0, char* verb);
static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** argv,
cli_reader_opts_t* _, cli_writer_opts_t* __);
static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names,
static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, int do_lashed,
int show_counts, int show_num_distinct_only, char* output_field_name);
static void mapper_uniq_free(mapper_t* pmapper, context_t* _);
static sllv_t* mapper_uniq_process_unlashed(lrec_t* pinrec, context_t* pctx, void* pvstate);
static sllv_t* mapper_uniq_process_num_distinct_only(lrec_t* pinrec, context_t* pctx, void* pvstate);
static sllv_t* mapper_uniq_process_with_counts(lrec_t* pinrec, context_t* pctx, void* pvstate);
static sllv_t* mapper_uniq_process_no_counts(lrec_t* pinrec, context_t* pctx, void* pvstate);
@ -55,8 +57,14 @@ mapper_setup_t mapper_uniq_setup = {
static void mapper_count_distinct_usage(FILE* o, char* argv0, char* verb) {
fprintf(o, "Usage: %s %s [options]\n", argv0, verb);
fprintf(o, "-f {a,b,c} Field names for distinct count.\n");
fprintf(o, "-n Show only the number of distinct values.\n");
fprintf(o, "-n Show only the number of distinct values. Not compatible with -u.\n");
fprintf(o, "-o {name} Field name for output count. Default \"%s\".\n", DEFAULT_OUTPUT_FIELD_NAME);
fprintf(o, " Ignored with -u.\n");
fprintf(o, "-u Do unlashed counts for multiple field names. With -f a,b and\n");
fprintf(o, " without -u, computes counts for distinct combinations of a\n");
fprintf(o, " and b field values. With -f a,b and with -u, computes counts\n");
fprintf(o, " for distinct a field values and counts for distinct b field\n");
fprintf(o, " values separately.\n");
fprintf(o, "Prints number of records having distinct values for specified field names.\n");
fprintf(o, "Same as uniq -c.\n");
}
@ -68,6 +76,7 @@ static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** ar
slls_t* pfield_names = NULL;
int show_num_distinct_only = FALSE;
char* output_field_name = DEFAULT_OUTPUT_FIELD_NAME;
int do_lashed = TRUE;
char* verb = argv[(*pargi)++];
@ -75,6 +84,7 @@ static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** ar
ap_define_string_list_flag(pstate, "-f", &pfield_names);
ap_define_true_flag(pstate, "-n", &show_num_distinct_only);
ap_define_string_flag(pstate, "-o", &output_field_name);
ap_define_false_flag(pstate, "-u", &do_lashed);
if (!ap_parse(pstate, verb, pargi, argc, argv)) {
mapper_count_distinct_usage(stderr, argv[0], verb);
@ -85,8 +95,12 @@ static mapper_t* mapper_count_distinct_parse_cli(int* pargi, int argc, char** ar
mapper_count_distinct_usage(stderr, argv[0], verb);
return NULL;
}
if (!do_lashed && show_num_distinct_only) {
mapper_count_distinct_usage(stderr, argv[0], verb);
return NULL;
}
return mapper_uniq_alloc(pstate, pfield_names, TRUE, show_num_distinct_only,
return mapper_uniq_alloc(pstate, pfield_names, do_lashed, TRUE, show_num_distinct_only,
output_field_name);
}
@ -108,6 +122,7 @@ static mapper_t* mapper_uniq_parse_cli(int* pargi, int argc, char** argv,
int show_counts = FALSE;
int show_num_distinct_only = FALSE;
char* output_field_name = DEFAULT_OUTPUT_FIELD_NAME;
int do_lashed = TRUE;
char* verb = argv[(*pargi)++];
@ -128,12 +143,12 @@ static mapper_t* mapper_uniq_parse_cli(int* pargi, int argc, char** argv,
return NULL;
}
return mapper_uniq_alloc(pstate, pgroup_by_field_names, show_counts, show_num_distinct_only,
return mapper_uniq_alloc(pstate, pgroup_by_field_names, do_lashed, show_counts, show_num_distinct_only,
output_field_name);
}
// ----------------------------------------------------------------
static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names,
static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_names, int do_lashed,
int show_counts, int show_num_distinct_only, char* output_field_name)
{
mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
@ -145,11 +160,13 @@ static mapper_t* mapper_uniq_alloc(ap_state_t* pargp, slls_t* pgroup_by_field_na
pstate->show_counts = show_counts;
pstate->show_num_distinct_only = show_num_distinct_only;
pstate->pcounts_by_group = lhmslv_alloc();
pstate->pcounts_unlashed = lhmsll_alloc();
pstate->pcounts_unlashed = lhmsv_alloc();
pstate->output_field_name = output_field_name;
pmapper->pvstate = pstate;
if (show_num_distinct_only)
if (!do_lashed)
pmapper->pprocess_func = mapper_uniq_process_unlashed;
else if (show_num_distinct_only)
pmapper->pprocess_func = mapper_uniq_process_num_distinct_only;
else if (show_counts)
pmapper->pprocess_func = mapper_uniq_process_with_counts;
@ -169,7 +186,11 @@ static void mapper_uniq_free(mapper_t* pmapper, context_t* _) {
free(pcount);
}
lhmslv_free(pstate->pcounts_by_group);
lhmsll_free(pstate->pcounts_unlashed);
for (lhmsve_t* pb = pstate->pcounts_unlashed->phead; pb != NULL; pb = pb->pnext) {
lhmsll_t* pmap = pb->pvvalue;
lhmsll_free(pmap);
}
lhmsv_free(pstate->pcounts_unlashed);
pstate->pgroup_by_field_names = NULL;
pstate->pcounts_by_group = NULL;
pstate->pcounts_unlashed = NULL;
@ -179,10 +200,50 @@ static void mapper_uniq_free(mapper_t* pmapper, context_t* _) {
}
// ----------------------------------------------------------------
static sllv_t* mapper_uniq_process_unlashed(lrec_t* pinrec, context_t* pctx, void* pvstate) {
mapper_uniq_state_t* pstate = pvstate;
if (pinrec != NULL) {
for (sllse_t* pe = pstate->pgroup_by_field_names->phead; pe != NULL; pe = pe->pnext) {
char* field_name = pe->value;
lhmsll_t* pcounts_for_field_name = lhmsv_get(pstate->pcounts_unlashed, field_name);
if (pcounts_for_field_name == NULL) {
pcounts_for_field_name = lhmsll_alloc();
lhmsv_put(pstate->pcounts_unlashed, field_name, pcounts_for_field_name, NO_FREE);
}
char* field_value = lrec_get(pinrec, field_name);
if (field_value != NULL) {
if (!lhmsll_test_and_increment(pcounts_for_field_name, field_value)) {
lhmsll_put(pcounts_for_field_name, mlr_strdup_or_die(field_value), 1LL, FREE_ENTRY_KEY);
}
}
}
lrec_free(pinrec);
return NULL;
}
else {
sllv_t* poutrecs = sllv_alloc();
for (lhmsve_t* pe = pstate->pcounts_unlashed->phead; pe != NULL; pe = pe->pnext) {
lrec_t* poutrec = lrec_unbacked_alloc();
char* field_name= pe->key;
lhmsll_t* pcounts_for_field_name = pe->pvvalue;
lrec_put(poutrec, "field", field_name, NO_FREE);
for (lhmslle_t* pf = pcounts_for_field_name->phead; pf != NULL; pf = pf->pnext) {
char* field_value = pf->key;
lrec_put(poutrec, mlr_paste_2_strings(field_value, "_count"), mlr_alloc_string_from_ll(pf->value),
FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
}
sllv_append(poutrecs, poutrec);
}
sllv_append(poutrecs, NULL);
return poutrecs;
}
}
static sllv_t* mapper_uniq_process_num_distinct_only(lrec_t* pinrec, context_t* pctx, void* pvstate) {
mapper_uniq_state_t* pstate = pvstate;
if (pinrec != NULL) {
slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names);
slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec,
pstate->pgroup_by_field_names);
if (pgroup_by_field_values != NULL) {
unsigned long long* pcount = lhmslv_get(pstate->pcounts_by_group, pgroup_by_field_values);
if (pcount == NULL) {
@ -213,7 +274,8 @@ static sllv_t* mapper_uniq_process_num_distinct_only(lrec_t* pinrec, context_t*
static sllv_t* mapper_uniq_process_with_counts(lrec_t* pinrec, context_t* pctx, void* pvstate) {
mapper_uniq_state_t* pstate = pvstate;
if (pinrec != NULL) {
slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names);
slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec,
pstate->pgroup_by_field_names);
if (pgroup_by_field_values != NULL) {
unsigned long long* pcount = lhmslv_get(pstate->pcounts_by_group, pgroup_by_field_values);
if (pcount == NULL) {
@ -259,7 +321,8 @@ static sllv_t* mapper_uniq_process_no_counts(lrec_t* pinrec, context_t* pctx, vo
return sllv_single(NULL);
}
slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names);
slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec,
pstate->pgroup_by_field_names);
if (pgroup_by_field_values == NULL) {
lrec_free(pinrec);
return NULL;

4
c/reg_test/expected/out
View file

@ -918,6 +918,10 @@ a=zee,b=wye,count=2
a=hat,b=wye,count=2
a=pan,b=wye,count=2
mlr count-distinct -f a,b -u ./reg_test/input/small ./reg_test/input/abixy
field=a,pan_count=4,eks_count=6,wye_count=4,zee_count=4,hat_count=2
field=b,pan_count=8,wye_count=10,zee_count=2
mlr count-distinct -f a -n ./reg_test/input/small ./reg_test/input/abixy
count=5

5
c/reg_test/run
View file

@ -261,8 +261,9 @@ run_mlr uniq -f a,b $indir/abixy-het
run_mlr uniq -c -g a $indir/abixy-het
run_mlr uniq -c -g a,b $indir/abixy-het
run_mlr count-distinct -f a $indir/small $indir/abixy
run_mlr count-distinct -f a,b $indir/small $indir/abixy
run_mlr count-distinct -f a $indir/small $indir/abixy
run_mlr count-distinct -f a,b $indir/small $indir/abixy
run_mlr count-distinct -f a,b -u $indir/small $indir/abixy
run_mlr count-distinct -f a -n $indir/small $indir/abixy
run_mlr count-distinct -f a,b -n $indir/small $indir/abixy

2
c/todo.txt
View file

@ -22,8 +22,6 @@ FUNDAM:
----------------------------------------------------------------
airable:
! count-distinct --unlashed
! !autoreconf doc note w/ as-of-5.2.0 caveat
* reg_test/run --mlrexec flag

10
doc/manpage.html
View file

@ -566,8 +566,14 @@ VERBS
count-distinct
Usage: mlr count-distinct [options]
-f {a,b,c} Field names for distinct count.
-n Show only the number of distinct values.
-n Show only the number of distinct values. Not compatible with -u.
-o {name} Field name for output count. Default "count".
Ignored with -u.
-u Do unlashed counts for multiple field names. With -f a,b and
without -u, computes counts for distinct combinations of a
and b field values. With -f a,b and with -u, computes counts
for distinct a field values and counts for distinct b field
values separately.
Prints number of records having distinct values for specified field names.
Same as uniq -c.
@ -2214,7 +2220,7 @@ SEE ALSO
2017-04-27 MILLER(1)
2017-05-05 MILLER(1)
</pre>
</div>
<p/>

10
doc/manpage.txt
View file

@ -373,8 +373,14 @@ VERBS
count-distinct
Usage: mlr count-distinct [options]
-f {a,b,c} Field names for distinct count.
-n Show only the number of distinct values.
-n Show only the number of distinct values. Not compatible with -u.
-o {name} Field name for output count. Default "count".
Ignored with -u.
-u Do unlashed counts for multiple field names. With -f a,b and
without -u, computes counts for distinct combinations of a
and b field values. With -f a,b and with -u, computes counts
for distinct a field values and counts for distinct b field
values separately.
Prints number of records having distinct values for specified field names.
Same as uniq -c.
@ -2021,4 +2027,4 @@ SEE ALSO
2017-04-27 MILLER(1)
2017-05-05 MILLER(1)

12
doc/mlr.1
View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2017-04-27
.\" Date: 2017-05-05
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2017-04-27" "\ \&" "\ \&"
.TH "MILLER" "1" "2017-05-05" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -508,8 +508,14 @@ Useful for doing a well-formatted check on input data.
.nf
Usage: mlr count-distinct [options]
-f {a,b,c} Field names for distinct count.
-n Show only the number of distinct values.
-n Show only the number of distinct values. Not compatible with -u.
-o {name} Field name for output count. Default "count".
Ignored with -u.
-u Do unlashed counts for multiple field names. With -f a,b and
without -u, computes counts for distinct combinations of a
and b field values. With -f a,b and with -u, computes counts
for distinct a field values and counts for distinct b field
values separately.
Prints number of records having distinct values for specified field names.
Same as uniq -c.
.fi

10
doc/reference-dsl.html
View file

@ -1419,6 +1419,16 @@ $ mlr --from ../c/s put '
}
$* = f({"a": $a, "x": $x});
'
a=pan,x=69.358029
a=eks,x=151.735993
a=wye,x=40.920661
a=eks,x=76.279879
a=wye,x=114.657784
a=zee,x=105.425232
a=eks,x=122.356812
a=zee,x=119.710802
a=hat,x=6.288375
a=pan,x=100.525201
</pre>
</div>
<p/>

8
doc/reference-verbs.html
View file

@ -604,8 +604,14 @@ Useful for doing a well-formatted check on input data.
$ mlr count-distinct --help
Usage: mlr count-distinct [options]
-f {a,b,c} Field names for distinct count.
-n Show only the number of distinct values.
-n Show only the number of distinct values. Not compatible with -u.
-o {name} Field name for output count. Default "count".
Ignored with -u.
-u Do unlashed counts for multiple field names. With -f a,b and
without -u, computes counts for distinct combinations of a
and b field values. With -f a,b and with -u, computes counts
for distinct a field values and counts for distinct b field
values separately.
Prints number of records having distinct values for specified field names.
Same as uniq -c.
</pre>