Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-19 01:15:21 +00:00
Code Issues Projects Releases Wiki Activity

join dev checkpoint

Browse source
This commit is contained in:
John Kerl 2015-06-11 19:01:17 -04:00
parent 1cd5e25e70
commit 718b53b54f
4 changed files with 98 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

2
c/input/lrec_reader_mmap.h
View file

@ -12,7 +12,7 @@ typedef void lrec_reader_mmap_sof_func_t(void* pvstate);
typedef struct _lrec_reader_mmap_t {
void* pvstate;
lrec_reader_mmap_process_func_t* pprocess_func;
lrec_reader_mmap_sof_func_t* psof_func;
lrec_reader_mmap_sof_func_t* psof_func;
} lrec_reader_mmap_t;
#endif // LREC_READER_MMAP_H

2
c/input/lrec_reader_stdio.h
View file

@ -12,7 +12,7 @@ typedef void lrec_reader_stdio_free_func_t(void* pvstate);
typedef struct _lrec_reader_stdio_t {
void* pvstate;
lrec_reader_stdio_process_func_t* pprocess_func;
lrec_reader_stdio_sof_func_t* psof_func;
lrec_reader_stdio_sof_func_t* psof_func;
lrec_reader_stdio_free_func_t* pfree_func;
} lrec_reader_stdio_t;

11
c/lib/mlrmath.c
View file

@ -154,13 +154,20 @@ double qnorm(double x) {
}
// ----------------------------------------------------------------
// This is essentially Newton-Raphson.
// This is a tangent-following method not unlike Newton-Raphson:
// * We can compute qnorm(y) = integral from -infinity to y of (1/sqrt(2pi)) exp(-t^2/2) dt.
// * We can compute derivative of qnorm(y) = (1/sqrt(2pi)) exp(-y^2/2).
// * We cannot explicitly compute invqnorm(y).
// * If dx/dy = (1/sqrt(2pi)) exp(-y^2/2) then dy/dx = sqrt(2pi) exp(y^2/2).
//
// This means we *can* compute the derivative of invqnorm even though we
// can't compute the function itself. So the essence of the method is to
// follow the tangent line to form successive approximations: we have known function input x
// and unknown function output y and initial guess y0. At each step we find the intersection
// of the tangent line at y_n with the vertical line at x, to find y_{n+1}. Specificall:
//
// * Even though we can't compute y = q^-1(x) we can compute x = q(y).
// * Start with linear approximation for y.
// * Start with initial guess for y (y0 = 0.0 or y0 = x both are OK).
// * Find x = q(y). Since q (and therefore q^-1) are 1-1, we're done if qnorm(invqnorm(x)) is small.
// * Else iterate: using point-slope form, (y_{n+1} - y_n) / (x_{n+1} - x_n) = m = sqrt(2pi) exp(y_n^2/2).
// Here x_2 = x (the input) and x_1 = q(y_1).

126
c/mapping/mapper_join.c
View file

@ -1,7 +1,9 @@
#include "lib/mlr_globals.h"
#include "lib/mlrutil.h"
#include "containers/lrec.h"
#include "containers/sllv.h"
#include "containers/lhmsi.h"
#include "containers/lhmslv.h"
#include "containers/mixutil.h"
#include "mapping/mappers.h"
#include "cli/argparse.h"
@ -9,14 +11,25 @@ typedef struct _mapper_join_state_t {
slls_t* pleft_field_names;
slls_t* pright_field_names;
slls_t* poutput_field_names;
int allow_unsorted_input;
int emit_pairables;
int emit_left_unpairables;
int emit_right_unpairables;
char* left_file_name;
lhmslv_t* precords_by_key_field_names; // For unsorted input
} mapper_join_state_t;
// ----------------------------------------------------------------
static sllv_t* mapper_join_process(lrec_t* pinrec, context_t* pctx, void* pvstate) {
static sllv_t* mapper_join_process_unsorted(lrec_t* pinrec, context_t* pctx, void* pvstate) {
if (pinrec == NULL) {
return sllv_single(NULL);
}
//mapper_join_state_t* pstate = (mapper_join_state_t*)pvstate;
//int num_found = 0;
return sllv_single(pinrec);
}
// ----------------------------------------------------------------
static sllv_t* mapper_join_process_sorted(lrec_t* pinrec, context_t* pctx, void* pvstate) {
if (pinrec == NULL) {
return sllv_single(NULL);
}
@ -36,6 +49,28 @@ static void mapper_join_free(void* pvstate) {
slls_free(pstate->poutput_field_names);
}
// ----------------------------------------------------------------
static lhmslv_t* ingest_left_file(char* left_file_name) {
lhmslv_t* precords_by_key_field_names = lhmslv_alloc();
while (TRUE) {
lrec_t* pinrec = NULL; // need reader ...
if (pinrec == NULL)
break;
slls_t* pkey_field_names = mlr_keys_from_record(pinrec);
sllv_t* plist = lhmslv_get(precords_by_key_field_names, pkey_field_names);
if (plist == NULL) {
plist = sllv_alloc();
sllv_add(plist, pinrec);
lhmslv_put(precords_by_key_field_names, slls_copy(pkey_field_names), plist);
} else {
sllv_add(plist, pinrec);
}
}
return precords_by_key_field_names;
}
// ----------------------------------------------------------------
static mapper_t* mapper_join_alloc(
slls_t* pleft_field_names,
slls_t* pright_field_names,
@ -43,7 +78,8 @@ static mapper_t* mapper_join_alloc(
int allow_unsorted_input,
int emit_pairables,
int emit_left_unpairables,
int emit_right_unpairables)
int emit_right_unpairables,
char* left_file_name)
{
mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
@ -51,13 +87,20 @@ static mapper_t* mapper_join_alloc(
pstate->pleft_field_names = pleft_field_names;
pstate->pright_field_names = pright_field_names;
pstate->poutput_field_names = poutput_field_names;
pstate->allow_unsorted_input = allow_unsorted_input;
pstate->emit_pairables = emit_pairables;
pstate->emit_left_unpairables = emit_left_unpairables;
pstate->emit_right_unpairables = emit_right_unpairables;
pmapper->pvstate = (void*)pstate;
pmapper->pprocess_func = mapper_join_process;
if (allow_unsorted_input) {
pstate->precords_by_key_field_names = ingest_left_file(left_file_name);
pstate->left_file_name = NULL;
pmapper->pprocess_func = mapper_join_process_unsorted;
} else {
pstate->precords_by_key_field_names = NULL;
pstate->left_file_name = left_file_name;
pmapper->pprocess_func = mapper_join_process_sorted;
}
pmapper->pfree_func = mapper_join_free;
return pmapper;
@ -66,20 +109,21 @@ static mapper_t* mapper_join_alloc(
// ----------------------------------------------------------------
static void mapper_join_usage(char* argv0, char* verb) {
fprintf(stdout, "Usage: %s %s [options]\n", argv0, verb);
fprintf(stdout, "-1 {a,b,c}\n");
fprintf(stdout, "-2 {a,b,c}\n");
fprintf(stdout, "-j {a,b,c}\n");
fprintf(stdout, "-a 1\n");
fprintf(stdout, "-a 2\n");
fprintf(stdout, "-v 1\n");
fprintf(stdout, "-v 2\n");
fprintf(stdout, "-o {a,b,c} -- do i want this?? or use then cut -f {a,b,c}...\n");
fprintf(stdout, "-e EMPTY\n");
fprintf(stdout, "xxx write me up.\n");
fprintf(stdout, "-f {left file name}\n");
fprintf(stdout, "-l {a,b,c}\n");
fprintf(stdout, "-r {a,b,c}\n");
fprintf(stdout, "-o {a,b,c}\n");
fprintf(stdout, "--np\n");
fprintf(stdout, "--ul\n");
fprintf(stdout, "--ur\n");
fprintf(stdout, "-u\n");
fprintf(stdout, "-e EMPTY\n");
}
// ----------------------------------------------------------------
static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
char* left_file_name = NULL;
slls_t* pleft_field_names = NULL;
slls_t* pright_field_names = NULL;
slls_t* poutput_field_names = NULL;
@ -90,36 +134,40 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) {
char* verb = argv[(*pargi)++];
int argi = *pargi;
while (argv[argi][0] == '-') {
// if (streq(argv[argi], "--at-least")) {
// criterion = HAVING_FIELDS_AT_LEAST;
// } else if (streq(argv[argi], "--which-are")) {
// criterion = HAVING_FIELDS_WHICH_ARE;
// } else if (streq(argv[argi], "--at-most")) {
// criterion = HAVING_FIELDS_AT_MOST;
// } else {
// mapper_join_usage(argv[0], verb);
// return NULL;
// }
ap_state_t* pstate = ap_alloc();
ap_define_string_flag(pstate, "-f", &left_file_name);
ap_define_string_list_flag(pstate, "-l", &pleft_field_names);
ap_define_string_list_flag(pstate, "-r", &pright_field_names);
ap_define_string_list_flag(pstate, "-o", &poutput_field_names);
ap_define_false_flag(pstate, "--np", &emit_pairables);
ap_define_int_flag(pstate, "--ul", &emit_left_unpairables);
ap_define_int_flag(pstate, "--ur", &emit_left_unpairables);
ap_define_true_flag(pstate, "-u", &allow_unsorted_input);
if (argc - argi < 2) {
return NULL;
}
if (pleft_field_names != NULL)
slls_free(pleft_field_names);
pleft_field_names = slls_from_line(argv[argi+1], ',', FALSE);
argi += 2;
}
if (pleft_field_names == NULL) {
if (!ap_parse(pstate, verb, pargi, argc, argv)) {
mapper_join_usage(argv[0], verb);
return NULL;
}
*pargi = argi;
if (left_file_name == NULL) {
fprintf(stderr, "%s %s: need left file name\n", MLR_GLOBALS.argv0, verb);
mapper_join_usage(argv[0], verb);
return NULL;
}
if (pleft_field_names == NULL) {
fprintf(stderr, "%s %s: need left field names\n", MLR_GLOBALS.argv0, verb);
mapper_join_usage(argv[0], verb);
return NULL;
}
if (pright_field_names == NULL)
pright_field_names = slls_copy(pleft_field_names);
if (poutput_field_names == NULL)
poutput_field_names = slls_copy(pleft_field_names);
return mapper_join_alloc(pleft_field_names, pright_field_names, poutput_field_names,
allow_unsorted_input, emit_pairables, emit_left_unpairables, emit_right_unpairables);
allow_unsorted_input, emit_pairables, emit_left_unpairables, emit_right_unpairables,
left_file_name);
}
// ----------------------------------------------------------------