diff --git a/c/input/lrec_reader_mmap.h b/c/input/lrec_reader_mmap.h index d9ef1695e..c7b0d87d7 100644 --- a/c/input/lrec_reader_mmap.h +++ b/c/input/lrec_reader_mmap.h @@ -12,7 +12,7 @@ typedef void lrec_reader_mmap_sof_func_t(void* pvstate); typedef struct _lrec_reader_mmap_t { void* pvstate; lrec_reader_mmap_process_func_t* pprocess_func; - lrec_reader_mmap_sof_func_t* psof_func; + lrec_reader_mmap_sof_func_t* psof_func; } lrec_reader_mmap_t; #endif // LREC_READER_MMAP_H diff --git a/c/input/lrec_reader_stdio.h b/c/input/lrec_reader_stdio.h index 2db607457..4b2bed287 100644 --- a/c/input/lrec_reader_stdio.h +++ b/c/input/lrec_reader_stdio.h @@ -12,7 +12,7 @@ typedef void lrec_reader_stdio_free_func_t(void* pvstate); typedef struct _lrec_reader_stdio_t { void* pvstate; lrec_reader_stdio_process_func_t* pprocess_func; - lrec_reader_stdio_sof_func_t* psof_func; + lrec_reader_stdio_sof_func_t* psof_func; lrec_reader_stdio_free_func_t* pfree_func; } lrec_reader_stdio_t; diff --git a/c/lib/mlrmath.c b/c/lib/mlrmath.c index d64022d87..7de96ee95 100644 --- a/c/lib/mlrmath.c +++ b/c/lib/mlrmath.c @@ -154,13 +154,20 @@ double qnorm(double x) { } // ---------------------------------------------------------------- -// This is essentially Newton-Raphson. +// This is a tangent-following method not unlike Newton-Raphson: // * We can compute qnorm(y) = integral from -infinity to y of (1/sqrt(2pi)) exp(-t^2/2) dt. // * We can compute derivative of qnorm(y) = (1/sqrt(2pi)) exp(-y^2/2). // * We cannot explicitly compute invqnorm(y). // * If dx/dy = (1/sqrt(2pi)) exp(-y^2/2) then dy/dx = sqrt(2pi) exp(y^2/2). +// +// This means we *can* compute the derivative of invqnorm even though we +// can't compute the function itself. So the essence of the method is to +// follow the tangent line to form successive approximations: we have known function input x +// and unknown function output y and initial guess y0. At each step we find the intersection +// of the tangent line at y_n with the vertical line at x, to find y_{n+1}. Specificall: +// // * Even though we can't compute y = q^-1(x) we can compute x = q(y). -// * Start with linear approximation for y. +// * Start with initial guess for y (y0 = 0.0 or y0 = x both are OK). // * Find x = q(y). Since q (and therefore q^-1) are 1-1, we're done if qnorm(invqnorm(x)) is small. // * Else iterate: using point-slope form, (y_{n+1} - y_n) / (x_{n+1} - x_n) = m = sqrt(2pi) exp(y_n^2/2). // Here x_2 = x (the input) and x_1 = q(y_1). diff --git a/c/mapping/mapper_join.c b/c/mapping/mapper_join.c index 1bfb53a32..6e4492896 100644 --- a/c/mapping/mapper_join.c +++ b/c/mapping/mapper_join.c @@ -1,7 +1,9 @@ +#include "lib/mlr_globals.h" #include "lib/mlrutil.h" #include "containers/lrec.h" #include "containers/sllv.h" -#include "containers/lhmsi.h" +#include "containers/lhmslv.h" +#include "containers/mixutil.h" #include "mapping/mappers.h" #include "cli/argparse.h" @@ -9,14 +11,25 @@ typedef struct _mapper_join_state_t { slls_t* pleft_field_names; slls_t* pright_field_names; slls_t* poutput_field_names; - int allow_unsorted_input; int emit_pairables; int emit_left_unpairables; int emit_right_unpairables; + char* left_file_name; + lhmslv_t* precords_by_key_field_names; // For unsorted input } mapper_join_state_t; // ---------------------------------------------------------------- -static sllv_t* mapper_join_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { +static sllv_t* mapper_join_process_unsorted(lrec_t* pinrec, context_t* pctx, void* pvstate) { + if (pinrec == NULL) { + return sllv_single(NULL); + } + //mapper_join_state_t* pstate = (mapper_join_state_t*)pvstate; + //int num_found = 0; + return sllv_single(pinrec); +} + +// ---------------------------------------------------------------- +static sllv_t* mapper_join_process_sorted(lrec_t* pinrec, context_t* pctx, void* pvstate) { if (pinrec == NULL) { return sllv_single(NULL); } @@ -36,6 +49,28 @@ static void mapper_join_free(void* pvstate) { slls_free(pstate->poutput_field_names); } +// ---------------------------------------------------------------- +static lhmslv_t* ingest_left_file(char* left_file_name) { + lhmslv_t* precords_by_key_field_names = lhmslv_alloc(); + while (TRUE) { + lrec_t* pinrec = NULL; // need reader ... + if (pinrec == NULL) + break; + + slls_t* pkey_field_names = mlr_keys_from_record(pinrec); + sllv_t* plist = lhmslv_get(precords_by_key_field_names, pkey_field_names); + if (plist == NULL) { + plist = sllv_alloc(); + sllv_add(plist, pinrec); + lhmslv_put(precords_by_key_field_names, slls_copy(pkey_field_names), plist); + } else { + sllv_add(plist, pinrec); + } + } + return precords_by_key_field_names; +} + +// ---------------------------------------------------------------- static mapper_t* mapper_join_alloc( slls_t* pleft_field_names, slls_t* pright_field_names, @@ -43,7 +78,8 @@ static mapper_t* mapper_join_alloc( int allow_unsorted_input, int emit_pairables, int emit_left_unpairables, - int emit_right_unpairables) + int emit_right_unpairables, + char* left_file_name) { mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t)); @@ -51,13 +87,20 @@ static mapper_t* mapper_join_alloc( pstate->pleft_field_names = pleft_field_names; pstate->pright_field_names = pright_field_names; pstate->poutput_field_names = poutput_field_names; - pstate->allow_unsorted_input = allow_unsorted_input; pstate->emit_pairables = emit_pairables; pstate->emit_left_unpairables = emit_left_unpairables; pstate->emit_right_unpairables = emit_right_unpairables; pmapper->pvstate = (void*)pstate; - pmapper->pprocess_func = mapper_join_process; + if (allow_unsorted_input) { + pstate->precords_by_key_field_names = ingest_left_file(left_file_name); + pstate->left_file_name = NULL; + pmapper->pprocess_func = mapper_join_process_unsorted; + } else { + pstate->precords_by_key_field_names = NULL; + pstate->left_file_name = left_file_name; + pmapper->pprocess_func = mapper_join_process_sorted; + } pmapper->pfree_func = mapper_join_free; return pmapper; @@ -66,20 +109,21 @@ static mapper_t* mapper_join_alloc( // ---------------------------------------------------------------- static void mapper_join_usage(char* argv0, char* verb) { fprintf(stdout, "Usage: %s %s [options]\n", argv0, verb); - fprintf(stdout, "-1 {a,b,c}\n"); - fprintf(stdout, "-2 {a,b,c}\n"); - fprintf(stdout, "-j {a,b,c}\n"); - fprintf(stdout, "-a 1\n"); - fprintf(stdout, "-a 2\n"); - fprintf(stdout, "-v 1\n"); - fprintf(stdout, "-v 2\n"); - fprintf(stdout, "-o {a,b,c} -- do i want this?? or use then cut -f {a,b,c}...\n"); - fprintf(stdout, "-e EMPTY\n"); fprintf(stdout, "xxx write me up.\n"); + fprintf(stdout, "-f {left file name}\n"); + fprintf(stdout, "-l {a,b,c}\n"); + fprintf(stdout, "-r {a,b,c}\n"); + fprintf(stdout, "-o {a,b,c}\n"); + fprintf(stdout, "--np\n"); + fprintf(stdout, "--ul\n"); + fprintf(stdout, "--ur\n"); + fprintf(stdout, "-u\n"); + fprintf(stdout, "-e EMPTY\n"); } // ---------------------------------------------------------------- static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) { + char* left_file_name = NULL; slls_t* pleft_field_names = NULL; slls_t* pright_field_names = NULL; slls_t* poutput_field_names = NULL; @@ -90,36 +134,40 @@ static mapper_t* mapper_join_parse_cli(int* pargi, int argc, char** argv) { char* verb = argv[(*pargi)++]; - int argi = *pargi; - while (argv[argi][0] == '-') { -// if (streq(argv[argi], "--at-least")) { -// criterion = HAVING_FIELDS_AT_LEAST; -// } else if (streq(argv[argi], "--which-are")) { -// criterion = HAVING_FIELDS_WHICH_ARE; -// } else if (streq(argv[argi], "--at-most")) { -// criterion = HAVING_FIELDS_AT_MOST; -// } else { -// mapper_join_usage(argv[0], verb); -// return NULL; -// } + ap_state_t* pstate = ap_alloc(); + ap_define_string_flag(pstate, "-f", &left_file_name); + ap_define_string_list_flag(pstate, "-l", &pleft_field_names); + ap_define_string_list_flag(pstate, "-r", &pright_field_names); + ap_define_string_list_flag(pstate, "-o", &poutput_field_names); + ap_define_false_flag(pstate, "--np", &emit_pairables); + ap_define_int_flag(pstate, "--ul", &emit_left_unpairables); + ap_define_int_flag(pstate, "--ur", &emit_left_unpairables); + ap_define_true_flag(pstate, "-u", &allow_unsorted_input); - if (argc - argi < 2) { - return NULL; - } - if (pleft_field_names != NULL) - slls_free(pleft_field_names); - pleft_field_names = slls_from_line(argv[argi+1], ',', FALSE); - argi += 2; - } - - if (pleft_field_names == NULL) { + if (!ap_parse(pstate, verb, pargi, argc, argv)) { mapper_join_usage(argv[0], verb); return NULL; } - *pargi = argi; + if (left_file_name == NULL) { + fprintf(stderr, "%s %s: need left file name\n", MLR_GLOBALS.argv0, verb); + mapper_join_usage(argv[0], verb); + return NULL; + } + if (pleft_field_names == NULL) { + fprintf(stderr, "%s %s: need left field names\n", MLR_GLOBALS.argv0, verb); + mapper_join_usage(argv[0], verb); + return NULL; + } + + if (pright_field_names == NULL) + pright_field_names = slls_copy(pleft_field_names); + if (poutput_field_names == NULL) + poutput_field_names = slls_copy(pleft_field_names); + return mapper_join_alloc(pleft_field_names, pright_field_names, poutput_field_names, - allow_unsorted_input, emit_pairables, emit_left_unpairables, emit_right_unpairables); + allow_unsorted_input, emit_pairables, emit_left_unpairables, emit_right_unpairables, + left_file_name); } // ----------------------------------------------------------------