From 03e7f98804be17567ed42f3b104ab9d701da283f Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 4 Nov 2015 09:25:30 -0500 Subject: [PATCH] join neaten iterate --- c/containers/join_bucket_keeper.c | 15 +++-- c/containers/join_bucket_keeper.h | 2 +- c/containers/lrec.c | 5 ++ c/mapping/mapper_join.c | 21 +++++++ c/todo.txt | 1 + c/unit_test/test_join_bucket_keeper.c | 81 +++++++++++++++++++++------ 6 files changed, 104 insertions(+), 21 deletions(-) diff --git a/c/containers/join_bucket_keeper.c b/c/containers/join_bucket_keeper.c index ea2efd8ad..58514123a 100644 --- a/c/containers/join_bucket_keeper.c +++ b/c/containers/join_bucket_keeper.c @@ -35,7 +35,8 @@ // Private methods static int join_bucket_keeper_get_state(join_bucket_keeper_t* pkeeper); -static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper); +static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, + sllv_t** pprecords_left_unpaired); static void join_bucket_keeper_advance_to(join_bucket_keeper_t* pkeeper, slls_t* pright_field_values, sllv_t** pprecords_paired, sllv_t** pprecords_left_unpaired); static void join_bucket_keeper_fill(join_bucket_keeper_t* pkeeper); @@ -115,9 +116,11 @@ void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, slls_t* pright_field *pprecords_left_unpaired = NULL; int cmp = 0; + //printf("[istate] %d\n", pkeeper->state); if (pkeeper->state == LEFT_STATE_0_PREFILL) { - join_bucket_keeper_initial_fill(pkeeper); + join_bucket_keeper_initial_fill(pkeeper, pprecords_left_unpaired); pkeeper->state = join_bucket_keeper_get_state(pkeeper); + //printf("[nstate] %d\n", pkeeper->state); } if (pright_field_values != NULL) { // Not right EOF @@ -159,9 +162,13 @@ static int join_bucket_keeper_get_state(join_bucket_keeper_t* pkeeper) { } } -static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper) { +// XXX for het case need to keep peeking, skipping when left field values aren't all had +static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, + sllv_t** pprecords_left_unpaired) +{ pkeeper->prec_peek = pkeeper->plrec_reader->pprocess_func(pkeeper->plrec_reader->pvstate, pkeeper->pvhandle, pkeeper->pctx); + //printf("[recpeek] "); lrec_print(pkeeper->prec_peek); // XXX if (pkeeper->prec_peek == NULL) { pkeeper->leof = TRUE; return; @@ -169,7 +176,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper) { join_bucket_keeper_fill(pkeeper); } -// xxx preconditions: +// Preconditions: // * prec_peek != NULL static void join_bucket_keeper_fill(join_bucket_keeper_t* pkeeper) { slls_t* pleft_field_values = mlr_selected_values_from_record(pkeeper->prec_peek, diff --git a/c/containers/join_bucket_keeper.h b/c/containers/join_bucket_keeper.h index 1a6ce4eec..78c185280 100644 --- a/c/containers/join_bucket_keeper.h +++ b/c/containers/join_bucket_keeper.h @@ -25,8 +25,8 @@ typedef struct _join_bucket_keeper_t { slls_t* pleft_field_names; join_bucket_t* pbucket; - lrec_t* prec_peek; + int leof; int state; } join_bucket_keeper_t; diff --git a/c/containers/lrec.c b/c/containers/lrec.c index 78a20f61d..d1670aeb3 100644 --- a/c/containers/lrec.c +++ b/c/containers/lrec.c @@ -425,6 +425,11 @@ void lrec_print(lrec_t* prec) { char ors = '\n'; char ofs = ','; char ops = '='; + if (prec == NULL) { + fputs("NULL", output_stream); + fputc(ors, output_stream); + return; + } int nf = 0; for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) { if (nf > 0) diff --git a/c/mapping/mapper_join.c b/c/mapping/mapper_join.c index 0eb387b92..40f95d10b 100644 --- a/c/mapping/mapper_join.c +++ b/c/mapping/mapper_join.c @@ -258,6 +258,20 @@ static sllv_t* mapper_join_process_sorted(lrec_t* pright_rec, context_t* pctx, v } join_bucket_keeper_t* pkeeper = pstate->pjoin_bucket_keeper; // keystroke-saver + // $ cat a + // x=2 + // a=3 + // + // $ cat b + // a=1 + // + // $ mlr join -j a -f a b + // [rightrec] a=1 + // mlr: internal coding error: failed transition from prefill state. + + // printf("[rightrec] "); // XXX + // lrec_print(pright_rec); + sllv_t* pleft_records = NULL; sllv_t* pbucket_left_unpaired = NULL; sllv_t* pout_recs = sllv_alloc(); @@ -274,8 +288,15 @@ static sllv_t* mapper_join_process_sorted(lrec_t* pright_rec, context_t* pctx, v slls_t* pright_field_values = mlr_selected_values_from_record(pright_rec, pstate->popts->pright_join_field_names); + + // printf("[right-field-values] "); // XXX + // slls_print(pright_field_values); + // printf("\n"); + if (pright_field_values != NULL) { + // printf("pre-emit\n"); join_bucket_keeper_emit(pkeeper, pright_field_values, &pleft_records, &pbucket_left_unpaired); + // printf("post-emit\n"); } if (pstate->popts->emit_left_unpairables) { diff --git a/c/todo.txt b/c/todo.txt index 273c83e4e..bd82d8931 100644 --- a/c/todo.txt +++ b/c/todo.txt @@ -18,6 +18,7 @@ TOP OF LIST ! R @ covers x 2 ! be very clear streaming vs. dataframe -- each has things the other can't do ! emph mlr has light stats but for heavyweight analysis use R et al. + MINOR: brew version bump?!? MINOR: filter/put option for no auto-convert: all inputs should stay MT_STRING until converted. diff --git a/c/unit_test/test_join_bucket_keeper.c b/c/unit_test/test_join_bucket_keeper.c index 8a3139977..efe6e9bba 100644 --- a/c/unit_test/test_join_bucket_keeper.c +++ b/c/unit_test/test_join_bucket_keeper.c @@ -33,6 +33,23 @@ static sllv_t* make_records_113335() { return precords; } +// ---------------------------------------------------------------- +#if 0 +static sllv_t* make_records_het() { + sllv_t* precords = sllv_alloc(); + sllv_add(precords, lrec_literal_2("x","1", "b","10")); + sllv_add(precords, lrec_literal_2("l","1", "b","11")); + sllv_add(precords, lrec_literal_2("l","1", "b","12")); + sllv_add(precords, lrec_literal_2("x","3", "b","13")); + sllv_add(precords, lrec_literal_2("l","3", "b","14")); + sllv_add(precords, lrec_literal_2("l","3", "b","15")); + sllv_add(precords, lrec_literal_2("x","3", "b","16")); + sllv_add(precords, lrec_literal_2("l","5", "b","17")); + sllv_add(precords, lrec_literal_2("l","5", "b","18")); + return precords; +} +#endif + // ---------------------------------------------------------------- static void set_up( sllv_t* precords, @@ -99,9 +116,9 @@ static int list_has_length(sllv_t* plist, int length, char* desc, char* rval) { } // ---------------------------------------------------------------- -static char* test00() { +static char* test_left_empty_right_empty() { printf("----------------------------------------------------------------\n"); - printf("test00 enter\n"); + printf("test_left_empty_right_empty enter\n"); slls_t* pleft_field_names; lrec_reader_t* preader; @@ -115,15 +132,15 @@ static char* test00() { mu_assert_lf(list_has_length(precords_left_unpaired, 0, "unpaired", "(eof)")); printf("\n"); - printf("test00 exit\n"); + printf("test_left_empty_right_empty exit\n"); printf("\n"); return 0; } // ---------------------------------------------------------------- -static char* test01() { +static char* test_left_empty() { printf("----------------------------------------------------------------\n"); - printf("test01 enter\n"); + printf("test_left_empty enter\n"); slls_t* pleft_field_names; lrec_reader_t* preader; @@ -143,15 +160,15 @@ static char* test01() { mu_assert_lf(list_has_length(precords_left_unpaired, 0, "unpaired", "(eof)")); printf("\n"); - printf("test01 exit\n"); + printf("test_left_empty exit\n"); printf("\n"); return 0; } // ---------------------------------------------------------------- -static char* test0() { +static char* test_right_empty() { printf("----------------------------------------------------------------\n"); - printf("test0 enter\n"); + printf("test_right_empty enter\n"); slls_t* pleft_field_names; lrec_reader_t* preader; @@ -165,7 +182,7 @@ static char* test0() { mu_assert_lf(list_has_length(precords_left_unpaired, 6, "unpaired", "(eof)")); printf("\n"); - printf("test0 exit\n"); + printf("test_right_empty exit\n"); printf("\n"); return 0; } @@ -307,9 +324,9 @@ static char* test4() { } // ---------------------------------------------------------------- -static char* test5() { +static char* test_walk_through_all() { printf("----------------------------------------------------------------\n"); - printf("test5 enter\n"); + printf("test_walk_through_all enter\n"); slls_t* pleft_field_names; lrec_reader_t* preader; @@ -407,21 +424,53 @@ static char* test5() { mu_assert_lf(list_has_length(precords_left_unpaired, 0, "unpaired", pright_field_values->phead->value)); printf("\n"); - printf("test5 exit\n"); + printf("test_walk_through_all exit\n"); + printf("\n"); + return 0; +} + +// ---------------------------------------------------------------- +// xxx give all of these more descriptive names. +static char* testhet6() { + printf("----------------------------------------------------------------\n"); + printf("testhet6 enter\n"); +#if 1 + printf("testhet6 stub!!\n"); +#else + slls_t* pleft_field_names; + lrec_reader_t* preader; + set_up(make_records_het(), &pleft_field_names, &preader); + join_bucket_keeper_t* pkeeper = join_bucket_keeper_alloc_from_reader(preader, NULL, pleft_field_names); + sllv_t* precords_paired; + sllv_t* precords_left_unpaired; + + slls_t* pright_field_values = slls_single_no_free("1"); + emit(pkeeper, pright_field_values, &precords_paired, &precords_left_unpaired); + mu_assert_lf(list_is_null(precords_paired, "paired", pright_field_values->phead->value)); + mu_assert_lf(list_is_null(precords_left_unpaired, "unpaired", pright_field_values->phead->value)); + printf("\n"); + + emit(pkeeper, NULL, &precords_paired, &precords_left_unpaired); + mu_assert_lf(list_is_null(precords_paired, "paired", "(eof)")); + mu_assert_lf(list_has_length(precords_left_unpaired, 0, "unpaired", "(eof)")); + printf("\n"); +#endif + printf("testhet6 exit\n"); printf("\n"); return 0; } // ================================================================ static char * run_all_tests() { - mu_run_test(test00); - mu_run_test(test01); - mu_run_test(test0); + mu_run_test(test_left_empty_right_empty); + mu_run_test(test_left_empty); + mu_run_test(test_right_empty); mu_run_test(test1); mu_run_test(test2); mu_run_test(test3); mu_run_test(test4); - mu_run_test(test5); + mu_run_test(test_walk_through_all); + mu_run_test(testhet6); return 0; }