diff --git a/.gitignore b/.gitignore index 2d0d3fae3..ad0d3d278 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ mlrp mlr-unit-tester test-mlrutil test-lrec +test-join-bucket-keeper a.out *.dSYM catc diff --git a/c/Makefile b/c/Makefile index b7d5c8aee..b7fe98c5c 100644 --- a/c/Makefile +++ b/c/Makefile @@ -21,6 +21,18 @@ input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ containers/test_lrec.c +TEST_JOIN_BUCKET_KEEPER_SRCS = lib/mlrutil.c lib/mlr_globals.c containers/lrec.c \ +containers/sllv.c containers/slls.c containers/lhmslv.c containers/hss.c containers/mixutil.c \ +containers/header_keeper.c \ +containers/join_bucket_keeper.c \ +input/lrec_reader_in_memory.c input/lrec_readers.c \ +input/lrec_reader_mmap_csv.c input/lrec_reader_stdio_csv.c \ +input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \ +input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ +input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ +input/file_reader_mmap.c input/file_reader_stdio.c \ +containers/test_join_bucket_keeper.c + # ---------------------------------------------------------------- # mlr for default use. # mlrg for valgrind. @@ -65,10 +77,11 @@ test: unit-test reg-test # Unfortunately --error-exitcode=1 doesn't work well since there are # valgrind-detected errors in stdlibs. :( -unit-test: test-mlrutil test-lrec +unit-test: test-mlrutil test-lrec test-join-bucket-keeper #valgrind --leak-check=full --error-exitcode=1 ./a.out valgrind --leak-check=full ./test-mlrutil valgrind --leak-check=full ./test-lrec + valgrind --leak-check=full ./test-join-bucket-keeper @echo @echo DONE @@ -88,6 +101,9 @@ test-lrec: .always test-mlrutil: .always $(CCDEBUG) -D__TEST_MLRUTIL_MAIN__ lib/mlrutil.c lib/test_mlrutil.c -o test-mlrutil +test-join-bucket-keeper: .always + $(CCDEBUG) -D__TEST_JOIN_BUCKET_KEEPER_MAIN__ $(TEST_JOIN_BUCKET_KEEPER_SRCS) -o test-join-bucket-keeper + # ---------------------------------------------------------------- # Standalone mains dheap: diff --git a/c/containers/join_bucket_keeper.c b/c/containers/join_bucket_keeper.c index 428f8db86..e05dbaef4 100644 --- a/c/containers/join_bucket_keeper.c +++ b/c/containers/join_bucket_keeper.c @@ -27,7 +27,8 @@ join_bucket_keeper_t* join_bucket_keeper_alloc( char ifs, int allow_repeat_ifs, char ips, - int allow_repeat_ips + int allow_repeat_ips, + slls_t* pleft_field_names ) { join_bucket_keeper_t* pkeeper = mlr_malloc_or_die(sizeof(join_bucket_keeper_t)); @@ -44,6 +45,7 @@ join_bucket_keeper_t* join_bucket_keeper_alloc( pkeeper->pctx->filenum = 1; pkeeper->pctx->filename = left_file_name; + pkeeper->pleft_field_names = slls_copy(pleft_field_names); // xxx be sure the caller frees its own pkeeper->pleft_field_values = NULL; pkeeper->precords = sllv_alloc(); pkeeper->prec_peek = NULL; @@ -85,7 +87,7 @@ static int join_bucket_keeper_get_state(join_bucket_keeper_t* pkeeper) { } // xxx put bucket & bucket-keeper into separate files with separate UTs -static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_t* pleft_field_names) { +static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper) { pkeeper->prec_peek = pkeeper->plrec_reader->pprocess_func(pkeeper->pvhandle, pkeeper->plrec_reader->pvstate, pkeeper->pctx); if (pkeeper->prec_peek == NULL) { @@ -93,7 +95,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_ return; } pkeeper->pleft_field_values = mlr_selected_values_from_record(pkeeper->prec_peek, - pleft_field_names); + pkeeper->pleft_field_names); sllv_add(pkeeper->precords, pkeeper->prec_peek); pkeeper->prec_peek = NULL; @@ -106,7 +108,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_ } // xxx make a function to compare w/o copy slls_t* pnext_field_values = mlr_selected_values_from_record(pkeeper->prec_peek, - pleft_field_names); + pkeeper->pleft_field_names); int cmp = slls_compare_lexically(pkeeper->pleft_field_values, pnext_field_values); if (cmp != 0) { break; @@ -158,8 +160,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_ // *ppbucket_paired = pfoo; // xxx cmt re who frees -void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, - slls_t* pleft_field_names, slls_t* pright_field_values, +void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, slls_t* pright_field_values, sllv_t** ppbucket_paired, sllv_t** ppbucket_left_unpaired) { *ppbucket_paired = NULL; @@ -168,22 +169,12 @@ void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, if (pkeeper->state == LEFT_STATE_0_PREFILL) { // try fill Lv & peek; next state is 1,2,3 & continue from there. - join_bucket_keeper_initial_fill(pkeeper, pleft_field_names); + join_bucket_keeper_initial_fill(pkeeper); pkeeper->state = join_bucket_keeper_get_state(pkeeper); } -// -//typedef struct _join_bucket_keeper_t { -// lrec_reader_t* plrec_reader; -// void* pvhandle; -// context_t* pctx; -// -// int state; -// slls_t* pleft_field_values; -// sllv_t* precords; -// lrec_t* prec_peek; -// int leof; -//} join_bucket_keeper_t; + // xxx drain on pright_field_values == NULL, for returning the final + // left-unpaireds after right EOF. switch (pkeeper->state) { case LEFT_STATE_1_FULL: @@ -262,8 +253,6 @@ void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, } -// xxx need a drain-hook for returning the final left-unpaireds after right EOF. - // ---------------------------------------------------------------- // +-----------+-----------+-----------+-----------+-----------+-----------+ diff --git a/c/containers/join_bucket_keeper.h b/c/containers/join_bucket_keeper.h index 2dd30d24d..5ee55fdda 100644 --- a/c/containers/join_bucket_keeper.h +++ b/c/containers/join_bucket_keeper.h @@ -12,6 +12,8 @@ typedef struct _join_bucket_keeper_t { void* pvhandle; context_t* pctx; + slls_t* pleft_field_names; + int state; slls_t* pleft_field_values; sllv_t* precords; @@ -28,14 +30,13 @@ join_bucket_keeper_t* join_bucket_keeper_alloc( char ifs, int allow_repeat_ifs, char ips, - int allow_repeat_ips + int allow_repeat_ips, + slls_t* pleft_field_names ); void join_bucket_keeper_free(join_bucket_keeper_t* pkeeper); -void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, - slls_t* pleft_field_names, slls_t* pright_field_values, +void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, slls_t* pright_field_values, sllv_t** ppbucket_paired, sllv_t** ppbucket_left_unpaired); - #endif // JOIN_BUCKET_KEEPER_H diff --git a/c/containers/test_join_bucket_keeper.c b/c/containers/test_join_bucket_keeper.c new file mode 100644 index 000000000..955224543 --- /dev/null +++ b/c/containers/test_join_bucket_keeper.c @@ -0,0 +1,289 @@ +#include +#include +#ifdef MLR_USE_MCHECK +#include +#endif // MLR_USE_MCHECK +#include "lib/minunit.h" +#include "lib/mlrutil.h" +#include "containers/lrec.h" +#include "containers/sllv.h" +#include "input/lrec_readers.h" + +#ifdef __TEST_JOIN_BUCKET_KEEPER_MAIN__ +int tests_run = 0; +int tests_failed = 0; +int assertions_run = 0; +int assertions_failed = 0; + +// ---------------------------------------------------------------- +static char* test_lrec_unbacked_api() { + lrec_t* prec = lrec_unbacked_alloc(); + mu_assert_lf(prec->field_count == 0); + + lrec_put_no_free(prec, "x", "3"); + mu_assert_lf(prec->field_count == 1); + mu_assert_lf(streq(lrec_get(prec, "x"), "3")); + + lrec_put_no_free(prec, "y", "4"); + mu_assert_lf(prec->field_count == 2); + mu_assert_lf(streq(lrec_get(prec, "x"), "3")); + mu_assert_lf(streq(lrec_get(prec, "y"), "4")); + + lrec_put_no_free(prec, "x", "5"); + mu_assert_lf(prec->field_count == 2); + mu_assert_lf(streq(lrec_get(prec, "x"), "5")); + mu_assert_lf(streq(lrec_get(prec, "y"), "4")); + + lrec_remove(prec, "x"); + mu_assert_lf(prec->field_count == 1); + mu_assert_lf(lrec_get(prec, "x") == NULL); + + // Non-replacing-rename case + //lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "y", "z"); + //lrec_dump_titled("After rename", prec); + mu_assert_lf(prec->field_count == 1); + mu_assert_lf(lrec_get(prec, "x") == NULL); + mu_assert_lf(lrec_get(prec, "y") == NULL); + mu_assert_lf(streq(lrec_get(prec, "z"), "4")); + + lrec_free(prec); + + // Replacing-rename case + prec = lrec_unbacked_alloc(); + + lrec_put_no_free(prec, "x", "3"); + lrec_put_no_free(prec, "y", "4"); + lrec_put_no_free(prec, "z", "5"); + mu_assert_lf(prec->field_count == 3); + + //lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "y", "z"); + //lrec_dump_titled("After rename", prec); + + mu_assert_lf(prec->field_count == 2); + mu_assert_lf(streq(lrec_get(prec, "x"), "3")); + mu_assert_lf(lrec_get(prec, "y") == NULL); + mu_assert_lf(streq(lrec_get(prec, "z"), "4")); + + lrec_free(prec); + + return NULL; +} + +// ---------------------------------------------------------------- +static char* test_lrec_dkvp_api() { + char* line = strdup("w=2,x=3,y=4,z=5"); + lrec_t* prec = lrec_parse_stdio_dkvp(line, ',', '=', FALSE); + mu_assert_lf(prec->field_count == 4); + + mu_assert_lf(streq(lrec_get(prec, "w"), "2")); + mu_assert_lf(streq(lrec_get(prec, "x"), "3")); + mu_assert_lf(streq(lrec_get(prec, "y"), "4")); + mu_assert_lf(streq(lrec_get(prec, "z"), "5")); + + lrec_remove(prec, "w"); + mu_assert_lf(prec->field_count == 3); + mu_assert_lf(lrec_get(prec, "w") == NULL); + + // Non-replacing-rename case + //lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "x", "u"); + //lrec_dump_titled("After rename", prec); + mu_assert_lf(prec->field_count == 3); + mu_assert_lf(lrec_get(prec, "x") == NULL); + mu_assert_lf(streq(lrec_get(prec, "u"), "3")); + + // Replacing-rename case + //lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "y", "z"); + //lrec_dump_titled("After rename", prec); + + mu_assert_lf(prec->field_count == 2); + mu_assert_lf(streq(lrec_get(prec, "u"), "3")); + mu_assert_lf(lrec_get(prec, "y") == NULL); + mu_assert_lf(streq(lrec_get(prec, "z"), "4")); + + lrec_free(prec); + + return NULL; +} + +// ---------------------------------------------------------------- +static char* test_lrec_nidx_api() { + char* line = strdup("a,b,c,d"); + lrec_t* prec = lrec_parse_stdio_nidx(line, ',', FALSE); + mu_assert_lf(prec->field_count == 4); + + mu_assert_lf(streq(lrec_get(prec, "1"), "a")); + mu_assert_lf(streq(lrec_get(prec, "2"), "b")); + mu_assert_lf(streq(lrec_get(prec, "3"), "c")); + mu_assert_lf(streq(lrec_get(prec, "4"), "d")); + + lrec_remove(prec, "1"); + mu_assert_lf(prec->field_count == 3); + mu_assert_lf(lrec_get(prec, "1") == NULL); + + // Non-replacing-rename case + lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "2", "u"); + lrec_dump_titled("After rename", prec); + mu_assert_lf(prec->field_count == 3); + mu_assert_lf(lrec_get(prec, "2") == NULL); + mu_assert_lf(streq(lrec_get(prec, "u"), "b")); + + // Replacing-rename case + lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "3", "4"); + lrec_dump_titled("After rename", prec); + + mu_assert_lf(prec->field_count == 2); + mu_assert_lf(streq(lrec_get(prec, "u"), "b")); + mu_assert_lf(lrec_get(prec, "3") == NULL); + mu_assert_lf(streq(lrec_get(prec, "4"), "c")); + + lrec_free(prec); + + return NULL; +} + +// ---------------------------------------------------------------- +static char* test_lrec_csv_api() { + char* hdr_line = strdup("w,x,y,z"); + slls_t* hdr_fields = split_csv_header_line(hdr_line, ',', FALSE); + header_keeper_t* pheader_keeper = header_keeper_alloc(hdr_line, hdr_fields); + + char* data_line_1 = strdup("2,3,4,5"); + lrec_t* prec_1 = lrec_parse_stdio_csv_data_line(pheader_keeper, data_line_1, ',', FALSE); + + char* data_line_2 = strdup("6,7,8,9"); + lrec_t* prec_2 = lrec_parse_stdio_csv_data_line(pheader_keeper, data_line_2, ',', FALSE); + + mu_assert_lf(prec_1->field_count == 4); + mu_assert_lf(prec_2->field_count == 4); + + mu_assert_lf(streq(lrec_get(prec_1, "w"), "2")); + mu_assert_lf(streq(lrec_get(prec_1, "x"), "3")); + mu_assert_lf(streq(lrec_get(prec_1, "y"), "4")); + mu_assert_lf(streq(lrec_get(prec_1, "z"), "5")); + + mu_assert_lf(streq(lrec_get(prec_2, "w"), "6")); + mu_assert_lf(streq(lrec_get(prec_2, "x"), "7")); + mu_assert_lf(streq(lrec_get(prec_2, "y"), "8")); + mu_assert_lf(streq(lrec_get(prec_2, "z"), "9")); + + lrec_remove(prec_1, "w"); + mu_assert_lf(prec_1->field_count == 3); + mu_assert_lf(prec_2->field_count == 4); + mu_assert_lf(lrec_get(prec_1, "w") == NULL); + mu_assert_lf(streq(lrec_get(prec_2, "w"), "6")); + + // Non-replacing-rename case + //lrec_dump_titled("Before rename", prec_1); + lrec_rename(prec_1, "x", "u"); + //lrec_dump_titled("After rename", prec_1); + mu_assert_lf(prec_1->field_count == 3); + mu_assert_lf(lrec_get(prec_1, "x") == NULL); + mu_assert_lf(streq(lrec_get(prec_1, "u"), "3")); + + // Replacing-rename case + //lrec_dump_titled("Before rename", prec_2); + lrec_rename(prec_2, "y", "z"); + //lrec_dump_titled("After rename", prec_2); + + mu_assert_lf(prec_2->field_count == 3); + mu_assert_lf(streq(lrec_get(prec_2, "w"), "6")); + mu_assert_lf(streq(lrec_get(prec_2, "x"), "7")); + mu_assert_lf(lrec_get(prec_2, "y") == NULL); + mu_assert_lf(streq(lrec_get(prec_2, "z"), "8")); + + lrec_free(prec_1); + lrec_free(prec_2); + + // xxx need a test case for alloc1,free1,alloc2,free2 w/ same hdr. + return NULL; +} + +// ---------------------------------------------------------------- +static char* test_lrec_xtab_api() { + char* line_1 = strdup("w 2"); + char* line_2 = strdup("x 3"); + char* line_3 = strdup("y 4"); + char* line_4 = strdup("z 5"); + slls_t* pxtab_lines = slls_alloc(); + slls_add_with_free(pxtab_lines, line_1); + slls_add_with_free(pxtab_lines, line_2); + slls_add_with_free(pxtab_lines, line_3); + slls_add_with_free(pxtab_lines, line_4); + + lrec_t* prec = lrec_parse_stdio_xtab(pxtab_lines, ' ', TRUE); + mu_assert_lf(prec->field_count == 4); + + mu_assert_lf(streq(lrec_get(prec, "w"), "2")); + mu_assert_lf(streq(lrec_get(prec, "x"), "3")); + mu_assert_lf(streq(lrec_get(prec, "y"), "4")); + mu_assert_lf(streq(lrec_get(prec, "z"), "5")); + + lrec_remove(prec, "w"); + mu_assert_lf(prec->field_count == 3); + mu_assert_lf(lrec_get(prec, "w") == NULL); + + // Non-replacing-rename case + //lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "x", "u"); + //lrec_dump_titled("After rename", prec); + mu_assert_lf(prec->field_count == 3); + mu_assert_lf(lrec_get(prec, "x") == NULL); + mu_assert_lf(streq(lrec_get(prec, "u"), "3")); + + // Replacing-rename case + //lrec_dump_titled("Before rename", prec); + lrec_rename(prec, "y", "z"); + //lrec_dump_titled("After rename", prec); + + mu_assert_lf(prec->field_count == 2); + mu_assert_lf(streq(lrec_get(prec, "u"), "3")); + mu_assert_lf(lrec_get(prec, "y") == NULL); + mu_assert_lf(streq(lrec_get(prec, "z"), "4")); + + lrec_free(prec); + + return NULL; +} + +// ================================================================ +static char * run_all_tests() { + mu_run_test(test_lrec_unbacked_api); + mu_run_test(test_lrec_dkvp_api); + mu_run_test(test_lrec_nidx_api); + mu_run_test(test_lrec_csv_api); + mu_run_test(test_lrec_xtab_api); + return 0; +} + +int main(int argc, char **argv) { +#ifdef MLR_USE_MCHECK + if (mcheck(NULL) != 0) { + printf("Could not set up mcheck\n"); + exit(1); + } + printf("Set up mcheck\n"); +#endif // MLR_USE_MCHECK + + char *result = run_all_tests(); + printf("\n"); + if (result != 0) { + //printf("%s\n", result); + printf("Not all unit tests passed\n"); + } + else { + printf("TEST_LREC: ALL UNIT TESTS PASSED\n"); + } + printf("Tests run: %d\n", tests_run); + printf("Tests failed: %d\n", tests_failed); + printf("Assertions run: %d\n", assertions_run); + printf("Assertions failed: %d\n", assertions_failed); + + return result != 0; +} +#endif // __TEST_JOIN_BUCKET_KEEPER_MAIN__