join iterate

This commit is contained in:
John Kerl 2015-07-10 08:42:46 -04:00
parent c641ff36c1
commit d2a29c6573
5 changed files with 322 additions and 26 deletions

1
.gitignore vendored
View file

@ -5,6 +5,7 @@ mlrp
mlr-unit-tester
test-mlrutil
test-lrec
test-join-bucket-keeper
a.out
*.dSYM
catc

View file

@ -21,6 +21,18 @@ input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \
input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \
containers/test_lrec.c
TEST_JOIN_BUCKET_KEEPER_SRCS = lib/mlrutil.c lib/mlr_globals.c containers/lrec.c \
containers/sllv.c containers/slls.c containers/lhmslv.c containers/hss.c containers/mixutil.c \
containers/header_keeper.c \
containers/join_bucket_keeper.c \
input/lrec_reader_in_memory.c input/lrec_readers.c \
input/lrec_reader_mmap_csv.c input/lrec_reader_stdio_csv.c \
input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \
input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \
input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \
input/file_reader_mmap.c input/file_reader_stdio.c \
containers/test_join_bucket_keeper.c
# ----------------------------------------------------------------
# mlr for default use.
# mlrg for valgrind.
@ -65,10 +77,11 @@ test: unit-test reg-test
# Unfortunately --error-exitcode=1 doesn't work well since there are
# valgrind-detected errors in stdlibs. :(
unit-test: test-mlrutil test-lrec
unit-test: test-mlrutil test-lrec test-join-bucket-keeper
#valgrind --leak-check=full --error-exitcode=1 ./a.out
valgrind --leak-check=full ./test-mlrutil
valgrind --leak-check=full ./test-lrec
valgrind --leak-check=full ./test-join-bucket-keeper
@echo
@echo DONE
@ -88,6 +101,9 @@ test-lrec: .always
test-mlrutil: .always
$(CCDEBUG) -D__TEST_MLRUTIL_MAIN__ lib/mlrutil.c lib/test_mlrutil.c -o test-mlrutil
test-join-bucket-keeper: .always
$(CCDEBUG) -D__TEST_JOIN_BUCKET_KEEPER_MAIN__ $(TEST_JOIN_BUCKET_KEEPER_SRCS) -o test-join-bucket-keeper
# ----------------------------------------------------------------
# Standalone mains
dheap:

View file

@ -27,7 +27,8 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
char ifs,
int allow_repeat_ifs,
char ips,
int allow_repeat_ips
int allow_repeat_ips,
slls_t* pleft_field_names
) {
join_bucket_keeper_t* pkeeper = mlr_malloc_or_die(sizeof(join_bucket_keeper_t));
@ -44,6 +45,7 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
pkeeper->pctx->filenum = 1;
pkeeper->pctx->filename = left_file_name;
pkeeper->pleft_field_names = slls_copy(pleft_field_names); // xxx be sure the caller frees its own
pkeeper->pleft_field_values = NULL;
pkeeper->precords = sllv_alloc();
pkeeper->prec_peek = NULL;
@ -85,7 +87,7 @@ static int join_bucket_keeper_get_state(join_bucket_keeper_t* pkeeper) {
}
// xxx put bucket & bucket-keeper into separate files with separate UTs
static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_t* pleft_field_names) {
static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper) {
pkeeper->prec_peek = pkeeper->plrec_reader->pprocess_func(pkeeper->pvhandle,
pkeeper->plrec_reader->pvstate, pkeeper->pctx);
if (pkeeper->prec_peek == NULL) {
@ -93,7 +95,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_
return;
}
pkeeper->pleft_field_values = mlr_selected_values_from_record(pkeeper->prec_peek,
pleft_field_names);
pkeeper->pleft_field_names);
sllv_add(pkeeper->precords, pkeeper->prec_peek);
pkeeper->prec_peek = NULL;
@ -106,7 +108,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_
}
// xxx make a function to compare w/o copy
slls_t* pnext_field_values = mlr_selected_values_from_record(pkeeper->prec_peek,
pleft_field_names);
pkeeper->pleft_field_names);
int cmp = slls_compare_lexically(pkeeper->pleft_field_values, pnext_field_values);
if (cmp != 0) {
break;
@ -158,8 +160,7 @@ static void join_bucket_keeper_initial_fill(join_bucket_keeper_t* pkeeper, slls_
// *ppbucket_paired = pfoo;
// xxx cmt re who frees
void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper,
slls_t* pleft_field_names, slls_t* pright_field_values,
void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, slls_t* pright_field_values,
sllv_t** ppbucket_paired, sllv_t** ppbucket_left_unpaired)
{
*ppbucket_paired = NULL;
@ -168,22 +169,12 @@ void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper,
if (pkeeper->state == LEFT_STATE_0_PREFILL) {
// try fill Lv & peek; next state is 1,2,3 & continue from there.
join_bucket_keeper_initial_fill(pkeeper, pleft_field_names);
join_bucket_keeper_initial_fill(pkeeper);
pkeeper->state = join_bucket_keeper_get_state(pkeeper);
}
//
//typedef struct _join_bucket_keeper_t {
// lrec_reader_t* plrec_reader;
// void* pvhandle;
// context_t* pctx;
//
// int state;
// slls_t* pleft_field_values;
// sllv_t* precords;
// lrec_t* prec_peek;
// int leof;
//} join_bucket_keeper_t;
// xxx drain on pright_field_values == NULL, for returning the final
// left-unpaireds after right EOF.
switch (pkeeper->state) {
case LEFT_STATE_1_FULL:
@ -262,8 +253,6 @@ void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper,
}
// xxx need a drain-hook for returning the final left-unpaireds after right EOF.
// ----------------------------------------------------------------
// +-----------+-----------+-----------+-----------+-----------+-----------+

View file

@ -12,6 +12,8 @@ typedef struct _join_bucket_keeper_t {
void* pvhandle;
context_t* pctx;
slls_t* pleft_field_names;
int state;
slls_t* pleft_field_values;
sllv_t* precords;
@ -28,14 +30,13 @@ join_bucket_keeper_t* join_bucket_keeper_alloc(
char ifs,
int allow_repeat_ifs,
char ips,
int allow_repeat_ips
int allow_repeat_ips,
slls_t* pleft_field_names
);
void join_bucket_keeper_free(join_bucket_keeper_t* pkeeper);
void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper,
slls_t* pleft_field_names, slls_t* pright_field_values,
void join_bucket_keeper_emit(join_bucket_keeper_t* pkeeper, slls_t* pright_field_values,
sllv_t** ppbucket_paired, sllv_t** ppbucket_left_unpaired);
#endif // JOIN_BUCKET_KEEPER_H

View file

@ -0,0 +1,289 @@
#include <stdio.h>
#include <string.h>
#ifdef MLR_USE_MCHECK
#include <mcheck.h>
#endif // MLR_USE_MCHECK
#include "lib/minunit.h"
#include "lib/mlrutil.h"
#include "containers/lrec.h"
#include "containers/sllv.h"
#include "input/lrec_readers.h"
#ifdef __TEST_JOIN_BUCKET_KEEPER_MAIN__
int tests_run = 0;
int tests_failed = 0;
int assertions_run = 0;
int assertions_failed = 0;
// ----------------------------------------------------------------
static char* test_lrec_unbacked_api() {
lrec_t* prec = lrec_unbacked_alloc();
mu_assert_lf(prec->field_count == 0);
lrec_put_no_free(prec, "x", "3");
mu_assert_lf(prec->field_count == 1);
mu_assert_lf(streq(lrec_get(prec, "x"), "3"));
lrec_put_no_free(prec, "y", "4");
mu_assert_lf(prec->field_count == 2);
mu_assert_lf(streq(lrec_get(prec, "x"), "3"));
mu_assert_lf(streq(lrec_get(prec, "y"), "4"));
lrec_put_no_free(prec, "x", "5");
mu_assert_lf(prec->field_count == 2);
mu_assert_lf(streq(lrec_get(prec, "x"), "5"));
mu_assert_lf(streq(lrec_get(prec, "y"), "4"));
lrec_remove(prec, "x");
mu_assert_lf(prec->field_count == 1);
mu_assert_lf(lrec_get(prec, "x") == NULL);
// Non-replacing-rename case
//lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "y", "z");
//lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 1);
mu_assert_lf(lrec_get(prec, "x") == NULL);
mu_assert_lf(lrec_get(prec, "y") == NULL);
mu_assert_lf(streq(lrec_get(prec, "z"), "4"));
lrec_free(prec);
// Replacing-rename case
prec = lrec_unbacked_alloc();
lrec_put_no_free(prec, "x", "3");
lrec_put_no_free(prec, "y", "4");
lrec_put_no_free(prec, "z", "5");
mu_assert_lf(prec->field_count == 3);
//lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "y", "z");
//lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 2);
mu_assert_lf(streq(lrec_get(prec, "x"), "3"));
mu_assert_lf(lrec_get(prec, "y") == NULL);
mu_assert_lf(streq(lrec_get(prec, "z"), "4"));
lrec_free(prec);
return NULL;
}
// ----------------------------------------------------------------
static char* test_lrec_dkvp_api() {
char* line = strdup("w=2,x=3,y=4,z=5");
lrec_t* prec = lrec_parse_stdio_dkvp(line, ',', '=', FALSE);
mu_assert_lf(prec->field_count == 4);
mu_assert_lf(streq(lrec_get(prec, "w"), "2"));
mu_assert_lf(streq(lrec_get(prec, "x"), "3"));
mu_assert_lf(streq(lrec_get(prec, "y"), "4"));
mu_assert_lf(streq(lrec_get(prec, "z"), "5"));
lrec_remove(prec, "w");
mu_assert_lf(prec->field_count == 3);
mu_assert_lf(lrec_get(prec, "w") == NULL);
// Non-replacing-rename case
//lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "x", "u");
//lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 3);
mu_assert_lf(lrec_get(prec, "x") == NULL);
mu_assert_lf(streq(lrec_get(prec, "u"), "3"));
// Replacing-rename case
//lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "y", "z");
//lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 2);
mu_assert_lf(streq(lrec_get(prec, "u"), "3"));
mu_assert_lf(lrec_get(prec, "y") == NULL);
mu_assert_lf(streq(lrec_get(prec, "z"), "4"));
lrec_free(prec);
return NULL;
}
// ----------------------------------------------------------------
static char* test_lrec_nidx_api() {
char* line = strdup("a,b,c,d");
lrec_t* prec = lrec_parse_stdio_nidx(line, ',', FALSE);
mu_assert_lf(prec->field_count == 4);
mu_assert_lf(streq(lrec_get(prec, "1"), "a"));
mu_assert_lf(streq(lrec_get(prec, "2"), "b"));
mu_assert_lf(streq(lrec_get(prec, "3"), "c"));
mu_assert_lf(streq(lrec_get(prec, "4"), "d"));
lrec_remove(prec, "1");
mu_assert_lf(prec->field_count == 3);
mu_assert_lf(lrec_get(prec, "1") == NULL);
// Non-replacing-rename case
lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "2", "u");
lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 3);
mu_assert_lf(lrec_get(prec, "2") == NULL);
mu_assert_lf(streq(lrec_get(prec, "u"), "b"));
// Replacing-rename case
lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "3", "4");
lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 2);
mu_assert_lf(streq(lrec_get(prec, "u"), "b"));
mu_assert_lf(lrec_get(prec, "3") == NULL);
mu_assert_lf(streq(lrec_get(prec, "4"), "c"));
lrec_free(prec);
return NULL;
}
// ----------------------------------------------------------------
static char* test_lrec_csv_api() {
char* hdr_line = strdup("w,x,y,z");
slls_t* hdr_fields = split_csv_header_line(hdr_line, ',', FALSE);
header_keeper_t* pheader_keeper = header_keeper_alloc(hdr_line, hdr_fields);
char* data_line_1 = strdup("2,3,4,5");
lrec_t* prec_1 = lrec_parse_stdio_csv_data_line(pheader_keeper, data_line_1, ',', FALSE);
char* data_line_2 = strdup("6,7,8,9");
lrec_t* prec_2 = lrec_parse_stdio_csv_data_line(pheader_keeper, data_line_2, ',', FALSE);
mu_assert_lf(prec_1->field_count == 4);
mu_assert_lf(prec_2->field_count == 4);
mu_assert_lf(streq(lrec_get(prec_1, "w"), "2"));
mu_assert_lf(streq(lrec_get(prec_1, "x"), "3"));
mu_assert_lf(streq(lrec_get(prec_1, "y"), "4"));
mu_assert_lf(streq(lrec_get(prec_1, "z"), "5"));
mu_assert_lf(streq(lrec_get(prec_2, "w"), "6"));
mu_assert_lf(streq(lrec_get(prec_2, "x"), "7"));
mu_assert_lf(streq(lrec_get(prec_2, "y"), "8"));
mu_assert_lf(streq(lrec_get(prec_2, "z"), "9"));
lrec_remove(prec_1, "w");
mu_assert_lf(prec_1->field_count == 3);
mu_assert_lf(prec_2->field_count == 4);
mu_assert_lf(lrec_get(prec_1, "w") == NULL);
mu_assert_lf(streq(lrec_get(prec_2, "w"), "6"));
// Non-replacing-rename case
//lrec_dump_titled("Before rename", prec_1);
lrec_rename(prec_1, "x", "u");
//lrec_dump_titled("After rename", prec_1);
mu_assert_lf(prec_1->field_count == 3);
mu_assert_lf(lrec_get(prec_1, "x") == NULL);
mu_assert_lf(streq(lrec_get(prec_1, "u"), "3"));
// Replacing-rename case
//lrec_dump_titled("Before rename", prec_2);
lrec_rename(prec_2, "y", "z");
//lrec_dump_titled("After rename", prec_2);
mu_assert_lf(prec_2->field_count == 3);
mu_assert_lf(streq(lrec_get(prec_2, "w"), "6"));
mu_assert_lf(streq(lrec_get(prec_2, "x"), "7"));
mu_assert_lf(lrec_get(prec_2, "y") == NULL);
mu_assert_lf(streq(lrec_get(prec_2, "z"), "8"));
lrec_free(prec_1);
lrec_free(prec_2);
// xxx need a test case for alloc1,free1,alloc2,free2 w/ same hdr.
return NULL;
}
// ----------------------------------------------------------------
static char* test_lrec_xtab_api() {
char* line_1 = strdup("w 2");
char* line_2 = strdup("x 3");
char* line_3 = strdup("y 4");
char* line_4 = strdup("z 5");
slls_t* pxtab_lines = slls_alloc();
slls_add_with_free(pxtab_lines, line_1);
slls_add_with_free(pxtab_lines, line_2);
slls_add_with_free(pxtab_lines, line_3);
slls_add_with_free(pxtab_lines, line_4);
lrec_t* prec = lrec_parse_stdio_xtab(pxtab_lines, ' ', TRUE);
mu_assert_lf(prec->field_count == 4);
mu_assert_lf(streq(lrec_get(prec, "w"), "2"));
mu_assert_lf(streq(lrec_get(prec, "x"), "3"));
mu_assert_lf(streq(lrec_get(prec, "y"), "4"));
mu_assert_lf(streq(lrec_get(prec, "z"), "5"));
lrec_remove(prec, "w");
mu_assert_lf(prec->field_count == 3);
mu_assert_lf(lrec_get(prec, "w") == NULL);
// Non-replacing-rename case
//lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "x", "u");
//lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 3);
mu_assert_lf(lrec_get(prec, "x") == NULL);
mu_assert_lf(streq(lrec_get(prec, "u"), "3"));
// Replacing-rename case
//lrec_dump_titled("Before rename", prec);
lrec_rename(prec, "y", "z");
//lrec_dump_titled("After rename", prec);
mu_assert_lf(prec->field_count == 2);
mu_assert_lf(streq(lrec_get(prec, "u"), "3"));
mu_assert_lf(lrec_get(prec, "y") == NULL);
mu_assert_lf(streq(lrec_get(prec, "z"), "4"));
lrec_free(prec);
return NULL;
}
// ================================================================
static char * run_all_tests() {
mu_run_test(test_lrec_unbacked_api);
mu_run_test(test_lrec_dkvp_api);
mu_run_test(test_lrec_nidx_api);
mu_run_test(test_lrec_csv_api);
mu_run_test(test_lrec_xtab_api);
return 0;
}
int main(int argc, char **argv) {
#ifdef MLR_USE_MCHECK
if (mcheck(NULL) != 0) {
printf("Could not set up mcheck\n");
exit(1);
}
printf("Set up mcheck\n");
#endif // MLR_USE_MCHECK
char *result = run_all_tests();
printf("\n");
if (result != 0) {
//printf("%s\n", result);
printf("Not all unit tests passed\n");
}
else {
printf("TEST_LREC: ALL UNIT TESTS PASSED\n");
}
printf("Tests run: %d\n", tests_run);
printf("Tests failed: %d\n", tests_failed);
printf("Assertions run: %d\n", assertions_run);
printf("Assertions failed: %d\n", assertions_failed);
return result != 0;
}
#endif // __TEST_JOIN_BUCKET_KEEPER_MAIN__