diff --git a/c/Makefile b/c/Makefile index 9112d84be..b239018c5 100644 --- a/c/Makefile +++ b/c/Makefile @@ -99,9 +99,10 @@ mlrp: .always dsls tests: unit-test reg-test -unit-test: test-mlrutil test-lrec test-join-bucket-keeper +unit-test: test-mlrutil test-lrec test-string-builder test-join-bucket-keeper ./test-mlrutil ./test-lrec + ./test-string-builder ./test-join-bucket-keeper @echo @echo DONE @@ -116,10 +117,11 @@ dev-tests: dev-unit-test reg-test # Unfortunately --error-exitcode=1 doesn't work well since there are # valgrind-detected errors in stdlibs. :( -dev-unit-test: test-mlrutil test-lrec test-join-bucket-keeper +dev-unit-test: test-mlrutil test-lrec test-string-builder test-join-bucket-keeper #valgrind --leak-check=full --error-exitcode=1 ./a.out valgrind --leak-check=full ./test-mlrutil valgrind --leak-check=full ./test-lrec + valgrind --leak-check=full ./test-string-builder valgrind --leak-check=full ./test-join-bucket-keeper @echo @echo DONE @@ -138,6 +140,9 @@ test-lrec: .always test-mlrutil: .always $(CCDEBUG) -D__TEST_MLRUTIL_MAIN__ lib/mlrutil.c lib/test_mlrutil.c -o test-mlrutil +test-string-builder: .always + $(CCDEBUG) -D__TEST_STRING_BUILDER_MAIN__ lib/mlrutil.c lib/mlr_globals.c lib/string_builder.c lib/test_string_builder.c -o test-string-builder + test-join-bucket-keeper: .always $(CCDEBUG) -D__TEST_JOIN_BUCKET_KEEPER_MAIN__ $(TEST_JOIN_BUCKET_KEEPER_SRCS) -o test-join-bucket-keeper diff --git a/c/csv-rfc.txt b/c/csv-rfc.txt index 7dd4a2c28..dc546678c 100644 --- a/c/csv-rfc.txt +++ b/c/csv-rfc.txt @@ -1,3 +1,30 @@ +================================================================ +--> parser option: write a grammar like the below. be careful to + preserve the streaming property! + +--> state-machine option: + * each field either does or does not begin & end w/ " + * possibility of "" inside + * pretty simple except for CRLF being a double terminator, & fact I cannot + use getline anymore + * use fgetc & imitate mmap-style seeking + * still char irs needs to be char* irs ... and for rfc-csv it's non-parameterizable. + * for general string-fs/rs/ps (for other formats) there will need to be some rework. + + * char p,q = lagged-by-one fgetc: while p != EOF. + * have a char* get_field(FILE* fp) function. + * enter the function looking at start of field. + + * aux-fcn if SOF isn't DQUOTE: just string-build until COMMA, CRLF, or EOF + * aux-fcn if SOF is DQUOTE: just string-build until DQUOTE COMMA, + DQUOTE CRLF, or DQUOTE EOF. with special case that DQUOTE DQUOTE + maps to DQUOTE. + +--> regardless: need a string_builder_t with private int curlen, char* buf, + public fcn append_char, public fcn append_string, + private fcn realloc resizing with minimal copy, public fcn zterm & produce. + +================================================================ Definition of the CSV Format While there are various specifications and implementations for the @@ -86,4 +113,3 @@ The ABNF grammar [2] appears as follows: CRLF = CR LF ;as per section 6.1 of RFC 2234 [2] TEXTDATA = %x20-21 / %x23-2B / %x2D-7E - diff --git a/c/lib/string_builder.c b/c/lib/string_builder.c new file mode 100644 index 000000000..099c653c0 --- /dev/null +++ b/c/lib/string_builder.c @@ -0,0 +1,52 @@ +#include +#include +#include "string_builder.h" +#include "../lib/mlrutil.h" +#include "../lib/mlr_globals.h" + +static void sb_enlarge(string_builder_t* psb); + +// typedef struct _string_builder_t { +// int used_length; +// int alloc_length; +// char* buffer; +// } string_builder_t; + +void sb_init(string_builder_t* psb, int alloc_length) { + if (alloc_length < 1) { + fprintf(stderr, "%s: string_builder alloc_length must be >= 1; got %d.\n", + MLR_GLOBALS.argv0, alloc_length); + exit(1); + } + psb->used_length = 0; + psb->alloc_length = alloc_length; + psb->buffer = mlr_malloc_or_die(alloc_length); // xxx malloc ... +} + +void sb_append_char(string_builder_t* psb, char c) { + if (psb->used_length >= psb->alloc_length) + sb_enlarge(psb); + psb->buffer[psb->used_length++] = c; +} + +void sb_append_string(string_builder_t* psb, char* s) { + for (char* p = s; *p; p++) + sb_append_char(psb, *p); +} + +char* sb_finish(string_builder_t* psb) { + sb_append_char(psb, '\0'); + char* rv = psb->buffer; + psb->used_length = 0; + psb->alloc_length = 0; + psb->buffer = NULL; + return rv; +} + +static void sb_enlarge(string_builder_t* psb) { + int new_alloc_length = psb->alloc_length * 2; + char* new_buffer = mlr_malloc_or_die(new_alloc_length); + memcpy(new_buffer, psb->buffer, psb->used_length); + psb->alloc_length = new_alloc_length; + psb->buffer = new_buffer; +} diff --git a/c/lib/string_builder.h b/c/lib/string_builder.h new file mode 100644 index 000000000..4f40da249 --- /dev/null +++ b/c/lib/string_builder.h @@ -0,0 +1,16 @@ +#ifndef STRING_BUILDER_H +#define STRING_BUILDER_H + +typedef struct _string_builder_t { + int used_length; + int alloc_length; + char* buffer; +} string_builder_t; + +void sb_init(string_builder_t* psb, int alloc_length); +void sb_append_char(string_builder_t* psb, char c); +void sb_append_string(string_builder_t* psb, char* s); +// The caller should free() the return value: +char* sb_finish(string_builder_t* psb); + +#endif // STRING_BUILDER_H diff --git a/c/lib/test_string_builder.c b/c/lib/test_string_builder.c new file mode 100644 index 000000000..de4d23913 --- /dev/null +++ b/c/lib/test_string_builder.c @@ -0,0 +1,95 @@ +#include +#include +#include "lib/minunit.h" +#include "lib/mlrutil.h" +#include "lib/string_builder.h" + +#ifdef __TEST_STRING_BUILDER_MAIN__ +int tests_run = 0; +int tests_failed = 0; +int assertions_run = 0; +int assertions_failed = 0; + +// ---------------------------------------------------------------- +static char * test_simple() { + string_builder_t sb; + string_builder_t* psb = &sb; + + sb_init(psb, 1); + mu_assert("error: case 0", streq("", sb_finish(psb))); + + sb_init(psb, 1); + sb_append_char(psb, 'a'); + mu_assert("error: case 1", streq("a", sb_finish(psb))); + + sb_init(psb, 1); + sb_append_char(psb, 'a'); + sb_append_char(psb, 'b'); + mu_assert("error: case 2", streq("ab", sb_finish(psb))); + + sb_init(psb, 1); + sb_append_char(psb, 'a'); + sb_append_char(psb, 'b'); + sb_append_char(psb, 'b'); + sb_append_char(psb, 'c'); + sb_append_char(psb, 'c'); + sb_append_char(psb, 'e'); + mu_assert("error: case 3", streq("abbcce", sb_finish(psb))); + + sb_init(psb, 1); + sb_append_string(psb, ""); + mu_assert("error: case 4", streq("", sb_finish(psb))); + + sb_init(psb, 1); + sb_append_string(psb, "hello"); + mu_assert("error: case 5", streq("hello", sb_finish(psb))); + + sb_init(psb, 1); + sb_append_string(psb, "hello"); + sb_append_char(psb, ','); + sb_append_char(psb, ' '); + sb_append_string(psb, "world"); + sb_append_char(psb, '!'); + mu_assert("error: case 5", streq("hello, world!", sb_finish(psb))); + + sb_init(psb, 2); + sb_append_string(psb, "hello"); + sb_append_char(psb, ','); + sb_append_char(psb, ' '); + sb_append_string(psb, "world"); + sb_append_char(psb, '!'); + mu_assert("error: case 6", streq("hello, world!", sb_finish(psb))); + + sb_init(psb, 32768); + sb_append_string(psb, "hello"); + sb_append_char(psb, ','); + sb_append_char(psb, ' '); + sb_append_string(psb, "world"); + sb_append_char(psb, '!'); + mu_assert("error: case 7", streq("hello, world!", sb_finish(psb))); + + return 0; +} + +// ================================================================ +static char * all_tests() { + mu_run_test(test_simple); + return 0; +} + +int main(int argc, char **argv) { + char *result = all_tests(); + printf("\n"); + if (result != 0) { + //printf("%s\n", result); + printf("Not all unit tests passed\n"); + } + else { + printf("TEST_STRING_BUILDER: ALL UNIT TESTS PASSED\n"); + } + printf("Tests passed: %d of %d\n", tests_run - tests_failed, tests_run); + printf("Assertions passed: %d of %d\n", assertions_run - assertions_failed, assertions_run); + + return result != 0; +} +#endif // __TEST_STRING_BUILDER_MAIN__