From 2de70a96da0b80ea773bd593583cd9b34e163e79 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 27 Aug 2015 22:15:12 -0400 Subject: [PATCH] fix UTF-8 alignment for pprint and xtab formats --- c/mapping/mapper_cat.c | 3 ++- c/output/lrec_writer_pprint.c | 31 +++++++++++++++++++++++-------- c/output/lrec_writer_xtab.c | 7 ++++++- c/test/expected/out | 30 ++++++++++++++++++++++++++++++ c/test/input/utf8-align.dkvp | 3 +++ c/test/input/utf8-align.nidx | 4 ++++ c/test/output/out | 30 ++++++++++++++++++++++++++++++ c/test/run | 4 ++++ 8 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 c/test/input/utf8-align.dkvp create mode 100644 c/test/input/utf8-align.nidx diff --git a/c/mapping/mapper_cat.c b/c/mapping/mapper_cat.c index 282bca43a..43a8432e9 100644 --- a/c/mapping/mapper_cat.c +++ b/c/mapping/mapper_cat.c @@ -4,8 +4,9 @@ // ---------------------------------------------------------------- static sllv_t* mapper_cat_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { - if (pinrec != NULL) + if (pinrec != NULL) { return sllv_single(pinrec); + } else return sllv_single(NULL); } diff --git a/c/output/lrec_writer_pprint.c b/c/output/lrec_writer_pprint.c index 10eb8910b..2b8a7fd2c 100644 --- a/c/output/lrec_writer_pprint.c +++ b/c/output/lrec_writer_pprint.c @@ -79,12 +79,20 @@ static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, in fputc(' ', output_stream); } if (left_align) { - if (pe->pnext == NULL) + if (pe->pnext == NULL) { fprintf(output_stream, "%s", pe->key); - else - fprintf(output_stream, "%-*s", max_widths[j], pe->key); + } else { + // "%-*s" fprintf format isn't correct for non-ASCII UTF-8 + fprintf(output_stream, "%s", pe->key); + int d = max_widths[j] - strlen_for_utf8_display(pe->key); + for (int i = 0; i < d; i++) + fputc(' ', output_stream); + } } else { - fprintf(output_stream, "%*s", max_widths[j], pe->key); + int d = max_widths[j] - strlen_for_utf8_display(pe->key); + for (int i = 0; i < d; i++) + fputc(' ', output_stream); + fprintf(output_stream, "%s", pe->key); } } fputc('\n', output_stream); @@ -99,12 +107,19 @@ static void print_and_free_record_list(sllv_t* precords, FILE* output_stream, in if (*value == 0) // empty string value = "-"; if (left_align) { - if (pe->pnext == NULL) + if (pe->pnext == NULL) { fprintf(output_stream, "%s", value); - else - fprintf(output_stream, "%-*s", max_widths[j], value); + } else { + fprintf(output_stream, "%s", value); + int d = max_widths[j] - strlen_for_utf8_display(value); + for (int i = 0; i < d; i++) + fputc(' ', output_stream); + } } else { - fprintf(output_stream, "%*s", max_widths[j], value); + int d = max_widths[j] - strlen_for_utf8_display(value); + for (int i = 0; i < d; i++) + fputc(' ', output_stream); + fprintf(output_stream, "%s", value); } } fputc('\n', output_stream); diff --git a/c/output/lrec_writer_xtab.c b/c/output/lrec_writer_xtab.c index 126af16fb..39bad0a5a 100644 --- a/c/output/lrec_writer_xtab.c +++ b/c/output/lrec_writer_xtab.c @@ -24,7 +24,12 @@ static void lrec_writer_xtab_process(FILE* output_stream, lrec_t* prec, void* pv } for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) { - fprintf(output_stream, "%-*s %s\n", max_key_width, pe->key, pe->value); + // "%-*s" fprintf format isn't correct for non-ASCII UTF-8 + fprintf(output_stream, "%s", pe->key); + int d = max_key_width - strlen_for_utf8_display(pe->key); + for (int i = 0; i < d; i++) + fputc(' ', output_stream); + fprintf(output_stream, " %s\n", pe->value); } lrec_free(prec); // xxx cmt mem-mgmt } diff --git a/c/test/expected/out b/c/test/expected/out index 4eacf3530..aa334cf25 100644 --- a/c/test/expected/out +++ b/c/test/expected/out @@ -1887,6 +1887,36 @@ français langue françois nom vendredi jour +./test/../mlr --inidx --ifs space --opprint cat test/input/utf8-align.nidx +1 2 +191º test +191 test2 +francois français +françois francais + +./test/../mlr --inidx --ifs space --opprint --right cat test/input/utf8-align.nidx + 1 2 + 191º test + 191 test2 +francois français +françois francais + +./test/../mlr --oxtab cat test/input/utf8-align.dkvp +191º test +1912 test2 +cois çais +çois cais + +191º test +1912 test2 +ois çais +çois cais + +191º test +1912 test2 +coise çais +çois cais + ================================================================ RFC-CSV diff --git a/c/test/input/utf8-align.dkvp b/c/test/input/utf8-align.dkvp new file mode 100644 index 000000000..925b12366 --- /dev/null +++ b/c/test/input/utf8-align.dkvp @@ -0,0 +1,3 @@ +191º=test,1912=test2,cois=çais,çois=cais +191º=test,1912=test2,ois=çais,çois=cais +191º=test,1912=test2,coise=çais,çois=cais diff --git a/c/test/input/utf8-align.nidx b/c/test/input/utf8-align.nidx new file mode 100644 index 000000000..5a3bbe6e6 --- /dev/null +++ b/c/test/input/utf8-align.nidx @@ -0,0 +1,4 @@ +191º test +191 test2 +francois français +françois francais diff --git a/c/test/output/out b/c/test/output/out index 4eacf3530..aa334cf25 100644 --- a/c/test/output/out +++ b/c/test/output/out @@ -1887,6 +1887,36 @@ français langue françois nom vendredi jour +./test/../mlr --inidx --ifs space --opprint cat test/input/utf8-align.nidx +1 2 +191º test +191 test2 +francois français +françois francais + +./test/../mlr --inidx --ifs space --opprint --right cat test/input/utf8-align.nidx + 1 2 + 191º test + 191 test2 +francois français +françois francais + +./test/../mlr --oxtab cat test/input/utf8-align.dkvp +191º test +1912 test2 +cois çais +çois cais + +191º test +1912 test2 +ois çais +çois cais + +191º test +1912 test2 +coise çais +çois cais + ================================================================ RFC-CSV diff --git a/c/test/run b/c/test/run index a28b5194d..89ef24188 100755 --- a/c/test/run +++ b/c/test/run @@ -374,6 +374,10 @@ run_command $mlr --icsvlite --opprint cat test/input/utf8-2.csv run_command $mlr --icsvlite --oxtab cat test/input/utf8-1.csv run_command $mlr --icsvlite --oxtab cat test/input/utf8-2.csv +run_command $mlr --inidx --ifs space --opprint cat test/input/utf8-align.nidx +run_command $mlr --inidx --ifs space --opprint --right cat test/input/utf8-align.nidx +run_command $mlr --oxtab cat test/input/utf8-align.dkvp + # ================================================================ announce RFC-CSV