diff --git a/c/containers/slls.c b/c/containers/slls.c
index b7e88aff6..5443ff232 100644
--- a/c/containers/slls.c
+++ b/c/containers/slls.c
@@ -108,6 +108,7 @@ slls_t* slls_from_line(char* line, char ifs, int allow_repeat_ifs) {
char* walker = line;
char* piece;
while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
+ mlr_rstrip(piece); // https://github.com/johnkerl/miller/issues/313
slls_append_no_free(plist, piece);
}
diff --git a/c/lib/mlrutil.c b/c/lib/mlrutil.c
index a532a25e8..4a401cad9 100644
--- a/c/lib/mlrutil.c
+++ b/c/lib/mlrutil.c
@@ -485,6 +485,25 @@ char* mlr_alloc_unbackslash(char* input) {
return output;
}
+// Destructively removes final LF, CR, or CR/LF in the string.
+void mlr_rstrip(char* s) {
+ if (s == NULL) {
+ return;
+ }
+ int len = strlen(s);
+ if (len >= 2) {
+ if (s[len-2] == '\r' && s[len-1] == '\n') {
+ s[len-2] = 0;
+ return;
+ }
+ }
+ if (len >= 1) {
+ if (s[len-1] == '\r' || s[len-1] == '\n') {
+ s[len-1] = 0;
+ }
+ }
+}
+
// Does a strdup even if there's nothing to expand, so the caller can unconditionally
// free what we return.
char* mlr_alloc_double_backslash(char* input) {
diff --git a/c/lib/mlrutil.h b/c/lib/mlrutil.h
index 3f8db81b5..41d995276 100644
--- a/c/lib/mlrutil.h
+++ b/c/lib/mlrutil.h
@@ -140,6 +140,9 @@ int power_of_two_above(int n);
// "\t", "\n", "\\" to single characters such as tab, newline, backslash, etc.
char* mlr_alloc_unbackslash(char* input);
+// Destructively removes final LF, CR, or CR/LF in the string.
+void mlr_rstrip(char* s);
+
// Miller DSL literals are unbackslashed: e.g. the two-character sequence "\t" is converted to a tab character, and
// users need to type "\\t" to get a backslash followed by a t. Well and good, but the system regex library handles
// backslashes not quite as I want. Namely, without this function,
diff --git a/c/unit_test/test_mlrutil.c b/c/unit_test/test_mlrutil.c
index 5db94da57..815a8d175 100644
--- a/c/unit_test/test_mlrutil.c
+++ b/c/unit_test/test_mlrutil.c
@@ -171,17 +171,60 @@ static char * test_unbackslash() {
return 0;
}
+// ----------------------------------------------------------------
+static char * test_rstrip() {
+
+ char* a = NULL;
+ mlr_rstrip(a);
+ mu_assert_lf(a == NULL);
+
+ a = mlr_strdup_or_die("");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, ""));
+
+ a = mlr_strdup_or_die("foo");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, "foo"));
+
+ a = mlr_strdup_or_die("\r");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, ""));
+
+ a = mlr_strdup_or_die("\n");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, ""));
+
+ a = mlr_strdup_or_die("\r\n");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, ""));
+
+ a = mlr_strdup_or_die("x\r");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, "x"));
+
+ a = mlr_strdup_or_die("x\n");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, "x"));
+
+ a = mlr_strdup_or_die("x\r\n");
+ mlr_rstrip(a);
+ mu_assert_lf(streq(a, "x"));
+
+ return 0;
+}
+
// ================================================================
static char * all_tests() {
mu_run_test(test_canonical_mod);
mu_run_test(test_power_of_two_above);
mu_run_test(test_streq);
mu_run_test(test_streqn);
- mu_run_test(test_strdup_quoted);
+mu_run_test(test_strdup_quoted);
mu_run_test(test_starts_or_ends_with);
mu_run_test(test_scanners);
mu_run_test(test_paste);
mu_run_test(test_unbackslash);
+ mu_run_test(test_rstrip);
return 0;
}
diff --git a/doc/cookbook.html b/doc/cookbook.html
index 150202aa3..2a4d1938c 100644
--- a/doc/cookbook.html
+++ b/doc/cookbook.html
@@ -858,7 +858,7 @@ date,qoh
$ wc -l data/miss-date.csv
-1372 data/miss-date.csv
+ 1372 data/miss-date.csv
diff --git a/doc/data-examples.html b/doc/data-examples.html
index c8bcf0b6d..6f4c434e3 100644
--- a/doc/data-examples.html
+++ b/doc/data-examples.html
@@ -208,7 +208,7 @@ St. Johns - - -
$ wc -l data/colored-shapes.dkvp
-10078 data/colored-shapes.dkvp
+ 10078 data/colored-shapes.dkvp
diff --git a/doc/data-sharing.html b/doc/data-sharing.html
index f6b274e37..6c8f3801f 100644
--- a/doc/data-sharing.html
+++ b/doc/data-sharing.html
@@ -417,11 +417,11 @@ wye pan 5 0.5732889198020006 0.8636244699032729 {5}
$ mlr --opprint put '$o = system("echo -n ".$a."| sha1sum")' data/small
a b i x y o
-pan pan 1 0.3467901443380824 0.7268028627434533 bd2bd8216b9cb4aa5a12daa6cfc98eef2ee20e56 -
-eks pan 2 0.7586799647899636 0.5221511083334797 16191338e81a46c7d127f5c8899f5c92e3cd38e3 -
-wye wye 3 0.20460330576630303 0.33831852551664776 14ba3c3e96a2474ab6dc7409ebf9d6b9cc3d84f0 -
-eks wye 4 0.38139939387114097 0.13418874328430463 16191338e81a46c7d127f5c8899f5c92e3cd38e3 -
-wye pan 5 0.5732889198020006 0.8636244699032729 14ba3c3e96a2474ab6dc7409ebf9d6b9cc3d84f0 -
+pan pan 1 0.3467901443380824 0.7268028627434533 f29c748220331c273ef16d5115f6ecd799947f13 -
+eks pan 2 0.7586799647899636 0.5221511083334797 456d988ecb3bf1b75f057fc6e9fe70db464e9388 -
+wye wye 3 0.20460330576630303 0.33831852551664776 eab0de043d67f441c7fd1e335f0ca38708e6ebf7 -
+eks wye 4 0.38139939387114097 0.13418874328430463 456d988ecb3bf1b75f057fc6e9fe70db464e9388 -
+wye pan 5 0.5732889198020006 0.8636244699032729 eab0de043d67f441c7fd1e335f0ca38708e6ebf7 -
diff --git a/doc/manpage.html b/doc/manpage.html
index 05613f552..ac8f60bfe 100644
--- a/doc/manpage.html
+++ b/doc/manpage.html
@@ -2374,7 +2374,7 @@ SEE ALSO
- 2020-03-17 MILLER(1)
+ 2020-04-06 MILLER(1)
diff --git a/doc/manpage.txt b/doc/manpage.txt
index 3170900e8..afaa837d0 100644
--- a/doc/manpage.txt
+++ b/doc/manpage.txt
@@ -2300,4 +2300,4 @@ SEE ALSO
- 2020-03-17 MILLER(1)
+ 2020-04-06 MILLER(1)
diff --git a/doc/mlr.1 b/doc/mlr.1
index 0755a3a4c..a658217c4 100644
--- a/doc/mlr.1
+++ b/doc/mlr.1
@@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
-.\" Date: 2020-03-17
+.\" Date: 2020-04-06
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
-.TH "MILLER" "1" "2020-03-17" "\ \&" "\ \&"
+.TH "MILLER" "1" "2020-04-06" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/reference-dsl.html b/doc/reference-dsl.html
index 2103a2bd3..fa46e1a6e 100644
--- a/doc/reference-dsl.html
+++ b/doc/reference-dsl.html
@@ -2025,7 +2025,7 @@ subsequent assignments are done unconditionally:
$ mlr put '$x > 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp
x=-1,y=nan,z=nan
-x=0,y=-inf,z=-nan
+x=0,y=-inf,z=nan
x=1,y=0.000000,z=0.000000
x=2,y=0.301030,z=0.548662
x=3,y=0.477121,z=0.690740
diff --git a/doc/reference-verbs.html b/doc/reference-verbs.html
index 604058e62..2fac96979 100644
--- a/doc/reference-verbs.html
+++ b/doc/reference-verbs.html
@@ -3968,7 +3968,7 @@ Options:
$ wc -l data/colored-shapes.dkvp
-10078 data/colored-shapes.dkvp
+ 10078 data/colored-shapes.dkvp
@@ -4140,7 +4140,7 @@ color=purple,shape=square,flag=0
$ wc -l data/repeats.dkvp
-57 data/repeats.dkvp
+ 57 data/repeats.dkvp