handle CR and/or LF in field-name args following {verb} -f

c/containers/slls.c

@@ -108,6 +108,7 @@ slls_t* slls_from_line(char* line, char ifs, int allow_repeat_ifs) {
 	char* walker = line;
 	char* piece;
 	while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
+		mlr_rstrip(piece); // https://github.com/johnkerl/miller/issues/313
 		slls_append_no_free(plist, piece);
 	}
 

c/lib/mlrutil.c

@@ -485,6 +485,25 @@ char* mlr_alloc_unbackslash(char* input) {
 	return output;
 }
 
+// Destructively removes final LF, CR, or CR/LF in the string.
+void mlr_rstrip(char* s) {
+	if (s == NULL) {
+		return;
+	}
+	int len = strlen(s);
+	if (len >= 2) {
+		if (s[len-2] == '\r' && s[len-1] == '\n') {
+			s[len-2] = 0;
+			return;
+		}
+	}
+	if (len >= 1) {
+		if (s[len-1] == '\r' || s[len-1] == '\n') {
+			s[len-1] = 0;
+		}
+	}
+}
+
 // Does a strdup even if there's nothing to expand, so the caller can unconditionally
 // free what we return.
 char* mlr_alloc_double_backslash(char* input) {

c/lib/mlrutil.h

@@ -140,6 +140,9 @@ int power_of_two_above(int n);
 // "\t", "\n", "\\" to single characters such as tab, newline, backslash, etc.
 char* mlr_alloc_unbackslash(char* input);
 
+// Destructively removes final LF, CR, or CR/LF in the string.
+void mlr_rstrip(char* s);
+
 // Miller DSL literals are unbackslashed: e.g. the two-character sequence "\t" is converted to a tab character, and
 // users need to type "\\t" to get a backslash followed by a t. Well and good, but the system regex library handles
 // backslashes not quite as I want. Namely, without this function,

c/unit_test/test_mlrutil.c

@@ -171,17 +171,60 @@ static char * test_unbackslash() {
 	return 0;
 }
 
+// ----------------------------------------------------------------
+static char * test_rstrip() {
+
+	char* a = NULL;
+	mlr_rstrip(a);
+	mu_assert_lf(a == NULL);
+
+	a = mlr_strdup_or_die("");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, ""));
+
+	a = mlr_strdup_or_die("foo");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, "foo"));
+
+	a = mlr_strdup_or_die("\r");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, ""));
+
+	a = mlr_strdup_or_die("\n");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, ""));
+
+	a = mlr_strdup_or_die("\r\n");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, ""));
+
+	a = mlr_strdup_or_die("x\r");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, "x"));
+
+	a = mlr_strdup_or_die("x\n");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, "x"));
+
+	a = mlr_strdup_or_die("x\r\n");
+	mlr_rstrip(a);
+	mu_assert_lf(streq(a, "x"));
+
+	return 0;
+}
+
 // ================================================================
 static char * all_tests() {
 	mu_run_test(test_canonical_mod);
 	mu_run_test(test_power_of_two_above);
 	mu_run_test(test_streq);
 	mu_run_test(test_streqn);
-	mu_run_test(test_strdup_quoted);
+mu_run_test(test_strdup_quoted);
 	mu_run_test(test_starts_or_ends_with);
 	mu_run_test(test_scanners);
 	mu_run_test(test_paste);
 	mu_run_test(test_unbackslash);
+	mu_run_test(test_rstrip);
 	return 0;
 }
 

doc/cookbook.html

@@ -858,7 +858,7 @@ date,qoh
 <div class="pokipanel">
 <pre>
 $ wc -l data/miss-date.csv
-1372 data/miss-date.csv
+    1372 data/miss-date.csv
 </pre>
 </div>
 <p/>

doc/data-examples.html

@@ -208,7 +208,7 @@ St. Johns  -                      -                       -
 <div class="pokipanel">
 <pre>
 $ wc -l data/colored-shapes.dkvp
-10078 data/colored-shapes.dkvp
+   10078 data/colored-shapes.dkvp
 </pre>
 </div>
 <p/>

doc/data-sharing.html

@@ -417,11 +417,11 @@ wye pan 5 0.5732889198020006  0.8636244699032729  {5}
 <pre>
 $ mlr --opprint put '$o = system("echo -n ".$a."| sha1sum")' data/small
 a   b   i x                   y                   o
-pan pan 1 0.3467901443380824  0.7268028627434533  bd2bd8216b9cb4aa5a12daa6cfc98eef2ee20e56  -
-eks pan 2 0.7586799647899636  0.5221511083334797  16191338e81a46c7d127f5c8899f5c92e3cd38e3  -
-wye wye 3 0.20460330576630303 0.33831852551664776 14ba3c3e96a2474ab6dc7409ebf9d6b9cc3d84f0  -
-eks wye 4 0.38139939387114097 0.13418874328430463 16191338e81a46c7d127f5c8899f5c92e3cd38e3  -
-wye pan 5 0.5732889198020006  0.8636244699032729  14ba3c3e96a2474ab6dc7409ebf9d6b9cc3d84f0  -
+pan pan 1 0.3467901443380824  0.7268028627434533  f29c748220331c273ef16d5115f6ecd799947f13  -
+eks pan 2 0.7586799647899636  0.5221511083334797  456d988ecb3bf1b75f057fc6e9fe70db464e9388  -
+wye wye 3 0.20460330576630303 0.33831852551664776 eab0de043d67f441c7fd1e335f0ca38708e6ebf7  -
+eks wye 4 0.38139939387114097 0.13418874328430463 456d988ecb3bf1b75f057fc6e9fe70db464e9388  -
+wye pan 5 0.5732889198020006  0.8636244699032729  eab0de043d67f441c7fd1e335f0ca38708e6ebf7  -
 </pre>
 </div>
 <p/>

doc/manpage.html

@@ -2374,7 +2374,7 @@ SEE ALSO
 
 
 
-				  2020-03-17			     MILLER(1)
+				  2020-04-06			     MILLER(1)
 </pre>
 </div>
 <p/>

doc/manpage.txt

@@ -2300,4 +2300,4 @@ SEE ALSO
 
 
 
-				  2020-03-17			     MILLER(1)
+				  2020-04-06			     MILLER(1)

doc/mlr.1

@@ -2,12 +2,12 @@
 .\"     Title: mlr
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: ./mkman.rb
-.\"      Date: 2020-03-17
+.\"      Date: 2020-04-06
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MILLER" "1" "2020-03-17" "\ \&" "\ \&"
+.TH "MILLER" "1" "2020-04-06" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Portability definitions
 .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/reference-dsl.html

@@ -2025,7 +2025,7 @@ subsequent assignments are done unconditionally:
 <pre>
 $ mlr put '$x &gt; 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp
 x=-1,y=nan,z=nan
-x=0,y=-inf,z=-nan
+x=0,y=-inf,z=nan
 x=1,y=0.000000,z=0.000000
 x=2,y=0.301030,z=0.548662
 x=3,y=0.477121,z=0.690740

doc/reference-verbs.html

@@ -3968,7 +3968,7 @@ Options:
 <div class="pokipanel">
 <pre>
 $ wc -l data/colored-shapes.dkvp
-10078 data/colored-shapes.dkvp
+   10078 data/colored-shapes.dkvp
 </pre>
 </div>
 <p/>
@@ -4140,7 +4140,7 @@ color=purple,shape=square,flag=0
 <div class="pokipanel">
 <pre>
 $ wc -l data/repeats.dkvp
-57 data/repeats.dkvp
+      57 data/repeats.dkvp
 </pre>
 </div>
 <p/>