From ff28e681774fdfae9ea5ae829b8164457e63c080 Mon Sep 17 00:00:00 2001
From: John Kerl <kerl.john.r@gmail.com>
Date: Sat, 6 Jan 2018 12:17:17 -0500
Subject: [PATCH] skip/pass-comment CRLF UT

---
 c/oo                    |  48 ++++-----
 c/reg_test/expected/out | 227 ++++++++++++++++++++++++++++++++++++++++
 c/reg_test/run          | 104 ++++++++++++++++++
 c/todo.txt              |   2 +
 4 files changed, 352 insertions(+), 29 deletions(-)

diff --git a/c/oo b/c/oo
index 6a5e538e9..dfae07fee 100755
--- a/c/oo
+++ b/c/oo
@@ -6,6 +6,13 @@ run_mlr() {
   ./mlr "$@"
 }
 
+run_cat() {
+  echo cat "$@"
+  echo cat "$@" >> $outfile
+  cat "$@" >> $outfile
+  echo >> $outfile
+}
+
 mlr_expect_fail() {
   echo
   echo ................................................................
@@ -37,35 +44,18 @@ flags="-a"
 vee=
 
 # ----------------------------------------------------------------
-announce cat 1.csv
-cat 1.csv
-announce
-run_mlr --pass-comments --csv cat       1.csv
-run_mlr --pass-comments --csv cat     < 1.csv
-run_mlr --pass-comments --csvlite cat   1.csv
-run_mlr --pass-comments --csvlite cat < 1.csv
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+run_mlr termcvt --lf2crlf < $indir/comments/comments1.dkvp > $outdir/comments1-crlf.dkvp
 
-announce cat 2.csv
-cat 2.csv
-announce
-run_mlr --pass-comments --csv cat       2.csv
-run_mlr --pass-comments --csv cat     < 2.csv
-run_mlr --pass-comments --csvlite cat   2.csv
-run_mlr --pass-comments --csvlite cat < 2.csv
+mention input comments1-crlf.dkvp
+run_cat $outdir/comments1-crlf.dkvp
 
-announce cat 3.csv
-cat 3.csv
-announce
-run_mlr --pass-comments --csv cat       3.csv
-run_mlr --pass-comments --csv cat     < 3.csv
-run_mlr --pass-comments --csvlite cat   3.csv
-run_mlr --pass-comments --csvlite cat < 3.csv
-
-announce cat 4.csv
-cat 4.csv
-announce
-run_mlr --pass-comments --csv cat       4.csv
-run_mlr --pass-comments --csv cat     < 4.csv
-run_mlr --pass-comments --csvlite cat   4.csv
-run_mlr --pass-comments --csvlite cat < 4.csv
+mention skip comments1-crlf.dkvp
+run_mlr --skip-comments --idkvp --oxtab cat < $outdir/comments1-crlf.dkvp
+run_mlr --skip-comments --idkvp --oxtab cat   $outdir/comments1-crlf.dkvp
 
+mention pass comments1-crlf.dkvp
+run_mlr --pass-comments --idkvp --oxtab cat < $outdir/comments1-crlf.dkvp
+run_mlr --pass-comments --idkvp --oxtab cat   $outdir/comments1-crlf.dkvp
diff --git a/c/reg_test/expected/out b/c/reg_test/expected/out
index fb852706c..36f1b47d4 100644
--- a/c/reg_test/expected/out
+++ b/c/reg_test/expected/out
@@ -45116,6 +45116,59 @@ c 6
 # hello world 3
 
 
+---------------------------------------------------------------- input comments1-crlf.dkvp
+cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp
+# hello world 1
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
+---------------------------------------------------------------- skip comments1-crlf.dkvp
+mlr --skip-comments --idkvp --oxtab cat
+a 1
+b 2
+c 3
+
+a 4
+b 5
+c 6
+
+mlr --skip-comments --idkvp --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp
+a 1
+b 2
+c 3
+
+a 4
+b 5
+c 6
+
+
+---------------------------------------------------------------- pass comments1-crlf.dkvp
+mlr --pass-comments --idkvp --oxtab cat
+# hello world 1
+a 1
+b 2
+c 3
+
+a 4
+b 5
+c 6
+
+mlr --pass-comments --idkvp --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.dkvp
+# hello world 1
+a 1
+b 2
+c 3
+
+a 4
+b 5
+c 6
+
+
+================================================================
+SKIP/PASS COMMENTS DKVP WITH ALTERNATE PREFIX
+
+
 ---------------------------------------------------------------- input comments1-atat.dkvp
 cat ./reg_test/input/comments/comments1-atat.dkvp
 @@ hello world 1
@@ -45414,6 +45467,55 @@ mlr --pass-comments --inidx --oxtab cat ./reg_test/input/comments/comments3.nidx
 # hello world 3
 
 
+---------------------------------------------------------------- input comments1-crlf.nidx
+cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx
+# hello world 1
+aX1 bX2 cX3
+aX4 bX5 cX6
+
+
+---------------------------------------------------------------- skip comments1-crlf.nidx
+mlr --skip-comments --inidx --oxtab cat
+1 aX1
+2 bX2
+3 cX3
+
+1 aX4
+2 bX5
+3 cX6
+
+mlr --skip-comments --inidx --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx
+1 aX1
+2 bX2
+3 cX3
+
+1 aX4
+2 bX5
+3 cX6
+
+
+---------------------------------------------------------------- pass comments1-crlf.nidx
+mlr --pass-comments --inidx --oxtab cat
+# hello world 1
+1 aX1
+2 bX2
+3 cX3
+
+1 aX4
+2 bX5
+3 cX6
+
+mlr --pass-comments --inidx --oxtab cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.nidx
+# hello world 1
+1 aX1
+2 bX2
+3 cX3
+
+1 aX4
+2 bX5
+3 cX6
+
+
 ================================================================
 SKIP/PASS COMMENTS JSON
 
@@ -45505,6 +45607,35 @@ a=1,b=2,c=3
 a=4,b=5,c=6
 
 
+---------------------------------------------------------------- input comments1-crlf.json
+cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json
+# hello world 1
+{ "a":1, "b":2, "c":3 }
+{ "a":4, "b":5, "c":6 }
+
+
+---------------------------------------------------------------- skip comments1-crlf.json
+mlr --skip-comments --ijson --odkvp cat
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+mlr --skip-comments --ijson --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
+---------------------------------------------------------------- pass comments1-crlf.json
+mlr --pass-comments --ijson --odkvp cat
+# hello world 1
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+mlr --pass-comments --ijson --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.json
+# hello world 1
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
 ================================================================
 SKIP/PASS COMMENTS XTAB
 
@@ -45594,6 +45725,42 @@ x=2,z=5
 #hello world6
 
 
+---------------------------------------------------------------- input comments1-crlf.xtab
+cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab
+x 1
+#hello world3
+z 3
+
+x 2
+#hello world4
+z 5
+
+
+
+---------------------------------------------------------------- skip comments1-crlf.xtab
+mlr --skip-comments --ixtab --odkvp cat
+x=1,z=3
+x=2,z=5
+
+mlr --skip-comments --ixtab --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab
+x=1,z=3
+x=2,z=5
+
+
+---------------------------------------------------------------- pass comments1-crlf.xtab
+mlr --pass-comments --ixtab --odkvp cat
+#hello world3
+x=1,z=3
+#hello world4
+x=2,z=5
+
+mlr --pass-comments --ixtab --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.xtab
+#hello world3
+x=1,z=3
+#hello world4
+x=2,z=5
+
+
 ================================================================
 SKIP/PASS COMMENTS CSVLITE
 
@@ -45667,6 +45834,36 @@ a=4,b=5,c=6
 # haha
 
 
+---------------------------------------------------------------- input comments1-crlf.csv
+cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv
+# hello
+a,b,c
+1,2,3
+4,5,6
+
+
+---------------------------------------------------------------- skip comments1-crlf.csv
+mlr --skip-comments --icsvlite --odkvp cat
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+mlr --skip-comments --icsvlite --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
+---------------------------------------------------------------- pass comments1-crlf.csv
+mlr --pass-comments --icsvlite --odkvp cat
+# hello
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+mlr --pass-comments --icsvlite --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv
+# hello
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
 ================================================================
 SKIP/PASS COMMENTS CSV
 
@@ -45740,6 +45937,36 @@ a=4,b=5,c=6
 # haha
 
 
+---------------------------------------------------------------- input comments1-crlf.csv
+cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv
+# hello
+a,b,c
+1,2,3
+4,5,6
+
+
+---------------------------------------------------------------- skip comments1-crlf.csv
+mlr --skip-comments --icsv --odkvp cat
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+mlr --skip-comments --icsv --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
+---------------------------------------------------------------- pass comments1-crlf.csv
+mlr --pass-comments --icsv --odkvp cat
+# hello
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+mlr --pass-comments --icsv --odkvp cat /Users/kerl/pub_http_internet/miller-releases/miller-head/c/output-regtest/comments1-crlf.csv
+# hello
+a=1,b=2,c=3
+a=4,b=5,c=6
+
+
 ================================================================
 MMAP AT PAGE BOUNDARIES
 
diff --git a/c/reg_test/run b/c/reg_test/run
index 24441419f..d1765402f 100755
--- a/c/reg_test/run
+++ b/c/reg_test/run
@@ -5203,6 +5203,25 @@ mention pass comments3.dkvp
 run_mlr --pass-comments --idkvp --oxtab cat < $indir/comments/comments3.dkvp
 run_mlr --pass-comments --idkvp --oxtab cat   $indir/comments/comments3.dkvp
 
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+mlr termcvt --lf2crlf < $indir/comments/comments1.dkvp > $outdir/comments1-crlf.dkvp
+
+mention input comments1-crlf.dkvp
+run_cat $outdir/comments1-crlf.dkvp
+
+mention skip comments1-crlf.dkvp
+run_mlr --skip-comments --idkvp --oxtab cat < $outdir/comments1-crlf.dkvp
+run_mlr --skip-comments --idkvp --oxtab cat   $outdir/comments1-crlf.dkvp
+
+mention pass comments1-crlf.dkvp
+run_mlr --pass-comments --idkvp --oxtab cat < $outdir/comments1-crlf.dkvp
+run_mlr --pass-comments --idkvp --oxtab cat   $outdir/comments1-crlf.dkvp
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+announce SKIP/PASS COMMENTS DKVP WITH ALTERNATE PREFIX
 
 mention input comments1-atat.dkvp
 run_cat $indir/comments/comments1-atat.dkvp
@@ -5277,6 +5296,23 @@ mention pass comments3.nidx
 run_mlr --pass-comments --inidx --oxtab cat < $indir/comments/comments3.nidx
 run_mlr --pass-comments --inidx --oxtab cat   $indir/comments/comments3.nidx
 
+
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+mlr termcvt --lf2crlf < $indir/comments/comments1.nidx > $outdir/comments1-crlf.nidx
+
+mention input comments1-crlf.nidx
+run_cat $outdir/comments1-crlf.nidx
+
+mention skip comments1-crlf.nidx
+run_mlr --skip-comments --inidx --oxtab cat < $outdir/comments1-crlf.nidx
+run_mlr --skip-comments --inidx --oxtab cat   $outdir/comments1-crlf.nidx
+
+mention pass comments1-crlf.nidx
+run_mlr --pass-comments --inidx --oxtab cat < $outdir/comments1-crlf.nidx
+run_mlr --pass-comments --inidx --oxtab cat   $outdir/comments1-crlf.nidx
+
 # ----------------------------------------------------------------
 announce SKIP/PASS COMMENTS JSON
 
@@ -5315,6 +5351,23 @@ mention pass comments3.json
 run_mlr --pass-comments --ijson --odkvp cat < $indir/comments/comments3.json
 run_mlr --pass-comments --ijson --odkvp cat   $indir/comments/comments3.json
 
+
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+mlr termcvt --lf2crlf < $indir/comments/comments1.json > $outdir/comments1-crlf.json
+
+mention input comments1-crlf.json
+run_cat $outdir/comments1-crlf.json
+
+mention skip comments1-crlf.json
+run_mlr --skip-comments --ijson --odkvp cat < $outdir/comments1-crlf.json
+run_mlr --skip-comments --ijson --odkvp cat   $outdir/comments1-crlf.json
+
+mention pass comments1-crlf.json
+run_mlr --pass-comments --ijson --odkvp cat < $outdir/comments1-crlf.json
+run_mlr --pass-comments --ijson --odkvp cat   $outdir/comments1-crlf.json
+
 # ----------------------------------------------------------------
 announce SKIP/PASS COMMENTS XTAB
 
@@ -5341,6 +5394,23 @@ mention pass comments2.xtab
 run_mlr --pass-comments --ixtab --odkvp cat < $indir/comments/comments2.xtab
 run_mlr --pass-comments --ixtab --odkvp cat   $indir/comments/comments2.xtab
 
+
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+mlr termcvt --lf2crlf < $indir/comments/comments1.xtab > $outdir/comments1-crlf.xtab
+
+mention input comments1-crlf.xtab
+run_cat $outdir/comments1-crlf.xtab
+
+mention skip comments1-crlf.xtab
+run_mlr --skip-comments --ixtab --odkvp cat < $outdir/comments1-crlf.xtab
+run_mlr --skip-comments --ixtab --odkvp cat   $outdir/comments1-crlf.xtab
+
+mention pass comments1-crlf.xtab
+run_mlr --pass-comments --ixtab --odkvp cat < $outdir/comments1-crlf.xtab
+run_mlr --pass-comments --ixtab --odkvp cat   $outdir/comments1-crlf.xtab
+
 # ----------------------------------------------------------------
 announce SKIP/PASS COMMENTS CSVLITE
 
@@ -5367,6 +5437,23 @@ mention pass comments2.csv
 run_mlr --pass-comments --icsvlite --odkvp cat < $indir/comments/comments2.csv
 run_mlr --pass-comments --icsvlite --odkvp cat   $indir/comments/comments2.csv
 
+
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+mlr termcvt --lf2crlf < $indir/comments/comments1.csv > $outdir/comments1-crlf.csv
+
+mention input comments1-crlf.csv
+run_cat $outdir/comments1-crlf.csv
+
+mention skip comments1-crlf.csv
+run_mlr --skip-comments --icsvlite --odkvp cat < $outdir/comments1-crlf.csv
+run_mlr --skip-comments --icsvlite --odkvp cat   $outdir/comments1-crlf.csv
+
+mention pass comments1-crlf.csv
+run_mlr --pass-comments --icsvlite --odkvp cat < $outdir/comments1-crlf.csv
+run_mlr --pass-comments --icsvlite --odkvp cat   $outdir/comments1-crlf.csv
+
 # ----------------------------------------------------------------
 announce SKIP/PASS COMMENTS CSV
 
@@ -5393,6 +5480,23 @@ mention pass comments2.csv
 run_mlr --pass-comments --icsv --odkvp cat < $indir/comments/comments2.csv
 run_mlr --pass-comments --icsv --odkvp cat   $indir/comments/comments2.csv
 
+
+# It's annoying trying to check in text files (especially CSV) with CRLF
+# to Git, given that it likes to 'fix' line endings for multi-platform use.
+# It's easy to simply create CRLF on the fly.
+mlr termcvt --lf2crlf < $indir/comments/comments1.csv > $outdir/comments1-crlf.csv
+
+mention input comments1-crlf.csv
+run_cat $outdir/comments1-crlf.csv
+
+mention skip comments1-crlf.csv
+run_mlr --skip-comments --icsv --odkvp cat < $outdir/comments1-crlf.csv
+run_mlr --skip-comments --icsv --odkvp cat   $outdir/comments1-crlf.csv
+
+mention pass comments1-crlf.csv
+run_mlr --pass-comments --icsv --odkvp cat < $outdir/comments1-crlf.csv
+run_mlr --pass-comments --icsv --odkvp cat   $outdir/comments1-crlf.csv
+
 # ----------------------------------------------------------------
 announce MMAP AT PAGE BOUNDARIES
 
diff --git a/c/todo.txt b/c/todo.txt
index 47a7955ca..ac90c7113 100644
--- a/c/todo.txt
+++ b/c/todo.txt
@@ -24,6 +24,8 @@ BUGFIXES
 
 * valgrinds
 
+* aux-list -> help ...
+
 ================================================================
 FUNDAM: