From cfc986b5f4678d8f53e8c549db2aef02c46bb7c6 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sat, 4 Sep 2021 14:19:15 -0400 Subject: [PATCH] Fix non-auto-infer of true/false/Infinity/NaN/etc in non-JSON data files --- docs6/docs/10min.md | 11 ++---- docs6/docs/data-cleaning-examples.md | 14 +++---- docs6/docs/keystroke-savers.md | 4 +- docs6/docs/proofreads.txt | 7 +++- docs6/docs/record-heterogeneity.md | 3 ++ docs6/docs/reference-main-data-types.md | 2 +- docs6/docs/reference-main-maps.md | 4 +- docs6/docs/repl.md | 4 +- docs6/docs/special-symbols-and-formatting.md | 4 +- .../cases/dsl-min-max-types/0001/expout | 12 +++--- .../cases/dsl-min-max-types/0002/expout | 16 ++++---- .../cases/dsl-min-max-types/0003/expout | 8 ++-- .../cases/dsl-min-max-types/0004/expout | 8 ++-- go/src/input/record_reader_csv.go | 6 +-- go/src/input/record_reader_csvlite.go | 12 +++--- go/src/input/record_reader_dkvp.go | 4 +- go/src/input/record_reader_nidx.go | 2 +- go/src/input/record_reader_xtab.go | 2 +- go/src/types/mlrval_new.go | 38 +++++++++++++++++-- 19 files changed, 97 insertions(+), 64 deletions(-) diff --git a/docs6/docs/10min.md b/docs6/docs/10min.md index d75a63584..db6900811 100644 --- a/docs6/docs/10min.md +++ b/docs6/docs/10min.md @@ -87,7 +87,7 @@ purple,square,false,10,91,72.3735,8.2430 { "color": "yellow", "shape": "circle", - "flag": true, + "flag": "true", "k": 9, "index": 87, "quantity": 63.5058, @@ -96,7 +96,7 @@ purple,square,false,10,91,72.3735,8.2430 { "color": "purple", "shape": "square", - "flag": false, + "flag": "false", "k": 10, "index": 91, "quantity": 72.3735, @@ -212,14 +212,9 @@ red square false 4 48 77.5542 7.4670 red square false 6 64 77.1991 9.5310 -
+
 mlr --icsv --opprint filter '$color == "red" && $flag == true' example.csv
 

-
-color shape  flag k index quantity rate
-red   square true 2 15    79.2778  0.0130
-red   circle true 3 16    13.8103  2.9010
-

 
 You can use `put` to create new fields which are computed from other fields:
 
diff --git a/docs6/docs/data-cleaning-examples.md b/docs6/docs/data-cleaning-examples.md
index b03c96b23..be3b74286 100644
--- a/docs6/docs/data-cleaning-examples.md
+++ b/docs6/docs/data-cleaning-examples.md
@@ -58,9 +58,9 @@ A second option is to flag badly formatted data within the output stream:
 
 name   reachable format_ok
-barney false     false
-betty  true      false
-fred   true      false
+barney false     true
+betty  true      true
+fred   true      true
 wilma  1         false
@@ -72,9 +72,6 @@ Or perhaps to flag badly formatted data outside the output stream: ' data/het-bool.csv 
-Malformed at NR=1
-Malformed at NR=2
-Malformed at NR=3
 Malformed at NR=4
 name   reachable
 barney false
@@ -89,5 +86,8 @@ A third way is to abort the process on first instance of bad data:
 mlr --csv put '$reachable = asserting_string($reachable)' data/het-bool.csv
 
-Miller: is_string type-assertion failed at NR=1 FNR=1 FILENAME=data/het-bool.csv
+name,reachable
+barney,false
+Miller: is_string type-assertion failed at NR=4 FNR=4 FILENAME=data/het-bool.csv
+betty,true
diff --git a/docs6/docs/keystroke-savers.md b/docs6/docs/keystroke-savers.md index fa48095e1..df88f3885 100644 --- a/docs6/docs/keystroke-savers.md +++ b/docs6/docs/keystroke-savers.md @@ -34,7 +34,7 @@ red square true 2 15 79.2778 0.0130 { "color": "yellow", "shape": "triangle", - "flag": true, + "flag": "true", "k": 1, "index": 11, "quantity": 43.6498, @@ -43,7 +43,7 @@ red square true 2 15 79.2778 0.0130 { "color": "red", "shape": "square", - "flag": true, + "flag": "true", "k": 2, "index": 15, "quantity": 79.2778, diff --git a/docs6/docs/proofreads.txt b/docs6/docs/proofreads.txt index 44274ffe6..49505c85c 100644 --- a/docs6/docs/proofreads.txt +++ b/docs6/docs/proofreads.txt @@ -3,14 +3,17 @@ TOP: c data-types: ! don't auto-infer bools; then, data-types page ! don't auto-infer Infinity, etc -- ?!? then, data-types page + RT-check + https://github.com/johnkerl/miller/issues/241 + https://github.com/johnkerl/miller/issues/357 MlrvalPointerFromInferredType -> mk variant -E flatten/unflatten page c! seps \001 etc ! mlrc --iusv --oxtab cat regtest/input/example.usv mlr --iusv --oxtab cat regtest/input/example.usv C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ? -? twi-dm re all-contribs: all-contributors.org +E flatten/unflatten page +? twi-dm re all-contribs: all-contributors.org C flags LUTs ---------------------------------------------------------------- diff --git a/docs6/docs/record-heterogeneity.md b/docs6/docs/record-heterogeneity.md index 6d57396e7..cf2f63136 100644 --- a/docs6/docs/record-heterogeneity.md +++ b/docs6/docs/record-heterogeneity.md @@ -126,6 +126,9 @@ If you `mlr csv cat` this, you'll get an error message: 
 mlr :  Miller: CSV header/data length mismatch 3 != 2 at filename data/het/ragged.csv row 3.
+
+a,b,c
+1,2,3
There are two kinds of raggedness here. Since CSVs form records by zipping the diff --git a/docs6/docs/reference-main-data-types.md b/docs6/docs/reference-main-data-types.md index ed33a0350..afe993d1e 100644 --- a/docs6/docs/reference-main-data-types.md +++ b/docs6/docs/reference-main-data-types.md @@ -92,7 +92,7 @@ ta float b 3 tb int c true -tc bool +tc string d 1.2true td string e 7 diff --git a/docs6/docs/reference-main-maps.md b/docs6/docs/reference-main-maps.md index ad23b1ae4..81debf235 100644 --- a/docs6/docs/reference-main-maps.md +++ b/docs6/docs/reference-main-maps.md @@ -82,7 +82,7 @@ The current record, accessible using `$*`, is a map. { "color": "yellow", "shape": "triangle", - "flag": true, + "flag": "true", "k": 1, "index": 11, "quantity": 43.6498, @@ -92,7 +92,7 @@ Color is yellow { "color": "red", "shape": "square", - "flag": true, + "flag": "true", "k": 2, "index": 15, "quantity": 79.2778, diff --git a/docs6/docs/repl.md b/docs6/docs/repl.md index 3cb2e90a4..61bf3c42e 100644 --- a/docs6/docs/repl.md +++ b/docs6/docs/repl.md @@ -48,7 +48,7 @@ HELLO { "color": "yellow", "shape": "triangle", - "flag": true, + "flag": "true", "k": 1, "index": 11, "quantity": 43.6498, @@ -58,7 +58,7 @@ HELLO { "color": "red", "shape": "square", - "flag": true, + "flag": "true", "k": 2, "index": 15, "quantity": 79.2778, diff --git a/docs6/docs/special-symbols-and-formatting.md b/docs6/docs/special-symbols-and-formatting.md index 44b2bad10..ea50e6ecb 100644 --- a/docs6/docs/special-symbols-and-formatting.md +++ b/docs6/docs/special-symbols-and-formatting.md @@ -83,8 +83,8 @@ characters as delimiters -- here, control-A: mlr --icsv --odkvp --ofs '\001' cat commas.csv | cat -v 
-Name=Xiao, Lin\001Role=administrator
-Name=Khavari, Darius\001Role=tester
+Name=Xiao, Lin^ARole=administrator
+Name=Khavari, Darius^ARole=tester
## How can I handle field names with special symbols in them? diff --git a/go/regtest/cases/dsl-min-max-types/0001/expout b/go/regtest/cases/dsl-min-max-types/0001/expout index 6e03b4091..e021b5638 100644 --- a/go/regtest/cases/dsl-min-max-types/0001/expout +++ b/go/regtest/cases/dsl-min-max-types/0001/expout @@ -1,6 +1,6 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "min": { @@ -12,19 +12,19 @@ }, "b": { "n": 1, - "b": true, - "v": true, - "s": true + "b": "true", + "v": "", + "s": "abc" }, "v": { "n": 1, - "b": true, + "b": "", "v": "", "s": "" }, "s": { "n": 1, - "b": true, + "b": "abc", "v": "", "s": "abc" } diff --git a/go/regtest/cases/dsl-min-max-types/0002/expout b/go/regtest/cases/dsl-min-max-types/0002/expout index 34f50b766..4c4057af9 100644 --- a/go/regtest/cases/dsl-min-max-types/0002/expout +++ b/go/regtest/cases/dsl-min-max-types/0002/expout @@ -1,30 +1,30 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "max": { "n": { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc" }, "b": { - "n": true, - "b": true, - "v": "", - "s": "abc" + "n": "true", + "b": "true", + "v": "true", + "s": "true" }, "v": { "n": "", - "b": "", + "b": "true", "v": "", "s": "abc" }, "s": { "n": "abc", - "b": "abc", + "b": "true", "v": "abc", "s": "abc" } diff --git a/go/regtest/cases/dsl-min-max-types/0003/expout b/go/regtest/cases/dsl-min-max-types/0003/expout index 714c9de22..bd0cd7c8f 100644 --- a/go/regtest/cases/dsl-min-max-types/0003/expout +++ b/go/regtest/cases/dsl-min-max-types/0003/expout @@ -1,12 +1,12 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "le": { "n": { "n": true, - "b": false, + "b": true, "v": false, "s": true }, @@ -18,13 +18,13 @@ }, "v": { "n": true, - "b": false, + "b": true, "v": true, "s": true }, "s": { "n": false, - "b": false, + "b": true, "v": false, "s": true } diff --git a/go/regtest/cases/dsl-min-max-types/0004/expout b/go/regtest/cases/dsl-min-max-types/0004/expout index 1575d57b5..2e62305c0 100644 --- a/go/regtest/cases/dsl-min-max-types/0004/expout +++ b/go/regtest/cases/dsl-min-max-types/0004/expout @@ -1,6 +1,6 @@ { "n": 1, - "b": true, + "b": "true", "v": "", "s": "abc", "ge": { @@ -11,10 +11,10 @@ "s": false }, "b": { - "n": false, + "n": true, "b": true, - "v": false, - "s": false + "v": true, + "s": true }, "v": { "n": false, diff --git a/go/src/input/record_reader_csv.go b/go/src/input/record_reader_csv.go index 8e3809fdb..0cc3a719b 100644 --- a/go/src/input/record_reader_csv.go +++ b/go/src/input/record_reader_csv.go @@ -140,7 +140,7 @@ func (reader *RecordReaderCSV) processHandle( if nh == nd { for i := 0; i < nh; i++ { key := header[i] - value := types.MlrvalPointerFromInferredType(csvRecord[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i]) record.PutReference(key, value) } @@ -160,13 +160,13 @@ func (reader *RecordReaderCSV) processHandle( n := lib.IntMin2(nh, nd) for i = 0; i < n; i++ { key := header[i] - value := types.MlrvalPointerFromInferredType(csvRecord[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i]) record.PutReference(key, value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalPointerFromInferredType(csvRecord[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i]) record.PutCopy(key, value) } if nh > nd { diff --git a/go/src/input/record_reader_csvlite.go b/go/src/input/record_reader_csvlite.go index 308d39401..03335c0f3 100644 --- a/go/src/input/record_reader_csvlite.go +++ b/go/src/input/record_reader_csvlite.go @@ -193,7 +193,7 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader( record := types.NewMlrmap() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalPointerFromInferredType(field) + value := types.MlrvalPointerFromInferredTypeForDataFiles(field) record.PutCopy(headerStrings[i], value) } } else { @@ -202,14 +202,14 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys for i = nh; i < nd; i++ { key := strconv.Itoa(i + 1) - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(key, value) } } @@ -306,7 +306,7 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader( record := types.NewMlrmap() if !reader.readerOptions.AllowRaggedCSVInput { for i, field := range fields { - value := types.MlrvalPointerFromInferredType(field) + value := types.MlrvalPointerFromInferredTypeForDataFiles(field) record.PutCopy(headerStrings[i], value) } } else { @@ -315,13 +315,13 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader( n := lib.IntMin2(nh, nd) var i int for i = 0; i < n; i++ { - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(headerStrings[i], value) } if nh < nd { // if header shorter than data: use 1-up itoa keys key := strconv.Itoa(i + 1) - value := types.MlrvalPointerFromInferredType(fields[i]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i]) record.PutCopy(key, value) } if nh > nd { diff --git a/go/src/input/record_reader_dkvp.go b/go/src/input/record_reader_dkvp.go index 44fdd9786..8a094d8d1 100644 --- a/go/src/input/record_reader_dkvp.go +++ b/go/src/input/record_reader_dkvp.go @@ -122,11 +122,11 @@ func (reader *RecordReaderDKVP) recordFromDKVPLine( // "a=". Here we use the positional index as the key. This way // DKVP is a generalization of NIDX. key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up - value := types.MlrvalPointerFromInferredType(kv[0]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[0]) record.PutReference(key, value) } else { key := kv[0] - value := types.MlrvalPointerFromInferredType(kv[1]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/go/src/input/record_reader_nidx.go b/go/src/input/record_reader_nidx.go index 0ebdc4d13..6eed7d769 100644 --- a/go/src/input/record_reader_nidx.go +++ b/go/src/input/record_reader_nidx.go @@ -121,7 +121,7 @@ func recordFromNIDXLine( for _, value := range values { i++ key := strconv.Itoa(i) - mval := types.MlrvalPointerFromInferredType(value) + mval := types.MlrvalPointerFromInferredTypeForDataFiles(value) record.PutReference(key, mval) } return record diff --git a/go/src/input/record_reader_xtab.go b/go/src/input/record_reader_xtab.go index f1385d97e..276b7cf64 100644 --- a/go/src/input/record_reader_xtab.go +++ b/go/src/input/record_reader_xtab.go @@ -165,7 +165,7 @@ func (reader *RecordReaderXTAB) recordFromXTABLines( value := types.MLRVAL_VOID record.PutReference(key, value) } else { - value := types.MlrvalPointerFromInferredType(kv[1]) + value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1]) record.PutReference(key, value) } } diff --git a/go/src/types/mlrval_new.go b/go/src/types/mlrval_new.go index 68c70c3d7..bb8299422 100644 --- a/go/src/types/mlrval_new.go +++ b/go/src/types/mlrval_new.go @@ -101,10 +101,42 @@ func MlrvalPointerFromBoolString(input string) *Mlrval { } } +var floatNamesToNotInfer = map[string]bool{ + "Inf": true, + "+Inf": true, + "-Inf": true, + "Infinity": true, + "+Infinity": true, + "-Infinity": true, + "nan": true, + "NaN": true, +} + +// MlrvalPointerFromInferredTypeForDataFiles is for parsing field values from +// data files (except JSON, which is typed -- "true" and true are distinct). +// Mostly the same as MlrvalPointerFromInferredType, except it doesn't +// auto-infer true/false to bool; don't auto-infer NaN/Inf to float; etc. +func MlrvalPointerFromInferredTypeForDataFiles(input string) *Mlrval { + if input == "" { + return MLRVAL_VOID + } + + _, iok := lib.TryIntFromString(input) + if iok { + return MlrvalPointerFromIntString(input) + } + + if floatNamesToNotInfer[input] == false { + _, fok := lib.TryFloat64FromString(input) + if fok { + return MlrvalPointerFromFloat64String(input) + } + } + + return MlrvalPointerFromString(input) +} + func MlrvalPointerFromInferredType(input string) *Mlrval { - // xxx the parsing has happened so stash it ... - // xxx emphasize the invariant that a non-invalid printrep always - // matches the nval ... if input == "" { return MLRVAL_VOID }