Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-29 02:30:12 +00:00
Code Issues Projects Releases Wiki Activity

Fix non-auto-infer of true/false/Infinity/NaN/etc in non-JSON data files

Browse source
This commit is contained in:
John Kerl 2021-09-04 14:19:15 -04:00
parent a916087ca0
commit cfc986b5f4
19 changed files with 97 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

11
docs6/docs/10min.md
View file

@ -87,7 +87,7 @@ purple,square,false,10,91,72.3735,8.2430
{
"color": "yellow",
"shape": "circle",
"flag": true,
"flag": "true",
"k": 9,
"index": 87,
"quantity": 63.5058,
@ -96,7 +96,7 @@ purple,square,false,10,91,72.3735,8.2430
{
"color": "purple",
"shape": "square",
"flag": false,
"flag": "false",
"k": 10,
"index": 91,
"quantity": 72.3735,
@ -212,14 +212,9 @@ red square false 4 48 77.5542 7.4670
red square false 6 64 77.1991 9.5310
</pre>
<pre class="pre-highlight-in-pair">
<pre class="pre-highlight-non-pair">
<b>mlr --icsv --opprint filter '$color == "red" && $flag == true' example.csv</b>
</pre>
<pre class="pre-non-highlight-in-pair">
color shape flag k index quantity rate
red square true 2 15 79.2778 0.0130
red circle true 3 16 13.8103 2.9010
</pre>
You can use `put` to create new fields which are computed from other fields:

14
docs6/docs/data-cleaning-examples.md
View file

@ -58,9 +58,9 @@ A second option is to flag badly formatted data within the output stream:
</pre>
<pre class="pre-non-highlight-in-pair">
name reachable format_ok
barney false false
betty true false
fred true false
barney false true
betty true true
fred true true
wilma 1 false
</pre>
@ -72,9 +72,6 @@ Or perhaps to flag badly formatted data outside the output stream:
<b>' data/het-bool.csv</b>
</pre>
<pre class="pre-non-highlight-in-pair">
Malformed at NR=1
Malformed at NR=2
Malformed at NR=3
Malformed at NR=4
name reachable
barney false
@ -89,5 +86,8 @@ A third way is to abort the process on first instance of bad data:
<b>mlr --csv put '$reachable = asserting_string($reachable)' data/het-bool.csv</b>
</pre>
<pre class="pre-non-highlight-in-pair">
Miller: is_string type-assertion failed at NR=1 FNR=1 FILENAME=data/het-bool.csv
name,reachable
barney,false
Miller: is_string type-assertion failed at NR=4 FNR=4 FILENAME=data/het-bool.csv
betty,true
</pre>

4
docs6/docs/keystroke-savers.md
View file

@ -34,7 +34,7 @@ red square true 2 15 79.2778 0.0130
{
"color": "yellow",
"shape": "triangle",
"flag": true,
"flag": "true",
"k": 1,
"index": 11,
"quantity": 43.6498,
@ -43,7 +43,7 @@ red square true 2 15 79.2778 0.0130
{
"color": "red",
"shape": "square",
"flag": true,
"flag": "true",
"k": 2,
"index": 15,
"quantity": 79.2778,

7
docs6/docs/proofreads.txt
View file

@ -3,14 +3,17 @@ TOP:
c data-types:
! don't auto-infer bools; then, data-types page
! don't auto-infer Infinity, etc -- ?!? then, data-types page
RT-check
https://github.com/johnkerl/miller/issues/241
https://github.com/johnkerl/miller/issues/357
MlrvalPointerFromInferredType -> mk variant
E flatten/unflatten page
c! seps \001 etc !
mlrc --iusv --oxtab cat regtest/input/example.usv
mlr --iusv --oxtab cat regtest/input/example.usv
C! repifs !! https://pkg.go.dev/regexp#Regexp.Split 2-for-1 -- get regexp as well ?
? twi-dm re all-contribs: all-contributors.org
E flatten/unflatten page
? twi-dm re all-contribs: all-contributors.org
C flags LUTs
----------------------------------------------------------------

3
docs6/docs/record-heterogeneity.md
View file

@ -126,6 +126,9 @@ If you `mlr csv cat` this, you'll get an error message:
</pre>
<pre class="pre-non-highlight-in-pair">
mlr : Miller: CSV header/data length mismatch 3 != 2 at filename data/het/ragged.csv row 3.
a,b,c
1,2,3
</pre>
There are two kinds of raggedness here. Since CSVs form records by zipping the

2
docs6/docs/reference-main-data-types.md
View file

@ -92,7 +92,7 @@ ta float
b 3
tb int
c true
tc bool
tc string
d 1.2true
td string
e 7

4
docs6/docs/reference-main-maps.md
View file

@ -82,7 +82,7 @@ The current record, accessible using `$*`, is a map.
{
"color": "yellow",
"shape": "triangle",
"flag": true,
"flag": "true",
"k": 1,
"index": 11,
"quantity": 43.6498,
@ -92,7 +92,7 @@ Color is yellow
{
"color": "red",
"shape": "square",
"flag": true,
"flag": "true",
"k": 2,
"index": 15,
"quantity": 79.2778,

4
docs6/docs/repl.md
View file

@ -48,7 +48,7 @@ HELLO
{
"color": "yellow",
"shape": "triangle",
"flag": true,
"flag": "true",
"k": 1,
"index": 11,
"quantity": 43.6498,
@ -58,7 +58,7 @@ HELLO
{
"color": "red",
"shape": "square",
"flag": true,
"flag": "true",
"k": 2,
"index": 15,
"quantity": 79.2778,

4
docs6/docs/special-symbols-and-formatting.md
View file

@ -83,8 +83,8 @@ characters as delimiters -- here, control-A:
<b>mlr --icsv --odkvp --ofs '\001' cat commas.csv | cat -v</b>
</pre>
<pre class="pre-non-highlight-in-pair">
Name=Xiao, Lin\001Role=administrator
Name=Khavari, Darius\001Role=tester
Name=Xiao, Lin^ARole=administrator
Name=Khavari, Darius^ARole=tester
</pre>
## How can I handle field names with special symbols in them?

12
go/regtest/cases/dsl-min-max-types/0001/expout
View file

@ -1,6 +1,6 @@
{
"n": 1,
"b": true,
"b": "true",
"v": "",
"s": "abc",
"min": {
@ -12,19 +12,19 @@
},
"b": {
"n": 1,
"b": true,
"v": true,
"s": true
"b": "true",
"v": "",
"s": "abc"
},
"v": {
"n": 1,
"b": true,
"b": "",
"v": "",
"s": ""
},
"s": {
"n": 1,
"b": true,
"b": "abc",
"v": "",
"s": "abc"
}

16
go/regtest/cases/dsl-min-max-types/0002/expout
View file

@ -1,30 +1,30 @@
{
"n": 1,
"b": true,
"b": "true",
"v": "",
"s": "abc",
"max": {
"n": {
"n": 1,
"b": true,
"b": "true",
"v": "",
"s": "abc"
},
"b": {
"n": true,
"b": true,
"v": "",
"s": "abc"
"n": "true",
"b": "true",
"v": "true",
"s": "true"
},
"v": {
"n": "",
"b": "",
"b": "true",
"v": "",
"s": "abc"
},
"s": {
"n": "abc",
"b": "abc",
"b": "true",
"v": "abc",
"s": "abc"
}

8
go/regtest/cases/dsl-min-max-types/0003/expout
View file

@ -1,12 +1,12 @@
{
"n": 1,
"b": true,
"b": "true",
"v": "",
"s": "abc",
"le": {
"n": {
"n": true,
"b": false,
"b": true,
"v": false,
"s": true
},
@ -18,13 +18,13 @@
},
"v": {
"n": true,
"b": false,
"b": true,
"v": true,
"s": true
},
"s": {
"n": false,
"b": false,
"b": true,
"v": false,
"s": true
}

8
go/regtest/cases/dsl-min-max-types/0004/expout
View file

@ -1,6 +1,6 @@
{
"n": 1,
"b": true,
"b": "true",
"v": "",
"s": "abc",
"ge": {
@ -11,10 +11,10 @@
"s": false
},
"b": {
"n": false,
"n": true,
"b": true,
"v": false,
"s": false
"v": true,
"s": true
},
"v": {
"n": false,

6
go/src/input/record_reader_csv.go
View file

@ -140,7 +140,7 @@ func (reader *RecordReaderCSV) processHandle(
if nh == nd {
for i := 0; i < nh; i++ {
key := header[i]
value := types.MlrvalPointerFromInferredType(csvRecord[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i])
record.PutReference(key, value)
}
@ -160,13 +160,13 @@ func (reader *RecordReaderCSV) processHandle(
n := lib.IntMin2(nh, nd)
for i = 0; i < n; i++ {
key := header[i]
value := types.MlrvalPointerFromInferredType(csvRecord[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i])
record.PutReference(key, value)
}
if nh < nd {
// if header shorter than data: use 1-up itoa keys
key := strconv.Itoa(i + 1)
value := types.MlrvalPointerFromInferredType(csvRecord[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(csvRecord[i])
record.PutCopy(key, value)
}
if nh > nd {

12
go/src/input/record_reader_csvlite.go
View file

@ -193,7 +193,7 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader(
record := types.NewMlrmap()
if !reader.readerOptions.AllowRaggedCSVInput {
for i, field := range fields {
value := types.MlrvalPointerFromInferredType(field)
value := types.MlrvalPointerFromInferredTypeForDataFiles(field)
record.PutCopy(headerStrings[i], value)
}
} else {
@ -202,14 +202,14 @@ func (reader *RecordReaderCSVLite) processHandleExplicitCSVHeader(
n := lib.IntMin2(nh, nd)
var i int
for i = 0; i < n; i++ {
value := types.MlrvalPointerFromInferredType(fields[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(headerStrings[i], value)
}
if nh < nd {
// if header shorter than data: use 1-up itoa keys
for i = nh; i < nd; i++ {
key := strconv.Itoa(i + 1)
value := types.MlrvalPointerFromInferredType(fields[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(key, value)
}
}
@ -306,7 +306,7 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader(
record := types.NewMlrmap()
if !reader.readerOptions.AllowRaggedCSVInput {
for i, field := range fields {
value := types.MlrvalPointerFromInferredType(field)
value := types.MlrvalPointerFromInferredTypeForDataFiles(field)
record.PutCopy(headerStrings[i], value)
}
} else {
@ -315,13 +315,13 @@ func (reader *RecordReaderCSVLite) processHandleImplicitCSVHeader(
n := lib.IntMin2(nh, nd)
var i int
for i = 0; i < n; i++ {
value := types.MlrvalPointerFromInferredType(fields[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(headerStrings[i], value)
}
if nh < nd {
// if header shorter than data: use 1-up itoa keys
key := strconv.Itoa(i + 1)
value := types.MlrvalPointerFromInferredType(fields[i])
value := types.MlrvalPointerFromInferredTypeForDataFiles(fields[i])
record.PutCopy(key, value)
}
if nh > nd {

4
go/src/input/record_reader_dkvp.go
View file

@ -122,11 +122,11 @@ func (reader *RecordReaderDKVP) recordFromDKVPLine(
// "a=". Here we use the positional index as the key. This way
// DKVP is a generalization of NIDX.
key := strconv.Itoa(i + 1) // Miller userspace indices are 1-up
value := types.MlrvalPointerFromInferredType(kv[0])
value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[0])
record.PutReference(key, value)
} else {
key := kv[0]
value := types.MlrvalPointerFromInferredType(kv[1])
value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1])
record.PutReference(key, value)
}
}

2
go/src/input/record_reader_nidx.go
View file

@ -121,7 +121,7 @@ func recordFromNIDXLine(
for _, value := range values {
i++
key := strconv.Itoa(i)
mval := types.MlrvalPointerFromInferredType(value)
mval := types.MlrvalPointerFromInferredTypeForDataFiles(value)
record.PutReference(key, mval)
}
return record

2
go/src/input/record_reader_xtab.go
View file

@ -165,7 +165,7 @@ func (reader *RecordReaderXTAB) recordFromXTABLines(
value := types.MLRVAL_VOID
record.PutReference(key, value)
} else {
value := types.MlrvalPointerFromInferredType(kv[1])
value := types.MlrvalPointerFromInferredTypeForDataFiles(kv[1])
record.PutReference(key, value)
}
}

38
go/src/types/mlrval_new.go
View file

@ -101,10 +101,42 @@ func MlrvalPointerFromBoolString(input string) *Mlrval {
}
}
var floatNamesToNotInfer = map[string]bool{
"Inf": true,
"+Inf": true,
"-Inf": true,
"Infinity": true,
"+Infinity": true,
"-Infinity": true,
"nan": true,
"NaN": true,
}
// MlrvalPointerFromInferredTypeForDataFiles is for parsing field values from
// data files (except JSON, which is typed -- "true" and true are distinct).
// Mostly the same as MlrvalPointerFromInferredType, except it doesn't
// auto-infer true/false to bool; don't auto-infer NaN/Inf to float; etc.
func MlrvalPointerFromInferredTypeForDataFiles(input string) *Mlrval {
if input == "" {
return MLRVAL_VOID
}
_, iok := lib.TryIntFromString(input)
if iok {
return MlrvalPointerFromIntString(input)
}
if floatNamesToNotInfer[input] == false {
_, fok := lib.TryFloat64FromString(input)
if fok {
return MlrvalPointerFromFloat64String(input)
}
}
return MlrvalPointerFromString(input)
}
func MlrvalPointerFromInferredType(input string) *Mlrval {
// xxx the parsing has happened so stash it ...
// xxx emphasize the invariant that a non-invalid printrep always
// matches the nval ...
if input == "" {
return MLRVAL_VOID
}