Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-01-23 02:14:13 +00:00
Code Issues Projects Releases Wiki Activity

Can't use ${field_name} if it contains UTF-8 characters also encodeable as Latin-1 (#1363)

Browse source 
* unit-test data

* docgen

* windows unit-test accommodations
This commit is contained in:
John Kerl 2023-08-20 12:20:15 -04:00 • committed by GitHub
parent 9d1d2e07ca
commit 2107d520fa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 46 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/src/manpage.md
View file

@ -3470,5 +3470,5 @@ MILLER(1) MILLER(1)
2023-08-19 MILLER(1)
2023-08-20 MILLER(1)
</pre>

2
docs/src/manpage.txt
View file

@ -3449,4 +3449,4 @@ MILLER(1) MILLER(1)
2023-08-19 MILLER(1)
2023-08-20 MILLER(1)

7
internal/pkg/parsing/lexer/lexer.go
View file

@ -12,7 +12,7 @@ import (
const (
NoState = -1
NumStates = 336
NumSymbols = 652
NumSymbols = 653
)
type Lexer struct {
@ -779,6 +779,7 @@ Lexer symbols:
647: 'A'-'Z'
648: 'a'-'z'
649: '0'-'9'
650: \u0100-\U0010ffff
651: .
650: \u00a0-\u00ff
651: \u0100-\U0010ffff
652: .
*/

12
internal/pkg/parsing/lexer/transitiontable.go
View file

@ -1500,6 +1500,8 @@ var TransTab = TransitionTable{
return 159
case r == 126: // ['~','~']
return 159
case 160 <= r && r <= 255: // [\u00a0,\u00ff]
return 159
case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff]
return 159
}
@ -1840,6 +1842,8 @@ var TransTab = TransitionTable{
return 184
case r == 126: // ['~','~']
return 184
case 160 <= r && r <= 255: // [\u00a0,\u00ff]
return 184
case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff]
return 184
}
@ -3144,6 +3148,8 @@ var TransTab = TransitionTable{
return 239
case r == 126: // ['~','~']
return 159
case 160 <= r && r <= 255: // [\u00a0,\u00ff]
return 159
case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff]
return 159
}
@ -3444,6 +3450,8 @@ var TransTab = TransitionTable{
return 254
case r == 126: // ['~','~']
return 184
case 160 <= r && r <= 255: // [\u00a0,\u00ff]
return 184
case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff]
return 184
}
@ -4604,6 +4612,8 @@ var TransTab = TransitionTable{
return 239
case r == 126: // ['~','~']
return 159
case 160 <= r && r <= 255: // [\u00a0,\u00ff]
return 159
case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff]
return 159
}
@ -4792,6 +4802,8 @@ var TransTab = TransitionTable{
return 254
case r == 126: // ['~','~']
return 184
case 160 <= r && r <= 255: // [\u00a0,\u00ff]
return 184
case 256 <= r && r <= 1114111: // [\u0100,\U0010ffff]
return 184
}

1
internal/pkg/parsing/mlr.bnf
View file

@ -271,6 +271,7 @@ _braced_char
| ':' | ';' | '<' | '=' | '>' | '?' | '@' | '['
| ']' | '^' | '_' | '`' | '|' | '~'
| ( '\\' '{' ) | ( '\\' '}' )
| '\u00a0'-'\u00ff'
| '\u0100'-'\U0010FFFF'
;
braced_field_name: '$' '{' _braced_char { _braced_char } '}' ;

2
man/manpage.txt
View file

@ -3449,4 +3449,4 @@ MILLER(1) MILLER(1)
2023-08-19 MILLER(1)
2023-08-20 MILLER(1)

4
man/mlr.1
View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2023-08-19
.\" Date: 2023-08-20
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2023-08-19" "\ \&" "\ \&"
.TH "MILLER" "1" "2023-08-20" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

1
test/cases/dsl-utf8-field-names/0001/cmd Normal file
View file

@ -0,0 +1 @@
mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv

0
test/cases/dsl-utf8-field-names/0001/experr Normal file
View file

3
test/cases/dsl-utf8-field-names/0001/expout Normal file
View file

@ -0,0 +1,3 @@
año ποσότητα
2021 130
2022 145

1
test/cases/dsl-utf8-field-names/0001/mlr Normal file
View file

@ -0,0 +1 @@
$año > 2020

1
test/cases/dsl-utf8-field-names/0002/cmd Normal file
View file

@ -0,0 +1 @@
mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv

0
test/cases/dsl-utf8-field-names/0002/experr Normal file
View file

3
test/cases/dsl-utf8-field-names/0002/expout Normal file
View file

@ -0,0 +1,3 @@
año ποσότητα
2021 130
2022 145

1
test/cases/dsl-utf8-field-names/0002/mlr Normal file
View file

@ -0,0 +1 @@
${año} > 2020

1
test/cases/dsl-utf8-field-names/0003/cmd Normal file
View file

@ -0,0 +1 @@
mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv

0
test/cases/dsl-utf8-field-names/0003/experr Normal file
View file

3
test/cases/dsl-utf8-field-names/0003/expout Normal file
View file

@ -0,0 +1,3 @@
año ποσότητα
2021 130
2022 145

1
test/cases/dsl-utf8-field-names/0003/mlr Normal file
View file

@ -0,0 +1 @@
$ποσότητα > 100

1
test/cases/dsl-utf8-field-names/0004/cmd Normal file
View file

@ -0,0 +1 @@
mlr --c2p filter -f ${CASEDIR}/mlr test/input/datos-plurilingües.csv

0
test/cases/dsl-utf8-field-names/0004/experr Normal file
View file

3
test/cases/dsl-utf8-field-names/0004/expout Normal file
View file

@ -0,0 +1,3 @@
año ποσότητα
2021 130
2022 145

1
test/cases/dsl-utf8-field-names/0004/mlr Normal file
View file

@ -0,0 +1 @@
${ποσότητα} > 100

4
test/input/datos-plurilingües.csv Normal file
View file

@ -0,0 +1,4 @@
año,ποσότητα
2020,100
2021,130
2022,145
1 año ποσότητα
2 2020 100
3 2021 130
4 2022 145