mirror of
https://github.com/johnkerl/miller.git
synced 2026-01-23 02:14:13 +00:00
Address some staticcheck issues (#823)
* address some staticcheck issues * address some staticcheck issues * address some staticcheck issues * address some staticcheck issues
This commit is contained in:
John Kerl
GitHub
parent
a343741b73
commit
a977617797
83 changed files with 316 additions and 641 deletions
11
cmd/experiments/cli-parser/arch/args_notwindows.go
Normal file
11
cmd/experiments/cli-parser/arch/args_notwindows.go
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
// +build !windows
|
||||
|
||||
package arch
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func GetMainArgs() []string {
|
||||
return os.Args
|
||||
}
|
||||
26
cmd/experiments/cli-parser/arch/args_windows.go
Normal file
26
cmd/experiments/cli-parser/arch/args_windows.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// +build windows
|
||||
|
||||
package arch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
shellquote "github.com/kballard/go-shellquote"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func GetMainArgs() []string {
|
||||
commandLine := windows.UTF16PtrToString(windows.GetCommandLine())
|
||||
|
||||
split, err := shellquote.Split(commandLine)
|
||||
if err != nil {
|
||||
fmt.Fprintf(
|
||||
os.Stderr,
|
||||
"%s: could not parse obtain Windows command line: %v\n",
|
||||
os.Args[0], err,
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
return split
|
||||
}
|
||||
27
cmd/experiments/cli-parser/cliparse.go
Normal file
27
cmd/experiments/cli-parser/cliparse.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
// go mod init cliparse.go
|
||||
// go get golang.org/x/sys
|
||||
// go get github.com/kballard/go-shellquote
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"cliparse/arch"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("-- os.Args:")
|
||||
for i, arg := range os.Args {
|
||||
fmt.Printf("args[%d] <<%s>>\n", i, arg)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
fmt.Println("-- canonical args:")
|
||||
args := arch.GetMainArgs()
|
||||
|
||||
for i, arg := range args {
|
||||
fmt.Printf("args[%d] <<%s>>\n", i, arg)
|
||||
}
|
||||
}
|
||||
8
cmd/experiments/cli-parser/go.mod
Normal file
8
cmd/experiments/cli-parser/go.mod
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
module cliparse
|
||||
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 // indirect
|
||||
)
|
||||
4
cmd/experiments/cli-parser/go.sum
Normal file
4
cmd/experiments/cli-parser/go.sum
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 h1:64ChN/hjER/taL4YJuA+gpLfIMT+/NFherRZixbxOhg=
|
||||
golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
12
cmd/experiments/cli-parser/msys2in.sh
Normal file
12
cmd/experiments/cli-parser/msys2in.sh
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/home/kerlj/git/miller/experiments/cli-parser/cliparse.exe '$c=$a."bbb"'
|
||||
# <<C:\tools\msys64\home\kerlj\git\miller\experiments\cli-parser\cliparse.exe "$c=$a.\"bbb\"">>
|
||||
#
|
||||
# args[0] <<C:\tools\msys64\home\kerlj\git\miller\experiments\cli-parser\cliparse.exe>>
|
||||
# args[1] <<$c=$a."bbb">>
|
||||
|
||||
$ /c/users/kerlj/git/miller/experiments/cli-parser/cliparse.exe '$c=$a."bbb"'
|
||||
# <<C:\users\kerlj\git\miller\experiments\cli-parser\cliparse.exe "$c=$a.\"bbb\"">>
|
||||
#
|
||||
# args[0] <<C:\users\kerlj\git\miller\experiments\cli-parser\cliparse.exe>>
|
||||
# args[1] <<$c=$a."bbb">>
|
||||
|
||||
12
cmd/experiments/cli-parser/msys2out.bat
Normal file
12
cmd/experiments/cli-parser/msys2out.bat
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
\users\kerlj\git\miller\experiments\cli-parser\cliparse.exe '$c=$a."bbb"'
|
||||
|
||||
rem <<\users\kerlj\git\miller\experiments\cli-parser\cliparse.exe '$c=$a."bbb"'>>
|
||||
rem
|
||||
rem args[0] <<\users\kerlj\git\miller\experiments\cli-parser\cliparse.exe>>
|
||||
rem args[1] <<'$c=$a.bbb'>>
|
||||
|
||||
\tools\msys64\home\kerlj\git\miller\experiments\cli-parser\cliparse.exe '$c=$a."bbb"'
|
||||
rem <<\tools\msys64\home\kerlj\git\miller\experiments\cli-parser\cliparse.exe '$c=$a."bbb"'>>
|
||||
rem
|
||||
rem args[0] <<\tools\msys64\home\kerlj\git\miller\experiments\cli-parser\cliparse.exe>>
|
||||
rem args[1] <<'$c=$a.bbb'>>
|
||||
1
cmd/experiments/dsl-parser/.vimrc
Normal file
1
cmd/experiments/dsl-parser/.vimrc
Normal file
|
|
@ -0,0 +1 @@
|
|||
map \f :w<C-m>:!clear;echo Building ...; echo; build; echo; main<C-m>
|
||||
1
cmd/experiments/dsl-parser/README.md
Normal file
1
cmd/experiments/dsl-parser/README.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
Running the GOCC parser-generator for the full Miller grammar takes a few minutes. That's a bit painful for experimentation; hence this.
|
||||
62
cmd/experiments/dsl-parser/one/build
Executable file
62
cmd/experiments/dsl-parser/one/build
Executable file
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Setup
|
||||
us=$(basename $0)
|
||||
set -euo pipefail
|
||||
|
||||
verbose="false"
|
||||
|
||||
if [ $# -eq 1 ]; then
|
||||
bnf="$1"
|
||||
elif [ $# -eq 2 ]; then
|
||||
if [ "$1" = "-v" ]; then
|
||||
verbose="true"
|
||||
else
|
||||
echo "Usage: $0 {.bnf file}" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
bnf="$2"
|
||||
else
|
||||
echo "Usage: $0 {.bnf file}" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
dir=src
|
||||
mkdir -p $dir
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Run the parser-generator
|
||||
|
||||
# Build the bin/gocc executable:
|
||||
go get github.com/goccmack/gocc
|
||||
#go get github.com/johnkerl/gocc
|
||||
bingocc="$GOPATH/bin/gocc"
|
||||
|
||||
if [ ! -x "$bingocc" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f $dir/*.txt
|
||||
if [ "$verbose" = "true" ]; then
|
||||
lr1="$dir/LR1_conflicts.txt"
|
||||
# The -o specifies the package name within the autogen
|
||||
$bingocc -v -o $dir $bnf || expand -2 $lr1
|
||||
else
|
||||
$bingocc -o $dir $bnf
|
||||
fi
|
||||
|
||||
echo "Parser-autogen OK"
|
||||
|
||||
# Code-gen directories:
|
||||
# $dir/errors/
|
||||
# $dir/lexer/
|
||||
# $dir/parser/
|
||||
# $dir/token/
|
||||
# $dir/util/
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Compile the main and the parser-autogen
|
||||
|
||||
go build main.go
|
||||
echo "Compile OK"
|
||||
5
cmd/experiments/dsl-parser/one/go.mod
Normal file
5
cmd/experiments/dsl-parser/one/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
module one
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 // indirect
|
||||
26
cmd/experiments/dsl-parser/one/go.sum
Normal file
26
cmd/experiments/dsl-parser/one/go.sum
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 h1:MBgZdx/wBJWTR2Q79mQfP6c8uXdQiu5JowfEz3KhFac=
|
||||
github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808/go.mod h1:dWhnuKE5wcnGTExA2DH6Iicu21YnWwOPMrc/GyhtbCk=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
74
cmd/experiments/dsl-parser/one/main.go
Normal file
74
cmd/experiments/dsl-parser/one/main.go
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"one/src/lexer"
|
||||
"one/src/parser"
|
||||
)
|
||||
|
||||
func parseOne(input string) {
|
||||
theLexer := lexer.NewLexer([]byte(input))
|
||||
theParser := parser.NewParser()
|
||||
_, err := theParser.Parse(theLexer)
|
||||
|
||||
green := "\033[32;01m"
|
||||
red := "\033[31;01m"
|
||||
textdefault := "\033[0m"
|
||||
|
||||
if err != nil {
|
||||
//fmt.Println(err)
|
||||
fmt.Printf("%sFail%s %s\n", red, textdefault, input)
|
||||
} else {
|
||||
fmt.Printf("%sOK%s %s\n", green, textdefault, input)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) == 1 {
|
||||
|
||||
fmt.Println("EXPECT OK")
|
||||
goods := []string{
|
||||
"",
|
||||
";",
|
||||
";;",
|
||||
"x",
|
||||
"x;x",
|
||||
"x;x;x",
|
||||
"x;x;x;x",
|
||||
"x;",
|
||||
"x;;",
|
||||
";x",
|
||||
";;x",
|
||||
"x ; {}",
|
||||
"{} ; x",
|
||||
"{} x",
|
||||
"{ x }",
|
||||
"{ x; x }",
|
||||
"x; { x; x }",
|
||||
"{ x; x } x",
|
||||
"{ x; x } ; x",
|
||||
"{};{}",
|
||||
"{} {}",
|
||||
}
|
||||
for _, input := range goods {
|
||||
parseOne(input)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("EXPECT FAIL")
|
||||
bads := []string{
|
||||
"x x",
|
||||
"x {}",
|
||||
}
|
||||
for _, input := range bads {
|
||||
parseOne(input)
|
||||
}
|
||||
|
||||
} else {
|
||||
for _, arg := range os.Args[1:] {
|
||||
parseOne(arg)
|
||||
}
|
||||
}
|
||||
}
|
||||
9
cmd/experiments/dsl-parser/one/run
Executable file
9
cmd/experiments/dsl-parser/one/run
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: $0 {.bnf file} [expressions]" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
bnf="$1"
|
||||
shift
|
||||
./build $bnf && echo && main "$@"
|
||||
29
cmd/experiments/dsl-parser/one/semi1.bnf
Normal file
29
cmd/experiments/dsl-parser/one/semi1.bnf
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
||||
|
||||
Root
|
||||
: StatementBlock
|
||||
;
|
||||
|
||||
StatementBlock
|
||||
: ";"
|
||||
| BracelessStatement
|
||||
| ";" BracelessStatement
|
||||
| BracefulStatement
|
||||
| BracelessStatement ";" NonEmptyStatementBlock
|
||||
| BracefulStatement NonEmptyStatementBlock
|
||||
;
|
||||
|
||||
NonEmptyStatementBlock
|
||||
: BracelessStatement
|
||||
| BracelessStatement ";" StatementBlock
|
||||
| BracefulStatement
|
||||
| BracefulStatement StatementBlock
|
||||
;
|
||||
|
||||
BracelessStatement
|
||||
: "x"
|
||||
;
|
||||
|
||||
BracefulStatement
|
||||
: "{" StatementBlock "}"
|
||||
;
|
||||
25
cmd/experiments/dsl-parser/one/semi2.bnf
Normal file
25
cmd/experiments/dsl-parser/one/semi2.bnf
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
||||
|
||||
Root
|
||||
: StatementBlock
|
||||
;
|
||||
|
||||
StatementBlock
|
||||
: empty
|
||||
| ";" StatementBlock
|
||||
| BracelessStatement
|
||||
| BracefulStatement
|
||||
| BracelessStatement ";" StatementBlock
|
||||
| BracefulStatement ";" BracefulStatement
|
||||
| BracefulStatement BracefulStatement
|
||||
| BracefulStatement ";" BracelessStatement
|
||||
| BracefulStatement BracelessStatement
|
||||
;
|
||||
|
||||
BracelessStatement
|
||||
: "x"
|
||||
;
|
||||
|
||||
BracefulStatement
|
||||
: "{" StatementBlock "}"
|
||||
;
|
||||
42
cmd/experiments/dsl-parser/two/brackets.bnf
Normal file
42
cmd/experiments/dsl-parser/two/brackets.bnf
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
// ================================================================
|
||||
// LEXER
|
||||
|
||||
_string_literal_element
|
||||
: 'A'-'Z' | 'a'-'z' | '0'-'9'
|
||||
| ' ' | '!' | '#' | '$' | '%' | '&' | '\'' | '\\'
|
||||
| '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/'
|
||||
| ':' | ';' | '<' | '=' | '>' | '?' | '@' | '['
|
||||
| ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~'
|
||||
| ( '\\' '"' )
|
||||
| ( '\\' '[' )
|
||||
| ( '\\' ']' )
|
||||
| ( '\\' 'b' )
|
||||
| ( '\\' 'f' )
|
||||
| ( '\\' 'n' )
|
||||
| ( '\\' 'r' )
|
||||
| ( '\\' 't' )
|
||||
| ( '\\' '\\' )
|
||||
| ( '\\' '0' )
|
||||
| ( '\\' 'x' )
|
||||
| '\u0100'-'\U0010FFFF'
|
||||
;
|
||||
string_literal : '"' {_string_literal_element} '"' ;
|
||||
|
||||
// ================================================================
|
||||
// IMPORT
|
||||
|
||||
<< import "two/src/dsl" >>
|
||||
|
||||
// ================================================================
|
||||
// PARSER
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Root
|
||||
: StringLiteral
|
||||
<< dsl.NewAST($0) >>
|
||||
;
|
||||
|
||||
StringLiteral
|
||||
: string_literal
|
||||
<< dsl.NewASTNodeStripDoubleQuotePair($0, dsl.NodeTypeStringLiteral) >>
|
||||
;
|
||||
81
cmd/experiments/dsl-parser/two/build
Executable file
81
cmd/experiments/dsl-parser/two/build
Executable file
|
|
@ -0,0 +1,81 @@
|
|||
#!/bin/bash
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Setup
|
||||
us=$(basename $0)
|
||||
set -euo pipefail
|
||||
|
||||
verbose="false"
|
||||
|
||||
if [ $# -eq 1 ]; then
|
||||
bnf="$1"
|
||||
elif [ $# -eq 2 ]; then
|
||||
if [ "$1" = "-v" ]; then
|
||||
verbose="true"
|
||||
else
|
||||
echo "Usage: $0 {.bnf file}" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
bnf="$2"
|
||||
else
|
||||
echo "Usage: $0 {.bnf file}" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
dir=src/parsing
|
||||
mkdir -p $dir
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Run the parser-generator
|
||||
|
||||
# Build the bin/gocc executable:
|
||||
go get github.com/goccmack/gocc
|
||||
#go get github.com/johnkerl/gocc
|
||||
bingocc="$GOPATH/bin/gocc"
|
||||
if [ ! -x "$bingocc" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f $dir/*.txt
|
||||
if [ "$verbose" = "true" ]; then
|
||||
lr1="$dir/LR1_conflicts.txt"
|
||||
# The -o specifies the package name within the autogen
|
||||
$bingocc -v -o $dir $bnf || expand -2 $lr1
|
||||
else
|
||||
$bingocc -o $dir $bnf
|
||||
fi
|
||||
|
||||
echo "Parser-autogen OK"
|
||||
|
||||
# Code-gen directories:
|
||||
# $dir/errors/
|
||||
# $dir/lexer/
|
||||
# $dir/parser/
|
||||
# $dir/token/
|
||||
# $dir/util/
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Override GOCC codegen with customized error handling
|
||||
cp ../../../go/src/parsing/errors.go.template src/parsing/errors/errors.go
|
||||
sed -i .bak 's:miller/src:two/src:' src/parsing/errors/errors.go
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Copy AST files from the main Miller tree
|
||||
|
||||
rm -rf ./src/lib/
|
||||
rm -rf ./src/dsl/
|
||||
|
||||
mkdir -p ./src/lib/
|
||||
mkdir -p ./src/dsl/
|
||||
|
||||
cp ../../../go/src/lib/*.go ./src/lib/
|
||||
cp ../../../go/src/dsl/ast*.go ./src/dsl/
|
||||
|
||||
# Different path to autogen between main Miller tree and here
|
||||
sed -i .bak 's:miller/src:two/src:' src/dsl/ast*go
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Compile the main and the parser-autogen
|
||||
|
||||
go build main.go
|
||||
echo "Compile OK"
|
||||
103
cmd/experiments/dsl-parser/two/emit01.bnf
Normal file
103
cmd/experiments/dsl-parser/two/emit01.bnf
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
// ================================================================
|
||||
// LEXER
|
||||
|
||||
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
||||
!comment : '#' {.} '\n' ;
|
||||
|
||||
_letter : 'a'-'z' | 'A'-'Z' ;
|
||||
_decdig : '0'-'9' ;
|
||||
_idchar : _letter | _decdig | '_' ;
|
||||
|
||||
emit : 'e' 'm' 'i' 't' ;
|
||||
|
||||
// ================================================================
|
||||
// IMPORT
|
||||
|
||||
<< import "miller/src/dsl" >>
|
||||
|
||||
// ================================================================
|
||||
// PARSER
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Root
|
||||
: EmitStatement
|
||||
<< dsl.NewAST($0) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Examples:
|
||||
// emit @a
|
||||
// emit (@a, @b)
|
||||
// emit @a, "x", "y"
|
||||
// emit (@a, @b), "x", "y"
|
||||
// First argument (single or in parentheses) must be non-indexed
|
||||
// oosvar/localvar/fieldname, so we can use their names as keys in the emitted
|
||||
// record. These restrictions are enforced in the CST logic, to keep this
|
||||
// parser/AST logic simpler.
|
||||
EmitStatement
|
||||
: emit Rvalue
|
||||
<< dsl.NewASTNodeUnary($0, $1, dsl.NodeTypeEmitStatement) >>
|
||||
|
||||
| emit Rvalue "," EmitArgs
|
||||
<< dsl.NewASTNodeBinary($0, $1, $3, dsl.NodeTypeEmitStatement) >>
|
||||
|
||||
| emit "(" EmitArgs ")"
|
||||
<< dsl.NewASTNodeUnary($0, $2, dsl.NodeTypeEmitStatement) >>
|
||||
|
||||
| emit "(" EmitArgs ")" "," EmitArgs
|
||||
<< dsl.NewASTNodeBinary($0, $2, $5, dsl.NodeTypeEmitStatement) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
EmitArgs
|
||||
: Rvalue
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeFunctionCallsite,
|
||||
) >>
|
||||
|
||||
| Rvalue "," EmitArgs
|
||||
<< dsl.PrependChild(
|
||||
$2,
|
||||
$0,
|
||||
) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
FcnArgs
|
||||
: Rvalue
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeFunctionCallsite,
|
||||
) >>
|
||||
|
||||
// Allow trailing final comma, especially for multiline statements
|
||||
| Rvalue ","
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeFunctionCallsite,
|
||||
) >>
|
||||
|
||||
// Allow trailing final comma, especially for multiline statements
|
||||
| Rvalue "," FcnArgs
|
||||
<< dsl.PrependChild(
|
||||
$2,
|
||||
$0,
|
||||
) >>
|
||||
;
|
||||
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Rvalue
|
||||
: Literal
|
||||
| "(" Literal ")"
|
||||
;
|
||||
|
||||
Literal
|
||||
: "x" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
| "y" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
| "z" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
;
|
||||
118
cmd/experiments/dsl-parser/two/emit02.bnf
Normal file
118
cmd/experiments/dsl-parser/two/emit02.bnf
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
// ================================================================
|
||||
// LEXER
|
||||
|
||||
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
||||
!comment : '#' {.} '\n' ;
|
||||
|
||||
_letter : 'a'-'z' | 'A'-'Z' ;
|
||||
_decdig : '0'-'9' ;
|
||||
_idchar : _letter | _decdig | '_' ;
|
||||
|
||||
emit : 'e' 'm' 'i' 't' ;
|
||||
|
||||
// ================================================================
|
||||
// IMPORT
|
||||
|
||||
<< import "two/src/dsl" >>
|
||||
|
||||
// ================================================================
|
||||
// PARSER
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Root
|
||||
: EmitStatement
|
||||
<< dsl.NewAST($0) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Examples:
|
||||
// emit @a
|
||||
// emit (@a, @b)
|
||||
// emit @a, "x", "y"
|
||||
// emit (@a, @b), "x", "y"
|
||||
// First argument (single or in parentheses) must be non-indexed
|
||||
// oosvar/localvar/fieldname, so we can use their names as keys in the emitted
|
||||
// record. These restrictions are enforced in the CST logic, to keep this
|
||||
// parser/AST logic simpler.
|
||||
|
||||
EmitStatement
|
||||
: emit Emittable
|
||||
<<
|
||||
dsl.NewASTNodeUnary($0, $1, dsl.NodeTypeEmitStatement)
|
||||
/*
|
||||
dsl.NewASTNodeUnary(
|
||||
$0,
|
||||
dsl.NewASTNodeNestable(
|
||||
$1,
|
||||
),
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
*/
|
||||
>>
|
||||
|
||||
| emit "(" EmittableList ")"
|
||||
<< dsl.NewASTNodeUnary($0, $2, dsl.NodeTypeEmitStatement) >>
|
||||
|
||||
| emit Emittable "," EmitKeys
|
||||
<< dsl.NewASTNodeBinary($0, $1, $3, dsl.NodeTypeEmitStatement) >>
|
||||
|
||||
| emit "(" EmittableList ")" "," EmitKeys
|
||||
<< dsl.NewASTNodeBinary($0, $2, $5, dsl.NodeTypeEmitStatement) >>
|
||||
;
|
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
EmittableList
|
||||
: Emittable
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeEmittableList,
|
||||
) >>
|
||||
|
||||
// Allow trailing final comma, especially for multiline statements
|
||||
| Emittable "," EmittableList
|
||||
<< dsl.PrependChild(
|
||||
$2,
|
||||
$0,
|
||||
) >>
|
||||
;
|
||||
|
||||
Emittable
|
||||
: Literal
|
||||
;
|
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
EmitKeys
|
||||
|
||||
: Rvalue
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeEmitKeys,
|
||||
) >>
|
||||
|
||||
| Rvalue "," EmitKeys
|
||||
<< dsl.PrependChild(
|
||||
$2,
|
||||
$0,
|
||||
) >>
|
||||
;
|
||||
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Rvalue
|
||||
: Literal
|
||||
<< dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStringLiteral) >>
|
||||
| "(" Literal ")"
|
||||
<< dsl.NewASTNodeUnary(nil, $1, dsl.NodeTypeStringLiteral) >>
|
||||
| "[" Literal "]"
|
||||
<< dsl.NewASTNodeUnary(nil, $1, dsl.NodeTypeStringLiteral) >>
|
||||
| "[" Literal "," Literal "]"
|
||||
<< dsl.NewASTNodeBinary(nil, $1, $2, dsl.NodeTypeStringLiteral) >>
|
||||
;
|
||||
|
||||
Literal
|
||||
: "x" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
| "y" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
| "z" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
;
|
||||
5
cmd/experiments/dsl-parser/two/go.mod
Normal file
5
cmd/experiments/dsl-parser/two/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
module two
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 // indirect
|
||||
26
cmd/experiments/dsl-parser/two/go.sum
Normal file
26
cmd/experiments/dsl-parser/two/go.sum
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808 h1:MBgZdx/wBJWTR2Q79mQfP6c8uXdQiu5JowfEz3KhFac=
|
||||
github.com/goccmack/gocc v0.0.0-20210322175033-34358ebe5808/go.mod h1:dWhnuKE5wcnGTExA2DH6Iicu21YnWwOPMrc/GyhtbCk=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
123
cmd/experiments/dsl-parser/two/main.go
Normal file
123
cmd/experiments/dsl-parser/two/main.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"two/src/dsl"
|
||||
|
||||
"two/src/parsing/lexer"
|
||||
"two/src/parsing/parser"
|
||||
)
|
||||
|
||||
const GREEN = "\033[32;01m"
|
||||
const RED = "\033[31;01m"
|
||||
const TEXTDEFAULT = "\033[0m"
|
||||
|
||||
func parseOne(input string, printError bool) bool {
|
||||
theLexer := lexer.NewLexer([]byte(input))
|
||||
theParser := parser.NewParser()
|
||||
iast, err := theParser.Parse(theLexer)
|
||||
if err == nil {
|
||||
fmt.Printf("%sOK%s %s\n", GREEN, TEXTDEFAULT, input)
|
||||
iast.(*dsl.AST).Print()
|
||||
fmt.Println()
|
||||
return true
|
||||
} else {
|
||||
if printError {
|
||||
fmt.Println(err)
|
||||
}
|
||||
fmt.Printf("%sFail%s %s\n", RED, TEXTDEFAULT, input)
|
||||
fmt.Println()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
printError := false
|
||||
args := os.Args[1:] // os.Args[0] is program name in Go
|
||||
if len(args) >= 1 && args[0] == "-v" {
|
||||
printError = true
|
||||
args = args[1:]
|
||||
}
|
||||
|
||||
if len(args) == 0 {
|
||||
ok := true
|
||||
|
||||
fmt.Println("----------------------------------------------------------------")
|
||||
fmt.Println("EXPECT OK")
|
||||
goods := []string{
|
||||
"",
|
||||
";",
|
||||
";;",
|
||||
";;;",
|
||||
"x",
|
||||
"x;x",
|
||||
"x;x;x",
|
||||
"x;x;x;x",
|
||||
"x;",
|
||||
"x;;",
|
||||
";x",
|
||||
";;x",
|
||||
"x ; {}",
|
||||
"{} ; x",
|
||||
"{} x",
|
||||
"{ x }",
|
||||
"{ x; x }",
|
||||
"x; { x; x }",
|
||||
"{ x; x } x",
|
||||
"{ x; x } ; x",
|
||||
"{};{}",
|
||||
"{} {}",
|
||||
"{} {};",
|
||||
"{};{};",
|
||||
"{} {} {}",
|
||||
"{};{}; {}",
|
||||
"{} ; ; {}",
|
||||
"x; x;",
|
||||
"{x; x;}",
|
||||
"{x; x;} x",
|
||||
"{x; x;} x;",
|
||||
"{x; x;} {};",
|
||||
"{} x; {}",
|
||||
"{} x; {};",
|
||||
"{} x; x; {}",
|
||||
"{} x; x; {};",
|
||||
"{} x; x; x; {}",
|
||||
"{} x; x; x; {};",
|
||||
"{} {} ;;; {;;} x; x; x; x; x; {}",
|
||||
}
|
||||
for _, input := range goods {
|
||||
if parseOne(input, printError) == false {
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("----------------------------------------------------------------")
|
||||
fmt.Println("EXPECT FAIL")
|
||||
bads := []string{
|
||||
"x x",
|
||||
"x {}",
|
||||
}
|
||||
for _, input := range bads {
|
||||
if parseOne(input, printError) == true {
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("----------------------------------------------------------------")
|
||||
if ok {
|
||||
fmt.Printf("%sALL AS EXPECTED%s\n", GREEN, TEXTDEFAULT)
|
||||
} else {
|
||||
fmt.Printf("%sNOT ALL AS EXPECTED%s\n", RED, TEXTDEFAULT)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
} else {
|
||||
for _, arg := range args {
|
||||
parseOne(arg, printError)
|
||||
}
|
||||
}
|
||||
}
|
||||
15
cmd/experiments/dsl-parser/two/mktags
Executable file
15
cmd/experiments/dsl-parser/two/mktags
Executable file
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ~/.ctags needs to look like
|
||||
# --langdef=Go
|
||||
# --langmap=Go:.go
|
||||
# --regex-Go=/func([ \t]+\([^)]+\))?[ \t]+([a-zA-Z0-9_]+)/\2/d,func/
|
||||
# --regex-Go=/var[ \t]+([a-zA-Z_][a-zA-Z0-9_]+)/\1/d,var/
|
||||
# --regex-Go=/type[ \t]+([a-zA-Z_][a-zA-Z0-9_]+)/\1/d,type/
|
||||
#
|
||||
# See also https://stackoverflow.com/questions/8204367/ctag-database-for-go
|
||||
|
||||
ctags -f gosource.tags -R `pwd`
|
||||
mv gosource.tags tags
|
||||
9
cmd/experiments/dsl-parser/two/run
Executable file
9
cmd/experiments/dsl-parser/two/run
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: $0 {.bnf file} [expressions]" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
bnf="$1"
|
||||
shift
|
||||
./build $bnf && echo && main "$@"
|
||||
93
cmd/experiments/dsl-parser/two/semi3.bnf
Normal file
93
cmd/experiments/dsl-parser/two/semi3.bnf
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
// ================================================================
|
||||
// LEXER
|
||||
|
||||
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
||||
|
||||
// ================================================================
|
||||
// IMPORT
|
||||
|
||||
<< import "two/src/dsl" >>
|
||||
|
||||
// ================================================================
|
||||
// PARSER
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Root
|
||||
: StatementBlock
|
||||
<< dsl.NewAST($0) >>
|
||||
;
|
||||
|
||||
// ================================================================
|
||||
StatementBlock
|
||||
|
||||
// Empty statement. This allows for 'mlr put ""', as well as repeated semicolons.
|
||||
: empty
|
||||
<< dsl.NewASTNodeZary(nil, dsl.NodeTypeStatementBlock) >>
|
||||
|
||||
| NonEmptyStatementBlock
|
||||
<< dsl.Wrap($0) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
NonEmptyStatementBlock
|
||||
// ---------------------- Terminal rules
|
||||
|
||||
// Things not ending in a curly brace, like assignments -- and also do-while.
|
||||
: BracelessStatement
|
||||
<< dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStatementBlock) >>
|
||||
|
||||
// Things ending in a curly brace, like for/do/while, begin/end, and pattern-action blocks
|
||||
| BracefulStatement
|
||||
<< dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStatementBlock) >>
|
||||
|
||||
// ---------------------- Recursive rules
|
||||
|
||||
// So statements can start with a semicolon
|
||||
| ";" StatementBlock
|
||||
<< dsl.Wrap($1) >>
|
||||
|
||||
// Normal case for sequential statements like '$x=1; $y=2'
|
||||
| BracelessStatement ";" StatementBlock
|
||||
<<dsl.PrependChild($2, $0) >>
|
||||
|
||||
// For 'begin {...} ; $x=1'
|
||||
| BracefulStatement ";" StatementBlock
|
||||
<<dsl.PrependChild($2, $0) >>
|
||||
|
||||
// These are for things like 'begin {...} begin {...} ...' -- where people
|
||||
// shouldn't have to put semicolons after the closing curly braces.
|
||||
//
|
||||
// We get LR-1 conflicts with the following, so we need a pair of more
|
||||
// explicit lookahead-by-more production rules instead. (By using ternaries
|
||||
// we are effectively getting lookahead-by-two.)
|
||||
//
|
||||
// | BracefulStatement StatementBlock
|
||||
// <<dsl.PrependChild($1, $0) >>
|
||||
|
||||
// E.g. 'begin {...} begin {...} $x=1'
|
||||
| BracefulStatement BracefulStatement StatementBlock
|
||||
<<dsl.PrependTwoChildren($2, $0, $1) >>
|
||||
|
||||
// E.g. 'begin {...} $x=1'
|
||||
| BracefulStatement BracelessStatement
|
||||
<< dsl.NewASTNodeBinary(nil, $0, $1, dsl.NodeTypeStatementBlock) >>
|
||||
|
||||
// E.g. 'begin {...} $x=1 ;'
|
||||
| BracefulStatement BracelessStatement ";"
|
||||
<< dsl.NewASTNodeBinary(nil, $0, $1, dsl.NodeTypeStatementBlock) >>
|
||||
|
||||
| BracefulStatement BracelessStatement ";" NonEmptyStatementBlock
|
||||
<<dsl.PrependTwoChildren($3, $0, $1) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
BracelessStatement
|
||||
: "x"
|
||||
<< dsl.NewASTNodeZary($0, dsl.NodeTypeBareBoolean) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
BracefulStatement
|
||||
: "{" StatementBlock "}"
|
||||
<< dsl.Wrap($1) >>
|
||||
;
|
||||
219
cmd/experiments/dsl-parser/two/temp.bnf
Normal file
219
cmd/experiments/dsl-parser/two/temp.bnf
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
// ================================================================
|
||||
// LEXER
|
||||
|
||||
!whitespace : ' ' | '\t' | '\n' | '\r' ;
|
||||
!comment : '#' {.} '\n' ;
|
||||
|
||||
_letter : 'a'-'z' | 'A'-'Z' ;
|
||||
_decdig : '0'-'9' ;
|
||||
_idchar : _letter | _decdig | '_' ;
|
||||
|
||||
emitf : 'e' 'm' 'i' 't' 'f';
|
||||
emit : 'e' 'm' 'i' 't' ;
|
||||
|
||||
stdout : 's' 't' 'd' 'o' 'u' 't' ;
|
||||
stderr : 's' 't' 'd' 'e' 'r' 'r' ;
|
||||
|
||||
// ================================================================
|
||||
// IMPORT
|
||||
|
||||
<< import "two/src/dsl" >>
|
||||
|
||||
// ================================================================
|
||||
// PARSER
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Root
|
||||
: EmitFStatement
|
||||
<< dsl.NewAST($0) >>
|
||||
| EmitStatement
|
||||
<< dsl.NewAST($0) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Redirector
|
||||
: ">" RedirectTarget
|
||||
<< dsl.NewASTNodeUnary($0, $1, dsl.NodeTypeRedirectWrite) >>
|
||||
| ">>" RedirectTarget
|
||||
<< dsl.NewASTNodeUnary($0, $1, dsl.NodeTypeRedirectAppend) >>
|
||||
| "|" RedirectTarget
|
||||
<< dsl.NewASTNodeUnary($0, $1, dsl.NodeTypeRedirectPipe) >>
|
||||
;
|
||||
|
||||
RedirectTarget
|
||||
: stdout
|
||||
<< dsl.NewASTNodeZary($0, dsl.NodeTypeRedirectTargetStdout) >>
|
||||
| stderr
|
||||
<< dsl.NewASTNodeZary($0, dsl.NodeTypeRedirectTargetStderr) >>
|
||||
| Rvalue
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
EmitStatement
|
||||
: emit Emittable
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$1,
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit Redirector "," Emittable
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$3,
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
$1,
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit "(" EmittableList ")"
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$2,
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit Redirector "," "(" EmittableList ")"
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$4,
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
$1,
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit Emittable "," EmitKeys
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$1,
|
||||
$3,
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit Redirector "," Emittable "," EmitKeys
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$3,
|
||||
$5,
|
||||
$1,
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit "(" EmittableList ")" "," EmitKeys
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$2,
|
||||
$5,
|
||||
dsl.NewASTNodeNestable(nil, dsl.NodeTypeNoOp),
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
|
||||
| emit Redirector "," "(" EmittableList ")" "," EmitKeys
|
||||
<<
|
||||
dsl.NewASTNodeTernary(
|
||||
$0,
|
||||
$4,
|
||||
$7,
|
||||
$1,
|
||||
dsl.NodeTypeEmitStatement,
|
||||
)
|
||||
>>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Examples:
|
||||
// emitf @a
|
||||
// emitf @a, b, $c
|
||||
// Each argument must be a non-indexed oosvar/localvar/fieldname, so we can use
|
||||
// their names as keys in the emitted record.
|
||||
EmitFStatement
|
||||
|
||||
: emitf EmittableList
|
||||
<< dsl.AdoptChildren(
|
||||
dsl.NewASTNodeNestable(
|
||||
$0,
|
||||
dsl.NodeTypeEmitFStatement,
|
||||
),
|
||||
$1,
|
||||
) >>
|
||||
|
||||
| emitf Redirector "," EmittableList
|
||||
// TODO
|
||||
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
EmittableList
|
||||
|
||||
: Emittable
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeEmittableList,
|
||||
) >>
|
||||
|
||||
// Allow trailing final comma, especially for multiline statements
|
||||
| Emittable "," EmittableList
|
||||
<< dsl.PrependChild(
|
||||
$2,
|
||||
$0,
|
||||
) >>
|
||||
;
|
||||
|
||||
Emittable
|
||||
: Literal
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
EmitKeys
|
||||
|
||||
: Rvalue
|
||||
<< dsl.NewASTNodeUnary(
|
||||
nil,
|
||||
$0,
|
||||
dsl.NodeTypeEmitKeys,
|
||||
) >>
|
||||
|
||||
| Rvalue "," EmitKeys
|
||||
<< dsl.PrependChild(
|
||||
$2,
|
||||
$0,
|
||||
) >>
|
||||
;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
Rvalue
|
||||
: Literal
|
||||
<< dsl.NewASTNodeUnary(nil, $0, dsl.NodeTypeStringLiteral) >>
|
||||
| "(" Literal ")"
|
||||
<< dsl.NewASTNodeUnary(nil, $1, dsl.NodeTypeStringLiteral) >>
|
||||
| "[" Literal "]"
|
||||
<< dsl.NewASTNodeUnary(nil, $1, dsl.NodeTypeStringLiteral) >>
|
||||
| "[" Literal "," Literal "]"
|
||||
<< dsl.NewASTNodeBinary(nil, $1, $2, dsl.NodeTypeStringLiteral) >>
|
||||
;
|
||||
|
||||
Literal
|
||||
: "x" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
| "y" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
| "z" << dsl.NewASTNodeZary($0, dsl.NodeTypeStringLiteral) >>
|
||||
;
|
||||
99
cmd/experiments/gc-raw-notes.txt
Normal file
99
cmd/experiments/gc-raw-notes.txt
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
* stack optimization:
|
||||
k StackVariable opaque struct
|
||||
- name string
|
||||
- frameSetIndex int
|
||||
- frameIndex int
|
||||
k convert StackFrame from map of name to *TypeGatedMlrvalVariable to array of *TypeGatedMlrvalVariable
|
||||
k convert StackFrameSet from list of StackFrame to array of StackFrame
|
||||
- comments about S/SF indef large b/c recursion; SF/SFS known statically
|
||||
o Define/Set/Get need to modify the indices:
|
||||
? add comments for retain in CST node?
|
||||
- within Stack: map from name to idxpair
|
||||
- within Set/Get (re-use): fastcall to indexer if non-trivial
|
||||
o mods to use the indices when set
|
||||
|
||||
o profile mand.mlr
|
||||
- c 3.2s
|
||||
- go pre-alloccate 45s
|
||||
|
||||
Why 5MB goal with GOGC=100????
|
||||
|
||||
GOGC=100000 GODEBUG=gctrace=1 mlr -n put -q -f u/mand.mlr 1> /dev/null
|
||||
|
||||
* u/mand.mlr silent option
|
||||
|
||||
https://blog.twitch.tv/en/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap-26c2462549a2/
|
||||
ballast := make([]byte, 10<<30)
|
||||
|
||||
https://dave.cheney.net/2014/07/11/visualising-the-go-garbage-collector
|
||||
go get -u -v github.com/davecheney/gcvis
|
||||
gcvis mlr -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null
|
||||
|
||||
* duffcopy/madvise (w/ GOGC=off), & heavy GC (w/ GOGC != off), are both indicators of the same thing: lots of allocation and lots of copying
|
||||
|
||||
https://github.com/golang/go/issues/23044
|
||||
"It has often been noted that programs which make a lot of allocations while
|
||||
maintaining a small live heap end up doing excessive garbage collections."
|
||||
yyyuuuuuuuuup
|
||||
--> maybe plan out the CST w/ mlrvals for all slots pre-allocated, and point to them -- ?
|
||||
--> and/or: Evaluate return *types.Mlrval ?
|
||||
o or instead of Evaluate(state*State) -> Mlrval, do Evaluate(state *State, poutput *Mlrval)
|
||||
o and not type BinaryFunc func(*Mlrval, *Mlrval) Mlrval but type BinaryFunc func(out *Mlrval, in1 *Mlrval, in2 *Mlrval)
|
||||
o idea: don't generate ANY garbage w/ this:
|
||||
zt = zr*zr - zi*zi + cr;
|
||||
zi = 2*zr*zi + ci;
|
||||
zr = zt;
|
||||
o leaf evaluables:
|
||||
- literals & local vars etc just point to storage
|
||||
|
||||
i https://hub.packtpub.com/implementing-memory-management-with-golang-garbage-collector/
|
||||
|
||||
i mlr --cpuprofile cpu.pprof -n put -q -s iheight=100 -s iwidth=100 -f u/mand.mlr > /dev/null
|
||||
|
||||
? SetGCPercent
|
||||
? SetMaxHeap
|
||||
|
||||
mlr --cpuprofile cpu.pprof -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null
|
||||
go tool pprof mlr cpu.pprof
|
||||
top10
|
||||
|
||||
go tool pprof --pdf mlr cpu.pprof > mlr-call-graph.pdf
|
||||
mv mlr-call-graph.pdf ~/Desktop
|
||||
|
||||
runtime.duffcopy
|
||||
https://stackoverflow.com/questions/45786687/runtime-duffcopy-is-called-a-lot
|
||||
runtime.madvise
|
||||
|
||||
GOGC=off
|
||||
GODEBUG=gctrace=1
|
||||
|
||||
export PATH=${PATH}:~/git/brendangregg/FlameGraph/
|
||||
go-torch cpu.pprof
|
||||
mv torch.svg ~/Desktop/
|
||||
|
||||
i mlr --cpuprofile cpu.pprof -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null
|
||||
|
||||
i https://hub.packtpub.com/implementing-memory-management-with-golang-garbage-collector/
|
||||
|
||||
i https://golang.org/pkg/runtime/
|
||||
|
||||
! flame-graph readme; & profile-readme out of mlr.go & into separate .md file
|
||||
i mlr --cpuprofile cpu.pprof -n put -q -s iheight=100 -s iwidth=100 -f u/mand.mlr > /dev/null
|
||||
i GODEBUG=gctrace=1 mlr -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null| head -n 100
|
||||
gc 1 @0.129s 1%: 0.012+3.9+0.002 ms clock, 0.048+3.7/3.7/7.5+0.010 ms cpu, 10240->10240->0 MB, 10241 MB goal, 4 P
|
||||
gc 2 @0.140s 2%: 0.009+2.1+0.002 ms clock, 0.038+2.0/2.0/4.0+0.011 ms cpu, 4->4->0 MB, 5 MB goal, 4 P
|
||||
gc 3 @0.149s 2%: 0.032+2.1+0.021 ms clock, 0.12+2.0/2.0/4.0+0.087 ms cpu, 4->4->0 MB, 5 MB goal, 4 P
|
||||
gc 4 @0.157s 3%: 0.035+2.1+0.014 ms clock, 0.14+2.0/1.9/4.1+0.059 ms cpu, 4->4->0 MB, 5 MB goal, 4 P
|
||||
gc 5 @0.166s 3%: 0.034+2.2+0.024 ms clock, 0.13+2.0/1.9/4.0+0.098 ms cpu, 4->4->0 MB, 5 MB goal, 4 P
|
||||
|
||||
mem.Alloc: 176104
|
||||
mem.TotalAlloc: 176104
|
||||
mem.HeapAlloc: 176104
|
||||
mem.NumGC: 0
|
||||
|
||||
mem.Alloc: 1179440
|
||||
mem.TotalAlloc: 16,529,643,664
|
||||
mem.HeapAlloc: 1179440
|
||||
mem.NumGC: 4254
|
||||
|
||||
19
cmd/experiments/line-parser/README.txt
Normal file
19
cmd/experiments/line-parser/README.txt
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
$ ls -l ~/tmp/huge
|
||||
-rw-r--r-- 1 kerl staff 614295000 Aug 25 2020 /Users/kerl/tmp/huge
|
||||
|
||||
$ wc -l ~/tmp/huge
|
||||
10000000 /Users/kerl/tmp/huge
|
||||
|
||||
$ justtime read-string ~/tmp/huge > /dev/null
|
||||
TIME IN SECONDS 8.707 -- read-string /Users/kerl/tmp/huge
|
||||
$ justtime read-string ~/tmp/huge > /dev/null
|
||||
TIME IN SECONDS 8.540 -- read-string /Users/kerl/tmp/huge
|
||||
$ justtime read-string ~/tmp/huge > /dev/null
|
||||
TIME IN SECONDS 8.549 -- read-string /Users/kerl/tmp/huge
|
||||
|
||||
$ justtime scanner ~/tmp/huge > /dev/null
|
||||
TIME IN SECONDS 8.774 -- scanner /Users/kerl/tmp/huge
|
||||
$ justtime scanner ~/tmp/huge > /dev/null
|
||||
TIME IN SECONDS 8.873 -- scanner /Users/kerl/tmp/huge
|
||||
$ justtime scanner ~/tmp/huge > /dev/null
|
||||
TIME IN SECONDS 8.777 -- scanner /Users/kerl/tmp/huge
|
||||
32
cmd/experiments/line-parser/read-string.go
Normal file
32
cmd/experiments/line-parser/read-string.go
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
filename := os.Args[1]
|
||||
handle, err := os.Open(filename)
|
||||
if err != nil {
|
||||
fmt.Println("OERR", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer handle.Close()
|
||||
|
||||
lineReader := bufio.NewReader(handle)
|
||||
|
||||
eof := false
|
||||
for !eof {
|
||||
|
||||
line, err := lineReader.ReadString('\n') // TODO: auto-detect
|
||||
if err != nil {
|
||||
if line != "" {
|
||||
fmt.Println(line)
|
||||
}
|
||||
break
|
||||
}
|
||||
fmt.Print(line)
|
||||
}
|
||||
}
|
||||
26
cmd/experiments/line-parser/scanner.go
Normal file
26
cmd/experiments/line-parser/scanner.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
filename := os.Args[1]
|
||||
|
||||
handle, err := os.Open(filename)
|
||||
if err != nil {
|
||||
fmt.Println("OERR", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer handle.Close()
|
||||
|
||||
scanner := bufio.NewScanner(handle)
|
||||
for scanner.Scan() {
|
||||
fmt.Println(scanner.Text()) // Println will add back the final '\n'
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "reading standard input:", err)
|
||||
}
|
||||
}
|
||||
100
cmd/experiments/line-parser/splitter.go
Normal file
100
cmd/experiments/line-parser/splitter.go
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// From https://pkg.go.dev/bufio#SplitFunc:
|
||||
//
|
||||
// SplitFunc is the signature of the split function used to tokenize the input.
|
||||
// The arguments are an initial substring of the remaining unprocessed data and
|
||||
// a flag, atEOF, that reports whether the Reader has no more data to give. The
|
||||
// return values are the number of bytes to advance the input and the next
|
||||
// token to return to the user, if any, plus an error, if any.
|
||||
//
|
||||
// Scanning stops if the function returns an error, in which case some of the
|
||||
// input may be discarded. If that error is ErrFinalToken, scanning stops with
|
||||
// no error.
|
||||
//
|
||||
// Otherwise, the Scanner advances the input. If the token is not nil, the
|
||||
// Scanner returns it to the user. If the token is nil, the Scanner reads more
|
||||
// data and continues scanning; if there is no more data--if atEOF was
|
||||
// true--the Scanner returns. If the data does not yet hold a complete token,
|
||||
// for instance if it has no newline while scanning lines, a SplitFunc can
|
||||
// return (0, nil, nil) to signal the Scanner to read more data into the slice
|
||||
// and try again with a longer slice starting at the same point in the input.
|
||||
//
|
||||
// The function is never called with an empty data slice unless atEOF is true.
|
||||
// If atEOF is true, however, data may be non-empty and, as always, holds
|
||||
// unprocessed text.
|
||||
|
||||
func main() {
|
||||
filename := os.Args[1]
|
||||
handle, err := os.Open(filename)
|
||||
if err != nil {
|
||||
fmt.Println("OERR", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
irs := "xy\n"
|
||||
irsbytes := []byte(irs)
|
||||
irslen := len(irsbytes)
|
||||
|
||||
scanner := bufio.NewScanner(handle)
|
||||
|
||||
// Custom splitter
|
||||
splitter := func(
|
||||
data []byte,
|
||||
atEOF bool,
|
||||
) (
|
||||
advance int,
|
||||
token []byte,
|
||||
err error,
|
||||
) {
|
||||
datalen := len(data)
|
||||
// Example:
|
||||
// datalen = 10
|
||||
// irslen = 3
|
||||
// 0123456789 <--- data
|
||||
// xy* <--- IRS
|
||||
end := datalen - irslen
|
||||
for i := 0; i <= end; i++ {
|
||||
if data[i] == irsbytes[0] {
|
||||
match := true
|
||||
for j := 1; j < irslen; j++ {
|
||||
if data[i+j] != irsbytes[j] {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return i + irslen, data[:i], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if !atEOF {
|
||||
return 0, nil, nil
|
||||
}
|
||||
// There is one final token to be delivered, which may be the empty string.
|
||||
// Returning bufio.ErrFinalToken here tells Scan there are no more tokens after this
|
||||
// but does not trigger an error to be returned from Scan itself.
|
||||
return 0, data, bufio.ErrFinalToken
|
||||
}
|
||||
scanner.Split(splitter)
|
||||
|
||||
// Consume input
|
||||
atFirst := true
|
||||
for scanner.Scan() {
|
||||
if atFirst {
|
||||
atFirst = false
|
||||
} else {
|
||||
fmt.Println()
|
||||
}
|
||||
fmt.Printf("%s\n", scanner.Text())
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "reading input:", err)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue