diff --git a/go/gc-raw-notes.txt b/go/gc-raw-notes.txt index b3ca5717b..1f4e5a166 100644 --- a/go/gc-raw-notes.txt +++ b/go/gc-raw-notes.txt @@ -17,6 +17,10 @@ - c 3.2s - go pre-alloccate 45s +Why 5MB goal with GOGC=100???? + +GOGC=100000 GODEBUG=gctrace=1 mlr -n put -q -f u/mand.mlr 1> /dev/null + * u/mand.mlr silent option https://blog.twitch.tv/en/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap-26c2462549a2/ @@ -46,3 +50,50 @@ https://github.com/golang/go/issues/23044 i https://hub.packtpub.com/implementing-memory-management-with-golang-garbage-collector/ i mlr --cpuprofile cpu.pprof -n put -q -s iheight=100 -s iwidth=100 -f u/mand.mlr > /dev/null + +? SetGCPercent +? SetMaxHeap + +mlr --cpuprofile cpu.pprof -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null +go tool pprof mlr cpu.pprof +top10 + +go tool pprof --pdf mlr cpu.pprof > mlr-call-graph.pdf +mv mlr-call-graph.pdf ~/Desktop + +runtime.duffcopy + https://stackoverflow.com/questions/45786687/runtime-duffcopy-is-called-a-lot +runtime.madvise + +GOGC=off +GODEBUG=gctrace=1 + +export PATH=${PATH}:~/git/brendangregg/FlameGraph/ +go-torch cpu.pprof +mv torch.svg ~/Desktop/ + +i mlr --cpuprofile cpu.pprof -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null + +i https://hub.packtpub.com/implementing-memory-management-with-golang-garbage-collector/ + +i https://golang.org/pkg/runtime/ + +! flame-graph readme; & profile-readme out of mlr.go & into separate .md file +i mlr --cpuprofile cpu.pprof -n put -q -s iheight=100 -s iwidth=100 -f u/mand.mlr > /dev/null +i GODEBUG=gctrace=1 mlr -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null| head -n 100 + gc 1 @0.129s 1%: 0.012+3.9+0.002 ms clock, 0.048+3.7/3.7/7.5+0.010 ms cpu, 10240->10240->0 MB, 10241 MB goal, 4 P + gc 2 @0.140s 2%: 0.009+2.1+0.002 ms clock, 0.038+2.0/2.0/4.0+0.011 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + gc 3 @0.149s 2%: 0.032+2.1+0.021 ms clock, 0.12+2.0/2.0/4.0+0.087 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + gc 4 @0.157s 3%: 0.035+2.1+0.014 ms clock, 0.14+2.0/1.9/4.1+0.059 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + gc 5 @0.166s 3%: 0.034+2.2+0.024 ms clock, 0.13+2.0/1.9/4.0+0.098 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + +mem.Alloc: 176104 +mem.TotalAlloc: 176104 +mem.HeapAlloc: 176104 +mem.NumGC: 0 + +mem.Alloc: 1179440 +mem.TotalAlloc: 16,529,643,664 +mem.HeapAlloc: 1179440 +mem.NumGC: 4254 + diff --git a/go/gg b/go/gg index a8d205f87..1e5e18b0c 100755 --- a/go/gg +++ b/go/gg @@ -52,8 +52,39 @@ mention() { # ================================================================ -run_mlr --from r put -q 'for (k in $*) { print k }; print' +run_mlr --from r put -q ' + a = 1; + b = 2; + print; + print "OUT1a ". a; + print "OUT1b ". b; + if (NR == 1) { + a = 3; + c = 4; + print "IN1a ". a; + print "IN1c ". c; + } else { + b = 5; + d = 6; + print "IN2a ". b; + print "IN2c ". d; + } + print "OUT2a ". a; + print "OUT2a ". b; +' -run_mlr --from r put -q 'for (k,v in $*) { print k; print v }; print' +# OUT1a 1 +# OUT1b 2 +# IN1a 3 +# IN1c 4 +# OUT2a 3 +# OUT2a 2 +# +# OUT1a 1 +# OUT1b 2 +# IN2a 5 +# IN2c 6 +# OUT2a 1 +# OUT2a 5 # ================================================================ diff --git a/go/mlr.go b/go/mlr.go index b0180d613..a41a84ef6 100644 --- a/go/mlr.go +++ b/go/mlr.go @@ -51,9 +51,6 @@ func main() { // found then this function will not return. auxents.Dispatch(os.Args) - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Start of Miller main per se - options, recordTransformers, err := cli.ParseCommandLine(os.Args) if err != nil { fmt.Fprintln(os.Stderr, os.Args[0], ": ", err) diff --git a/go/src/dsl/cst/for.go b/go/src/dsl/cst/for.go index c60f691c9..e91fcd758 100644 --- a/go/src/dsl/cst/for.go +++ b/go/src/dsl/cst/for.go @@ -33,7 +33,7 @@ import ( // ================================================================ type ForLoopOneVariableNode struct { - variableName string + indexVariable *runtime.StackVariable indexableNode IEvaluable statementBlockNode *StatementBlockNode } @@ -44,7 +44,7 @@ func NewForLoopOneVariableNode( statementBlockNode *StatementBlockNode, ) *ForLoopOneVariableNode { return &ForLoopOneVariableNode{ - variableName, + runtime.NewStackVariable(variableName), indexableNode, statementBlockNode, } @@ -128,7 +128,7 @@ func (this *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(this.variableName, &mapkey) + err := state.Stack.SetAtScope(this.indexVariable, &mapkey) if err != nil { return nil, err } @@ -163,7 +163,7 @@ func (this *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for _, element := range arrayval { - err := state.Stack.SetAtScope(this.variableName, &element) + err := state.Stack.SetAtScope(this.indexVariable, &element) if err != nil { return nil, err } @@ -207,21 +207,21 @@ func (this *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay // ================================================================ type ForLoopTwoVariableNode struct { - keyVariableName string - valueVariableName string + keyIndexVariable *runtime.StackVariable + valueIndexVariable *runtime.StackVariable indexableNode IEvaluable statementBlockNode *StatementBlockNode } func NewForLoopTwoVariableNode( - keyVariableName string, - valueVariableName string, + keyIndexVariable *runtime.StackVariable, + valueIndexVariable *runtime.StackVariable, indexableNode IEvaluable, statementBlockNode *StatementBlockNode, ) *ForLoopTwoVariableNode { return &ForLoopTwoVariableNode{ - keyVariableName, - valueVariableName, + keyIndexVariable, + valueIndexVariable, indexableNode, statementBlockNode, } @@ -258,10 +258,12 @@ func (this *RootNode) BuildForLoopTwoVariableNode(astNode *dsl.ASTNode) (*ForLoo lib.InternalCodingErrorIf(keyVariableASTNode.Type != dsl.NodeTypeLocalVariable) lib.InternalCodingErrorIf(keyVariableASTNode.Token == nil) keyVariableName := string(keyVariableASTNode.Token.Lit) + keyIndexVariable := runtime.NewStackVariable(keyVariableName) lib.InternalCodingErrorIf(valueVariableASTNode.Type != dsl.NodeTypeLocalVariable) lib.InternalCodingErrorIf(valueVariableASTNode.Token == nil) valueVariableName := string(valueVariableASTNode.Token.Lit) + valueIndexVariable := runtime.NewStackVariable(valueVariableName) // TODO: error if loop-over node isn't map/array (inasmuch as can be // detected at CST-build time) @@ -276,8 +278,8 @@ func (this *RootNode) BuildForLoopTwoVariableNode(astNode *dsl.ASTNode) (*ForLoo } return NewForLoopTwoVariableNode( - keyVariableName, - valueVariableName, + keyIndexVariable, + valueIndexVariable, indexableNode, statementBlockNode, ), nil @@ -311,11 +313,11 @@ func (this *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(this.keyVariableName, &mapkey) + err := state.Stack.SetAtScope(this.keyIndexVariable, &mapkey) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, pe.Value) + err = state.Stack.SetAtScope(this.valueIndexVariable, pe.Value) if err != nil { return nil, err } @@ -352,11 +354,11 @@ func (this *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay for zindex, element := range arrayval { mindex := types.MlrvalFromInt(int(zindex + 1)) - err := state.Stack.SetAtScope(this.keyVariableName, &mindex) + err := state.Stack.SetAtScope(this.keyIndexVariable, &mindex) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, &element) + err = state.Stack.SetAtScope(this.valueIndexVariable, &element) if err != nil { return nil, err } @@ -400,21 +402,21 @@ func (this *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay // ================================================================ type ForLoopMultivariableNode struct { - keyVariableNames []string - valueVariableName string + keyIndexVariables []*runtime.StackVariable + valueIndexVariable *runtime.StackVariable indexableNode IEvaluable statementBlockNode *StatementBlockNode } func NewForLoopMultivariableNode( - keyVariableNames []string, - valueVariableName string, + keyIndexVariables []*runtime.StackVariable, + valueIndexVariable *runtime.StackVariable, indexableNode IEvaluable, statementBlockNode *StatementBlockNode, ) *ForLoopMultivariableNode { return &ForLoopMultivariableNode{ - keyVariableNames, - valueVariableName, + keyIndexVariables, + valueIndexVariable, indexableNode, statementBlockNode, } @@ -449,15 +451,17 @@ func (this *RootNode) BuildForLoopMultivariableNode( lib.InternalCodingErrorIf(keyVariablesASTNode.Type != dsl.NodeTypeParameterList) lib.InternalCodingErrorIf(keyVariablesASTNode.Children == nil) - keyVariableNames := make([]string, len(keyVariablesASTNode.Children)) + keyIndexVariables := make([]*runtime.StackVariable, len(keyVariablesASTNode.Children)) for i, keyVariableASTNode := range keyVariablesASTNode.Children { lib.InternalCodingErrorIf(keyVariableASTNode.Token == nil) - keyVariableNames[i] = string(keyVariableASTNode.Token.Lit) + keyIndexVariableName := string(keyVariableASTNode.Token.Lit) + keyIndexVariables[i] = runtime.NewStackVariable(keyIndexVariableName) } lib.InternalCodingErrorIf(valueVariableASTNode.Type != dsl.NodeTypeLocalVariable) lib.InternalCodingErrorIf(valueVariableASTNode.Token == nil) valueVariableName := string(valueVariableASTNode.Token.Lit) + valueIndexVariable := runtime.NewStackVariable(valueVariableName) // TODO: error if loop-over node isn't map/array (inasmuch as can be // detected at CST-build time) @@ -472,8 +476,8 @@ func (this *RootNode) BuildForLoopMultivariableNode( } return NewForLoopMultivariableNode( - keyVariableNames, - valueVariableName, + keyIndexVariables, + valueIndexVariable, indexableNode, statementBlockNode, ), nil @@ -506,7 +510,7 @@ func (this *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // from any of the latter is a break from all. However, at this point, the // break has been "broken" and should not be returned to the caller. // Return-statements should, though. - blockExitPayload, err := this.executeOuter(indexMlrval, this.keyVariableNames, state) + blockExitPayload, err := this.executeOuter(indexMlrval, this.keyIndexVariables, state) if blockExitPayload == nil { return nil, err } else { @@ -521,11 +525,11 @@ func (this *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // ---------------------------------------------------------------- func (this *ForLoopMultivariableNode) executeOuter( mlrval *types.Mlrval, - keyVariableNames []string, + keyIndexVariables []*runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { - if len(keyVariableNames) == 1 { - return this.executeInner(mlrval, keyVariableNames[0], state) + if len(keyIndexVariables) == 1 { + return this.executeInner(mlrval, keyIndexVariables[0], state) } // else, recurse @@ -535,12 +539,12 @@ func (this *ForLoopMultivariableNode) executeOuter( for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(keyVariableNames[0], &mapkey) + err := state.Stack.SetAtScope(keyIndexVariables[0], &mapkey) if err != nil { return nil, err } - blockExitPayload, err := this.executeOuter(pe.Value, keyVariableNames[1:], state) + blockExitPayload, err := this.executeOuter(pe.Value, keyIndexVariables[1:], state) if err != nil { return nil, err } @@ -568,12 +572,12 @@ func (this *ForLoopMultivariableNode) executeOuter( for zindex, element := range arrayval { mindex := types.MlrvalFromInt(int(zindex + 1)) - err := state.Stack.SetAtScope(keyVariableNames[0], &mindex) + err := state.Stack.SetAtScope(keyIndexVariables[0], &mindex) if err != nil { return nil, err } - blockExitPayload, err := this.executeOuter(&element, keyVariableNames[1:], state) + blockExitPayload, err := this.executeOuter(&element, keyIndexVariables[1:], state) if err != nil { return nil, err } @@ -614,7 +618,7 @@ func (this *ForLoopMultivariableNode) executeOuter( // ---------------------------------------------------------------- func (this *ForLoopMultivariableNode) executeInner( mlrval *types.Mlrval, - keyVariableName string, + keyIndexVariable *runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { if mlrval.IsMap() { @@ -623,11 +627,11 @@ func (this *ForLoopMultivariableNode) executeInner( for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(keyVariableName, &mapkey) + err := state.Stack.SetAtScope(keyIndexVariable, &mapkey) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, pe.Value) + err = state.Stack.SetAtScope(this.valueIndexVariable, pe.Value) if err != nil { return nil, err } @@ -661,11 +665,11 @@ func (this *ForLoopMultivariableNode) executeInner( for zindex, element := range arrayval { mindex := types.MlrvalFromInt(int(zindex + 1)) - err := state.Stack.SetAtScope(keyVariableName, &mindex) + err := state.Stack.SetAtScope(keyIndexVariable, &mindex) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, &element) + err = state.Stack.SetAtScope(this.valueIndexVariable, &element) if err != nil { return nil, err } diff --git a/go/src/dsl/cst/leaves.go b/go/src/dsl/cst/leaves.go index bac507ef2..137775d8a 100644 --- a/go/src/dsl/cst/leaves.go +++ b/go/src/dsl/cst/leaves.go @@ -173,18 +173,18 @@ func (this *FullOosvarRvalueNode) Evaluate( // ---------------------------------------------------------------- type LocalVariableNode struct { - variableName string + stackVariable *runtime.StackVariable } func (this *RootNode) BuildLocalVariableNode(variableName string) *LocalVariableNode { return &LocalVariableNode{ - variableName: variableName, + stackVariable: runtime.NewStackVariable(variableName), } } func (this *LocalVariableNode) Evaluate( state *runtime.State, ) *types.Mlrval { - value := state.Stack.Get(this.variableName) + value := state.Stack.Get(this.stackVariable) if value == nil { return types.MLRVAL_ABSENT } else { diff --git a/go/src/dsl/cst/lvalues.go b/go/src/dsl/cst/lvalues.go index 3430aee08..90e19615c 100644 --- a/go/src/dsl/cst/lvalues.go +++ b/go/src/dsl/cst/lvalues.go @@ -779,8 +779,8 @@ func (this *FullOosvarLvalueNode) UnassignIndexed( // ---------------------------------------------------------------- type LocalVariableLvalueNode struct { - variableName string - typeName string + stackVariable *runtime.StackVariable + typeName string // a = 1; // b = 1; @@ -804,19 +804,19 @@ func (this *RootNode) BuildLocalVariableLvalueNode(astNode *dsl.ASTNode) (IAssig defineTypedAtScope = true } return NewLocalVariableLvalueNode( - variableName, + runtime.NewStackVariable(variableName), typeName, defineTypedAtScope, ), nil } func NewLocalVariableLvalueNode( - variableName string, + stackVariable *runtime.StackVariable, typeName string, defineTypedAtScope bool, ) *LocalVariableLvalueNode { return &LocalVariableLvalueNode{ - variableName: variableName, + stackVariable: stackVariable, typeName: typeName, defineTypedAtScope: defineTypedAtScope, } @@ -839,15 +839,15 @@ func (this *LocalVariableLvalueNode) AssignIndexed( var err error = nil if indices == nil { if this.defineTypedAtScope { - err = state.Stack.DefineTypedAtScope(this.variableName, this.typeName, rvalue) + err = state.Stack.DefineTypedAtScope(this.stackVariable, this.typeName, rvalue) } else { - err = state.Stack.Set(this.variableName, rvalue) + err = state.Stack.Set(this.stackVariable, rvalue) } } else { // There is no 'map x[1] = {}' in the DSL grammar. lib.InternalCodingErrorIf(this.defineTypedAtScope) - err = state.Stack.SetIndexed(this.variableName, indices, rvalue) + err = state.Stack.SetIndexed(this.stackVariable, indices, rvalue) } return err } @@ -863,9 +863,9 @@ func (this *LocalVariableLvalueNode) UnassignIndexed( state *runtime.State, ) { if indices == nil { - state.Stack.Unset(this.variableName) + state.Stack.Unset(this.stackVariable) } else { - state.Stack.UnsetIndexed(this.variableName, indices) + state.Stack.UnsetIndexed(this.stackVariable, indices) } } diff --git a/go/src/dsl/cst/udf.go b/go/src/dsl/cst/udf.go index 2155c1583..6935edbc5 100644 --- a/go/src/dsl/cst/udf.go +++ b/go/src/dsl/cst/udf.go @@ -129,7 +129,7 @@ func (this *UDFCallsite) Evaluate( for i, _ := range arguments { err := state.Stack.DefineTypedAtScope( - this.udf.signature.typeGatedParameterNames[i].Name, + runtime.NewStackVariable(this.udf.signature.typeGatedParameterNames[i].Name), this.udf.signature.typeGatedParameterNames[i].TypeName, arguments[i], ) diff --git a/go/src/dsl/cst/uds.go b/go/src/dsl/cst/uds.go index 3df093bc1..535ec87b4 100644 --- a/go/src/dsl/cst/uds.go +++ b/go/src/dsl/cst/uds.go @@ -123,7 +123,7 @@ func (this *UDSCallsite) Execute(state *runtime.State) (*BlockExitPayload, error for i, _ := range arguments { err := state.Stack.DefineTypedAtScope( - this.uds.signature.typeGatedParameterNames[i].Name, + runtime.NewStackVariable(this.uds.signature.typeGatedParameterNames[i].Name), this.uds.signature.typeGatedParameterNames[i].TypeName, arguments[i], ) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index c7c58496b..e1ece25e7 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -34,56 +34,98 @@ import ( "miller/src/types" ) +// ================================================================ +// STACK VARIABLE + +// StackVariable is an opaque handle which a callsite can hold onto, which +// keeps stack-offset information in it that is private to us. +type StackVariable struct { + name string + + // Type like "int" or "num" or "var" is stored in the stack itself. A + // StackVariable can appear in the CST (concrete syntax tree) on either the + // left-hand side or right-hande side of an assignment -- in the latter + // case the callsite won't know the type until the value is read off the + // stack. + + // TODO: comment + frameSetOffset int + offsetInFrame int +} + +// TODO: be sure to invalidate slot 0 for struct uninit +func NewStackVariable(name string) *StackVariable { + return &StackVariable{ + name: name, + frameSetOffset: -1, + offsetInFrame: -1, + } +} + // ================================================================ // STACK METHODS type Stack struct { - stackFrameSets *list.List // list of *StackFrameSet + // list of *StackFrameSet + stackFrameSets *list.List + + // Invariant: equal to the head of the stackFrameSets list. This is cached + // since all sets/gets in between frameset-push and frameset-pop will all + // and only be operating on the head. + head *StackFrameSet } func NewStack() *Stack { stackFrameSets := list.New() - stackFrameSets.PushFront(newStackFrameSet()) + head := newStackFrameSet() + stackFrameSets.PushFront(head) return &Stack{ stackFrameSets: stackFrameSets, + head: head, } } // For when a user-defined function/subroutine is being entered func (this *Stack) PushStackFrameSet() { - this.stackFrameSets.PushFront(newStackFrameSet()) + this.head = newStackFrameSet() + this.stackFrameSets.PushFront(this.head) } // For when a user-defined function/subroutine is being exited func (this *Stack) PopStackFrameSet() { this.stackFrameSets.Remove(this.stackFrameSets.Front()) + this.head = this.stackFrameSets.Front().Value.(*StackFrameSet) } // ---------------------------------------------------------------- -// Delegations to topmost frameset +// All of these are simply delegations to the head frameset // For when an if/for/etc block is being entered func (this *Stack) PushStackFrame() { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.pushStackFrame() + this.head.pushStackFrame() } // For when an if/for/etc block is being exited func (this *Stack) PopStackFrame() { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.popStackFrame() + this.head.popStackFrame() +} + +// Returns nil on no-such +func (this *Stack) Get( + stackVariable *StackVariable, +) *types.Mlrval { + return this.head.get(stackVariable) } // For 'num a = 2', setting a variable at the current frame regardless of outer // scope. It's an error to define it again in the same scope, whether the type // is the same or not. func (this *Stack) DefineTypedAtScope( - variableName string, + stackVariable *StackVariable, typeName string, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.defineTypedAtScope(variableName, typeName, mlrval) + return this.head.defineTypedAtScope(stackVariable, typeName, mlrval) } // For untyped declarations at the current scope -- these are in binds of @@ -92,11 +134,10 @@ func (this *Stack) DefineTypedAtScope( // E.g. 'for (int i = 0; i < 10; i += 1)' uses DefineTypedAtScope // E.g. 'for (i = 0; i < 10; i += 1)' uses Set. func (this *Stack) SetAtScope( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.setAtScope(variableName, mlrval) + return this.head.setAtScope(stackVariable, mlrval) } // For 'a = 2', checking for outer-scoped to maybe reuse, else insert new in @@ -107,42 +148,34 @@ func (this *Stack) SetAtScope( // However if it waa previously assigned untyped with 'a = "hello"' then the // assignment is OK. func (this *Stack) Set( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.set(variableName, mlrval) + return this.head.set(stackVariable, mlrval) } // E.g. 'x[1] = 2' where the variable x may or may not have been already set. func (this *Stack) SetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.setIndexed(variableName, indices, mlrval) + return this.head.setIndexed(stackVariable, indices, mlrval) } // E.g. 'unset x' -func (this *Stack) Unset(variableName string) { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.unset(variableName) +func (this *Stack) Unset( + stackVariable *StackVariable, +) { + this.head.unset(stackVariable) } // E.g. 'unset x[1]' func (this *Stack) UnsetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, ) { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.unsetIndexed(variableName, indices) -} - -// Returns nil on no-such -func (this *Stack) Get(variableName string) *types.Mlrval { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.get(variableName) + this.head.unsetIndexed(stackVariable, indices) } func (this *Stack) Dump() { @@ -156,89 +189,159 @@ func (this *Stack) Dump() { // ================================================================ // STACKFRAMESET METHODS +const stackFrameSetInitCap = 6 + type StackFrameSet struct { - stackFrames *list.List // list of *StackFrame + stackFrames []*StackFrame } func newStackFrameSet() *StackFrameSet { - stackFrames := list.New() - stackFrames.PushFront(newStackFrame()) + stackFrames := make([]*StackFrame, 1, stackFrameSetInitCap) + stackFrames[0] = newStackFrame() return &StackFrameSet{ stackFrames: stackFrames, } } func (this *StackFrameSet) pushStackFrame() { - this.stackFrames.PushFront(newStackFrame()) + this.stackFrames = append(this.stackFrames, newStackFrame()) } func (this *StackFrameSet) popStackFrame() { - this.stackFrames.Remove(this.stackFrames.Front()) + this.stackFrames = this.stackFrames[0 : len(this.stackFrames)-1] } func (this *StackFrameSet) dump() { - fmt.Printf(" STACK FRAMES (count %d):\n", this.stackFrames.Len()) - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + fmt.Printf(" STACK FRAMES (count %d):\n", len(this.stackFrames)) + for _, stackFrame := range this.stackFrames { fmt.Printf(" VARIABLES (count %d):\n", len(stackFrame.vars)) - for k, v := range stackFrame.vars { - fmt.Printf(" %-16s %s\n", k, v.ValueString()) + for _, v := range stackFrame.vars { + fmt.Printf(" %-16s %s\n", v.GetName(), v.ValueString()) } } } +// Returns nil on no-such +func (this *StackFrameSet) get( + stackVariable *StackVariable, +) *types.Mlrval { + // TODO: comment + fso := stackVariable.frameSetOffset + oif := stackVariable.offsetInFrame + if fso >= 0 && fso < len(this.stackFrames) && oif >= 0 && oif < len(this.stackFrames[fso].vars) { + return this.stackFrames[fso].vars[oif].GetValue() + } else { + return this.getUncached(stackVariable) + } +} + +// Returns nil on no-such +func (this *StackFrameSet) getUncached( + stackVariable *StackVariable, +) *types.Mlrval { + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] + mlrval := stackFrame.get(stackVariable) + if mlrval != nil { + // TODO: comment + stackVariable.frameSetOffset = offset + return mlrval + } + } + return nil +} + // See Stack.DefineTypedAtScope comments above func (this *StackFrameSet) defineTypedAtScope( - variableName string, + stackVariable *StackVariable, typeName string, mlrval *types.Mlrval, ) error { - return this.stackFrames.Front().Value.(*StackFrame).defineTyped(variableName, typeName, mlrval) + offset := len(this.stackFrames) - 1 + // TODO: comment + stackVariable.frameSetOffset = offset + return this.stackFrames[offset].defineTyped( + stackVariable, typeName, mlrval, + ) } // See Stack.SetAtScope comments above func (this *StackFrameSet) setAtScope( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - return this.stackFrames.Front().Value.(*StackFrame).set(variableName, mlrval) + offset := len(this.stackFrames) - 1 + // TODO: comment + stackVariable.frameSetOffset = offset + return this.stackFrames[offset].set(stackVariable, mlrval) } // See Stack.Set comments above func (this *StackFrameSet) set( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - return stackFrame.set(variableName, mlrval) + fso := stackVariable.frameSetOffset + oif := stackVariable.offsetInFrame + if fso >= 0 && fso < len(this.stackFrames) && oif >= 0 && oif < len(this.stackFrames[fso].vars) { + return this.stackFrames[fso].vars[oif].Assign(mlrval.Copy()) + } else { + return this.setUncached(stackVariable, mlrval) + } +} + +// See Stack.Set comments above +func (this *StackFrameSet) setUncached( + stackVariable *StackVariable, + mlrval *types.Mlrval, +) error { + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] + if stackFrame.has(stackVariable) { + // TODO: comment + stackVariable.frameSetOffset = offset + return stackFrame.set(stackVariable, mlrval) } } - return this.setAtScope(variableName, mlrval) + return this.setAtScope(stackVariable, mlrval) } // See Stack.SetIndexed comments above func (this *StackFrameSet) setIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, mlrval *types.Mlrval, ) error { - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - return stackFrame.setIndexed(variableName, indices, mlrval) + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] + if stackFrame.has(stackVariable) { + // TODO: comment + stackVariable.frameSetOffset = offset + return stackFrame.setIndexed(stackVariable, indices, mlrval) } } - return this.stackFrames.Front().Value.(*StackFrame).setIndexed(variableName, indices, mlrval) + // TODO: comment + offset := numStackFrames - 1 + stackVariable.frameSetOffset = offset + return this.stackFrames[offset].setIndexed(stackVariable, indices, mlrval) } // See Stack.Unset comments above -func (this *StackFrameSet) unset(variableName string) { - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - stackFrame.unset(variableName) +func (this *StackFrameSet) unset( + stackVariable *StackVariable, +) { + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] + if stackFrame.has(stackVariable) { + stackFrame.unset(stackVariable) return } } @@ -246,134 +349,130 @@ func (this *StackFrameSet) unset(variableName string) { // See Stack.UnsetIndexed comments above func (this *StackFrameSet) unsetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, ) { - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - stackFrame.unsetIndexed(variableName, indices) + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] + if stackFrame.has(stackVariable) { + stackFrame.unsetIndexed(stackVariable, indices) return } } } -// Returns nil on no-such -func (this *StackFrameSet) get(variableName string) *types.Mlrval { - // Scope-walk - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) - mlrval := stackFrame.get(variableName) - if mlrval != nil { - return mlrval - } - } - return nil -} - // ================================================================ // STACKFRAME METHODS +const stackFrameInitCap = 10 + type StackFrame struct { // TODO: just a map for now. In the C impl, pre-computation of // name-to-array-slot indices was an important optimization, especially for // compute-intensive scenarios. - vars map[string]*types.TypeGatedMlrvalVariable + //vars map[string]*types.TypeGatedMlrvalVariable + + // TODO: comment + vars []*types.TypeGatedMlrvalVariable + namesToOffsets map[string]int } func newStackFrame() *StackFrame { + vars := make([]*types.TypeGatedMlrvalVariable, 0, stackFrameInitCap) + namesToOffsets := make(map[string]int) return &StackFrame{ - vars: make(map[string]*types.TypeGatedMlrvalVariable), + vars: vars, + namesToOffsets: namesToOffsets, } } // Returns nil on no such -func (this *StackFrame) get(variableName string) *types.Mlrval { - slot := this.vars[variableName] - if slot == nil { - return nil +func (this *StackFrame) get( + stackVariable *StackVariable, +) *types.Mlrval { + offset, ok := this.namesToOffsets[stackVariable.name] + if ok { + // TODO: comment + stackVariable.offsetInFrame = offset + return this.vars[offset].GetValue() } else { - return slot.GetValue() + return nil } } -func (this *StackFrame) has(variableName string) bool { - return this.vars[variableName] != nil -} - -func (this *StackFrame) clear() { - this.vars = make(map[string]*types.TypeGatedMlrvalVariable) +func (this *StackFrame) has( + stackVariable *StackVariable, +) bool { + _, ok := this.namesToOffsets[stackVariable.name] + return ok } // TODO: audit for honor of error-return at callsites func (this *StackFrame) set( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - slot := this.vars[variableName] - if slot == nil { - slot, err := types.NewTypeGatedMlrvalVariable(variableName, "any", mlrval) + offset, ok := this.namesToOffsets[stackVariable.name] + if !ok { + slot, err := types.NewTypeGatedMlrvalVariable(stackVariable.name, "any", mlrval) if err != nil { return err - } else { - this.vars[variableName] = slot - return nil } - this.vars[variableName] = slot + this.vars = append(this.vars, slot) + offsetInFrame := len(this.vars) - 1 + this.namesToOffsets[stackVariable.name] = offsetInFrame + // TODO: comment + stackVariable.offsetInFrame = offsetInFrame return nil } else { - return slot.Assign(mlrval) + return this.vars[offset].Assign(mlrval) } } // TODO: audit for honor of error-return at callsites func (this *StackFrame) defineTyped( - variableName string, + stackVariable *StackVariable, typeName string, mlrval *types.Mlrval, ) error { - slot := this.vars[variableName] - if slot == nil { - slot, err := types.NewTypeGatedMlrvalVariable(variableName, typeName, mlrval) + _, ok := this.namesToOffsets[stackVariable.name] + if !ok { + slot, err := types.NewTypeGatedMlrvalVariable(stackVariable.name, typeName, mlrval) if err != nil { return err - } else { - this.vars[variableName] = slot - return nil } - this.vars[variableName] = slot + this.vars = append(this.vars, slot) + offsetInFrame := len(this.vars) - 1 + this.namesToOffsets[stackVariable.name] = offsetInFrame + // TODO: comment + stackVariable.offsetInFrame = offsetInFrame return nil } else { return errors.New( fmt.Sprintf( "%s: variable %s has already been defined in the same scope.", - lib.MlrExeName(), variableName, + lib.MlrExeName(), stackVariable.name, ), ) } } -func (this *StackFrame) unset(variableName string) { - slot := this.vars[variableName] - if slot != nil { - slot.Unassign() - } -} - // TODO: audit for honor of error-return at callsites func (this *StackFrame) setIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, mlrval *types.Mlrval, ) error { - value := this.get(variableName) + value := this.get(stackVariable) if value == nil { lib.InternalCodingErrorIf(len(indices) < 1) leadingIndex := indices[0] if leadingIndex.IsString() || leadingIndex.IsInt() { newval := types.MlrvalEmptyMap() newval.PutIndexed(indices, mlrval) - return this.set(variableName, &newval) + return this.set(stackVariable, &newval) } else { return errors.New( fmt.Sprintf( @@ -389,11 +488,20 @@ func (this *StackFrame) setIndexed( } } +func (this *StackFrame) unset( + stackVariable *StackVariable, +) { + offset, ok := this.namesToOffsets[stackVariable.name] + if ok { + this.vars[offset].Unassign() + } +} + func (this *StackFrame) unsetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, ) { - value := this.get(variableName) + value := this.get(stackVariable) if value == nil { return } diff --git a/go/src/types/mlrval_typing.go b/go/src/types/mlrval_typing.go index 36204fc27..ec22f958a 100644 --- a/go/src/types/mlrval_typing.go +++ b/go/src/types/mlrval_typing.go @@ -77,6 +77,10 @@ func NewTypeGatedMlrvalVariable( }, nil } +func (this *TypeGatedMlrvalVariable) GetName() string { + return this.typeGatedMlrvalName.Name +} + func (this *TypeGatedMlrvalVariable) GetValue() *Mlrval { return this.value } diff --git a/go/todo.txt b/go/todo.txt index 6c9110c90..1dfd6f7a1 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -295,6 +295,26 @@ GOCC UPSTREAMS: * support "abc" (not just 'a' 'b' 'c') in the lexer part ---------------------------------------------------------------- +<<<<<<< HEAD +======= +FLAME GRAPHS + +go get -u github.com/google/pprof +ll ~/go/bin/pprof +go get -u github.com/uber/go-torch + +cd ~/git +mkdir brendangregg +cd brendangregg +git clone https://github.com/brendangregg/FlameGraph + +cd /path/to/mlr/go +export PATH=${PATH}:~/git/brendangregg/FlameGraph/ +go-torch cpu.pprof +mv torch.svg ~/Desktop/ + +---------------------------------------------------------------- +>>>>>>> cb2647ab... perf-debug [WIP] NITS/NON-IMMEDIATE: * "Miller: " prefixes on all errors.New diff --git a/go/u/mand.mlr b/go/u/mand.mlr index 60c6b6256..ebe5423bd 100644 --- a/go/u/mand.mlr +++ b/go/u/mand.mlr @@ -12,17 +12,6 @@ begin { @silent ??= false; } -# Override defaults -@rcorn = $rcorn; -@icorn = $icorn; -@side = $side; -@iheight = $iheight; -@iwidth = $iwidth; -@maxits = $maxits; -@levelstep = $levelstep; -@chars = $chars; -@silent = $silent; - end { if (!@silent) { print "RCORN = ".@rcorn;