From 583e5c1c1d692e06fa2169c3ce591c309b2013d6 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 14:27:44 -0500 Subject: [PATCH 01/11] plan --- go/gg | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/go/gg b/go/gg index a8d205f87..1e5e18b0c 100755 --- a/go/gg +++ b/go/gg @@ -52,8 +52,39 @@ mention() { # ================================================================ -run_mlr --from r put -q 'for (k in $*) { print k }; print' +run_mlr --from r put -q ' + a = 1; + b = 2; + print; + print "OUT1a ". a; + print "OUT1b ". b; + if (NR == 1) { + a = 3; + c = 4; + print "IN1a ". a; + print "IN1c ". c; + } else { + b = 5; + d = 6; + print "IN2a ". b; + print "IN2c ". d; + } + print "OUT2a ". a; + print "OUT2a ". b; +' -run_mlr --from r put -q 'for (k,v in $*) { print k; print v }; print' +# OUT1a 1 +# OUT1b 2 +# IN1a 3 +# IN1c 4 +# OUT2a 3 +# OUT2a 2 +# +# OUT1a 1 +# OUT1b 2 +# IN2a 5 +# IN2c 6 +# OUT2a 1 +# OUT2a 5 # ================================================================ From dbcb3fbc837e4d8087e98e2262d913e242fe5b39 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 14:55:58 -0500 Subject: [PATCH 02/11] new StackVariable type wrapping string-name and offset-indices --- go/src/dsl/cst/for.go | 80 ++++++++++---------- go/src/dsl/cst/leaves.go | 4 +- go/src/dsl/cst/lvalues.go | 20 ++--- go/src/dsl/cst/udf.go | 2 +- go/src/dsl/cst/uds.go | 2 +- go/src/runtime/stack.go | 149 ++++++++++++++++++++++++-------------- 6 files changed, 149 insertions(+), 108 deletions(-) diff --git a/go/src/dsl/cst/for.go b/go/src/dsl/cst/for.go index c60f691c9..970f8f553 100644 --- a/go/src/dsl/cst/for.go +++ b/go/src/dsl/cst/for.go @@ -33,7 +33,7 @@ import ( // ================================================================ type ForLoopOneVariableNode struct { - variableName string + indexVariable *runtime.StackVariable indexableNode IEvaluable statementBlockNode *StatementBlockNode } @@ -44,7 +44,7 @@ func NewForLoopOneVariableNode( statementBlockNode *StatementBlockNode, ) *ForLoopOneVariableNode { return &ForLoopOneVariableNode{ - variableName, + runtime.NewStackVariable(variableName), indexableNode, statementBlockNode, } @@ -128,7 +128,7 @@ func (this *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(this.variableName, &mapkey) + err := state.Stack.SetAtScope(this.indexVariable, &mapkey) if err != nil { return nil, err } @@ -163,7 +163,7 @@ func (this *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay state.Stack.PushStackFrame() defer state.Stack.PopStackFrame() for _, element := range arrayval { - err := state.Stack.SetAtScope(this.variableName, &element) + err := state.Stack.SetAtScope(this.indexVariable, &element) if err != nil { return nil, err } @@ -207,21 +207,21 @@ func (this *ForLoopOneVariableNode) Execute(state *runtime.State) (*BlockExitPay // ================================================================ type ForLoopTwoVariableNode struct { - keyVariableName string - valueVariableName string + keyIndexVariable *runtime.StackVariable + valueIndexVariable *runtime.StackVariable indexableNode IEvaluable statementBlockNode *StatementBlockNode } func NewForLoopTwoVariableNode( - keyVariableName string, - valueVariableName string, + keyIndexVariable *runtime.StackVariable, + valueIndexVariable *runtime.StackVariable, indexableNode IEvaluable, statementBlockNode *StatementBlockNode, ) *ForLoopTwoVariableNode { return &ForLoopTwoVariableNode{ - keyVariableName, - valueVariableName, + keyIndexVariable, + valueIndexVariable, indexableNode, statementBlockNode, } @@ -258,10 +258,12 @@ func (this *RootNode) BuildForLoopTwoVariableNode(astNode *dsl.ASTNode) (*ForLoo lib.InternalCodingErrorIf(keyVariableASTNode.Type != dsl.NodeTypeLocalVariable) lib.InternalCodingErrorIf(keyVariableASTNode.Token == nil) keyVariableName := string(keyVariableASTNode.Token.Lit) + keyIndexVariable := runtime.NewStackVariable(keyVariableName) lib.InternalCodingErrorIf(valueVariableASTNode.Type != dsl.NodeTypeLocalVariable) lib.InternalCodingErrorIf(valueVariableASTNode.Token == nil) valueVariableName := string(valueVariableASTNode.Token.Lit) + valueIndexVariable := runtime.NewStackVariable(valueVariableName) // TODO: error if loop-over node isn't map/array (inasmuch as can be // detected at CST-build time) @@ -276,8 +278,8 @@ func (this *RootNode) BuildForLoopTwoVariableNode(astNode *dsl.ASTNode) (*ForLoo } return NewForLoopTwoVariableNode( - keyVariableName, - valueVariableName, + keyIndexVariable, + valueIndexVariable, indexableNode, statementBlockNode, ), nil @@ -311,11 +313,11 @@ func (this *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(this.keyVariableName, &mapkey) + err := state.Stack.SetAtScope(this.keyIndexVariable, &mapkey) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, pe.Value) + err = state.Stack.SetAtScope(this.valueIndexVariable, pe.Value) if err != nil { return nil, err } @@ -352,11 +354,11 @@ func (this *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay for zindex, element := range arrayval { mindex := types.MlrvalFromInt(int(zindex + 1)) - err := state.Stack.SetAtScope(this.keyVariableName, &mindex) + err := state.Stack.SetAtScope(this.keyIndexVariable, &mindex) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, &element) + err = state.Stack.SetAtScope(this.valueIndexVariable, &element) if err != nil { return nil, err } @@ -400,21 +402,21 @@ func (this *ForLoopTwoVariableNode) Execute(state *runtime.State) (*BlockExitPay // ================================================================ type ForLoopMultivariableNode struct { - keyVariableNames []string - valueVariableName string + keyIndexVariables []*runtime.StackVariable + valueIndexVariable *runtime.StackVariable indexableNode IEvaluable statementBlockNode *StatementBlockNode } func NewForLoopMultivariableNode( - keyVariableNames []string, - valueVariableName string, + keyIndexVariables []*runtime.StackVariable, + valueIndexVariable *runtime.StackVariable, indexableNode IEvaluable, statementBlockNode *StatementBlockNode, ) *ForLoopMultivariableNode { return &ForLoopMultivariableNode{ - keyVariableNames, - valueVariableName, + keyIndexVariables, + valueIndexVariable, indexableNode, statementBlockNode, } @@ -449,15 +451,17 @@ func (this *RootNode) BuildForLoopMultivariableNode( lib.InternalCodingErrorIf(keyVariablesASTNode.Type != dsl.NodeTypeParameterList) lib.InternalCodingErrorIf(keyVariablesASTNode.Children == nil) - keyVariableNames := make([]string, len(keyVariablesASTNode.Children)) + keyIndexVariables := make([]*runtime.StackVariable, len(keyVariablesASTNode.Children)) for i, keyVariableASTNode := range keyVariablesASTNode.Children { lib.InternalCodingErrorIf(keyVariableASTNode.Token == nil) - keyVariableNames[i] = string(keyVariableASTNode.Token.Lit) + keyIndexVariableName := string(keyVariableASTNode.Token.Lit) + keyIndexVariables[i] = runtime.NewStackVariable(keyIndexVariableName) } lib.InternalCodingErrorIf(valueVariableASTNode.Type != dsl.NodeTypeLocalVariable) lib.InternalCodingErrorIf(valueVariableASTNode.Token == nil) valueVariableName := string(valueVariableASTNode.Token.Lit) + valueIndexVariable := runtime.NewStackVariable(valueVariableName) // TODO: error if loop-over node isn't map/array (inasmuch as can be // detected at CST-build time) @@ -472,8 +476,8 @@ func (this *RootNode) BuildForLoopMultivariableNode( } return NewForLoopMultivariableNode( - keyVariableNames, - valueVariableName, + keyIndexVariables, + valueIndexVariable, indexableNode, statementBlockNode, ), nil @@ -521,11 +525,11 @@ func (this *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // ---------------------------------------------------------------- func (this *ForLoopMultivariableNode) executeOuter( mlrval *types.Mlrval, - keyVariableNames []string, + keyIndexVariables []*runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { - if len(keyVariableNames) == 1 { - return this.executeInner(mlrval, keyVariableNames[0], state) + if len(keyIndexVariables) == 1 { + return this.executeInner(mlrval, keyIndexVariables[0], state) } // else, recurse @@ -535,12 +539,12 @@ func (this *ForLoopMultivariableNode) executeOuter( for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(keyVariableNames[0], &mapkey) + err := state.Stack.SetAtScope(keyIndexVariables[0], &mapkey) if err != nil { return nil, err } - blockExitPayload, err := this.executeOuter(pe.Value, keyVariableNames[1:], state) + blockExitPayload, err := this.executeOuter(pe.Value, keyIndexVariables[1:], state) if err != nil { return nil, err } @@ -568,12 +572,12 @@ func (this *ForLoopMultivariableNode) executeOuter( for zindex, element := range arrayval { mindex := types.MlrvalFromInt(int(zindex + 1)) - err := state.Stack.SetAtScope(keyVariableNames[0], &mindex) + err := state.Stack.SetAtScope(keyIndexVariables[0], &mindex) if err != nil { return nil, err } - blockExitPayload, err := this.executeOuter(&element, keyVariableNames[1:], state) + blockExitPayload, err := this.executeOuter(&element, keyIndexVariables[1:], state) if err != nil { return nil, err } @@ -614,7 +618,7 @@ func (this *ForLoopMultivariableNode) executeOuter( // ---------------------------------------------------------------- func (this *ForLoopMultivariableNode) executeInner( mlrval *types.Mlrval, - keyVariableName string, + keyIndexVariable *runtime.StackVariable, state *runtime.State, ) (*BlockExitPayload, error) { if mlrval.IsMap() { @@ -623,11 +627,11 @@ func (this *ForLoopMultivariableNode) executeInner( for pe := mapval.Head; pe != nil; pe = pe.Next { mapkey := types.MlrvalFromString(pe.Key) - err := state.Stack.SetAtScope(keyVariableName, &mapkey) + err := state.Stack.SetAtScope(keyIndexVariable, &mapkey) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, pe.Value) + err = state.Stack.SetAtScope(this.valueIndexVariable, pe.Value) if err != nil { return nil, err } @@ -661,11 +665,11 @@ func (this *ForLoopMultivariableNode) executeInner( for zindex, element := range arrayval { mindex := types.MlrvalFromInt(int(zindex + 1)) - err := state.Stack.SetAtScope(keyVariableName, &mindex) + err := state.Stack.SetAtScope(keyIndexVariable, &mindex) if err != nil { return nil, err } - err = state.Stack.SetAtScope(this.valueVariableName, &element) + err = state.Stack.SetAtScope(this.valueIndexVariable, &element) if err != nil { return nil, err } diff --git a/go/src/dsl/cst/leaves.go b/go/src/dsl/cst/leaves.go index bac507ef2..11e829269 100644 --- a/go/src/dsl/cst/leaves.go +++ b/go/src/dsl/cst/leaves.go @@ -173,12 +173,12 @@ func (this *FullOosvarRvalueNode) Evaluate( // ---------------------------------------------------------------- type LocalVariableNode struct { - variableName string + stackVariable *runtime.StackVariable } func (this *RootNode) BuildLocalVariableNode(variableName string) *LocalVariableNode { return &LocalVariableNode{ - variableName: variableName, + stackVariable: runtime.NewStackVariable(variableName), } } func (this *LocalVariableNode) Evaluate( diff --git a/go/src/dsl/cst/lvalues.go b/go/src/dsl/cst/lvalues.go index 3430aee08..90e19615c 100644 --- a/go/src/dsl/cst/lvalues.go +++ b/go/src/dsl/cst/lvalues.go @@ -779,8 +779,8 @@ func (this *FullOosvarLvalueNode) UnassignIndexed( // ---------------------------------------------------------------- type LocalVariableLvalueNode struct { - variableName string - typeName string + stackVariable *runtime.StackVariable + typeName string // a = 1; // b = 1; @@ -804,19 +804,19 @@ func (this *RootNode) BuildLocalVariableLvalueNode(astNode *dsl.ASTNode) (IAssig defineTypedAtScope = true } return NewLocalVariableLvalueNode( - variableName, + runtime.NewStackVariable(variableName), typeName, defineTypedAtScope, ), nil } func NewLocalVariableLvalueNode( - variableName string, + stackVariable *runtime.StackVariable, typeName string, defineTypedAtScope bool, ) *LocalVariableLvalueNode { return &LocalVariableLvalueNode{ - variableName: variableName, + stackVariable: stackVariable, typeName: typeName, defineTypedAtScope: defineTypedAtScope, } @@ -839,15 +839,15 @@ func (this *LocalVariableLvalueNode) AssignIndexed( var err error = nil if indices == nil { if this.defineTypedAtScope { - err = state.Stack.DefineTypedAtScope(this.variableName, this.typeName, rvalue) + err = state.Stack.DefineTypedAtScope(this.stackVariable, this.typeName, rvalue) } else { - err = state.Stack.Set(this.variableName, rvalue) + err = state.Stack.Set(this.stackVariable, rvalue) } } else { // There is no 'map x[1] = {}' in the DSL grammar. lib.InternalCodingErrorIf(this.defineTypedAtScope) - err = state.Stack.SetIndexed(this.variableName, indices, rvalue) + err = state.Stack.SetIndexed(this.stackVariable, indices, rvalue) } return err } @@ -863,9 +863,9 @@ func (this *LocalVariableLvalueNode) UnassignIndexed( state *runtime.State, ) { if indices == nil { - state.Stack.Unset(this.variableName) + state.Stack.Unset(this.stackVariable) } else { - state.Stack.UnsetIndexed(this.variableName, indices) + state.Stack.UnsetIndexed(this.stackVariable, indices) } } diff --git a/go/src/dsl/cst/udf.go b/go/src/dsl/cst/udf.go index 2155c1583..6935edbc5 100644 --- a/go/src/dsl/cst/udf.go +++ b/go/src/dsl/cst/udf.go @@ -129,7 +129,7 @@ func (this *UDFCallsite) Evaluate( for i, _ := range arguments { err := state.Stack.DefineTypedAtScope( - this.udf.signature.typeGatedParameterNames[i].Name, + runtime.NewStackVariable(this.udf.signature.typeGatedParameterNames[i].Name), this.udf.signature.typeGatedParameterNames[i].TypeName, arguments[i], ) diff --git a/go/src/dsl/cst/uds.go b/go/src/dsl/cst/uds.go index 3df093bc1..535ec87b4 100644 --- a/go/src/dsl/cst/uds.go +++ b/go/src/dsl/cst/uds.go @@ -123,7 +123,7 @@ func (this *UDSCallsite) Execute(state *runtime.State) (*BlockExitPayload, error for i, _ := range arguments { err := state.Stack.DefineTypedAtScope( - this.uds.signature.typeGatedParameterNames[i].Name, + runtime.NewStackVariable(this.uds.signature.typeGatedParameterNames[i].Name), this.uds.signature.typeGatedParameterNames[i].TypeName, arguments[i], ) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index c7c58496b..0e4716a32 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -34,6 +34,27 @@ import ( "miller/src/types" ) +// ================================================================ +// STACK VARIABLE + +// StackVariable is an opaque handle which a callsite can hold onto, which +// keeps stack-offset information in it that is private to us. +type StackVariable struct { + name string + // Type like "int" or "num" or "var" is stored in the stack itself + frameSetIndex int + indexInFrame int +} + +// TODO: be sure to invalidate slot 0 for struct uninit +func NewStackVariable(name string) *StackVariable { + return &StackVariable{ + name: name, + frameSetIndex: 0, + indexInFrame: 0, + } +} + // ================================================================ // STACK METHODS @@ -78,12 +99,12 @@ func (this *Stack) PopStackFrame() { // scope. It's an error to define it again in the same scope, whether the type // is the same or not. func (this *Stack) DefineTypedAtScope( - variableName string, + stackVariable *StackVariable, typeName string, mlrval *types.Mlrval, ) error { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.defineTypedAtScope(variableName, typeName, mlrval) + return head.defineTypedAtScope(stackVariable, typeName, mlrval) } // For untyped declarations at the current scope -- these are in binds of @@ -92,11 +113,11 @@ func (this *Stack) DefineTypedAtScope( // E.g. 'for (int i = 0; i < 10; i += 1)' uses DefineTypedAtScope // E.g. 'for (i = 0; i < 10; i += 1)' uses Set. func (this *Stack) SetAtScope( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.setAtScope(variableName, mlrval) + return head.setAtScope(stackVariable, mlrval) } // For 'a = 2', checking for outer-scoped to maybe reuse, else insert new in @@ -107,42 +128,46 @@ func (this *Stack) SetAtScope( // However if it waa previously assigned untyped with 'a = "hello"' then the // assignment is OK. func (this *Stack) Set( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.set(variableName, mlrval) + return head.set(stackVariable, mlrval) } // E.g. 'x[1] = 2' where the variable x may or may not have been already set. func (this *Stack) SetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, mlrval *types.Mlrval, ) error { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.setIndexed(variableName, indices, mlrval) + return head.setIndexed(stackVariable, indices, mlrval) } // E.g. 'unset x' -func (this *Stack) Unset(variableName string) { +func (this *Stack) Unset( + stackVariable *StackVariable, +) { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.unset(variableName) + head.unset(stackVariable) } // E.g. 'unset x[1]' func (this *Stack) UnsetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, ) { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.unsetIndexed(variableName, indices) + head.unsetIndexed(stackVariable, indices) } // Returns nil on no-such -func (this *Stack) Get(variableName string) *types.Mlrval { +func (this *Stack) Get( + stackVariable *StackVariable, +) *types.Mlrval { head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.get(variableName) + return head.get(stackVariable) } func (this *Stack) Dump() { @@ -189,56 +214,60 @@ func (this *StackFrameSet) dump() { // See Stack.DefineTypedAtScope comments above func (this *StackFrameSet) defineTypedAtScope( - variableName string, + stackVariable *StackVariable, typeName string, mlrval *types.Mlrval, ) error { - return this.stackFrames.Front().Value.(*StackFrame).defineTyped(variableName, typeName, mlrval) + return this.stackFrames.Front().Value.(*StackFrame).defineTyped( + stackVariable, typeName, mlrval, + ) } // See Stack.SetAtScope comments above func (this *StackFrameSet) setAtScope( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - return this.stackFrames.Front().Value.(*StackFrame).set(variableName, mlrval) + return this.stackFrames.Front().Value.(*StackFrame).set(stackVariable, mlrval) } // See Stack.Set comments above func (this *StackFrameSet) set( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - return stackFrame.set(variableName, mlrval) + if stackFrame.has(stackVariable) { + return stackFrame.set(stackVariable, mlrval) } } - return this.setAtScope(variableName, mlrval) + return this.setAtScope(stackVariable, mlrval) } // See Stack.SetIndexed comments above func (this *StackFrameSet) setIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, mlrval *types.Mlrval, ) error { for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - return stackFrame.setIndexed(variableName, indices, mlrval) + if stackFrame.has(stackVariable) { + return stackFrame.setIndexed(stackVariable, indices, mlrval) } } - return this.stackFrames.Front().Value.(*StackFrame).setIndexed(variableName, indices, mlrval) + return this.stackFrames.Front().Value.(*StackFrame).setIndexed(stackVariable, indices, mlrval) } // See Stack.Unset comments above -func (this *StackFrameSet) unset(variableName string) { +func (this *StackFrameSet) unset( + stackVariable *StackVariable, +) { for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - stackFrame.unset(variableName) + if stackFrame.has(stackVariable) { + stackFrame.unset(stackVariable) return } } @@ -246,24 +275,26 @@ func (this *StackFrameSet) unset(variableName string) { // See Stack.UnsetIndexed comments above func (this *StackFrameSet) unsetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, ) { for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) - if stackFrame.has(variableName) { - stackFrame.unsetIndexed(variableName, indices) + if stackFrame.has(stackVariable) { + stackFrame.unsetIndexed(stackVariable, indices) return } } } // Returns nil on no-such -func (this *StackFrameSet) get(variableName string) *types.Mlrval { +func (this *StackFrameSet) get( + stackVariable *StackVariable, +) *types.Mlrval { // Scope-walk for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) - mlrval := stackFrame.get(variableName) + mlrval := stackFrame.get(stackVariable) if mlrval != nil { return mlrval } @@ -288,8 +319,10 @@ func newStackFrame() *StackFrame { } // Returns nil on no such -func (this *StackFrame) get(variableName string) *types.Mlrval { - slot := this.vars[variableName] +func (this *StackFrame) get( + stackVariable *StackVariable, +) *types.Mlrval { + slot := this.vars[stackVariable.name] if slot == nil { return nil } else { @@ -297,8 +330,10 @@ func (this *StackFrame) get(variableName string) *types.Mlrval { } } -func (this *StackFrame) has(variableName string) bool { - return this.vars[variableName] != nil +func (this *StackFrame) has( + stackVariable *StackVariable, +) bool { + return this.vars[stackVariable.name] != nil } func (this *StackFrame) clear() { @@ -307,19 +342,19 @@ func (this *StackFrame) clear() { // TODO: audit for honor of error-return at callsites func (this *StackFrame) set( - variableName string, + stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - slot := this.vars[variableName] + slot := this.vars[stackVariable.name] if slot == nil { - slot, err := types.NewTypeGatedMlrvalVariable(variableName, "any", mlrval) + slot, err := types.NewTypeGatedMlrvalVariable(stackVariable.name, "any", mlrval) if err != nil { return err } else { - this.vars[variableName] = slot + this.vars[stackVariable.name] = slot return nil } - this.vars[variableName] = slot + this.vars[stackVariable.name] = slot return nil } else { return slot.Assign(mlrval) @@ -328,33 +363,35 @@ func (this *StackFrame) set( // TODO: audit for honor of error-return at callsites func (this *StackFrame) defineTyped( - variableName string, + stackVariable *StackVariable, typeName string, mlrval *types.Mlrval, ) error { - slot := this.vars[variableName] + slot := this.vars[stackVariable.name] if slot == nil { - slot, err := types.NewTypeGatedMlrvalVariable(variableName, typeName, mlrval) + slot, err := types.NewTypeGatedMlrvalVariable(stackVariable.name, typeName, mlrval) if err != nil { return err } else { - this.vars[variableName] = slot + this.vars[stackVariable.name] = slot return nil } - this.vars[variableName] = slot + this.vars[stackVariable.name] = slot return nil } else { return errors.New( fmt.Sprintf( "%s: variable %s has already been defined in the same scope.", - lib.MlrExeName(), variableName, + lib.MlrExeName(), stackVariable.name, ), ) } } -func (this *StackFrame) unset(variableName string) { - slot := this.vars[variableName] +func (this *StackFrame) unset( + stackVariable *StackVariable, +) { + slot := this.vars[stackVariable.name] if slot != nil { slot.Unassign() } @@ -362,18 +399,18 @@ func (this *StackFrame) unset(variableName string) { // TODO: audit for honor of error-return at callsites func (this *StackFrame) setIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, mlrval *types.Mlrval, ) error { - value := this.get(variableName) + value := this.get(stackVariable) if value == nil { lib.InternalCodingErrorIf(len(indices) < 1) leadingIndex := indices[0] if leadingIndex.IsString() || leadingIndex.IsInt() { newval := types.MlrvalEmptyMap() newval.PutIndexed(indices, mlrval) - return this.set(variableName, &newval) + return this.set(stackVariable, &newval) } else { return errors.New( fmt.Sprintf( @@ -390,10 +427,10 @@ func (this *StackFrame) setIndexed( } func (this *StackFrame) unsetIndexed( - variableName string, + stackVariable *StackVariable, indices []*types.Mlrval, ) { - value := this.get(variableName) + value := this.get(stackVariable) if value == nil { return } From 269393d32cb2fc057b68a287cfbdb7e77eafb3b5 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 15:14:38 -0500 Subject: [PATCH 03/11] plan --- go/src/runtime/stack.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 0e4716a32..5b113d333 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -186,6 +186,7 @@ type StackFrameSet struct { } func newStackFrameSet() *StackFrameSet { + // TODO: to array stackFrames := list.New() stackFrames.PushFront(newStackFrame()) return &StackFrameSet{ @@ -194,10 +195,12 @@ func newStackFrameSet() *StackFrameSet { } func (this *StackFrameSet) pushStackFrame() { + // TODO: to array this.stackFrames.PushFront(newStackFrame()) } func (this *StackFrameSet) popStackFrame() { + // TODO: to array this.stackFrames.Remove(this.stackFrames.Front()) } @@ -218,6 +221,7 @@ func (this *StackFrameSet) defineTypedAtScope( typeName string, mlrval *types.Mlrval, ) error { + // TODO: to array return this.stackFrames.Front().Value.(*StackFrame).defineTyped( stackVariable, typeName, mlrval, ) @@ -228,6 +232,7 @@ func (this *StackFrameSet) setAtScope( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { + // TODO: to array return this.stackFrames.Front().Value.(*StackFrame).set(stackVariable, mlrval) } @@ -236,6 +241,7 @@ func (this *StackFrameSet) set( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { + // TODO: to array for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) if stackFrame.has(stackVariable) { @@ -251,6 +257,7 @@ func (this *StackFrameSet) setIndexed( indices []*types.Mlrval, mlrval *types.Mlrval, ) error { + // TODO: to array for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { stackFrame := entry.Value.(*StackFrame) if stackFrame.has(stackVariable) { From bd8ceaadc7228b197167c5ef19aefa9ae78d9a50 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 20:12:47 -0500 Subject: [PATCH 04/11] cache frameset-head in stack object --- go/src/runtime/stack.go | 52 ++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 5b113d333..83f7ca228 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -41,7 +41,14 @@ import ( // keeps stack-offset information in it that is private to us. type StackVariable struct { name string - // Type like "int" or "num" or "var" is stored in the stack itself + + // Type like "int" or "num" or "var" is stored in the stack itself. A + // StackVariable can appear in the CST (concrete syntax tree) on either the + // left-hand side or right-hande side of an assignment -- in the latter + // case the callsite won't know the type until the value is read off the + // stack. + + // TODO: comment frameSetIndex int indexInFrame int } @@ -59,25 +66,35 @@ func NewStackVariable(name string) *StackVariable { // STACK METHODS type Stack struct { - stackFrameSets *list.List // list of *StackFrameSet + // list of *StackFrameSet + stackFrameSets *list.List + + // Invariant: equal to the head of the stackFrameSets list. This is cached + // since all sets/gets in between frameset-push and frameset-pop will all + // and only be operating on the head. + head *StackFrameSet } func NewStack() *Stack { stackFrameSets := list.New() - stackFrameSets.PushFront(newStackFrameSet()) + head := newStackFrameSet() + stackFrameSets.PushFront(head) return &Stack{ stackFrameSets: stackFrameSets, + head: head, } } // For when a user-defined function/subroutine is being entered func (this *Stack) PushStackFrameSet() { - this.stackFrameSets.PushFront(newStackFrameSet()) + this.head = newStackFrameSet() + this.stackFrameSets.PushFront(this.head) } // For when a user-defined function/subroutine is being exited func (this *Stack) PopStackFrameSet() { this.stackFrameSets.Remove(this.stackFrameSets.Front()) + this.head = this.stackFrameSets.Front().Value.(*StackFrameSet) } // ---------------------------------------------------------------- @@ -85,14 +102,12 @@ func (this *Stack) PopStackFrameSet() { // For when an if/for/etc block is being entered func (this *Stack) PushStackFrame() { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.pushStackFrame() + this.head.pushStackFrame() } // For when an if/for/etc block is being exited func (this *Stack) PopStackFrame() { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.popStackFrame() + this.head.popStackFrame() } // For 'num a = 2', setting a variable at the current frame regardless of outer @@ -103,8 +118,7 @@ func (this *Stack) DefineTypedAtScope( typeName string, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.defineTypedAtScope(stackVariable, typeName, mlrval) + return this.head.defineTypedAtScope(stackVariable, typeName, mlrval) } // For untyped declarations at the current scope -- these are in binds of @@ -116,8 +130,7 @@ func (this *Stack) SetAtScope( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.setAtScope(stackVariable, mlrval) + return this.head.setAtScope(stackVariable, mlrval) } // For 'a = 2', checking for outer-scoped to maybe reuse, else insert new in @@ -131,8 +144,7 @@ func (this *Stack) Set( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.set(stackVariable, mlrval) + return this.head.set(stackVariable, mlrval) } // E.g. 'x[1] = 2' where the variable x may or may not have been already set. @@ -141,16 +153,14 @@ func (this *Stack) SetIndexed( indices []*types.Mlrval, mlrval *types.Mlrval, ) error { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.setIndexed(stackVariable, indices, mlrval) + return this.head.setIndexed(stackVariable, indices, mlrval) } // E.g. 'unset x' func (this *Stack) Unset( stackVariable *StackVariable, ) { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.unset(stackVariable) + this.head.unset(stackVariable) } // E.g. 'unset x[1]' @@ -158,16 +168,14 @@ func (this *Stack) UnsetIndexed( stackVariable *StackVariable, indices []*types.Mlrval, ) { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - head.unsetIndexed(stackVariable, indices) + this.head.unsetIndexed(stackVariable, indices) } // Returns nil on no-such func (this *Stack) Get( stackVariable *StackVariable, ) *types.Mlrval { - head := this.stackFrameSets.Front().Value.(*StackFrameSet) - return head.get(stackVariable) + return this.head.get(stackVariable) } func (this *Stack) Dump() { From 882316ef4560b2ae388038af820c8017cc2baa51 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 20:51:54 -0500 Subject: [PATCH 05/11] convert StackFrame vars from linked list to array --- go/src/runtime/stack.go | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 83f7ca228..24613490f 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -49,16 +49,16 @@ type StackVariable struct { // stack. // TODO: comment - frameSetIndex int - indexInFrame int + frameSetOffset int + offsetInFrame int } // TODO: be sure to invalidate slot 0 for struct uninit func NewStackVariable(name string) *StackVariable { return &StackVariable{ - name: name, - frameSetIndex: 0, - indexInFrame: 0, + name: name, + frameSetOffset: -1, + offsetInFrame: -1, } } @@ -98,7 +98,7 @@ func (this *Stack) PopStackFrameSet() { } // ---------------------------------------------------------------- -// Delegations to topmost frameset +// All of these are simply delegations to the head frameset // For when an if/for/etc block is being entered func (this *Stack) PushStackFrame() { @@ -320,16 +320,25 @@ func (this *StackFrameSet) get( // ================================================================ // STACKFRAME METHODS +const stackFrameInitCap = 10 + type StackFrame struct { // TODO: just a map for now. In the C impl, pre-computation of // name-to-array-slot indices was an important optimization, especially for // compute-intensive scenarios. - vars map[string]*types.TypeGatedMlrvalVariable + //vars map[string]*types.TypeGatedMlrvalVariable + + // TODO: comment + vars []*types.TypeGatedMlrvalVariable + namesToOffsets map[string]int } func newStackFrame() *StackFrame { + vars := make([]*types.TypeGatedMlrvalVariable, 0, stackFrameInitCap) + namesToOffsets := make(map[string]int) return &StackFrame{ - vars: make(map[string]*types.TypeGatedMlrvalVariable), + vars: vars, + namesToOffsets: namesToOffsets, } } @@ -337,22 +346,19 @@ func newStackFrame() *StackFrame { func (this *StackFrame) get( stackVariable *StackVariable, ) *types.Mlrval { - slot := this.vars[stackVariable.name] - if slot == nil { + offset, ok := this.namesToOffsets[stackVariable.name] + if !ok { return nil } else { - return slot.GetValue() + return this.vars[offset].GetValue() } } func (this *StackFrame) has( stackVariable *StackVariable, ) bool { - return this.vars[stackVariable.name] != nil -} - -func (this *StackFrame) clear() { - this.vars = make(map[string]*types.TypeGatedMlrvalVariable) + _, ok := this.namesToOffsets[stackVariable.name] + return ok } // TODO: audit for honor of error-return at callsites @@ -360,19 +366,18 @@ func (this *StackFrame) set( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - slot := this.vars[stackVariable.name] - if slot == nil { + offset, ok := this.namesToOffsets[stackVariable.name] + if !ok { slot, err := types.NewTypeGatedMlrvalVariable(stackVariable.name, "any", mlrval) if err != nil { return err - } else { - this.vars[stackVariable.name] = slot - return nil } - this.vars[stackVariable.name] = slot + this.vars = append(this.vars, slot) + this.namesToOffsets[stackVariable.name] = len(this.vars) - 1 return nil } else { return slot.Assign(mlrval) + return this.vars[offset].Assign(mlrval) } } @@ -382,16 +387,14 @@ func (this *StackFrame) defineTyped( typeName string, mlrval *types.Mlrval, ) error { - slot := this.vars[stackVariable.name] - if slot == nil { + _, ok := this.namesToOffsets[stackVariable.name] + if !ok { slot, err := types.NewTypeGatedMlrvalVariable(stackVariable.name, typeName, mlrval) if err != nil { return err - } else { - this.vars[stackVariable.name] = slot - return nil } - this.vars[stackVariable.name] = slot + this.vars = append(this.vars, slot) + this.namesToOffsets[stackVariable.name] = len(this.vars) - 1 return nil } else { return errors.New( @@ -406,9 +409,9 @@ func (this *StackFrame) defineTyped( func (this *StackFrame) unset( stackVariable *StackVariable, ) { - slot := this.vars[stackVariable.name] - if slot != nil { - slot.Unassign() + offset, ok := this.namesToOffsets[stackVariable.name] + if ok { + this.vars[offset].Unassign() } } From 986506a1cc1744815a5c4d617daf891c056f9a59 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 21:07:21 -0500 Subject: [PATCH 06/11] convert StackFrameSet stack-frames from linked list to array --- go/src/runtime/stack.go | 65 +++++++++++++++++++---------------- go/src/types/mlrval_typing.go | 4 +++ 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 24613490f..5ddd83296 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -189,36 +189,34 @@ func (this *Stack) Dump() { // ================================================================ // STACKFRAMESET METHODS +const stackFrameSetInitCap = 6 + type StackFrameSet struct { - stackFrames *list.List // list of *StackFrame + stackFrames []*StackFrame } func newStackFrameSet() *StackFrameSet { - // TODO: to array - stackFrames := list.New() - stackFrames.PushFront(newStackFrame()) + stackFrames := make([]*StackFrame, 1, stackFrameSetInitCap) + stackFrames[0] = newStackFrame() return &StackFrameSet{ stackFrames: stackFrames, } } func (this *StackFrameSet) pushStackFrame() { - // TODO: to array - this.stackFrames.PushFront(newStackFrame()) + this.stackFrames = append(this.stackFrames, newStackFrame()) } func (this *StackFrameSet) popStackFrame() { - // TODO: to array - this.stackFrames.Remove(this.stackFrames.Front()) + this.stackFrames = this.stackFrames[0 : len(this.stackFrames)-1] } func (this *StackFrameSet) dump() { - fmt.Printf(" STACK FRAMES (count %d):\n", this.stackFrames.Len()) - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + fmt.Printf(" STACK FRAMES (count %d):\n", len(this.stackFrames)) + for _, stackFrame := range this.stackFrames { fmt.Printf(" VARIABLES (count %d):\n", len(stackFrame.vars)) - for k, v := range stackFrame.vars { - fmt.Printf(" %-16s %s\n", k, v.ValueString()) + for _, v := range stackFrame.vars { + fmt.Printf(" %-16s %s\n", v.GetName(), v.ValueString()) } } } @@ -229,8 +227,8 @@ func (this *StackFrameSet) defineTypedAtScope( typeName string, mlrval *types.Mlrval, ) error { - // TODO: to array - return this.stackFrames.Front().Value.(*StackFrame).defineTyped( + offset := len(this.stackFrames) - 1 + return this.stackFrames[offset].defineTyped( stackVariable, typeName, mlrval, ) } @@ -240,8 +238,8 @@ func (this *StackFrameSet) setAtScope( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - // TODO: to array - return this.stackFrames.Front().Value.(*StackFrame).set(stackVariable, mlrval) + offset := len(this.stackFrames) - 1 + return this.stackFrames[offset].set(stackVariable, mlrval) } // See Stack.Set comments above @@ -249,9 +247,10 @@ func (this *StackFrameSet) set( stackVariable *StackVariable, mlrval *types.Mlrval, ) error { - // TODO: to array - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] if stackFrame.has(stackVariable) { return stackFrame.set(stackVariable, mlrval) } @@ -265,22 +264,25 @@ func (this *StackFrameSet) setIndexed( indices []*types.Mlrval, mlrval *types.Mlrval, ) error { - // TODO: to array - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] if stackFrame.has(stackVariable) { return stackFrame.setIndexed(stackVariable, indices, mlrval) } } - return this.stackFrames.Front().Value.(*StackFrame).setIndexed(stackVariable, indices, mlrval) + return this.stackFrames[numStackFrames-1].setIndexed(stackVariable, indices, mlrval) } // See Stack.Unset comments above func (this *StackFrameSet) unset( stackVariable *StackVariable, ) { - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] if stackFrame.has(stackVariable) { stackFrame.unset(stackVariable) return @@ -293,8 +295,10 @@ func (this *StackFrameSet) unsetIndexed( stackVariable *StackVariable, indices []*types.Mlrval, ) { - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] if stackFrame.has(stackVariable) { stackFrame.unsetIndexed(stackVariable, indices) return @@ -307,8 +311,9 @@ func (this *StackFrameSet) get( stackVariable *StackVariable, ) *types.Mlrval { // Scope-walk - for entry := this.stackFrames.Front(); entry != nil; entry = entry.Next() { - stackFrame := entry.Value.(*StackFrame) + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] mlrval := stackFrame.get(stackVariable) if mlrval != nil { return mlrval diff --git a/go/src/types/mlrval_typing.go b/go/src/types/mlrval_typing.go index 36204fc27..ec22f958a 100644 --- a/go/src/types/mlrval_typing.go +++ b/go/src/types/mlrval_typing.go @@ -77,6 +77,10 @@ func NewTypeGatedMlrvalVariable( }, nil } +func (this *TypeGatedMlrvalVariable) GetName() string { + return this.typeGatedMlrvalName.Name +} + func (this *TypeGatedMlrvalVariable) GetValue() *Mlrval { return this.value } From 780f8cca4a8a358514d0651ba1f271e417c41216 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 21:12:29 -0500 Subject: [PATCH 07/11] neaten --- go/src/runtime/stack.go | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 5ddd83296..16ae94a56 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -110,6 +110,13 @@ func (this *Stack) PopStackFrame() { this.head.popStackFrame() } +// Returns nil on no-such +func (this *Stack) Get( + stackVariable *StackVariable, +) *types.Mlrval { + return this.head.get(stackVariable) +} + // For 'num a = 2', setting a variable at the current frame regardless of outer // scope. It's an error to define it again in the same scope, whether the type // is the same or not. @@ -171,13 +178,6 @@ func (this *Stack) UnsetIndexed( this.head.unsetIndexed(stackVariable, indices) } -// Returns nil on no-such -func (this *Stack) Get( - stackVariable *StackVariable, -) *types.Mlrval { - return this.head.get(stackVariable) -} - func (this *Stack) Dump() { fmt.Printf("STACK FRAMESETS (count %d):\n", this.stackFrameSets.Len()) for entry := this.stackFrameSets.Front(); entry != nil; entry = entry.Next() { @@ -221,6 +221,22 @@ func (this *StackFrameSet) dump() { } } +// Returns nil on no-such +func (this *StackFrameSet) get( + stackVariable *StackVariable, +) *types.Mlrval { + // Scope-walk + numStackFrames := len(this.stackFrames) + for offset := numStackFrames - 1; offset >= 0; offset-- { + stackFrame := this.stackFrames[offset] + mlrval := stackFrame.get(stackVariable) + if mlrval != nil { + return mlrval + } + } + return nil +} + // See Stack.DefineTypedAtScope comments above func (this *StackFrameSet) defineTypedAtScope( stackVariable *StackVariable, @@ -306,22 +322,6 @@ func (this *StackFrameSet) unsetIndexed( } } -// Returns nil on no-such -func (this *StackFrameSet) get( - stackVariable *StackVariable, -) *types.Mlrval { - // Scope-walk - numStackFrames := len(this.stackFrames) - for offset := numStackFrames - 1; offset >= 0; offset-- { - stackFrame := this.stackFrames[offset] - mlrval := stackFrame.get(stackVariable) - if mlrval != nil { - return mlrval - } - } - return nil -} - // ================================================================ // STACKFRAME METHODS @@ -411,15 +411,6 @@ func (this *StackFrame) defineTyped( } } -func (this *StackFrame) unset( - stackVariable *StackVariable, -) { - offset, ok := this.namesToOffsets[stackVariable.name] - if ok { - this.vars[offset].Unassign() - } -} - // TODO: audit for honor of error-return at callsites func (this *StackFrame) setIndexed( stackVariable *StackVariable, @@ -449,6 +440,15 @@ func (this *StackFrame) setIndexed( } } +func (this *StackFrame) unset( + stackVariable *StackVariable, +) { + offset, ok := this.namesToOffsets[stackVariable.name] + if ok { + this.vars[offset].Unassign() + } +} + func (this *StackFrame) unsetIndexed( stackVariable *StackVariable, indices []*types.Mlrval, From f3d3c1ea29f85d414f2bbcfe2fc7200e41d449f2 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 17 Feb 2021 21:33:56 -0500 Subject: [PATCH 08/11] cache frameset/frame offsets in StackVariable on get/set --- go/src/runtime/stack.go | 61 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 16ae94a56..95a998120 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -224,6 +224,20 @@ func (this *StackFrameSet) dump() { // Returns nil on no-such func (this *StackFrameSet) get( stackVariable *StackVariable, +) *types.Mlrval { + // TODO: comment + fso := stackVariable.frameSetOffset + oif := stackVariable.offsetInFrame + if fso >= 0 && oif > 0 { + return this.stackFrames[fso].vars[oif].GetValue() + } else { + return this.getUncached(stackVariable) + } +} + +// Returns nil on no-such +func (this *StackFrameSet) getUncached( + stackVariable *StackVariable, ) *types.Mlrval { // Scope-walk numStackFrames := len(this.stackFrames) @@ -231,6 +245,8 @@ func (this *StackFrameSet) get( stackFrame := this.stackFrames[offset] mlrval := stackFrame.get(stackVariable) if mlrval != nil { + // TODO: comment + stackVariable.frameSetOffset = offset return mlrval } } @@ -244,6 +260,8 @@ func (this *StackFrameSet) defineTypedAtScope( mlrval *types.Mlrval, ) error { offset := len(this.stackFrames) - 1 + // TODO: comment + stackVariable.frameSetOffset = offset return this.stackFrames[offset].defineTyped( stackVariable, typeName, mlrval, ) @@ -255,6 +273,8 @@ func (this *StackFrameSet) setAtScope( mlrval *types.Mlrval, ) error { offset := len(this.stackFrames) - 1 + // TODO: comment + stackVariable.frameSetOffset = offset return this.stackFrames[offset].set(stackVariable, mlrval) } @@ -262,12 +282,28 @@ func (this *StackFrameSet) setAtScope( func (this *StackFrameSet) set( stackVariable *StackVariable, mlrval *types.Mlrval, +) error { + fso := stackVariable.frameSetOffset + oif := stackVariable.offsetInFrame + if fso >= 0 && oif > 0 { + return this.stackFrames[fso].vars[oif].Assign(mlrval.Copy()) + } else { + return this.setUncached(stackVariable, mlrval) + } +} + +// See Stack.Set comments above +func (this *StackFrameSet) setUncached( + stackVariable *StackVariable, + mlrval *types.Mlrval, ) error { // Scope-walk numStackFrames := len(this.stackFrames) for offset := numStackFrames - 1; offset >= 0; offset-- { stackFrame := this.stackFrames[offset] if stackFrame.has(stackVariable) { + // TODO: comment + stackVariable.frameSetOffset = offset return stackFrame.set(stackVariable, mlrval) } } @@ -285,10 +321,15 @@ func (this *StackFrameSet) setIndexed( for offset := numStackFrames - 1; offset >= 0; offset-- { stackFrame := this.stackFrames[offset] if stackFrame.has(stackVariable) { + // TODO: comment + stackVariable.frameSetOffset = offset return stackFrame.setIndexed(stackVariable, indices, mlrval) } } - return this.stackFrames[numStackFrames-1].setIndexed(stackVariable, indices, mlrval) + // TODO: comment + offset := numStackFrames - 1 + stackVariable.frameSetOffset = offset + return this.stackFrames[offset].setIndexed(stackVariable, indices, mlrval) } // See Stack.Unset comments above @@ -352,10 +393,12 @@ func (this *StackFrame) get( stackVariable *StackVariable, ) *types.Mlrval { offset, ok := this.namesToOffsets[stackVariable.name] - if !ok { - return nil - } else { + if ok { + // TODO: comment + stackVariable.offsetInFrame = offset return this.vars[offset].GetValue() + } else { + return nil } } @@ -378,7 +421,10 @@ func (this *StackFrame) set( return err } this.vars = append(this.vars, slot) - this.namesToOffsets[stackVariable.name] = len(this.vars) - 1 + offsetInFrame := len(this.vars) - 1 + this.namesToOffsets[stackVariable.name] = offsetInFrame + // TODO: comment + stackVariable.offsetInFrame = offsetInFrame return nil } else { return slot.Assign(mlrval) @@ -399,7 +445,10 @@ func (this *StackFrame) defineTyped( return err } this.vars = append(this.vars, slot) - this.namesToOffsets[stackVariable.name] = len(this.vars) - 1 + offsetInFrame := len(this.vars) - 1 + this.namesToOffsets[stackVariable.name] = offsetInFrame + // TODO: comment + stackVariable.offsetInFrame = offsetInFrame return nil } else { return errors.New( From d442067dbdac8f7c32e396c7b70d7761ed2560ac Mon Sep 17 00:00:00 2001 From: John Kerl Date: Fri, 19 Feb 2021 09:39:47 -0500 Subject: [PATCH 09/11] perf-debug [WIP] --- go/gc-raw-notes.txt | 51 +++++++++++++++++++++++++++++++++++++++++ go/mlr.go | 3 --- go/src/runtime/stack.go | 5 ++-- go/todo.txt | 20 ++++++++++++++++ go/u/mand.mlr | 11 --------- 5 files changed, 73 insertions(+), 17 deletions(-) diff --git a/go/gc-raw-notes.txt b/go/gc-raw-notes.txt index b3ca5717b..1f4e5a166 100644 --- a/go/gc-raw-notes.txt +++ b/go/gc-raw-notes.txt @@ -17,6 +17,10 @@ - c 3.2s - go pre-alloccate 45s +Why 5MB goal with GOGC=100???? + +GOGC=100000 GODEBUG=gctrace=1 mlr -n put -q -f u/mand.mlr 1> /dev/null + * u/mand.mlr silent option https://blog.twitch.tv/en/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap-26c2462549a2/ @@ -46,3 +50,50 @@ https://github.com/golang/go/issues/23044 i https://hub.packtpub.com/implementing-memory-management-with-golang-garbage-collector/ i mlr --cpuprofile cpu.pprof -n put -q -s iheight=100 -s iwidth=100 -f u/mand.mlr > /dev/null + +? SetGCPercent +? SetMaxHeap + +mlr --cpuprofile cpu.pprof -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null +go tool pprof mlr cpu.pprof +top10 + +go tool pprof --pdf mlr cpu.pprof > mlr-call-graph.pdf +mv mlr-call-graph.pdf ~/Desktop + +runtime.duffcopy + https://stackoverflow.com/questions/45786687/runtime-duffcopy-is-called-a-lot +runtime.madvise + +GOGC=off +GODEBUG=gctrace=1 + +export PATH=${PATH}:~/git/brendangregg/FlameGraph/ +go-torch cpu.pprof +mv torch.svg ~/Desktop/ + +i mlr --cpuprofile cpu.pprof -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null + +i https://hub.packtpub.com/implementing-memory-management-with-golang-garbage-collector/ + +i https://golang.org/pkg/runtime/ + +! flame-graph readme; & profile-readme out of mlr.go & into separate .md file +i mlr --cpuprofile cpu.pprof -n put -q -s iheight=100 -s iwidth=100 -f u/mand.mlr > /dev/null +i GODEBUG=gctrace=1 mlr -n put -q -s iheight=500 -s iwidth=500 -f u/mand.mlr > /dev/null| head -n 100 + gc 1 @0.129s 1%: 0.012+3.9+0.002 ms clock, 0.048+3.7/3.7/7.5+0.010 ms cpu, 10240->10240->0 MB, 10241 MB goal, 4 P + gc 2 @0.140s 2%: 0.009+2.1+0.002 ms clock, 0.038+2.0/2.0/4.0+0.011 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + gc 3 @0.149s 2%: 0.032+2.1+0.021 ms clock, 0.12+2.0/2.0/4.0+0.087 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + gc 4 @0.157s 3%: 0.035+2.1+0.014 ms clock, 0.14+2.0/1.9/4.1+0.059 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + gc 5 @0.166s 3%: 0.034+2.2+0.024 ms clock, 0.13+2.0/1.9/4.0+0.098 ms cpu, 4->4->0 MB, 5 MB goal, 4 P + +mem.Alloc: 176104 +mem.TotalAlloc: 176104 +mem.HeapAlloc: 176104 +mem.NumGC: 0 + +mem.Alloc: 1179440 +mem.TotalAlloc: 16,529,643,664 +mem.HeapAlloc: 1179440 +mem.NumGC: 4254 + diff --git a/go/mlr.go b/go/mlr.go index b0180d613..a41a84ef6 100644 --- a/go/mlr.go +++ b/go/mlr.go @@ -51,9 +51,6 @@ func main() { // found then this function will not return. auxents.Dispatch(os.Args) - // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // Start of Miller main per se - options, recordTransformers, err := cli.ParseCommandLine(os.Args) if err != nil { fmt.Fprintln(os.Stderr, os.Args[0], ": ", err) diff --git a/go/src/runtime/stack.go b/go/src/runtime/stack.go index 95a998120..e1ece25e7 100644 --- a/go/src/runtime/stack.go +++ b/go/src/runtime/stack.go @@ -228,7 +228,7 @@ func (this *StackFrameSet) get( // TODO: comment fso := stackVariable.frameSetOffset oif := stackVariable.offsetInFrame - if fso >= 0 && oif > 0 { + if fso >= 0 && fso < len(this.stackFrames) && oif >= 0 && oif < len(this.stackFrames[fso].vars) { return this.stackFrames[fso].vars[oif].GetValue() } else { return this.getUncached(stackVariable) @@ -285,7 +285,7 @@ func (this *StackFrameSet) set( ) error { fso := stackVariable.frameSetOffset oif := stackVariable.offsetInFrame - if fso >= 0 && oif > 0 { + if fso >= 0 && fso < len(this.stackFrames) && oif >= 0 && oif < len(this.stackFrames[fso].vars) { return this.stackFrames[fso].vars[oif].Assign(mlrval.Copy()) } else { return this.setUncached(stackVariable, mlrval) @@ -427,7 +427,6 @@ func (this *StackFrame) set( stackVariable.offsetInFrame = offsetInFrame return nil } else { - return slot.Assign(mlrval) return this.vars[offset].Assign(mlrval) } } diff --git a/go/todo.txt b/go/todo.txt index 6c9110c90..1dfd6f7a1 100644 --- a/go/todo.txt +++ b/go/todo.txt @@ -295,6 +295,26 @@ GOCC UPSTREAMS: * support "abc" (not just 'a' 'b' 'c') in the lexer part ---------------------------------------------------------------- +<<<<<<< HEAD +======= +FLAME GRAPHS + +go get -u github.com/google/pprof +ll ~/go/bin/pprof +go get -u github.com/uber/go-torch + +cd ~/git +mkdir brendangregg +cd brendangregg +git clone https://github.com/brendangregg/FlameGraph + +cd /path/to/mlr/go +export PATH=${PATH}:~/git/brendangregg/FlameGraph/ +go-torch cpu.pprof +mv torch.svg ~/Desktop/ + +---------------------------------------------------------------- +>>>>>>> cb2647ab... perf-debug [WIP] NITS/NON-IMMEDIATE: * "Miller: " prefixes on all errors.New diff --git a/go/u/mand.mlr b/go/u/mand.mlr index 60c6b6256..ebe5423bd 100644 --- a/go/u/mand.mlr +++ b/go/u/mand.mlr @@ -12,17 +12,6 @@ begin { @silent ??= false; } -# Override defaults -@rcorn = $rcorn; -@icorn = $icorn; -@side = $side; -@iheight = $iheight; -@iwidth = $iwidth; -@maxits = $maxits; -@levelstep = $levelstep; -@chars = $chars; -@silent = $silent; - end { if (!@silent) { print "RCORN = ".@rcorn; From b071f6b1ed15f16d3fb4286c6db936a901f4ec7d Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 22 Feb 2021 01:31:22 -0500 Subject: [PATCH 10/11] merge --- go/src/dsl/cst/leaves.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/src/dsl/cst/leaves.go b/go/src/dsl/cst/leaves.go index 11e829269..137775d8a 100644 --- a/go/src/dsl/cst/leaves.go +++ b/go/src/dsl/cst/leaves.go @@ -184,7 +184,7 @@ func (this *RootNode) BuildLocalVariableNode(variableName string) *LocalVariable func (this *LocalVariableNode) Evaluate( state *runtime.State, ) *types.Mlrval { - value := state.Stack.Get(this.variableName) + value := state.Stack.Get(this.stackVariable) if value == nil { return types.MLRVAL_ABSENT } else { From 5dbfb639ae8deac684687e0e4356fe4de37cacf7 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Mon, 22 Feb 2021 23:10:39 -0500 Subject: [PATCH 11/11] merge --- go/src/dsl/cst/for.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/src/dsl/cst/for.go b/go/src/dsl/cst/for.go index 970f8f553..e91fcd758 100644 --- a/go/src/dsl/cst/for.go +++ b/go/src/dsl/cst/for.go @@ -510,7 +510,7 @@ func (this *ForLoopMultivariableNode) Execute(state *runtime.State) (*BlockExitP // from any of the latter is a break from all. However, at this point, the // break has been "broken" and should not be returned to the caller. // Return-statements should, though. - blockExitPayload, err := this.executeOuter(indexMlrval, this.keyVariableNames, state) + blockExitPayload, err := this.executeOuter(indexMlrval, this.keyIndexVariables, state) if blockExitPayload == nil { return nil, err } else {