From f00fa1cc7e8b6975eae2efefc74527a98abddf3c Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 3 Dec 2020 08:31:44 -0500 Subject: [PATCH] missed a spot in the mapper->transformer bulk-rename --- go/src/miller/transformers/altkv.go | 2 +- go/src/miller/transformers/bootstrap.go | 2 +- go/src/miller/transformers/cat.go | 2 +- go/src/miller/transformers/check.go | 2 +- go/src/miller/transformers/count-similar.go | 2 +- go/src/miller/transformers/count.go | 2 +- go/src/miller/transformers/cut.go | 2 +- go/src/miller/transformers/decimate.go | 2 +- go/src/miller/transformers/fill-down.go | 2 +- go/src/miller/transformers/gap.go | 2 +- go/src/miller/transformers/grep.go | 2 +- go/src/miller/transformers/group-by.go | 2 +- go/src/miller/transformers/group-like.go | 2 +- go/src/miller/transformers/head.go | 2 +- go/src/miller/transformers/label.go | 2 +- go/src/miller/transformers/nothing.go | 2 +- go/src/miller/transformers/put_or_filter.go | 2 +- go/src/miller/transformers/regularize.go | 2 +- .../transformers/remove-empty-columns.go | 2 +- go/src/miller/transformers/rename.go | 2 +- go/src/miller/transformers/reorder.go | 2 +- go/src/miller/transformers/sample.go | 2 +- go/src/miller/transformers/sec2gmt.go | 2 +- go/src/miller/transformers/seqgen.go | 2 +- go/src/miller/transformers/shuffle.go | 2 +- .../transformers/skip-trivial-records.go | 2 +- .../transformers/sort-within-records.go | 2 +- go/src/miller/transformers/sort.go | 2 +- go/src/miller/transformers/step.go | 348 +++++++++--------- go/src/miller/transformers/tac.go | 2 +- go/src/miller/transformers/tail.go | 2 +- go/src/miller/transformers/unsparsify.go | 2 +- .../miller/transforming/chain_transformer.go | 2 +- .../miller/transforming/record_transformer.go | 2 +- go/src/miller/types/mlrmap_accessors.go | 17 + 35 files changed, 225 insertions(+), 206 deletions(-) diff --git a/go/src/miller/transformers/altkv.go b/go/src/miller/transformers/altkv.go index 0c76fe86a..182000634 100644 --- a/go/src/miller/transformers/altkv.go +++ b/go/src/miller/transformers/altkv.go @@ -81,7 +81,7 @@ func NewTransformerAltkv() (*TransformerAltkv, error) { } // ---------------------------------------------------------------- -func (this *TransformerAltkv) Map( +func (this *TransformerAltkv) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/bootstrap.go b/go/src/miller/transformers/bootstrap.go index 031a8f595..ffbaad092 100644 --- a/go/src/miller/transformers/bootstrap.go +++ b/go/src/miller/transformers/bootstrap.go @@ -102,7 +102,7 @@ func NewTransformerBootstrap(nout int) (*TransformerBootstrap, error) { } // ---------------------------------------------------------------- -func (this *TransformerBootstrap) Map( +func (this *TransformerBootstrap) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/cat.go b/go/src/miller/transformers/cat.go index 6e087f84e..7868b9fe8 100644 --- a/go/src/miller/transformers/cat.go +++ b/go/src/miller/transformers/cat.go @@ -145,7 +145,7 @@ func NewTransformerCat( } // ---------------------------------------------------------------- -func (this *TransformerCat) Map( +func (this *TransformerCat) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/check.go b/go/src/miller/transformers/check.go index ac1f93d6e..6bca23100 100644 --- a/go/src/miller/transformers/check.go +++ b/go/src/miller/transformers/check.go @@ -80,7 +80,7 @@ func NewTransformerCheck() (*TransformerCheck, error) { return &TransformerCheck{}, nil } -func (this *TransformerCheck) Map( +func (this *TransformerCheck) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/count-similar.go b/go/src/miller/transformers/count-similar.go index 3e2f02e62..2ac2dec34 100644 --- a/go/src/miller/transformers/count-similar.go +++ b/go/src/miller/transformers/count-similar.go @@ -122,7 +122,7 @@ func NewTransformerCountSimilar( } // ---------------------------------------------------------------- -func (this *TransformerCountSimilar) Map( +func (this *TransformerCountSimilar) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/count.go b/go/src/miller/transformers/count.go index 1e81ae05a..6621aaac9 100644 --- a/go/src/miller/transformers/count.go +++ b/go/src/miller/transformers/count.go @@ -149,7 +149,7 @@ func NewTransformerCount( } // ---------------------------------------------------------------- -func (this *TransformerCount) Map( +func (this *TransformerCount) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/cut.go b/go/src/miller/transformers/cut.go index 5819be586..25efaaf57 100644 --- a/go/src/miller/transformers/cut.go +++ b/go/src/miller/transformers/cut.go @@ -180,7 +180,7 @@ func NewTransformerCut( // } // ---------------------------------------------------------------- -func (this *TransformerCut) Map( +func (this *TransformerCut) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/decimate.go b/go/src/miller/transformers/decimate.go index 2b3de7ba2..695146557 100644 --- a/go/src/miller/transformers/decimate.go +++ b/go/src/miller/transformers/decimate.go @@ -139,7 +139,7 @@ func NewTransformerDecimate( } // ---------------------------------------------------------------- -func (this *TransformerDecimate) Map( +func (this *TransformerDecimate) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/fill-down.go b/go/src/miller/transformers/fill-down.go index c3a115e9d..14de73953 100644 --- a/go/src/miller/transformers/fill-down.go +++ b/go/src/miller/transformers/fill-down.go @@ -121,7 +121,7 @@ func NewTransformerFillDown( } // ---------------------------------------------------------------- -func (this *TransformerFillDown) Map( +func (this *TransformerFillDown) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/gap.go b/go/src/miller/transformers/gap.go index 444ca0934..aab396413 100644 --- a/go/src/miller/transformers/gap.go +++ b/go/src/miller/transformers/gap.go @@ -129,7 +129,7 @@ func NewTransformerGap( } // ---------------------------------------------------------------- -func (this *TransformerGap) Map( +func (this *TransformerGap) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/grep.go b/go/src/miller/transformers/grep.go index 62ca93ab3..33e267010 100644 --- a/go/src/miller/transformers/grep.go +++ b/go/src/miller/transformers/grep.go @@ -140,7 +140,7 @@ func NewTransformerGrep( } // ---------------------------------------------------------------- -func (this *TransformerGrep) Map( +func (this *TransformerGrep) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/group-by.go b/go/src/miller/transformers/group-by.go index 3f64b4147..2fc5f55c4 100644 --- a/go/src/miller/transformers/group-by.go +++ b/go/src/miller/transformers/group-by.go @@ -110,7 +110,7 @@ func NewTransformerGroupBy( } // ---------------------------------------------------------------- -func (this *TransformerGroupBy) Map( +func (this *TransformerGroupBy) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/group-like.go b/go/src/miller/transformers/group-like.go index 112ef5cc1..4ca1b5a30 100644 --- a/go/src/miller/transformers/group-like.go +++ b/go/src/miller/transformers/group-like.go @@ -90,7 +90,7 @@ func NewTransformerGroupLike() (*TransformerGroupLike, error) { } // ---------------------------------------------------------------- -func (this *TransformerGroupLike) Map( +func (this *TransformerGroupLike) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/head.go b/go/src/miller/transformers/head.go index 2858d0910..81387c1a7 100644 --- a/go/src/miller/transformers/head.go +++ b/go/src/miller/transformers/head.go @@ -135,7 +135,7 @@ func NewTransformerHead( } // ---------------------------------------------------------------- -func (this *TransformerHead) Map( +func (this *TransformerHead) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/label.go b/go/src/miller/transformers/label.go index d63587e97..f9223a6a0 100644 --- a/go/src/miller/transformers/label.go +++ b/go/src/miller/transformers/label.go @@ -123,7 +123,7 @@ func NewTransformerLabel( } // ---------------------------------------------------------------- -func (this *TransformerLabel) Map( +func (this *TransformerLabel) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/nothing.go b/go/src/miller/transformers/nothing.go index eaba4475a..d52b3bb9e 100644 --- a/go/src/miller/transformers/nothing.go +++ b/go/src/miller/transformers/nothing.go @@ -79,7 +79,7 @@ func NewTransformerNothing() (*TransformerNothing, error) { return &TransformerNothing{}, nil } -func (this *TransformerNothing) Map( +func (this *TransformerNothing) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/put_or_filter.go b/go/src/miller/transformers/put_or_filter.go index 090ee4ee9..76f0e4568 100644 --- a/go/src/miller/transformers/put_or_filter.go +++ b/go/src/miller/transformers/put_or_filter.go @@ -327,7 +327,7 @@ func BuildASTFromString(dslString string) (*dsl.AST, error) { return astRootNode, nil } -func (this *TransformerPut) Map( +func (this *TransformerPut) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/regularize.go b/go/src/miller/transformers/regularize.go index 0f282acf3..57e4fefa3 100644 --- a/go/src/miller/transformers/regularize.go +++ b/go/src/miller/transformers/regularize.go @@ -88,7 +88,7 @@ func NewTransformerRegularize() (*TransformerRegularize, error) { } // ---------------------------------------------------------------- -func (this *TransformerRegularize) Map( +func (this *TransformerRegularize) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/remove-empty-columns.go b/go/src/miller/transformers/remove-empty-columns.go index c635793a7..1795009be 100644 --- a/go/src/miller/transformers/remove-empty-columns.go +++ b/go/src/miller/transformers/remove-empty-columns.go @@ -86,7 +86,7 @@ func NewTransformerRemoveEmptyColumns() (*TransformerRemoveEmptyColumns, error) } // ---------------------------------------------------------------- -func (this *TransformerRemoveEmptyColumns) Map( +func (this *TransformerRemoveEmptyColumns) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/rename.go b/go/src/miller/transformers/rename.go index 02aa42dd9..8c3fef27b 100644 --- a/go/src/miller/transformers/rename.go +++ b/go/src/miller/transformers/rename.go @@ -115,7 +115,7 @@ func NewTransformerRename( } // ---------------------------------------------------------------- -func (this *TransformerRename) Map( +func (this *TransformerRename) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/reorder.go b/go/src/miller/transformers/reorder.go index ba3b4e0a3..30c525cb9 100644 --- a/go/src/miller/transformers/reorder.go +++ b/go/src/miller/transformers/reorder.go @@ -132,7 +132,7 @@ func NewTransformerReorder( } // ---------------------------------------------------------------- -func (this *TransformerReorder) Map( +func (this *TransformerReorder) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/sample.go b/go/src/miller/transformers/sample.go index 30257ef6f..ba0ee4412 100644 --- a/go/src/miller/transformers/sample.go +++ b/go/src/miller/transformers/sample.go @@ -122,7 +122,7 @@ func NewTransformerSample( } // ---------------------------------------------------------------- -func (this *TransformerSample) Map( +func (this *TransformerSample) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/sec2gmt.go b/go/src/miller/transformers/sec2gmt.go index 111a7afde..061009f87 100644 --- a/go/src/miller/transformers/sec2gmt.go +++ b/go/src/miller/transformers/sec2gmt.go @@ -137,7 +137,7 @@ func NewTransformerSec2GMT( } // ---------------------------------------------------------------- -func (this *TransformerSec2GMT) Map( +func (this *TransformerSec2GMT) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/seqgen.go b/go/src/miller/transformers/seqgen.go index 5787211c7..4996fa47b 100644 --- a/go/src/miller/transformers/seqgen.go +++ b/go/src/miller/transformers/seqgen.go @@ -185,7 +185,7 @@ func NewTransformerSeqgen( } // ---------------------------------------------------------------- -func (this *TransformerSeqgen) Map( +func (this *TransformerSeqgen) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/shuffle.go b/go/src/miller/transformers/shuffle.go index 4584a2ee2..ab2d2101d 100644 --- a/go/src/miller/transformers/shuffle.go +++ b/go/src/miller/transformers/shuffle.go @@ -91,7 +91,7 @@ func NewTransformerShuffle() (*TransformerShuffle, error) { } // ---------------------------------------------------------------- -func (this *TransformerShuffle) Map( +func (this *TransformerShuffle) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/skip-trivial-records.go b/go/src/miller/transformers/skip-trivial-records.go index f3f25b1d7..962388156 100644 --- a/go/src/miller/transformers/skip-trivial-records.go +++ b/go/src/miller/transformers/skip-trivial-records.go @@ -81,7 +81,7 @@ func NewTransformerSkipTrivialRecords() (*TransformerSkipTrivialRecords, error) } // ---------------------------------------------------------------- -func (this *TransformerSkipTrivialRecords) Map( +func (this *TransformerSkipTrivialRecords) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/sort-within-records.go b/go/src/miller/transformers/sort-within-records.go index 71656aad4..f8e83f1ee 100644 --- a/go/src/miller/transformers/sort-within-records.go +++ b/go/src/miller/transformers/sort-within-records.go @@ -87,7 +87,7 @@ func NewTransformerSortWithinRecords() (*TransformerSortWithinRecords, error) { } // ---------------------------------------------------------------- -func (this *TransformerSortWithinRecords) Map( +func (this *TransformerSortWithinRecords) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/sort.go b/go/src/miller/transformers/sort.go index e06c0761c..5f1d29436 100644 --- a/go/src/miller/transformers/sort.go +++ b/go/src/miller/transformers/sort.go @@ -246,7 +246,7 @@ type GroupingKeysAndMlrvals struct { mlrvals []types.Mlrval } -func (this *TransformerSort) Map( +func (this *TransformerSort) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/step.go b/go/src/miller/transformers/step.go index 7b420a649..ad06283d6 100644 --- a/go/src/miller/transformers/step.go +++ b/go/src/miller/transformers/step.go @@ -161,40 +161,21 @@ func transformerStepUsage( } // ---------------------------------------------------------------- -// Multilevel hashmap structure: -// { -// ["s","t"] : { <--- group-by field names -// ["x","y"] : { <--- value field names -// "corr" : C stats2_corr_t object, -// "cov" : C stats2_cov_t object -// } -// }, -// ["u","v"] : { -// ["x","y"] : { -// "corr" : C stats2_corr_t object, -// "cov" : C stats2_cov_t object -// } -// }, -// ["u","w"] : { -// ["x","y"] : { -// "corr" : C stats2_corr_t object, -// "cov" : C stats2_cov_t object -// } -// }, -// } - type tStepper struct { } type TransformerStep struct { // Input: + stepperNames []string valueFieldNames []string + groupByFieldNames []string // State: valueFieldValues []types.Mlrval // scratch space used per-record - // Map from group-by field names to value-field names to array of stepper objects + // Map from group-by field names to value-field names to array of stepper objects. + // See the Transform method below for more details. groups map[string]map[string][]tStepper } @@ -224,24 +205,133 @@ func NewTransformerStep( // TODO: flesh out this := &TransformerStep{ + stepperNames: stepperNames, valueFieldNames: valueFieldNames, + groupByFieldNames: groupByFieldNames, } return this, nil } // ---------------------------------------------------------------- -func (this *TransformerStep) Map( +// Multilevel hashmap structure: +// { +// ["s","t"] : { <--- group-by field names +// ["x","y"] : { <--- value field names +// "corr" : C stats2_corr_t object, +// "cov" : C stats2_cov_t object +// } +// }, +// ["u","v"] : { +// ["x","y"] : { +// "corr" : C stats2_corr_t object, +// "cov" : C stats2_cov_t object +// } +// }, +// ["u","w"] : { +// ["x","y"] : { +// "corr" : C stats2_corr_t object, +// "cov" : C stats2_cov_t object +// } +// }, +// } + +func (this *TransformerStep) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { inrec := inrecAndContext.Record - if inrec != nil { // not end of record stream - // TODO: flesh out - outputChannel <- inrecAndContext - } else { + + if inrec == nil { // end of record stream outputChannel <- inrecAndContext } + + // ["s", "t"] + groupingKey, groupByFieldValues, gok := inrec.GetSelectedValuesAndJoined(this.groupByFieldNames) + if !gok { // current record doesn't have fields to be stepped; pass it along + outputChannel <- inrecAndContext + return + } + + // ["x", "y"] + valueFieldValues, vok := inrec.GetSelectedValues(this.valueFieldNames) + if !vok { // current record doesn't have fields to be stepped; pass it along + outputChannel <- inrecAndContext + return + } + + groupToAccField := this.groups[groupingKey] + if groupToAccField == nil { + // Populate the groups data structure on first reference if needed + groupToAccField := make(map[string][]tStepper) + this.groups[groupingKey] = groupToAccField + } + + // for x=1 and y=2: + for i, valueFieldName := range this.valueFieldNames { + // TODO: make it sparse in the GetSelectedValues() ... no `vok` return ... + valueFieldValue := valueFieldValues[i] + // char* value_field_sval = pstate->pvalue_field_values->strings[i]; + // if (value_field_sval == NULL) // Key not present + // continue; + + // int have_dval = FALSE; + // int have_nval = FALSE; + // double value_field_dval = -999.0; + // mv_t value_field_nval = mv_absent(); + // + // lhmsv_t* pacc_field_to_acc_state = lhmsv_get(pgroup_to_acc_field, value_field_name); + // if (pacc_field_to_acc_state == NULL) { + // pacc_field_to_acc_state = lhmsv_alloc(); + // lhmsv_put(pgroup_to_acc_field, value_field_name, pacc_field_to_acc_state, NO_FREE); + // } + + // for "delta", "rsum" + for _, stepperName := range this.stepperNames { + // tStepper* stepper = lhmsv_get(pacc_field_to_acc_state, stepperName); + // if (stepper == NULL) { + // stepper = make_step(stepperName, value_field_name, pstate->allow_int_float, + // pstate->pstring_alphas, pstate->pewma_suffixes); + // if (stepper == NULL) { + // fprintf(stderr, "mlr step: stepper \"%s\" not found.\n", + // stepperName); + // exit(1); + // } + // lhmsv_put(pacc_field_to_acc_state, stepperName, stepper, NO_FREE); + // } + // + // if (*value_field_sval == 0) { // Key present with null value + // if (stepper->pzprocess_func != NULL) { + // stepper->pzprocess_func(stepper->pvstate, pinrec); + // } + // } else { + // + // if (stepper->pdprocess_func != NULL) { + // if (!have_dval) { + // value_field_dval = mlr_double_from_string_or_die(value_field_sval); + // have_dval = TRUE; + // } + // stepper->pdprocess_func(stepper->pvstate, value_field_dval, pinrec); + // } + // + // if (stepper->pnprocess_func != NULL) { + // if (!have_nval) { + // value_field_nval = pstate->allow_int_float + // ? mv_scan_number_or_die(value_field_sval) + // : mv_from_float(mlr_double_from_string_or_die(value_field_sval)); + // have_nval = TRUE; + // } + // stepper->pnprocess_func(stepper->pvstate, &value_field_nval, pinrec); + // } + // + // if (stepper->psprocess_func != NULL) { + // stepper->psprocess_func(stepper->pvstate, value_field_sval, pinrec); + // } + // } + } + } + + outputChannel <- inrecAndContext } // ---------------------------------------------------------------- @@ -262,95 +352,7 @@ func (this *TransformerStep) Map( //}; // ---------------------------------------------------------------- -//// ---------------------------------------------------------------- -//static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstate) { -// mapper_step_state_t* pstate = pvstate; -// if (pinrec == NULL) -// return sllv_single(NULL); -// -// // ["s", "t"] -// mlr_reference_values_from_record_into_string_array(pinrec, pstate->pvalue_field_names, pstate->pvalue_field_values); -// slls_t* pgroup_by_field_values = mlr_reference_selected_values_from_record(pinrec, pstate->pgroup_by_field_names); -// -// if (pgroup_by_field_values == NULL) { -// return sllv_single(pinrec); -// } -// -// lhmsv_t* pgroup_to_acc_field = lhmslv_get(pstate->groups, pgroup_by_field_values); -// if (pgroup_to_acc_field == NULL) { -// pgroup_to_acc_field = lhmsv_alloc(); -// lhmslv_put(pstate->groups, slls_copy(pgroup_by_field_values), pgroup_to_acc_field, FREE_ENTRY_KEY); -// } -// -// // for x=1 and y=2 -// int n = pstate->pvalue_field_names->length; -// for (int i = 0; i < n; i++) { -// char* value_field_name = pstate->pvalue_field_names->strings[i]; -// char* value_field_sval = pstate->pvalue_field_values->strings[i]; -// if (value_field_sval == NULL) // Key not present -// continue; -// -// int have_dval = FALSE; -// int have_nval = FALSE; -// double value_field_dval = -999.0; -// mv_t value_field_nval = mv_absent(); -// -// lhmsv_t* pacc_field_to_acc_state = lhmsv_get(pgroup_to_acc_field, value_field_name); -// if (pacc_field_to_acc_state == NULL) { -// pacc_field_to_acc_state = lhmsv_alloc(); -// lhmsv_put(pgroup_to_acc_field, value_field_name, pacc_field_to_acc_state, NO_FREE); -// } -// -// // for "delta", "rsum" -// sllse_t* pc = pstate->pstepper_names->phead; -// for ( ; pc != NULL; pc = pc->pnext) { -// char* step_name = pc->value; -// step_t* pstep = lhmsv_get(pacc_field_to_acc_state, step_name); -// if (pstep == NULL) { -// pstep = make_step(step_name, value_field_name, pstate->allow_int_float, -// pstate->pstring_alphas, pstate->pewma_suffixes); -// if (pstep == NULL) { -// fprintf(stderr, "mlr step: stepper \"%s\" not found.\n", -// step_name); -// exit(1); -// } -// lhmsv_put(pacc_field_to_acc_state, step_name, pstep, NO_FREE); -// } -// -// if (*value_field_sval == 0) { // Key present with null value -// if (pstep->pzprocess_func != NULL) { -// pstep->pzprocess_func(pstep->pvstate, pinrec); -// } -// } else { -// -// if (pstep->pdprocess_func != NULL) { -// if (!have_dval) { -// value_field_dval = mlr_double_from_string_or_die(value_field_sval); -// have_dval = TRUE; -// } -// pstep->pdprocess_func(pstep->pvstate, value_field_dval, pinrec); -// } -// -// if (pstep->pnprocess_func != NULL) { -// if (!have_nval) { -// value_field_nval = pstate->allow_int_float -// ? mv_scan_number_or_die(value_field_sval) -// : mv_from_float(mlr_double_from_string_or_die(value_field_sval)); -// have_nval = TRUE; -// } -// pstep->pnprocess_func(pstep->pvstate, &value_field_nval, pinrec); -// } -// -// if (pstep->psprocess_func != NULL) { -// pstep->psprocess_func(pstep->pvstate, value_field_sval, pinrec); -// } -// } -// } -// } -// return sllv_single(pinrec); -//} - -//static step_t* make_step(char* step_name, char* input_field_name, int allow_int_float, +//static tStepper* make_step(char* step_name, char* input_field_name, int allow_int_float, // slls_t* pstring_alphas, slls_t* pewma_suffixes) //{ // for (int i = 0; i < step_lookup_table_length; i++) @@ -381,18 +383,18 @@ func (this *TransformerStep) Map( // step_delta_state_t* pstate = pvstate; // lrec_put(prec, pstate->output_field_name, "", NO_FREE); //} -//static step_t* step_delta_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_delta_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // step_delta_state_t* pstate = mlr_malloc_or_die(sizeof(step_delta_state_t)); // pstate->prev = mv_absent(); // pstate->allow_int_float = allow_int_float; // pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_delta"); -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = NULL; -// pstep->pnprocess_func = step_delta_nprocess; -// pstep->psprocess_func = NULL; -// pstep->pzprocess_func = step_delta_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = NULL; +// stepper->pnprocess_func = step_delta_nprocess; +// stepper->psprocess_func = NULL; +// stepper->pzprocess_func = step_delta_zprocess; +// return stepper; //} //// ---------------------------------------------------------------- @@ -410,18 +412,18 @@ func (this *TransformerStep) Map( // step_shift_state_t* pstate = pvstate; // lrec_put(prec, pstate->output_field_name, "", NO_FREE); //} -//static step_t* step_shift_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_shift_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // step_shift_state_t* pstate = mlr_malloc_or_die(sizeof(step_shift_state_t)); // pstate->prev = mlr_strdup_or_die(""); // pstate->allow_int_float = allow_int_float; // pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_shift"); -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = NULL; -// pstep->pnprocess_func = NULL; -// pstep->psprocess_func = step_shift_sprocess; -// pstep->pzprocess_func = step_shift_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = NULL; +// stepper->pnprocess_func = NULL; +// stepper->psprocess_func = step_shift_sprocess; +// stepper->pzprocess_func = step_shift_zprocess; +// return stepper; //} //// ---------------------------------------------------------------- @@ -445,18 +447,18 @@ func (this *TransformerStep) Map( // step_from_first_state_t* pstate = pvstate; // lrec_put(prec, pstate->output_field_name, "", NO_FREE); //} -//static step_t* step_from_first_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_from_first_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // step_from_first_state_t* pstate = mlr_malloc_or_die(sizeof(step_from_first_state_t)); // pstate->first = mv_absent(); // pstate->allow_int_float = allow_int_float; // pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_from_first"); -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = NULL; -// pstep->pnprocess_func = step_from_first_nprocess; -// pstep->psprocess_func = NULL; -// pstep->pzprocess_func = step_from_first_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = NULL; +// stepper->pnprocess_func = step_from_first_nprocess; +// stepper->psprocess_func = NULL; +// stepper->pzprocess_func = step_from_first_zprocess; +// return stepper; //} //// ---------------------------------------------------------------- @@ -481,19 +483,19 @@ func (this *TransformerStep) Map( // step_ratio_state_t* pstate = pvstate; // lrec_put(prec, pstate->output_field_name, "", NO_FREE); //} -//static step_t* step_ratio_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_ratio_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // step_ratio_state_t* pstate = mlr_malloc_or_die(sizeof(step_ratio_state_t)); // pstate->prev = -999.0; // pstate->have_prev = FALSE; // pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_ratio"); // -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = step_ratio_dprocess; -// pstep->pnprocess_func = NULL; -// pstep->psprocess_func = NULL; -// pstep->pzprocess_func = step_ratio_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = step_ratio_dprocess; +// stepper->pnprocess_func = NULL; +// stepper->psprocess_func = NULL; +// stepper->pzprocess_func = step_ratio_zprocess; +// return stepper; //} //// ---------------------------------------------------------------- @@ -512,18 +514,18 @@ func (this *TransformerStep) Map( // step_rsum_state_t* pstate = pvstate; // lrec_put(prec, pstate->output_field_name, "", NO_FREE); //} -//static step_t* step_rsum_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_rsum_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // step_rsum_state_t* pstate = mlr_malloc_or_die(sizeof(step_rsum_state_t)); // pstate->allow_int_float = allow_int_float; // pstate->rsum = pstate->allow_int_float ? mv_from_int(0LL) : mv_from_float(0.0); // pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_rsum"); -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = NULL; -// pstep->pnprocess_func = step_rsum_nprocess; -// pstep->psprocess_func = NULL; -// pstep->pzprocess_func = step_rsum_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = NULL; +// stepper->pnprocess_func = step_rsum_nprocess; +// stepper->psprocess_func = NULL; +// stepper->pzprocess_func = step_rsum_zprocess; +// return stepper; //} //// ---------------------------------------------------------------- @@ -542,19 +544,19 @@ func (this *TransformerStep) Map( // step_counter_state_t* pstate = pvstate; // lrec_put(prec, pstate->output_field_name, "", NO_FREE); //} -//static step_t* step_counter_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_counter_alloc(char* input_field_name, int allow_int_float, slls_t* unused1, slls_t* unused2) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // step_counter_state_t* pstate = mlr_malloc_or_die(sizeof(step_counter_state_t)); // pstate->counter = allow_int_float ? mv_from_int(0LL) : mv_from_float(0.0); // pstate->one = allow_int_float ? mv_from_int(1LL) : mv_from_float(1.0); // pstate->output_field_name = mlr_paste_2_strings(input_field_name, "_counter"); // -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = NULL; -// pstep->pnprocess_func = NULL; -// pstep->psprocess_func = step_counter_sprocess; -// pstep->pzprocess_func = step_counter_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = NULL; +// stepper->pnprocess_func = NULL; +// stepper->psprocess_func = step_counter_sprocess; +// stepper->pzprocess_func = step_counter_zprocess; +// return stepper; //} //// ---------------------------------------------------------------- @@ -592,8 +594,8 @@ func (this *TransformerStep) Map( // lrec_put(prec, pstate->output_field_names[i], "", NO_FREE); //} -//static step_t* step_ewma_alloc(char* input_field_name, int unused, slls_t* pstring_alphas, slls_t* pewma_suffixes) { -// step_t* pstep = mlr_malloc_or_die(sizeof(step_t)); +//static tStepper* step_ewma_alloc(char* input_field_name, int unused, slls_t* pstring_alphas, slls_t* pewma_suffixes) { +// tStepper* stepper = mlr_malloc_or_die(sizeof(tStepper)); // // step_ewma_state_t* pstate = mlr_malloc_or_die(sizeof(step_ewma_state_t)); // int n = pstring_alphas->length; @@ -616,10 +618,10 @@ func (this *TransformerStep) Map( // } // pstate->have_prevs = FALSE; // -// pstep->pvstate = (void*)pstate; -// pstep->pdprocess_func = step_ewma_dprocess; -// pstep->pnprocess_func = NULL; -// pstep->psprocess_func = NULL; -// pstep->pzprocess_func = step_ewma_zprocess; -// return pstep; +// stepper->pvstate = (void*)pstate; +// stepper->pdprocess_func = step_ewma_dprocess; +// stepper->pnprocess_func = NULL; +// stepper->psprocess_func = NULL; +// stepper->pzprocess_func = step_ewma_zprocess; +// return stepper; //} diff --git a/go/src/miller/transformers/tac.go b/go/src/miller/transformers/tac.go index 26d7eac8c..bdaf3c988 100644 --- a/go/src/miller/transformers/tac.go +++ b/go/src/miller/transformers/tac.go @@ -81,7 +81,7 @@ func NewTransformerTac() (*TransformerTac, error) { }, nil } -func (this *TransformerTac) Map( +func (this *TransformerTac) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/tail.go b/go/src/miller/transformers/tail.go index bcd433ed1..68fff6008 100644 --- a/go/src/miller/transformers/tail.go +++ b/go/src/miller/transformers/tail.go @@ -122,7 +122,7 @@ func NewTransformerTail( } // ---------------------------------------------------------------- -func (this *TransformerTail) Map( +func (this *TransformerTail) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transformers/unsparsify.go b/go/src/miller/transformers/unsparsify.go index 13b0bfb6e..4aacd5b22 100644 --- a/go/src/miller/transformers/unsparsify.go +++ b/go/src/miller/transformers/unsparsify.go @@ -131,7 +131,7 @@ func NewTransformerUnsparsify( } // ---------------------------------------------------------------- -func (this *TransformerUnsparsify) Map( +func (this *TransformerUnsparsify) Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) { diff --git a/go/src/miller/transforming/chain_transformer.go b/go/src/miller/transforming/chain_transformer.go index 5ea95436e..57e66dba6 100644 --- a/go/src/miller/transforming/chain_transformer.go +++ b/go/src/miller/transforming/chain_transformer.go @@ -48,7 +48,7 @@ func runSingleTransformer( ) { for { recordAndContext := <-inputChannel - recordTransformer.Map(recordAndContext, outputChannel) + recordTransformer.Transform(recordAndContext, outputChannel) if recordAndContext.Record == nil { // end of stream break } diff --git a/go/src/miller/transforming/record_transformer.go b/go/src/miller/transforming/record_transformer.go index b8b748fe2..2c1ed2f82 100644 --- a/go/src/miller/transforming/record_transformer.go +++ b/go/src/miller/transforming/record_transformer.go @@ -8,7 +8,7 @@ import ( ) type IRecordTransformer interface { - Map( + Transform( inrecAndContext *types.RecordAndContext, outputChannel chan<- *types.RecordAndContext, ) diff --git a/go/src/miller/types/mlrmap_accessors.go b/go/src/miller/types/mlrmap_accessors.go index 17d57952f..2f66aba40 100644 --- a/go/src/miller/types/mlrmap_accessors.go +++ b/go/src/miller/types/mlrmap_accessors.go @@ -395,6 +395,23 @@ func (this *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) ( return buffer.String(), mlrvals, true } +// As above but only returns the array. For step. +func (this *Mlrmap) GetSelectedValues(selectedFieldNames []string) ( + []Mlrval, + bool, +) { + mlrvals := make([]Mlrval, 0, len(selectedFieldNames)) + + for _, selectedFieldName := range selectedFieldNames { + entry := this.findEntry(&selectedFieldName) + if entry == nil { + return mlrvals, false + } + mlrvals = append(mlrvals, *entry.Value.Copy()) + } + return mlrvals, true +} + // ---------------------------------------------------------------- func (this *Mlrmap) Rename(oldKey *string, newKey *string) bool { entry := this.findEntry(oldKey)