From 8a0fbdc8759b11ef55c5698edd6d306cb4ccc934 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Thu, 7 Jan 2016 22:47:21 +0000 Subject: [PATCH] free-flags for lhmsv --- c/containers/lhmsv.c | 21 +++++++++++---------- c/containers/lhmsv.h | 4 +++- c/mapping/mapper_histogram.c | 4 ++-- c/mapping/mapper_merge_fields.c | 11 +++++------ c/mapping/mapper_stats1.c | 4 ++-- c/mapping/mapper_stats2.c | 2 +- c/mapping/mapper_step.c | 4 ++-- c/mapping/mapper_top.c | 2 +- c/mapping/stats1_accumulators.c | 4 ++-- c/unit_test/test_multiple_containers.c | 8 ++++---- 10 files changed, 33 insertions(+), 31 deletions(-) diff --git a/c/containers/lhmsv.c b/c/containers/lhmsv.c index 9b26a85e4..d33907913 100644 --- a/c/containers/lhmsv.c +++ b/c/containers/lhmsv.c @@ -2,7 +2,7 @@ // Array-only (open addressing) string-to-void linked hash map with linear // probing for collisions. // -// Keys are strduped; memory management of the void* values is left to the +// Keys are not strduped; memory management of the void* values is left to the // caller. // // John Kerl 2012-08-13 @@ -23,6 +23,7 @@ #include "lib/mlr_globals.h" #include "lib/mlrutil.h" #include "containers/lhmsv.h" +#include "containers/free_flags.h" // ---------------------------------------------------------------- // Allow compile-time override, e.g using gcc -D. @@ -44,7 +45,7 @@ #define EMPTY 0xce // ---------------------------------------------------------------- -static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue); +static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags); static void lhmsv_enlarge(lhmsv_t* pmap); static void lhmsv_init(lhmsv_t *pmap, int length) { @@ -78,7 +79,8 @@ void lhmsv_free(lhmsv_t* pmap) { if (pmap == NULL) return; for (lhmsve_t* pe = pmap->phead; pe != NULL; pe = pe->pnext) { - free(pe->key); + if (pe->free_flags & FREE_ENTRY_KEY) + free(pe->key); } free(pmap->entries); free(pmap->states); @@ -129,13 +131,13 @@ static int lhmsv_find_index_for_key(lhmsv_t* pmap, char* key) { } // ---------------------------------------------------------------- -void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue) { +void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags) { if ((pmap->num_occupied + pmap->num_freed) >= (pmap->array_length*LOAD_FACTOR)) lhmsv_enlarge(pmap); - lhmsv_put_no_enlarge(pmap, key, pvvalue); + lhmsv_put_no_enlarge(pmap, key, pvvalue, free_flags); } -static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue) { +static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags) { int index = lhmsv_find_index_for_key(pmap, key); lhmsve_t* pe = &pmap->entries[index]; @@ -148,8 +150,9 @@ static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue) { else if (pmap->states[index] == EMPTY) { // End of chain. pe->ideal_index = mlr_canonical_mod(mlr_string_hash_func(key), pmap->array_length); - pe->key = mlr_strdup_or_die(key); + pe->key = key; pe->pvvalue = pvvalue; + pe->free_flags = free_flags; pmap->states[index] = OCCUPIED; if (pmap->phead == NULL) { @@ -244,9 +247,7 @@ static void lhmsv_enlarge(lhmsv_t* pmap) { lhmsv_init(pmap, pmap->array_length*ENLARGEMENT_FACTOR); for (lhmsve_t* pe = old_head; pe != NULL; pe = pe->pnext) { - // xxx implement free-flags here so we can do this without the redundant strdups - lhmsv_put_no_enlarge(pmap, pe->key, pe->pvvalue); - free(pe->key); + lhmsv_put_no_enlarge(pmap, pe->key, pe->pvvalue, pe->free_flags); } free(old_entries); free(old_states); diff --git a/c/containers/lhmsv.h b/c/containers/lhmsv.h index 273ecbe74..f84b7d5aa 100644 --- a/c/containers/lhmsv.h +++ b/c/containers/lhmsv.h @@ -17,12 +17,14 @@ #define LHMSV_H #include "containers/sllv.h" +#include "containers/free_flags.h" // ---------------------------------------------------------------- typedef struct _lhmsve_t { int ideal_index; char* key; void* pvvalue; + char free_flags; struct _lhmsve_t *pprev; struct _lhmsve_t *pnext; } lhmsve_t; @@ -42,7 +44,7 @@ typedef struct _lhmsv_t { // ---------------------------------------------------------------- lhmsv_t* lhmsv_alloc(); void lhmsv_free(lhmsv_t* pmap); -void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue); +void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags); void* lhmsv_get(lhmsv_t* pmap, char* key); int lhmsv_has_key(lhmsv_t* pmap, char* key); void lhmsv_remove(lhmsv_t* pmap, char* key); diff --git a/c/mapping/mapper_histogram.c b/c/mapping/mapper_histogram.c index 0d0bf0bb5..04d594c08 100644 --- a/c/mapping/mapper_histogram.c +++ b/c/mapping/mapper_histogram.c @@ -114,14 +114,14 @@ static mapper_t* mapper_histogram_alloc(ap_state_t* pargp, slls_t* value_field_n unsigned long long* pcounts = mlr_malloc_or_die(nbins * sizeof(unsigned long long)); for (int i = 0; i < nbins; i++) pcounts[i] = 0LL; - lhmsv_put(pstate->pcounts_by_field, value_field_name, pcounts); + lhmsv_put(pstate->pcounts_by_field, value_field_name, pcounts, NO_FREE); } if (do_auto) { pstate->pvectors_by_field = lhmsv_alloc(); for (sllse_t* pe = pstate->value_field_names->phead; pe != NULL; pe = pe->pnext) { char* value_field_name = pe->value; dvector_t* pvector = dvector_alloc(DVECTOR_INITIAL_SIZE); - lhmsv_put(pstate->pvectors_by_field, value_field_name, pvector); + lhmsv_put(pstate->pvectors_by_field, value_field_name, pvector, NO_FREE); } } else { pstate->pvectors_by_field = NULL; diff --git a/c/mapping/mapper_merge_fields.c b/c/mapping/mapper_merge_fields.c index b63a002f7..8591e2c43 100644 --- a/c/mapping/mapper_merge_fields.c +++ b/c/mapping/mapper_merge_fields.c @@ -251,7 +251,7 @@ static sllv_t* mapper_merge_fields_process_by_name_list(lrec_t* pinrec, context_ char* acc_name = pa->value; stats1_acc_t* pacc = make_stats1_acc(pstate->output_field_basename, acc_name, pstate->allow_int_float); - lhmsv_put(paccs, acc_name, pacc); + lhmsv_put(paccs, acc_name, pacc, NO_FREE); } for (sllse_t* pb = pstate->pvalue_field_names->phead; pb != NULL; pb = pb->pnext) { @@ -315,7 +315,7 @@ static sllv_t* mapper_merge_fields_process_by_name_regex(lrec_t* pinrec, context char* acc_name = pa->value; stats1_acc_t* pacc = make_stats1_acc(pstate->output_field_basename, acc_name, pstate->allow_int_float); - lhmsv_put(paccs, acc_name, pacc); + lhmsv_put(paccs, acc_name, pacc, NO_FREE); } for (lrece_t* pb = pinrec->phead; pb != NULL; /* increment inside loop */ ) { @@ -411,11 +411,10 @@ static sllv_t* mapper_merge_fields_process_by_collapsing(lrec_t* pinrec, context for (sllse_t* pc = pstate->paccumulator_names->phead; pc != NULL; pc = pc->pnext) { char* acc_name = pc->value; stats1_acc_t* pacc = make_stats1_acc(short_name, acc_name, pstate->allow_int_float); - // xxx implement free-flags here (& for all lhm's) for copy-reduction - lhmsv_put(acc_map_for_short_name, acc_name, pacc); + lhmsv_put(acc_map_for_short_name, acc_name, pacc, NO_FREE); } - // xxx implement free-flags here (& for all lhm's) for copy-reduction - lhmsv_put(short_names_to_acc_maps, short_name, acc_map_for_short_name); + lhmsv_put(short_names_to_acc_maps, mlr_strdup_or_die(short_name), acc_map_for_short_name, + FREE_ENTRY_KEY); } char* value_field_sval = lrec_get(pinrec, field_name); diff --git a/c/mapping/mapper_stats1.c b/c/mapping/mapper_stats1.c index b704d1597..1d72c695a 100644 --- a/c/mapping/mapper_stats1.c +++ b/c/mapping/mapper_stats1.c @@ -252,14 +252,14 @@ static void mapper_stats1_ingest(lrec_t* pinrec, mapper_stats1_state_t* pstate) lhmsv_t* acc_field_to_acc_state = lhmsv_get(pgroup_to_acc_field, value_field_name); if (acc_field_to_acc_state == NULL) { acc_field_to_acc_state = lhmsv_alloc(); - lhmsv_put(pgroup_to_acc_field, value_field_name, acc_field_to_acc_state); + lhmsv_put(pgroup_to_acc_field, value_field_name, acc_field_to_acc_state, NO_FREE); } // Look up presence of all accumulators at this level's hashmap. char* presence = lhmsv_get(acc_field_to_acc_state, fake_acc_name_for_setups); if (presence == NULL) { make_stats1_accs(value_field_name, pstate->paccumulator_names, pstate->allow_int_float, acc_field_to_acc_state); - lhmsv_put(acc_field_to_acc_state, fake_acc_name_for_setups, fake_acc_name_for_setups); + lhmsv_put(acc_field_to_acc_state, fake_acc_name_for_setups, fake_acc_name_for_setups, NO_FREE); } if (value_field_sval == NULL) diff --git a/c/mapping/mapper_stats2.c b/c/mapping/mapper_stats2.c index 64988fbce..162b91f85 100644 --- a/c/mapping/mapper_stats2.c +++ b/c/mapping/mapper_stats2.c @@ -334,7 +334,7 @@ static void mapper_stats2_ingest(lrec_t* pinrec, context_t* pctx, mapper_stats2_ stats2_acc_name); exit(1); } - lhmsv_put(pacc_fields_to_acc_state, stats2_acc_name, pstats2_acc); + lhmsv_put(pacc_fields_to_acc_state, stats2_acc_name, pstats2_acc, NO_FREE); } if (sval1 == NULL || sval2 == NULL) continue; diff --git a/c/mapping/mapper_step.c b/c/mapping/mapper_step.c index 445ec2261..f40bcbf64 100644 --- a/c/mapping/mapper_step.c +++ b/c/mapping/mapper_step.c @@ -230,7 +230,7 @@ static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstat lhmsv_t* pacc_field_to_acc_state = lhmsv_get(pgroup_to_acc_field, value_field_name); if (pacc_field_to_acc_state == NULL) { pacc_field_to_acc_state = lhmsv_alloc(); - lhmsv_put(pgroup_to_acc_field, value_field_name, pacc_field_to_acc_state); + lhmsv_put(pgroup_to_acc_field, value_field_name, pacc_field_to_acc_state, NO_FREE); } // for "delta", "rsum" @@ -245,7 +245,7 @@ static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstat step_name); exit(1); } - lhmsv_put(pacc_field_to_acc_state, step_name, pstep); + lhmsv_put(pacc_field_to_acc_state, step_name, pstep, NO_FREE); } if (pstep->pdprocess_func != NULL) { diff --git a/c/mapping/mapper_top.c b/c/mapping/mapper_top.c index 31c780dd6..12991fbda 100644 --- a/c/mapping/mapper_top.c +++ b/c/mapping/mapper_top.c @@ -194,7 +194,7 @@ static void mapper_top_ingest(lrec_t* pinrec, mapper_top_state_t* pstate) { top_keeper_t* ptop_keeper_for_group = lhmsv_get(group_to_acc_field, value_field_name); if (ptop_keeper_for_group == NULL) { ptop_keeper_for_group = top_keeper_alloc(pstate->top_count); - lhmsv_put(group_to_acc_field, value_field_name, ptop_keeper_for_group); + lhmsv_put(group_to_acc_field, value_field_name, ptop_keeper_for_group, NO_FREE); } // The top-keeper object will free the record if it isn't retained, or diff --git a/c/mapping/stats1_accumulators.c b/c/mapping/stats1_accumulators.c index ad0d362d8..a05ff1d57 100644 --- a/c/mapping/stats1_accumulators.c +++ b/c/mapping/stats1_accumulators.c @@ -51,7 +51,7 @@ void make_stats1_accs( } else { stats1_percentile_reuse(ppercentile_acc); } - lhmsv_put(acc_field_to_acc_state, stats1_acc_name, ppercentile_acc); + lhmsv_put(acc_field_to_acc_state, stats1_acc_name, ppercentile_acc, NO_FREE); } else { stats1_acc_t* pstats1_acc = make_stats1_acc(value_field_name, stats1_acc_name, allow_int_float); if (pstats1_acc == NULL) { @@ -59,7 +59,7 @@ void make_stats1_accs( MLR_GLOBALS.argv0, stats1_acc_name); exit(1); } - lhmsv_put(acc_field_to_acc_state, stats1_acc_name, pstats1_acc); + lhmsv_put(acc_field_to_acc_state, stats1_acc_name, pstats1_acc, NO_FREE); } } } diff --git a/c/unit_test/test_multiple_containers.c b/c/unit_test/test_multiple_containers.c index 6ad1b23ff..1c014f4b4 100644 --- a/c/unit_test/test_multiple_containers.c +++ b/c/unit_test/test_multiple_containers.c @@ -387,7 +387,7 @@ static char* test_lhmsv() { mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL); mu_assert_lf(lhmsv_check_counts(pmap)); - lhmsv_put(pmap, "x", "3"); + lhmsv_put(pmap, "x", "3", NO_FREE); mu_assert_lf(pmap->num_occupied == 1); mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL); mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "3")); @@ -395,7 +395,7 @@ static char* test_lhmsv() { mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL); mu_assert_lf(lhmsv_check_counts(pmap)); - lhmsv_put(pmap, "y", "5"); + lhmsv_put(pmap, "y", "5", NO_FREE); mu_assert_lf(pmap->num_occupied == 2); mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL); mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "3")); @@ -403,7 +403,7 @@ static char* test_lhmsv() { mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL); mu_assert_lf(lhmsv_check_counts(pmap)); - lhmsv_put(pmap, "x", "4"); + lhmsv_put(pmap, "x", "4", NO_FREE); mu_assert_lf(pmap->num_occupied == 2); mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL); mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4")); @@ -411,7 +411,7 @@ static char* test_lhmsv() { mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL); mu_assert_lf(lhmsv_check_counts(pmap)); - lhmsv_put(pmap, "z", "7"); + lhmsv_put(pmap, "z", "7", NO_FREE); mu_assert_lf(pmap->num_occupied == 3); mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL); mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4"));