Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-25 08:53:55 +00:00
Code Issues Projects Releases Wiki Activity

free-flags for lhmsv

Browse source
This commit is contained in:
John Kerl 2016-01-07 22:47:21 +00:00
parent 4fe9420f36
commit 8a0fbdc875
10 changed files with 33 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

21
c/containers/lhmsv.c
View file

@ -2,7 +2,7 @@
// Array-only (open addressing) string-to-void linked hash map with linear
// probing for collisions.
//
// Keys are strduped; memory management of the void* values is left to the
// Keys are not strduped; memory management of the void* values is left to the
// caller.
//
// John Kerl 2012-08-13
@ -23,6 +23,7 @@
#include "lib/mlr_globals.h"
#include "lib/mlrutil.h"
#include "containers/lhmsv.h"
#include "containers/free_flags.h"
// ----------------------------------------------------------------
// Allow compile-time override, e.g using gcc -D.
@ -44,7 +45,7 @@
#define EMPTY 0xce
// ----------------------------------------------------------------
static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue);
static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags);
static void lhmsv_enlarge(lhmsv_t* pmap);
static void lhmsv_init(lhmsv_t *pmap, int length) {
@ -78,7 +79,8 @@ void lhmsv_free(lhmsv_t* pmap) {
if (pmap == NULL)
return;
for (lhmsve_t* pe = pmap->phead; pe != NULL; pe = pe->pnext) {
free(pe->key);
if (pe->free_flags & FREE_ENTRY_KEY)
free(pe->key);
}
free(pmap->entries);
free(pmap->states);
@ -129,13 +131,13 @@ static int lhmsv_find_index_for_key(lhmsv_t* pmap, char* key) {
}
// ----------------------------------------------------------------
void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue) {
void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags) {
if ((pmap->num_occupied + pmap->num_freed) >= (pmap->array_length*LOAD_FACTOR))
lhmsv_enlarge(pmap);
lhmsv_put_no_enlarge(pmap, key, pvvalue);
lhmsv_put_no_enlarge(pmap, key, pvvalue, free_flags);
}
static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue) {
static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags) {
int index = lhmsv_find_index_for_key(pmap, key);
lhmsve_t* pe = &pmap->entries[index];
@ -148,8 +150,9 @@ static void lhmsv_put_no_enlarge(lhmsv_t* pmap, char* key, void* pvvalue) {
else if (pmap->states[index] == EMPTY) {
// End of chain.
pe->ideal_index = mlr_canonical_mod(mlr_string_hash_func(key), pmap->array_length);
pe->key = mlr_strdup_or_die(key);
pe->key = key;
pe->pvvalue = pvvalue;
pe->free_flags = free_flags;
pmap->states[index] = OCCUPIED;
if (pmap->phead == NULL) {
@ -244,9 +247,7 @@ static void lhmsv_enlarge(lhmsv_t* pmap) {
lhmsv_init(pmap, pmap->array_length*ENLARGEMENT_FACTOR);
for (lhmsve_t* pe = old_head; pe != NULL; pe = pe->pnext) {
// xxx implement free-flags here so we can do this without the redundant strdups
lhmsv_put_no_enlarge(pmap, pe->key, pe->pvvalue);
free(pe->key);
lhmsv_put_no_enlarge(pmap, pe->key, pe->pvvalue, pe->free_flags);
}
free(old_entries);
free(old_states);

4
c/containers/lhmsv.h
View file

@ -17,12 +17,14 @@
#define LHMSV_H
#include "containers/sllv.h"
#include "containers/free_flags.h"
// ----------------------------------------------------------------
typedef struct _lhmsve_t {
int ideal_index;
char* key;
void* pvvalue;
char free_flags;
struct _lhmsve_t *pprev;
struct _lhmsve_t *pnext;
} lhmsve_t;
@ -42,7 +44,7 @@ typedef struct _lhmsv_t {
// ----------------------------------------------------------------
lhmsv_t* lhmsv_alloc();
void lhmsv_free(lhmsv_t* pmap);
void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue);
void lhmsv_put(lhmsv_t* pmap, char* key, void* pvvalue, char free_flags);
void* lhmsv_get(lhmsv_t* pmap, char* key);
int lhmsv_has_key(lhmsv_t* pmap, char* key);
void lhmsv_remove(lhmsv_t* pmap, char* key);

4
c/mapping/mapper_histogram.c
View file

@ -114,14 +114,14 @@ static mapper_t* mapper_histogram_alloc(ap_state_t* pargp, slls_t* value_field_n
unsigned long long* pcounts = mlr_malloc_or_die(nbins * sizeof(unsigned long long));
for (int i = 0; i < nbins; i++)
pcounts[i] = 0LL;
lhmsv_put(pstate->pcounts_by_field, value_field_name, pcounts);
lhmsv_put(pstate->pcounts_by_field, value_field_name, pcounts, NO_FREE);
}
if (do_auto) {
pstate->pvectors_by_field = lhmsv_alloc();
for (sllse_t* pe = pstate->value_field_names->phead; pe != NULL; pe = pe->pnext) {
char* value_field_name = pe->value;
dvector_t* pvector = dvector_alloc(DVECTOR_INITIAL_SIZE);
lhmsv_put(pstate->pvectors_by_field, value_field_name, pvector);
lhmsv_put(pstate->pvectors_by_field, value_field_name, pvector, NO_FREE);
}
} else {
pstate->pvectors_by_field = NULL;

11
c/mapping/mapper_merge_fields.c
View file

@ -251,7 +251,7 @@ static sllv_t* mapper_merge_fields_process_by_name_list(lrec_t* pinrec, context_
char* acc_name = pa->value;
stats1_acc_t* pacc = make_stats1_acc(pstate->output_field_basename, acc_name,
pstate->allow_int_float);
lhmsv_put(paccs, acc_name, pacc);
lhmsv_put(paccs, acc_name, pacc, NO_FREE);
}
for (sllse_t* pb = pstate->pvalue_field_names->phead; pb != NULL; pb = pb->pnext) {
@ -315,7 +315,7 @@ static sllv_t* mapper_merge_fields_process_by_name_regex(lrec_t* pinrec, context
char* acc_name = pa->value;
stats1_acc_t* pacc = make_stats1_acc(pstate->output_field_basename, acc_name,
pstate->allow_int_float);
lhmsv_put(paccs, acc_name, pacc);
lhmsv_put(paccs, acc_name, pacc, NO_FREE);
}
for (lrece_t* pb = pinrec->phead; pb != NULL; /* increment inside loop */ ) {
@ -411,11 +411,10 @@ static sllv_t* mapper_merge_fields_process_by_collapsing(lrec_t* pinrec, context
for (sllse_t* pc = pstate->paccumulator_names->phead; pc != NULL; pc = pc->pnext) {
char* acc_name = pc->value;
stats1_acc_t* pacc = make_stats1_acc(short_name, acc_name, pstate->allow_int_float);
// xxx implement free-flags here (& for all lhm's) for copy-reduction
lhmsv_put(acc_map_for_short_name, acc_name, pacc);
lhmsv_put(acc_map_for_short_name, acc_name, pacc, NO_FREE);
}
// xxx implement free-flags here (& for all lhm's) for copy-reduction
lhmsv_put(short_names_to_acc_maps, short_name, acc_map_for_short_name);
lhmsv_put(short_names_to_acc_maps, mlr_strdup_or_die(short_name), acc_map_for_short_name,
FREE_ENTRY_KEY);
}
char* value_field_sval = lrec_get(pinrec, field_name);

4
c/mapping/mapper_stats1.c
View file

@ -252,14 +252,14 @@ static void mapper_stats1_ingest(lrec_t* pinrec, mapper_stats1_state_t* pstate)
lhmsv_t* acc_field_to_acc_state = lhmsv_get(pgroup_to_acc_field, value_field_name);
if (acc_field_to_acc_state == NULL) {
acc_field_to_acc_state = lhmsv_alloc();
lhmsv_put(pgroup_to_acc_field, value_field_name, acc_field_to_acc_state);
lhmsv_put(pgroup_to_acc_field, value_field_name, acc_field_to_acc_state, NO_FREE);
}
// Look up presence of all accumulators at this level's hashmap.
char* presence = lhmsv_get(acc_field_to_acc_state, fake_acc_name_for_setups);
if (presence == NULL) {
make_stats1_accs(value_field_name, pstate->paccumulator_names, pstate->allow_int_float, acc_field_to_acc_state);
lhmsv_put(acc_field_to_acc_state, fake_acc_name_for_setups, fake_acc_name_for_setups);
lhmsv_put(acc_field_to_acc_state, fake_acc_name_for_setups, fake_acc_name_for_setups, NO_FREE);
}
if (value_field_sval == NULL)

2
c/mapping/mapper_stats2.c
View file

@ -334,7 +334,7 @@ static void mapper_stats2_ingest(lrec_t* pinrec, context_t* pctx, mapper_stats2_
stats2_acc_name);
exit(1);
}
lhmsv_put(pacc_fields_to_acc_state, stats2_acc_name, pstats2_acc);
lhmsv_put(pacc_fields_to_acc_state, stats2_acc_name, pstats2_acc, NO_FREE);
}
if (sval1 == NULL || sval2 == NULL)
continue;

4
c/mapping/mapper_step.c
View file

@ -230,7 +230,7 @@ static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstat
lhmsv_t* pacc_field_to_acc_state = lhmsv_get(pgroup_to_acc_field, value_field_name);
if (pacc_field_to_acc_state == NULL) {
pacc_field_to_acc_state = lhmsv_alloc();
lhmsv_put(pgroup_to_acc_field, value_field_name, pacc_field_to_acc_state);
lhmsv_put(pgroup_to_acc_field, value_field_name, pacc_field_to_acc_state, NO_FREE);
}
// for "delta", "rsum"
@ -245,7 +245,7 @@ static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstat
step_name);
exit(1);
}
lhmsv_put(pacc_field_to_acc_state, step_name, pstep);
lhmsv_put(pacc_field_to_acc_state, step_name, pstep, NO_FREE);
}
if (pstep->pdprocess_func != NULL) {

2
c/mapping/mapper_top.c
View file

@ -194,7 +194,7 @@ static void mapper_top_ingest(lrec_t* pinrec, mapper_top_state_t* pstate) {
top_keeper_t* ptop_keeper_for_group = lhmsv_get(group_to_acc_field, value_field_name);
if (ptop_keeper_for_group == NULL) {
ptop_keeper_for_group = top_keeper_alloc(pstate->top_count);
lhmsv_put(group_to_acc_field, value_field_name, ptop_keeper_for_group);
lhmsv_put(group_to_acc_field, value_field_name, ptop_keeper_for_group, NO_FREE);
}
// The top-keeper object will free the record if it isn't retained, or

4
c/mapping/stats1_accumulators.c
View file

@ -51,7 +51,7 @@ void make_stats1_accs(
} else {
stats1_percentile_reuse(ppercentile_acc);
}
lhmsv_put(acc_field_to_acc_state, stats1_acc_name, ppercentile_acc);
lhmsv_put(acc_field_to_acc_state, stats1_acc_name, ppercentile_acc, NO_FREE);
} else {
stats1_acc_t* pstats1_acc = make_stats1_acc(value_field_name, stats1_acc_name, allow_int_float);
if (pstats1_acc == NULL) {
@ -59,7 +59,7 @@ void make_stats1_accs(
MLR_GLOBALS.argv0, stats1_acc_name);
exit(1);
}
lhmsv_put(acc_field_to_acc_state, stats1_acc_name, pstats1_acc);
lhmsv_put(acc_field_to_acc_state, stats1_acc_name, pstats1_acc, NO_FREE);
}
}
}

8
c/unit_test/test_multiple_containers.c
View file

@ -387,7 +387,7 @@ static char* test_lhmsv() {
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "x", "3");
lhmsv_put(pmap, "x", "3", NO_FREE);
mu_assert_lf(pmap->num_occupied == 1);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "3"));
@ -395,7 +395,7 @@ static char* test_lhmsv() {
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "y", "5");
lhmsv_put(pmap, "y", "5", NO_FREE);
mu_assert_lf(pmap->num_occupied == 2);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "3"));
@ -403,7 +403,7 @@ static char* test_lhmsv() {
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "x", "4");
lhmsv_put(pmap, "x", "4", NO_FREE);
mu_assert_lf(pmap->num_occupied == 2);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4"));
@ -411,7 +411,7 @@ static char* test_lhmsv() {
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "z", "7");
lhmsv_put(pmap, "z", "7", NO_FREE);
mu_assert_lf(pmap->num_occupied == 3);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4"));