// ----------------------------------------------------------------
static char* test_lhmsv() {
lhmsv_t *pmap = lhmsv_alloc();
mu_assert_lf(pmap->num_occupied == 0);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf(!lhmsv_has_key(pmap, "x")); mu_assert_lf(lhmsv_get(pmap, "x") == NULL);
mu_assert_lf(!lhmsv_has_key(pmap, "y")); mu_assert_lf(lhmsv_get(pmap, "y") == NULL);
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "x", "3");
mu_assert_lf(pmap->num_occupied == 1);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "3"));
mu_assert_lf(!lhmsv_has_key(pmap, "y")); mu_assert_lf(lhmsv_get(pmap, "y") == NULL);
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "y", "5");
mu_assert_lf(pmap->num_occupied == 2);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "3"));
mu_assert_lf( lhmsv_has_key(pmap, "y")); mu_assert_lf(streq(lhmsv_get(pmap, "y"), "5"));
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "x", "4");
mu_assert_lf(pmap->num_occupied == 2);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4"));
mu_assert_lf( lhmsv_has_key(pmap, "y")); mu_assert_lf(streq(lhmsv_get(pmap, "y"), "5"));
mu_assert_lf(!lhmsv_has_key(pmap, "z")); mu_assert_lf(lhmsv_get(pmap, "z") == NULL);
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_put(pmap, "z", "7");
mu_assert_lf(pmap->num_occupied == 3);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4"));
mu_assert_lf( lhmsv_has_key(pmap, "y")); mu_assert_lf(streq(lhmsv_get(pmap, "y"), "5"));
mu_assert_lf( lhmsv_has_key(pmap, "z")); mu_assert_lf(streq(lhmsv_get(pmap, "z"), "7"));
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_remove(pmap, "y");
mu_assert_lf(pmap->num_occupied == 2);
mu_assert_lf(!lhmsv_has_key(pmap, "w")); mu_assert_lf(lhmsv_get(pmap, "w") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "x")); mu_assert_lf(streq(lhmsv_get(pmap, "x"), "4"));
mu_assert_lf(!lhmsv_has_key(pmap, "y")); mu_assert_lf(lhmsv_get(pmap, "y") == NULL);
mu_assert_lf( lhmsv_has_key(pmap, "z")); mu_assert_lf(streq(lhmsv_get(pmap, "z"), "7"));
mu_assert_lf(lhmsv_check_counts(pmap));
lhmsv_free(pmap);
return NULL;
}
static sllv_t* mapper_step_process(lrec_t* pinrec, context_t* pctx, void* pvstate) {
mapper_step_state_t* pstate = pvstate;
if (pinrec == NULL)
return sllv_single(NULL);
// ["s", "t"]
slls_t* pvalue_field_values = mlr_selected_values_from_record(pinrec, pstate->pvalue_field_names);
slls_t* pgroup_by_field_values = mlr_selected_values_from_record(pinrec, pstate->pgroup_by_field_names);
if (pgroup_by_field_values->length != pstate->pgroup_by_field_names->length) {
lrec_free(pinrec);
return NULL;
}
lhmsv_t* group_to_acc_field = lhmslv_get(pstate->groups, pgroup_by_field_values);
if (group_to_acc_field == NULL) {
group_to_acc_field = lhmsv_alloc();
lhmslv_put(pstate->groups, slls_copy(pgroup_by_field_values), group_to_acc_field);
}
sllse_t* pa = pstate->pvalue_field_names->phead;
sllse_t* pb = pvalue_field_values->phead;
// for x=1 and y=2
for ( ; pa != NULL && pb != NULL; pa = pa->pnext, pb = pb->pnext) {
char* value_field_name = pa->value;
char* value_field_sval = pb->value;
int have_dval = FALSE;
double value_field_dval = -999.0;
lhmsv_t* acc_field_to_acc_state = lhmsv_get(group_to_acc_field, value_field_name);
if (acc_field_to_acc_state == NULL) {
acc_field_to_acc_state = lhmsv_alloc();
lhmsv_put(group_to_acc_field, value_field_name, acc_field_to_acc_state);
}
// for "delta", "rsum"
sllse_t* pc = pstate->pstepper_names->phead;
for ( ; pc != NULL; pc = pc->pnext) {
char* step_name = pc->value;
step_t* pstep = lhmsv_get(acc_field_to_acc_state, step_name);
if (pstep == NULL) {
pstep = make_step(step_name, value_field_name);
if (pstep == NULL) {
fprintf(stderr, "mlr step: stepper \"%s\" not found.\n",
step_name);
exit(1);
}
lhmsv_put(acc_field_to_acc_state, step_name, pstep);
}
if (pstep->psprocess_func != NULL) {
pstep->psprocess_func(pstep->pvstate, value_field_sval, pinrec);
}
if (pstep->pdprocess_func != NULL) {
if (!have_dval) {
value_field_dval = mlr_double_from_string_or_die(value_field_sval);
have_dval = TRUE;
}
pstep->pdprocess_func(pstep->pvstate, value_field_dval, pinrec);
}
}
}
return sllv_single(pinrec);
}
// ----------------------------------------------------------------
static void mapper_stats1_ingest(lrec_t* pinrec, mapper_stats1_state_t* pstate) {
// E.g. ["s", "t"]
// To do: make value_field_values into a hashmap. Then accept partial
// population on that, but retain full-population requirement on group-by.
// E.g. if accumulating stats of x,y on a,b then skip record with x,y,a but
// process record with x,a,b.
slls_t* pvalue_field_values = mlr_selected_values_from_record(pinrec, pstate->pvalue_field_names);
slls_t* pgroup_by_field_values = mlr_selected_values_from_record(pinrec, pstate->pgroup_by_field_names);
if (pvalue_field_values->length != pstate->pvalue_field_names->length)
return;
if (pgroup_by_field_values->length != pstate->pgroup_by_field_names->length)
return;
lhmsv_t* group_to_acc_field = lhmslv_get(pstate->groups, pgroup_by_field_values);
if (group_to_acc_field == NULL) {
group_to_acc_field = lhmsv_alloc();
lhmslv_put(pstate->groups, slls_copy(pgroup_by_field_values), group_to_acc_field);
}
sllse_t* pa = pstate->pvalue_field_names->phead;
sllse_t* pb = pvalue_field_values->phead;
// for x=1 and y=2
for ( ; pa != NULL && pb != NULL; pa = pa->pnext, pb = pb->pnext) {
char* value_field_name = pa->value;
char* value_field_sval = pb->value;
int have_dval = FALSE;
double value_field_dval = -999.0;
lhmsv_t* acc_field_to_acc_state = lhmsv_get(group_to_acc_field, value_field_name);
if (acc_field_to_acc_state == NULL) {
acc_field_to_acc_state = lhmsv_alloc();
lhmsv_put(group_to_acc_field, value_field_name, acc_field_to_acc_state);
}
// Look up presence of all accumulators at this level's hashmap.
char* presence = lhmsv_get(acc_field_to_acc_state, fake_acc_name_for_setups);
if (presence == NULL) {
make_accs(pstate->paccumulator_names, acc_field_to_acc_state);
lhmsv_put(acc_field_to_acc_state, fake_acc_name_for_setups, fake_acc_name_for_setups);
}
// There isn't a one-to-one mapping between user-specified acc_names
// and internal acc_t's. Here in the ingestor we feed each datum into
// an acc_t. In the emitter, we loop over the acc_names in
// user-specified order. Example: they ask for p10,mean,p90. Then there
// is only one percentiles accumulator to be told about each point. In
// the emitter it will be asked to produce output twice: once for the
// 10th percentile & once for the 90th.
for (lhmsve_t* pc = acc_field_to_acc_state->phead; pc != NULL; pc = pc->pnext) {
char* acc_name = pc->key;
if (streq(acc_name, fake_acc_name_for_setups))
continue;
acc_t* pacc = pc->pvvalue;
if (pacc->psingest_func != NULL) {
pacc->psingest_func(pacc->pvstate, value_field_sval);
}
if (pacc->pdingest_func != NULL) {
if (!have_dval) {
value_field_dval = mlr_double_from_string_or_die(value_field_sval);
have_dval = TRUE;
}
pacc->pdingest_func(pacc->pvstate, value_field_dval);
}
}
}
}
// ----------------------------------------------------------------
multi_lrec_writer_t* multi_lrec_writer_alloc(cli_writer_opts_t* pwriter_opts) {
multi_lrec_writer_t* pmlw = mlr_malloc_or_die(sizeof(multi_lrec_writer_t));
pmlw->pnames_to_lrec_writers_and_fps = lhmsv_alloc();
pmlw->pwriter_opts = pwriter_opts;
return pmlw;
}
