Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-21 02:23:20 +00:00
Code Issues Projects Releases Wiki Activity

PCA-linreg feature

Browse source
This commit is contained in:
John Kerl 2015-05-05 20:28:05 -07:00
parent 40f1209df4
commit 17d2aa06a8
9 changed files with 143 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

107
c/mapping/mapper_stats2.c
View file

@ -29,7 +29,7 @@ typedef struct _stats2_t {
stats2_get_func_t* pget_func;
} stats2_t;
typedef stats2_t* stats2_alloc_func_t(static_context_t* pstatx);
typedef stats2_t* stats2_alloc_func_t(static_context_t* pstatx, int do_verbose);
// xxx move to mlrstat.h/c
@ -75,24 +75,24 @@ typedef stats2_t* stats2_alloc_func_t(static_context_t* pstatx);
// b = ----------------------------------------
// D
typedef struct _stats2_linreg_state_t {
typedef struct _stats2_linreg_ols_state_t {
unsigned long long count;
double sumx;
double sumy;
double sumx2;
double sumxy;
static_context_t* pstatx;
} stats2_linreg_state_t;
void stats2_linreg_put(void* pvstate, double x, double y) {
stats2_linreg_state_t* pstate = pvstate;
} stats2_linreg_ols_state_t;
void stats2_linreg_ols_put(void* pvstate, double x, double y) {
stats2_linreg_ols_state_t* pstate = pvstate;
pstate->count++;
pstate->sumx += x;
pstate->sumy += y;
pstate->sumx2 += x*x;
pstate->sumxy += x*y;
}
void stats2_linreg_get(void* pvstate, char* name1, char* name2, lrec_t* poutrec) {
stats2_linreg_state_t* pstate = pvstate;
void stats2_linreg_ols_get(void* pvstate, char* name1, char* name2, lrec_t* poutrec) {
stats2_linreg_ols_state_t* pstate = pvstate;
int n = pstate->count;
double sumx = pstate->sumx;
double sumy = pstate->sumy;
@ -117,17 +117,17 @@ void stats2_linreg_get(void* pvstate, char* name1, char* name2, lrec_t* poutrec)
//
// return [m, b, math.sqrt(var_m), math.sqrt(var_b)]
char* key = mlr_paste_4_strings(name1, "_", name2, "_m");
char* key = mlr_paste_4_strings(name1, "_", name2, "_ols_m");
char* val = mlr_alloc_string_from_double(m, pstate->pstatx->ofmt);
lrec_put(poutrec, key, val, LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
key = mlr_paste_4_strings(name1, "_", name2, "_b");
key = mlr_paste_4_strings(name1, "_", name2, "_ols_b");
val = mlr_alloc_string_from_double(b, pstate->pstatx->ofmt);
lrec_put(poutrec, key, val, LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
}
stats2_t* stats2_linreg_alloc(static_context_t* pstatx) {
stats2_t* stats2_linreg_ols_alloc(static_context_t* pstatx, int do_verbose) {
stats2_t* pstats2 = mlr_malloc_or_die(sizeof(stats2_t));
stats2_linreg_state_t* pstate = mlr_malloc_or_die(sizeof(stats2_linreg_state_t));
stats2_linreg_ols_state_t* pstate = mlr_malloc_or_die(sizeof(stats2_linreg_ols_state_t));
pstate->count = 0LL;
pstate->sumx = 0.0;
pstate->sumy = 0.0;
@ -135,8 +135,8 @@ stats2_t* stats2_linreg_alloc(static_context_t* pstatx) {
pstate->sumxy = 0.0;
pstate->pstatx = pstatx;
pstats2->pvstate = (void*)pstate;
pstats2->pput_func = &stats2_linreg_put;
pstats2->pget_func = &stats2_linreg_get;
pstats2->pput_func = &stats2_linreg_ols_put;
pstats2->pget_func = &stats2_linreg_ols_get;
return pstats2;
}
@ -183,7 +183,7 @@ void stats2_r2_get(void* pvstate, char* name1, char* name2, lrec_t* poutrec) {
lrec_put(poutrec, key, val, LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
}
}
stats2_t* stats2_r2_alloc(static_context_t* pstatx) {
stats2_t* stats2_r2_alloc(static_context_t* pstatx, int do_verbose) {
stats2_t* pstats2 = mlr_malloc_or_die(sizeof(stats2_t));
stats2_r2_state_t* pstate = mlr_malloc_or_die(sizeof(stats2_r2_state_t));
pstate->count = 0LL;
@ -220,7 +220,7 @@ typedef struct _stats2_corr_cov_state_t {
double sumxy;
double sumy2;
int do_which;
// xxx do_verbose;
int do_verbose;
static_context_t* pstatx;
} stats2_corr_cov_state_t;
void stats2_corr_cov_put(void* pvstate, double x, double y) {
@ -272,12 +272,14 @@ void stats2_corr_cov_get(void* pvstate, char* name1, char* name2, lrec_t* poutre
lrec_put(poutrec, keym, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyb, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyq, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyl1, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyl2, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv11, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv12, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv21, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv22, "", LREC_FREE_ENTRY_KEY);
if (pstate->do_verbose) {
lrec_put(poutrec, keyl1, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyl2, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv11, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv12, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv21, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyv22, "", LREC_FREE_ENTRY_KEY);
}
} else {
double Q[2][2];
mlr_get_cov_matrix(pstate->count,
@ -295,12 +297,14 @@ void stats2_corr_cov_get(void* pvstate, char* name1, char* name2, lrec_t* poutre
lrec_put(poutrec, keym, mlr_alloc_string_from_double(m, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyb, mlr_alloc_string_from_double(b, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyq, mlr_alloc_string_from_double(q, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyl1, mlr_alloc_string_from_double(l1, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyl2, mlr_alloc_string_from_double(l2, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv11, mlr_alloc_string_from_double(v1[0], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv12, mlr_alloc_string_from_double(v1[1], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv21, mlr_alloc_string_from_double(v2[0], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv22, mlr_alloc_string_from_double(v2[1], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
if (pstate->do_verbose) {
lrec_put(poutrec, keyl1, mlr_alloc_string_from_double(l1, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyl2, mlr_alloc_string_from_double(l2, pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv11, mlr_alloc_string_from_double(v1[0], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv12, mlr_alloc_string_from_double(v1[1], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv21, mlr_alloc_string_from_double(v2[0], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
lrec_put(poutrec, keyv22, mlr_alloc_string_from_double(v2[1], pstate->pstatx->ofmt), LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
}
}
} else {
char* suffix = (pstate->do_which == DO_CORR) ? "corr" : "cov";
@ -319,7 +323,7 @@ void stats2_corr_cov_get(void* pvstate, char* name1, char* name2, lrec_t* poutre
}
}
}
stats2_t* stats2_corr_cov_alloc(int do_which, static_context_t* pstatx) {
stats2_t* stats2_corr_cov_alloc(int do_which, int do_verbose, static_context_t* pstatx) {
stats2_t* pstats2 = mlr_malloc_or_die(sizeof(stats2_t));
stats2_corr_cov_state_t* pstate = mlr_malloc_or_die(sizeof(stats2_corr_cov_state_t));
pstate->count = 0LL;
@ -329,23 +333,24 @@ stats2_t* stats2_corr_cov_alloc(int do_which, static_context_t* pstatx) {
pstate->sumxy = 0.0;
pstate->sumy2 = 0.0;
pstate->do_which = do_which;
pstate->do_verbose = do_verbose;
pstate->pstatx = pstatx;
pstats2->pvstate = (void*)pstate;
pstats2->pput_func = &stats2_corr_cov_put;
pstats2->pget_func = &stats2_corr_cov_get;
return pstats2;
}
stats2_t* stats2_corr_alloc(static_context_t* pstatx) {
return stats2_corr_cov_alloc(DO_CORR, pstatx);
stats2_t* stats2_corr_alloc(static_context_t* pstatx, int do_verbose) {
return stats2_corr_cov_alloc(DO_CORR, do_verbose, pstatx);
}
stats2_t* stats2_cov_alloc(static_context_t* pstatx) {
return stats2_corr_cov_alloc(DO_COV, pstatx);
stats2_t* stats2_cov_alloc(static_context_t* pstatx, int do_verbose) {
return stats2_corr_cov_alloc(DO_COV, do_verbose, pstatx);
}
stats2_t* stats2_covx_alloc(static_context_t* pstatx) {
return stats2_corr_cov_alloc(DO_COVX, pstatx);
stats2_t* stats2_covx_alloc(static_context_t* pstatx, int do_verbose) {
return stats2_corr_cov_alloc(DO_COVX, do_verbose, pstatx);
}
stats2_t* stats2_linreg_pca_alloc(static_context_t* pstatx) {
return stats2_corr_cov_alloc(DO_LINREG_PCA, pstatx);
stats2_t* stats2_linreg_pca_alloc(static_context_t* pstatx, int do_verbose) {
return stats2_corr_cov_alloc(DO_LINREG_PCA, do_verbose, pstatx);
}
// ----------------------------------------------------------------
@ -355,19 +360,19 @@ typedef struct _stats2_lookup_t {
static_context_t* pstatx;
} stats2_lookup_t;
static stats2_lookup_t stats2_lookup_table[] = {
{"linreg", stats2_linreg_alloc},
{"r2", stats2_r2_alloc},
{"corr", stats2_corr_alloc},
{"cov", stats2_cov_alloc},
{"covx", stats2_covx_alloc},
{"linregpca", stats2_linreg_pca_alloc},
{"linreg-ols", stats2_linreg_ols_alloc},
{"r2", stats2_r2_alloc},
{"corr", stats2_corr_alloc},
{"cov", stats2_cov_alloc},
{"covx", stats2_covx_alloc},
{"linreg-pca", stats2_linreg_pca_alloc},
};
static int stats2_lookup_table_length = sizeof(stats2_lookup_table) / sizeof(stats2_lookup_table[0]);
static stats2_t* make_stats2(char* stats2_name, static_context_t* pstatx) {
static stats2_t* make_stats2(char* stats2_name, int do_verbose, static_context_t* pstatx) {
for (int i = 0; i < stats2_lookup_table_length; i++)
if (streq(stats2_name, stats2_lookup_table[i].name))
return stats2_lookup_table[i].pnew_func(pstatx);
return stats2_lookup_table[i].pnew_func(pstatx, do_verbose);
return NULL;
}
@ -378,7 +383,7 @@ typedef struct _mapper_stats2_state_t {
slls_t* pgroup_by_field_names;
lhmslv_t* pmaps_level_1;
int do_verbose;
} mapper_stats2_state_t;
// given: accumulate count,sum on values x,y group by a,b
@ -439,7 +444,7 @@ sllv_t* mapper_stats2_func(lrec_t* pinrec, context_t* pctx, void* pvstate) {
char* stats2_name = pc->value;
stats2_t* pstats2 = lhmsv_get(pmaps_level_3, stats2_name);
if (pstats2 == NULL) {
pstats2 = make_stats2(stats2_name, &pctx->statx);
pstats2 = make_stats2(stats2_name, pstate->do_verbose, &pctx->statx);
if (pstats2 == NULL) {
fprintf(stderr, "mlr stats2: accumulator \"%s\" not found.\n",
stats2_name);
@ -506,7 +511,7 @@ static void mapper_stats2_free(void* pvstate) {
}
mapper_t* mapper_stats2_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_name_pairs,
slls_t* pgroup_by_field_names)
slls_t* pgroup_by_field_names, int do_verbose)
{
mapper_t* pmapper = mlr_malloc_or_die(sizeof(mapper_t));
@ -515,6 +520,7 @@ mapper_t* mapper_stats2_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_n
pstate->pvalue_field_name_pairs = pvalue_field_name_pairs; // xxx validate length is even
pstate->pgroup_by_field_names = pgroup_by_field_names;
pstate->pmaps_level_1 = lhmslv_alloc();
pstate->do_verbose = do_verbose;
pmapper->pvstate = pstate;
pmapper->pmapper_process_func = mapper_stats2_func;
@ -526,7 +532,7 @@ mapper_t* mapper_stats2_alloc(slls_t* paccumulator_names, slls_t* pvalue_field_n
// ----------------------------------------------------------------
void mapper_stats2_usage(char* argv0, char* verb) {
fprintf(stdout, "Usage: %s %s [options]\n", argv0, verb);
fprintf(stdout, "-a {linreg,corr,...} Names of accumulators: one or more of\n");
fprintf(stdout, "-a {linreg-ols,corr,...} Names of accumulators: one or more of\n");
fprintf(stdout, " ");
for (int i = 0; i < stats2_lookup_table_length; i++) {
fprintf(stdout, " %s", stats2_lookup_table[i].name);
@ -535,12 +541,14 @@ void mapper_stats2_usage(char* argv0, char* verb) {
fprintf(stdout, "-f {a,b,c,d} Value-field names on which to compute statistics.\n");
fprintf(stdout, " There must be an even number of these.\n");
fprintf(stdout, "-g {d,e,f} Group-by-field names\n");
fprintf(stdout, "-v Print additional output for linreg-pca.\n");
}
mapper_t* mapper_stats2_parse_cli(int* pargi, int argc, char** argv) {
slls_t* paccumulator_names = NULL;
slls_t* pvalue_field_names = NULL;
slls_t* pgroup_by_field_names = slls_alloc();
int do_verbose = FALSE;
char* verb = argv[(*pargi)++];
@ -548,6 +556,7 @@ mapper_t* mapper_stats2_parse_cli(int* pargi, int argc, char** argv) {
ap_define_string_list_flag(pstate, "-a", &paccumulator_names);
ap_define_string_list_flag(pstate, "-f", &pvalue_field_names);
ap_define_string_list_flag(pstate, "-g", &pgroup_by_field_names);
ap_define_true_flag(pstate, "-v", &do_verbose);
if (!ap_parse(pstate, verb, pargi, argc, argv)) {
mapper_stats2_usage(argv[0], verb);
@ -563,7 +572,7 @@ mapper_t* mapper_stats2_parse_cli(int* pargi, int argc, char** argv) {
return NULL;
}
return mapper_stats2_alloc(paccumulator_names, pvalue_field_names, pgroup_by_field_names);
return mapper_stats2_alloc(paccumulator_names, pvalue_field_names, pgroup_by_field_names, do_verbose);
}
// ----------------------------------------------------------------

54
c/test/expected/out
View file

@ -231,33 +231,33 @@ zee wye 8.000000 8.000000 1 8.000000 8.000000 0.598554 0.598554 1
hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1 0.031442 0.031442 0.749551 0.749551 1 0.749551 0.749551
pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618
./test/../mlr --opprint stats2 -a linreg,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_m x_y_b x_y_r2 x_y_corr x_y_cov xy_y2_m xy_y2_b xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_m x2_x2_b x2_x2_r2 x2_x2_corr x2_x2_cov
cat pan 0.054420 0.481777 0.002504 0.050036 0.003777 0.950908 0.105754 0.435336 0.659800 0.041616 1.000000 0.000000 1.000000 1.000000 0.066303
pan wye -0.145486 0.584799 0.019479 -0.139568 -0.012683 0.908151 0.126628 0.438850 0.662457 0.046203 1.000000 0.000000 1.000000 1.000000 0.093192
wye cat 0.185913 0.377639 0.033002 0.181665 0.014494 0.969266 0.040602 0.561236 0.749157 0.052090 1.000000 0.000000 1.000000 1.000000 0.086883
dog hat 0.100096 0.448757 0.010462 0.102283 0.008036 0.919149 0.090504 0.507155 0.712148 0.045034 1.000000 0.000000 1.000000 1.000000 0.081226
dog pan -0.066834 0.590647 0.005924 -0.076969 -0.005709 0.726118 0.164937 0.315011 0.561259 0.034107 1.000000 0.000000 1.000000 1.000000 0.098975
pan pan 0.094932 0.461566 0.009768 0.098832 0.007175 0.822261 0.123441 0.465390 0.682195 0.039784 1.000000 0.000000 1.000000 1.000000 0.080908
hat hat 0.043668 0.405219 0.001324 0.036392 0.003037 1.128896 0.015188 0.708725 0.841858 0.060975 1.000000 0.000000 1.000000 1.000000 0.084636
wye hat 0.043018 0.496029 0.002197 0.046876 0.004023 0.720402 0.165623 0.353558 0.594608 0.038763 1.000000 0.000000 1.000000 1.000000 0.091675
pan hat 0.120797 0.448197 0.013060 0.114278 0.008987 0.962678 0.076920 0.622353 0.788893 0.054965 1.000000 0.000000 1.000000 1.000000 0.079553
cat hat 0.172391 0.464384 0.030150 0.173639 0.015030 0.904257 0.133482 0.498171 0.705812 0.055626 1.000000 0.000000 1.000000 1.000000 0.089895
hat wye -0.022975 0.496361 0.000514 -0.022665 -0.002000 0.971929 0.096088 0.386354 0.621574 0.040126 1.000000 0.000000 1.000000 1.000000 0.075309
dog dog 0.078397 0.489236 0.007619 0.087288 0.008214 0.776967 0.150999 0.408257 0.638950 0.049648 1.000000 0.000000 1.000000 1.000000 0.106525
wye dog 0.116403 0.425576 0.011048 0.105109 0.007867 0.925781 0.071192 0.501559 0.708208 0.046440 1.000000 0.000000 1.000000 1.000000 0.081433
wye wye -0.188354 0.613934 0.031156 -0.176512 -0.015876 0.876717 0.159179 0.325193 0.570257 0.042026 1.000000 0.000000 1.000000 1.000000 0.087513
dog wye 0.029527 0.502643 0.000913 0.030211 0.002391 0.904925 0.120816 0.432413 0.657581 0.042857 1.000000 0.000000 1.000000 1.000000 0.083924
cat dog 0.057573 0.408644 0.003442 0.058671 0.005320 0.884325 0.079999 0.479596 0.692528 0.044762 1.000000 0.000000 1.000000 1.000000 0.098206
hat pan -0.154393 0.564981 0.025143 -0.158566 -0.012756 0.911165 0.104362 0.397150 0.630199 0.035622 1.000000 0.000000 1.000000 1.000000 0.087879
cat wye -0.014851 0.564875 0.000224 -0.014982 -0.000966 0.878820 0.086362 0.463961 0.681147 0.041096 1.000000 0.000000 1.000000 1.000000 0.081922
hat cat -0.022859 0.498539 0.000610 -0.024689 -0.002116 0.840965 0.111121 0.373515 0.611158 0.036575 1.000000 0.000000 1.000000 1.000000 0.093075
dog cat 0.104057 0.428559 0.008705 0.093300 0.007122 1.080443 0.023866 0.547103 0.739664 0.050357 1.000000 0.000000 1.000000 1.000000 0.075381
hat dog 0.041849 0.427228 0.001918 0.043789 0.003856 0.776135 0.114930 0.372058 0.609966 0.040583 1.000000 0.000000 1.000000 1.000000 0.104033
pan dog 0.119510 0.467833 0.011427 0.106896 0.009302 0.948592 0.107556 0.541263 0.735706 0.056609 1.000000 0.000000 1.000000 1.000000 0.088101
cat cat 0.016257 0.425410 0.000273 0.016510 0.001350 0.930954 0.072476 0.446764 0.668404 0.036267 1.000000 0.000000 1.000000 1.000000 0.086429
pan cat -0.188523 0.616919 0.037036 -0.192447 -0.016206 0.781770 0.176617 0.278739 0.527958 0.032984 1.000000 0.000000 1.000000 1.000000 0.093193
wye pan 0.229443 0.444446 0.046722 0.216152 0.020367 0.887659 0.145052 0.462545 0.680107 0.064496 1.000000 0.000000 1.000000 1.000000 0.103497
./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_ols_m x_y_ols_b x_x_pca_m x_y_pca_b y_x_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_xy_pca_m xy_y2_pca_b y2_xy_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov
cat pan 0.054420 0.481777 3.636062 -1.221602 0.177683 0.002504 0.050036 0.003777 0.950908 0.105754 1.715574 -0.081719 0.830612 0.435336 0.659800 0.041616 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.066303
pan wye -0.145486 0.584799 -1.340927 1.199920 0.254025 0.019479 -0.139568 -0.012683 0.908151 0.126628 1.595150 -0.045034 0.824114 0.438850 0.662457 0.046203 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093192
wye cat 0.185913 0.377639 1.135325 -0.145894 0.309499 0.033002 0.181665 0.014494 0.969266 0.040602 1.406365 -0.081379 0.868480 0.561236 0.749157 0.052090 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.086883
dog hat 0.100096 0.448757 0.810749 0.097346 0.189256 0.010462 0.102283 0.008036 0.919149 0.090504 1.425774 -0.038344 0.846209 0.507155 0.712148 0.045034 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.081226
dog pan -0.066834 0.590647 -0.254112 0.688837 0.275316 0.005924 -0.076969 -0.005709 0.726118 0.164937 1.566309 -0.075073 0.749025 0.315011 0.561259 0.034107 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.098975
pan pan 0.094932 0.461566 0.672369 0.189898 0.192719 0.009768 0.098832 0.007175 0.822261 0.123441 1.312543 0.003200 0.820351 0.465390 0.682195 0.039784 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.080908
hat hat 0.043668 0.405219 10.170494 -5.125282 0.310513 0.001324 0.036392 0.003037 1.128896 0.015188 1.414166 -0.052514 0.922308 0.708725 0.841858 0.060975 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.084636
wye hat 0.043018 0.496029 0.254879 0.395780 0.177794 0.002197 0.046876 0.004023 0.720402 0.165623 1.376136 0.002792 0.760716 0.353558 0.594608 0.038763 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.091675
pan hat 0.120797 0.448197 1.597359 -0.325695 0.225137 0.013060 0.114278 0.008987 0.962678 0.076920 1.285796 -0.012566 0.887704 0.622353 0.788893 0.054965 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.079553
cat hat 0.172391 0.464384 0.959329 0.086790 0.296109 0.030150 0.173639 0.015030 0.904257 0.133482 1.415658 -0.008369 0.841567 0.498171 0.705812 0.055626 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.089895
hat wye -0.022975 0.496361 -1.765884 1.344268 0.051493 0.000514 -0.022665 -0.002000 0.971929 0.096088 1.989422 -0.142072 0.825656 0.386354 0.621574 0.040126 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.075309
dog dog 0.078397 0.489236 0.354494 0.351041 0.242210 0.007619 0.087288 0.008214 0.776967 0.150999 1.354405 -0.006432 0.792265 0.408257 0.638950 0.049648 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.106525
wye dog 0.116403 0.425576 2.367821 -0.777734 0.254607 0.011048 0.105109 0.007867 0.925781 0.071192 1.453590 -0.070509 0.845204 0.501559 0.708208 0.046440 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.081433
wye wye -0.188354 0.613934 -1.433772 1.217887 0.316070 0.031156 -0.176512 -0.015876 0.876717 0.159179 2.044493 -0.118503 0.795455 0.325193 0.570257 0.042026 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.087513
dog wye 0.029527 0.502643 0.496713 0.282511 0.073039 0.000913 0.030211 0.002391 0.904925 0.120816 1.609123 -0.052245 0.821822 0.432413 0.657581 0.042857 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.083924
cat dog 0.057573 0.408644 0.728479 0.071114 0.116103 0.003442 0.058671 0.005320 0.884325 0.079999 1.418207 -0.040344 0.832998 0.479596 0.692528 0.044762 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.098206
hat pan -0.154393 0.564981 -0.845852 0.911026 0.276955 0.025143 -0.158566 -0.012756 0.911165 0.104362 1.763740 -0.092987 0.814584 0.397150 0.630199 0.035622 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.087879
cat wye -0.014851 0.564875 -0.572708 0.892322 0.034146 0.000224 -0.014982 -0.000966 0.878820 0.086362 1.447244 -0.098657 0.827119 0.463961 0.681147 0.041096 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.081922
hat cat -0.022859 0.498539 -0.156242 0.565723 0.149344 0.000610 -0.024689 -0.002116 0.840965 0.111121 1.663518 -0.088942 0.793883 0.373515 0.611158 0.036575 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093075
dog cat 0.104057 0.428559 2.712382 -1.005787 0.250036 0.008705 0.093300 0.007122 1.080443 0.023866 1.653922 -0.133367 0.875586 0.547103 0.739664 0.050357 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.075381
hat dog 0.041849 0.427228 0.403977 0.254919 0.118494 0.001918 0.043789 0.003856 0.776135 0.114930 1.475403 -0.036508 0.779056 0.372058 0.609966 0.040583 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.104033
pan dog 0.119510 0.467833 2.492496 -0.761490 0.266455 0.011427 0.106896 0.009302 0.948592 0.107556 1.408389 -0.022846 0.860243 0.541263 0.735706 0.056609 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.088101
cat cat 0.016257 0.425410 0.432946 0.225535 0.044275 0.000273 0.016510 0.001350 0.930954 0.072476 1.624993 -0.072669 0.830029 0.446764 0.668404 0.036267 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.086429
pan cat -0.188523 0.616919 -0.898665 0.953923 0.324264 0.037036 -0.192447 -0.016206 0.781770 0.176617 2.020454 -0.113587 0.762332 0.278739 0.527958 0.032984 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093193
wye pan 0.229443 0.444446 1.313689 -0.098124 0.365811 0.046722 0.216152 0.020367 0.887659 0.145052 1.471906 -0.030176 0.827911 0.462545 0.680107 0.064496 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.103497
./test/../mlr --opprint step -a rsum,delta,counter -f x,y -g a ./test/input/abixy
a b i x y x_rsum x_delta x_counter y_rsum y_delta y_counter

54
c/test/output/out
View file

@ -231,33 +231,33 @@ zee wye 8.000000 8.000000 1 8.000000 8.000000 0.598554 0.598554 1
hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1 0.031442 0.031442 0.749551 0.749551 1 0.749551 0.749551
pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618
./test/../mlr --opprint stats2 -a linreg,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_m x_y_b x_y_r2 x_y_corr x_y_cov xy_y2_m xy_y2_b xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_m x2_x2_b x2_x2_r2 x2_x2_corr x2_x2_cov
cat pan 0.054420 0.481777 0.002504 0.050036 0.003777 0.950908 0.105754 0.435336 0.659800 0.041616 1.000000 0.000000 1.000000 1.000000 0.066303
pan wye -0.145486 0.584799 0.019479 -0.139568 -0.012683 0.908151 0.126628 0.438850 0.662457 0.046203 1.000000 0.000000 1.000000 1.000000 0.093192
wye cat 0.185913 0.377639 0.033002 0.181665 0.014494 0.969266 0.040602 0.561236 0.749157 0.052090 1.000000 0.000000 1.000000 1.000000 0.086883
dog hat 0.100096 0.448757 0.010462 0.102283 0.008036 0.919149 0.090504 0.507155 0.712148 0.045034 1.000000 0.000000 1.000000 1.000000 0.081226
dog pan -0.066834 0.590647 0.005924 -0.076969 -0.005709 0.726118 0.164937 0.315011 0.561259 0.034107 1.000000 0.000000 1.000000 1.000000 0.098975
pan pan 0.094932 0.461566 0.009768 0.098832 0.007175 0.822261 0.123441 0.465390 0.682195 0.039784 1.000000 0.000000 1.000000 1.000000 0.080908
hat hat 0.043668 0.405219 0.001324 0.036392 0.003037 1.128896 0.015188 0.708725 0.841858 0.060975 1.000000 0.000000 1.000000 1.000000 0.084636
wye hat 0.043018 0.496029 0.002197 0.046876 0.004023 0.720402 0.165623 0.353558 0.594608 0.038763 1.000000 0.000000 1.000000 1.000000 0.091675
pan hat 0.120797 0.448197 0.013060 0.114278 0.008987 0.962678 0.076920 0.622353 0.788893 0.054965 1.000000 0.000000 1.000000 1.000000 0.079553
cat hat 0.172391 0.464384 0.030150 0.173639 0.015030 0.904257 0.133482 0.498171 0.705812 0.055626 1.000000 0.000000 1.000000 1.000000 0.089895
hat wye -0.022975 0.496361 0.000514 -0.022665 -0.002000 0.971929 0.096088 0.386354 0.621574 0.040126 1.000000 0.000000 1.000000 1.000000 0.075309
dog dog 0.078397 0.489236 0.007619 0.087288 0.008214 0.776967 0.150999 0.408257 0.638950 0.049648 1.000000 0.000000 1.000000 1.000000 0.106525
wye dog 0.116403 0.425576 0.011048 0.105109 0.007867 0.925781 0.071192 0.501559 0.708208 0.046440 1.000000 0.000000 1.000000 1.000000 0.081433
wye wye -0.188354 0.613934 0.031156 -0.176512 -0.015876 0.876717 0.159179 0.325193 0.570257 0.042026 1.000000 0.000000 1.000000 1.000000 0.087513
dog wye 0.029527 0.502643 0.000913 0.030211 0.002391 0.904925 0.120816 0.432413 0.657581 0.042857 1.000000 0.000000 1.000000 1.000000 0.083924
cat dog 0.057573 0.408644 0.003442 0.058671 0.005320 0.884325 0.079999 0.479596 0.692528 0.044762 1.000000 0.000000 1.000000 1.000000 0.098206
hat pan -0.154393 0.564981 0.025143 -0.158566 -0.012756 0.911165 0.104362 0.397150 0.630199 0.035622 1.000000 0.000000 1.000000 1.000000 0.087879
cat wye -0.014851 0.564875 0.000224 -0.014982 -0.000966 0.878820 0.086362 0.463961 0.681147 0.041096 1.000000 0.000000 1.000000 1.000000 0.081922
hat cat -0.022859 0.498539 0.000610 -0.024689 -0.002116 0.840965 0.111121 0.373515 0.611158 0.036575 1.000000 0.000000 1.000000 1.000000 0.093075
dog cat 0.104057 0.428559 0.008705 0.093300 0.007122 1.080443 0.023866 0.547103 0.739664 0.050357 1.000000 0.000000 1.000000 1.000000 0.075381
hat dog 0.041849 0.427228 0.001918 0.043789 0.003856 0.776135 0.114930 0.372058 0.609966 0.040583 1.000000 0.000000 1.000000 1.000000 0.104033
pan dog 0.119510 0.467833 0.011427 0.106896 0.009302 0.948592 0.107556 0.541263 0.735706 0.056609 1.000000 0.000000 1.000000 1.000000 0.088101
cat cat 0.016257 0.425410 0.000273 0.016510 0.001350 0.930954 0.072476 0.446764 0.668404 0.036267 1.000000 0.000000 1.000000 1.000000 0.086429
pan cat -0.188523 0.616919 0.037036 -0.192447 -0.016206 0.781770 0.176617 0.278739 0.527958 0.032984 1.000000 0.000000 1.000000 1.000000 0.093193
wye pan 0.229443 0.444446 0.046722 0.216152 0.020367 0.887659 0.145052 0.462545 0.680107 0.064496 1.000000 0.000000 1.000000 1.000000 0.103497
./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_ols_m x_y_ols_b x_x_pca_m x_y_pca_b y_x_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_xy_pca_m xy_y2_pca_b y2_xy_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov
cat pan 0.054420 0.481777 3.636062 -1.221602 0.177683 0.002504 0.050036 0.003777 0.950908 0.105754 1.715574 -0.081719 0.830612 0.435336 0.659800 0.041616 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.066303
pan wye -0.145486 0.584799 -1.340927 1.199920 0.254025 0.019479 -0.139568 -0.012683 0.908151 0.126628 1.595150 -0.045034 0.824114 0.438850 0.662457 0.046203 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093192
wye cat 0.185913 0.377639 1.135325 -0.145894 0.309499 0.033002 0.181665 0.014494 0.969266 0.040602 1.406365 -0.081379 0.868480 0.561236 0.749157 0.052090 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.086883
dog hat 0.100096 0.448757 0.810749 0.097346 0.189256 0.010462 0.102283 0.008036 0.919149 0.090504 1.425774 -0.038344 0.846209 0.507155 0.712148 0.045034 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.081226
dog pan -0.066834 0.590647 -0.254112 0.688837 0.275316 0.005924 -0.076969 -0.005709 0.726118 0.164937 1.566309 -0.075073 0.749025 0.315011 0.561259 0.034107 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.098975
pan pan 0.094932 0.461566 0.672369 0.189898 0.192719 0.009768 0.098832 0.007175 0.822261 0.123441 1.312543 0.003200 0.820351 0.465390 0.682195 0.039784 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.080908
hat hat 0.043668 0.405219 10.170494 -5.125282 0.310513 0.001324 0.036392 0.003037 1.128896 0.015188 1.414166 -0.052514 0.922308 0.708725 0.841858 0.060975 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.084636
wye hat 0.043018 0.496029 0.254879 0.395780 0.177794 0.002197 0.046876 0.004023 0.720402 0.165623 1.376136 0.002792 0.760716 0.353558 0.594608 0.038763 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.091675
pan hat 0.120797 0.448197 1.597359 -0.325695 0.225137 0.013060 0.114278 0.008987 0.962678 0.076920 1.285796 -0.012566 0.887704 0.622353 0.788893 0.054965 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.079553
cat hat 0.172391 0.464384 0.959329 0.086790 0.296109 0.030150 0.173639 0.015030 0.904257 0.133482 1.415658 -0.008369 0.841567 0.498171 0.705812 0.055626 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.089895
hat wye -0.022975 0.496361 -1.765884 1.344268 0.051493 0.000514 -0.022665 -0.002000 0.971929 0.096088 1.989422 -0.142072 0.825656 0.386354 0.621574 0.040126 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.075309
dog dog 0.078397 0.489236 0.354494 0.351041 0.242210 0.007619 0.087288 0.008214 0.776967 0.150999 1.354405 -0.006432 0.792265 0.408257 0.638950 0.049648 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.106525
wye dog 0.116403 0.425576 2.367821 -0.777734 0.254607 0.011048 0.105109 0.007867 0.925781 0.071192 1.453590 -0.070509 0.845204 0.501559 0.708208 0.046440 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.081433
wye wye -0.188354 0.613934 -1.433772 1.217887 0.316070 0.031156 -0.176512 -0.015876 0.876717 0.159179 2.044493 -0.118503 0.795455 0.325193 0.570257 0.042026 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.087513
dog wye 0.029527 0.502643 0.496713 0.282511 0.073039 0.000913 0.030211 0.002391 0.904925 0.120816 1.609123 -0.052245 0.821822 0.432413 0.657581 0.042857 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.083924
cat dog 0.057573 0.408644 0.728479 0.071114 0.116103 0.003442 0.058671 0.005320 0.884325 0.079999 1.418207 -0.040344 0.832998 0.479596 0.692528 0.044762 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.098206
hat pan -0.154393 0.564981 -0.845852 0.911026 0.276955 0.025143 -0.158566 -0.012756 0.911165 0.104362 1.763740 -0.092987 0.814584 0.397150 0.630199 0.035622 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.087879
cat wye -0.014851 0.564875 -0.572708 0.892322 0.034146 0.000224 -0.014982 -0.000966 0.878820 0.086362 1.447244 -0.098657 0.827119 0.463961 0.681147 0.041096 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.081922
hat cat -0.022859 0.498539 -0.156242 0.565723 0.149344 0.000610 -0.024689 -0.002116 0.840965 0.111121 1.663518 -0.088942 0.793883 0.373515 0.611158 0.036575 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093075
dog cat 0.104057 0.428559 2.712382 -1.005787 0.250036 0.008705 0.093300 0.007122 1.080443 0.023866 1.653922 -0.133367 0.875586 0.547103 0.739664 0.050357 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.075381
hat dog 0.041849 0.427228 0.403977 0.254919 0.118494 0.001918 0.043789 0.003856 0.776135 0.114930 1.475403 -0.036508 0.779056 0.372058 0.609966 0.040583 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.104033
pan dog 0.119510 0.467833 2.492496 -0.761490 0.266455 0.011427 0.106896 0.009302 0.948592 0.107556 1.408389 -0.022846 0.860243 0.541263 0.735706 0.056609 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.088101
cat cat 0.016257 0.425410 0.432946 0.225535 0.044275 0.000273 0.016510 0.001350 0.930954 0.072476 1.624993 -0.072669 0.830029 0.446764 0.668404 0.036267 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.086429
pan cat -0.188523 0.616919 -0.898665 0.953923 0.324264 0.037036 -0.192447 -0.016206 0.781770 0.176617 2.020454 -0.113587 0.762332 0.278739 0.527958 0.032984 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093193
wye pan 0.229443 0.444446 1.313689 -0.098124 0.365811 0.046722 0.216152 0.020367 0.887659 0.145052 1.471906 -0.030176 0.827911 0.462545 0.680107 0.064496 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.103497
./test/../mlr --opprint step -a rsum,delta,counter -f x,y -g a ./test/input/abixy
a b i x y x_rsum x_delta x_counter y_rsum y_delta y_counter

2
c/test/run
View file

@ -82,7 +82,7 @@ announce STATS
run_command $mlr count-distinct -f a,b $indir/small $indir/abixy
run_command $mlr --opprint stats1 -a avg,sum,count,min,max -f i,x,y -g a,b $indir/abixy
run_command $mlr --opprint stats2 -a linreg,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b $indir/abixy-wide
run_command $mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b $indir/abixy-wide
run_command $mlr --opprint step -a rsum,delta,counter -f x,y -g a $indir/abixy
run_command $mlr --opprint histogram -f x,y --lo 0 --hi 1 --nbins 20 $indir/small

6
doc/content-for-data-examples.html
View file

@ -19,11 +19,11 @@ cat flins.csv | mlr --icsv --oxtab stats1 -a min,avg,max -f eq_site_deductible,h
echo
cat flins.csv | mlr --icsv --oxtab stats1 -a min,avg,max -f eq_site_deductible,hu_site_deductible,fl_site_deductible,fr_site_deductible -g county
echo
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg,r2 -f eq_site_deductible,tiv_2012 -g county
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg-ols,r2 -f eq_site_deductible,tiv_2012 -g county
echo
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg,r2 -f tiv_2011,tiv_2012
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012
echo
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg,r2 -f tiv_2011,tiv_2012 -g county
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012 -g county
</pre>
<p/> xxx plaintext hardlinks

2
doc/content-for-reference.html
View file

@ -442,7 +442,7 @@ fields, optionally categorized by one or more fields.
<table><tr><td>
POKI_RUN_COMMAND{{mlr --oxtab put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' then stats2 -a cov,corr -f x,y,y,y,x2,xy,x2,y2 data/medium}}HERE
</td></tr><tr><td>
POKI_RUN_COMMAND{{mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' then stats2 -a linreg,r2 -f x,y,y,y,xy,y2 -g a data/medium}}HERE
POKI_RUN_COMMAND{{mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' then stats2 -a linreg-ols,r2 -f x,y,y,y,xy,y2 -g a data/medium}}HERE
</td></tr></table>
<p/>Here&rsquo;s an example simple line-fit. The <tt>x</tt> and <tt>y</tt>

6
doc/data-examples.html
View file

@ -167,11 +167,11 @@ cat flins.csv | mlr --icsv --oxtab stats1 -a min,avg,max -f eq_site_deductible,h
echo
cat flins.csv | mlr --icsv --oxtab stats1 -a min,avg,max -f eq_site_deductible,hu_site_deductible,fl_site_deductible,fr_site_deductible -g county
echo
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg,r2 -f eq_site_deductible,tiv_2012 -g county
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg-ols,r2 -f eq_site_deductible,tiv_2012 -g county
echo
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg,r2 -f tiv_2011,tiv_2012
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012
echo
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg,r2 -f tiv_2011,tiv_2012 -g county
cat flins.csv | mlr --icsv --opprint stats2 -a corr,linreg-ols,r2 -f tiv_2011,tiv_2012 -g county
</pre>
<p/> xxx plaintext hardlinks

14
doc/data/linreg-example.txt
View file

@ -1,13 +1,13 @@
mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)' data/medium > data/medium-squares
mlr --ofs newline stats2 -a linreg -f x,y data/medium-squares
x_y_m=0.764675
x_y_b=0.124841
mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares
x_y_ols_m=0.764675
x_y_ols_b=0.124841
# Set x_y_m and x_y_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg -f x,y data/medium-squares)
# Set x_y_ols_m and x_y_ols_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares)
# In addition to x and y, make a new yfit which is the line fit. Plot using your favorite tool.
mlr --onidx put '$yfit='$x_y_m'*$x+'$x_y_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg example'
mlr --onidx put '$yfit='$x_y_ols_m'*$x+'$x_y_ols_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg-ols example'

32
doc/reference.html
View file

@ -1122,8 +1122,8 @@ zee 0.504242 0.502997
<pre>
$ mlr stats2 --help
Usage: mlr stats2 [options]
-a {linreg,corr,...} Names of accumulators: one or more of
linreg r2 corr cov covx
-a {linreg-ols,corr,...} Names of accumulators: one or more of
linreg-ols r2 corr cov covx linreg-pca
-f {a,b,c,d} Value-field names on which to compute statistics.
There must be an even number of these.
-g {d,e,f} Group-by-field names
@ -1154,13 +1154,13 @@ x2_y2_corr -0.003425
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' then stats2 -a linreg,r2 -f x,y,y,y,xy,y2 -g a data/medium
a x_y_m x_y_b x_y_r2 y_y_m y_y_b y_y_r2 xy_y2_m xy_y2_b xy_y2_r2
pan 0.017026 0.500403 0.000287 1.000000 0.000000 1.000000 0.878132 0.119082 0.417498
eks 0.040780 0.481402 0.001646 1.000000 0.000000 1.000000 0.897873 0.107341 0.455632
wye -0.039153 0.525510 0.001505 1.000000 0.000000 1.000000 0.853832 0.126745 0.389917
zee 0.002781 0.504307 0.000008 1.000000 0.000000 1.000000 0.852444 0.124017 0.393566
hat -0.018621 0.517901 0.000352 1.000000 0.000000 1.000000 0.841230 0.135573 0.368794
$ mlr --opprint put '$x2=$x*$x; $xy=$x*$y; $y2=$y**2' then stats2 -a linreg-ols,r2 -f x,y,y,y,xy,y2 -g a data/medium
a x_y_ols_m x_y_ols_b x_y_r2 y_y_ols_m y_y_ols_b y_y_r2 xy_y2_ols_m xy_y2_ols_b xy_y2_r2
pan 0.017026 0.500403 0.000287 1.000000 0.000000 1.000000 0.878132 0.119082 0.417498
eks 0.040780 0.481402 0.001646 1.000000 0.000000 1.000000 0.897873 0.107341 0.455632
wye -0.039153 0.525510 0.001505 1.000000 0.000000 1.000000 0.853832 0.126745 0.389917
zee 0.002781 0.504307 0.000008 1.000000 0.000000 1.000000 0.852444 0.124017 0.393566
hat -0.018621 0.517901 0.000352 1.000000 0.000000 1.000000 0.841230 0.135573 0.368794
</pre>
</div>
<p/>
@ -1176,16 +1176,16 @@ distributed on the unit interval. Here we remove half the data and fit a line to
mlr filter '($x&lt;.5 &amp;&amp; $y&lt;.5) || ($x&gt;.5 &amp;&amp; $y&gt;.5)' data/medium &gt; data/medium-squares
mlr --ofs newline stats2 -a linreg -f x,y data/medium-squares
x_y_m=0.764675
x_y_b=0.124841
mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares
x_y_ols_m=0.764675
x_y_ols_b=0.124841
# Set x_y_m and x_y_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg -f x,y data/medium-squares)
# Set x_y_ols_m and x_y_ols_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares)
# In addition to x and y, make a new yfit which is the line fit. Plot using your favorite tool.
mlr --onidx put '$yfit='$x_y_m'*$x+'$x_y_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg example'
mlr --onidx put '$yfit='$x_y_ols_m'*$x+'$x_y_ols_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg-ols example'
</pre>
</div>
<p/>