Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

update docs re linreg-pca

Browse source
This commit is contained in:
John Kerl 2015-05-05 20:57:19 -07:00
parent 4679e39c9d
commit b0e9f7ca36
7 changed files with 28 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

16
c/mapping/mapper_stats2.c
View file

@ -239,15 +239,15 @@ void stats2_corr_cov_get(void* pvstate, char* name1, char* name2, lrec_t* poutre
lrec_put(poutrec, key11, val11, LREC_FREE_ENTRY_KEY|LREC_FREE_ENTRY_VALUE);
}
} else if (pstate->do_which == DO_LINREG_PCA) {
char* keym = mlr_paste_4_strings(name1, "_", name1, "_pca_m");
char* keym = mlr_paste_4_strings(name1, "_", name2, "_pca_m");
char* keyb = mlr_paste_4_strings(name1, "_", name2, "_pca_b");
char* keyq = mlr_paste_4_strings(name2, "_", name1, "_pca_quality");
char* keyl1 = mlr_paste_4_strings(name2, "_", name1, "_pca_eival1");
char* keyl2 = mlr_paste_4_strings(name2, "_", name1, "_pca_eival2");
char* keyv11 = mlr_paste_4_strings(name2, "_", name1, "_pca_eivec11");
char* keyv12 = mlr_paste_4_strings(name2, "_", name1, "_pca_eivec12");
char* keyv21 = mlr_paste_4_strings(name2, "_", name1, "_pca_eivec21");
char* keyv22 = mlr_paste_4_strings(name2, "_", name1, "_pca_eivec22");
char* keyq = mlr_paste_4_strings(name1, "_", name2, "_pca_quality");
char* keyl1 = mlr_paste_4_strings(name1, "_", name2, "_pca_eival1");
char* keyl2 = mlr_paste_4_strings(name1, "_", name2, "_pca_eival2");
char* keyv11 = mlr_paste_4_strings(name1, "_", name2, "_pca_eivec11");
char* keyv12 = mlr_paste_4_strings(name1, "_", name2, "_pca_eivec12");
char* keyv21 = mlr_paste_4_strings(name1, "_", name2, "_pca_eivec21");
char* keyv22 = mlr_paste_4_strings(name1, "_", name2, "_pca_eivec22");
if (pstate->count < 2LL) {
lrec_put(poutrec, keym, "", LREC_FREE_ENTRY_KEY);
lrec_put(poutrec, keyb, "", LREC_FREE_ENTRY_KEY);

2
c/test/expected/out
View file

@ -232,7 +232,7 @@ hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1
pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618
./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_ols_m x_y_ols_b x_x_pca_m x_y_pca_b y_x_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_xy_pca_m xy_y2_pca_b y2_xy_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov
a b x_y_ols_m x_y_ols_b x_y_pca_m x_y_pca_b x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_pca_m xy_y2_pca_b xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov
cat pan 0.054420 0.481777 3.636062 -1.221602 0.177683 0.002504 0.050036 0.003777 0.950908 0.105754 1.715574 -0.081719 0.830612 0.435336 0.659800 0.041616 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.066303
pan wye -0.145486 0.584799 -1.340927 1.199920 0.254025 0.019479 -0.139568 -0.012683 0.908151 0.126628 1.595150 -0.045034 0.824114 0.438850 0.662457 0.046203 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093192
wye cat 0.185913 0.377639 1.135325 -0.145894 0.309499 0.033002 0.181665 0.014494 0.969266 0.040602 1.406365 -0.081379 0.868480 0.561236 0.749157 0.052090 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.086883

2
c/test/output/out
View file

@ -232,7 +232,7 @@ hat wye 9.000000 9.000000 1 9.000000 9.000000 0.031442 0.031442 1
pan wye 10.000000 10.000000 1 10.000000 10.000000 0.502626 0.502626 1 0.502626 0.502626 0.952618 0.952618 1 0.952618 0.952618
./test/../mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2,x2,x2 -g a,b ./test/input/abixy-wide
a b x_y_ols_m x_y_ols_b x_x_pca_m x_y_pca_b y_x_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_xy_pca_m xy_y2_pca_b y2_xy_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov
a b x_y_ols_m x_y_ols_b x_y_pca_m x_y_pca_b x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_pca_m xy_y2_pca_b xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov x2_x2_ols_m x2_x2_ols_b x2_x2_pca_m x2_x2_pca_b x2_x2_pca_quality x2_x2_r2 x2_x2_corr x2_x2_cov
cat pan 0.054420 0.481777 3.636062 -1.221602 0.177683 0.002504 0.050036 0.003777 0.950908 0.105754 1.715574 -0.081719 0.830612 0.435336 0.659800 0.041616 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.066303
pan wye -0.145486 0.584799 -1.340927 1.199920 0.254025 0.019479 -0.139568 -0.012683 0.908151 0.126628 1.595150 -0.045034 0.824114 0.438850 0.662457 0.046203 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.093192
wye cat 0.185913 0.377639 1.135325 -0.145894 0.309499 0.033002 0.181665 0.014494 0.969266 0.040602 1.406365 -0.081379 0.868480 0.561236 0.749157 0.052090 1.000000 0.000000 1.000000 0.000000 1.000000 1.000000 1.000000 0.086883

6
doc/content-for-reference.html
View file

@ -458,11 +458,7 @@ plotting; here&rsquo;s a screenshot.
<img src="data/linreg-example.jpg"/>
</center>
<p/>
(Perhaps surprisingly, the fit line doesn&rsquo;t have slope 1. This appears to
be due to Miller&rsquo;s doing RMS minimization of vertical residuals; a PCA
analysis gives slope 1 on the dominant eigenvector &mdash; thanks Drew Kunas for
a good conversation!)
<p/> (Thanks Drew Kunas for a good conversation about PCA!)
<!-- ================================================================ -->
<h2>step</h2>

BIN
doc/data/linreg-example.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 109 KiB

After

Width:  |  Height:  |  Size: 137 KiB

Before After
Before After

15
doc/data/linreg-example.txt
View file

@ -1,13 +1,14 @@
mlr filter '($x<.5 && $y<.5) || ($x>.5 && $y>.5)' data/medium > data/medium-squares
mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares
x_y_ols_m=0.764675
x_y_ols_b=0.124841
mlr --ofs newline stats2 -a linreg-pca -f x,y data/medium-squares
x_y_pca_m=1.014419
x_y_pca_b=0.000308
x_y_pca_quality=0.861354
# Set x_y_ols_m and x_y_ols_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares)
# Set x_y_pca_m and x_y_pca_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg-pca -f x,y data/medium-squares)
# In addition to x and y, make a new yfit which is the line fit. Plot using your favorite tool.
mlr --onidx put '$yfit='$x_y_ols_m'*$x+'$x_y_ols_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg-ols example'
mlr --onidx put '$yfit='$x_y_pca_m'*$x+'$x_y_pca_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg-pca example' -xmin 0 -xmax 1 -ymin 0 -ymax 1

21
doc/reference.html
View file

@ -1177,16 +1177,17 @@ distributed on the unit interval. Here we remove half the data and fit a line to
mlr filter '($x&lt;.5 &amp;&amp; $y&lt;.5) || ($x&gt;.5 &amp;&amp; $y&gt;.5)' data/medium &gt; data/medium-squares
mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares
x_y_ols_m=0.764675
x_y_ols_b=0.124841
mlr --ofs newline stats2 -a linreg-pca -f x,y data/medium-squares
x_y_pca_m=1.014419
x_y_pca_b=0.000308
x_y_pca_quality=0.861354
# Set x_y_ols_m and x_y_ols_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg-ols -f x,y data/medium-squares)
# Set x_y_pca_m and x_y_pca_b as shell variables
eval $(mlr --ofs newline stats2 -a linreg-pca -f x,y data/medium-squares)
# In addition to x and y, make a new yfit which is the line fit. Plot using your favorite tool.
mlr --onidx put '$yfit='$x_y_ols_m'*$x+'$x_y_ols_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg-ols example'
mlr --onidx put '$yfit='$x_y_pca_m'*$x+'$x_y_pca_b then cut -x -f a,b,i data/medium-squares \
| pgr -p -title 'linreg-pca example' -xmin 0 -xmax 1 -ymin 0 -ymax 1
</pre>
</div>
<p/>
@ -1198,11 +1199,7 @@ plotting; here&rsquo;s a screenshot.
<img src="data/linreg-example.jpg"/>
</center>
<p/>
(Perhaps surprisingly, the fit line doesn&rsquo;t have slope 1. This appears to
be due to Miller&rsquo;s doing RMS minimization of vertical residuals; a PCA
analysis gives slope 1 on the dominant eigenvector &mdash; thanks Drew Kunas for
a good conversation!)
<p/> (Thanks Drew Kunas for a good conversation about PCA!)
<!-- ================================================================ -->
<h2>step</h2> <a id="step"/>