mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-26 17:28:13 +00:00
Port stats2 verb from C to Go (#512)
* Port stats2 verb from C to Go * testing stats2 * neaten
This commit is contained in:
John Kerl
GitHub
parent
b97cadf374
commit
0dfe488199
71 changed files with 3529 additions and 926 deletions
3
go/build
3
go/build
|
|
@ -45,3 +45,6 @@ if [ "$do_wips" = "true" ]; then
|
|||
mlr regtest $verbose regtest/cases-pending-go-port
|
||||
fi
|
||||
echo
|
||||
|
||||
# Run the auto-formatter
|
||||
go fmt ./...
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 regtest/input/abixy-wide
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
x_y_ols_m x_y_ols_b x_y_ols_n x_y_pca_m x_y_pca_b x_y_pca_n x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_ols_n xy_y2_pca_m xy_y2_pca_b xy_y2_pca_n xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov
|
||||
0.028351 0.487644 2000 1.332924 -0.170590 2000 0.056909 0.000791 0.028120 0.002330 0.893610 0.107060 2000 1.529534 -0.055477 2000 0.824336 0.447971 0.669306 0.045036
|
||||
|
|
@ -1 +0,0 @@
|
|||
mlr --opprint stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 -g a,b regtest/input/abixy-wide
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
a b x_y_ols_m x_y_ols_b x_y_ols_n x_y_pca_m x_y_pca_b x_y_pca_n x_y_pca_quality x_y_r2 x_y_corr x_y_cov xy_y2_ols_m xy_y2_ols_b xy_y2_ols_n xy_y2_pca_m xy_y2_pca_b xy_y2_pca_n xy_y2_pca_quality xy_y2_r2 xy_y2_corr xy_y2_cov
|
||||
cat pan 0.054420 0.481777 89 3.636062 -1.221602 89 0.177683 0.002504 0.050036 0.003777 0.950908 0.105754 89 1.715574 -0.081719 89 0.830612 0.435336 0.659800 0.041616
|
||||
pan wye -0.145486 0.584799 78 -1.340927 1.199920 78 0.254025 0.019479 -0.139568 -0.012683 0.908151 0.126628 78 1.595150 -0.045034 78 0.824114 0.438850 0.662457 0.046203
|
||||
wye cat 0.185913 0.377639 74 1.135325 -0.145894 74 0.309499 0.033002 0.181665 0.014494 0.969266 0.040602 74 1.406365 -0.081379 74 0.868480 0.561236 0.749157 0.052090
|
||||
dog hat 0.100096 0.448757 88 0.810749 0.097346 88 0.189256 0.010462 0.102283 0.008036 0.919149 0.090504 88 1.425774 -0.038344 88 0.846209 0.507155 0.712148 0.045034
|
||||
dog pan -0.066834 0.590647 87 -0.254112 0.688837 87 0.275316 0.005924 -0.076969 -0.005709 0.726118 0.164937 87 1.566309 -0.075073 87 0.749025 0.315011 0.561259 0.034107
|
||||
pan pan 0.094932 0.461566 77 0.672369 0.189898 77 0.192719 0.009768 0.098832 0.007175 0.822261 0.123441 77 1.312543 0.003200 77 0.820351 0.465390 0.682195 0.039784
|
||||
hat hat 0.043668 0.405219 88 10.170494 -5.125282 88 0.310513 0.001324 0.036392 0.003037 1.128896 0.015188 88 1.414166 -0.052514 88 0.922308 0.708725 0.841858 0.060975
|
||||
wye hat 0.043018 0.496029 87 0.254879 0.395780 87 0.177794 0.002197 0.046876 0.004023 0.720402 0.165623 87 1.376136 0.002792 87 0.760716 0.353558 0.594608 0.038763
|
||||
pan hat 0.120797 0.448197 67 1.597359 -0.325695 67 0.225137 0.013060 0.114278 0.008987 0.962678 0.076920 67 1.285796 -0.012566 67 0.887704 0.622353 0.788893 0.054965
|
||||
cat hat 0.172391 0.464384 90 0.959329 0.086790 90 0.296109 0.030150 0.173639 0.015030 0.904257 0.133482 90 1.415658 -0.008369 90 0.841567 0.498171 0.705812 0.055626
|
||||
hat wye -0.022975 0.496361 70 -1.765884 1.344268 70 0.051493 0.000514 -0.022665 -0.002000 0.971929 0.096088 70 1.989422 -0.142072 70 0.825656 0.386354 0.621574 0.040126
|
||||
dog dog 0.078397 0.489236 87 0.354494 0.351041 87 0.242210 0.007619 0.087288 0.008214 0.776967 0.150999 87 1.354405 -0.006432 87 0.792265 0.408257 0.638950 0.049648
|
||||
wye dog 0.116403 0.425576 76 2.367821 -0.777734 76 0.254607 0.011048 0.105109 0.007867 0.925781 0.071192 76 1.453590 -0.070509 76 0.845204 0.501559 0.708208 0.046440
|
||||
wye wye -0.188354 0.613934 67 -1.433772 1.217887 67 0.316070 0.031156 -0.176512 -0.015876 0.876717 0.159179 67 2.044493 -0.118503 67 0.795455 0.325193 0.570257 0.042026
|
||||
dog wye 0.029527 0.502643 79 0.496713 0.282511 79 0.073039 0.000913 0.030211 0.002391 0.904925 0.120816 79 1.609123 -0.052245 79 0.821822 0.432413 0.657581 0.042857
|
||||
cat dog 0.057573 0.408644 78 0.728479 0.071114 78 0.116103 0.003442 0.058671 0.005320 0.884325 0.079999 78 1.418207 -0.040344 78 0.832998 0.479596 0.692528 0.044762
|
||||
hat pan -0.154393 0.564981 85 -0.845852 0.911026 85 0.276955 0.025143 -0.158566 -0.012756 0.911165 0.104362 85 1.763740 -0.092987 85 0.814584 0.397150 0.630199 0.035622
|
||||
cat wye -0.014851 0.564875 77 -0.572708 0.892322 77 0.034146 0.000224 -0.014982 -0.000966 0.878820 0.086362 77 1.447244 -0.098657 77 0.827119 0.463961 0.681147 0.041096
|
||||
hat cat -0.022859 0.498539 88 -0.156242 0.565723 88 0.149344 0.000610 -0.024689 -0.002116 0.840965 0.111121 88 1.663518 -0.088942 88 0.793883 0.373515 0.611158 0.036575
|
||||
dog cat 0.104057 0.428559 83 2.712382 -1.005787 83 0.250036 0.008705 0.093300 0.007122 1.080443 0.023866 83 1.653922 -0.133367 83 0.875586 0.547103 0.739664 0.050357
|
||||
hat dog 0.041849 0.427228 78 0.403977 0.254919 78 0.118494 0.001918 0.043789 0.003856 0.776135 0.114930 78 1.475403 -0.036508 78 0.779056 0.372058 0.609966 0.040583
|
||||
pan dog 0.119510 0.467833 73 2.492496 -0.761490 73 0.266455 0.011427 0.106896 0.009302 0.948592 0.107556 73 1.408389 -0.022846 73 0.860243 0.541263 0.735706 0.056609
|
||||
cat cat 0.016257 0.425410 79 0.432946 0.225535 79 0.044275 0.000273 0.016510 0.001350 0.930954 0.072476 79 1.624993 -0.072669 79 0.830029 0.446764 0.668404 0.036267
|
||||
pan cat -0.188523 0.616919 89 -0.898665 0.953923 89 0.324264 0.037036 -0.192447 -0.016206 0.781770 0.176617 89 2.020454 -0.113587 89 0.762332 0.278739 0.527958 0.032984
|
||||
wye pan 0.229443 0.444446 66 1.313689 -0.098124 66 0.365811 0.046722 0.216152 0.020367 0.887659 0.145052 66 1.471906 -0.030176 66 0.827911 0.462545 0.680107 0.064496
|
||||
|
|
@ -1 +0,0 @@
|
|||
mlr --oxtab stats2 -s -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 regtest/input/abixy-wide-short
|
||||
|
|
@ -1,579 +0,0 @@
|
|||
a cat
|
||||
b pan
|
||||
i 1
|
||||
x 0.5117389009583777
|
||||
y 0.08295224980036853
|
||||
x2 0.2618767027540883
|
||||
xy 0.0424498931448654
|
||||
y2 0.006881075746942741
|
||||
x_y_ols_m
|
||||
x_y_ols_b
|
||||
x_y_ols_n 1
|
||||
x_y_pca_m
|
||||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
xy_y2_pca_m
|
||||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b wye
|
||||
i 2
|
||||
x 0.5225940442098578
|
||||
y 0.511678736087022
|
||||
x2 0.27310453504361476
|
||||
xy 0.2674002600279053
|
||||
y2 0.26181512896361225
|
||||
x_y_ols_m 39.495240
|
||||
x_y_ols_b -20.128298
|
||||
x_y_ols_n 2
|
||||
x_y_pca_m 39.495240
|
||||
x_y_pca_b -20.128298
|
||||
x_y_pca_n 2
|
||||
x_y_pca_quality 1.000000
|
||||
x_y_r2 1.000000
|
||||
x_y_corr 1.000000
|
||||
x_y_cov 0.002327
|
||||
xy_y2_ols_m 1.133290
|
||||
xy_y2_ols_b -0.041227
|
||||
xy_y2_ols_n 2
|
||||
xy_y2_pca_m 1.133290
|
||||
xy_y2_pca_b -0.041227
|
||||
xy_y2_pca_n 2
|
||||
xy_y2_pca_quality 1.000000
|
||||
xy_y2_r2 1.000000
|
||||
xy_y2_corr 1.000000
|
||||
xy_y2_cov 0.028674
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 3
|
||||
x 0.8150401717873625
|
||||
y 0.07989551500795256
|
||||
x2 0.6642904816271734
|
||||
xy 0.06511805427712146
|
||||
y2 0.006383293318385972
|
||||
x_y_ols_m -0.689881
|
||||
x_y_ols_b 0.650125
|
||||
x_y_ols_n 3
|
||||
x_y_pca_m -2.057762
|
||||
x_y_pca_b 1.493365
|
||||
x_y_pca_n 3
|
||||
x_y_pca_quality 0.725269
|
||||
x_y_r2 0.228338
|
||||
x_y_corr -0.477847
|
||||
x_y_cov -0.020424
|
||||
xy_y2_ols_m 1.184397
|
||||
xy_y2_ols_b -0.056344
|
||||
xy_y2_ols_n 3
|
||||
xy_y2_pca_m 1.190469
|
||||
xy_y2_pca_b -0.057103
|
||||
xy_y2_pca_n 3
|
||||
xy_y2_pca_quality 0.997884
|
||||
xy_y2_r2 0.991315
|
||||
xy_y2_corr 0.995648
|
||||
xy_y2_cov 0.018168
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 4
|
||||
x 0.4488733555675044
|
||||
y 0.5730530513123552
|
||||
x2 0.20148728933843124
|
||||
xy 0.25722824606077416
|
||||
y2 0.32838979961840076
|
||||
x_y_ols_m -1.054065
|
||||
x_y_ols_b 0.917520
|
||||
x_y_ols_n 4
|
||||
x_y_pca_m -2.068130
|
||||
x_y_pca_b 1.500163
|
||||
x_y_pca_n 4
|
||||
x_y_pca_quality 0.845806
|
||||
x_y_r2 0.416082
|
||||
x_y_corr -0.645044
|
||||
x_y_cov -0.028205
|
||||
xy_y2_ols_m 1.365739
|
||||
xy_y2_ols_b -0.064987
|
||||
xy_y2_ols_n 4
|
||||
xy_y2_pca_m 1.406929
|
||||
xy_y2_pca_b -0.071497
|
||||
xy_y2_pca_n 4
|
||||
xy_y2_pca_quality 0.989979
|
||||
xy_y2_r2 0.956155
|
||||
xy_y2_corr 0.977832
|
||||
xy_y2_cov 0.019937
|
||||
|
||||
a dog
|
||||
b pan
|
||||
i 5
|
||||
x 0.2946557960430134
|
||||
y 0.6850437256584863
|
||||
x2 0.08682203814174191
|
||||
xy 0.20185210430817294
|
||||
y2 0.46928490606405937
|
||||
x_y_ols_m -1.176419
|
||||
x_y_ols_b 0.996592
|
||||
x_y_ols_n 5
|
||||
x_y_pca_m -1.681618
|
||||
x_y_pca_b 1.258579
|
||||
x_y_pca_n 5
|
||||
x_y_pca_quality 0.899128
|
||||
x_y_r2 0.607344
|
||||
x_y_corr -0.779323
|
||||
x_y_cov -0.042043
|
||||
xy_y2_ols_m 1.565652
|
||||
xy_y2_ols_b -0.046615
|
||||
xy_y2_ols_n 5
|
||||
xy_y2_pca_m 2.169036
|
||||
xy_y2_pca_b -0.147265
|
||||
xy_y2_pca_n 5
|
||||
xy_y2_pca_quality 0.936719
|
||||
xy_y2_r2 0.667168
|
||||
xy_y2_corr 0.816804
|
||||
xy_y2_cov 0.017742
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 6
|
||||
x 0.048709182664292916
|
||||
y 0.5851879044762575
|
||||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_ols_m -0.752322
|
||||
x_y_ols_b 0.750859
|
||||
x_y_ols_n 6
|
||||
x_y_pca_m -1.066519
|
||||
x_y_pca_b 0.889190
|
||||
x_y_pca_n 6
|
||||
x_y_pca_quality 0.836548
|
||||
x_y_r2 0.515958
|
||||
x_y_corr -0.718302
|
||||
x_y_cov -0.049192
|
||||
xy_y2_ols_m 0.917730
|
||||
xy_y2_ols_b 0.103935
|
||||
xy_y2_ols_n 6
|
||||
xy_y2_pca_m 2.512667
|
||||
xy_y2_pca_b -0.125351
|
||||
xy_y2_pca_n 6
|
||||
xy_y2_pca_quality 0.807995
|
||||
xy_y2_r2 0.286403
|
||||
xy_y2_corr 0.535166
|
||||
xy_y2_cov 0.011246
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 7
|
||||
x 0.8500003149528544
|
||||
y 0.2984098741712895
|
||||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_ols_m -0.612818
|
||||
x_y_ols_b 0.707992
|
||||
x_y_ols_n 7
|
||||
x_y_pca_m -0.842190
|
||||
x_y_pca_b 0.822403
|
||||
x_y_pca_n 7
|
||||
x_y_pca_quality 0.820362
|
||||
x_y_r2 0.476303
|
||||
x_y_corr -0.690147
|
||||
x_y_cov -0.048089
|
||||
xy_y2_ols_m 0.592008
|
||||
xy_y2_ols_b 0.120493
|
||||
xy_y2_ols_n 7
|
||||
xy_y2_pca_m 3.299160
|
||||
xy_y2_pca_b -0.311183
|
||||
xy_y2_pca_n 7
|
||||
xy_y2_pca_quality 0.722139
|
||||
xy_y2_r2 0.126356
|
||||
xy_y2_corr 0.355466
|
||||
xy_y2_cov 0.007067
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 8
|
||||
x 0.616507208914765
|
||||
y 0.25924335982487057
|
||||
x2 0.38008113864387366
|
||||
xy 0.15982540019531707
|
||||
y2 0.06720711961328732
|
||||
x_y_ols_m -0.627947
|
||||
x_y_ols_b 0.706893
|
||||
x_y_ols_n 8
|
||||
x_y_pca_m -0.857225
|
||||
x_y_pca_b 0.824631
|
||||
x_y_pca_n 8
|
||||
x_y_pca_quality 0.826128
|
||||
x_y_r2 0.489375
|
||||
x_y_corr -0.699554
|
||||
x_y_cov -0.043324
|
||||
xy_y2_ols_m 0.591342
|
||||
xy_y2_ols_b 0.102111
|
||||
xy_y2_ols_n 8
|
||||
xy_y2_pca_m 3.718931
|
||||
xy_y2_pca_b -0.396750
|
||||
xy_y2_pca_n 8
|
||||
xy_y2_pca_quality 0.737121
|
||||
xy_y2_r2 0.115022
|
||||
xy_y2_corr 0.339150
|
||||
xy_y2_cov 0.006050
|
||||
|
||||
a hat
|
||||
b hat
|
||||
i 9
|
||||
x 0.33786884067769307
|
||||
y 0.6036735617015514
|
||||
x2 0.11415535350088835
|
||||
xy 0.203962486439877
|
||||
y2 0.3644217690974368
|
||||
x_y_ols_m -0.661274
|
||||
x_y_ols_b 0.735462
|
||||
x_y_ols_n 9
|
||||
x_y_pca_m -0.890432
|
||||
x_y_pca_b 0.848665
|
||||
x_y_pca_n 9
|
||||
x_y_pca_quality 0.838021
|
||||
x_y_r2 0.516776
|
||||
x_y_corr -0.718871
|
||||
x_y_cov -0.042188
|
||||
xy_y2_ols_m 0.667659
|
||||
xy_y2_ols_b 0.105305
|
||||
xy_y2_ols_n 9
|
||||
xy_y2_pca_m 3.726998
|
||||
xy_y2_pca_b -0.397782
|
||||
xy_y2_pca_n 9
|
||||
xy_y2_pca_quality 0.764687
|
||||
xy_y2_r2 0.134705
|
||||
xy_y2_corr 0.367022
|
||||
xy_y2_cov 0.006124
|
||||
|
||||
a wye
|
||||
b hat
|
||||
i 10
|
||||
x 0.3834648944206174
|
||||
y 0.4999709279216641
|
||||
x2 0.14704532525301522
|
||||
xy 0.19172129908885902
|
||||
y2 0.24997092876684981
|
||||
x_y_ols_m -0.664725
|
||||
x_y_ols_b 0.738937
|
||||
x_y_ols_n 10
|
||||
x_y_pca_m -0.888643
|
||||
x_y_pca_b 0.847077
|
||||
x_y_pca_n 10
|
||||
x_y_pca_quality 0.841594
|
||||
x_y_r2 0.524349
|
||||
x_y_corr -0.724120
|
||||
x_y_cov -0.038508
|
||||
xy_y2_ols_m 0.673183
|
||||
xy_y2_ols_b 0.106048
|
||||
xy_y2_ols_n 10
|
||||
xy_y2_pca_m 3.679574
|
||||
xy_y2_pca_b -0.396534
|
||||
xy_y2_pca_n 10
|
||||
xy_y2_pca_quality 0.765015
|
||||
xy_y2_r2 0.137573
|
||||
xy_y2_corr 0.370908
|
||||
xy_y2_cov 0.005539
|
||||
|
||||
a pan
|
||||
b hat
|
||||
i 11
|
||||
x 0.025474999754416028
|
||||
y 0.7861954915044592
|
||||
x2 0.0006489756124874967
|
||||
xy 0.020028329952999087
|
||||
y2 0.6181033508619382
|
||||
x_y_ols_m -0.702240
|
||||
x_y_ols_b 0.761330
|
||||
x_y_ols_n 11
|
||||
x_y_pca_m -0.861995
|
||||
x_y_pca_b 0.831839
|
||||
x_y_pca_n 11
|
||||
x_y_pca_quality 0.884552
|
||||
x_y_r2 0.623708
|
||||
x_y_corr -0.789752
|
||||
x_y_cov -0.049973
|
||||
xy_y2_ols_m -0.038359
|
||||
xy_y2_ols_b 0.260804
|
||||
xy_y2_ols_n 11
|
||||
xy_y2_pca_m -82.143959
|
||||
xy_y2_pca_b 12.888187
|
||||
xy_y2_pca_n 11
|
||||
xy_y2_pca_quality 0.759203
|
||||
xy_y2_r2 0.000355
|
||||
xy_y2_corr -0.018828
|
||||
xy_y2_cov -0.000360
|
||||
|
||||
a cat
|
||||
b hat
|
||||
i 12
|
||||
x 0.6335445699880142
|
||||
y 0.15467178563525052
|
||||
x2 0.4013787221612979
|
||||
xy 0.0979914699195631
|
||||
y2 0.02392336127159689
|
||||
x_y_ols_m -0.740466
|
||||
x_y_ols_b 0.765334
|
||||
x_y_ols_n 12
|
||||
x_y_pca_m -0.911766
|
||||
x_y_pca_b 0.843681
|
||||
x_y_pca_n 12
|
||||
x_y_pca_quality 0.887843
|
||||
x_y_r2 0.635324
|
||||
x_y_corr -0.797072
|
||||
x_y_cov -0.050182
|
||||
xy_y2_ols_m 0.085108
|
||||
xy_y2_ols_b 0.222963
|
||||
xy_y2_ols_n 12
|
||||
xy_y2_pca_m 41.549828
|
||||
xy_y2_pca_b -5.961259
|
||||
xy_y2_pca_n 12
|
||||
xy_y2_pca_quality 0.780003
|
||||
xy_y2_r2 0.001597
|
||||
xy_y2_corr 0.039969
|
||||
xy_y2_cov 0.000747
|
||||
|
||||
a hat
|
||||
b wye
|
||||
i 13
|
||||
x 0.35922068401384877
|
||||
y 0.8502678133887914
|
||||
x2 0.1290394998233774
|
||||
xy 0.30543378552048117
|
||||
y2 0.7229553544849566
|
||||
x_y_ols_m -0.782612
|
||||
x_y_ols_b 0.811286
|
||||
x_y_ols_n 13
|
||||
x_y_pca_m -1.047326
|
||||
x_y_pca_b 0.930360
|
||||
x_y_pca_n 13
|
||||
x_y_pca_quality 0.861099
|
||||
x_y_r2 0.571131
|
||||
x_y_corr -0.755732
|
||||
x_y_cov -0.049199
|
||||
xy_y2_ols_m 0.659118
|
||||
xy_y2_ols_b 0.166913
|
||||
xy_y2_ols_n 13
|
||||
xy_y2_pca_m 6.994463
|
||||
xy_y2_pca_b -0.854132
|
||||
xy_y2_pca_n 13
|
||||
xy_y2_pca_quality 0.838549
|
||||
xy_y2_r2 0.078760
|
||||
xy_y2_corr 0.280642
|
||||
xy_y2_cov 0.006543
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 14
|
||||
x 0.5440047442770544
|
||||
y 0.933608851612059
|
||||
x2 0.2959411617959433
|
||||
xy 0.5078876445760125
|
||||
y2 0.8716254878083876
|
||||
x_y_ols_m -0.719761
|
||||
x_y_ols_b 0.821739
|
||||
x_y_ols_n 14
|
||||
x_y_pca_m -1.259269
|
||||
x_y_pca_b 1.068052
|
||||
x_y_pca_n 14
|
||||
x_y_pca_quality 0.775265
|
||||
x_y_r2 0.388115
|
||||
x_y_corr -0.622988
|
||||
x_y_cov -0.042223
|
||||
xy_y2_ols_m 1.175278
|
||||
xy_y2_ols_b 0.097368
|
||||
xy_y2_ols_n 14
|
||||
xy_y2_pca_m 3.119062
|
||||
xy_y2_pca_b -0.264044
|
||||
xy_y2_pca_n 14
|
||||
xy_y2_pca_quality 0.866432
|
||||
xy_y2_r2 0.322054
|
||||
xy_y2_corr 0.567498
|
||||
xy_y2_cov 0.020862
|
||||
|
||||
a wye
|
||||
b dog
|
||||
i 15
|
||||
x 0.4689175303764642
|
||||
y 0.09048353045392021
|
||||
x2 0.21988365029436224
|
||||
xy 0.04242931364019586
|
||||
y2 0.008187269283405506
|
||||
x_y_ols_m -0.725720
|
||||
x_y_ols_b 0.798215
|
||||
x_y_ols_n 15
|
||||
x_y_pca_m -1.432456
|
||||
x_y_pca_b 1.121457
|
||||
x_y_pca_n 15
|
||||
x_y_pca_quality 0.758098
|
||||
x_y_r2 0.343569
|
||||
x_y_corr -0.586148
|
||||
x_y_cov -0.039539
|
||||
xy_y2_ols_m 1.249777
|
||||
xy_y2_ols_b 0.074959
|
||||
xy_y2_ols_n 15
|
||||
xy_y2_pca_m 2.985574
|
||||
xy_y2_pca_b -0.231176
|
||||
xy_y2_pca_n 15
|
||||
xy_y2_pca_quality 0.877117
|
||||
xy_y2_r2 0.362173
|
||||
xy_y2_corr 0.601808
|
||||
xy_y2_cov 0.022316
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 16
|
||||
x 0.3959177828066379
|
||||
y 0.6339858483805666
|
||||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_ols_m -0.734975
|
||||
x_y_ols_b 0.810106
|
||||
x_y_ols_n 16
|
||||
x_y_pca_m -1.451312
|
||||
x_y_pca_b 1.134988
|
||||
x_y_pca_n 16
|
||||
x_y_pca_quality 0.761173
|
||||
x_y_r2 0.346218
|
||||
x_y_corr -0.588403
|
||||
x_y_cov -0.037547
|
||||
xy_y2_ols_m 1.253418
|
||||
xy_y2_ols_b 0.075130
|
||||
xy_y2_ols_n 16
|
||||
xy_y2_pca_m 2.945315
|
||||
xy_y2_pca_b -0.231155
|
||||
xy_y2_pca_n 16
|
||||
xy_y2_pca_quality 0.877563
|
||||
xy_y2_r2 0.368261
|
||||
xy_y2_corr 0.606845
|
||||
xy_y2_cov 0.021325
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 17
|
||||
x 0.34033844788864975
|
||||
y 0.8845934733681523
|
||||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_ols_m -0.779408
|
||||
x_y_ols_b 0.849058
|
||||
x_y_ols_n 17
|
||||
x_y_pca_m -1.579107
|
||||
x_y_pca_b 1.206423
|
||||
x_y_pca_n 17
|
||||
x_y_pca_quality 0.772982
|
||||
x_y_r2 0.349689
|
||||
x_y_corr -0.591345
|
||||
x_y_cov -0.037916
|
||||
xy_y2_ols_m 1.392136
|
||||
xy_y2_ols_b 0.068451
|
||||
xy_y2_ols_n 17
|
||||
xy_y2_pca_m 3.095298
|
||||
xy_y2_pca_b -0.251899
|
||||
xy_y2_pca_n 17
|
||||
xy_y2_pca_quality 0.896358
|
||||
xy_y2_r2 0.398830
|
||||
xy_y2_corr 0.631530
|
||||
xy_y2_cov 0.023385
|
||||
|
||||
a wye
|
||||
b wye
|
||||
i 18
|
||||
x 0.6770613653962891
|
||||
y 0.896307226056897
|
||||
x2 0.4584120925122874
|
||||
xy 0.6068549942886431
|
||||
y2 0.8033666434818095
|
||||
x_y_ols_m -0.628521
|
||||
x_y_ols_b 0.811643
|
||||
x_y_ols_n 18
|
||||
x_y_pca_m -1.834767
|
||||
x_y_pca_b 1.366109
|
||||
x_y_pca_n 18
|
||||
x_y_pca_quality 0.694668
|
||||
x_y_r2 0.218178
|
||||
x_y_corr -0.467095
|
||||
x_y_cov -0.030628
|
||||
xy_y2_ols_m 1.292066
|
||||
xy_y2_ols_b 0.083495
|
||||
xy_y2_ols_n 18
|
||||
xy_y2_pca_m 2.354328
|
||||
xy_y2_pca_b -0.141020
|
||||
xy_y2_pca_n 18
|
||||
xy_y2_pca_quality 0.888372
|
||||
xy_y2_r2 0.477918
|
||||
xy_y2_corr 0.691316
|
||||
xy_y2_cov 0.033015
|
||||
|
||||
a dog
|
||||
b wye
|
||||
i 19
|
||||
x 0.4865373244199632
|
||||
y 0.44117766146315884
|
||||
x2 0.23671856805373653
|
||||
xy 0.2146493990021416
|
||||
y2 0.1946377289741016
|
||||
x_y_ols_m -0.630507
|
||||
x_y_ols_b 0.809155
|
||||
x_y_ols_n 19
|
||||
x_y_pca_m -1.839104
|
||||
x_y_pca_b 1.366411
|
||||
x_y_pca_n 19
|
||||
x_y_pca_quality 0.695696
|
||||
x_y_r2 0.218821
|
||||
x_y_corr -0.467783
|
||||
x_y_cov -0.029041
|
||||
xy_y2_ols_m 1.290872
|
||||
xy_y2_ols_b 0.075000
|
||||
xy_y2_ols_n 19
|
||||
xy_y2_pca_m 2.393419
|
||||
xy_y2_pca_b -0.158220
|
||||
xy_y2_pca_n 19
|
||||
xy_y2_pca_quality 0.887359
|
||||
xy_y2_r2 0.469361
|
||||
xy_y2_corr 0.685099
|
||||
xy_y2_cov 0.031153
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 20
|
||||
x 0.3223311725542929
|
||||
y 0.08115611029827985
|
||||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_ols_m -0.548880
|
||||
x_y_ols_b 0.745846
|
||||
x_y_ols_n 20
|
||||
x_y_pca_m -2.251324
|
||||
x_y_pca_b 1.518994
|
||||
x_y_pca_n 20
|
||||
x_y_pca_quality 0.661764
|
||||
x_y_r2 0.151247
|
||||
x_y_corr -0.388905
|
||||
x_y_cov -0.024479
|
||||
xy_y2_ols_m 1.329401
|
||||
xy_y2_ols_b 0.062098
|
||||
xy_y2_ols_n 20
|
||||
xy_y2_pca_m 2.337844
|
||||
xy_y2_pca_b -0.141870
|
||||
xy_y2_pca_n 20
|
||||
xy_y2_pca_quality 0.894994
|
||||
xy_y2_r2 0.499340
|
||||
xy_y2_corr 0.706640
|
||||
xy_y2_cov 0.032678
|
||||
|
|
@ -1 +0,0 @@
|
|||
mlr --oxtab stats2 -s -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 -g a,b regtest/input/abixy-wide-short
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
a b i x y x2 xy y2 x_y_ols_fit x_y_pca_fit xy_y2_ols_fit xy_y2_pca_fit
|
||||
cat pan 1 0.5117389009583777 0.08295224980036853 0.2618767027540883 0.0424498931448654 0.006881075746942741 0.464963 0.366904 0.118531 -0.042629
|
||||
pan wye 2 0.5225940442098578 0.511678736087022 0.27310453504361476 0.2674002600279053 0.26181512896361225 0.459005 0.342466 0.417581 0.483270
|
||||
wye cat 3 0.8150401717873625 0.07989551500795256 0.6642904816271734 0.06511805427712146 0.006383293318385972 0.298487 -0.315925 0.148666 0.010366
|
||||
dog hat 4 0.4488733555675044 0.5730530513123552 0.20148728933843124 0.25722824606077416 0.32838979961840076 0.499469 0.508435 0.404058 0.459490
|
||||
dog pan 5 0.2946557960430134 0.6850437256584863 0.08682203814174191 0.20185210430817294 0.46928490606405937 0.584116 0.855629 0.330441 0.330029
|
||||
wye cat 6 0.048709182664292916 0.5851879044762575 0.0023725844758234536 0.02850402453206882 0.34244488354531344 0.719111 1.409334 0.099992 -0.075232
|
||||
dog hat 7 0.8500003149528544 0.2984098741712895 0.7225005354199517 0.25364848703063775 0.08904845300292483 0.279298 -0.394632 0.399299 0.451121
|
||||
pan pan 8 0.616507208914765 0.25924335982487057 0.38008113864387366 0.15982540019531707 0.06720711961328732 0.407458 0.131037 0.274570 0.231777
|
||||
hat hat 9 0.33786884067769307 0.6036735617015514 0.11415535350088835 0.203962486439877 0.3644217690974368 0.560397 0.758342 0.333246 0.334963
|
||||
wye hat 10 0.3834648944206174 0.4999709279216641 0.14704532525301522 0.19172129908885902 0.24997092876684981 0.535370 0.655691 0.316973 0.306345
|
||||
pan hat 11 0.025474999754416028 0.7861954915044592 0.0006489756124874967 0.020028329952999087 0.6181033508619382 0.731863 1.461642 0.088724 -0.095047
|
||||
cat hat 12 0.6335445699880142 0.15467178563525052 0.4013787221612979 0.0979914699195631 0.02392336127159689 0.398106 0.092680 0.192368 0.087219
|
||||
hat wye 13 0.35922068401384877 0.8502678133887914 0.1290394998233774 0.30543378552048117 0.7229553544849566 0.548677 0.710272 0.468142 0.572187
|
||||
dog dog 14 0.5440047442770544 0.933608851612059 0.2959411617959433 0.5078876445760125 0.8716254878083876 0.447253 0.294263 0.737285 1.045492
|
||||
wye dog 15 0.4689175303764642 0.09048353045392021 0.21988365029436224 0.04242931364019586 0.008187269283405506 0.488467 0.463309 0.118504 -0.042677
|
||||
pan pan 16 0.3959177828066379 0.6339858483805666 0.15675089074252413 0.25100627142161924 0.4019380559468268 0.528535 0.627655 0.395786 0.444944
|
||||
dog hat 17 0.34033844788864975 0.8845934733681523 0.11583025911125516 0.3010611697385466 0.782505613125532 0.559041 0.752782 0.462329 0.561964
|
||||
wye wye 18 0.6770613653962891 0.896307226056897 0.4584120925122874 0.6068549942886431 0.8033666434818095 0.374221 -0.005290 0.868852 1.276863
|
||||
dog wye 19 0.4865373244199632 0.44117766146315884 0.23671856805373653 0.2146493990021416 0.1946377289741016 0.478796 0.423641 0.347454 0.359947
|
||||
dog dog 20 0.3223311725542929 0.08115611029827985 0.10389738480022534 0.026159144192390068 0.006586314238746564 0.568925 0.793322 0.096874 -0.080714
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
a b i x y x2 xy y2 x_y_ols_fit x_y_pca_fit xy_y2_ols_fit xy_y2_pca_fit
|
||||
cat pan 1 0.5117389009583777 0.08295224980036853 0.2618767027540883 0.0424498931448654 0.006881075746942741 0.082952 0.082952 0.006881 0.006881
|
||||
cat hat 12 0.6335445699880142 0.15467178563525052 0.4013787221612979 0.0979914699195631 0.02392336127159689 0.154672 0.154672 0.023923 0.023923
|
||||
pan wye 2 0.5225940442098578 0.511678736087022 0.27310453504361476 0.2674002600279053 0.26181512896361225 0.445016 0.435835 0.237402 0.033364
|
||||
pan pan 8 0.616507208914765 0.25924335982487057 0.38008113864387366 0.15982540019531707 0.06720711961328732 0.372165 0.356477 0.353122 0.385517
|
||||
pan hat 11 0.025474999754416028 0.7861954915044592 0.0006489756124874967 0.020028329952999087 0.6181033508619382 0.830642 0.855912 0.503503 0.843152
|
||||
pan pan 16 0.3959177828066379 0.6339858483805666 0.15675089074252413 0.25100627142161924 0.4019380559468268 0.543281 0.542880 0.255037 0.087031
|
||||
wye cat 3 0.8150401717873625 0.07989551500795256 0.6642904816271734 0.06511805427712146 0.006383293318385972 0.337827 -0.227660 0.137066 0.112494
|
||||
wye cat 6 0.048709182664292916 0.5851879044762575 0.0023725844758234536 0.02850402453206882 0.34244488354531344 0.548640 1.271347 0.093480 0.061521
|
||||
wye hat 10 0.3834648944206174 0.4999709279216641 0.14704532525301522 0.19172129908885902 0.24997092876684981 0.456551 0.616537 0.287780 0.288747
|
||||
wye dog 15 0.4689175303764642 0.09048353045392021 0.21988365029436224 0.04242931364019586 0.008187269283405506 0.433043 0.449384 0.110057 0.080907
|
||||
wye wye 18 0.6770613653962891 0.896307226056897 0.4584120925122874 0.6068549942886431 0.8033666434818095 0.375784 0.042238 0.781971 0.866684
|
||||
dog hat 4 0.4488733555675044 0.5730530513123552 0.20148728933843124 0.25722824606077416 0.32838979961840076 0.563043 0.668750 0.401991 0.406622
|
||||
dog pan 5 0.2946557960430134 0.6850437256584863 0.08682203814174191 0.20185210430817294 0.46928490606405937 0.610235 1.504956 0.297579 0.255112
|
||||
dog hat 7 0.8500003149528544 0.2984098741712895 0.7225005354199517 0.25364848703063775 0.08904845300292483 0.440294 -1.506261 0.395241 0.396827
|
||||
dog dog 14 0.5440047442770544 0.933608851612059 0.2959411617959433 0.5078876445760125 0.8716254878083876 0.533932 0.152924 0.874610 1.092430
|
||||
dog hat 17 0.34033844788864975 0.8845934733681523 0.11583025911125516 0.3010611697385466 0.782505613125532 0.596256 1.257254 0.484638 0.526549
|
||||
dog wye 19 0.4865373244199632 0.44117766146315884 0.23671856805373653 0.2146493990021416 0.1946377289741016 0.551517 0.464527 0.321709 0.290125
|
||||
dog dog 20 0.3223311725542929 0.08115611029827985 0.10389738480022534 0.026159144192390068 0.006586314238746564 0.601766 1.354893 -0.033690 -0.225587
|
||||
hat hat 9 0.33786884067769307 0.6036735617015514 0.11415535350088835 0.203962486439877 0.3644217690974368 0.603674 0.603674 0.364422 0.364422
|
||||
hat wye 13 0.35922068401384877 0.8502678133887914 0.1290394998233774 0.30543378552048117 0.7229553544849566 0.850268 0.850268 0.722955 0.722955
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
x_y_logistic_m x_y_logistic_b x_y_logistic_n
|
||||
0.145457 0.145449 22
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
g x_y_logistic_m x_y_logistic_b x_y_logistic_n
|
||||
red 0.145458 -0.036371 11
|
||||
blue 0.145457 0.327269 11
|
||||
|
|
@ -1 +0,0 @@
|
|||
x_y_cov -0.011481
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
a pan
|
||||
x_y_cov 0.017595
|
||||
|
||||
a eks
|
||||
x_y_cov 0.034641
|
||||
|
||||
a wye
|
||||
|
||||
a zee
|
||||
x_y_cov
|
||||
|
|
@ -740,6 +740,33 @@ Notes:
|
|||
In particular, 1 and 1.0 are distinct text for count and mode.
|
||||
* When there are mode ties, the first-encountered datum wins.
|
||||
================================================================
|
||||
Usage: mlr stats2 [options]
|
||||
Computes bivariate statistics for one or more given field-name pairs,
|
||||
accumulated across the input record stream.
|
||||
-a {linreg-ols,corr,...} Names of accumulators: one or more of:
|
||||
linreg-ols Linear regression using ordinary least squares
|
||||
linreg-pca Linear regression using principal component analysis
|
||||
r2 Quality metric for linreg-ols (linreg-pca emits its own)
|
||||
logireg Logistic regression
|
||||
corr Sample correlation
|
||||
cov Sample covariance
|
||||
covx Sample-covariance matrix
|
||||
-f {a,b,c,d} Value-field name-pairs on which to compute statistics.
|
||||
There must be an even number of names.
|
||||
-g {e,f,g} Optional group-by-field names.
|
||||
-v Print additional output for linreg-pca.
|
||||
-s Print iterative stats. Useful in tail -f contexts (in which
|
||||
case please avoid pprint-format output since end of input
|
||||
stream will never be seen).
|
||||
--fit Rather than printing regression parameters, applies them to
|
||||
the input data to compute new fit fields. All input records are
|
||||
held in memory until end of input stream. Has effect only for
|
||||
linreg-ols, linreg-pca, and logireg.
|
||||
Only one of -s or --fit may be used.
|
||||
Example: mlr stats2 -a linreg-pca -f x,y
|
||||
Example: mlr stats2 -a linreg-ols,r2 -f x,y -g size,shape
|
||||
Example: mlr stats2 -a corr -f x,y
|
||||
================================================================
|
||||
Usage: mlr step [options]
|
||||
Computes values dependent on the previous record, optionally grouped by category.
|
||||
Options:
|
||||
|
|
|
|||
1
go/regtest/cases/verb-stats2/0001/cmd
Normal file
1
go/regtest/cases/verb-stats2/0001/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -a linreg-ols,linreg-pca,r2,corr,cov -f x,y,xy,y2 regtest/input/abixy-wide
|
||||
20
go/regtest/cases/verb-stats2/0001/expout
Normal file
20
go/regtest/cases/verb-stats2/0001/expout
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
x_y_ols_m 0.02835121661906505
|
||||
x_y_ols_b 0.48764386016506256
|
||||
x_y_ols_n 2000
|
||||
x_y_pca_m 1.332923806361005
|
||||
x_y_pca_b -0.17059009916254853
|
||||
x_y_pca_n 2000
|
||||
x_y_pca_quality 0.05690935192950697
|
||||
x_y_r2 0.0007907285950881155
|
||||
x_y_corr 0.028119896783027547
|
||||
x_y_cov 0.002330213751075865
|
||||
xy_y2_ols_m 0.8936101982287672
|
||||
xy_y2_ols_b 0.10706003374034685
|
||||
xy_y2_ols_n 2000
|
||||
xy_y2_pca_m 1.5295336661747538
|
||||
xy_y2_pca_b -0.055476815789010814
|
||||
xy_y2_pca_n 2000
|
||||
xy_y2_pca_quality 0.8243359107998629
|
||||
xy_y2_r2 0.44797076692663085
|
||||
xy_y2_corr 0.6693061832424908
|
||||
xy_y2_cov 0.04503611750776758
|
||||
1
go/regtest/cases/verb-stats2/0002a/cmd
Normal file
1
go/regtest/cases/verb-stats2/0002a/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --opprint stats2 -a linreg-ols,linreg-pca -f x,y,xy,y2 -g a,b regtest/input/abixy-wide
|
||||
26
go/regtest/cases/verb-stats2/0002a/expout
Normal file
26
go/regtest/cases/verb-stats2/0002a/expout
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
a b x_y_ols_m x_y_ols_b x_y_ols_n x_y_pca_m x_y_pca_b x_y_pca_n x_y_pca_quality xy_y2_ols_m xy_y2_ols_b xy_y2_ols_n xy_y2_pca_m xy_y2_pca_b xy_y2_pca_n xy_y2_pca_quality
|
||||
cat pan 0.054420331072315566 0.48177726045953384 89 3.6360622649153744 -1.2216016718286058 89 0.17768332852207824 0.9509076277310625 0.10575421166162527 89 1.7155741289015414 -0.08171887473852824 89 0.8306116484518189
|
||||
pan wye -0.145485601632497 0.58479925355158 78 -1.3409269688160623 1.1999197350989401 78 0.25402520581032506 0.9081508099888997 0.1266278500839982 78 1.5951504732062534 -0.045033646350741585 78 0.8241143431220244
|
||||
wye cat 0.18591317368264998 0.3776388056437748 74 1.1353251244073408 -0.14589418164098722 74 0.3094986708351688 0.9692656916661936 0.04060201647770494 74 1.4063652501785224 -0.08137930778911018 74 0.8684797887109428
|
||||
dog hat 0.10009629366113551 0.4487570574039835 88 0.810749113172465 0.09734600380923053 88 0.18925579914870694 0.9191492617650554 0.09050352496253057 88 1.425774239771295 -0.03834448605203988 88 0.846209331302479
|
||||
dog pan -0.06683364788050715 0.5906471638899853 87 -0.25411243437740655 0.6888372402454463 87 0.27531567900095133 0.7261179615725861 0.1649366501202624 87 1.5663091673224863 -0.07507301171050912 87 0.7490252493797314
|
||||
pan pan 0.09493201929368525 0.46156550484329756 77 0.6723687119545303 0.1898977821683261 77 0.19271863773389442 0.8222608507297753 0.123440869361421 77 1.3125431665778664 0.0031998944803106055 77 0.820350784165558
|
||||
hat hat 0.043668138280056557 0.4052194008419142 88 10.170494157514183 -5.125281654155784 88 0.310513062033242 1.1288963329304458 0.015188485398760106 88 1.4141658093813985 -0.052513603405017095 88 0.9223081294982979
|
||||
wye hat 0.04301822051501528 0.49602850983141966 87 0.2548788534378015 0.3957802061740632 87 0.17779411111552057 0.7204023437597552 0.16562337121409948 87 1.3761363275226155 0.002791611275949124 87 0.7607158833703676
|
||||
pan hat 0.12079685289316795 0.4481972614219893 67 1.597359121580881 -0.3256947919863026 67 0.22513723664544083 0.9626784393603379 0.07691964115366853 67 1.2857964222817417 -0.012565821206783734 67 0.8877035574889709
|
||||
cat hat 0.17239115140356195 0.464384061249257 90 0.9593291515129574 0.08679014008638403 90 0.29610907139881526 0.9042569821868581 0.13348247368673333 90 1.4156584666483505 -0.008368625196926882 90 0.8415668817297943
|
||||
hat wye -0.022975185871119196 0.4963605898461038 70 -1.7658841771766842 1.3442676285684412 70 0.05149295472045501 0.9719290909997436 0.09608767485929498 70 1.9894215654621845 -0.1420717660852956 70 0.8256560273405081
|
||||
dog dog 0.07839654110401448 0.48923550753966766 87 0.3544939585482934 0.3510409568753121 87 0.24221037762901476 0.7769666147811132 0.15099885883063824 87 1.3544045965803233 -0.006431584287105274 87 0.7922646832859046
|
||||
wye dog 0.1164033041847666 0.4255764393282477 76 2.367820908790329 -0.7777335974173947 76 0.25460728261082 0.9257814718275479 0.07119237315601917 76 1.4535901339159096 -0.07050931004494287 76 0.8452036138880259
|
||||
wye wye -0.18835420176241532 0.6139344128398031 67 -1.433771562778627 1.217886972908627 67 0.3160698090045916 0.8767168554487523 0.15917898616435514 67 2.0444927228375374 -0.11850334104657884 67 0.7954545208353714
|
||||
dog wye 0.029526530188510737 0.5026433511471881 79 0.49671335977671177 0.2825108623122976 79 0.07303881962119818 0.9049249754278408 0.12081572949798884 79 1.6091234435062964 -0.05224483555930731 79 0.821822318108098
|
||||
cat dog 0.057572942807262946 0.40864390334149303 78 0.7284785925720797 0.07111408484797838 78 0.11610297783946633 0.884325159054163 0.07999850046677684 78 1.4182070985231408 -0.04034419049874843 78 0.8329980414844023
|
||||
hat pan -0.15439307771396008 0.5649808693260537 85 -0.845852219735721 0.9110257253709291 85 0.2769547823946832 0.9111652684145581 0.1043616629348576 85 1.7637395632402355 -0.09298668279005579 85 0.8145840691753903
|
||||
cat wye -0.014850955996988408 0.564875404625477 77 -0.5727082493682792 0.8923215124847966 77 0.03414594662667991 0.8788196250355562 0.08636189010681794 77 1.4472439001911792 -0.09865744134791715 77 0.8271186634379826
|
||||
hat cat -0.022859047329439403 0.498538563067491 88 -0.1562416879196635 0.5657234820446683 88 0.14934383521234407 0.8409651161475857 0.11112113794588172 88 1.6635178923692764 -0.08894189096058164 88 0.7938832928207415
|
||||
dog cat 0.10405714462328876 0.42855891492111386 83 2.712382336210859 -1.0057872369256355 83 0.25003573106375554 1.0804425307042143 0.023865527372970832 83 1.65392159213901 -0.1333666540015272 83 0.8755855607391829
|
||||
hat dog 0.04184851765028569 0.4272278900137946 78 0.40397678519397046 0.254918697221277 78 0.11849398287687107 0.7761352110156776 0.11493014978514286 78 1.4754027187006336 -0.036507960449088495 78 0.7790557026056879
|
||||
pan dog 0.11950976972069548 0.4678325963579821 73 2.4924955152312864 -0.7614897496264781 73 0.2664545653464373 0.9485923358165189 0.10755567780994264 73 1.4083894540121666 -0.02284647681822416 73 0.8602431907465251
|
||||
cat cat 0.016256623666497577 0.42540980787799243 79 0.43294642731785266 0.22553505739756807 79 0.044275436886916375 0.9309543521264523 0.07247577667743145 79 1.6249929057631094 -0.07266917609915541 79 0.8300288539499573
|
||||
pan cat -0.1885230952275368 0.6169191441347157 89 -0.898664748101113 0.9539226634016469 89 0.32426391280104794 0.7817700559806444 0.17661687979553184 89 2.02045351816123 -0.1135866985084365 89 0.7623319064814484
|
||||
wye pan 0.229442895943327 0.44444638507611284 66 1.3136892717403466 -0.09812357007892536 66 0.3658106271699061 0.8876586720872541 0.14505160298768832 66 1.471905869648956 -0.030175818697552925 66 0.8279107564841395
|
||||
1
go/regtest/cases/verb-stats2/0002b/cmd
Normal file
1
go/regtest/cases/verb-stats2/0002b/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --opprint stats2 -a r2 -f x,y,xy,y2 -g a,b regtest/input/abixy-wide
|
||||
26
go/regtest/cases/verb-stats2/0002b/expout
Normal file
26
go/regtest/cases/verb-stats2/0002b/expout
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
a b x_y_r2 xy_y2_r2
|
||||
cat pan 0.0025036353028391202 0.43533632188113097
|
||||
pan wye 0.019479356864070648 0.43884971912526155
|
||||
wye cat 0.03300209357863516 0.5612355257896368
|
||||
dog hat 0.010461895189883821 0.5071550264632348
|
||||
dog pan 0.005924249459274809 0.3150114896724656
|
||||
pan pan 0.009767730657475817 0.4653896082028348
|
||||
hat hat 0.0013243941394434865 0.7087246275194752
|
||||
wye hat 0.0021973396998447837 0.35355828321386507
|
||||
pan hat 0.0130595608737085 0.6223529503781107
|
||||
cat hat 0.030150458622914638 0.4981710213113751
|
||||
hat wye 0.0005137010849484458 0.3863539051352603
|
||||
dog dog 0.00761927702536921 0.4082573818120529
|
||||
wye dog 0.01104782129018003 0.5015592442180044
|
||||
wye wye 0.0311563234553737 0.3251925624534024
|
||||
dog wye 0.0009126837539977595 0.43241340912531395
|
||||
cat dog 0.003442323302149995 0.47959568421506393
|
||||
hat pan 0.02514304893876954 0.39715027726718594
|
||||
cat wye 0.00022446233812577454 0.4639614234932371
|
||||
hat cat 0.0006095379218843187 0.37351456675510697
|
||||
dog cat 0.0087049377058871 0.5471025319399344
|
||||
hat dog 0.0019175197040864338 0.37205849281239917
|
||||
pan dog 0.011426710432476266 0.5412633821266005
|
||||
cat cat 0.0002725948352221859 0.44676399108840026
|
||||
pan cat 0.03703580705963691 0.2787393270934291
|
||||
wye pan 0.04672155745662957 0.46254536720439005
|
||||
1
go/regtest/cases/verb-stats2/0002c/cmd
Normal file
1
go/regtest/cases/verb-stats2/0002c/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --opprint stats2 -a corr,cov -f x,y,xy,y2 -g a,b regtest/input/abixy-wide
|
||||
26
go/regtest/cases/verb-stats2/0002c/expout
Normal file
26
go/regtest/cases/verb-stats2/0002c/expout
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
a b x_y_corr x_y_cov xy_y2_corr xy_y2_cov
|
||||
cat pan 0.05003633982256386 0.003776726550325581 0.6598002136110073 0.04161601570081189
|
||||
pan wye -0.139568466582071 -0.012682975821884686 0.6624573338149874 0.04620305695467854
|
||||
wye cat 0.18166478353999968 0.014493586640460984 0.7491565429131861 0.05208959439852251
|
||||
dog hat 0.10228340622937716 0.008035805065738578 0.7121481773221319 0.04503356932816789
|
||||
dog pan -0.07696914615139468 -0.005708694853629292 0.5612588437365293 0.03410721121170538
|
||||
pan pan 0.09883183018378165 0.007175430796907774 0.6821946996296839 0.03978366739154019
|
||||
hat hat 0.03639222636008246 0.003037253679771858 0.841857842821147 0.06097488433368237
|
||||
wye hat 0.04687579012501871 0.004023341674960352 0.5946076716742436 0.03876271503751961
|
||||
pan hat 0.11427843573355559 0.008986955347275335 0.7888934974875322 0.0549651361597052
|
||||
cat hat 0.17363887416968166 0.015030011072489035 0.7058123130913593 0.05562626424184947
|
||||
hat wye -0.02266497484994137 -0.00200017258648794 0.6215737326619105 0.04012644173569002
|
||||
dog dog 0.08728847017429733 0.008213768509488021 0.6389502185710974 0.04964809679992539
|
||||
wye dog 0.10510861663146331 0.007867061043408133 0.7082084751102634 0.046439738967672506
|
||||
wye wye -0.17651153915643497 -0.015876262436306236 0.5702565759843568 0.04202643083075441
|
||||
dog wye 0.03021065629869283 0.002391095757839724 0.6575814847798818 0.04285746060130249
|
||||
cat dog 0.058671315837894714 0.005319826839472508 0.6925284717721462 0.044761718519245
|
||||
hat pan -0.15856559821969426 -0.012755930959939613 0.6301986014481358 0.03562169671732862
|
||||
cat wye -0.01498206721803674 -0.0009662836178657299 0.6811471379175265 0.041096380677337487
|
||||
hat cat -0.024688821800246515 -0.0021164484854253865 0.611158381072457 0.03657526735671884
|
||||
dog cat 0.093300255658209 0.007122052131758338 0.7396637965589057 0.050357273273392594
|
||||
hat dog 0.04378949307866459 0.003856238010660333 0.6099659767662451 0.04058253044685895
|
||||
pan dog 0.10689579239837425 0.009301974754383657 0.7357060432853604 0.05660935218447732
|
||||
cat cat 0.016510446245398026 0.0013501358521011662 0.6684040627407949 0.03626747560069507
|
||||
pan cat -0.1924468941283205 -0.016205786732663062 0.5279576944163511 0.03298391263240368
|
||||
wye pan 0.216151700101178 0.020366960303036363 0.680106879250894 0.06449614008868144
|
||||
1
go/regtest/cases/verb-stats2/0003a/cmd
Normal file
1
go/regtest/cases/verb-stats2/0003a/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -s -a linreg-ols,linreg-pca -f x,y,xy,y2 regtest/input/abixy-wide-short
|
||||
459
go/regtest/cases/verb-stats2/0003a/expout
Normal file
459
go/regtest/cases/verb-stats2/0003a/expout
Normal file
|
|
@ -0,0 +1,459 @@
|
|||
a cat
|
||||
b pan
|
||||
i 1
|
||||
x 0.5117389009583777
|
||||
y 0.08295224980036853
|
||||
x2 0.2618767027540883
|
||||
xy 0.0424498931448654
|
||||
y2 0.006881075746942741
|
||||
x_y_ols_m
|
||||
x_y_ols_b
|
||||
x_y_ols_n 1
|
||||
x_y_pca_m
|
||||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
xy_y2_pca_m
|
||||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
|
||||
a pan
|
||||
b wye
|
||||
i 2
|
||||
x 0.5225940442098578
|
||||
y 0.511678736087022
|
||||
x2 0.27310453504361476
|
||||
xy 0.2674002600279053
|
||||
y2 0.26181512896361225
|
||||
x_y_ols_m 39.49523984664168
|
||||
x_y_ols_b -20.128298382407923
|
||||
x_y_ols_n 2
|
||||
x_y_pca_m 39.495239846622475
|
||||
x_y_pca_b -20.12829838239774
|
||||
x_y_pca_n 2
|
||||
x_y_pca_quality 0.9999999999999997
|
||||
xy_y2_ols_m 1.1332902308588784
|
||||
xy_y2_ols_b -0.04122697345513659
|
||||
xy_y2_ols_n 2
|
||||
xy_y2_pca_m 1.133290230858878
|
||||
xy_y2_pca_b -0.04122697345513651
|
||||
xy_y2_pca_n 2
|
||||
xy_y2_pca_quality 0.9999999999999998
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 3
|
||||
x 0.8150401717873625
|
||||
y 0.07989551500795256
|
||||
x2 0.6642904816271734
|
||||
xy 0.06511805427712146
|
||||
y2 0.006383293318385972
|
||||
x_y_ols_m -0.6898814114631506
|
||||
x_y_ols_b 0.6501248790475598
|
||||
x_y_ols_n 3
|
||||
x_y_pca_m -2.0577615709439026
|
||||
x_y_pca_b 1.493365143767772
|
||||
x_y_pca_n 3
|
||||
x_y_pca_quality 0.7252691906017982
|
||||
xy_y2_ols_m 1.1843973414257218
|
||||
xy_y2_ols_b -0.05634394999795996
|
||||
xy_y2_ols_n 3
|
||||
xy_y2_pca_m 1.190468615411739
|
||||
xy_y2_pca_b -0.057102794905784626
|
||||
xy_y2_pca_n 3
|
||||
xy_y2_pca_quality 0.9978838471856216
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 4
|
||||
x 0.4488733555675044
|
||||
y 0.5730530513123552
|
||||
x2 0.20148728933843124
|
||||
xy 0.25722824606077416
|
||||
y2 0.32838979961840076
|
||||
x_y_ols_m -1.054065239487934
|
||||
x_y_ols_b 0.9175203176675149
|
||||
x_y_ols_n 4
|
||||
x_y_pca_m -2.068129715127662
|
||||
x_y_pca_b 1.5001628436800138
|
||||
x_y_pca_n 4
|
||||
x_y_pca_quality 0.8458056817963171
|
||||
xy_y2_ols_m 1.3657391836082693
|
||||
xy_y2_ols_b -0.06498654266258976
|
||||
xy_y2_ols_n 4
|
||||
xy_y2_pca_m 1.4069291069366794
|
||||
xy_y2_pca_b -0.07149657352473901
|
||||
xy_y2_pca_n 4
|
||||
xy_y2_pca_quality 0.9899789519513221
|
||||
|
||||
a dog
|
||||
b pan
|
||||
i 5
|
||||
x 0.2946557960430134
|
||||
y 0.6850437256584863
|
||||
x2 0.08682203814174191
|
||||
xy 0.20185210430817294
|
||||
y2 0.46928490606405937
|
||||
x_y_ols_m -1.1764185073376179
|
||||
x_y_ols_b 0.9965922988650124
|
||||
x_y_ols_n 5
|
||||
x_y_pca_m -1.6816177412515287
|
||||
x_y_pca_b 1.2585787468036602
|
||||
x_y_pca_n 5
|
||||
x_y_pca_quality 0.899128386250364
|
||||
xy_y2_ols_m 1.5656522050547186
|
||||
xy_y2_ols_b -0.04661515199207446
|
||||
xy_y2_ols_n 5
|
||||
xy_y2_pca_m 2.16903640420161
|
||||
xy_y2_pca_b -0.14726549621390256
|
||||
xy_y2_pca_n 5
|
||||
xy_y2_pca_quality 0.9367189476137213
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 6
|
||||
x 0.048709182664292916
|
||||
y 0.5851879044762575
|
||||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_ols_m -0.7523222295732452
|
||||
x_y_ols_b 0.7508590331663864
|
||||
x_y_ols_n 6
|
||||
x_y_pca_m -1.066519249068385
|
||||
x_y_pca_b 0.8891901072731907
|
||||
x_y_pca_n 6
|
||||
x_y_pca_quality 0.8365479608708519
|
||||
xy_y2_ols_m 0.917729586268825
|
||||
xy_y2_ols_b 0.10393484378678483
|
||||
xy_y2_ols_n 6
|
||||
xy_y2_pca_m 2.512666899176228
|
||||
xy_y2_pca_b -0.12535137253589826
|
||||
xy_y2_pca_n 6
|
||||
xy_y2_pca_quality 0.8079949804514893
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 7
|
||||
x 0.8500003149528544
|
||||
y 0.2984098741712895
|
||||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_ols_m -0.6128184014107156
|
||||
x_y_ols_b 0.707992142487572
|
||||
x_y_ols_n 7
|
||||
x_y_pca_m -0.8421903017002842
|
||||
x_y_pca_b 0.8224032318994109
|
||||
x_y_pca_n 7
|
||||
x_y_pca_quality 0.8203615305314821
|
||||
xy_y2_ols_m 0.5920075177565502
|
||||
xy_y2_ols_b 0.12049258797969493
|
||||
xy_y2_ols_n 7
|
||||
xy_y2_pca_m 3.299159815322068
|
||||
xy_y2_pca_b -0.31118259623763944
|
||||
xy_y2_pca_n 7
|
||||
xy_y2_pca_quality 0.7221392843612484
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 8
|
||||
x 0.616507208914765
|
||||
y 0.25924335982487057
|
||||
x2 0.38008113864387366
|
||||
xy 0.15982540019531707
|
||||
y2 0.06720711961328732
|
||||
x_y_ols_m -0.6279470616816101
|
||||
x_y_ols_b 0.7068932069737344
|
||||
x_y_ols_n 8
|
||||
x_y_pca_m -0.8572248854425552
|
||||
x_y_pca_b 0.8246307792689245
|
||||
x_y_pca_n 8
|
||||
x_y_pca_quality 0.8261284434430209
|
||||
xy_y2_ols_m 0.5913423595084192
|
||||
xy_y2_ols_b 0.10211076956976824
|
||||
xy_y2_ols_n 8
|
||||
xy_y2_pca_m 3.7189306551818038
|
||||
xy_y2_pca_b -0.3967499118324849
|
||||
xy_y2_pca_n 8
|
||||
xy_y2_pca_quality 0.7371213165894916
|
||||
|
||||
a hat
|
||||
b hat
|
||||
i 9
|
||||
x 0.33786884067769307
|
||||
y 0.6036735617015514
|
||||
x2 0.11415535350088835
|
||||
xy 0.203962486439877
|
||||
y2 0.3644217690974368
|
||||
x_y_ols_m -0.6612741212160769
|
||||
x_y_ols_b 0.7354616293171811
|
||||
x_y_ols_n 9
|
||||
x_y_pca_m -0.8904323114806357
|
||||
x_y_pca_b 0.8486654650728969
|
||||
x_y_pca_n 9
|
||||
x_y_pca_quality 0.8380205883020141
|
||||
xy_y2_ols_m 0.6676589428453942
|
||||
xy_y2_ols_b 0.10530539635259668
|
||||
xy_y2_ols_n 9
|
||||
xy_y2_pca_m 3.726998487762793
|
||||
xy_y2_pca_b -0.39778224133440676
|
||||
xy_y2_pca_n 9
|
||||
xy_y2_pca_quality 0.7646865863526708
|
||||
|
||||
a wye
|
||||
b hat
|
||||
i 10
|
||||
x 0.3834648944206174
|
||||
y 0.4999709279216641
|
||||
x2 0.14704532525301522
|
||||
xy 0.19172129908885902
|
||||
y2 0.24997092876684981
|
||||
x_y_ols_m -0.6647247565265036
|
||||
x_y_ols_b 0.7389365682903344
|
||||
x_y_ols_n 10
|
||||
x_y_pca_m -0.8886428400960342
|
||||
x_y_pca_b 0.8470767478460183
|
||||
x_y_pca_n 10
|
||||
x_y_pca_quality 0.8415935537959993
|
||||
xy_y2_ols_m 0.6731829764451408
|
||||
xy_y2_ols_b 0.10604804724513589
|
||||
xy_y2_ols_n 10
|
||||
xy_y2_pca_m 3.6795744733068814
|
||||
xy_y2_pca_b -0.39653350237146834
|
||||
xy_y2_pca_n 10
|
||||
xy_y2_pca_quality 0.7650151371487569
|
||||
|
||||
a pan
|
||||
b hat
|
||||
i 11
|
||||
x 0.025474999754416028
|
||||
y 0.7861954915044592
|
||||
x2 0.0006489756124874967
|
||||
xy 0.020028329952999087
|
||||
y2 0.6181033508619382
|
||||
x_y_ols_m -0.70223960498624
|
||||
x_y_ols_b 0.7613297195217132
|
||||
x_y_ols_n 11
|
||||
x_y_pca_m -0.8619949917244347
|
||||
x_y_pca_b 0.8318388880570915
|
||||
x_y_pca_n 11
|
||||
x_y_pca_quality 0.8845517771159492
|
||||
xy_y2_ols_m -0.03835863158678937
|
||||
xy_y2_ols_b 0.26080395324768824
|
||||
xy_y2_ols_n 11
|
||||
xy_y2_pca_m -82.14395885058943
|
||||
xy_y2_pca_b 12.888186856873364
|
||||
xy_y2_pca_n 11
|
||||
xy_y2_pca_quality 0.7592026204676313
|
||||
|
||||
a cat
|
||||
b hat
|
||||
i 12
|
||||
x 0.6335445699880142
|
||||
y 0.15467178563525052
|
||||
x2 0.4013787221612979
|
||||
xy 0.0979914699195631
|
||||
y2 0.02392336127159689
|
||||
x_y_ols_m -0.7404659033780294
|
||||
x_y_ols_b 0.7653335640026381
|
||||
x_y_ols_n 12
|
||||
x_y_pca_m -0.9117657607302022
|
||||
x_y_pca_b 0.843681440555544
|
||||
x_y_pca_n 12
|
||||
x_y_pca_quality 0.887843401973305
|
||||
xy_y2_ols_m 0.08510768219225334
|
||||
xy_y2_ols_b 0.22296285776181185
|
||||
xy_y2_ols_n 12
|
||||
xy_y2_pca_m 41.54982838981989
|
||||
xy_y2_pca_b -5.961258548214565
|
||||
xy_y2_pca_n 12
|
||||
xy_y2_pca_quality 0.7800031828051545
|
||||
|
||||
a hat
|
||||
b wye
|
||||
i 13
|
||||
x 0.35922068401384877
|
||||
y 0.8502678133887914
|
||||
x2 0.1290394998233774
|
||||
xy 0.30543378552048117
|
||||
y2 0.7229553544849566
|
||||
x_y_ols_m -0.7826124145128263
|
||||
x_y_ols_b 0.8112862389491498
|
||||
x_y_ols_n 13
|
||||
x_y_pca_m -1.0473258482718806
|
||||
x_y_pca_b 0.9303603069535002
|
||||
x_y_pca_n 13
|
||||
x_y_pca_quality 0.8610987201824412
|
||||
xy_y2_ols_m 0.659117803529104
|
||||
xy_y2_ols_b 0.16691304890558747
|
||||
xy_y2_ols_n 13
|
||||
xy_y2_pca_m 6.994462896842802
|
||||
xy_y2_pca_b -0.8541320246783631
|
||||
xy_y2_pca_n 13
|
||||
xy_y2_pca_quality 0.8385494624215741
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 14
|
||||
x 0.5440047442770544
|
||||
y 0.933608851612059
|
||||
x2 0.2959411617959433
|
||||
xy 0.5078876445760125
|
||||
y2 0.8716254878083876
|
||||
x_y_ols_m -0.7197613814345103
|
||||
x_y_ols_b 0.8217392871635399
|
||||
x_y_ols_n 14
|
||||
x_y_pca_m -1.2592693974547642
|
||||
x_y_pca_b 1.068051583561268
|
||||
x_y_pca_n 14
|
||||
x_y_pca_quality 0.7752647153111026
|
||||
xy_y2_ols_m 1.1752775714542227
|
||||
xy_y2_ols_b 0.097367634601089
|
||||
xy_y2_ols_n 14
|
||||
xy_y2_pca_m 3.119061534206634
|
||||
xy_y2_pca_b -0.2640444890351227
|
||||
xy_y2_pca_n 14
|
||||
xy_y2_pca_quality 0.8664323235597384
|
||||
|
||||
a wye
|
||||
b dog
|
||||
i 15
|
||||
x 0.4689175303764642
|
||||
y 0.09048353045392021
|
||||
x2 0.21988365029436224
|
||||
xy 0.04242931364019586
|
||||
y2 0.008187269283405506
|
||||
x_y_ols_m -0.7257201388626029
|
||||
x_y_ols_b 0.7982148681466954
|
||||
x_y_ols_n 15
|
||||
x_y_pca_m -1.4324555126604852
|
||||
x_y_pca_b 1.1214574998226792
|
||||
x_y_pca_n 15
|
||||
x_y_pca_quality 0.7580983592284677
|
||||
xy_y2_ols_m 1.249776957077547
|
||||
xy_y2_ols_b 0.07495874925483838
|
||||
xy_y2_ols_n 15
|
||||
xy_y2_pca_m 2.985573767524858
|
||||
xy_y2_pca_b -0.23117572624575122
|
||||
xy_y2_pca_n 15
|
||||
xy_y2_pca_quality 0.8771173766793461
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 16
|
||||
x 0.3959177828066379
|
||||
y 0.6339858483805666
|
||||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_ols_m -0.7349753586940665
|
||||
x_y_ols_b 0.8101059492015656
|
||||
x_y_ols_n 16
|
||||
x_y_pca_m -1.4513117438236791
|
||||
x_y_pca_b 1.1349883637723468
|
||||
x_y_pca_n 16
|
||||
x_y_pca_quality 0.7611730087427229
|
||||
xy_y2_ols_m 1.253417590933342
|
||||
xy_y2_ols_b 0.07512952448589368
|
||||
xy_y2_ols_n 16
|
||||
xy_y2_pca_m 2.9453145837655694
|
||||
xy_y2_pca_b -0.23115533574793906
|
||||
xy_y2_pca_n 16
|
||||
xy_y2_pca_quality 0.8775634304266631
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 17
|
||||
x 0.34033844788864975
|
||||
y 0.8845934733681523
|
||||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_ols_m -0.7794081706150736
|
||||
x_y_ols_b 0.8490576114465344
|
||||
x_y_ols_n 17
|
||||
x_y_pca_m -1.5791074959231262
|
||||
x_y_pca_b 1.2064230813554957
|
||||
x_y_pca_n 17
|
||||
x_y_pca_quality 0.7729819554887778
|
||||
xy_y2_ols_m 1.392136411139648
|
||||
xy_y2_ols_b 0.06845073643613857
|
||||
xy_y2_ols_n 17
|
||||
xy_y2_pca_m 3.0952981761498575
|
||||
xy_y2_pca_b -0.2518987873244982
|
||||
xy_y2_pca_n 17
|
||||
xy_y2_pca_quality 0.8963583403613385
|
||||
|
||||
a wye
|
||||
b wye
|
||||
i 18
|
||||
x 0.6770613653962891
|
||||
y 0.896307226056897
|
||||
x2 0.4584120925122874
|
||||
xy 0.6068549942886431
|
||||
y2 0.8033666434818095
|
||||
x_y_ols_m -0.6285214590366035
|
||||
x_y_ols_b 0.8116426257553877
|
||||
x_y_ols_n 18
|
||||
x_y_pca_m -1.834766518166325
|
||||
x_y_pca_b 1.366108770279015
|
||||
x_y_pca_n 18
|
||||
x_y_pca_quality 0.6946679729176459
|
||||
xy_y2_ols_m 1.2920660443574936
|
||||
xy_y2_ols_b 0.08349512533860959
|
||||
xy_y2_ols_n 18
|
||||
xy_y2_pca_m 2.3543283527110175
|
||||
xy_y2_pca_b -0.14102010585196273
|
||||
xy_y2_pca_n 18
|
||||
xy_y2_pca_quality 0.8883717678074635
|
||||
|
||||
a dog
|
||||
b wye
|
||||
i 19
|
||||
x 0.4865373244199632
|
||||
y 0.44117766146315884
|
||||
x2 0.23671856805373653
|
||||
xy 0.2146493990021416
|
||||
y2 0.1946377289741016
|
||||
x_y_ols_m -0.6305072480270196
|
||||
x_y_ols_b 0.8091547641722724
|
||||
x_y_ols_n 19
|
||||
x_y_pca_m -1.8391037461242377
|
||||
x_y_pca_b 1.3664112671193678
|
||||
x_y_pca_n 19
|
||||
x_y_pca_quality 0.6956956431789203
|
||||
xy_y2_ols_m 1.290872216528924
|
||||
xy_y2_ols_b 0.075000360108138
|
||||
xy_y2_ols_n 19
|
||||
xy_y2_pca_m 2.3934189799754977
|
||||
xy_y2_pca_b -0.15822034898756637
|
||||
xy_y2_pca_n 19
|
||||
xy_y2_pca_quality 0.8873586980686771
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 20
|
||||
x 0.3223311725542929
|
||||
y 0.08115611029827985
|
||||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_ols_m -0.5488798201620805
|
||||
x_y_ols_b 0.7458461529809686
|
||||
x_y_ols_n 20
|
||||
x_y_pca_m -2.251324465594883
|
||||
x_y_pca_b 1.518994477612712
|
||||
x_y_pca_n 20
|
||||
x_y_pca_quality 0.6617637845836841
|
||||
xy_y2_ols_m 1.329401049119439
|
||||
xy_y2_ols_b 0.062098387824211336
|
||||
xy_y2_ols_n 20
|
||||
xy_y2_pca_m 2.3378444783356844
|
||||
xy_y2_pca_b -0.14186997400904056
|
||||
xy_y2_pca_n 20
|
||||
xy_y2_pca_quality 0.8949943706453538
|
||||
1
go/regtest/cases/verb-stats2/0003b/cmd
Normal file
1
go/regtest/cases/verb-stats2/0003b/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -s -a r2 -f x,y,xy,y2 regtest/input/abixy-wide-short
|
||||
219
go/regtest/cases/verb-stats2/0003b/expout
Normal file
219
go/regtest/cases/verb-stats2/0003b/expout
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
a cat
|
||||
b pan
|
||||
i 1
|
||||
x 0.5117389009583777
|
||||
y 0.08295224980036853
|
||||
x2 0.2618767027540883
|
||||
xy 0.0424498931448654
|
||||
y2 0.006881075746942741
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a pan
|
||||
b wye
|
||||
i 2
|
||||
x 0.5225940442098578
|
||||
y 0.511678736087022
|
||||
x2 0.27310453504361476
|
||||
xy 0.2674002600279053
|
||||
y2 0.26181512896361225
|
||||
x_y_r2 1.0000000000004867
|
||||
xy_y2_r2 1.0000000000000009
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 3
|
||||
x 0.8150401717873625
|
||||
y 0.07989551500795256
|
||||
x2 0.6642904816271734
|
||||
xy 0.06511805427712146
|
||||
y2 0.006383293318385972
|
||||
x_y_r2 0.228337666410322
|
||||
xy_y2_r2 0.9913145292717177
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 4
|
||||
x 0.4488733555675044
|
||||
y 0.5730530513123552
|
||||
x2 0.20148728933843124
|
||||
xy 0.25722824606077416
|
||||
y2 0.32838979961840076
|
||||
x_y_r2 0.41608239185231793
|
||||
xy_y2_r2 0.9561552818459209
|
||||
|
||||
a dog
|
||||
b pan
|
||||
i 5
|
||||
x 0.2946557960430134
|
||||
y 0.6850437256584863
|
||||
x2 0.08682203814174191
|
||||
xy 0.20185210430817294
|
||||
y2 0.46928490606405937
|
||||
x_y_r2 0.6073436443301099
|
||||
xy_y2_r2 0.6671679816113195
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 6
|
||||
x 0.048709182664292916
|
||||
y 0.5851879044762575
|
||||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_r2 0.5159580589872577
|
||||
xy_y2_r2 0.2864030478923904
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 7
|
||||
x 0.8500003149528544
|
||||
y 0.2984098741712895
|
||||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_r2 0.47630286054993914
|
||||
xy_y2_r2 0.12635639473434646
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 8
|
||||
x 0.616507208914765
|
||||
y 0.25924335982487057
|
||||
x2 0.38008113864387366
|
||||
xy 0.15982540019531707
|
||||
y2 0.06720711961328732
|
||||
x_y_r2 0.4893751959095168
|
||||
xy_y2_r2 0.11502244832137472
|
||||
|
||||
a hat
|
||||
b hat
|
||||
i 9
|
||||
x 0.33786884067769307
|
||||
y 0.6036735617015514
|
||||
x2 0.11415535350088835
|
||||
xy 0.203962486439877
|
||||
y2 0.3644217690974368
|
||||
x_y_r2 0.516776049297009
|
||||
xy_y2_r2 0.13470488600289762
|
||||
|
||||
a wye
|
||||
b hat
|
||||
i 10
|
||||
x 0.3834648944206174
|
||||
y 0.4999709279216641
|
||||
x2 0.14704532525301522
|
||||
xy 0.19172129908885902
|
||||
y2 0.24997092876684981
|
||||
x_y_r2 0.5243493923775143
|
||||
xy_y2_r2 0.13757283589319558
|
||||
|
||||
a pan
|
||||
b hat
|
||||
i 11
|
||||
x 0.025474999754416028
|
||||
y 0.7861954915044592
|
||||
x2 0.0006489756124874967
|
||||
xy 0.020028329952999087
|
||||
y2 0.6181033508619382
|
||||
x_y_r2 0.6237080132488391
|
||||
xy_y2_r2 0.0003545109684238707
|
||||
|
||||
a cat
|
||||
b hat
|
||||
i 12
|
||||
x 0.6335445699880142
|
||||
y 0.15467178563525052
|
||||
x2 0.4013787221612979
|
||||
xy 0.0979914699195631
|
||||
y2 0.02392336127159689
|
||||
x_y_r2 0.6353236984494468
|
||||
xy_y2_r2 0.0015974988868297253
|
||||
|
||||
a hat
|
||||
b wye
|
||||
i 13
|
||||
x 0.35922068401384877
|
||||
y 0.8502678133887914
|
||||
x2 0.1290394998233774
|
||||
xy 0.30543378552048117
|
||||
y2 0.7229553544849566
|
||||
x_y_r2 0.5711311780609561
|
||||
xy_y2_r2 0.07876014176187277
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 14
|
||||
x 0.5440047442770544
|
||||
y 0.933608851612059
|
||||
x2 0.2959411617959433
|
||||
xy 0.5078876445760125
|
||||
y2 0.8716254878083876
|
||||
x_y_r2 0.3881145737555198
|
||||
xy_y2_r2 0.3220542860660139
|
||||
|
||||
a wye
|
||||
b dog
|
||||
i 15
|
||||
x 0.4689175303764642
|
||||
y 0.09048353045392021
|
||||
x2 0.21988365029436224
|
||||
xy 0.04242931364019586
|
||||
y2 0.008187269283405506
|
||||
x_y_r2 0.343569469054952
|
||||
xy_y2_r2 0.36217309022695093
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 16
|
||||
x 0.3959177828066379
|
||||
y 0.6339858483805666
|
||||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_r2 0.3462178323525664
|
||||
xy_y2_r2 0.36826111947863793
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 17
|
||||
x 0.34033844788864975
|
||||
y 0.8845934733681523
|
||||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_r2 0.34968865803197713
|
||||
xy_y2_r2 0.39883033745273555
|
||||
|
||||
a wye
|
||||
b wye
|
||||
i 18
|
||||
x 0.6770613653962891
|
||||
y 0.896307226056897
|
||||
x2 0.4584120925122874
|
||||
xy 0.6068549942886431
|
||||
y2 0.8033666434818095
|
||||
x_y_r2 0.21817801153291994
|
||||
xy_y2_r2 0.47791761376746933
|
||||
|
||||
a dog
|
||||
b wye
|
||||
i 19
|
||||
x 0.4865373244199632
|
||||
y 0.44117766146315884
|
||||
x2 0.23671856805373653
|
||||
xy 0.2146493990021416
|
||||
y2 0.1946377289741016
|
||||
x_y_r2 0.21882087937391942
|
||||
xy_y2_r2 0.4693608611953701
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 20
|
||||
x 0.3223311725542929
|
||||
y 0.08115611029827985
|
||||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_r2 0.1512468137008115
|
||||
xy_y2_r2 0.4993396013801769
|
||||
1
go/regtest/cases/verb-stats2/0003c/cmd
Normal file
1
go/regtest/cases/verb-stats2/0003c/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -s -a corr,cov -f x,y,xy,y2 regtest/input/abixy-wide-short
|
||||
259
go/regtest/cases/verb-stats2/0003c/expout
Normal file
259
go/regtest/cases/verb-stats2/0003c/expout
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
a cat
|
||||
b pan
|
||||
i 1
|
||||
x 0.5117389009583777
|
||||
y 0.08295224980036853
|
||||
x2 0.2618767027540883
|
||||
xy 0.0424498931448654
|
||||
y2 0.006881075746942741
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b wye
|
||||
i 2
|
||||
x 0.5225940442098578
|
||||
y 0.511678736087022
|
||||
x2 0.27310453504361476
|
||||
xy 0.2674002600279053
|
||||
y2 0.26181512896361225
|
||||
x_y_corr 1.0000000000002434
|
||||
x_y_cov 0.00232694371217268
|
||||
xy_y2_corr 1.0000000000000002
|
||||
xy_y2_cov 0.028673754401035118
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 3
|
||||
x 0.8150401717873625
|
||||
y 0.07989551500795256
|
||||
x2 0.6642904816271734
|
||||
xy 0.06511805427712146
|
||||
y2 0.006383293318385972
|
||||
x_y_corr -0.4778469068753316
|
||||
x_y_cov -0.020424425846716165
|
||||
xy_y2_corr 0.9956477937863962
|
||||
xy_y2_cov 0.018167590020509827
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 4
|
||||
x 0.4488733555675044
|
||||
y 0.5730530513123552
|
||||
x2 0.20148728933843124
|
||||
xy 0.25722824606077416
|
||||
y2 0.32838979961840076
|
||||
x_y_corr -0.6450444882737297
|
||||
x_y_cov -0.028204957584219037
|
||||
xy_y2_corr 0.9778319292424037
|
||||
xy_y2_cov 0.019936848957115713
|
||||
|
||||
a dog
|
||||
b pan
|
||||
i 5
|
||||
x 0.2946557960430134
|
||||
y 0.6850437256584863
|
||||
x2 0.08682203814174191
|
||||
xy 0.20185210430817294
|
||||
y2 0.46928490606405937
|
||||
x_y_corr -0.7793225547423286
|
||||
x_y_cov -0.04204302461843579
|
||||
xy_y2_corr 0.8168035146908463
|
||||
xy_y2_cov 0.017742165206082017
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 6
|
||||
x 0.048709182664292916
|
||||
y 0.5851879044762575
|
||||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_corr -0.7183022058905697
|
||||
x_y_cov -0.0491921119068774
|
||||
xy_y2_corr 0.5351663740299744
|
||||
xy_y2_cov 0.011245653255343102
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 7
|
||||
x 0.8500003149528544
|
||||
y 0.2984098741712895
|
||||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_corr -0.6901469847430611
|
||||
x_y_cov -0.0480891212244187
|
||||
xy_y2_corr 0.35546644670678323
|
||||
xy_y2_cov 0.00706654969009118
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 8
|
||||
x 0.616507208914765
|
||||
y 0.25924335982487057
|
||||
x2 0.38008113864387366
|
||||
xy 0.15982540019531707
|
||||
y2 0.06720711961328732
|
||||
x_y_corr -0.6995535690063464
|
||||
x_y_cov -0.04332432088186592
|
||||
xy_y2_corr 0.3391495957853624
|
||||
xy_y2_cov 0.006050247133323179
|
||||
|
||||
a hat
|
||||
b hat
|
||||
i 9
|
||||
x 0.33786884067769307
|
||||
y 0.6036735617015514
|
||||
x2 0.11415535350088835
|
||||
xy 0.203962486439877
|
||||
y2 0.3644217690974368
|
||||
x_y_corr -0.7188713718719146
|
||||
x_y_cov -0.042187528038522076
|
||||
xy_y2_corr 0.36702164241757895
|
||||
xy_y2_cov 0.006123821180645354
|
||||
|
||||
a wye
|
||||
b hat
|
||||
i 10
|
||||
x 0.3834648944206174
|
||||
y 0.4999709279216641
|
||||
x2 0.14704532525301522
|
||||
xy 0.19172129908885902
|
||||
y2 0.24997092876684981
|
||||
x_y_corr -0.7241197362159897
|
||||
x_y_cov -0.03850784756718855
|
||||
xy_y2_corr 0.3709081232504834
|
||||
xy_y2_cov 0.005538524953729523
|
||||
|
||||
a pan
|
||||
b hat
|
||||
i 11
|
||||
x 0.025474999754416028
|
||||
y 0.7861954915044592
|
||||
x2 0.0006489756124874967
|
||||
xy 0.020028329952999087
|
||||
y2 0.6181033508619382
|
||||
x_y_corr -0.7897518681515343
|
||||
x_y_cov -0.04997335883595264
|
||||
xy_y2_corr -0.018828461658454355
|
||||
xy_y2_cov -0.0003595317188710478
|
||||
|
||||
a cat
|
||||
b hat
|
||||
i 12
|
||||
x 0.6335445699880142
|
||||
y 0.15467178563525052
|
||||
x2 0.4013787221612979
|
||||
xy 0.0979914699195631
|
||||
y2 0.02392336127159689
|
||||
x_y_corr -0.7970719531193194
|
||||
x_y_cov -0.05018247496334472
|
||||
xy_y2_corr 0.039968723857908096
|
||||
xy_y2_cov 0.0007472725121354662
|
||||
|
||||
a hat
|
||||
b wye
|
||||
i 13
|
||||
x 0.35922068401384877
|
||||
y 0.8502678133887914
|
||||
x2 0.1290394998233774
|
||||
xy 0.30543378552048117
|
||||
y2 0.7229553544849566
|
||||
x_y_corr -0.7557322131952274
|
||||
x_y_cov -0.04919887105688747
|
||||
xy_y2_corr 0.2806423734254552
|
||||
xy_y2_cov 0.006543446048123162
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 14
|
||||
x 0.5440047442770544
|
||||
y 0.933608851612059
|
||||
x2 0.2959411617959433
|
||||
xy 0.5078876445760125
|
||||
y2 0.8716254878083876
|
||||
x_y_corr -0.6229884218470836
|
||||
x_y_cov -0.042223184813877435
|
||||
xy_y2_corr 0.5674982696590483
|
||||
xy_y2_cov 0.020862057783783267
|
||||
|
||||
a wye
|
||||
b dog
|
||||
i 15
|
||||
x 0.4689175303764642
|
||||
y 0.09048353045392021
|
||||
x2 0.21988365029436224
|
||||
xy 0.04242931364019586
|
||||
y2 0.008187269283405506
|
||||
x_y_corr -0.5861479924515245
|
||||
x_y_cov -0.03953923219970396
|
||||
xy_y2_corr 0.6018081839149005
|
||||
xy_y2_cov 0.02231565699284971
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 16
|
||||
x 0.3959177828066379
|
||||
y 0.6339858483805666
|
||||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_corr -0.5884027807145092
|
||||
x_y_cov -0.037547412564836985
|
||||
xy_y2_corr 0.6068452187161385
|
||||
xy_y2_cov 0.02132506373236575
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 17
|
||||
x 0.34033844788864975
|
||||
y 0.8845934733681523
|
||||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_corr -0.5913447877778054
|
||||
x_y_cov -0.03791619972919136
|
||||
xy_y2_corr 0.6315301556162899
|
||||
xy_y2_cov 0.023384664048332934
|
||||
|
||||
a wye
|
||||
b wye
|
||||
i 18
|
||||
x 0.6770613653962891
|
||||
y 0.896307226056897
|
||||
x2 0.4584120925122874
|
||||
xy 0.6068549942886431
|
||||
y2 0.8033666434818095
|
||||
x_y_corr -0.4670952917049365
|
||||
x_y_cov -0.030627517334870798
|
||||
xy_y2_corr 0.6913158567308214
|
||||
xy_y2_cov 0.033014850643411614
|
||||
|
||||
a dog
|
||||
b wye
|
||||
i 19
|
||||
x 0.4865373244199632
|
||||
y 0.44117766146315884
|
||||
x2 0.23671856805373653
|
||||
xy 0.2146493990021416
|
||||
y2 0.1946377289741016
|
||||
x_y_corr -0.46778294044772495
|
||||
x_y_cov -0.029041346125282332
|
||||
xy_y2_corr 0.6850991615783586
|
||||
xy_y2_cov 0.031152619343953587
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 20
|
||||
x 0.3223311725542929
|
||||
y 0.08115611029827985
|
||||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_corr -0.3889046331696387
|
||||
x_y_cov -0.024479265410859593
|
||||
xy_y2_corr 0.7066396545483253
|
||||
xy_y2_cov 0.03267793137969429
|
||||
1
go/regtest/cases/verb-stats2/0004a/cmd
Normal file
1
go/regtest/cases/verb-stats2/0004a/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -s -a linreg-ols,linreg-pca -f x,y,xy,y2 -g a,b regtest/input/abixy-wide-short
|
||||
|
|
@ -13,9 +13,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -23,9 +20,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b wye
|
||||
|
|
@ -42,9 +36,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -52,9 +43,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b cat
|
||||
|
|
@ -71,9 +59,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -81,9 +66,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b hat
|
||||
|
|
@ -100,9 +82,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -110,9 +89,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b pan
|
||||
|
|
@ -129,9 +105,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -139,9 +112,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b cat
|
||||
|
|
@ -151,26 +121,20 @@ y 0.5851879044762575
|
|||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_ols_m -0.659366
|
||||
x_y_ols_b 0.617305
|
||||
x_y_ols_m -0.6593657266118429
|
||||
x_y_ols_b 0.617305070096368
|
||||
x_y_ols_n 2
|
||||
x_y_pca_m -0.659366
|
||||
x_y_pca_b 0.617305
|
||||
x_y_pca_m -0.6593657266118429
|
||||
x_y_pca_b 0.617305070096368
|
||||
x_y_pca_n 2
|
||||
x_y_pca_quality 1.000000
|
||||
x_y_r2 1.000000
|
||||
x_y_corr -1.000000
|
||||
x_y_cov -0.193611
|
||||
xy_y2_ols_m -9.178492
|
||||
xy_y2_ols_b 0.604069
|
||||
x_y_pca_quality 1
|
||||
xy_y2_ols_m -9.17849230382342
|
||||
xy_y2_ols_b 0.6040688533409013
|
||||
xy_y2_ols_n 2
|
||||
xy_y2_pca_m -9.178492
|
||||
xy_y2_pca_b 0.604069
|
||||
xy_y2_pca_m -9.178492303823418
|
||||
xy_y2_pca_b 0.604068853340901
|
||||
xy_y2_pca_n 2
|
||||
xy_y2_pca_quality 1.000000
|
||||
xy_y2_r2 1.000000
|
||||
xy_y2_corr -1.000000
|
||||
xy_y2_cov -0.006152
|
||||
xy_y2_pca_quality 1
|
||||
|
||||
a dog
|
||||
b hat
|
||||
|
|
@ -180,26 +144,20 @@ y 0.2984098741712895
|
|||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_ols_m -0.684679
|
||||
x_y_ols_b 0.880387
|
||||
x_y_ols_m -0.6846789294888149
|
||||
x_y_ols_b 0.8803871798783659
|
||||
x_y_ols_n 2
|
||||
x_y_pca_m -0.684679
|
||||
x_y_pca_b 0.880387
|
||||
x_y_pca_m -0.6846789294888159
|
||||
x_y_pca_b 0.8803871798783665
|
||||
x_y_pca_n 2
|
||||
x_y_pca_quality 1.000000
|
||||
x_y_r2 1.000000
|
||||
x_y_corr -1.000000
|
||||
x_y_cov -0.055083
|
||||
xy_y2_ols_m 66.859625
|
||||
xy_y2_ols_b -16.869794
|
||||
x_y_pca_quality 0.999999999999999
|
||||
xy_y2_ols_m 66.85962507534806
|
||||
xy_y2_ols_b -16.86979429079411
|
||||
xy_y2_ols_n 2
|
||||
xy_y2_pca_m 66.859625
|
||||
xy_y2_pca_b -16.869794
|
||||
xy_y2_pca_m 66.85962507547774
|
||||
xy_y2_pca_b -16.869794290827695
|
||||
xy_y2_pca_n 2
|
||||
xy_y2_pca_quality 1.000000
|
||||
xy_y2_r2 1.000000
|
||||
xy_y2_corr 1.000000
|
||||
xy_y2_cov 0.000428
|
||||
xy_y2_pca_quality 0.9999999999999996
|
||||
|
||||
a pan
|
||||
b pan
|
||||
|
|
@ -216,9 +174,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -226,9 +181,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a hat
|
||||
b hat
|
||||
|
|
@ -245,9 +197,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -255,9 +204,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b hat
|
||||
|
|
@ -274,9 +220,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -284,9 +227,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b hat
|
||||
|
|
@ -303,9 +243,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -313,9 +250,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a cat
|
||||
b hat
|
||||
|
|
@ -332,9 +266,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -342,9 +273,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a hat
|
||||
b wye
|
||||
|
|
@ -361,9 +289,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -371,9 +296,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b dog
|
||||
|
|
@ -390,9 +312,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -400,9 +319,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b dog
|
||||
|
|
@ -419,9 +335,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -429,9 +342,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b pan
|
||||
|
|
@ -441,26 +351,20 @@ y 0.6339858483805666
|
|||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_ols_m -1.698823
|
||||
x_y_ols_b 1.306580
|
||||
x_y_ols_m -1.698823443930671
|
||||
x_y_ols_b 1.3065802596815352
|
||||
x_y_ols_n 2
|
||||
x_y_pca_m -1.698823
|
||||
x_y_pca_b 1.306580
|
||||
x_y_pca_m -1.698823443930662
|
||||
x_y_pca_b 1.3065802596815315
|
||||
x_y_pca_n 2
|
||||
x_y_pca_quality 1.000000
|
||||
x_y_r2 1.000000
|
||||
x_y_corr -1.000000
|
||||
x_y_cov -0.041332
|
||||
xy_y2_ols_m 3.671065
|
||||
xy_y2_ols_b -0.519522
|
||||
x_y_pca_quality 0.9999999999999987
|
||||
xy_y2_ols_m 3.6710653433302842
|
||||
xy_y2_ols_b -0.5195223680276352
|
||||
xy_y2_ols_n 2
|
||||
xy_y2_pca_m 3.671065
|
||||
xy_y2_pca_b -0.519522
|
||||
xy_y2_pca_m 3.6710653433302878
|
||||
xy_y2_pca_b -0.5195223680276352
|
||||
xy_y2_pca_n 2
|
||||
xy_y2_pca_quality 1.000000
|
||||
xy_y2_r2 1.000000
|
||||
xy_y2_corr 1.000000
|
||||
xy_y2_cov 0.015261
|
||||
xy_y2_pca_quality 0.9999999999999999
|
||||
|
||||
a dog
|
||||
b hat
|
||||
|
|
@ -470,26 +374,20 @@ y 0.8845934733681523
|
|||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_ols_m -1.023812
|
||||
x_y_ols_b 1.144767
|
||||
x_y_ols_m -1.0238122955114224
|
||||
x_y_ols_b 1.1447673068767552
|
||||
x_y_ols_n 3
|
||||
x_y_pca_m -1.098934
|
||||
x_y_pca_b 1.185814
|
||||
x_y_pca_m -1.0989338028258004
|
||||
x_y_pca_b 1.1858140019243817
|
||||
x_y_pca_n 3
|
||||
x_y_pca_quality 0.967833
|
||||
x_y_r2 0.878279
|
||||
x_y_corr -0.937166
|
||||
x_y_cov -0.073789
|
||||
xy_y2_ols_m 12.826246
|
||||
xy_y2_ols_b -3.071390
|
||||
x_y_pca_quality 0.9678328196858587
|
||||
xy_y2_ols_m 12.826245725176594
|
||||
xy_y2_ols_b -3.0713903965115823
|
||||
xy_y2_ols_n 3
|
||||
xy_y2_pca_m 13.871436
|
||||
xy_y2_pca_b -3.354267
|
||||
xy_y2_pca_m 13.871436291653312
|
||||
xy_y2_pca_b -3.354267008712503
|
||||
xy_y2_pca_n 3
|
||||
xy_y2_pca_quality 0.999579
|
||||
xy_y2_r2 0.924260
|
||||
xy_y2_corr 0.961385
|
||||
xy_y2_cov 0.008940
|
||||
xy_y2_pca_quality 0.9995788673419361
|
||||
|
||||
a wye
|
||||
b wye
|
||||
|
|
@ -506,9 +404,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -516,9 +411,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b wye
|
||||
|
|
@ -535,9 +427,6 @@ x_y_pca_m
|
|||
x_y_pca_b
|
||||
x_y_pca_n
|
||||
x_y_pca_quality
|
||||
x_y_r2
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_ols_m
|
||||
xy_y2_ols_b
|
||||
xy_y2_ols_n 1
|
||||
|
|
@ -545,9 +434,6 @@ xy_y2_pca_m
|
|||
xy_y2_pca_b
|
||||
xy_y2_pca_n
|
||||
xy_y2_pca_quality
|
||||
xy_y2_r2
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b dog
|
||||
|
|
@ -557,23 +443,17 @@ y 0.08115611029827985
|
|||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_ols_m 3.845532
|
||||
x_y_ols_b -1.158379
|
||||
x_y_ols_m 3.845531673842952
|
||||
x_y_ols_b -1.158378623226191
|
||||
x_y_ols_n 2
|
||||
x_y_pca_m 3.845532
|
||||
x_y_pca_b -1.158379
|
||||
x_y_pca_m 3.8455316738429635
|
||||
x_y_pca_b -1.1583786232261952
|
||||
x_y_pca_n 2
|
||||
x_y_pca_quality 1.000000
|
||||
x_y_r2 1.000000
|
||||
x_y_corr 1.000000
|
||||
x_y_cov 0.094483
|
||||
xy_y2_ols_m 1.795699
|
||||
xy_y2_ols_b -0.040388
|
||||
x_y_pca_quality 0.9999999999999998
|
||||
xy_y2_ols_m 1.7956985581728517
|
||||
xy_y2_ols_b -0.040387623270564124
|
||||
xy_y2_ols_n 2
|
||||
xy_y2_pca_m 1.795699
|
||||
xy_y2_pca_b -0.040388
|
||||
xy_y2_pca_m 1.7956985581728517
|
||||
xy_y2_pca_b -0.040387623270564
|
||||
xy_y2_pca_n 2
|
||||
xy_y2_pca_quality 1.000000
|
||||
xy_y2_r2 1.000000
|
||||
xy_y2_corr 1.000000
|
||||
xy_y2_cov 0.208357
|
||||
xy_y2_pca_quality 0.9999999999999999
|
||||
1
go/regtest/cases/verb-stats2/0004b/cmd
Normal file
1
go/regtest/cases/verb-stats2/0004b/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -s -a r2 -f x,y,xy,y2 -g a,b regtest/input/abixy-wide-short
|
||||
219
go/regtest/cases/verb-stats2/0004b/expout
Normal file
219
go/regtest/cases/verb-stats2/0004b/expout
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
a cat
|
||||
b pan
|
||||
i 1
|
||||
x 0.5117389009583777
|
||||
y 0.08295224980036853
|
||||
x2 0.2618767027540883
|
||||
xy 0.0424498931448654
|
||||
y2 0.006881075746942741
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a pan
|
||||
b wye
|
||||
i 2
|
||||
x 0.5225940442098578
|
||||
y 0.511678736087022
|
||||
x2 0.27310453504361476
|
||||
xy 0.2674002600279053
|
||||
y2 0.26181512896361225
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 3
|
||||
x 0.8150401717873625
|
||||
y 0.07989551500795256
|
||||
x2 0.6642904816271734
|
||||
xy 0.06511805427712146
|
||||
y2 0.006383293318385972
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 4
|
||||
x 0.4488733555675044
|
||||
y 0.5730530513123552
|
||||
x2 0.20148728933843124
|
||||
xy 0.25722824606077416
|
||||
y2 0.32838979961840076
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a dog
|
||||
b pan
|
||||
i 5
|
||||
x 0.2946557960430134
|
||||
y 0.6850437256584863
|
||||
x2 0.08682203814174191
|
||||
xy 0.20185210430817294
|
||||
y2 0.46928490606405937
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 6
|
||||
x 0.048709182664292916
|
||||
y 0.5851879044762575
|
||||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_r2 1
|
||||
xy_y2_r2 1.0000000000000004
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 7
|
||||
x 0.8500003149528544
|
||||
y 0.2984098741712895
|
||||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_r2 0.9999999999999954
|
||||
xy_y2_r2 0.9999999999980602
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 8
|
||||
x 0.616507208914765
|
||||
y 0.25924335982487057
|
||||
x2 0.38008113864387366
|
||||
xy 0.15982540019531707
|
||||
y2 0.06720711961328732
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a hat
|
||||
b hat
|
||||
i 9
|
||||
x 0.33786884067769307
|
||||
y 0.6036735617015514
|
||||
x2 0.11415535350088835
|
||||
xy 0.203962486439877
|
||||
y2 0.3644217690974368
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a wye
|
||||
b hat
|
||||
i 10
|
||||
x 0.3834648944206174
|
||||
y 0.4999709279216641
|
||||
x2 0.14704532525301522
|
||||
xy 0.19172129908885902
|
||||
y2 0.24997092876684981
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a pan
|
||||
b hat
|
||||
i 11
|
||||
x 0.025474999754416028
|
||||
y 0.7861954915044592
|
||||
x2 0.0006489756124874967
|
||||
xy 0.020028329952999087
|
||||
y2 0.6181033508619382
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a cat
|
||||
b hat
|
||||
i 12
|
||||
x 0.6335445699880142
|
||||
y 0.15467178563525052
|
||||
x2 0.4013787221612979
|
||||
xy 0.0979914699195631
|
||||
y2 0.02392336127159689
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a hat
|
||||
b wye
|
||||
i 13
|
||||
x 0.35922068401384877
|
||||
y 0.8502678133887914
|
||||
x2 0.1290394998233774
|
||||
xy 0.30543378552048117
|
||||
y2 0.7229553544849566
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 14
|
||||
x 0.5440047442770544
|
||||
y 0.933608851612059
|
||||
x2 0.2959411617959433
|
||||
xy 0.5078876445760125
|
||||
y2 0.8716254878083876
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a wye
|
||||
b dog
|
||||
i 15
|
||||
x 0.4689175303764642
|
||||
y 0.09048353045392021
|
||||
x2 0.21988365029436224
|
||||
xy 0.04242931364019586
|
||||
y2 0.008187269283405506
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 16
|
||||
x 0.3959177828066379
|
||||
y 0.6339858483805666
|
||||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_r2 1.0000000000000073
|
||||
xy_y2_r2 0.999999999999999
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 17
|
||||
x 0.34033844788864975
|
||||
y 0.8845934733681523
|
||||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_r2 0.8782792870256441
|
||||
xy_y2_r2 0.9242601735245941
|
||||
|
||||
a wye
|
||||
b wye
|
||||
i 18
|
||||
x 0.6770613653962891
|
||||
y 0.896307226056897
|
||||
x2 0.4584120925122874
|
||||
xy 0.6068549942886431
|
||||
y2 0.8033666434818095
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a dog
|
||||
b wye
|
||||
i 19
|
||||
x 0.4865373244199632
|
||||
y 0.44117766146315884
|
||||
x2 0.23671856805373653
|
||||
xy 0.2146493990021416
|
||||
y2 0.1946377289741016
|
||||
x_y_r2
|
||||
xy_y2_r2
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 20
|
||||
x 0.3223311725542929
|
||||
y 0.08115611029827985
|
||||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_r2 0.9999999999999969
|
||||
xy_y2_r2 1.0000000000000002
|
||||
1
go/regtest/cases/verb-stats2/0004c/cmd
Normal file
1
go/regtest/cases/verb-stats2/0004c/cmd
Normal file
|
|
@ -0,0 +1 @@
|
|||
mlr --oxtab stats2 -s -a corr,cov -f x,y,xy,y2 -g a,b regtest/input/abixy-wide-short
|
||||
259
go/regtest/cases/verb-stats2/0004c/expout
Normal file
259
go/regtest/cases/verb-stats2/0004c/expout
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
a cat
|
||||
b pan
|
||||
i 1
|
||||
x 0.5117389009583777
|
||||
y 0.08295224980036853
|
||||
x2 0.2618767027540883
|
||||
xy 0.0424498931448654
|
||||
y2 0.006881075746942741
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b wye
|
||||
i 2
|
||||
x 0.5225940442098578
|
||||
y 0.511678736087022
|
||||
x2 0.27310453504361476
|
||||
xy 0.2674002600279053
|
||||
y2 0.26181512896361225
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 3
|
||||
x 0.8150401717873625
|
||||
y 0.07989551500795256
|
||||
x2 0.6642904816271734
|
||||
xy 0.06511805427712146
|
||||
y2 0.006383293318385972
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 4
|
||||
x 0.4488733555675044
|
||||
y 0.5730530513123552
|
||||
x2 0.20148728933843124
|
||||
xy 0.25722824606077416
|
||||
y2 0.32838979961840076
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b pan
|
||||
i 5
|
||||
x 0.2946557960430134
|
||||
y 0.6850437256584863
|
||||
x2 0.08682203814174191
|
||||
xy 0.20185210430817294
|
||||
y2 0.46928490606405937
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b cat
|
||||
i 6
|
||||
x 0.048709182664292916
|
||||
y 0.5851879044762575
|
||||
x2 0.0023725844758234536
|
||||
xy 0.02850402453206882
|
||||
y2 0.34244488354531344
|
||||
x_y_corr -1
|
||||
x_y_cov -0.19361060830880272
|
||||
xy_y2_corr -1.0000000000000002
|
||||
xy_y2_cov -0.006152284530369208
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 7
|
||||
x 0.8500003149528544
|
||||
y 0.2984098741712895
|
||||
x2 0.7225005354199517
|
||||
xy 0.25364848703063775
|
||||
y2 0.08904845300292483
|
||||
x_y_corr -0.9999999999999977
|
||||
x_y_cov -0.05508339128126383
|
||||
xy_y2_corr 0.9999999999990301
|
||||
xy_y2_cov 0.00042839217341587854
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 8
|
||||
x 0.616507208914765
|
||||
y 0.25924335982487057
|
||||
x2 0.38008113864387366
|
||||
xy 0.15982540019531707
|
||||
y2 0.06720711961328732
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a hat
|
||||
b hat
|
||||
i 9
|
||||
x 0.33786884067769307
|
||||
y 0.6036735617015514
|
||||
x2 0.11415535350088835
|
||||
xy 0.203962486439877
|
||||
y2 0.3644217690974368
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b hat
|
||||
i 10
|
||||
x 0.3834648944206174
|
||||
y 0.4999709279216641
|
||||
x2 0.14704532525301522
|
||||
xy 0.19172129908885902
|
||||
y2 0.24997092876684981
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b hat
|
||||
i 11
|
||||
x 0.025474999754416028
|
||||
y 0.7861954915044592
|
||||
x2 0.0006489756124874967
|
||||
xy 0.020028329952999087
|
||||
y2 0.6181033508619382
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a cat
|
||||
b hat
|
||||
i 12
|
||||
x 0.6335445699880142
|
||||
y 0.15467178563525052
|
||||
x2 0.4013787221612979
|
||||
xy 0.0979914699195631
|
||||
y2 0.02392336127159689
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a hat
|
||||
b wye
|
||||
i 13
|
||||
x 0.35922068401384877
|
||||
y 0.8502678133887914
|
||||
x2 0.1290394998233774
|
||||
xy 0.30543378552048117
|
||||
y2 0.7229553544849566
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 14
|
||||
x 0.5440047442770544
|
||||
y 0.933608851612059
|
||||
x2 0.2959411617959433
|
||||
xy 0.5078876445760125
|
||||
y2 0.8716254878083876
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a wye
|
||||
b dog
|
||||
i 15
|
||||
x 0.4689175303764642
|
||||
y 0.09048353045392021
|
||||
x2 0.21988365029436224
|
||||
xy 0.04242931364019586
|
||||
y2 0.008187269283405506
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a pan
|
||||
b pan
|
||||
i 16
|
||||
x 0.3959177828066379
|
||||
y 0.6339858483805666
|
||||
x2 0.15675089074252413
|
||||
xy 0.25100627142161924
|
||||
y2 0.4019380559468268
|
||||
x_y_corr -1.0000000000000036
|
||||
x_y_cov -0.04133211524441621
|
||||
xy_y2_corr 0.9999999999999996
|
||||
xy_y2_cov 0.015260529200643996
|
||||
|
||||
a dog
|
||||
b hat
|
||||
i 17
|
||||
x 0.34033844788864975
|
||||
y 0.8845934733681523
|
||||
x2 0.11583025911125516
|
||||
xy 0.3010611697385466
|
||||
y2 0.782505613125532
|
||||
x_y_corr -0.937165560093648
|
||||
x_y_cov -0.07378920351961221
|
||||
xy_y2_corr 0.9613845086772409
|
||||
xy_y2_cov 0.008940112074263346
|
||||
|
||||
a wye
|
||||
b wye
|
||||
i 18
|
||||
x 0.6770613653962891
|
||||
y 0.896307226056897
|
||||
x2 0.4584120925122874
|
||||
xy 0.6068549942886431
|
||||
y2 0.8033666434818095
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b wye
|
||||
i 19
|
||||
x 0.4865373244199632
|
||||
y 0.44117766146315884
|
||||
x2 0.23671856805373653
|
||||
xy 0.2146493990021416
|
||||
y2 0.1946377289741016
|
||||
x_y_corr
|
||||
x_y_cov
|
||||
xy_y2_corr
|
||||
xy_y2_cov
|
||||
|
||||
a dog
|
||||
b dog
|
||||
i 20
|
||||
x 0.3223311725542929
|
||||
y 0.08115611029827985
|
||||
x2 0.10389738480022534
|
||||
xy 0.026159144192390068
|
||||
y2 0.006586314238746564
|
||||
x_y_corr 0.9999999999999983
|
||||
x_y_cov 0.09448312194594227
|
||||
xy_y2_corr 1.0000000000000002
|
||||
xy_y2_cov 0.20835701192839565
|
||||
0
go/regtest/cases/verb-stats2/0005/experr
Normal file
0
go/regtest/cases/verb-stats2/0005/experr
Normal file
21
go/regtest/cases/verb-stats2/0005/expout
Normal file
21
go/regtest/cases/verb-stats2/0005/expout
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
a b i x y x2 xy y2 x_y_ols_fit x_y_pca_fit xy_y2_ols_fit xy_y2_pca_fit
|
||||
cat pan 1 0.5117389009583777 0.08295224980036853 0.2618767027540883 0.0424498931448654 0.006881075746942741 0.4649629970529935 0.3669041698884796 0.11853132030600348 -0.04262872571437716
|
||||
pan wye 2 0.5225940442098578 0.511678736087022 0.27310453504361476 0.2674002600279053 0.26181512896361225 0.4590048279772875 0.3424657203088852 0.41758057404011945 0.48327024740272406
|
||||
wye cat 3 0.8150401717873625 0.07989551500795256 0.6642904816271734 0.06511805427712146 0.006383293318385972 0.29848705006544984 -0.3159254015748336 0.14866639749683316 0.010365909622691238
|
||||
dog hat 4 0.4488733555675044 0.5730530513123552 0.20148728933843124 0.25722824606077416 0.32838979961840076 0.4994686263015271 0.5084349102699182 0.4040578880005577 0.4594896607161131
|
||||
dog pan 5 0.2946557960430134 0.6850437256584863 0.08682203814174191 0.20185210430817294 0.46928490606405937 0.5841155326391647 0.85562867505174 0.33044078705846286 0.33002885348826017
|
||||
wye cat 6 0.048709182664292916 0.5851879044762575 0.0023725844758234536 0.02850402453206882 0.34244488354531344 0.7191106655599495 1.4093343029814591 0.09999166794126985 -0.07523199764639858
|
||||
dog hat 7 0.8500003149528544 0.2984098741712895 0.7225005354199517 0.25364848703063775 0.08904845300292483 0.27929813297193407 -0.39463202720400514 0.39929895259029957 0.4511207408337363
|
||||
pan pan 8 0.616507208914765 0.25924335982487057 0.38008113864387366 0.15982540019531707 0.06720711961328732 0.4074577870232062 0.13103671496728597 0.27457044251980006 0.23177695533537246
|
||||
hat hat 9 0.33786884067769307 0.6036735617015514 0.11415535350088835 0.203962486439877 0.3644217690974368 0.5603967644714258 0.758342090432842 0.33324633127839315 0.33496259870204276
|
||||
wye hat 10 0.3834648944206174 0.4999709279216641 0.14704532525301522 0.19172129908885902 0.24997092876684981 0.5353700106929089 0.6556905791068174 0.31697288397148227 0.3063446064451928
|
||||
pan hat 11 0.025474999754416028 0.7861954915044592 0.0006489756124874967 0.020028329952999087 0.6181033508619382 0.7318634396971356 1.4616419874045716 0.08872407067583861 -0.09504685341813644
|
||||
cat hat 12 0.6335445699880142 0.15467178563525052 0.4013787221612979 0.0979914699195631 0.02392336127159689 0.3981063233412847 0.09268008715390597 0.19236835074003447 0.08721884286640735
|
||||
hat wye 13 0.35922068401384877 0.8502678133887914 0.1290394998233774 0.30543378552048117 0.7229553544849566 0.5486771685409477 0.7102721631446056 0.46814238273166076 0.5721867149671821
|
||||
dog dog 14 0.5440047442770544 0.933608851612059 0.2959411617959433 0.5078876445760125 0.8716254878083876 0.4472529267748604 0.2942632874220916 0.7372847553583631 1.045492351477907
|
||||
wye dog 15 0.4689175303764642 0.09048353045392021 0.21988365029436224 0.04242931364019586 0.008187269283405506 0.488466783237088 0.4633089691298464 0.11850396189090542 -0.04267683739573573
|
||||
pan pan 16 0.3959177828066379 0.6339858483805666 0.15675089074252413 0.25100627142161924 0.4019380559468268 0.5285348715550915 0.627655086816047 0.3957863883876706 0.4449436516616201
|
||||
dog hat 17 0.34033844788864975 0.8845934733681523 0.11583025911125516 0.3010611697385466 0.782505613125532 0.5590412469096049 0.7527822032984056 0.46232942272376065 0.5619642193055028
|
||||
wye wye 18 0.6770613653962891 0.896307226056897 0.4584120925122874 0.6068549942886431 0.8033666434818095 0.37422083250356075 -0.005290339013030332 0.8688520538949046 1.276862623539097
|
||||
dog wye 19 0.4865373244199632 0.44117766146315884 0.23671856805373653 0.2146493990021416 0.1946377289741016 0.47879563385119933 0.4236410957209742 0.3474535240505155 0.3599469382261894
|
||||
dog dog 20 0.3223311725542929 0.08115611029827985 0.10389738480022534 0.026159144192390068 0.006586314238746564 0.5689250769567358 0.7933224228173466 0.09687438155764137 -0.08071396320087446
|
||||
0
go/regtest/cases/verb-stats2/0006/experr
Normal file
0
go/regtest/cases/verb-stats2/0006/experr
Normal file
21
go/regtest/cases/verb-stats2/0006/expout
Normal file
21
go/regtest/cases/verb-stats2/0006/expout
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
a b i x y x2 xy y2 x_y_ols_fit x_y_pca_fit xy_y2_ols_fit xy_y2_pca_fit
|
||||
cat pan 1 0.5117389009583777 0.08295224980036853 0.2618767027540883 0.0424498931448654 0.006881075746942741 0.08295224980036714 0.08295224980036847 0.006881075746942736 0.00688107574694273
|
||||
cat hat 12 0.6335445699880142 0.15467178563525052 0.4013787221612979 0.0979914699195631 0.02392336127159689 0.15467178563524925 0.15467178563525058 0.02392336127159691 0.0239233612715969
|
||||
pan wye 2 0.5225940442098578 0.511678736087022 0.27310453504361476 0.2674002600279053 0.26181512896361225 0.4450156174508846 0.43583531932235037 0.23740195053540492 0.03336393241196156
|
||||
pan pan 8 0.616507208914765 0.25924335982487057 0.38008113864387366 0.15982540019531707 0.06720711961328732 0.3721651570527497 0.35647661278247567 0.35312155341344 0.38551715605199943
|
||||
pan hat 11 0.025474999754416028 0.7861954915044592 0.0006489756124874967 0.020028329952999087 0.6181033508619382 0.8306415451369046 0.8559119346540124 0.5035029852566182 0.8431518582294276
|
||||
pan pan 16 0.3959177828066379 0.6339858483805666 0.15675089074252413 0.25100627142161924 0.4019380559468268 0.5432811161563802 0.5428795690380805 0.25503716618020145 0.08703070869227592
|
||||
wye cat 3 0.8150401717873625 0.07989551500795256 0.6642904816271734 0.06511805427712146 0.006383293318385972 0.3378266522291724 -0.22766037118357518 0.13706634945520504 0.11249378615988309
|
||||
wye cat 6 0.048709182664292916 0.5851879044762575 0.0023725844758234536 0.02850402453206882 0.34244488354531344 0.5486404440665242 1.2713467693109621 0.09347961774197075 0.061520804094258925
|
||||
wye hat 10 0.3834648944206174 0.4999709279216641 0.14704532525301522 0.19172129908885902 0.24997092876684981 0.4565508353514721 0.6165367598419671 0.28777966082950845 0.28874712325972834
|
||||
wye dog 15 0.4689175303764642 0.09048353045392021 0.21988365029436224 0.04242931364019586 0.008187269283405506 0.4330432459885998 0.44938429792929446 0.11005681081827151 0.08090718458582725
|
||||
wye wye 18 0.6770613653962891 0.896307226056897 0.4584120925122874 0.6068549942886431 0.8033666434818095 0.3757839262809238 0.04223764801804175 0.7819705795508083 0.8666841202960665
|
||||
dog hat 4 0.4488733555675044 0.5730530513123552 0.20148728933843124 0.25722824606077416 0.32838979961840076 0.5630428912856226 0.6687500310201933 0.4019910911716055 0.40662172437273886
|
||||
dog pan 5 0.2946557960430134 0.6850437256584863 0.08682203814174191 0.20185210430817294 0.46928490606405937 0.61023498409056 1.5049563033296673 0.2975793029036595 0.2551116873383739
|
||||
dog hat 7 0.8500003149528544 0.2984098741712895 0.7225005354199517 0.25364848703063775 0.08904845300292483 0.4402940891891301 -1.506260911084548 0.3952414508016656 0.39682744463304226
|
||||
dog dog 14 0.5440047442770544 0.933608851612059 0.2959411617959433 0.5078876445760125 0.8716254878083876 0.5339317489585992 0.15292379062502492 0.8746097174758546 1.0924299722951658
|
||||
dog hat 17 0.34033844788864975 0.8845934733681523 0.11583025911125516 0.3010611697385466 0.782505613125532 0.5962556424894984 1.2572535117225638 0.4846381265439033 0.5265493263969776
|
||||
dog wye 19 0.4865373244199632 0.44117766146315884 0.23671856805373653 0.2146493990021416 0.1946377289741016 0.5515173456929676 0.4645265490016426 0.32170861891774244 0.2901252966592472
|
||||
dog dog 20 0.3223311725542929 0.08115611029827985 0.10389738480022534 0.026159144192390068 0.006586314238746564 0.6017660461774006 1.3548934732692357 -0.03369000498227847 -0.22558714886339265
|
||||
hat hat 9 0.33786884067769307 0.6036735617015514 0.11415535350088835 0.203962486439877 0.3644217690974368 0.6036735617015432 0.6036735617015538 0.3644217690974366 0.36442176909743684
|
||||
hat wye 13 0.35922068401384877 0.8502678133887914 0.1290394998233774 0.30543378552048117 0.7229553544849566 0.8502678133887818 0.8502678133887898 0.7229553544849577 0.7229553544849565
|
||||
0
go/regtest/cases/verb-stats2/0007/experr
Normal file
0
go/regtest/cases/verb-stats2/0007/experr
Normal file
2
go/regtest/cases/verb-stats2/0007/expout
Normal file
2
go/regtest/cases/verb-stats2/0007/expout
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
x_y_logistic_m x_y_logistic_b x_y_logistic_n
|
||||
0.1454574781964489 0.14544868637515995 22
|
||||
0
go/regtest/cases/verb-stats2/0008/experr
Normal file
0
go/regtest/cases/verb-stats2/0008/experr
Normal file
3
go/regtest/cases/verb-stats2/0008/expout
Normal file
3
go/regtest/cases/verb-stats2/0008/expout
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
g x_y_logistic_m x_y_logistic_b x_y_logistic_n
|
||||
red 0.14545775092523985 -0.03637144999205283 11
|
||||
blue 0.145457205467658 0.32726882274237284 11
|
||||
0
go/regtest/cases/verb-stats2/0009/experr
Normal file
0
go/regtest/cases/verb-stats2/0009/experr
Normal file
1
go/regtest/cases/verb-stats2/0009/expout
Normal file
1
go/regtest/cases/verb-stats2/0009/expout
Normal file
|
|
@ -0,0 +1 @@
|
|||
x_y_cov -0.011480850237481244
|
||||
0
go/regtest/cases/verb-stats2/0010/experr
Normal file
0
go/regtest/cases/verb-stats2/0010/experr
Normal file
10
go/regtest/cases/verb-stats2/0010/expout
Normal file
10
go/regtest/cases/verb-stats2/0010/expout
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
a pan
|
||||
x_y_cov 0.01759507626713419
|
||||
|
||||
a eks
|
||||
x_y_cov 0.03464062871146781
|
||||
|
||||
a wye
|
||||
|
||||
a zee
|
||||
x_y_cov
|
||||
|
|
@ -58,6 +58,7 @@ var MAPPER_LOOKUP_TABLE = []transforming.TransformerSetup{
|
|||
transformers.SortSetup,
|
||||
transformers.SortWithinRecordsSetup,
|
||||
transformers.Stats1Setup,
|
||||
transformers.Stats2Setup,
|
||||
transformers.StepSetup,
|
||||
transformers.TacSetup,
|
||||
transformers.TailSetup,
|
||||
|
|
|
|||
430
go/src/lib/mlrmath.go
Normal file
430
go/src/lib/mlrmath.go
Normal file
|
|
@ -0,0 +1,430 @@
|
|||
// ================================================================
|
||||
// Non-mlrval math routines
|
||||
// ================================================================
|
||||
|
||||
package lib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Some wrappers around things which aren't one-liners from math.*.
|
||||
|
||||
func Sgn(a float64) float64 {
|
||||
if a > 0 {
|
||||
return 1.0
|
||||
} else if a < 0 {
|
||||
return -1.0
|
||||
} else if a == 0 {
|
||||
return 0.0
|
||||
} else {
|
||||
return math.NaN()
|
||||
}
|
||||
}
|
||||
|
||||
// Normal cumulative distribution function, expressed in terms of erfc library
|
||||
// function (which is awkward, but exists).
|
||||
func Qnorm(x float64) float64 {
|
||||
return 0.5 * math.Erfc(-x/math.Sqrt2)
|
||||
}
|
||||
|
||||
// This is a tangent-following method not unlike Newton-Raphson:
|
||||
// * We can compute qnorm(y) = integral from -infinity to y of (1/sqrt(2pi)) exp(-t^2/2) dt.
|
||||
// * We can compute derivative of qnorm(y) = (1/sqrt(2pi)) exp(-y^2/2).
|
||||
// * We cannot explicitly compute invqnorm(y).
|
||||
// * If dx/dy = (1/sqrt(2pi)) exp(-y^2/2) then dy/dx = sqrt(2pi) exp(y^2/2).
|
||||
//
|
||||
// This means we *can* compute the derivative of invqnorm even though we
|
||||
// can't compute the function itself. So the essence of the method is to
|
||||
// follow the tangent line to form successive approximations: we have known function input x
|
||||
// and unknown function output y and initial guess y0. At each step we find the intersection
|
||||
// of the tangent line at y_n with the vertical line at x, to find y_{n+1}. Specificall:
|
||||
//
|
||||
// * Even though we can't compute y = q^-1(x) we can compute x = q(y).
|
||||
// * Start with initial guess for y (y0 = 0.0 or y0 = x both are OK).
|
||||
// * Find x = q(y). Since q (and therefore q^-1) are 1-1, we're done if qnorm(invqnorm(x)) is small.
|
||||
// * Else iterate: using point-slope form, (y_{n+1} - y_n) / (x_{n+1} - x_n) = m = sqrt(2pi) exp(y_n^2/2).
|
||||
// Here x_2 = x (the input) and x_1 = q(y_1).
|
||||
// * Solve for y_{n+1} and repeat.
|
||||
|
||||
const INVQNORM_TOL float64 = 1e-9
|
||||
const INVQNORM_MAXITER int = 30
|
||||
|
||||
func Invqnorm(x float64) float64 {
|
||||
// Initial approximation is linear. Starting with y0 = 0.0 works just as well.
|
||||
y0 := x - 0.5
|
||||
if x <= 0.0 {
|
||||
return 0.0
|
||||
}
|
||||
if x >= 1.0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
y := y0
|
||||
niter := 0
|
||||
|
||||
for {
|
||||
|
||||
backx := Qnorm(y)
|
||||
err := math.Abs(x - backx)
|
||||
if err < INVQNORM_TOL {
|
||||
break
|
||||
}
|
||||
if niter > INVQNORM_MAXITER {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Miller: internal coding error: max iterations %d exceeded in invqnorm.\n",
|
||||
INVQNORM_MAXITER,
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
m := math.Sqrt2 * math.SqrtPi * math.Exp(y*y/2.0)
|
||||
delta_y := m * (x - backx)
|
||||
y += delta_y
|
||||
niter++
|
||||
}
|
||||
|
||||
return y
|
||||
}
|
||||
|
||||
const JACOBI_TOLERANCE = 1e-12
|
||||
const JACOBI_MAXITER = 20
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Jacobi real-symmetric eigensolver. Loosely adapted from Numerical Recipes.
|
||||
//
|
||||
// Note: this is coded for n=2 (to implement PCA linear regression on 2
|
||||
// variables) but the algorithm is quite general. Changing from 2 to n is a
|
||||
// matter of updating the top and bottom of the function: function signature to
|
||||
// take double** matrix, double* eigenvector_1, double* eigenvector_2, and n;
|
||||
// create copy-matrix and make-identity matrix functions; free temp matrices at
|
||||
// the end; etc.
|
||||
|
||||
func GetRealSymmetricEigensystem(
|
||||
matrix [2][2]float64,
|
||||
) (
|
||||
eigenvalue1 float64, // Output: dominant eigenvalue
|
||||
eigenvalue2 float64, // Output: less-dominant eigenvalue
|
||||
eigenvector1 [2]float64, // Output: corresponding to dominant eigenvalue
|
||||
eigenvector2 [2]float64, // Output: corresponding to less-dominant eigenvalue
|
||||
) {
|
||||
L := [2][2]float64{
|
||||
{matrix[0][0], matrix[0][1]},
|
||||
{matrix[1][0], matrix[1][1]},
|
||||
}
|
||||
V := [2][2]float64{
|
||||
{1.0, 0.0},
|
||||
{0.0, 1.0},
|
||||
}
|
||||
var P, PT_A [2][2]float64
|
||||
n := 2
|
||||
|
||||
found := false
|
||||
for iter := 0; iter < JACOBI_MAXITER; iter++ {
|
||||
sum := 0.0
|
||||
for i := 1; i < n; i++ {
|
||||
for j := 0; j < i; j++ {
|
||||
sum += math.Abs(L[i][j])
|
||||
}
|
||||
}
|
||||
if math.Abs(sum*sum) < JACOBI_TOLERANCE {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
|
||||
for p := 0; p < n; p++ {
|
||||
for q := p + 1; q < n; q++ {
|
||||
numer := L[p][p] - L[q][q]
|
||||
denom := L[p][q] + L[q][p]
|
||||
if math.Abs(denom) < JACOBI_TOLERANCE {
|
||||
continue
|
||||
}
|
||||
theta := numer / denom
|
||||
signTheta := 1.0
|
||||
if theta < 0 {
|
||||
signTheta = -1.0
|
||||
}
|
||||
t := signTheta / (math.Abs(theta) + math.Sqrt(theta*theta+1))
|
||||
c := 1.0 / math.Sqrt(t*t+1)
|
||||
s := t * c
|
||||
|
||||
for pi := 0; pi < n; pi++ {
|
||||
for pj := 0; pj < n; pj++ {
|
||||
if pi == pj {
|
||||
P[pi][pj] = 1.0
|
||||
} else {
|
||||
P[pi][pj] = 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
P[p][p] = c
|
||||
P[p][q] = -s
|
||||
P[q][p] = s
|
||||
P[q][q] = c
|
||||
|
||||
// L = P.transpose() * L * P
|
||||
// V = V * P
|
||||
matmul2t(&PT_A, &P, &L)
|
||||
matmul2(&L, &PT_A, &P)
|
||||
matmul2(&V, &V, &P)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"%s: Jacobi eigensolver: max iterations (%d) exceeded. Non-symmetric input?\n",
|
||||
MlrExeName(),
|
||||
JACOBI_MAXITER,
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
eigenvalue1 = L[0][0]
|
||||
eigenvalue2 = L[1][1]
|
||||
abs1 := math.Abs(eigenvalue1)
|
||||
abs2 := math.Abs(eigenvalue2)
|
||||
if abs1 > abs2 {
|
||||
eigenvector1[0] = V[0][0] // Column 0 of V
|
||||
eigenvector1[1] = V[1][0]
|
||||
eigenvector2[0] = V[0][1] // Column 1 of V
|
||||
eigenvector2[1] = V[1][1]
|
||||
} else {
|
||||
eigenvalue1, eigenvalue2 = eigenvalue2, eigenvalue1
|
||||
eigenvector1[0] = V[0][1]
|
||||
eigenvector1[1] = V[1][1]
|
||||
eigenvector2[0] = V[0][0]
|
||||
eigenvector2[1] = V[1][0]
|
||||
}
|
||||
|
||||
return eigenvalue1, eigenvalue2, eigenvector1, eigenvector2
|
||||
}
|
||||
|
||||
// C = A * B
|
||||
func matmul2(
|
||||
C *[2][2]float64, // Output
|
||||
A *[2][2]float64, // Input
|
||||
B *[2][2]float64, // Input
|
||||
) {
|
||||
var T [2][2]float64
|
||||
n := 2
|
||||
for i := 0; i < n; i++ {
|
||||
for j := 0; j < n; j++ {
|
||||
sum := 0.0
|
||||
for k := 0; k < n; k++ {
|
||||
sum += A[i][k] * B[k][j]
|
||||
}
|
||||
T[i][j] = sum
|
||||
}
|
||||
}
|
||||
// Needs copy in case C's memory is the same as A and/or B
|
||||
for i := 0; i < n; i++ {
|
||||
for j := 0; j < n; j++ {
|
||||
C[i][j] = T[i][j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// C = A^t * B
|
||||
func matmul2t(
|
||||
C *[2][2]float64, // Output
|
||||
A *[2][2]float64, // Input
|
||||
B *[2][2]float64, // Input
|
||||
) {
|
||||
var T [2][2]float64
|
||||
n := 2
|
||||
for i := 0; i < n; i++ {
|
||||
for j := 0; j < n; j++ {
|
||||
sum := 0.0
|
||||
for k := 0; k < n; k++ {
|
||||
sum += A[k][i] * B[k][j]
|
||||
}
|
||||
T[i][j] = sum
|
||||
}
|
||||
}
|
||||
// Needs copy in case C's memory is the same as A and/or B
|
||||
for i := 0; i < n; i++ {
|
||||
for j := 0; j < n; j++ {
|
||||
C[i][j] = T[i][j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Logisitic regression
|
||||
//
|
||||
// Real-valued x_0 .. x_{N-1}
|
||||
// 0/1-valued y_0 .. y_{N-1}
|
||||
// Model p(x_i == 1) as
|
||||
// p(x, m, b) = 1 / (1 + exp(-m*x-b)
|
||||
// which is the same as
|
||||
// log(p/(1-p)) = m*x + b
|
||||
// then
|
||||
// p(x, m, b) = 1 / (1 + exp(-m*x-b)
|
||||
// = exp(m*x+b) / (1 + exp(m*x+b)
|
||||
// and
|
||||
// 1-p = exp(-m*x-b) / (1 + exp(-m*x-b)
|
||||
// = 1 / (1 + exp(m*x+b)
|
||||
// Note for reference just below that
|
||||
// dp/dm = -1 / [1 + exp(-m*x-b)]**2 * (-x) * exp(-m*x-b)
|
||||
// = [x exp(-m*x-b)) ] / [1 + exp(-m*x-b)]**2
|
||||
// = x * p * (1-p)
|
||||
// and
|
||||
// dp/db = -1 / [1 + exp(-m*x-b)]**2 * (-1) * exp(-m*x-b)
|
||||
// = [exp(-m*x-b)) ] / [1 + exp(-m*x-b)]**2
|
||||
// = p * (1-p)
|
||||
// Write p_i for p(x_i, m, b)
|
||||
//
|
||||
// Maximum-likelihood equation:
|
||||
// L(m, b) = prod_{i=0}^{N-1} [ p_i**y_i * (1-p_i)**(1-y_i) ]
|
||||
//
|
||||
// Log-likelihood equation:
|
||||
// ell(m, b) = sum{i=0}^{N-1} [ y_i log(p_i) + (1-y_i) log(1-p_i) ]
|
||||
// = sum{i=0}^{N-1} [ log(1-p_i) + y_i log(p_i/(1-p_i)) ]
|
||||
// = sum{i=0}^{N-1} [ log(1-p_i) + y_i*(m*x_i+b) ]
|
||||
// Differentiate with respect to parameters:
|
||||
//
|
||||
// d ell/dm = sum{i=0}^{N-1} [ -1/(1-p_i) dp_i/dm + x_i*y_i ]
|
||||
// = sum{i=0}^{N-1} [ -1/(1-p_i) x_i*p_i*(1-p_i) + x_i*y_i ]
|
||||
// = sum{i=0}^{N-1} [ x_i(y_i-p_i) ]
|
||||
//
|
||||
// d ell/db = sum{i=0}^{N-1} [ -1/(1-p_i) dp_i/db + y_i ]
|
||||
// = sum{i=0}^{N-1} [ -1/(1-p_i) p_i*(1-p_i) + y_i ]
|
||||
// = sum{i=0}^{N-1} [ y_i - p_i ]
|
||||
//
|
||||
//
|
||||
// d2ell/dm2 = sum{i=0}^{N-1} [ -x_i dp_i/dm ]
|
||||
// = sum{i=0}^{N-1} [ -x_i**2 * p_i * (1-p_i) ]
|
||||
//
|
||||
// d2ell/dmdb = sum{i=0}^{N-1} [ -x_i dp_i/db ]
|
||||
// = sum{i=0}^{N-1} [ -x_i * p_i * (1-p_i) ]
|
||||
//
|
||||
// d2ell/dbdm = sum{i=0}^{N-1} [ -dp_i/dm ]
|
||||
// = sum{i=0}^{N-1} [ -x_i * p_i * (1-p_i) ]
|
||||
//
|
||||
// d2ell/db2 = sum{i=0}^{N-1} [ -dp_i/db ]
|
||||
// = sum{i=0}^{N-1} [ -p_i * (1-p_i) ]
|
||||
//
|
||||
// Newton-Raphson to minimize ell(m, b):
|
||||
// * Pick m0, b0
|
||||
// * [m_{j+1], b_{j+1}] = H^{-1} grad ell(m_j, b_j)
|
||||
// * grad ell =
|
||||
// [ d ell/dm ]
|
||||
// [ d ell/db ]
|
||||
// * H = Hessian of ell = Jacobian of grad ell =
|
||||
// [ d2ell/dm2 d2ell/dmdb ]
|
||||
// [ d2ell/dmdb d2ell/db2 ]
|
||||
|
||||
// p(x,m,b) for logistic regression:
|
||||
func lrp(x, m, b float64) float64 {
|
||||
return 1.0 / (1.0 + math.Exp(-m*x-b))
|
||||
}
|
||||
|
||||
// 1 - p(x,m,b) for logistic regression:
|
||||
func lrq(x, m, b float64) float64 {
|
||||
return 1.0 / (1.0 + math.Exp(m*x+b))
|
||||
}
|
||||
|
||||
func LogisticRegression(xs, ys []float64) (m, b float64) {
|
||||
m0 := -0.001
|
||||
b0 := 0.002
|
||||
tol := 1e-9
|
||||
maxits := 100
|
||||
return logisticRegressionAux(xs, ys, m0, b0, tol, maxits)
|
||||
}
|
||||
|
||||
// Supporting routine for mlr_logistic_regression():
|
||||
func logisticRegressionAux(
|
||||
xs, ys []float64,
|
||||
m0, b0, tol float64,
|
||||
maxits int,
|
||||
) (m, b float64) {
|
||||
|
||||
InternalCodingErrorIf(len(xs) != len(ys))
|
||||
n := len(xs)
|
||||
|
||||
its := 0
|
||||
done := false
|
||||
m = m0
|
||||
b = b0
|
||||
|
||||
for !done {
|
||||
// Compute derivatives
|
||||
dldm := 0.0
|
||||
dldb := 0.0
|
||||
d2ldm2 := 0.0
|
||||
d2ldmdb := 0.0
|
||||
d2ldb2 := 0.0
|
||||
ell0 := 0.0
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
xi := xs[i]
|
||||
yi := ys[i]
|
||||
pi := lrp(xi, m0, b0)
|
||||
qi := lrq(xi, m0, b0)
|
||||
dldm += xi * (yi - pi)
|
||||
dldb += yi - pi
|
||||
piqi := pi * qi
|
||||
xipiqi := xi * piqi
|
||||
xi2piqi := xi * xipiqi
|
||||
d2ldm2 -= xi2piqi
|
||||
d2ldmdb -= xipiqi
|
||||
d2ldb2 -= piqi
|
||||
ell0 += math.Log(qi) + yi*(m0*xi+b0)
|
||||
}
|
||||
|
||||
// Form the Hessian
|
||||
ha := d2ldm2
|
||||
hb := d2ldmdb
|
||||
hc := d2ldmdb
|
||||
hd := d2ldb2
|
||||
|
||||
// Invert the Hessian
|
||||
D := ha*hd - hb*hc
|
||||
Hinva := hd / D
|
||||
Hinvb := -hb / D
|
||||
Hinvc := -hc / D
|
||||
Hinvd := ha / D
|
||||
|
||||
// Compute H^-1 times grad ell
|
||||
Hinvgradm := Hinva*dldm + Hinvb*dldb
|
||||
Hinvgradb := Hinvc*dldm + Hinvd*dldb
|
||||
|
||||
// Update [m,b]
|
||||
m = m0 - Hinvgradm
|
||||
b = b0 - Hinvgradb
|
||||
|
||||
ell := 0.0
|
||||
for i := 0; i < n; i++ {
|
||||
xi := xs[i]
|
||||
yi := ys[i]
|
||||
qi := lrq(xi, m, b)
|
||||
ell += math.Log(qi) + yi*(m0*xi+b0)
|
||||
}
|
||||
|
||||
// Check for convergence
|
||||
dell := math.Max(ell, ell0)
|
||||
err := 0.0
|
||||
if dell != 0.0 {
|
||||
err = math.Abs(ell-ell0) / dell
|
||||
}
|
||||
|
||||
if err < tol {
|
||||
done = true
|
||||
}
|
||||
its++
|
||||
if its > maxits {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"mlr_logistic_regression: Newton-Raphson convergence failed after %d iterations. m=%e, b=%e.\n",
|
||||
its, m, b)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
m0 = m
|
||||
b0 = b
|
||||
}
|
||||
|
||||
return m, b
|
||||
}
|
||||
272
go/src/lib/stats.go
Normal file
272
go/src/lib/stats.go
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
// ================================================================
|
||||
// These are intended for streaming (i.e. single-pass) applications. Otherwise
|
||||
// the formulas look different (and are more intuitive).
|
||||
// ================================================================
|
||||
|
||||
package lib
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Univariate linear regression
|
||||
// ----------------------------------------------------------------
|
||||
// There are N (xi, yi) pairs.
|
||||
//
|
||||
// minimize E = sum (yi - m xi - b)^2
|
||||
//
|
||||
// Set the two partial derivatives to zero and solve for m and b:
|
||||
//
|
||||
// DE/Dm = sum 2 (yi - m xi - b) (-xi) = 0
|
||||
// DE/Db = sum 2 (yi - m xi - b) (-1) = 0
|
||||
//
|
||||
// sum (yi - m xi - b) (xi) = 0
|
||||
// sum (yi - m xi - b) = 0
|
||||
//
|
||||
// sum (xi yi - m xi^2 - b xi) = 0
|
||||
// sum (yi - m xi - b) = 0
|
||||
//
|
||||
// m sum(xi^2) + b sum(xi) = sum(xi yi)
|
||||
// m sum(xi) + b N = sum(yi)
|
||||
//
|
||||
// [ sum(xi^2) sum(xi) ] [ m ] = [ sum(xi yi) ]
|
||||
// [ sum(xi) N ] [ b ] = [ sum(yi) ]
|
||||
//
|
||||
// [ m ] = [ sum(xi^2) sum(xi) ]^-1 [ sum(xi yi) ]
|
||||
// [ b ] [ sum(xi) N ] [ sum(yi) ]
|
||||
//
|
||||
// = [ N -sum(xi) ] [ sum(xi yi) ] * 1/D
|
||||
// [ -sum(xi) sum(xi^2)] [ sum(yi) ]
|
||||
//
|
||||
// where
|
||||
//
|
||||
// D = N sum(xi^2) - sum(xi)^2.
|
||||
//
|
||||
// So
|
||||
//
|
||||
// N sum(xi yi) - sum(xi) sum(yi)
|
||||
// m = --------------------------------
|
||||
// D
|
||||
//
|
||||
// -sum(xi)sum(xi yi) + sum(xi^2) sum(yi)
|
||||
// b = ----------------------------------------
|
||||
// D
|
||||
//
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
func GetLinearRegressionOLS(
|
||||
nint int,
|
||||
sumx float64,
|
||||
sumx2 float64,
|
||||
sumxy float64,
|
||||
sumy float64,
|
||||
) (m, b float64) {
|
||||
|
||||
n := float64(nint)
|
||||
D := n*sumx2 - sumx*sumx
|
||||
m = (n*sumxy - sumx*sumy) / D
|
||||
b = (-sumx*sumxy + sumx2*sumy) / D
|
||||
return m, b
|
||||
}
|
||||
|
||||
// We would need a second pass through the data to compute the error-bars given
|
||||
// the data and the m and the b.
|
||||
//
|
||||
// # Young 1962, pp. 122-124. Compute sample variance of linear
|
||||
// # approximations, then variances of m and b.
|
||||
// var_z = 0.0
|
||||
// for i in range(0, N):
|
||||
// var_z += (m * xs[i] + b - ys[i])**2
|
||||
// var_z /= N
|
||||
//
|
||||
// var_m = (N * var_z) / D
|
||||
// var_b = (var_z * sumx2) / D
|
||||
//
|
||||
// output = [m, b, math.sqrt(var_m), math.sqrt(var_b)]
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func GetVar(
|
||||
nint int,
|
||||
sumx float64,
|
||||
sumx2 float64,
|
||||
) float64 {
|
||||
|
||||
n := float64(nint)
|
||||
mean := sumx / n
|
||||
numerator := sumx2 - mean*(2.0*sumx-n*mean)
|
||||
if numerator < 0.0 { // round-off error
|
||||
numerator = 0.0
|
||||
}
|
||||
denominator := n - 1.0
|
||||
return numerator / denominator
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Unbiased estimator:
|
||||
// (1/n) sum{(xi-mean)**3}
|
||||
// -----------------------------
|
||||
// [(1/(n-1)) sum{(xi-mean)**2}]**1.5
|
||||
|
||||
// mean = sumx / n; n mean = sumx
|
||||
|
||||
// sum{(xi-mean)^3}
|
||||
// = sum{xi^3 - 3 mean xi^2 + 3 mean^2 xi - mean^3}
|
||||
// = sum{xi^3} - 3 mean sum{xi^2} + 3 mean^2 sum{xi} - n mean^3
|
||||
// = sumx3 - 3 mean sumx2 + 3 mean^2 sumx - n mean^3
|
||||
// = sumx3 - 3 mean sumx2 + 3n mean^3 - n mean^3
|
||||
// = sumx3 - 3 mean sumx2 + 2n mean^3
|
||||
// = sumx3 - mean*(3 sumx2 + 2n mean^2)
|
||||
|
||||
// sum{(xi-mean)^2}
|
||||
// = sum{xi^2 - 2 mean xi + mean^2}
|
||||
// = sum{xi^2} - 2 mean sum{xi} + n mean^2
|
||||
// = sumx2 - 2 mean sumx + n mean^2
|
||||
// = sumx2 - 2 n mean^2 + n mean^2
|
||||
// = sumx2 - n mean^2
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func GetSkewness(
|
||||
nint int,
|
||||
sumx float64,
|
||||
sumx2 float64,
|
||||
sumx3 float64,
|
||||
) float64 {
|
||||
|
||||
n := float64(nint)
|
||||
mean := sumx / n
|
||||
numerator := sumx3 - mean*(3*sumx2-2*n*mean*mean)
|
||||
numerator = numerator / n
|
||||
denominator := (sumx2 - n*mean*mean) / (n - 1)
|
||||
denominator = math.Pow(denominator, 1.5)
|
||||
return numerator / denominator
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Unbiased:
|
||||
// (1/n) sum{(x-mean)**4}
|
||||
// ----------------------- - 3
|
||||
// [(1/n) sum{(x-mean)**2}]**2
|
||||
|
||||
// sum{(xi-mean)^4}
|
||||
// = sum{xi^4 - 4 mean xi^3 + 6 mean^2 xi^2 - 4 mean^3 xi + mean^4}
|
||||
// = sum{xi^4} - 4 mean sum{xi^3} + 6 mean^2 sum{xi^2} - 4 mean^3 sum{xi} + n mean^4
|
||||
// = sum{xi^4} - 4 mean sum{xi^3} + 6 mean^2 sum{xi^2} - 4 n mean^4 + n mean^4
|
||||
// = sum{xi^4} - 4 mean sum{xi^3} + 6 mean^2 sum{xi^2} - 3 n mean^4
|
||||
// = sum{xi^4} - mean*(4 sum{xi^3} - 6 mean sum{xi^2} + 3 n mean^3)
|
||||
// = sumx4 - mean*(4 sumx3 - 6 mean sumx2 + 3 n mean^3)
|
||||
// = sumx4 - mean*(4 sumx3 - mean*(6 sumx2 - 3 n mean^2))
|
||||
|
||||
func GetKurtosis(
|
||||
nint int,
|
||||
sumx float64,
|
||||
sumx2 float64,
|
||||
sumx3 float64,
|
||||
sumx4 float64,
|
||||
) float64 {
|
||||
|
||||
n := float64(nint)
|
||||
mean := sumx / n
|
||||
numerator := sumx4 - mean*(4*sumx3-mean*(6*sumx2-3*n*mean*mean))
|
||||
numerator = numerator / n
|
||||
denominator := (sumx2 - n*mean*mean) / n
|
||||
denominator = denominator * denominator
|
||||
return numerator/denominator - 3.0
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Non-streaming implementation:
|
||||
//
|
||||
// def find_sample_covariance(xs, ys):
|
||||
// n = len(xs)
|
||||
// mean_x = find_mean(xs)
|
||||
// mean_y = find_mean(ys)
|
||||
//
|
||||
// sum = 0.0
|
||||
// for k in range(0, n):
|
||||
// sum += (xs[k] - mean_x) * (ys[k] - mean_y)
|
||||
//
|
||||
// return sum / (n-1.0)
|
||||
|
||||
func GetCov(
|
||||
nint int,
|
||||
sumx float64,
|
||||
sumy float64,
|
||||
sumxy float64,
|
||||
) float64 {
|
||||
|
||||
n := float64(nint)
|
||||
meanx := sumx / n
|
||||
meany := sumy / n
|
||||
numerator := sumxy - meanx*sumy - meany*sumx + n*meanx*meany
|
||||
denominator := n - 1
|
||||
return numerator / denominator
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func GetCovMatrix(
|
||||
nint int,
|
||||
sumx float64,
|
||||
sumx2 float64,
|
||||
sumy float64,
|
||||
sumy2 float64,
|
||||
sumxy float64,
|
||||
) (Q [2][2]float64) {
|
||||
|
||||
n := float64(nint)
|
||||
denominator := n - 1
|
||||
|
||||
Q[0][0] = (sumx2 - sumx*sumx/n) / denominator
|
||||
Q[0][1] = (sumxy - sumx*sumy/n) / denominator
|
||||
Q[1][0] = Q[0][1]
|
||||
Q[1][1] = (sumy2 - sumy*sumy/n) / denominator
|
||||
|
||||
return Q
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Principal component analysis can be used for linear regression:
|
||||
//
|
||||
// * Compute the covariance matrix for the x's and y's.
|
||||
//
|
||||
// * Find its eigenvalues and eigenvectors of the cov. (This is real-symmetric
|
||||
// so Jacobi iteration is simple and fine.)
|
||||
//
|
||||
// * The principal eigenvector points in the direction of the fit.
|
||||
//
|
||||
// * The covariance matrix is computed on zero-mean data so the intercept
|
||||
// is zero. The fit equation is of the form (y - nu) = m*(x - mu) where mu
|
||||
// and nu are x and y means, respectively.
|
||||
//
|
||||
// * If the fit is perfect then the 2nd eigenvalue will be zero; if the fit is
|
||||
// good then the 2nd eigenvalue will be smaller; if the fit is bad then
|
||||
// they'll be about the same. I use 1 - |lambda2|/|lambda1| as an indication
|
||||
// of quality of the fit.
|
||||
//
|
||||
// Standard ("ordinary least-squares") linear regression is appropriate when
|
||||
// the errors are thought to be all in the y's. PCA ("total least-squares") is
|
||||
// appropriate when the x's and the y's are thought to both have errors.
|
||||
|
||||
func GetLinearRegressionPCA(
|
||||
eigenvalue_1 float64,
|
||||
eigenvalue_2 float64,
|
||||
eigenvector_1 [2]float64,
|
||||
eigenvector_2 [2]float64,
|
||||
x_mean float64,
|
||||
y_mean float64,
|
||||
) (m, b, quality float64) {
|
||||
|
||||
abs_1 := math.Abs(eigenvalue_1)
|
||||
abs_2 := math.Abs(eigenvalue_2)
|
||||
quality = 1.0
|
||||
if abs_1 == 0.0 {
|
||||
quality = 0.0
|
||||
} else if abs_2 > 0.0 {
|
||||
quality = 1.0 - abs_2/abs_1
|
||||
}
|
||||
a0 := eigenvector_1[0]
|
||||
a1 := eigenvector_1[1]
|
||||
m = a1 / a0
|
||||
b = y_mean - m*x_mean
|
||||
return m, b, quality
|
||||
}
|
||||
467
go/src/transformers/stats2.go
Normal file
467
go/src/transformers/stats2.go
Normal file
|
|
@ -0,0 +1,467 @@
|
|||
package transformers
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"miller/src/cliutil"
|
||||
"miller/src/lib"
|
||||
"miller/src/transformers/utils"
|
||||
"miller/src/transforming"
|
||||
"miller/src/types"
|
||||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
const verbNameStats2 = "stats2"
|
||||
|
||||
// For joining "x" and "y" into "x...y" for map keys. "," is another natural choice but would break
|
||||
// if we were ever asked to process field names with commas in them.
|
||||
const stats2KeySeparator = "\001"
|
||||
|
||||
var Stats2Setup = transforming.TransformerSetup{
|
||||
Verb: verbNameStats2,
|
||||
UsageFunc: transformerStats2Usage,
|
||||
ParseCLIFunc: transformerStats2ParseCLI,
|
||||
IgnoresInput: false,
|
||||
}
|
||||
|
||||
func transformerStats2Usage(
|
||||
o *os.File,
|
||||
doExit bool,
|
||||
exitCode int,
|
||||
) {
|
||||
argv0 := lib.MlrExeName()
|
||||
verb := verbNameStats2
|
||||
|
||||
fmt.Fprintf(o, "Usage: %s %s [options]\n", argv0, verb)
|
||||
fmt.Fprintf(o, "Computes bivariate statistics for one or more given field-name pairs,\n")
|
||||
fmt.Fprintf(o, "accumulated across the input record stream.\n")
|
||||
fmt.Fprintf(o, "-a {linreg-ols,corr,...} Names of accumulators: one or more of:\n")
|
||||
|
||||
utils.ListStats2Accumulators(o)
|
||||
|
||||
fmt.Fprintf(o, "-f {a,b,c,d} Value-field name-pairs on which to compute statistics.\n")
|
||||
fmt.Fprintf(o, " There must be an even number of names.\n")
|
||||
fmt.Fprintf(o, "-g {e,f,g} Optional group-by-field names.\n")
|
||||
fmt.Fprintf(o, "-v Print additional output for linreg-pca.\n")
|
||||
fmt.Fprintf(o, "-s Print iterative stats. Useful in tail -f contexts (in which\n")
|
||||
fmt.Fprintf(o, " case please avoid pprint-format output since end of input\n")
|
||||
fmt.Fprintf(o, " stream will never be seen).\n")
|
||||
fmt.Fprintf(o, "--fit Rather than printing regression parameters, applies them to\n")
|
||||
fmt.Fprintf(o, " the input data to compute new fit fields. All input records are\n")
|
||||
fmt.Fprintf(o, " held in memory until end of input stream. Has effect only for\n")
|
||||
fmt.Fprintf(o, " linreg-ols, linreg-pca, and logireg.\n")
|
||||
fmt.Fprintf(o, "Only one of -s or --fit may be used.\n")
|
||||
fmt.Fprintf(o, "Example: %s %s -a linreg-pca -f x,y\n", argv0, verb)
|
||||
fmt.Fprintf(o, "Example: %s %s -a linreg-ols,r2 -f x,y -g size,shape\n", argv0, verb)
|
||||
fmt.Fprintf(o, "Example: %s %s -a corr -f x,y\n", argv0, verb)
|
||||
|
||||
if doExit {
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func transformerStats2ParseCLI(
|
||||
pargi *int,
|
||||
argc int,
|
||||
args []string,
|
||||
_ *cliutil.TReaderOptions,
|
||||
__ *cliutil.TWriterOptions,
|
||||
) transforming.IRecordTransformer {
|
||||
|
||||
// Skip the verb name from the current spot in the mlr command line
|
||||
argi := *pargi
|
||||
verb := args[argi]
|
||||
argi++
|
||||
|
||||
argv0 := lib.MlrExeName()
|
||||
|
||||
var accumulatorNameList []string = nil
|
||||
var valueFieldNameList []string = nil
|
||||
groupByFieldNameList := make([]string, 0)
|
||||
doVerbose := false
|
||||
doIterativeStats := false
|
||||
doHoldAndFit := false
|
||||
|
||||
for argi < argc /* variable increment: 1 or 2 depending on flag */ {
|
||||
opt := args[argi]
|
||||
if !strings.HasPrefix(opt, "-") {
|
||||
break // No more flag options to process
|
||||
}
|
||||
argi++
|
||||
|
||||
if opt == "-h" || opt == "--help" {
|
||||
transformerStats2Usage(os.Stdout, true, 0)
|
||||
|
||||
} else if opt == "-a" {
|
||||
accumulatorNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
} else if opt == "-f" {
|
||||
valueFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
} else if opt == "-g" {
|
||||
groupByFieldNameList = cliutil.VerbGetStringArrayArgOrDie(verb, opt, args, &argi, argc)
|
||||
|
||||
} else if opt == "-v" {
|
||||
doVerbose = true
|
||||
|
||||
} else if opt == "-s" {
|
||||
doIterativeStats = true
|
||||
|
||||
} else if opt == "--fit" {
|
||||
doHoldAndFit = true
|
||||
|
||||
} else if opt == "-S" {
|
||||
// No-op pass-through for backward compatibility with Miller 5
|
||||
|
||||
} else if opt == "-F" {
|
||||
// The -F flag isn't used for stats2: all arithmetic here is
|
||||
// floating-point. Yet it is supported for step and stats1 for all
|
||||
// applicable stats1/step accumulators, so we accept here as well
|
||||
// for all applicable stats2 accumulators (i.e. none of them).
|
||||
|
||||
} else {
|
||||
transformerStats2Usage(os.Stderr, true, 1)
|
||||
}
|
||||
}
|
||||
|
||||
if doIterativeStats && doHoldAndFit {
|
||||
transformerStats2Usage(os.Stderr, true, 1)
|
||||
}
|
||||
if accumulatorNameList == nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: -a option is required.\n", argv0, verb)
|
||||
fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", argv0, verb)
|
||||
os.Exit(1)
|
||||
}
|
||||
if valueFieldNameList == nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: -f option is required.\n", argv0, verb)
|
||||
fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", argv0, verb)
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(valueFieldNameList)%2 != 0 {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: argument to -f must have even number of fields.\n", argv0, verb)
|
||||
fmt.Fprintf(os.Stderr, "Please see %s %s --help for more information.\n", argv0, verb)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
transformer, _ := NewTransformerStats2(
|
||||
accumulatorNameList,
|
||||
valueFieldNameList,
|
||||
groupByFieldNameList,
|
||||
doVerbose,
|
||||
doIterativeStats,
|
||||
doHoldAndFit,
|
||||
)
|
||||
|
||||
*pargi = argi
|
||||
return transformer
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type TransformerStats2 struct {
|
||||
// Input:
|
||||
accumulatorNameList []string
|
||||
valueFieldNameList []string
|
||||
groupByFieldNameList []string
|
||||
|
||||
doVerbose bool
|
||||
doIterativeStats bool
|
||||
doHoldAndFit bool
|
||||
|
||||
// State:
|
||||
accumulatorFactory *utils.Stats2AccumulatorFactory
|
||||
|
||||
// Accumulators are indexed by
|
||||
// groupByFieldName . value1FieldName+sep+value2FieldName . accumulatorName . accumulator object
|
||||
// This would be
|
||||
// namedAccumulators map[string]map[string]map[string]IStats2Accumulator
|
||||
// except we need maps that preserve insertion order.
|
||||
namedAccumulators *lib.OrderedMap
|
||||
|
||||
groupingKeysToGroupByFieldValues *lib.OrderedMap
|
||||
|
||||
// For hold-and-fit:
|
||||
// ordered map from grouping-key to list of RecordAndContext
|
||||
recordGroups *lib.OrderedMap
|
||||
}
|
||||
|
||||
func NewTransformerStats2(
|
||||
accumulatorNameList []string,
|
||||
valueFieldNameList []string,
|
||||
groupByFieldNameList []string,
|
||||
doVerbose bool,
|
||||
doIterativeStats bool,
|
||||
doHoldAndFit bool,
|
||||
) (*TransformerStats2, error) {
|
||||
for _, name := range accumulatorNameList {
|
||||
if !utils.ValidateStats2AccumulatorName(name) {
|
||||
return nil, errors.New(
|
||||
fmt.Sprintf(
|
||||
"%s stats2: accumulator \"%s\" not found.\n",
|
||||
lib.MlrExeName(), name,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
this := &TransformerStats2{
|
||||
accumulatorNameList: accumulatorNameList,
|
||||
valueFieldNameList: valueFieldNameList,
|
||||
groupByFieldNameList: groupByFieldNameList,
|
||||
doVerbose: doVerbose,
|
||||
doIterativeStats: doIterativeStats,
|
||||
doHoldAndFit: doHoldAndFit,
|
||||
accumulatorFactory: utils.NewStats2AccumulatorFactory(),
|
||||
namedAccumulators: lib.NewOrderedMap(),
|
||||
groupingKeysToGroupByFieldValues: lib.NewOrderedMap(),
|
||||
recordGroups: lib.NewOrderedMap(),
|
||||
}
|
||||
return this, nil
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Given: accumulate corr,cov on values x,y group by a,b.
|
||||
// Example input: Example output:
|
||||
// a b x y a b x_corr x_cov y_corr y_cov
|
||||
// s t 1 2 s t 2 6 2 8
|
||||
// u v 3 4 u v 1 3 1 4
|
||||
// s t 5 6 u w 1 7 1 9
|
||||
// u w 7 9
|
||||
//
|
||||
// Multilevel hashmap structure:
|
||||
// {
|
||||
// ["s","t"] : { <--- group-by field names
|
||||
// ["x","y"] : { <--- value field names
|
||||
// "corr" : stats2_corr object,
|
||||
// "cov" : stats2_cov object
|
||||
// }
|
||||
// },
|
||||
// ["u","v"] : {
|
||||
// ["x","y"] : {
|
||||
// "corr" : stats2_corr object,
|
||||
// "cov" : stats2_cov object
|
||||
// }
|
||||
// },
|
||||
// ["u","w"] : {
|
||||
// ["x","y"] : {
|
||||
// "corr" : stats2_corr object,
|
||||
// "cov" : stats2_cov object
|
||||
// }
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// In the iterative case, add to the current record its current group's stats fields.
|
||||
// In the non-iterative case, produce output only at the end of the input stream.
|
||||
// ================================================================
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *TransformerStats2) Transform(
|
||||
inrecAndContext *types.RecordAndContext,
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
) {
|
||||
if !inrecAndContext.EndOfStream {
|
||||
|
||||
this.ingest(inrecAndContext)
|
||||
|
||||
if this.doIterativeStats {
|
||||
// The input record is modified in this case, with new fields appended
|
||||
outputChannel <- inrecAndContext
|
||||
}
|
||||
// if this.doHoldAndFit, the input record is held by the ingestor
|
||||
|
||||
} else { // end of record stream
|
||||
if !this.doIterativeStats { // in the iterative case, already emitted per-record
|
||||
if this.doHoldAndFit {
|
||||
this.fit(outputChannel)
|
||||
} else {
|
||||
this.emit(outputChannel, &inrecAndContext.Context)
|
||||
}
|
||||
}
|
||||
outputChannel <- inrecAndContext // end-of-stream marker
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *TransformerStats2) ingest(
|
||||
inrecAndContext *types.RecordAndContext,
|
||||
) {
|
||||
inrec := inrecAndContext.Record
|
||||
|
||||
// E.g. if grouping by "a" and "b", and the current record has a=circle, b=blue,
|
||||
// then groupingKey is the string "circle,blue".
|
||||
groupingKey, groupByFieldValues, ok := inrec.GetSelectedValuesAndJoined(this.groupByFieldNameList)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
this.groupingKeysToGroupByFieldValues.Put(groupingKey, groupByFieldValues)
|
||||
|
||||
groupToValueFields := this.namedAccumulators.Get(groupingKey)
|
||||
if groupToValueFields == nil {
|
||||
groupToValueFields = lib.NewOrderedMap()
|
||||
this.namedAccumulators.Put(groupingKey, groupToValueFields)
|
||||
}
|
||||
|
||||
if this.doHoldAndFit { // Retain the input record in memory, for fitting and delivery at end of stream
|
||||
groupToRecords := this.recordGroups.Get(groupingKey)
|
||||
if groupToRecords == nil {
|
||||
groupToRecords = list.New()
|
||||
this.recordGroups.Put(groupingKey, groupToRecords)
|
||||
}
|
||||
groupToRecords.(*list.List).PushBack(inrecAndContext)
|
||||
}
|
||||
|
||||
// for [["x","y"]]
|
||||
n := len(this.valueFieldNameList)
|
||||
for i := 0; i < n; i += 2 {
|
||||
valueFieldName1 := this.valueFieldNameList[i]
|
||||
valueFieldName2 := this.valueFieldNameList[i+1]
|
||||
|
||||
key := valueFieldName1 + stats2KeySeparator + valueFieldName2
|
||||
|
||||
valueFieldsToAccumulator := groupToValueFields.(*lib.OrderedMap).Get(key)
|
||||
if valueFieldsToAccumulator == nil {
|
||||
valueFieldsToAccumulator = lib.NewOrderedMap()
|
||||
groupToValueFields.(*lib.OrderedMap).Put(key, valueFieldsToAccumulator)
|
||||
}
|
||||
|
||||
mval1 := inrec.Get(valueFieldName1)
|
||||
mval2 := inrec.Get(valueFieldName2)
|
||||
if mval1 == nil || mval2 == nil { // Key absent in current record
|
||||
continue
|
||||
}
|
||||
if mval1.IsVoid() || mval2.IsVoid() { // Key present in current record but with empty value
|
||||
continue
|
||||
}
|
||||
|
||||
// for ["corr", "cov"]
|
||||
for _, accumulatorName := range this.accumulatorNameList {
|
||||
accumulator := valueFieldsToAccumulator.(*lib.OrderedMap).Get(accumulatorName)
|
||||
if accumulator == nil {
|
||||
accumulator = this.accumulatorFactory.Make(
|
||||
valueFieldName1,
|
||||
valueFieldName2,
|
||||
accumulatorName,
|
||||
this.doVerbose,
|
||||
)
|
||||
if accumulator == nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: accumulator \"%s\" not found.\n",
|
||||
lib.MlrExeName(), verbNameStats2, accumulatorName,
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
valueFieldsToAccumulator.(*lib.OrderedMap).Put(accumulatorName, accumulator)
|
||||
}
|
||||
accumulator.(utils.IStats2Accumulator).Ingest(
|
||||
mval1.GetNumericToFloatValueOrDie(),
|
||||
mval2.GetNumericToFloatValueOrDie(),
|
||||
)
|
||||
}
|
||||
|
||||
if this.doIterativeStats {
|
||||
this.populateRecord(
|
||||
inrecAndContext.Record,
|
||||
valueFieldName1,
|
||||
valueFieldName2,
|
||||
valueFieldsToAccumulator.(*lib.OrderedMap),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func (this *TransformerStats2) emit(
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
context *types.Context,
|
||||
) {
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
outrec := types.NewMlrmapAsRecord()
|
||||
|
||||
// Add in a=s,b=t fields:
|
||||
groupingKey := pa.Key
|
||||
groupByFieldValues := this.groupingKeysToGroupByFieldValues.Get(groupingKey).([]*types.Mlrval)
|
||||
for i, groupByFieldName := range this.groupByFieldNameList {
|
||||
outrec.PutReference(groupByFieldName, groupByFieldValues[i].Copy())
|
||||
}
|
||||
|
||||
// Add in fields such as x_y_corr, etc.
|
||||
groupToValueFields := this.namedAccumulators.Get(groupingKey).(*lib.OrderedMap)
|
||||
|
||||
// For "x","y"
|
||||
for pc := groupToValueFields.Head; pc != nil; pc = pc.Next {
|
||||
|
||||
pairs := strings.Split(pc.Key, stats2KeySeparator)
|
||||
valueFieldName1 := pairs[0]
|
||||
valueFieldName2 := pairs[1]
|
||||
valueFieldsToAccumulator := pc.Value.(*lib.OrderedMap)
|
||||
|
||||
this.populateRecord(outrec, valueFieldName1, valueFieldName2, valueFieldsToAccumulator)
|
||||
|
||||
// For "corr", "linreg"
|
||||
for pd := valueFieldsToAccumulator.Head; pd != nil; pd = pd.Next {
|
||||
accumulator := pd.Value.(utils.IStats2Accumulator)
|
||||
accumulator.Populate(valueFieldName1, valueFieldName2, outrec)
|
||||
}
|
||||
}
|
||||
|
||||
outputChannel <- types.NewRecordAndContext(outrec, context)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *TransformerStats2) populateRecord(
|
||||
outrec *types.Mlrmap,
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
valueFieldsToAccumulator *lib.OrderedMap,
|
||||
) {
|
||||
// For "corr", "linreg"
|
||||
for pe := valueFieldsToAccumulator.Head; pe != nil; pe = pe.Next {
|
||||
accumulator := pe.Value.(utils.IStats2Accumulator)
|
||||
accumulator.Populate(valueFieldName1, valueFieldName2, outrec)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *TransformerStats2) fit(
|
||||
outputChannel chan<- *types.RecordAndContext,
|
||||
) {
|
||||
for pa := this.namedAccumulators.Head; pa != nil; pa = pa.Next {
|
||||
groupingKey := pa.Key
|
||||
groupToValueFields := pa.Value.(*lib.OrderedMap)
|
||||
recordsAndContexts := this.recordGroups.Get(groupingKey).(*list.List)
|
||||
|
||||
for recordsAndContexts.Front() != nil {
|
||||
recordAndContext := recordsAndContexts.Remove(recordsAndContexts.Front()).(*types.RecordAndContext)
|
||||
record := recordAndContext.Record
|
||||
|
||||
// For "x","y"
|
||||
for pb := groupToValueFields.Head; pb != nil; pb = pb.Next {
|
||||
pairs := strings.Split(pb.Key, stats2KeySeparator)
|
||||
valueFieldName1 := pairs[0]
|
||||
valueFieldName2 := pairs[1]
|
||||
valueFieldsToAccumulator := pb.Value.(*lib.OrderedMap)
|
||||
|
||||
// For "linreg-ols", "logireg"
|
||||
for pc := valueFieldsToAccumulator.Head; pc != nil; pc = pc.Next {
|
||||
accumulator := pc.Value.(utils.IStats2Accumulator)
|
||||
|
||||
// Note R2, cov, corr, etc have no non-trivial fit-function
|
||||
mval1 := record.Get(valueFieldName1)
|
||||
mval2 := record.Get(valueFieldName2)
|
||||
if mval1 != nil && mval2 != nil {
|
||||
accumulator.Fit(
|
||||
mval1.GetNumericToFloatValueOrDie(),
|
||||
mval2.GetNumericToFloatValueOrDie(),
|
||||
record,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
outputChannel <- recordAndContext
|
||||
}
|
||||
}
|
||||
}
|
||||
688
go/src/transformers/utils/stats2-accumulators.go
Normal file
688
go/src/transformers/utils/stats2-accumulators.go
Normal file
|
|
@ -0,0 +1,688 @@
|
|||
// ================================================================
|
||||
// For stats2
|
||||
// ================================================================
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
|
||||
"miller/src/lib"
|
||||
"miller/src/types"
|
||||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type IStats2Accumulator interface {
|
||||
Ingest(
|
||||
x float64,
|
||||
y float64,
|
||||
)
|
||||
|
||||
Populate(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
outrec *types.Mlrmap,
|
||||
)
|
||||
|
||||
Fit(
|
||||
x float64,
|
||||
y float64,
|
||||
outrec *types.Mlrmap,
|
||||
)
|
||||
}
|
||||
|
||||
type newStats2AccumulatorFunc func(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator
|
||||
|
||||
type stats2AccumulatorInfo struct {
|
||||
name string
|
||||
description string
|
||||
constructor newStats2AccumulatorFunc
|
||||
}
|
||||
|
||||
var stats2AccumulatorInfos []stats2AccumulatorInfo = []stats2AccumulatorInfo{
|
||||
{
|
||||
"linreg-ols",
|
||||
"Linear regression using ordinary least squares",
|
||||
NewStats2LinRegOLSAccumulator,
|
||||
},
|
||||
{
|
||||
"linreg-pca",
|
||||
"Linear regression using principal component analysis",
|
||||
NewStats2LinRegPCAAccumulator,
|
||||
},
|
||||
{
|
||||
"r2",
|
||||
"Quality metric for linreg-ols (linreg-pca emits its own)",
|
||||
NewStats2R2Accumulator,
|
||||
},
|
||||
{
|
||||
"logireg",
|
||||
"Logistic regression",
|
||||
NewStats2LogiRegAccumulator,
|
||||
},
|
||||
{
|
||||
"corr",
|
||||
"Sample correlation",
|
||||
NewStats2CorrAccumulator,
|
||||
},
|
||||
{
|
||||
"cov",
|
||||
"Sample covariance",
|
||||
NewStats2CovAccumulator,
|
||||
},
|
||||
{
|
||||
"covx",
|
||||
"Sample-covariance matrix",
|
||||
NewStats2CovXAccumulator,
|
||||
},
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
type Stats2AccumulatorFactory struct {
|
||||
}
|
||||
|
||||
func NewStats2AccumulatorFactory() *Stats2AccumulatorFactory {
|
||||
return &Stats2AccumulatorFactory{}
|
||||
}
|
||||
|
||||
func ListStats2Accumulators(o *os.File) {
|
||||
for _, info := range stats2AccumulatorInfos {
|
||||
fmt.Fprintf(o, " %-8s %s\n", info.name, info.description)
|
||||
}
|
||||
}
|
||||
|
||||
func ValidateStats2AccumulatorName(
|
||||
accumulatorName string,
|
||||
) bool {
|
||||
for _, info := range stats2AccumulatorInfos {
|
||||
if info.name == accumulatorName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *Stats2AccumulatorFactory) Make(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
// TODO: hashmapify the lookup table
|
||||
for _, info := range stats2AccumulatorInfos {
|
||||
if info.name == accumulatorName {
|
||||
return info.constructor(valueFieldName1, valueFieldName2, accumulatorName, doVerbose)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
type Stats2LinRegOLSAccumulator struct {
|
||||
count int
|
||||
sumx float64
|
||||
sumy float64
|
||||
sumx2 float64
|
||||
sumxy float64
|
||||
mOutputFieldName string
|
||||
bOutputFieldName string
|
||||
nOutputFieldName string
|
||||
fitOutputFieldName string
|
||||
fitReady bool
|
||||
m float64
|
||||
b float64
|
||||
}
|
||||
|
||||
func NewStats2LinRegOLSAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
prefix := valueFieldName1 + "_" + valueFieldName2 + "_"
|
||||
return &Stats2LinRegOLSAccumulator{
|
||||
count: 0,
|
||||
sumx: 0.0,
|
||||
sumy: 0.0,
|
||||
sumx2: 0.0,
|
||||
sumxy: 0.0,
|
||||
mOutputFieldName: prefix + "ols_m",
|
||||
bOutputFieldName: prefix + "ols_b",
|
||||
nOutputFieldName: prefix + "ols_n",
|
||||
fitOutputFieldName: prefix + "ols_fit",
|
||||
fitReady: false,
|
||||
m: -999.0,
|
||||
b: -999.0,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stats2LinRegOLSAccumulator) Ingest(
|
||||
x float64,
|
||||
y float64,
|
||||
) {
|
||||
this.count++
|
||||
this.sumx += x
|
||||
this.sumy += y
|
||||
this.sumx2 += x * x
|
||||
this.sumxy += x * y
|
||||
}
|
||||
|
||||
func (this *Stats2LinRegOLSAccumulator) Populate(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(this.mOutputFieldName, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(this.bOutputFieldName, types.MLRVAL_VOID)
|
||||
} else {
|
||||
|
||||
m, b := lib.GetLinearRegressionOLS(this.count, this.sumx, this.sumx2, this.sumxy, this.sumy)
|
||||
|
||||
outrec.PutReference(this.mOutputFieldName, types.MlrvalPointerFromFloat64(m))
|
||||
outrec.PutReference(this.bOutputFieldName, types.MlrvalPointerFromFloat64(b))
|
||||
}
|
||||
outrec.PutReference(this.nOutputFieldName, types.MlrvalPointerFromInt(this.count))
|
||||
}
|
||||
|
||||
func (this *Stats2LinRegOLSAccumulator) Fit(
|
||||
x float64,
|
||||
y float64,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
|
||||
if !this.fitReady {
|
||||
// Idea for hold-and-fit in stats2.go is:
|
||||
// * We've ingested say 10,000 records
|
||||
// * After the end of those we compute m and b
|
||||
// * Then for all 10,000 records we compute y = m*x + b
|
||||
// The fitReady flag keeps us from recomputing the linear fit 10,000 times
|
||||
this.m, this.b = lib.GetLinearRegressionOLS(this.count, this.sumx, this.sumx2, this.sumxy, this.sumy)
|
||||
this.fitReady = true
|
||||
}
|
||||
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(this.fitOutputFieldName, types.MLRVAL_VOID)
|
||||
} else {
|
||||
yfit := this.m*x + this.b
|
||||
outrec.PutReference(this.fitOutputFieldName, types.MlrvalPointerFromFloat64(yfit))
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
const LOGIREG_DVECTOR_INITIAL_SIZE = 16
|
||||
|
||||
type Stats2LogiRegAccumulator struct {
|
||||
xs []float64
|
||||
ys []float64
|
||||
mOutputFieldName string
|
||||
bOutputFieldName string
|
||||
nOutputFieldName string
|
||||
fitOutputFieldName string
|
||||
fitReady bool
|
||||
m float64
|
||||
b float64
|
||||
}
|
||||
|
||||
func NewStats2LogiRegAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
prefix := valueFieldName1 + "_" + valueFieldName2 + "_"
|
||||
return &Stats2LogiRegAccumulator{
|
||||
xs: make([]float64, 0, LOGIREG_DVECTOR_INITIAL_SIZE),
|
||||
ys: make([]float64, 0, LOGIREG_DVECTOR_INITIAL_SIZE),
|
||||
mOutputFieldName: prefix + "logistic_m",
|
||||
bOutputFieldName: prefix + "logistic_b",
|
||||
nOutputFieldName: prefix + "logistic_n",
|
||||
fitOutputFieldName: prefix + "logistic_fit",
|
||||
fitReady: false,
|
||||
m: -999.0,
|
||||
b: -999.0,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stats2LogiRegAccumulator) Ingest(
|
||||
x float64,
|
||||
y float64,
|
||||
) {
|
||||
this.xs = append(this.xs, x) // append is smart about cap-increase via doubling
|
||||
this.ys = append(this.ys, y) // append is smart about cap-increase via doubling
|
||||
}
|
||||
|
||||
func (this *Stats2LogiRegAccumulator) Populate(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
|
||||
if len(this.xs) < 2 {
|
||||
outrec.PutCopy(this.mOutputFieldName, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(this.bOutputFieldName, types.MLRVAL_VOID)
|
||||
} else {
|
||||
m, b := lib.LogisticRegression(this.xs, this.ys)
|
||||
outrec.PutCopy(this.mOutputFieldName, types.MlrvalPointerFromFloat64(m))
|
||||
outrec.PutCopy(this.bOutputFieldName, types.MlrvalPointerFromFloat64(b))
|
||||
}
|
||||
outrec.PutReference(this.nOutputFieldName, types.MlrvalPointerFromInt(len(this.xs)))
|
||||
}
|
||||
|
||||
func (this *Stats2LogiRegAccumulator) Fit(
|
||||
x float64,
|
||||
y float64,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
|
||||
if !this.fitReady {
|
||||
// Idea for hold-and-fit in stats2.go is:
|
||||
// * We've ingested say 10,000 records
|
||||
// * After the end of those we compute m and b
|
||||
// * Then for all 10,000 records we compute y = m*x + b
|
||||
// The fitReady flag keeps us from recomputing the linear fit 10,000 times
|
||||
this.m, this.b = lib.LogisticRegression(this.xs, this.ys)
|
||||
this.fitReady = true
|
||||
}
|
||||
|
||||
if len(this.xs) < 2 {
|
||||
outrec.PutCopy(this.fitOutputFieldName, types.MLRVAL_VOID)
|
||||
} else {
|
||||
yfit := 1.0 / (1.0 + math.Exp(-this.m*x-this.b))
|
||||
outrec.PutReference(this.fitOutputFieldName, types.MlrvalPointerFromFloat64(yfit))
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// http://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient
|
||||
// Alternatively, just use sqrt(corr) as defined above.
|
||||
|
||||
type Stats2R2Accumulator struct {
|
||||
count int
|
||||
sumx float64
|
||||
sumy float64
|
||||
sumx2 float64
|
||||
sumxy float64
|
||||
sumy2 float64
|
||||
r2OutputFieldName string
|
||||
}
|
||||
|
||||
func NewStats2R2Accumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
prefix := valueFieldName1 + "_" + valueFieldName2 + "_"
|
||||
return &Stats2R2Accumulator{
|
||||
count: 0,
|
||||
sumx: 0.0,
|
||||
sumy: 0.0,
|
||||
sumx2: 0.0,
|
||||
sumxy: 0.0,
|
||||
sumy2: 0.0,
|
||||
r2OutputFieldName: prefix + "r2",
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stats2R2Accumulator) Ingest(
|
||||
x float64,
|
||||
y float64,
|
||||
) {
|
||||
this.count++
|
||||
this.sumx += x
|
||||
this.sumy += y
|
||||
this.sumx2 += x * x
|
||||
this.sumxy += x * y
|
||||
this.sumy2 += y * y
|
||||
}
|
||||
|
||||
func (this *Stats2R2Accumulator) Populate(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(this.r2OutputFieldName, types.MLRVAL_VOID)
|
||||
} else {
|
||||
n := float64(this.count)
|
||||
sumx := this.sumx
|
||||
sumy := this.sumy
|
||||
sumx2 := this.sumx2
|
||||
sumy2 := this.sumy2
|
||||
sumxy := this.sumxy
|
||||
numerator := n*sumxy - sumx*sumy
|
||||
numerator = numerator * numerator
|
||||
denominator := (n*sumx2 - sumx*sumx) * (n*sumy2 - sumy*sumy)
|
||||
output := numerator / denominator
|
||||
outrec.PutReference(this.r2OutputFieldName, types.MlrvalPointerFromFloat64(output))
|
||||
}
|
||||
}
|
||||
|
||||
// Trivial function; there is no fit-feature here
|
||||
func (this *Stats2R2Accumulator) Fit(
|
||||
x float64,
|
||||
y float64,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Shared code for Corr, Cov, CovX, and LinRegPCA.
|
||||
// Corr(X,Y) = Cov(X,Y) / sigma_X sigma_Y.
|
||||
|
||||
type BivarMeasure int
|
||||
|
||||
const (
|
||||
DO_CORR BivarMeasure = iota
|
||||
DO_COV
|
||||
DO_COVX
|
||||
DO_LINREG_PCA
|
||||
)
|
||||
|
||||
type Stats2CorrCovAccumulator struct {
|
||||
count int
|
||||
sumx float64
|
||||
sumy float64
|
||||
sumx2 float64
|
||||
sumxy float64
|
||||
sumy2 float64
|
||||
|
||||
doWhich BivarMeasure
|
||||
doVerbose bool
|
||||
|
||||
corrOutputFieldName string
|
||||
|
||||
covOutputFieldName string
|
||||
|
||||
covx00OutputFieldName string
|
||||
covx01OutputFieldName string
|
||||
covx10OutputFieldName string
|
||||
covx11OutputFieldName string
|
||||
|
||||
pca_mOutputFieldName string
|
||||
pca_bOutputFieldName string
|
||||
pca_nOutputFieldName string
|
||||
pca_qOutputFieldName string
|
||||
pca_l1OutputFieldName string
|
||||
pca_l2OutputFieldName string
|
||||
pca_v11OutputFieldName string
|
||||
pca_v12OutputFieldName string
|
||||
pca_v21OutputFieldName string
|
||||
pca_v22OutputFieldName string
|
||||
pca_fitOutputFieldName string
|
||||
|
||||
fitReady bool
|
||||
m float64
|
||||
b float64
|
||||
q float64
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func NewStats2CorrCovAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
doWhich BivarMeasure,
|
||||
) IStats2Accumulator {
|
||||
prefix := valueFieldName1 + "_" + valueFieldName2 + "_"
|
||||
return &Stats2CorrCovAccumulator{
|
||||
count: 0,
|
||||
|
||||
sumx: 0.0,
|
||||
sumy: 0.0,
|
||||
sumx2: 0.0,
|
||||
sumxy: 0.0,
|
||||
sumy2: 0.0,
|
||||
doWhich: doWhich,
|
||||
doVerbose: doVerbose,
|
||||
|
||||
corrOutputFieldName: prefix + "corr",
|
||||
|
||||
covOutputFieldName: prefix + "cov",
|
||||
|
||||
covx00OutputFieldName: valueFieldName1 + "_" + valueFieldName1 + "_covx",
|
||||
covx01OutputFieldName: valueFieldName1 + "_" + valueFieldName2 + "_covx",
|
||||
covx10OutputFieldName: valueFieldName2 + "_" + valueFieldName1 + "_covx",
|
||||
covx11OutputFieldName: valueFieldName2 + "_" + valueFieldName2 + "_covx",
|
||||
|
||||
pca_mOutputFieldName: prefix + "pca_m",
|
||||
pca_bOutputFieldName: prefix + "pca_b",
|
||||
pca_nOutputFieldName: prefix + "pca_n",
|
||||
pca_qOutputFieldName: prefix + "pca_quality",
|
||||
pca_l1OutputFieldName: prefix + "pca_eival1",
|
||||
pca_l2OutputFieldName: prefix + "pca_eival2",
|
||||
pca_v11OutputFieldName: prefix + "pca_eivec11",
|
||||
pca_v12OutputFieldName: prefix + "pca_eivec12",
|
||||
pca_v21OutputFieldName: prefix + "pca_eivec21",
|
||||
pca_v22OutputFieldName: prefix + "pca_eivec22",
|
||||
pca_fitOutputFieldName: prefix + "pca_fit",
|
||||
|
||||
fitReady: false,
|
||||
m: -999.0,
|
||||
b: -999.0,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stats2CorrCovAccumulator) Ingest(
|
||||
x float64,
|
||||
y float64,
|
||||
) {
|
||||
this.count++
|
||||
this.sumx += x
|
||||
this.sumy += y
|
||||
this.sumx2 += x * x
|
||||
this.sumxy += x * y
|
||||
this.sumy2 += y * y
|
||||
}
|
||||
|
||||
func (this *Stats2CorrCovAccumulator) Populate(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
|
||||
if this.doWhich == DO_COVX {
|
||||
key00 := this.covx00OutputFieldName
|
||||
key01 := this.covx01OutputFieldName
|
||||
key10 := this.covx10OutputFieldName
|
||||
key11 := this.covx11OutputFieldName
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(key00, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(key01, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(key10, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(key11, types.MLRVAL_VOID)
|
||||
} else {
|
||||
Q := lib.GetCovMatrix(
|
||||
this.count,
|
||||
this.sumx,
|
||||
this.sumx2,
|
||||
this.sumy,
|
||||
this.sumy2,
|
||||
this.sumxy,
|
||||
)
|
||||
outrec.PutReference(key00, types.MlrvalPointerFromFloat64(Q[0][0]))
|
||||
outrec.PutReference(key01, types.MlrvalPointerFromFloat64(Q[0][1]))
|
||||
outrec.PutReference(key10, types.MlrvalPointerFromFloat64(Q[1][0]))
|
||||
outrec.PutReference(key11, types.MlrvalPointerFromFloat64(Q[1][1]))
|
||||
}
|
||||
|
||||
} else if this.doWhich == DO_LINREG_PCA {
|
||||
keym := this.pca_mOutputFieldName
|
||||
keyb := this.pca_bOutputFieldName
|
||||
keyn := this.pca_nOutputFieldName
|
||||
keyq := this.pca_qOutputFieldName
|
||||
|
||||
keyl1 := this.pca_l1OutputFieldName
|
||||
keyl2 := this.pca_l2OutputFieldName
|
||||
keyv11 := this.pca_v11OutputFieldName
|
||||
keyv12 := this.pca_v12OutputFieldName
|
||||
keyv21 := this.pca_v21OutputFieldName
|
||||
keyv22 := this.pca_v22OutputFieldName
|
||||
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(keym, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyb, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyn, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyq, types.MLRVAL_VOID)
|
||||
|
||||
if this.doVerbose {
|
||||
|
||||
outrec.PutCopy(keyl1, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyl2, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyv11, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyv12, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyv21, types.MLRVAL_VOID)
|
||||
outrec.PutCopy(keyv22, types.MLRVAL_VOID)
|
||||
}
|
||||
} else {
|
||||
Q := lib.GetCovMatrix(
|
||||
this.count,
|
||||
this.sumx,
|
||||
this.sumx2,
|
||||
this.sumy,
|
||||
this.sumy2,
|
||||
this.sumxy,
|
||||
)
|
||||
|
||||
l1, l2, v1, v2 := lib.GetRealSymmetricEigensystem(Q)
|
||||
|
||||
x_mean := this.sumx / float64(this.count)
|
||||
y_mean := this.sumy / float64(this.count)
|
||||
m, b, q := lib.GetLinearRegressionPCA(l1, l2, v1, v2, x_mean, y_mean)
|
||||
|
||||
outrec.PutReference(keym, types.MlrvalPointerFromFloat64(m))
|
||||
outrec.PutReference(keyb, types.MlrvalPointerFromFloat64(b))
|
||||
outrec.PutReference(keyn, types.MlrvalPointerFromInt(this.count))
|
||||
outrec.PutReference(keyq, types.MlrvalPointerFromFloat64(q))
|
||||
|
||||
if this.doVerbose {
|
||||
outrec.PutReference(keyl1, types.MlrvalPointerFromFloat64(l1))
|
||||
outrec.PutReference(keyl2, types.MlrvalPointerFromFloat64(l2))
|
||||
outrec.PutReference(keyv11, types.MlrvalPointerFromFloat64(v1[0]))
|
||||
outrec.PutReference(keyv12, types.MlrvalPointerFromFloat64(v1[1]))
|
||||
outrec.PutReference(keyv21, types.MlrvalPointerFromFloat64(v2[0]))
|
||||
outrec.PutReference(keyv22, types.MlrvalPointerFromFloat64(v2[1]))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
key := this.corrOutputFieldName
|
||||
if this.doWhich == DO_COV {
|
||||
key = this.covOutputFieldName
|
||||
}
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(key, types.MLRVAL_VOID)
|
||||
} else {
|
||||
output := lib.GetCov(this.count, this.sumx, this.sumy, this.sumxy)
|
||||
if this.doWhich == DO_CORR {
|
||||
sigmax := math.Sqrt(lib.GetVar(this.count, this.sumx, this.sumx2))
|
||||
sigmay := math.Sqrt(lib.GetVar(this.count, this.sumy, this.sumy2))
|
||||
output = output / sigmax / sigmay
|
||||
}
|
||||
outrec.PutReference(key, types.MlrvalPointerFromFloat64(output))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stats2CorrCovAccumulator) Fit(
|
||||
x float64,
|
||||
y float64,
|
||||
outrec *types.Mlrmap,
|
||||
) {
|
||||
if this.doWhich != DO_LINREG_PCA {
|
||||
return
|
||||
}
|
||||
|
||||
if !this.fitReady {
|
||||
// Idea for hold-and-fit in stats2.go is:
|
||||
// * We've ingested say 10,000 records
|
||||
// * After the end of those we compute m and b
|
||||
// * Then for all 10,000 records we compute y = m*x + b
|
||||
// The fitReady flag keeps us from recomputing the linear fit 10,000 times
|
||||
Q := lib.GetCovMatrix(this.count, this.sumx, this.sumx2, this.sumy, this.sumy2, this.sumxy)
|
||||
|
||||
l1, l2, v1, v2 := lib.GetRealSymmetricEigensystem(Q)
|
||||
|
||||
x_mean := this.sumx / float64(this.count)
|
||||
y_mean := this.sumy / float64(this.count)
|
||||
this.m, this.b, this.q = lib.GetLinearRegressionPCA(l1, l2, v1, v2, x_mean, y_mean)
|
||||
|
||||
this.fitReady = true
|
||||
}
|
||||
if this.count < 2 {
|
||||
outrec.PutCopy(this.pca_fitOutputFieldName, types.MLRVAL_VOID)
|
||||
} else {
|
||||
yfit := this.m*x + this.b
|
||||
outrec.PutCopy(this.pca_fitOutputFieldName, types.MlrvalPointerFromFloat64(yfit))
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
func NewStats2CorrAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
return NewStats2CorrCovAccumulator(
|
||||
valueFieldName1,
|
||||
valueFieldName2,
|
||||
accumulatorName,
|
||||
doVerbose,
|
||||
DO_CORR,
|
||||
)
|
||||
}
|
||||
|
||||
func NewStats2CovAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
return NewStats2CorrCovAccumulator(
|
||||
valueFieldName1,
|
||||
valueFieldName2,
|
||||
accumulatorName,
|
||||
doVerbose,
|
||||
DO_COV,
|
||||
)
|
||||
}
|
||||
|
||||
func NewStats2CovXAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
return NewStats2CorrCovAccumulator(
|
||||
valueFieldName1,
|
||||
valueFieldName2,
|
||||
accumulatorName,
|
||||
doVerbose,
|
||||
DO_COVX,
|
||||
)
|
||||
}
|
||||
|
||||
func NewStats2LinRegPCAAccumulator(
|
||||
valueFieldName1 string,
|
||||
valueFieldName2 string,
|
||||
accumulatorName string,
|
||||
doVerbose bool,
|
||||
) IStats2Accumulator {
|
||||
return NewStats2CorrCovAccumulator(
|
||||
valueFieldName1,
|
||||
valueFieldName2,
|
||||
accumulatorName,
|
||||
doVerbose,
|
||||
DO_LINREG_PCA,
|
||||
)
|
||||
}
|
||||
|
|
@ -498,6 +498,7 @@ func (this *Mlrmap) GetSelectedValuesJoined(selectedFieldNames []string) (string
|
|||
|
||||
// As with GetSelectedValuesJoined but also returning the array of mlrvals.
|
||||
// For sort.
|
||||
// TODO: put 'Copy' into the method name
|
||||
func (this *Mlrmap) GetSelectedValuesAndJoined(selectedFieldNames []string) (
|
||||
string,
|
||||
[]*Mlrval,
|
||||
|
|
|
|||
|
|
@ -5,90 +5,11 @@
|
|||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
|
||||
"miller/src/lib"
|
||||
)
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Some wrappers around things which aren't one-liners from math.*.
|
||||
|
||||
func mlrSgn(a float64) float64 {
|
||||
if a > 0 {
|
||||
return 1.0
|
||||
} else if a < 0 {
|
||||
return -1.0
|
||||
} else if a == 0 {
|
||||
return 0.0
|
||||
} else {
|
||||
return math.NaN()
|
||||
}
|
||||
}
|
||||
|
||||
// Normal cumulative distribution function, expressed in terms of erfc library
|
||||
// function (which is awkward, but exists).
|
||||
func mlrQnorm(x float64) float64 {
|
||||
return 0.5 * math.Erfc(-x/math.Sqrt2)
|
||||
}
|
||||
|
||||
// This is a tangent-following method not unlike Newton-Raphson:
|
||||
// * We can compute qnorm(y) = integral from -infinity to y of (1/sqrt(2pi)) exp(-t^2/2) dt.
|
||||
// * We can compute derivative of qnorm(y) = (1/sqrt(2pi)) exp(-y^2/2).
|
||||
// * We cannot explicitly compute invqnorm(y).
|
||||
// * If dx/dy = (1/sqrt(2pi)) exp(-y^2/2) then dy/dx = sqrt(2pi) exp(y^2/2).
|
||||
//
|
||||
// This means we *can* compute the derivative of invqnorm even though we
|
||||
// can't compute the function itself. So the essence of the method is to
|
||||
// follow the tangent line to form successive approximations: we have known function input x
|
||||
// and unknown function output y and initial guess y0. At each step we find the intersection
|
||||
// of the tangent line at y_n with the vertical line at x, to find y_{n+1}. Specificall:
|
||||
//
|
||||
// * Even though we can't compute y = q^-1(x) we can compute x = q(y).
|
||||
// * Start with initial guess for y (y0 = 0.0 or y0 = x both are OK).
|
||||
// * Find x = q(y). Since q (and therefore q^-1) are 1-1, we're done if qnorm(invqnorm(x)) is small.
|
||||
// * Else iterate: using point-slope form, (y_{n+1} - y_n) / (x_{n+1} - x_n) = m = sqrt(2pi) exp(y_n^2/2).
|
||||
// Here x_2 = x (the input) and x_1 = q(y_1).
|
||||
// * Solve for y_{n+1} and repeat.
|
||||
|
||||
const INVQNORM_TOL float64 = 1e-9
|
||||
const INVQNORM_MAXITER int = 30
|
||||
|
||||
func mlrInvqnorm(x float64) float64 {
|
||||
// Initial approximation is linear. Starting with y0 = 0.0 works just as well.
|
||||
y0 := x - 0.5
|
||||
if x <= 0.0 {
|
||||
return 0.0
|
||||
}
|
||||
if x >= 1.0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
y := y0
|
||||
niter := 0
|
||||
|
||||
for {
|
||||
|
||||
backx := mlrQnorm(y)
|
||||
err := math.Abs(x - backx)
|
||||
if err < INVQNORM_TOL {
|
||||
break
|
||||
}
|
||||
if niter > INVQNORM_MAXITER {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Miller: internal coding error: max iterations %d exceeded in invqnorm.\n",
|
||||
INVQNORM_MAXITER,
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
m := math.Sqrt2 * math.SqrtPi * math.Exp(y*y/2.0)
|
||||
delta_y := m * (x - backx)
|
||||
y += delta_y
|
||||
niter++
|
||||
}
|
||||
|
||||
return y
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
func math_unary_f_i(input1 *Mlrval, f mathLibUnaryFunc) *Mlrval {
|
||||
return MlrvalPointerFromFloat64(f(float64(input1.intval)))
|
||||
|
|
@ -126,13 +47,13 @@ func MlrvalErfc(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](inpu
|
|||
func MlrvalExp(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Exp) }
|
||||
func MlrvalExpm1(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Expm1) }
|
||||
func MlrvalFloor(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Floor) }
|
||||
func MlrvalInvqnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, mlrInvqnorm) }
|
||||
func MlrvalInvqnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Invqnorm) }
|
||||
func MlrvalLog(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log) }
|
||||
func MlrvalLog10(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log10) }
|
||||
func MlrvalLog1p(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Log1p) }
|
||||
func MlrvalQnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, mlrQnorm) }
|
||||
func MlrvalQnorm(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Qnorm) }
|
||||
func MlrvalRound(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Round) }
|
||||
func MlrvalSgn(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, mlrSgn) }
|
||||
func MlrvalSgn(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, lib.Sgn) }
|
||||
func MlrvalSin(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sin) }
|
||||
func MlrvalSinh(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sinh) }
|
||||
func MlrvalSqrt(input1 *Mlrval) *Mlrval { return mudispo[input1.mvtype](input1, math.Sqrt) }
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
================================================================
|
||||
TOP OF LIST:
|
||||
|
||||
* restore go fmt ./... note
|
||||
* mongo examples to doc :D
|
||||
! mrpl :h (and miller --help-xxx): substring ...
|
||||
! MlrvalPointerFromString -> NewMlrvalPointerFromString et al.
|
||||
|
||||
* blocker: regexes
|
||||
o finish stats1 -r
|
||||
|
|
@ -12,7 +12,7 @@ TOP OF LIST:
|
|||
|
||||
* blocker: remaining verbs:
|
||||
o format-values
|
||||
o merge-fields stats2
|
||||
o merge-fields
|
||||
|
||||
* blocker: remaining functions:
|
||||
o system
|
||||
|
|
@ -595,3 +595,4 @@ i https://en.wikipedia.org/wiki/Delimiter#Delimiter_collision
|
|||
* docs nest simplers now that we have getoptish
|
||||
* doc for DSL hostname/os/version functions
|
||||
? ENV as entire string-string map at RHS/LHS -- ?
|
||||
* mongo examples to doc :D
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue