Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-20 01:46:26 +00:00
Code Issues Projects Releases Wiki Activity
miller/doc/cookbook3.html
johnkerl e5c43cb4da fix 
missing data-sharing.html link
2019-09-17 22:39:56 -04:00

661 lines
20 KiB
HTML
Raw Blame History

<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<!-- PAGE GENERATED FROM template.html and content-for-cookbook3.html BY poki. -->
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
<head>
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
<meta name="description" content="Miller documentation"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
<meta name="keywords"
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
<title> Cookbook part 3 </title>
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
</head>
<!-- ================================================================ -->
<script type="text/javascript">
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type="text/javascript">
try {
var pageTracker = _gat._getTracker("UA-15651652-1");
pageTracker._trackPageview();
} catch(err) {}
</script>
<!-- ================================================================ -->
<script type="text/javascript">
function vis_expand_all_navbar_sections() {
//console.log('vis_expand_all_navbar_sections ENTER');
vis_expand_all_with_prefix('navbar_section_toggle_');
//console.log('vis_expand_all_navbar_sections EXIT');
}
function vis_collapse_all_navbar_sections() {
//console.log('vis_collapse_all_navbar_sections ENTER');
vis_collapse_all_with_prefix('navbar_section_toggle_');
//console.log('vis_collapse_all_navbar_sections EXIT');
}
function vis_expand_all_body_sections() {
//console.log('vis_expand_all_body_sections ENTER');
vis_expand_all_with_prefix('body_section_toggle_');
//console.log('vis_expand_all_body_sections EXIT');
}
function vis_collapse_all_body_sections() {
//console.log('vis_collapse_all_body_sections ENTER');
vis_collapse_all_with_prefix('body_section_toggle_');
//console.log('vis_collapse_all_body_sections EXIT');
}
function vis_expand_all_with_prefix(prefix) {
//console.log('vis_expand_all_with_prefix ENTER', prefix);
var divs = document.getElementsByTagName('div');
console.log("DIVS", divs);
for(var i = 0; i < divs.length; i++) {
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_expand_div(div);
}
}
//console.log('vis_expand_all_with_prefix EXIT', prefix);
}
function vis_collapse_all_with_prefix(prefix) {
var divs = document.getElementsByTagName('div');
for(var i = 0; i < divs.length; i++){
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_collapse_div(div);
}
}
}
function vis_toggle_by_name(divName) {
vis_toggle_div(document.getElementById(divName));
}
function vis_expand_by_name(divName) {
vis_expand_div(document.getElementById(divName));
}
function vis_collapse_by_name(divName) {
vis_collapse_div(document.getElementById(divName));
}
function vis_toggle_div(div) {
if (div != null) {
var state = div.style.display;
if (state == 'block') {
div.style.display = 'none';
} else {
div.style.display = 'block';
}
}
}
function vis_expand_div(div) {
if (div != null) {
div.style.display = 'block';
}
}
function vis_collapse_div(div) {
if (div != null) {
div.style.display = 'none';
}
}
</script>
<body bgcolor="#ffffff">
<!-- ================================================================ -->
<!-- navbar -->
<div class="pokinav">
<center><titleinbody>Miller</titleinbody></center>
<!-- NAVBAR GENERATED FROM template.html BY poki -->
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
href="javascript:;"
>
Overview
</button>
<button
style="font-weight:bold;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
href="javascript:;"
>
Using
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
href="javascript:;"
>
Reference
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
href="javascript:;"
>
Background
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
href="javascript:;"
>
Repository
</button>
<div id="navbar_section_toggle_overview" style="display: none;text-align: left">
<br/><a href="index.html">About</a>
<br/><a href="10-min.html">10-minute intro</a>
<br/><a href="file-formats.html">File formats</a>
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
<br/><a href="record-heterogeneity.html">Record-hetrogeneity</a>
<br/><a href="internationalization.html">Internationalization</a>
</div>
<div id="navbar_section_toggle_using" style="display: block;text-align: left">
<br/><a href="faq.html">FAQ</a>
<br/><a href="10-min.html">10-minute intro</a>
<br/><a href="data-sharing.html">Mixing with other languages</a>
<br/><a href="cookbook.html">Cookbook part 1</a>
<br/><a href="cookbook2.html">Cookbook part 2</a>
<br/><a href="cookbook3.html"><b>Cookbook part 3</b></a>
<br/><a href="data-examples.html">Data-diving examples</a>
</div>
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
<br/><a href="reference.html">Main reference</a>
<br/><a href="reference-verbs.html">Verbs reference</a>
<br/><a href="reference-dsl.html">DSL reference</a>
<br/><a href="manpage.html">Manpage</a>
<br/><a href="release-docs.html">Documents by release</a>
<br/><a href="build.html">Installation</a>
</div>
<div id="navbar_section_toggle_background" style="display: none;text-align: left">
<br/><a href="why.html">Why?</a>
<br/><a href="whyc.html">Why C?</a>
<br/><a href="etymology.html">Why call it Miller?</a>
<br/><a href="originality.html">How original is Miller?</a>
<br/><a href="performance.html">Performance</a>
</div>
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
<br/><a href="contact.html">Contact information</a>
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
</div>
</div>
<!-- page body -->
<p/>
<!-- BODY COPIED FROM content-for-cookbook3.html BY poki -->
<p/>
<center>
<boldmaroon>Stats with and without out-of-stream variables</boldmaroon>
</center>
<div class="pokitoc">
<center><titleinbody>Cookbook part 3</titleinbody></center>
&bull;&nbsp;<a href="#Overview">Overview</a><br/>
&bull;&nbsp;<a href="#Mean_without/with_oosvars">Mean without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Keyed_mean_without/with_oosvars">Keyed mean without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Variance_and_standard_deviation_without/with_oosvars">Variance and standard deviation without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Min/max_without/with_oosvars">Min/max without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Keyed_min/max_without/with_oosvars">Keyed min/max without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Delta_without/with_oosvars">Delta without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Keyed_delta_without/with_oosvars">Keyed delta without/with oosvars</a><br/>
&bull;&nbsp;<a href="#Exponentially_weighted_moving_averages_without/with_oosvars">Exponentially weighted moving averages without/with oosvars</a><br/>
</div>
<p/>
<p/>
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_expand_all_body_sections();" href="javascript:;">Expand all sections</button>
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_collapse_all_body_sections();" href="javascript:;">Collapse all sections</button>
<!-- ================================================================ -->
<a id="Overview"/><h1>Overview</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_overview');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_overview" style="display: block">
<p/> One of Miller&rsquo;s strengths is its compact notation: for example, given input of the form
<p/>
<div class="pokipanel">
<pre>
$ head -n 5 ../data/medium
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
</pre>
</div>
<p/>
you can simply do
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab stats1 -a sum -f x ../data/medium
x_sum 4986.019682
</pre>
</div>
<p/>
or
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint stats1 -a sum -f x -g b ../data/medium
b x_sum
pan 965.763670
wye 1023.548470
zee 979.742016
eks 1016.772857
hat 1000.192668
</pre>
</div>
<p/>
rather than the more tedious
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab put -q '
@x_sum += $x;
end {
emit @x_sum
}
' data/medium
x_sum 4986.019682
</pre>
</div>
<p/>
or
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put -q '
@x_sum[$b] += $x;
end {
emit @x_sum, "b"
}
' data/medium
b x_sum
pan 965.763670
wye 1023.548470
zee 979.742016
eks 1016.772857
hat 1000.192668
</pre>
</div>
<p/>
<p/> The former (<code>mlr stats1</code> et al.) has the advantages of being easier
to type, being less error-prone to type, and running faster.
<p/> Nonetheless, out-of-stream variables (which I whimsically call
<i>oosvars</i>), begin/end blocks, and emit statements give you the ability to
implement logic &mdash; if you wish to do so &mdash; which isn&rsquo;t present
in other Miller verbs. (If you find yourself often using the same
out-of-stream-variable logic over and over, please file a request at <a
href="https://github.com/johnkerl/miller/issues">https://github.com/johnkerl/miller/issues</a>
to get it implemented directly in C as a Miller verb of its own.)
<p/> The following examples compute some things using oosvars which are already
computable using Miller verbs, by way of providing food for thought.
</div>
<!-- ================================================================ -->
<a id="Mean_without/with_oosvars"/><h1>Mean without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_mean');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_mean" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint stats1 -a mean -f x data/medium
x_mean
0.498602
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put -q '
@x_sum += $x;
@x_count += 1;
end {
@x_mean = @x_sum / @x_count;
emit @x_mean
}
' data/medium
x_mean
0.498602
</pre>
</div>
<p/>
</div>
<!-- ================================================================ -->
<a id="Keyed_mean_without/with_oosvars"/><h1>Keyed mean without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_keyed_mean');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_keyed_mean" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint stats1 -a mean -f x -g a,b data/medium
a b x_mean
pan pan 0.513314
eks pan 0.485076
wye wye 0.491501
eks wye 0.483895
wye pan 0.499612
zee pan 0.519830
eks zee 0.495463
zee wye 0.514267
hat wye 0.493813
pan wye 0.502362
zee eks 0.488393
hat zee 0.509999
hat eks 0.485879
wye hat 0.497730
pan eks 0.503672
eks eks 0.522799
hat hat 0.479931
hat pan 0.464336
zee zee 0.512756
pan hat 0.492141
pan zee 0.496604
zee hat 0.467726
wye zee 0.505907
eks hat 0.500679
wye eks 0.530604
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put -q '
@x_sum[$a][$b] += $x;
@x_count[$a][$b] += 1;
end{
for ((a, b), v in @x_sum) {
@x_mean[a][b] = @x_sum[a][b] / @x_count[a][b];
}
emit @x_mean, "a", "b"
}
' data/medium
a b x_mean
pan pan 0.513314
pan wye 0.502362
pan eks 0.503672
pan hat 0.492141
pan zee 0.496604
eks pan 0.485076
eks wye 0.483895
eks zee 0.495463
eks eks 0.522799
eks hat 0.500679
wye wye 0.491501
wye pan 0.499612
wye hat 0.497730
wye zee 0.505907
wye eks 0.530604
zee pan 0.519830
zee wye 0.514267
zee eks 0.488393
zee zee 0.512756
zee hat 0.467726
hat wye 0.493813
hat zee 0.509999
hat eks 0.485879
hat hat 0.479931
hat pan 0.464336
</pre>
</div>
<p/>
</div>
<!-- ================================================================ -->
<a id="Variance_and_standard_deviation_without/with_oosvars"/><h1>Variance and standard deviation without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_var_stddev');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_var_stddev" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab stats1 -a count,sum,mean,var,stddev -f x data/medium
x_count 10000
x_sum 4986.019682
x_mean 0.498602
x_var 0.084270
x_stddev 0.290293
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ cat variance.mlr
@n += 1;
@sumx += $x;
@sumx2 += $x**2;
end {
@mean = @sumx / @n;
@var = (@sumx2 - @mean * (2 * @sumx - @n * @mean)) / (@n - 1);
@stddev = sqrt(@var);
emitf @n, @sumx, @sumx2, @mean, @var, @stddev
}
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab put -q -f variance.mlr data/medium
n 10000
sumx 4986.019682
sumx2 3328.652400
mean 0.498602
var 0.084270
stddev 0.290293
</pre>
</div>
<p/>
You can also do this keyed, of course, imitating the keyed-mean example above.
</div>
<!-- ================================================================ -->
<a id="Min/max_without/with_oosvars"/><h1>Min/max without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_min_max');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_min_max" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab stats1 -a min,max -f x data/medium
x_min 0.000045
x_max 0.999953
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab put -q '@x_min = min(@x_min, $x); @x_max = max(@x_max, $x); end{emitf @x_min, @x_max}' data/medium
x_min 0.000045
x_max 0.999953
</pre>
</div>
<p/>
</div>
<!-- ================================================================ -->
<a id="Keyed_min/max_without/with_oosvars"/><h1>Keyed min/max without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_keyed_min_max');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_keyed_min_max" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint stats1 -a min,max -f x -g a data/medium
a x_min x_max
pan 0.000204 0.999403
eks 0.000692 0.998811
wye 0.000187 0.999823
zee 0.000549 0.999490
hat 0.000045 0.999953
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint --from data/medium put -q '
@min[$a] = min(@min[$a], $x);
@max[$a] = max(@max[$a], $x);
end{
emit (@min, @max), "a";
}
'
a min max
pan 0.000204 0.999403
eks 0.000692 0.998811
wye 0.000187 0.999823
zee 0.000549 0.999490
hat 0.000045 0.999953
</pre>
</div>
<p/>
</div>
<!-- ================================================================ -->
<a id="Delta_without/with_oosvars"/><h1>Delta without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_delta');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_delta" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint step -a delta -f x data/small
a b i x y x_delta
pan pan 1 0.3467901443380824 0.7268028627434533 0
eks pan 2 0.7586799647899636 0.5221511083334797 0.411890
wye wye 3 0.20460330576630303 0.33831852551664776 -0.554077
eks wye 4 0.38139939387114097 0.13418874328430463 0.176796
wye pan 5 0.5732889198020006 0.8636244699032729 0.191890
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put '$x_delta = is_present(@last) ? $x - @last : 0; @last = $x' data/small
a b i x y x_delta
pan pan 1 0.3467901443380824 0.7268028627434533 0
eks pan 2 0.7586799647899636 0.5221511083334797 0.411890
wye wye 3 0.20460330576630303 0.33831852551664776 -0.554077
eks wye 4 0.38139939387114097 0.13418874328430463 0.176796
wye pan 5 0.5732889198020006 0.8636244699032729 0.191890
</pre>
</div>
<p/>
</div>
<!-- ================================================================ -->
<a id="Keyed_delta_without/with_oosvars"/><h1>Keyed delta without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_keyed_delta');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_keyed_delta" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint step -a delta -f x -g a data/small
a b i x y x_delta
pan pan 1 0.3467901443380824 0.7268028627434533 0
eks pan 2 0.7586799647899636 0.5221511083334797 0
wye wye 3 0.20460330576630303 0.33831852551664776 0
eks wye 4 0.38139939387114097 0.13418874328430463 -0.377281
wye pan 5 0.5732889198020006 0.8636244699032729 0.368686
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put '$x_delta = is_present(@last[$a]) ? $x - @last[$a] : 0; @last[$a]=$x' data/small
a b i x y x_delta
pan pan 1 0.3467901443380824 0.7268028627434533 0
eks pan 2 0.7586799647899636 0.5221511083334797 0
wye wye 3 0.20460330576630303 0.33831852551664776 0
eks wye 4 0.38139939387114097 0.13418874328430463 -0.377281
wye pan 5 0.5732889198020006 0.8636244699032729 0.368686
</pre>
</div>
<p/>
</div>
<!-- ================================================================ -->
<a id="Exponentially_weighted_moving_averages_without/with_oosvars"/><h1>Exponentially weighted moving averages without/with oosvars</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_ewma');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_ewma" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint step -a ewma -d 0.1 -f x data/small
a b i x y x_ewma_0.1
pan pan 1 0.3467901443380824 0.7268028627434533 0.346790
eks pan 2 0.7586799647899636 0.5221511083334797 0.387979
wye wye 3 0.20460330576630303 0.33831852551664776 0.369642
eks wye 4 0.38139939387114097 0.13418874328430463 0.370817
wye pan 5 0.5732889198020006 0.8636244699032729 0.391064
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint put '
begin{ @a=0.1 };
$e = NR==1 ? $x : @a * $x + (1 - @a) * @e;
@e=$e
' data/small
a b i x y e
pan pan 1 0.3467901443380824 0.7268028627434533 0.346790
eks pan 2 0.7586799647899636 0.5221511083334797 0.387979
wye wye 3 0.20460330576630303 0.33831852551664776 0.369642
eks wye 4 0.38139939387114097 0.13418874328430463 0.370817
wye pan 5 0.5732889198020006 0.8636244699032729 0.391064
</pre>
</div>
<p/>
</div>
</body>
</html>
Reference in a new issue View git blame Copy permalink