mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 00:45:47 +00:00
1617 lines
51 KiB
HTML
1617 lines
51 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<html lang="en">
|
|
|
|
<!-- PAGE GENERATED FROM template.html and content-for-cookbook.html BY poki. -->
|
|
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
|
|
<head>
|
|
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
|
|
<meta name="description" content="Miller documentation"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
|
|
<meta name="keywords"
|
|
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
|
|
|
|
<title> Cookbook </title>
|
|
<link rel="stylesheet" type="text/css" href="css/miller.css"/>
|
|
<link rel="stylesheet" type="text/css" href="css/poki-callbacks.css"/>
|
|
</head>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
|
|
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
|
|
</script>
|
|
<script type="text/javascript">
|
|
try {
|
|
var pageTracker = _gat._getTracker("UA-15651652-1");
|
|
pageTracker._trackPageview();
|
|
} catch(err) {}
|
|
</script>
|
|
|
|
<script type="text/javascript">
|
|
function toggle(divName) {
|
|
var eleDiv = document.getElementById(divName);
|
|
if (eleDiv != null) {
|
|
if (eleDiv.style.display == "block") {
|
|
eleDiv.style.display = "none";
|
|
} else {
|
|
eleDiv.style.display = "block";
|
|
}
|
|
}
|
|
}
|
|
</script>
|
|
|
|
<!--
|
|
The background image is from a screenshot of a Google search for "data analysis
|
|
tools", lightened and sepia-toned. Over this was placed a Mac Terminal app with
|
|
very light-grey font and translucent background, in which a few statistical
|
|
Miller commands were run with pretty-print-tabular output format.
|
|
-->
|
|
<body background="pix/sepia-overlay.jpg">
|
|
|
|
<!-- ================================================================ -->
|
|
<table width="100%">
|
|
<tr>
|
|
|
|
<!-- navbar -->
|
|
<td width="15%">
|
|
<!--
|
|
<img src="pix/mlr.jpg" />
|
|
<img style="border-width:1px; color:black;" src="pix/mlr.jpg" />
|
|
-->
|
|
|
|
<div class="pokinav">
|
|
<center><titleinbody>Miller</titleinbody></center>
|
|
|
|
<!-- PAGE LIST GENERATED FROM template.html BY poki -->
|
|
<br/><b>Overview:</b>
|
|
<br/>• <a href="index.html">About Miller</a>
|
|
<br/>• <a href="10-min.html">Miller in 10 minutes</a>
|
|
<br/>• <a href="file-formats.html">File formats</a>
|
|
<br/>• <a href="feature-comparison.html">Miller features in the context of the Unix toolkit</a>
|
|
<br/>• <a href="record-heterogeneity.html">Record-heterogeneity</a>
|
|
<br/>• <a href="internationalization.html">Internationalization</a>
|
|
<br/><b>Using Miller:</b>
|
|
<br/>• <a href="faq.html">FAQ</a>
|
|
<br/>• <a href="data-examples.html">Data-diving examples</a>
|
|
<br/>• <a href="cookbook.html"><b>Cookbook</b></a>
|
|
<br/>• <a href="manpage.html">Manpage</a>
|
|
<br/>• <a href="reference.html">Reference</a>
|
|
<br/>• <a href="reference-verbs.html">Reference: Verbs</a>
|
|
<br/>• <a href="reference-dsl.html">Reference: DSL</a>
|
|
<br/>• <a href="release-docs.html">Documents by release</a>
|
|
<br/>• <a href="build.html">Installation, portability, dependencies, and testing</a>
|
|
<br/><b>Background:</b>
|
|
<br/>• <a href="whyc.html">Why C?</a>
|
|
<br/>• <a href="etymology.html">Why call it Miller?</a>
|
|
<br/>• <a href="originality.html">How original is Miller?</a>
|
|
<br/>• <a href="performance.html">Performance</a>
|
|
<br/><b>Repository:</b>
|
|
<br/>• <a href="to-do.html">Things to do</a>
|
|
<br/>• <a href="contact.html">Contact information</a>
|
|
<br/>• <a href="https://github.com/johnkerl/miller">GitHub repo</a>
|
|
<br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/>
|
|
<br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/>
|
|
<br/> <br/> <br/> <br/> <br/> <br/>
|
|
</div>
|
|
</td>
|
|
|
|
<!-- page body -->
|
|
<td>
|
|
<!--
|
|
This is a visually gorgeous feature (here & in the CSS): it allows for
|
|
independent scroll of the nav and body panels. In particular the nav
|
|
stays on-screen as you scroll the body.
|
|
|
|
However, two problems:
|
|
|
|
(1) In Firefox & Chrome both I get janky end-of-body scrolls: there is
|
|
more content but I can't scroll down to it unless I repeatedly retry the
|
|
scrolldown. Which is weird.
|
|
|
|
(2) Worse, only the first page renders in PDF (again, Firefox & Chrome).
|
|
|
|
For now I'm disabling this separate-scroll feature. A frontender, I am
|
|
not ... maybe someday I'll find a config which gets *all* the features
|
|
I want; for now, it's a tradeoff.
|
|
-->
|
|
|
|
<!-- Implementation details: one bit is right here:
|
|
|
|
div style="overflow-y:scroll;height:1500px"
|
|
|
|
and the other bit is in css/poki-callbacks.css:
|
|
|
|
.pokinav {
|
|
display: inline-block;
|
|
background: #e8d9bc;
|
|
border: 1;
|
|
box-shadow: 0px 0px 3px 3px #C9C9C9;
|
|
margin: 10px;
|
|
padding-top: 10px;
|
|
padding-bottom: 10px;
|
|
padding-left: 10px;
|
|
padding-right: 10px;
|
|
overflow-y: scroll; < - - - - - - here
|
|
height: 1500px;
|
|
}
|
|
|
|
-->
|
|
<div>
|
|
<center> <titleinbody> Cookbook </titleinbody> </center>
|
|
<p/>
|
|
|
|
<!-- BODY COPIED FROM content-for-cookbook.html BY poki -->
|
|
<div class="pokitoc">
|
|
<center><b>Contents:</b></center>
|
|
• <a href="#Parsing_log-file_output">Parsing log-file output</a><br/>
|
|
• <a href="#Rectangularizing_data">Rectangularizing data</a><br/>
|
|
• <a href="#Bulk_rename_of_field_names">Bulk rename of field names</a><br/>
|
|
• <a href="#Headerless_CSV_on_input_or_output">Headerless CSV on input or output</a><br/>
|
|
• <a href="#Regularizing_ragged_CSV">Regularizing ragged CSV</a><br/>
|
|
• <a href="#Program_timing">Program timing</a><br/>
|
|
• <a href="#Memoization_with_out-of-stream_variables">Memoization with out-of-stream variables</a><br/>
|
|
• <a href="#Finding_missing_dates">Finding missing dates</a><br/>
|
|
• <a href="#Generating_random_variables">Generating random variables</a><br/>
|
|
• <a href="#Two-pass_algorithms">Two-pass algorithms</a><br/>
|
|
• <a href="#Two-pass_algorithms:_computation_of_percentages">Two-pass algorithms: computation of percentages</a><br/>
|
|
• <a href="#Two-pass_algorithms:_line-number_ratios">Two-pass algorithms: line-number ratios</a><br/>
|
|
• <a href="#Two-pass_algorithms:_records_having_max_value">Two-pass algorithms: records having max value</a><br/>
|
|
• <a href="#Filtering_paragraphs_of_text">Filtering paragraphs of text</a><br/>
|
|
• <a href="#Doing_arithmetic_on_fields_with_currency_symbols">Doing arithmetic on fields with currency symbols</a><br/>
|
|
• <a href="#Using_out-of-stream_variables">Using out-of-stream variables</a><br/>
|
|
• <a href="#Mean_without/with_oosvars">Mean without/with oosvars</a><br/>
|
|
• <a href="#Keyed_mean_without/with_oosvars">Keyed mean without/with oosvars</a><br/>
|
|
• <a href="#Variance_and_standard_deviation_without/with_oosvars">Variance and standard deviation without/with oosvars</a><br/>
|
|
• <a href="#Min/max_without/with_oosvars">Min/max without/with oosvars</a><br/>
|
|
• <a href="#Keyed_min/max_without/with_oosvars">Keyed min/max without/with oosvars</a><br/>
|
|
• <a href="#Delta_without/with_oosvars">Delta without/with oosvars</a><br/>
|
|
• <a href="#Keyed_delta_without/with_oosvars">Keyed delta without/with oosvars</a><br/>
|
|
• <a href="#Exponentially_weighted_moving_averages_without/with_oosvars">Exponentially weighted moving averages without/with oosvars</a><br/>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/><b>Disclaimer:</b> This page is about how to do some corner-case things in
|
|
ways you mightn’t have thought of. For an intro, please also see
|
|
<a href="10-min.html">Miller in 10 minutes</a>.
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Parsing_log-file_output"/><h1>Parsing log-file output</h1>
|
|
|
|
<p/>This, of course, depends highly on what’s in your log files. But, as
|
|
an example, suppose you have log-file lines such as
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
2015-10-08 08:29:09,445 INFO com.company.path.to.ClassName @ [sometext] various/sorts/of data {& punctuation} hits=1 status=0 time=2.378
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
I prefer to pre-filter with <tt>grep</tt> and/or <tt>sed</tt> to extract the structured text, then hand that to Miller. Example:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
grep 'various sorts' *.log | sed 's/.*} //' | mlr --fs space --repifs --oxtab stats1 -a min,p10,p50,p90,max -f time -g status
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Rectangularizing_data"/><h1>Rectangularizing data</h1>
|
|
|
|
<p/>Suppose you have a method (in whatever language) which is printing things of the form
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
outer=1
|
|
outer=2
|
|
outer=3
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
and then calls another method which prints things of the form
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
middle=10
|
|
middle=11
|
|
middle=12
|
|
middle=20
|
|
middle=21
|
|
middle=30
|
|
middle=31
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
and then, perhaps, that second method calls a third method which prints things of the form
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
inner1=100,inner2=101
|
|
inner1=120,inner2=121
|
|
inner1=200,inner2=201
|
|
inner1=210,inner2=211
|
|
inner1=300,inner2=301
|
|
inner1=312
|
|
inner1=313,inner2=314
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
with the result that your program’s output is
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
outer=1
|
|
middle=10
|
|
inner1=100,inner2=101
|
|
middle=11
|
|
middle=12
|
|
inner1=120,inner2=121
|
|
outer=2
|
|
middle=20
|
|
inner1=200,inner2=201
|
|
middle=21
|
|
inner1=210,inner2=211
|
|
outer=3
|
|
middle=30
|
|
inner1=300,inner2=301
|
|
middle=31
|
|
inner1=312
|
|
inner1=313,inner2=314
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
The idea here is that middles starting with a 1 belong to the outer value of 1,
|
|
and so on. (For example, the outer values might be account IDs, the middle
|
|
values might be invoice IDs, and the inner values might be invoice line-items.)
|
|
If you want all the middle and inner lines to have the context of which outers
|
|
they belong to, you can modify your software to pass all those through your
|
|
methods. Alternatively, you can use the following to rectangularize the data.
|
|
The idea is to use an out-of-stream variable to accumulate fields across
|
|
records. Clear that variable when you see an outer ID; accumulate fields; emit
|
|
output when you see the inner IDs.
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from data/rect.txt put -q '
|
|
is_present($outer) {
|
|
unset @r
|
|
}
|
|
for (k, v in $*) {
|
|
@r[k] = v
|
|
}
|
|
is_present($inner1) {
|
|
emit @r
|
|
}'
|
|
outer=1,middle=10,inner1=100,inner2=101
|
|
outer=1,middle=12,inner1=120,inner2=121
|
|
outer=2,middle=20,inner1=200,inner2=201
|
|
outer=2,middle=21,inner1=210,inner2=211
|
|
outer=3,middle=30,inner1=300,inner2=301
|
|
outer=3,middle=31,inner1=312,inner2=301
|
|
outer=3,middle=31,inner1=313,inner2=314
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Bulk_rename_of_field_names"/><h1>Bulk rename of field names</h1>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/spaces.csv
|
|
a b c,def,g h i
|
|
123,4567,890
|
|
2468,1357,3579
|
|
9987,3312,4543
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --csv rename -r -g ' ,_' data/spaces.csv
|
|
a_b_c,def,g_h_i
|
|
123,4567,890
|
|
2468,1357,3579
|
|
9987,3312,4543
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --csv --opprint rename -r -g ' ,_' data/spaces.csv
|
|
a_b_c def g_h_i
|
|
123 4567 890
|
|
2468 1357 3579
|
|
9987 3312 4543
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>You can also do this with a for-loop but it puts the modified fields after the unmodified fields:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/bulk-rename-for-loop.mlr
|
|
for (oldk,v in $*) {
|
|
@newk = gsub(oldk, " ", "_");
|
|
if (@newk != oldk) {
|
|
unset $[oldk];
|
|
$[@newk] = v
|
|
}
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --icsv --opprint put -f data/bulk-rename-for-loop.mlr data/spaces.csv
|
|
def a_b_c g_h_i
|
|
4567 123 890
|
|
1357 2468 3579
|
|
3312 9987 4543
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Headerless_CSV_on_input_or_output"/><h1>Headerless CSV on input or output</h1>
|
|
|
|
<p/>Sometimes we get CSV files which lack a header. For example:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/headerless.csv
|
|
John,23,present
|
|
Fred,34,present
|
|
Alice,56,missing
|
|
Carol,45,present
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> You can use Miller to add a header: the <tt>--implicit-csv-header</tt> applies positionally indexed labels:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --csv --implicit-csv-header cat data/headerless.csv
|
|
1,2,3
|
|
John,23,present
|
|
Fred,34,present
|
|
Alice,56,missing
|
|
Carol,45,present
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --icsv --implicit-csv-header --opprint cat data/headerless.csv
|
|
1 2 3
|
|
John 23 present
|
|
Fred 34 present
|
|
Alice 56 missing
|
|
Carol 45 present
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> Following that, you can rename the positionally indexed labels to names with meaning for your context.
|
|
For example:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --csv --implicit-csv-header label name,age,status data/headerless.csv
|
|
name,age,status
|
|
John,23,present
|
|
Fred,34,present
|
|
Alice,56,missing
|
|
Carol,45,present
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --icsv --implicit-csv-header --opprint label name,age,status data/headerless.csv
|
|
name age status
|
|
John 23 present
|
|
Fred 34 present
|
|
Alice 56 missing
|
|
Carol 45 present
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> Likewise, if you need to produce CSV which is lacking its header, you can pipe Miller’s output
|
|
to the system command <tt>sed 1d</tt>, or you can use Miller’s <tt>--headerless-csv-output</tt> option:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ head -5 data/colored-shapes.dkvp | mlr --ocsv cat
|
|
color,shape,flag,i,u,v,w,x
|
|
yellow,triangle,1,11,0.6321695890307647,0.9887207810889004,0.4364983936735774,5.7981881667050565
|
|
red,square,1,15,0.21966833570651523,0.001257332190235938,0.7927778364718627,2.944117399716207
|
|
red,circle,1,16,0.20901671281497636,0.29005231936593445,0.13810280912907674,5.065034003400998
|
|
red,square,0,48,0.9562743938458542,0.7467203085342884,0.7755423050923582,7.117831369597269
|
|
purple,triangle,0,51,0.4355354501763202,0.8591292672156728,0.8122903963006748,5.753094629505863
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ head -5 data/colored-shapes.dkvp | mlr --ocsv --headerless-csv-output cat
|
|
yellow,triangle,1,11,0.6321695890307647,0.9887207810889004,0.4364983936735774,5.7981881667050565
|
|
red,square,1,15,0.21966833570651523,0.001257332190235938,0.7927778364718627,2.944117399716207
|
|
red,circle,1,16,0.20901671281497636,0.29005231936593445,0.13810280912907674,5.065034003400998
|
|
red,square,0,48,0.9562743938458542,0.7467203085342884,0.7755423050923582,7.117831369597269
|
|
purple,triangle,0,51,0.4355354501763202,0.8591292672156728,0.8122903963006748,5.753094629505863
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Regularizing_ragged_CSV"/><h1>Regularizing ragged CSV</h1>
|
|
|
|
<p/>Miller handles compliant CSV: in particular, it’s an error if the
|
|
number of data fields in a given data line don’t match the number of
|
|
header lines. But in the event that you have a CSV file in which some lines
|
|
have less than the full number of fields, you can use Miller to pad them out.
|
|
The trick is to use NIDX format, for which each line stands on its own without
|
|
respect to a header line.
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/ragged.csv
|
|
a,b,c
|
|
1,2,3
|
|
4,5
|
|
6
|
|
7,8,9
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from data/ragged.csv --fs comma --nidx put '
|
|
@maxnf = max(@maxnf, NF);
|
|
@nf = NF;
|
|
while(@nf < @maxnf) {
|
|
@nf += 1;
|
|
$[@nf] = ""
|
|
}
|
|
'
|
|
a,b,c
|
|
1,2,3
|
|
4,5,
|
|
6,,
|
|
7,8,9
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
or, more simply,
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from data/ragged.csv --fs comma --nidx put '
|
|
@maxnf = max(@maxnf, NF);
|
|
while(NF < @maxnf) {
|
|
$[NF+1] = "";
|
|
}
|
|
'
|
|
a,b,c
|
|
1,2,3
|
|
4,5,
|
|
6,,
|
|
7,8,9
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Program_timing"/><h1>Program timing</h1>
|
|
|
|
This admittedly artificial example demonstrates using Miller time and stats
|
|
functions to introspectly acquire some information about Miller’s own
|
|
runtime. The <tt>delta</tt> function computes the difference between successive
|
|
timestamps.
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ ruby -e '10000.times{|i|puts "i=#{i+1}"}' > lines.txt
|
|
|
|
$ head -n 5 lines.txt
|
|
i=1
|
|
i=2
|
|
i=3
|
|
i=4
|
|
i=5
|
|
|
|
mlr --ofmt '%.9le' --opprint put '$t=systime()' then step -a delta -f t lines.txt | head -n 7
|
|
i t t_delta
|
|
1 1430603027.018016 1.430603027e+09
|
|
2 1430603027.018043 2.694129944e-05
|
|
3 1430603027.018048 5.006790161e-06
|
|
4 1430603027.018052 4.053115845e-06
|
|
5 1430603027.018055 2.861022949e-06
|
|
6 1430603027.018058 3.099441528e-06
|
|
|
|
mlr --ofmt '%.9le' --oxtab \
|
|
put '$t=systime()' then \
|
|
step -a delta -f t then \
|
|
filter '$i>1' then \
|
|
stats1 -a min,mean,max -f t_delta \
|
|
lines.txt
|
|
t_delta_min 2.861022949e-06
|
|
t_delta_mean 4.077508505e-06
|
|
t_delta_max 5.388259888e-05
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Memoization_with_out-of-stream_variables"/><h1>Memoization with out-of-stream variables</h1>
|
|
|
|
<p/> The recursive function for the Fibonacci sequence is famous for its computational complexity.
|
|
Namely, using
|
|
<i>f</i>(0)=1,
|
|
<i>f</i>(1)=1,
|
|
<i>f</i>(<i>n</i>)=<i>f</i>(<i>n</i>-1)+<i>f</i>(<i>n</i>-2) for <i>n</i>≥2,
|
|
the evaluation tree branches left as well as right at each non-trivial level, resulting in millions
|
|
or more paths to the root 0/1 nodes:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
|
|
func f(n) {
|
|
@fcount += 1; # count number of calls to the function
|
|
if (n < 2) {
|
|
return 1
|
|
} else {
|
|
return f(n-1) + f(n-2) # recurse
|
|
}
|
|
}
|
|
|
|
@fcount = 0;
|
|
$o = f($i);
|
|
$fcount = @fcount;
|
|
|
|
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
|
|
i o fcount seconds_delta
|
|
1 1 1 0
|
|
2 2 3 0.000031948
|
|
3 3 5 0.000013113
|
|
4 5 9 0.000015974
|
|
5 8 15 0.000021935
|
|
6 13 25 0.000030994
|
|
7 21 41 0.000045061
|
|
8 34 67 0.000071049
|
|
9 55 109 0.000113010
|
|
10 89 177 0.000178814
|
|
11 144 287 0.000289202
|
|
12 233 465 0.000463009
|
|
13 377 753 0.000734806
|
|
14 610 1219 0.001182079
|
|
15 987 1973 0.001869917
|
|
16 1597 3193 0.003251076
|
|
17 2584 5167 0.005389929
|
|
18 4181 8361 0.008499146
|
|
19 6765 13529 0.014052868
|
|
20 10946 21891 0.024805069
|
|
21 17711 35421 0.038400888
|
|
22 28657 57313 0.061202049
|
|
23 46368 92735 0.093353987
|
|
24 75025 150049 0.147068024
|
|
25 121393 242785 0.288206100
|
|
26 196418 392835 0.395586014
|
|
27 317811 635621 0.595448971
|
|
28 514229 1028457 0.958051920
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> Note that the time it takes to evaluate the function is blowing up exponentially as the input argument
|
|
increases. Using <tt>@</tt>-variables, which persist across records, we can cache and reuse the results
|
|
of previous computations:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --ofmt '%.9lf' --opprint seqgen --start 1 --stop 28 then put '
|
|
func f(n) {
|
|
@fcount += 1; # count number of calls to the function
|
|
if (is_present(@fcache[n])) { # cache hit
|
|
return @fcache[n]
|
|
} else { # cache miss
|
|
num rv = 1;
|
|
if (n >= 2) {
|
|
rv = f(n-1) + f(n-2) # recurse
|
|
}
|
|
@fcache[n] = rv;
|
|
return rv
|
|
}
|
|
}
|
|
@fcount = 0;
|
|
$o = f($i);
|
|
$fcount = @fcount;
|
|
' then put '$seconds=systime()' then step -a delta -f seconds then cut -x -f seconds
|
|
i o fcount seconds_delta
|
|
1 1 1 0
|
|
2 2 3 0.000037193
|
|
3 3 3 0.000012875
|
|
4 5 3 0.000011921
|
|
5 8 3 0.000011206
|
|
6 13 3 0.000010967
|
|
7 21 3 0.000010967
|
|
8 34 3 0.000010967
|
|
9 55 3 0.000010014
|
|
10 89 3 0.000013113
|
|
11 144 3 0.000010967
|
|
12 233 3 0.000015020
|
|
13 377 3 0.000010967
|
|
14 610 3 0.000010014
|
|
15 987 3 0.000010967
|
|
16 1597 3 0.000010014
|
|
17 2584 3 0.000010967
|
|
18 4181 3 0.000010014
|
|
19 6765 3 0.000010014
|
|
20 10946 3 0.000010967
|
|
21 17711 3 0.000010014
|
|
22 28657 3 0.000010014
|
|
23 46368 3 0.000012875
|
|
24 75025 3 0.000012159
|
|
25 121393 3 0.000010014
|
|
26 196418 3 0.000010967
|
|
27 317811 3 0.000010014
|
|
28 514229 3 0.000010967
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Finding_missing_dates"/><h1>Finding missing dates</h1>
|
|
|
|
<p/>Suppose you have some date-stamped data which may (or may not) be missing entries for one or more dates:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ head -n 10 data/miss-date.csv
|
|
date,qoh
|
|
2012-03-05,10055
|
|
2012-03-06,10486
|
|
2012-03-07,10430
|
|
2012-03-08,10674
|
|
2012-03-09,10880
|
|
2012-03-10,10718
|
|
2012-03-11,10795
|
|
2012-03-12,11043
|
|
2012-03-13,11177
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ wc -l data/miss-date.csv
|
|
1372 data/miss-date.csv
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>To find these, you can convert the dates to seconds since the epoch using <tt>strptime</tt>, then
|
|
compute adjacent differences (the <tt>cat -n</tt> simply inserts record-counters):
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from data/miss-date.csv --icsv \
|
|
cat -n \
|
|
then put '$datestamp = strptime($date, "%Y-%m-%d")' \
|
|
then step -a delta -f datestamp \
|
|
| head
|
|
n=1,date=2012-03-05,qoh=10055,datestamp=1330905600,datestamp_delta=0
|
|
n=2,date=2012-03-06,qoh=10486,datestamp=1330992000,datestamp_delta=86400
|
|
n=3,date=2012-03-07,qoh=10430,datestamp=1331078400,datestamp_delta=86400
|
|
n=4,date=2012-03-08,qoh=10674,datestamp=1331164800,datestamp_delta=86400
|
|
n=5,date=2012-03-09,qoh=10880,datestamp=1331251200,datestamp_delta=86400
|
|
n=6,date=2012-03-10,qoh=10718,datestamp=1331337600,datestamp_delta=86400
|
|
n=7,date=2012-03-11,qoh=10795,datestamp=1331424000,datestamp_delta=86400
|
|
n=8,date=2012-03-12,qoh=11043,datestamp=1331510400,datestamp_delta=86400
|
|
n=9,date=2012-03-13,qoh=11177,datestamp=1331596800,datestamp_delta=86400
|
|
n=10,date=2012-03-14,qoh=11498,datestamp=1331683200,datestamp_delta=86400
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>Then, filter for adjacent difference not being 86400 (the number of seconds in a day):
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from data/miss-date.csv --icsv \
|
|
cat -n \
|
|
then put '$datestamp = strptime($date, "%Y-%m-%d")' \
|
|
then step -a delta -f datestamp \
|
|
then filter '$datestamp_delta != 86400 && $n != 1'
|
|
n=774,date=2014-04-19,qoh=130140,datestamp=1397865600,datestamp_delta=259200
|
|
n=1119,date=2015-03-31,qoh=181625,datestamp=1427760000,datestamp_delta=172800
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> Given this, it’s now easy to see where the gaps are:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr cat -n then filter '$n >= 770 && $n <= 780' data/miss-date.csv
|
|
n=770,1=2014-04-12,2=129435
|
|
n=771,1=2014-04-13,2=129868
|
|
n=772,1=2014-04-14,2=129797
|
|
n=773,1=2014-04-15,2=129919
|
|
n=774,1=2014-04-16,2=130181
|
|
n=775,1=2014-04-19,2=130140
|
|
n=776,1=2014-04-20,2=130271
|
|
n=777,1=2014-04-21,2=130368
|
|
n=778,1=2014-04-22,2=130368
|
|
n=779,1=2014-04-23,2=130849
|
|
n=780,1=2014-04-24,2=131026
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr cat -n then filter '$n >= 1115 && $n <= 1125' data/miss-date.csv
|
|
n=1115,1=2015-03-25,2=181006
|
|
n=1116,1=2015-03-26,2=180995
|
|
n=1117,1=2015-03-27,2=181043
|
|
n=1118,1=2015-03-28,2=181112
|
|
n=1119,1=2015-03-29,2=181306
|
|
n=1120,1=2015-03-31,2=181625
|
|
n=1121,1=2015-04-01,2=181494
|
|
n=1122,1=2015-04-02,2=181718
|
|
n=1123,1=2015-04-03,2=181835
|
|
n=1124,1=2015-04-04,2=182104
|
|
n=1125,1=2015-04-05,2=182528
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Generating_random_variables"/><h1>Generating random variables</h1>
|
|
|
|
<p/>Here can chain together a few simple building blocks:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat expo-sample.sh
|
|
# Generate 100,000 pairs of independent and identically distributed
|
|
# exponentially distributed random variables with the same rate parameter
|
|
# (namely, 2.5). Then compute histograms of one of them, along with
|
|
# histograms for their sum and their product.
|
|
#
|
|
# See also https://en.wikipedia.org/wiki/Exponential_distribution
|
|
#
|
|
# Here I'm using a specified random-number seed so this example always
|
|
# produces the same output for this web document: in everyday practice we
|
|
# wouldn't do that.
|
|
|
|
mlr -n \
|
|
--seed 0.25 \
|
|
--opprint \
|
|
seqgen --stop 100000 \
|
|
then put '
|
|
# https://en.wikipedia.org/wiki/Inverse_transform_sampling
|
|
func expo_sample(lambda) {
|
|
return -log(1-urand())/lambda
|
|
}
|
|
$u = expo_sample(2.5);
|
|
$v = expo_sample(2.5);
|
|
$s = $u + $v;
|
|
$p = $u * $v;
|
|
' \
|
|
then histogram -f u,s,p --lo 0 --hi 2 --nbins 50 \
|
|
then bar -f u_count,s_count,p_count --auto -w 20
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>Namely:
|
|
|
|
<ul>
|
|
<li/> Set the Miller random-number seed so this webdoc looks the same every time I regenerate it.
|
|
<li/> Use pretty-printed tabular output.
|
|
<li/> Use pretty-printed tabular output.
|
|
<li/> Use <tt>seqgen</tt> to produce 100,000 records <tt>i=0</tt>, <tt>i=1</tt>, etc.
|
|
<li/> Send those to a <tt>put</tt> step which defines an inverse-transform-sampling function and
|
|
calls it twice, then computes the sum and product of samples.
|
|
<li/> Send those to a histogram, and from there to a bar-plotter. This is just for visualization; you
|
|
could just as well output CSV and send that off to your own plotting tool, etc.
|
|
</ul>
|
|
|
|
The output is as follows:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ sh expo-sample.sh
|
|
bin_lo bin_hi u_count s_count p_count
|
|
0.000000 0.040000 [78]*******************#[9497] [353]#...................[3732] [20]*******************#[39755]
|
|
0.040000 0.080000 [78]******************..[9497] [353]*****...............[3732] [20]*******.............[39755]
|
|
0.080000 0.120000 [78]****************....[9497] [353]*********...........[3732] [20]****................[39755]
|
|
0.120000 0.160000 [78]**************......[9497] [353]************........[3732] [20]***.................[39755]
|
|
0.160000 0.200000 [78]*************.......[9497] [353]**************......[3732] [20]**..................[39755]
|
|
0.200000 0.240000 [78]************........[9497] [353]****************....[3732] [20]*...................[39755]
|
|
0.240000 0.280000 [78]**********..........[9497] [353]******************..[3732] [20]*...................[39755]
|
|
0.280000 0.320000 [78]**********..........[9497] [353]******************..[3732] [20]*...................[39755]
|
|
0.320000 0.360000 [78]*********...........[9497] [353]*******************.[3732] [20]#...................[39755]
|
|
0.360000 0.400000 [78]********............[9497] [353]*******************.[3732] [20]#...................[39755]
|
|
0.400000 0.440000 [78]*******.............[9497] [353]*******************#[3732] [20]#...................[39755]
|
|
0.440000 0.480000 [78]******..............[9497] [353]******************..[3732] [20]#...................[39755]
|
|
0.480000 0.520000 [78]*****...............[9497] [353]******************..[3732] [20]#...................[39755]
|
|
0.520000 0.560000 [78]*****...............[9497] [353]******************..[3732] [20]#...................[39755]
|
|
0.560000 0.600000 [78]****................[9497] [353]*****************...[3732] [20]#...................[39755]
|
|
0.600000 0.640000 [78]****................[9497] [353]*****************...[3732] [20]#...................[39755]
|
|
0.640000 0.680000 [78]****................[9497] [353]****************....[3732] [20]#...................[39755]
|
|
0.680000 0.720000 [78]***.................[9497] [353]****************....[3732] [20]#...................[39755]
|
|
0.720000 0.760000 [78]***.................[9497] [353]**************......[3732] [20]#...................[39755]
|
|
0.760000 0.800000 [78]**..................[9497] [353]**************......[3732] [20]#...................[39755]
|
|
0.800000 0.840000 [78]**..................[9497] [353]*************.......[3732] [20]#...................[39755]
|
|
0.840000 0.880000 [78]**..................[9497] [353]************........[3732] [20]#...................[39755]
|
|
0.880000 0.920000 [78]**..................[9497] [353]***********.........[3732] [20]#...................[39755]
|
|
0.920000 0.960000 [78]*...................[9497] [353]***********.........[3732] [20]#...................[39755]
|
|
0.960000 1.000000 [78]*...................[9497] [353]**********..........[3732] [20]#...................[39755]
|
|
1.000000 1.040000 [78]*...................[9497] [353]*********...........[3732] [20]#...................[39755]
|
|
1.040000 1.080000 [78]*...................[9497] [353]*********...........[3732] [20]#...................[39755]
|
|
1.080000 1.120000 [78]*...................[9497] [353]********............[3732] [20]#...................[39755]
|
|
1.120000 1.160000 [78]*...................[9497] [353]********............[3732] [20]#...................[39755]
|
|
1.160000 1.200000 [78]#...................[9497] [353]*******.............[3732] [20]#...................[39755]
|
|
1.200000 1.240000 [78]#...................[9497] [353]******..............[3732] [20]#...................[39755]
|
|
1.240000 1.280000 [78]#...................[9497] [353]*****...............[3732] [20]#...................[39755]
|
|
1.280000 1.320000 [78]#...................[9497] [353]*****...............[3732] [20]#...................[39755]
|
|
1.320000 1.360000 [78]#...................[9497] [353]*****...............[3732] [20]#...................[39755]
|
|
1.360000 1.400000 [78]#...................[9497] [353]****................[3732] [20]#...................[39755]
|
|
1.400000 1.440000 [78]#...................[9497] [353]****................[3732] [20]#...................[39755]
|
|
1.440000 1.480000 [78]#...................[9497] [353]***.................[3732] [20]#...................[39755]
|
|
1.480000 1.520000 [78]#...................[9497] [353]***.................[3732] [20]#...................[39755]
|
|
1.520000 1.560000 [78]#...................[9497] [353]***.................[3732] [20]#...................[39755]
|
|
1.560000 1.600000 [78]#...................[9497] [353]**..................[3732] [20]#...................[39755]
|
|
1.600000 1.640000 [78]#...................[9497] [353]**..................[3732] [20]#...................[39755]
|
|
1.640000 1.680000 [78]#...................[9497] [353]*...................[3732] [20]#...................[39755]
|
|
1.680000 1.720000 [78]#...................[9497] [353]*...................[3732] [20]#...................[39755]
|
|
1.720000 1.760000 [78]#...................[9497] [353]*...................[3732] [20]#...................[39755]
|
|
1.760000 1.800000 [78]#...................[9497] [353]*...................[3732] [20]#...................[39755]
|
|
1.800000 1.840000 [78]#...................[9497] [353]#...................[3732] [20]#...................[39755]
|
|
1.840000 1.880000 [78]#...................[9497] [353]#...................[3732] [20]#...................[39755]
|
|
1.880000 1.920000 [78]#...................[9497] [353]#...................[3732] [20]#...................[39755]
|
|
1.920000 1.960000 [78]#...................[9497] [353]#...................[3732] [20]#...................[39755]
|
|
1.960000 2.000000 [78]#...................[9497] [353]#...................[3732] [20]#...................[39755]
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Two-pass_algorithms"/><h1>Two-pass algorithms</h1>
|
|
|
|
<p/>Miller is a streaming record processor; commands are performed once per
|
|
record. This makes Miller particularly suitable for single-pass algorithms,
|
|
allowing many of its verbs to process files that are (much) larger than the
|
|
amount of RAM present in your system. (Of course, Miller verbs such as
|
|
<tt>sort</tt>, </tt>tac</tt>, etc. all must ingest and retain all input records
|
|
before emitting any output records.) You can also use out-of-stream variables
|
|
to perform multi-pass computations, at the price of retaining all input records
|
|
in memory.
|
|
|
|
<a id="Two-pass_algorithms:_computation_of_percentages"/><h2>Two-pass algorithms: computation of percentages</h2>
|
|
|
|
<p/> For example, mapping numeric values down a column to the percentage
|
|
between their min and max values is two-pass: on the first pass you find the
|
|
min and max values, then on the second, map each record’s value to a
|
|
percentage.
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from data/small --opprint put -q '
|
|
# These are executed once per record, which is the first pass.
|
|
# The key is to use NR to index an out-of-stream variable to
|
|
# retain all the x-field values.
|
|
@x_min = min($x, @x_min);
|
|
@x_max = max($x, @x_max);
|
|
@x[NR] = $x;
|
|
|
|
# The second pass is in a for-loop in an end-block.
|
|
end {
|
|
for (nr, x in @x) {
|
|
@x_pct[nr] = 100 * (x - @x_min) / (@x_max - @x_min);
|
|
}
|
|
emit (@x, @x_pct), "NR"
|
|
}
|
|
'
|
|
NR x x_pct
|
|
1 0.346790 25.661943
|
|
2 0.758680 100.000000
|
|
3 0.204603 0.000000
|
|
4 0.381399 31.908236
|
|
5 0.573289 66.540542
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Two-pass_algorithms:_line-number_ratios"/><h2>Two-pass algorithms: line-number ratios</h2>
|
|
|
|
<p/>Similarly, finding the total record count requires first reading through
|
|
all the data:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint --from data/small put -q '
|
|
@records[NR] = $*;
|
|
end {
|
|
for((I,k),v in @records) {
|
|
@records[I]["I"] = I;
|
|
@records[I]["N"] = NR;
|
|
@records[I]["PCT"] = 100*I/NR
|
|
}
|
|
emit @records,"I"
|
|
}
|
|
' then reorder -f I,N,PCT
|
|
I N PCT a b i x y
|
|
1 5 20 pan pan 1 0.3467901443380824 0.7268028627434533
|
|
2 5 40 eks pan 2 0.7586799647899636 0.5221511083334797
|
|
3 5 60 wye wye 3 0.20460330576630303 0.33831852551664776
|
|
4 5 80 eks wye 4 0.38139939387114097 0.13418874328430463
|
|
5 5 100 wye pan 5 0.5732889198020006 0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Two-pass_algorithms:_records_having_max_value"/><h2>Two-pass algorithms: records having max value</h2>
|
|
|
|
<p/>The idea is to retain records having the largest value of <tt>n</tt> in the
|
|
following data:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --itsv --opprint cat data/maxrows.tsv
|
|
a b n score
|
|
purple red 5 0.743231
|
|
blue purple 2 0.093710
|
|
red purple 2 0.802103
|
|
purple red 5 0.389055
|
|
red purple 2 0.880457
|
|
orange red 2 0.540349
|
|
purple purple 1 0.634451
|
|
orange purple 5 0.257223
|
|
orange purple 5 0.693499
|
|
red red 4 0.981355
|
|
blue purple 5 0.157052
|
|
purple purple 1 0.441784
|
|
red purple 1 0.124912
|
|
orange blue 1 0.921944
|
|
blue purple 4 0.490909
|
|
purple red 5 0.454779
|
|
green purple 4 0.198278
|
|
orange blue 5 0.705700
|
|
red red 3 0.940705
|
|
purple red 5 0.072936
|
|
orange blue 3 0.389463
|
|
orange purple 2 0.664985
|
|
blue purple 1 0.371813
|
|
red purple 4 0.984571
|
|
green purple 5 0.203577
|
|
green purple 3 0.900873
|
|
purple purple 0 0.965677
|
|
blue purple 2 0.208785
|
|
purple purple 1 0.455077
|
|
red purple 4 0.477187
|
|
blue red 4 0.007487
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>Of course, the largest value of <tt>n</tt> isn’t known until after
|
|
all data have been read. Using an out-of-stream variable we can retain all
|
|
records as they are read, then filter them at the end:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/maxrows.mlr
|
|
# Retain all records
|
|
@records[NR] = $*;
|
|
# Track max value of n
|
|
@maxn = max(@maxn, $n);
|
|
|
|
# After all records have been read, loop through retained records
|
|
# and print those with the max n value
|
|
end {
|
|
for ((nr,k),v in @records) {
|
|
if (k == "n") {
|
|
if (@records[nr]["n"] == @maxn) {
|
|
emit @records[nr]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --itsv --opprint put -q -f data/maxrows.mlr data/maxrows.tsv
|
|
a b n score
|
|
purple red 5 0.743231
|
|
purple red 5 0.389055
|
|
orange purple 5 0.257223
|
|
orange purple 5 0.693499
|
|
blue purple 5 0.157052
|
|
purple red 5 0.454779
|
|
orange blue 5 0.705700
|
|
purple red 5 0.072936
|
|
green purple 5 0.203577
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Filtering_paragraphs_of_text"/><h1>Filtering paragraphs of text</h1>
|
|
|
|
<p/>The idea is to use a record separator which is a pair of newlines. Then, if
|
|
you want each paragraph to be a record with a single value, use a
|
|
field-separator which isn’t present in the input data (e.g. a control-A
|
|
which is octal 001). Or, if you want each paragraph to have its lines as
|
|
separate values, use newline as field separator.
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat paragraphs.txt
|
|
The quick brown fox jumped over the lazy dogs. The quick brown fox jumped
|
|
over the lazy dogs. The quick brown fox jumped over the lazy dogs. The quick
|
|
brown fox jumped over the lazy dogs. The quick brown fox jumped over the
|
|
lazy dogs.
|
|
|
|
Now is the time for all good people to come to the aid of their country. Now
|
|
is the time for all good people to come to the aid of their country. Now is
|
|
the time for all good people to come to the aid of their country. Now is the
|
|
time for all good people to come to the aid of their country. Now is the
|
|
time for all good people to come to the aid of their country.
|
|
|
|
Sphynx of black quartz, judge my vow. Sphynx of black quartz, judge my vow.
|
|
Sphynx of black quartz, judge my vow. Sphynx of black quartz, judge my vow.
|
|
Sphynx of black quartz, judge my vow.
|
|
|
|
The rain in Spain falls mainly on the plain. The rain in Spain falls mainly
|
|
on the plain. The rain in Spain falls mainly on the plain. The rain in Spain
|
|
falls mainly on the plain. The rain in Spain falls mainly on the plain. The
|
|
rain in Spain falls mainly on the plain. The rain in Spain falls mainly on
|
|
the plain. The rain in Spain falls mainly on the plain.
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from paragraphs.txt --nidx --rs '\n\n' --fs '\001' filter '$1 =~ "the"'
|
|
The quick brown fox jumped over the lazy dogs. The quick brown fox jumped
|
|
over the lazy dogs. The quick brown fox jumped over the lazy dogs. The quick
|
|
brown fox jumped over the lazy dogs. The quick brown fox jumped over the
|
|
lazy dogs.
|
|
|
|
Now is the time for all good people to come to the aid of their country. Now
|
|
is the time for all good people to come to the aid of their country. Now is
|
|
the time for all good people to come to the aid of their country. Now is the
|
|
time for all good people to come to the aid of their country. Now is the
|
|
time for all good people to come to the aid of their country.
|
|
|
|
The rain in Spain falls mainly on the plain. The rain in Spain falls mainly
|
|
on the plain. The rain in Spain falls mainly on the plain. The rain in Spain
|
|
falls mainly on the plain. The rain in Spain falls mainly on the plain. The
|
|
rain in Spain falls mainly on the plain. The rain in Spain falls mainly on
|
|
the plain. The rain in Spain falls mainly on the plain.
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --from paragraphs.txt --nidx --rs '\n\n' --fs '\n' cut -f 1,3
|
|
The quick brown fox jumped over the lazy dogs. The quick brown fox jumped
|
|
brown fox jumped over the lazy dogs. The quick brown fox jumped over the
|
|
|
|
Now is the time for all good people to come to the aid of their country. Now
|
|
the time for all good people to come to the aid of their country. Now is the
|
|
|
|
Sphynx of black quartz, judge my vow. Sphynx of black quartz, judge my vow.
|
|
Sphynx of black quartz, judge my vow.
|
|
|
|
The rain in Spain falls mainly on the plain. The rain in Spain falls mainly
|
|
falls mainly on the plain. The rain in Spain falls mainly on the plain. The
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Doing_arithmetic_on_fields_with_currency_symbols"/><h1>Doing arithmetic on fields with currency symbols</h1>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat sample.csv
|
|
EventOccurred,EventType,Description,Status,PaymentType,NameonAccount,TransactionNumber,Amount
|
|
10/1/2015,Charged Back,Reason: Authorization Revoked By Customer,Disputed,Checking,John,1,$230.36
|
|
10/1/2015,Charged Back,Reason: Authorization Revoked By Customer,Disputed,Checking,Fred,2,$32.25
|
|
10/1/2015,Charged Back,Reason: Customer Advises Not Authorized,Disputed,Checking,Bob,3,$39.02
|
|
10/1/2015,Charged Back,Reason: Authorization Revoked By Customer,Disputed,Checking,Alice,4,$57.54
|
|
10/1/2015,Charged Back,Reason: Authorization Revoked By Customer,Disputed,Checking,Jungle,5,$230.36
|
|
10/1/2015,Charged Back,Reason: Payment Stopped,Disputed,Checking,Joe,6,$281.96
|
|
10/2/2015,Charged Back,Reason: Customer Advises Not Authorized,Disputed,Checking,Joseph,7,$188.19
|
|
10/2/2015,Charged Back,Reason: Customer Advises Not Authorized,Disputed,Checking,Joseph,8,$188.19
|
|
10/2/2015,Charged Back,Reason: Payment Stopped,Disputed,Checking,Anthony,9,$250.00
|
|
|
|
$ mlr --icsv --opprint cat sample.csv
|
|
EventOccurred EventType Description Status PaymentType NameonAccount TransactionNumber Amount
|
|
10/1/2015 Charged Back Reason: Authorization Revoked By Customer Disputed Checking John 1 $230.36
|
|
10/1/2015 Charged Back Reason: Authorization Revoked By Customer Disputed Checking Fred 2 $32.25
|
|
10/1/2015 Charged Back Reason: Customer Advises Not Authorized Disputed Checking Bob 3 $39.02
|
|
10/1/2015 Charged Back Reason: Authorization Revoked By Customer Disputed Checking Alice 4 $57.54
|
|
10/1/2015 Charged Back Reason: Authorization Revoked By Customer Disputed Checking Jungle 5 $230.36
|
|
10/1/2015 Charged Back Reason: Payment Stopped Disputed Checking Joe 6 $281.96
|
|
10/2/2015 Charged Back Reason: Customer Advises Not Authorized Disputed Checking Joseph 7 $188.19
|
|
10/2/2015 Charged Back Reason: Customer Advises Not Authorized Disputed Checking Joseph 8 $188.19
|
|
10/2/2015 Charged Back Reason: Payment Stopped Disputed Checking Anthony 9 $250.00
|
|
|
|
$ mlr --csv put '$Amount = sub(string($Amount), "\$", "")' then stats1 -a sum -f Amount sample.csv
|
|
Amount_sum
|
|
1497.870000
|
|
|
|
$ mlr --csv --ofmt '%.2lf' put '$Amount = sub(string($Amount), "\$", "")' then stats1 -a sum -f Amount sample.csv
|
|
Amount_sum
|
|
1497.87
|
|
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<!-- ================================================================ -->
|
|
<a id="Using_out-of-stream_variables"/><h1>Using out-of-stream variables</h1>
|
|
|
|
<p/> One of Miller’s strengths is its compact notation: for example, given input of the form
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ head -n 5 ../data/medium
|
|
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
|
|
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
|
|
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
|
|
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
|
|
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
you can simply do
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --oxtab stats1 -a sum -f x ../data/medium
|
|
x_sum 4986.019682
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
or
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint stats1 -a sum -f x -g b ../data/medium
|
|
b x_sum
|
|
pan 965.763670
|
|
wye 1023.548470
|
|
zee 979.742016
|
|
eks 1016.772857
|
|
hat 1000.192668
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
rather than the more tedious
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --oxtab put -q '
|
|
@x_sum += $x;
|
|
end {
|
|
emit @x_sum
|
|
}
|
|
' data/medium
|
|
x_sum 4986.019682
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
or
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint put -q '
|
|
@x_sum[$b] += $x;
|
|
end {
|
|
emit @x_sum, "b"
|
|
}
|
|
' data/medium
|
|
b x_sum
|
|
pan 965.763670
|
|
wye 1023.548470
|
|
zee 979.742016
|
|
eks 1016.772857
|
|
hat 1000.192668
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> The former (<tt>mlr stats1</tt> et al.) has the advantages of being easier
|
|
to type, being less error-prone to type, and running faster.
|
|
|
|
<p/> Nonetheless, out-of-stream variables (which I whimsically call
|
|
<i>oosvars</i>), begin/end blocks, and emit statements give you the ability to
|
|
implement logic — if you wish to do so — which isn’t present
|
|
in other Miller verbs. (If you find yourself often using the same
|
|
out-of-stream-variable logic over and over, please file a request at <a
|
|
href="https://github.com/johnkerl/miller/issues">https://github.com/johnkerl/miller/issues</a>
|
|
to get it implemented directly in C as a Miller verb of its own.)
|
|
|
|
<p/> The following examples compute some things using oosvars which are already
|
|
computable using Miller verbs, by way of providing food for thought.
|
|
|
|
<a id="Mean_without/with_oosvars"/><h2>Mean without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint stats1 -a mean -f x data/medium
|
|
x_mean
|
|
0.498602
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint put -q '
|
|
@x_sum += $x;
|
|
@x_count += 1;
|
|
end {
|
|
@x_mean = @x_sum / @x_count;
|
|
emit @x_mean
|
|
}
|
|
' data/medium
|
|
x_mean
|
|
0.498602
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Keyed_mean_without/with_oosvars"/><h2>Keyed mean without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint stats1 -a mean -f x -g a,b data/medium
|
|
a b x_mean
|
|
pan pan 0.513314
|
|
eks pan 0.485076
|
|
wye wye 0.491501
|
|
eks wye 0.483895
|
|
wye pan 0.499612
|
|
zee pan 0.519830
|
|
eks zee 0.495463
|
|
zee wye 0.514267
|
|
hat wye 0.493813
|
|
pan wye 0.502362
|
|
zee eks 0.488393
|
|
hat zee 0.509999
|
|
hat eks 0.485879
|
|
wye hat 0.497730
|
|
pan eks 0.503672
|
|
eks eks 0.522799
|
|
hat hat 0.479931
|
|
hat pan 0.464336
|
|
zee zee 0.512756
|
|
pan hat 0.492141
|
|
pan zee 0.496604
|
|
zee hat 0.467726
|
|
wye zee 0.505907
|
|
eks hat 0.500679
|
|
wye eks 0.530604
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint put -q '
|
|
@x_sum[$a][$b] += $x;
|
|
@x_count[$a][$b] += 1;
|
|
end{
|
|
for ((a, b), v in @x_sum) {
|
|
@x_mean[a][b] = @x_sum[a][b] / @x_count[a][b];
|
|
}
|
|
emit @x_mean, "a", "b"
|
|
}
|
|
' data/medium
|
|
a b x_mean
|
|
pan pan 0.513314
|
|
pan wye 0.502362
|
|
pan eks 0.503672
|
|
pan hat 0.492141
|
|
pan zee 0.496604
|
|
eks pan 0.485076
|
|
eks wye 0.483895
|
|
eks zee 0.495463
|
|
eks eks 0.522799
|
|
eks hat 0.500679
|
|
wye wye 0.491501
|
|
wye pan 0.499612
|
|
wye hat 0.497730
|
|
wye zee 0.505907
|
|
wye eks 0.530604
|
|
zee pan 0.519830
|
|
zee wye 0.514267
|
|
zee eks 0.488393
|
|
zee zee 0.512756
|
|
zee hat 0.467726
|
|
hat wye 0.493813
|
|
hat zee 0.509999
|
|
hat eks 0.485879
|
|
hat hat 0.479931
|
|
hat pan 0.464336
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Variance_and_standard_deviation_without/with_oosvars"/><h2>Variance and standard deviation without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --oxtab stats1 -a count,sum,mean,var,stddev -f x data/medium
|
|
x_count 10000
|
|
x_sum 4986.019682
|
|
x_mean 0.498602
|
|
x_var 0.084270
|
|
x_stddev 0.290293
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat variance.mlr
|
|
@n += 1;
|
|
@sumx += $x;
|
|
@sumx2 += $x**2;
|
|
end {
|
|
@mean = @sumx / @n;
|
|
@var = (@sumx2 - @mean * (2 * @sumx - @n * @mean)) / (@n - 1);
|
|
@stddev = sqrt(@var);
|
|
emitf @n, @sumx, @sumx2, @mean, @var, @stddev
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --oxtab put -q -f variance.mlr data/medium
|
|
n 10000
|
|
sumx 4986.019682
|
|
sumx2 3328.652400
|
|
mean 0.498602
|
|
var 0.084270
|
|
stddev 0.290293
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
You can also do this keyed, of course, imitating the keyed-mean example above.
|
|
|
|
<a id="Min/max_without/with_oosvars"/><h2>Min/max without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --oxtab stats1 -a min,max -f x data/medium
|
|
x_min 0.000045
|
|
x_max 0.999953
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --oxtab put -q '@x_min = min(@x_min, $x); @x_max = max(@x_max, $x); end{emitf @x_min, @x_max}' data/medium
|
|
x_min 0.000045
|
|
x_max 0.999953
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Keyed_min/max_without/with_oosvars"/><h2>Keyed min/max without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint stats1 -a min,max -f x -g a data/medium
|
|
a x_min x_max
|
|
pan 0.000204 0.999403
|
|
eks 0.000692 0.998811
|
|
wye 0.000187 0.999823
|
|
zee 0.000549 0.999490
|
|
hat 0.000045 0.999953
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint --from data/medium put -q '
|
|
@min[$a] = min(@min[$a], $x);
|
|
@max[$a] = max(@max[$a], $x);
|
|
end{
|
|
emit (@min, @max), "a";
|
|
}
|
|
'
|
|
a min max
|
|
pan 0.000204 0.999403
|
|
eks 0.000692 0.998811
|
|
wye 0.000187 0.999823
|
|
zee 0.000549 0.999490
|
|
hat 0.000045 0.999953
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Delta_without/with_oosvars"/><h2>Delta without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint step -a delta -f x data/small
|
|
a b i x y x_delta
|
|
pan pan 1 0.3467901443380824 0.7268028627434533 0
|
|
eks pan 2 0.7586799647899636 0.5221511083334797 0.411890
|
|
wye wye 3 0.20460330576630303 0.33831852551664776 -0.554077
|
|
eks wye 4 0.38139939387114097 0.13418874328430463 0.176796
|
|
wye pan 5 0.5732889198020006 0.8636244699032729 0.191890
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint put '$x_delta = is_present(@last) ? $x - @last : 0; @last = $x' data/small
|
|
a b i x y x_delta
|
|
pan pan 1 0.3467901443380824 0.7268028627434533 0
|
|
eks pan 2 0.7586799647899636 0.5221511083334797 0.411890
|
|
wye wye 3 0.20460330576630303 0.33831852551664776 -0.554077
|
|
eks wye 4 0.38139939387114097 0.13418874328430463 0.176796
|
|
wye pan 5 0.5732889198020006 0.8636244699032729 0.191890
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Keyed_delta_without/with_oosvars"/><h2>Keyed delta without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint step -a delta -f x -g a data/small
|
|
a b i x y x_delta
|
|
pan pan 1 0.3467901443380824 0.7268028627434533 0
|
|
eks pan 2 0.7586799647899636 0.5221511083334797 0
|
|
wye wye 3 0.20460330576630303 0.33831852551664776 0
|
|
eks wye 4 0.38139939387114097 0.13418874328430463 -0.377281
|
|
wye pan 5 0.5732889198020006 0.8636244699032729 0.368686
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint put '$x_delta = is_present(@last[$a]) ? $x - @last[$a] : 0; @last[$a]=$x' data/small
|
|
a b i x y x_delta
|
|
pan pan 1 0.3467901443380824 0.7268028627434533 0
|
|
eks pan 2 0.7586799647899636 0.5221511083334797 0
|
|
wye wye 3 0.20460330576630303 0.33831852551664776 0
|
|
eks wye 4 0.38139939387114097 0.13418874328430463 -0.377281
|
|
wye pan 5 0.5732889198020006 0.8636244699032729 0.368686
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Exponentially_weighted_moving_averages_without/with_oosvars"/><h2>Exponentially weighted moving averages without/with oosvars</h2>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint step -a ewma -d 0.1 -f x data/small
|
|
a b i x y x_ewma_0.1
|
|
pan pan 1 0.3467901443380824 0.7268028627434533 0.346790
|
|
eks pan 2 0.7586799647899636 0.5221511083334797 0.387979
|
|
wye wye 3 0.20460330576630303 0.33831852551664776 0.369642
|
|
eks wye 4 0.38139939387114097 0.13418874328430463 0.370817
|
|
wye pan 5 0.5732889198020006 0.8636244699032729 0.391064
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint put '
|
|
begin{ @a=0.1 };
|
|
$e = NR==1 ? $x : @a * $x + (1 - @a) * @e;
|
|
@e=$e
|
|
' data/small
|
|
a b i x y e
|
|
pan pan 1 0.3467901443380824 0.7268028627434533 0.346790
|
|
eks pan 2 0.7586799647899636 0.5221511083334797 0.387979
|
|
wye wye 3 0.20460330576630303 0.33831852551664776 0.369642
|
|
eks wye 4 0.38139939387114097 0.13418874328430463 0.370817
|
|
wye pan 5 0.5732889198020006 0.8636244699032729 0.391064
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</div>
|
|
</td>
|
|
|
|
</table>
|
|
</body>
|
|
</html>
|