Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-21 02:23:20 +00:00
Code Issues Projects Releases Wiki Activity
miller/doc/10-min.html
johnkerl e5c43cb4da fix 
missing data-sharing.html link
2019-09-17 22:39:56 -04:00

1273 lines
38 KiB
HTML
Raw Blame History

<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<!-- PAGE GENERATED FROM template.html and content-for-10-min.html BY poki. -->
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
<head>
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
<meta name="description" content="Miller documentation"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
<meta name="keywords"
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
<title> 10-minute intro </title>
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
</head>
<!-- ================================================================ -->
<script type="text/javascript">
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type="text/javascript">
try {
var pageTracker = _gat._getTracker("UA-15651652-1");
pageTracker._trackPageview();
} catch(err) {}
</script>
<!-- ================================================================ -->
<script type="text/javascript">
function vis_expand_all_navbar_sections() {
//console.log('vis_expand_all_navbar_sections ENTER');
vis_expand_all_with_prefix('navbar_section_toggle_');
//console.log('vis_expand_all_navbar_sections EXIT');
}
function vis_collapse_all_navbar_sections() {
//console.log('vis_collapse_all_navbar_sections ENTER');
vis_collapse_all_with_prefix('navbar_section_toggle_');
//console.log('vis_collapse_all_navbar_sections EXIT');
}
function vis_expand_all_body_sections() {
//console.log('vis_expand_all_body_sections ENTER');
vis_expand_all_with_prefix('body_section_toggle_');
//console.log('vis_expand_all_body_sections EXIT');
}
function vis_collapse_all_body_sections() {
//console.log('vis_collapse_all_body_sections ENTER');
vis_collapse_all_with_prefix('body_section_toggle_');
//console.log('vis_collapse_all_body_sections EXIT');
}
function vis_expand_all_with_prefix(prefix) {
//console.log('vis_expand_all_with_prefix ENTER', prefix);
var divs = document.getElementsByTagName('div');
console.log("DIVS", divs);
for(var i = 0; i < divs.length; i++) {
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_expand_div(div);
}
}
//console.log('vis_expand_all_with_prefix EXIT', prefix);
}
function vis_collapse_all_with_prefix(prefix) {
var divs = document.getElementsByTagName('div');
for(var i = 0; i < divs.length; i++){
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_collapse_div(div);
}
}
}
function vis_toggle_by_name(divName) {
vis_toggle_div(document.getElementById(divName));
}
function vis_expand_by_name(divName) {
vis_expand_div(document.getElementById(divName));
}
function vis_collapse_by_name(divName) {
vis_collapse_div(document.getElementById(divName));
}
function vis_toggle_div(div) {
if (div != null) {
var state = div.style.display;
if (state == 'block') {
div.style.display = 'none';
} else {
div.style.display = 'block';
}
}
}
function vis_expand_div(div) {
if (div != null) {
div.style.display = 'block';
}
}
function vis_collapse_div(div) {
if (div != null) {
div.style.display = 'none';
}
}
</script>
<body bgcolor="#ffffff">
<!-- ================================================================ -->
<!-- navbar -->
<div class="pokinav">
<center><titleinbody>Miller</titleinbody></center>
<!-- NAVBAR GENERATED FROM template.html BY poki -->
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
href="javascript:;"
>
Overview
</button>
<button
style="font-weight:bold;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
href="javascript:;"
>
Using
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
href="javascript:;"
>
Reference
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
href="javascript:;"
>
Background
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
href="javascript:;"
>
Repository
</button>
<div id="navbar_section_toggle_overview" style="display: none;text-align: left">
<br/><a href="index.html">About</a>
<br/><a href="10-min.html"><b>10-minute intro</b></a>
<br/><a href="file-formats.html">File formats</a>
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
<br/><a href="record-heterogeneity.html">Record-hetrogeneity</a>
<br/><a href="internationalization.html">Internationalization</a>
</div>
<div id="navbar_section_toggle_using" style="display: block;text-align: left">
<br/><a href="faq.html">FAQ</a>
<br/><a href="10-min.html"><b>10-minute intro</b></a>
<br/><a href="data-sharing.html">Mixing with other languages</a>
<br/><a href="cookbook.html">Cookbook part 1</a>
<br/><a href="cookbook2.html">Cookbook part 2</a>
<br/><a href="cookbook3.html">Cookbook part 3</a>
<br/><a href="data-examples.html">Data-diving examples</a>
</div>
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
<br/><a href="reference.html">Main reference</a>
<br/><a href="reference-verbs.html">Verbs reference</a>
<br/><a href="reference-dsl.html">DSL reference</a>
<br/><a href="manpage.html">Manpage</a>
<br/><a href="release-docs.html">Documents by release</a>
<br/><a href="build.html">Installation</a>
</div>
<div id="navbar_section_toggle_background" style="display: none;text-align: left">
<br/><a href="why.html">Why?</a>
<br/><a href="whyc.html">Why C?</a>
<br/><a href="etymology.html">Why call it Miller?</a>
<br/><a href="originality.html">How original is Miller?</a>
<br/><a href="performance.html">Performance</a>
</div>
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
<br/><a href="contact.html">Contact information</a>
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
</div>
</div>
<!-- page body -->
<p/>
<!-- BODY COPIED FROM content-for-10-min.html BY poki -->
<div class="pokitoc">
<center><titleinbody>10-minute intro</titleinbody></center>
&bull;&nbsp;<a href="#CSV-file_examples">CSV-file examples</a><br/>
&bull;&nbsp;<a href="#Choices_for_printing_to_files">Choices for printing to files</a><br/>
&bull;&nbsp;<a href="#Other-format_examples">Other-format examples</a><br/>
&bull;&nbsp;<a href="#SQL-output_examples">SQL-output examples</a><br/>
&bull;&nbsp;<a href="#SQL-input_examples">SQL-input examples</a><br/>
&bull;&nbsp;<a href="#Log-processing_examples">Log-processing examples</a><br/>
&bull;&nbsp;<a href="#More">More</a><br/>
</div>
<p/>
<p/>
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_expand_all_body_sections();" href="javascript:;">Expand all sections</button>
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_collapse_all_body_sections();" href="javascript:;">Collapse all sections</button>
<a id="CSV-file_examples"/><h1>CSV-file examples</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_csv_file_examples');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_csv_file_examples" style="display: block">
<p/><boldmaroon> Sample CSV data file: </boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ cat example.csv
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
red,square,1,15,79.2778,0.0130
red,circle,1,16,13.8103,2.9010
red,square,0,48,77.5542,7.4670
purple,triangle,0,51,81.2290,8.5910
red,square,0,64,77.1991,9.5310
purple,triangle,0,65,80.1405,5.8240
yellow,circle,1,73,63.9785,4.2370
yellow,circle,1,87,63.5058,8.3350
purple,square,0,91,72.3735,8.2430
</pre>
</div>
<p/>
<p/><boldmaroon> <code>mlr cat</code> is like cat ...</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv cat example.csv
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
red,square,1,15,79.2778,0.0130
red,circle,1,16,13.8103,2.9010
red,square,0,48,77.5542,7.4670
purple,triangle,0,51,81.2290,8.5910
red,square,0,64,77.1991,9.5310
purple,triangle,0,65,80.1405,5.8240
yellow,circle,1,73,63.9785,4.2370
yellow,circle,1,87,63.5058,8.3350
purple,square,0,91,72.3735,8.2430
</pre>
</div>
<p/>
<p/><boldmaroon>... but it can also do format conversion (here, to pretty-printed tabular format): </boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint cat example.csv
color shape flag index quantity rate
yellow triangle 1 11 43.6498 9.8870
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
red square 0 48 77.5542 7.4670
purple triangle 0 51 81.2290 8.5910
red square 0 64 77.1991 9.5310
purple triangle 0 65 80.1405 5.8240
yellow circle 1 73 63.9785 4.2370
yellow circle 1 87 63.5058 8.3350
purple square 0 91 72.3735 8.2430
</pre>
</div>
<p/>
<p/><boldmaroon> <code>mlr head</code> and <code>mlr tail</code> count records rather than lines. The CSV
header is included either way:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv head -n 4 example.csv
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
red,square,1,15,79.2778,0.0130
red,circle,1,16,13.8103,2.9010
red,square,0,48,77.5542,7.4670
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv tail -n 4 example.csv
color,shape,flag,index,quantity,rate
purple,triangle,0,65,80.1405,5.8240
yellow,circle,1,73,63.9785,4.2370
yellow,circle,1,87,63.5058,8.3350
purple,square,0,91,72.3735,8.2430
</pre>
</div>
<p/>
<p/><boldmaroon> Sort primarily alphabetically on one field, then secondarily
numerically descending on another field: </boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint sort -f shape -nr index example.csv
color shape flag index quantity rate
yellow circle 1 87 63.5058 8.3350
yellow circle 1 73 63.9785 4.2370
red circle 1 16 13.8103 2.9010
purple square 0 91 72.3735 8.2430
red square 0 64 77.1991 9.5310
red square 0 48 77.5542 7.4670
red square 1 15 79.2778 0.0130
purple triangle 0 65 80.1405 5.8240
purple triangle 0 51 81.2290 8.5910
yellow triangle 1 11 43.6498 9.8870
</pre>
</div>
<p/>
<p/><boldmaroon> Use <code>cut</code> to retain only specified fields, in input-data order:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint cut -f flag,shape example.csv
shape flag
triangle 1
square 1
circle 1
square 0
triangle 0
square 0
triangle 0
circle 1
circle 1
square 0
</pre>
</div>
<p/>
<p/><boldmaroon> Use <code>cut -o</code> to retain only specified fields, in your specified order:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint cut -o -f flag,shape example.csv
flag shape
1 triangle
1 square
1 circle
0 square
0 triangle
0 square
0 triangle
1 circle
1 circle
0 square
</pre>
</div>
<p/>
<p/><boldmaroon> Use <code>cut -x</code> to omit specified fields:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint cut -x -f flag,shape example.csv
color index quantity rate
yellow 11 43.6498 9.8870
red 15 79.2778 0.0130
red 16 13.8103 2.9010
red 48 77.5542 7.4670
purple 51 81.2290 8.5910
red 64 77.1991 9.5310
purple 65 80.1405 5.8240
yellow 73 63.9785 4.2370
yellow 87 63.5058 8.3350
purple 91 72.3735 8.2430
</pre>
</div>
<p/>
<p/><boldmaroon> Use <code>filter</code> to retain specified records:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint filter '$color == "red"' example.csv
color shape flag index quantity rate
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
red square 0 48 77.5542 7.4670
red square 0 64 77.1991 9.5310
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint filter '$color == "red" &amp;&amp; $flag == 1' example.csv
color shape flag index quantity rate
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
</pre>
</div>
<p/>
<p/><boldmaroon> Use <code>put</code> to add/replace fields which are computed from other fields:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint put '$ratio = $quantity / $rate; $color_shape = $color . "_" . $shape' example.csv
color shape flag index quantity rate ratio color_shape
yellow triangle 1 11 43.6498 9.8870 4.414868 yellow_triangle
red square 1 15 79.2778 0.0130 6098.292308 red_square
red circle 1 16 13.8103 2.9010 4.760531 red_circle
red square 0 48 77.5542 7.4670 10.386260 red_square
purple triangle 0 51 81.2290 8.5910 9.455127 purple_triangle
red square 0 64 77.1991 9.5310 8.099790 red_square
purple triangle 0 65 80.1405 5.8240 13.760388 purple_triangle
yellow circle 1 73 63.9785 4.2370 15.099953 yellow_circle
yellow circle 1 87 63.5058 8.3350 7.619172 yellow_circle
purple square 0 91 72.3735 8.2430 8.779995 purple_square
</pre>
</div>
<p/>
<p/><boldmaroon> Even though Miller&rsquo;s main selling point is
name-indexing, sometimes you really want to refer to a field name by its
positional index. Use <code>$[[3]]</code> to access the name of field 3 or
<code>$[[[3]]]</code> to access the value of field 3:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint put '$[[3]] = "NEW"' example.csv
color shape NEW index quantity rate
yellow triangle 1 11 43.6498 9.8870
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
red square 0 48 77.5542 7.4670
purple triangle 0 51 81.2290 8.5910
red square 0 64 77.1991 9.5310
purple triangle 0 65 80.1405 5.8240
yellow circle 1 73 63.9785 4.2370
yellow circle 1 87 63.5058 8.3350
purple square 0 91 72.3735 8.2430
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint put '$[[[3]]] = "NEW"' example.csv
color shape flag index quantity rate
yellow triangle NEW 11 43.6498 9.8870
red square NEW 15 79.2778 0.0130
red circle NEW 16 13.8103 2.9010
red square NEW 48 77.5542 7.4670
purple triangle NEW 51 81.2290 8.5910
red square NEW 64 77.1991 9.5310
purple triangle NEW 65 80.1405 5.8240
yellow circle NEW 73 63.9785 4.2370
yellow circle NEW 87 63.5058 8.3350
purple square NEW 91 72.3735 8.2430
</pre>
</div>
<p/>
<p/><boldmaroon> JSON output:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --ojson put '$ratio = $quantity/$rate; $shape = toupper($shape)' example.csv
{ "color": "yellow", "shape": "TRIANGLE", "flag": 1, "index": 11, "quantity": 43.6498, "rate": 9.8870, "ratio": 4.414868 }
{ "color": "red", "shape": "SQUARE", "flag": 1, "index": 15, "quantity": 79.2778, "rate": 0.0130, "ratio": 6098.292308 }
{ "color": "red", "shape": "CIRCLE", "flag": 1, "index": 16, "quantity": 13.8103, "rate": 2.9010, "ratio": 4.760531 }
{ "color": "red", "shape": "SQUARE", "flag": 0, "index": 48, "quantity": 77.5542, "rate": 7.4670, "ratio": 10.386260 }
{ "color": "purple", "shape": "TRIANGLE", "flag": 0, "index": 51, "quantity": 81.2290, "rate": 8.5910, "ratio": 9.455127 }
{ "color": "red", "shape": "SQUARE", "flag": 0, "index": 64, "quantity": 77.1991, "rate": 9.5310, "ratio": 8.099790 }
{ "color": "purple", "shape": "TRIANGLE", "flag": 0, "index": 65, "quantity": 80.1405, "rate": 5.8240, "ratio": 13.760388 }
{ "color": "yellow", "shape": "CIRCLE", "flag": 1, "index": 73, "quantity": 63.9785, "rate": 4.2370, "ratio": 15.099953 }
{ "color": "yellow", "shape": "CIRCLE", "flag": 1, "index": 87, "quantity": 63.5058, "rate": 8.3350, "ratio": 7.619172 }
{ "color": "purple", "shape": "SQUARE", "flag": 0, "index": 91, "quantity": 72.3735, "rate": 8.2430, "ratio": 8.779995 }
</pre>
</div>
<p/>
<p/><boldmaroon> JSON output with vertical-formatting flags:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --ojson --jvstack --jlistwrap tail -n 2 example.csv
[
{
"color": "yellow",
"shape": "circle",
"flag": 1,
"index": 87,
"quantity": 63.5058,
"rate": 8.3350
}
,{
"color": "purple",
"shape": "square",
"flag": 0,
"index": 91,
"quantity": 72.3735,
"rate": 8.2430
}
]
</pre>
</div>
<p/>
<p/><boldmaroon> Use <code>then</code> to pipe commands together. Also, the
<code>-g</code> option for many Miller commands is for group-by: here, <code>head -n
1 -g shape</code> outputs the first record for each distinct value of the
<code>shape</code> field. This means we&rsquo;re finding the record with highest
<code>index</code> field for each distinct <code>shape</code> field:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint sort -f shape -nr index then head -n 1 -g shape example.csv
color shape flag index quantity rate
yellow circle 1 87 63.5058 8.3350
purple square 0 91 72.3735 8.2430
purple triangle 0 65 80.1405 5.8240
</pre>
</div>
<p/>
<p/><boldmaroon> Statistics can be computed with or without group-by field(s). Also, the first of these two
examples uses <code>--oxtab</code> output format which is a nice alternative to <code>--opprint</code> when you
have lots of columns:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --oxtab --from example.csv stats1 -a p0,p10,p25,p50,p75,p90,p99,p100 -f rate
rate_p0 0.013000
rate_p10 2.901000
rate_p25 4.237000
rate_p50 8.243000
rate_p75 8.591000
rate_p90 9.887000
rate_p99 9.887000
rate_p100 9.887000
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape
shape quantity_count quantity_min quantity_mean quantity_max
triangle 3 43.649800 68.339767 81.229000
square 4 72.373500 76.601150 79.277800
circle 3 13.810300 47.098200 63.978500
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --opprint --from example.csv stats1 -a count,min,mean,max -f quantity -g shape,color
shape color quantity_count quantity_min quantity_mean quantity_max
triangle yellow 1 43.649800 43.649800 43.649800
square red 3 77.199100 78.010367 79.277800
circle red 1 13.810300 13.810300 13.810300
triangle purple 2 80.140500 80.684750 81.229000
circle yellow 2 63.505800 63.742150 63.978500
square purple 1 72.373500 72.373500 72.373500
</pre>
</div>
<p/>
</div>
<a id="Choices_for_printing_to_files"/><h1>Choices for printing to files</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_printing_to_files');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_printing_to_files" style="display: block">
<p/>Often we want to print output <boldmaroon>to the screen</boldmaroon>. Miller does this by default, as we&rsquo;ve
seen in the previous examples.
<p/> Sometimes we want to print output to another file: <boldmaroon>just use '>
outputfilenamegoeshere'</boldmaroon> at the end of your command:
<table><tr><td>
<div class="pokipanel"> <pre>
% mlr --icsv --opprint cat example.csv &gt; newfile.csv
# Output goes to the new file;
# nothing is printed to the screen.
</pre> </div>
</td><td>
<div class="pokipanel"> <pre>
% cat newfile.csv
color shape flag index quantity rate
yellow triangle 1 11 43.6498 9.8870
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
red square 0 48 77.5542 7.4670
purple triangle 0 51 81.2290 8.5910
red square 0 64 77.1991 9.5310
purple triangle 0 65 80.1405 5.8240
yellow circle 1 73 63.9785 4.2370
yellow circle 1 87 63.5058 8.3350
purple square 0 91 72.3735 8.2430
</pre> </div>
</td></tr></table>
<p/> Other times we just want our files to be changed in-place: <boldmaroon>just use 'mlr -I'</boldmaroon>.
<table><tr><td>
<div class="pokipanel"> <pre>
% cp example.csv newfile.txt
% cat newfile.txt
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
red,square,1,15,79.2778,0.0130
red,circle,1,16,13.8103,2.9010
red,square,0,48,77.5542,7.4670
purple,triangle,0,51,81.2290,8.5910
red,square,0,64,77.1991,9.5310
purple,triangle,0,65,80.1405,5.8240
yellow,circle,1,73,63.9785,4.2370
yellow,circle,1,87,63.5058,8.3350
purple,square,0,91,72.3735,8.2430
</pre> </div>
</td><td>
<div class="pokipanel"> <pre>
% mlr -I --icsv --opprint cat newfile.txt
% cat newfile.txt
color shape flag index quantity rate
yellow triangle 1 11 43.6498 9.8870
red square 1 15 79.2778 0.0130
red circle 1 16 13.8103 2.9010
red square 0 48 77.5542 7.4670
purple triangle 0 51 81.2290 8.5910
red square 0 64 77.1991 9.5310
purple triangle 0 65 80.1405 5.8240
yellow circle 1 73 63.9785 4.2370
yellow circle 1 87 63.5058 8.3350
purple square 0 91 72.3735 8.2430
</pre> </div>
</td></tr></table>
<p/> Also using <code>mlr -I</code> you can bulk-operate on lots of files: e.g.
<table><tr><td>
<div class="pokipanel"> <pre>
mlr -I --csv cut -x -f unwanted_column_name *.csv
</pre> </div>
</td></tr></table>
<p/> If you like, you can first copy off your original data somewhere else, before doing in-place operations.
<p/><boldmaroon> Lastly, using <code>tee</code> within <code>put</code>, you can split your input data into separate files
per one or more field names:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --csv --from example.csv put -q 'tee &gt; $shape.".csv", $*'
</pre>
</div>
<p/>
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ cat circle.csv
color,shape,flag,index,quantity,rate
red,circle,1,16,13.8103,2.9010
yellow,circle,1,73,63.9785,4.2370
yellow,circle,1,87,63.5058,8.3350
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ cat square.csv
color,shape,flag,index,quantity,rate
red,square,1,15,79.2778,0.0130
red,square,0,48,77.5542,7.4670
red,square,0,64,77.1991,9.5310
purple,square,0,91,72.3735,8.2430
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ cat triangle.csv
color,shape,flag,index,quantity,rate
yellow,triangle,1,11,43.6498,9.8870
purple,triangle,0,51,81.2290,8.5910
purple,triangle,0,65,80.1405,5.8240
</pre>
</div>
<p/>
</td></tr></table>
</div>
<a id="Other-format_examples"/><h1>Other-format examples</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_other_format_examples');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_other_format_examples" style="display: block">
<p/>What&rsquo;s a CSV file, really? It&rsquo;s an array of rows, or
<i>records</i>, each being a list of key-value pairs, or <i>fields</i>: for CSV
it so happens that all the keys are shared in the header line and the values
vary data line by data line.
<p/>For example, if you have
<div class="pokipanel">
<pre>
shape,flag,index
circle,1,24
square,0,36
</pre>
</div>
<p/>then that&rsquo;s a way of saying
<div class="pokipanel">
<pre>
shape=circle,flag=1,index=24
shape=square,flag=0,index=36
</pre>
</div>
<p/>Data written this way are called <boldmaroon>DKVP</boldmaroon>, for <i>delimited key-value pairs</i>.
<p/>We&rsquo;ve also already seen other ways to write the same data:
<div class="pokipanel">
<pre>
CSV PPRINT JSON
shape,flag,index shape flag index [
circle,1,24 circle 1 24 {
square,0,36 square 0 36 "shape": "circle",
"flag": 1,
"index": 24
},
DKVP XTAB {
shape=circle,flag=1,index=24 shape circle "shape": "square",
shape=square,flag=0,index=36 flag 1 "flag": 0,
index 24 "index": 36
}
shape square ]
flag 0
index 36
</pre>
</div>
<p/><boldmaroon>Anything we can do with CSV input data, we can do with any
other format input data.</boldmaroon> And you can read from one format, do any
record-processing, and output to the same format as the input, or to a
different output format.
</div>
<a id="SQL-output_examples"/><h1>SQL-output examples</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_sql_output_examples');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_sql_output_examples" style="display: block">
<p/>I like to produce SQL-query output with header-column and tab delimiter:
this is CSV but with a tab instead of a comma, also known as TSV. Then I
post-process with <code>mlr --tsv</code> or <code>mlr --tsvlite</code>. This
means I can do some (or all, or none) of my data processing within SQL queries,
and some (or none, or all) of my data processing using Miller &mdash; whichever
is most convenient for my needs at the moment.
<p/>For example, using default output formatting in <code>mysql</code> we get
formatting like Miller&rsquo;s <code>--opprint --barred</code>:
<div class="pokipanel">
<pre>
$ mysql --database=mydb -e 'show columns in mytable'
+------------------+--------------+------+-----+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+------------------+--------------+------+-----+---------+-------+
| id | bigint(20) | NO | MUL | NULL | |
| category | varchar(256) | NO | | NULL | |
| is_permanent | tinyint(1) | NO | | NULL | |
| assigned_to | bigint(20) | YES | | NULL | |
| last_update_time | int(11) | YES | | NULL | |
+------------------+--------------+------+-----+---------+-------+
</pre>
</div>
<p/>Using <code>mysql</code>&rsquo;s <code>-B</code> we get TSV output:
<div class="pokipanel">
<pre>
$ mysql --database=mydb -B -e 'show columns in mytable' | mlr --itsvlite --opprint cat
Field Type Null Key Default Extra
id bigint(20) NO MUL NULL -
category varchar(256) NO - NULL -
is_permanent tinyint(1) NO - NULL -
assigned_to bigint(20) YES - NULL -
last_update_time int(11) YES - NULL -
</pre>
</div>
<p/>Since Miller handles TSV output, we can do as much or as little processing
as we want in the SQL query, then send the rest on to Miller. This includes
outputting as JSON, doing further selects/joins in Miller, doing stats, etc.
etc.
<div class="pokipanel">
<pre>
$ mysql --database=mydb -B -e 'show columns in mytable' | mlr --itsvlite --ojson --jlistwrap --jvstack cat
[
{
"Field": "id",
"Type": "bigint(20)",
"Null": "NO",
"Key": "MUL",
"Default": "NULL",
"Extra": ""
},
{
"Field": "category",
"Type": "varchar(256)",
"Null": "NO",
"Key": "",
"Default": "NULL",
"Extra": ""
},
{
"Field": "is_permanent",
"Type": "tinyint(1)",
"Null": "NO",
"Key": "",
"Default": "NULL",
"Extra": ""
},
{
"Field": "assigned_to",
"Type": "bigint(20)",
"Null": "YES",
"Key": "",
"Default": "NULL",
"Extra": ""
},
{
"Field": "last_update_time",
"Type": "int(11)",
"Null": "YES",
"Key": "",
"Default": "NULL",
"Extra": ""
}
]
</pre>
</div>
<div class="pokipanel">
<pre>
$ mysql --database=mydb -B -e 'select * from mytable' > query.tsv
$ mlr --from query.tsv --t2p stats1 -a count -f id -g category,assigned_to
category assigned_to id_count
special 10000978 207
special 10003924 385
special 10009872 168
standard 10000978 524
standard 10003924 392
standard 10009872 108
...
</pre>
</div>
<p/>Again, all the examples in the CSV section apply here &mdash; just change the input-format
flags.
</div>
<a id="SQL-input_examples"/><h1>SQL-input examples</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_sql_input_examples');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_sql_input_examples" style="display: block">
<p/> One use of NIDX (value-only, no keys) format is for loading up SQL tables.
<p/> Create and load SQL table:
<div class="pokipanel"><pre>
mysql> CREATE TABLE abixy(
a VARCHAR(32),
b VARCHAR(32),
i BIGINT(10),
x DOUBLE,
y DOUBLE
);
Query OK, 0 rows affected (0.01 sec)
bash$ mlr --onidx --fs comma cat data/medium > medium.nidx
mysql> LOAD DATA LOCAL INFILE 'medium.nidx' REPLACE INTO TABLE abixy FIELDS TERMINATED BY ',' ;
Query OK, 10000 rows affected (0.07 sec)
Records: 10000 Deleted: 0 Skipped: 0 Warnings: 0
mysql> SELECT COUNT(*) AS count FROM abixy;
+-------+
| count |
+-------+
| 10000 |
+-------+
1 row in set (0.00 sec)
mysql> SELECT * FROM abixy LIMIT 10;
+------+------+------+---------------------+---------------------+
| a | b | i | x | y |
+------+------+------+---------------------+---------------------+
| pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
| eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
| wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
| eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
| wye | pan | 5 | 0.5732889198020006 | 0.8636244699032729 |
| zee | pan | 6 | 0.5271261600918548 | 0.49322128674835697 |
| eks | zee | 7 | 0.6117840605678454 | 0.1878849191181694 |
| zee | wye | 8 | 0.5985540091064224 | 0.976181385699006 |
| hat | wye | 9 | 0.03144187646093577 | 0.7495507603507059 |
| pan | wye | 10 | 0.5026260055412137 | 0.9526183602969864 |
+------+------+------+---------------------+---------------------+
</pre></div>
<p/> Aggregate counts within SQL:
<div class="pokipanel"><pre>
mysql> SELECT a, b, COUNT(*) AS count FROM abixy GROUP BY a, b ORDER BY COUNT DESC;
+------+------+-------+
| a | b | count |
+------+------+-------+
| zee | wye | 455 |
| pan | eks | 429 |
| pan | pan | 427 |
| wye | hat | 426 |
| hat | wye | 423 |
| pan | hat | 417 |
| eks | hat | 417 |
| pan | zee | 413 |
| eks | eks | 413 |
| zee | hat | 409 |
| eks | wye | 407 |
| zee | zee | 403 |
| pan | wye | 395 |
| wye | pan | 392 |
| zee | eks | 391 |
| zee | pan | 389 |
| hat | eks | 389 |
| wye | eks | 386 |
| wye | zee | 385 |
| hat | zee | 385 |
| hat | hat | 381 |
| wye | wye | 377 |
| eks | pan | 371 |
| hat | pan | 363 |
| eks | zee | 357 |
+------+------+-------+
25 rows in set (0.01 sec)
</pre></div>
<p/> Aggregate counts within Miller:
<div class="pokipanel"><pre>
$ mlr --opprint uniq -c -g a,b then sort -nr count data/medium
a b count
zee wye 455
pan eks 429
pan pan 427
wye hat 426
hat wye 423
pan hat 417
eks hat 417
eks eks 413
pan zee 413
zee hat 409
eks wye 407
zee zee 403
pan wye 395
hat pan 363
eks zee 357
</pre></div>
<p/> Pipe SQL output to aggregate counts within Miller:
<div class="pokipanel"><pre>
$ mysql -D miller -B -e 'select * from abixy' | mlr --itsv --opprint uniq -c -g a,b then sort -nr count
a b count
zee wye 455
pan eks 429
pan pan 427
wye hat 426
hat wye 423
pan hat 417
eks hat 417
eks eks 413
pan zee 413
zee hat 409
eks wye 407
zee zee 403
pan wye 395
wye pan 392
zee eks 391
zee pan 389
hat eks 389
wye eks 386
hat zee 385
wye zee 385
hat hat 381
wye wye 377
eks pan 371
hat pan 363
eks zee 357
</pre></div>
</div>
<a id="Log-processing_examples"/><h1>Log-processing examples</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_log_processing_examples');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_log_processing_examples" style="display: block">
<p/>Another of my favorite use-cases for Miller is doing ad-hoc processing of
log-file data. Here&rsquo;s where DKVP format really shines: one, since the
field names and field values are present on every line, every line stands on
its own. That means you can <code>grep</code> or what have you. Also it means not
every line needs to have the same list of field names (&ldquo;schema &rdquo;).
<p/>Again, all the examples in the CSV section apply here &mdash; just change
the input-format flags. But there&rsquo;s more you can do when not all the
records have the same shape.
<p/>Writing a program &mdash; in any language whatsoever &mdash; you can have
it print out log lines as it goes along, with items for various events jumbled
together. After the program has finished running you can sort it all out,
filter it, analyze it, and learn from it.
<p/> Suppose your program has printed something like this:
<p/>
<div class="pokipanel">
<pre>
$ cat log.txt
op=enter,time=1472819681
op=cache,type=A9,hit=0
op=cache,type=A4,hit=1
time=1472819690,batch_size=100,num_filtered=237
op=cache,type=A1,hit=1
op=cache,type=A9,hit=0
op=cache,type=A1,hit=1
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A1,hit=1
time=1472819705,batch_size=100,num_filtered=348
op=cache,type=A4,hit=1
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A4,hit=1
time=1472819713,batch_size=100,num_filtered=493
op=cache,type=A9,hit=1
op=cache,type=A1,hit=1
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A9,hit=1
time=1472819720,batch_size=100,num_filtered=554
op=cache,type=A1,hit=0
op=cache,type=A4,hit=1
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A4,hit=0
op=cache,type=A4,hit=0
op=cache,type=A9,hit=0
time=1472819736,batch_size=100,num_filtered=612
op=cache,type=A1,hit=1
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
op=cache,type=A4,hit=1
op=cache,type=A1,hit=1
op=cache,type=A9,hit=0
op=cache,type=A9,hit=0
time=1472819742,batch_size=100,num_filtered=728
</pre>
</div>
<p/>
<p/> Each print statement simply contains local information: the current
timestamp, whether a particular cache was hit or not, etc. Then using either
the system <code>grep</code> command, or Miller&rsquo;s <code>having-fields</code>, or
<code>is_present</code>, we can pick out the parts we want and analyze them:
<p/>
<div class="pokipanel">
<pre>
$ grep op=cache log.txt \
| mlr --idkvp --opprint stats1 -a mean -f hit -g type then sort -f type
type hit_mean
A1 0.857143
A4 0.714286
A9 0.090909
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --from log.txt --opprint \
filter 'is_present($batch_size)' \
then step -a delta -f time,num_filtered \
then sec2gmt time
time batch_size num_filtered time_delta num_filtered_delta
2016-09-02T12:34:50Z 100 237 0 0
2016-09-02T12:35:05Z 100 348 15 111
2016-09-02T12:35:13Z 100 493 8 145
2016-09-02T12:35:20Z 100 554 7 61
2016-09-02T12:35:36Z 100 612 16 58
2016-09-02T12:35:42Z 100 728 6 116
</pre>
</div>
<p/>
<p/>Alternatively, we can simply group the similar data for a better look:
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint group-like log.txt
op time
enter 1472819681
op type hit
cache A9 0
cache A4 1
cache A1 1
cache A9 0
cache A1 1
cache A9 0
cache A9 0
cache A1 1
cache A4 1
cache A9 0
cache A9 0
cache A9 0
cache A9 0
cache A4 1
cache A9 1
cache A1 1
cache A9 0
cache A9 0
cache A9 1
cache A1 0
cache A4 1
cache A9 0
cache A9 0
cache A9 0
cache A4 0
cache A4 0
cache A9 0
cache A1 1
cache A9 0
cache A9 0
cache A9 0
cache A9 0
cache A4 1
cache A1 1
cache A9 0
cache A9 0
time batch_size num_filtered
1472819690 100 237
1472819705 100 348
1472819713 100 493
1472819720 100 554
1472819736 100 612
1472819742 100 728
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint group-like then sec2gmt time log.txt
op time
enter 2016-09-02T12:34:41Z
op type hit
cache A9 0
cache A4 1
cache A1 1
cache A9 0
cache A1 1
cache A9 0
cache A9 0
cache A1 1
cache A4 1
cache A9 0
cache A9 0
cache A9 0
cache A9 0
cache A4 1
cache A9 1
cache A1 1
cache A9 0
cache A9 0
cache A9 1
cache A1 0
cache A4 1
cache A9 0
cache A9 0
cache A9 0
cache A4 0
cache A4 0
cache A9 0
cache A1 1
cache A9 0
cache A9 0
cache A9 0
cache A9 0
cache A4 1
cache A1 1
cache A9 0
cache A9 0
time batch_size num_filtered
2016-09-02T12:34:50Z 100 237
2016-09-02T12:35:05Z 100 348
2016-09-02T12:35:13Z 100 493
2016-09-02T12:35:20Z 100 554
2016-09-02T12:35:36Z 100 612
2016-09-02T12:35:42Z 100 728
</pre>
</div>
<p/>
</div>
<a id="More"/><h1>More</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_more');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_more" style="display: block">
<p/>Please see the <a href="reference.html">reference</a> for complete
information, as well as the <a href="faq.html">FAQ</a> and the <a
href="cookbook.html">cookbook</a> for more tips.
</div>
</body>
</html>
Reference in a new issue View git blame Copy permalink