mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 17:04:50 +00:00
394 lines
13 KiB
HTML
394 lines
13 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<html lang="en">
|
|
|
|
<!-- PAGE GENERATED FROM template.html and content-for-index.html BY poki. -->
|
|
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
|
|
<head>
|
|
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
|
|
<meta name="description" content="Miller documentation"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
|
|
<meta name="keywords"
|
|
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
|
|
|
|
<title> About </title>
|
|
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
|
|
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
|
|
</head>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
|
|
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
|
|
</script>
|
|
<script type="text/javascript">
|
|
try {
|
|
var pageTracker = _gat._getTracker("UA-15651652-1");
|
|
pageTracker._trackPageview();
|
|
} catch(err) {}
|
|
</script>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
|
|
function vis_expand_all_navbar_sections() {
|
|
//console.log('vis_expand_all_navbar_sections ENTER');
|
|
vis_expand_all_with_prefix('navbar_section_toggle_');
|
|
//console.log('vis_expand_all_navbar_sections EXIT');
|
|
}
|
|
function vis_collapse_all_navbar_sections() {
|
|
//console.log('vis_collapse_all_navbar_sections ENTER');
|
|
vis_collapse_all_with_prefix('navbar_section_toggle_');
|
|
//console.log('vis_collapse_all_navbar_sections EXIT');
|
|
}
|
|
|
|
function vis_expand_all_body_sections() {
|
|
//console.log('vis_expand_all_body_sections ENTER');
|
|
vis_expand_all_with_prefix('body_section_toggle_');
|
|
//console.log('vis_expand_all_body_sections EXIT');
|
|
}
|
|
function vis_collapse_all_body_sections() {
|
|
//console.log('vis_collapse_all_body_sections ENTER');
|
|
vis_collapse_all_with_prefix('body_section_toggle_');
|
|
//console.log('vis_collapse_all_body_sections EXIT');
|
|
}
|
|
|
|
function vis_expand_all_with_prefix(prefix) {
|
|
//console.log('vis_expand_all_with_prefix ENTER', prefix);
|
|
var divs = document.getElementsByTagName('div');
|
|
console.log("DIVS", divs);
|
|
for(var i = 0; i < divs.length; i++) {
|
|
const div = divs[i];
|
|
if (div.id.startsWith(prefix)) {
|
|
vis_expand_div(div);
|
|
}
|
|
}
|
|
//console.log('vis_expand_all_with_prefix EXIT', prefix);
|
|
}
|
|
function vis_collapse_all_with_prefix(prefix) {
|
|
var divs = document.getElementsByTagName('div');
|
|
for(var i = 0; i < divs.length; i++){
|
|
const div = divs[i];
|
|
if (div.id.startsWith(prefix)) {
|
|
vis_collapse_div(div);
|
|
}
|
|
}
|
|
}
|
|
|
|
function vis_toggle_by_name(divName) {
|
|
vis_toggle_div(document.getElementById(divName));
|
|
}
|
|
function vis_expand_by_name(divName) {
|
|
vis_expand_div(document.getElementById(divName));
|
|
}
|
|
function vis_collapse_by_name(divName) {
|
|
vis_collapse_div(document.getElementById(divName));
|
|
}
|
|
|
|
function vis_toggle_div(div) {
|
|
if (div != null) {
|
|
var state = div.style.display;
|
|
if (state == 'block') {
|
|
div.style.display = 'none';
|
|
} else {
|
|
div.style.display = 'block';
|
|
}
|
|
}
|
|
}
|
|
function vis_expand_div(div) {
|
|
if (div != null) {
|
|
div.style.display = 'block';
|
|
}
|
|
}
|
|
function vis_collapse_div(div) {
|
|
if (div != null) {
|
|
div.style.display = 'none';
|
|
}
|
|
}
|
|
|
|
</script>
|
|
|
|
<body bgcolor="#ffffff">
|
|
|
|
<!-- ================================================================ -->
|
|
|
|
<!-- navbar -->
|
|
<div class="pokinav">
|
|
<center><titleinbody>Miller</titleinbody></center>
|
|
|
|
<!-- NAVBAR GENERATED FROM template.html BY poki -->
|
|
<button
|
|
style="font-weight:bold;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
|
|
href="javascript:;"
|
|
>
|
|
Overview
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
|
|
href="javascript:;"
|
|
>
|
|
Using
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
|
|
href="javascript:;"
|
|
>
|
|
Reference
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
|
|
href="javascript:;"
|
|
>
|
|
Background
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
|
|
href="javascript:;"
|
|
>
|
|
Repository
|
|
</button>
|
|
<div id="navbar_section_toggle_overview" style="display: block;text-align: left">
|
|
<br/><a href="index.html"><b>About</b></a>
|
|
<br/><a href="10-min.html">10-minute intro</a>
|
|
<br/><a href="file-formats.html">File formats</a>
|
|
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
|
|
<br/><a href="record-heterogeneity.html">Record-hetrogeneity</a>
|
|
<br/><a href="internationalization.html">Internationalization</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_using" style="display: none;text-align: left">
|
|
<br/><a href="faq.html">FAQ</a>
|
|
<br/><a href="10-min.html">10-minute intro</a>
|
|
<br/><a href="data-sharing.html">Mixing with other languages</a>
|
|
<br/><a href="cookbook.html">Cookbook part 1</a>
|
|
<br/><a href="cookbook2.html">Cookbook part 2</a>
|
|
<br/><a href="cookbook3.html">Cookbook part 3</a>
|
|
<br/><a href="data-examples.html">Data-diving examples</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
|
|
<br/><a href="reference.html">Main reference</a>
|
|
<br/><a href="reference-verbs.html">Verbs reference</a>
|
|
<br/><a href="reference-dsl.html">DSL reference</a>
|
|
<br/><a href="manpage.html">Manpage</a>
|
|
<br/><a href="release-docs.html">Documents by release</a>
|
|
<br/><a href="build.html">Installation</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_background" style="display: none;text-align: left">
|
|
<br/><a href="why.html">Why?</a>
|
|
<br/><a href="whyc.html">Why C?</a>
|
|
<br/><a href="etymology.html">Why call it Miller?</a>
|
|
<br/><a href="originality.html">How original is Miller?</a>
|
|
<br/><a href="performance.html">Performance</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
|
|
<br/><a href="contact.html">Contact information</a>
|
|
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- page body -->
|
|
<p/>
|
|
|
|
<!-- BODY COPIED FROM content-for-index.html BY poki -->
|
|
<div class="pokitoc">
|
|
<center><titleinbody>About</titleinbody></center>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>Miller is like awk, sed, cut, join, and sort for <b>name-indexed data such as
|
|
CSV, TSV, and tabular JSON</b>. You get to work with your data using named
|
|
fields, without needing to count positional column indices.
|
|
|
|
<p/>This is something the Unix toolkit always could have done, and arguably
|
|
always should have done. It operates on key-value-pair data while the familiar
|
|
Unix tools operate on integer-indexed fields: if the natural data structure for
|
|
the latter is the array, then Miller’s natural data structure is the
|
|
insertion-ordered hash map. This encompasses a <b>variety of data formats</b>,
|
|
including but not limited to the familiar CSV, TSV, and JSON. (Miller can handle
|
|
<b>positionally-indexed data</b> as a special case.)
|
|
|
|
<p/> Features:
|
|
<ul>
|
|
|
|
<li/> Miller is <b>multi-purpose</b>: it’s useful for <b>data
|
|
cleaning</b>, <b>data reduction</b>, <b>statistical reporting</b>,
|
|
<b>devops</b>, <b>system administration</b>, <b>log-file processing</b>,
|
|
<b>format conversion</b>, and <b>database-query post-processing</b>.
|
|
|
|
<li/> You can use Miller to snarf and munge <b>log-file data</b>, including
|
|
selecting out relevant substreams, then produce CSV format and load that into
|
|
all-in-memory/data-frame utilities for further statistical and/or graphical
|
|
processing.
|
|
|
|
<li/> Miller complements <b>data-analysis tools</b> such as <b>R</b>,
|
|
<b>pandas</b>, etc.: you can use Miller to <b>clean</b> and <b>prepare</b> your
|
|
data. While you can do <b>basic statistics</b> entirely in Miller, its
|
|
streaming-data feature and single-pass algorithms enable you to <b>reduce very
|
|
large data sets</b>.
|
|
|
|
<li/> Miller complements SQL <b>databases</b>: you can slice, dice, and
|
|
reformat data on the client side on its way into or out of a database.
|
|
(Examples <a href="10-min.html#SQL-input_examples">here</a> and <a
|
|
href="10-min.html#SQL-output_examples">here</a>). You can also reap some of the
|
|
benefits of databases for quick, setup-free one-off tasks when you just need to
|
|
query some data in disk files in a hurry.
|
|
|
|
<li/> Miller also goes beyond the classic Unix tools by stepping fully into our
|
|
modern, <b>no-SQL</b> world: its essential record-heterogeneity property allows
|
|
Miller to operate on data where records with different schema (field names) are
|
|
interleaved.
|
|
|
|
<li/> Miller is <b>streaming</b>: most operations need only a single record in
|
|
memory at a time, rather than ingesting all input before producing any output.
|
|
For those operations which require deeper retention (<code>sort</code>,
|
|
<code>tac</code>, <code>stats1</code>), Miller retains only as much data as needed.
|
|
This means that whenever functionally possible, you can operate on files which
|
|
are larger than your system’s available RAM, and you can use Miller in
|
|
<b>tail -f</b> contexts.
|
|
|
|
<li/> Miller is <b>pipe-friendly</b> and interoperates with the Unix toolkit
|
|
|
|
<li/> Miller’s I/O formats include <b>tabular pretty-printing</b>,
|
|
<b>positionally indexed</b> (Unix-toolkit style), CSV, JSON, and others
|
|
|
|
<li/> Miller does <b>conversion</b> between formats
|
|
|
|
<li/> Miller’s <b>processing is format-aware</b>: e.g. CSV <code>sort</code>
|
|
and <code>tac</code> keep header lines first
|
|
|
|
<li/> Miller has high-throughput <b>performance</b> on par with the Unix toolkit
|
|
|
|
<li/> Not unlike <a href="http://stedolan.github.io/jq/">jq</a> (for JSON),
|
|
Miller is written in portable, modern C, with <b>zero runtime dependencies</b>.
|
|
You can download or compile a single binary, <code>scp</code> it to a faraway
|
|
machine, and expect it to work.
|
|
|
|
</ul>
|
|
|
|
<p>Releases and release notes:
|
|
<a href="https://github.com/johnkerl/miller/releases">https://github.com/johnkerl/miller/releases</a>.
|
|
|
|
<p/> Examples:
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Column select
|
|
% mlr --csv cut -f hostname,uptime mydata.csv
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Add new columns as function of other columns
|
|
% mlr --nidx put '$sum = $7 < 0.0 ? 3.5 : $7 + 2.1*$8' *.dat
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Row filter
|
|
% mlr --csv filter '$status != "down" && $upsec >= 10000' *.csv
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Apply column labels and pretty-print
|
|
% grep -v '^#' /etc/group | mlr --ifs : --nidx --opprint label group,pass,gid,member then sort -f group
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Join multiple data sources on key columns
|
|
% mlr join -j account_id -f accounts.dat then group-by account_name balances.dat
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Multiple formats including JSON
|
|
% mlr --json put '$attr = sub($attr, "([0-9]+)_([0-9]+)_.*", "\1:\2")' data/*.json
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Aggregate per-column statistics
|
|
% mlr stats1 -a min,mean,max,p10,p50,p90 -f flag,u,v data/*
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Linear regression
|
|
% mlr stats2 -a linreg-pca -f u,v -g shape data/*
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Aggregate custom per-column statistics
|
|
% mlr put -q '@sum[$a][$b] += $x; end {emit @sum, "a", "b"}' data/*
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Iterate over data using DSL expressions
|
|
% mlr --from estimates.tbl put '
|
|
for (k,v in $*) {
|
|
if (is_numeric(v) && k =~ "^[t-z].*$") {
|
|
$sum += v; $count += 1
|
|
}
|
|
}
|
|
$mean = $sum / $count # no assignment if count unset
|
|
'
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Run DSL expressions from a script file
|
|
% mlr --from infile.dat put -f analyze.mlr
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Split/reduce output to multiple filenames
|
|
% mlr --from infile.dat put 'tee > "./taps/data-".$a."-".$b, $*'
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Compressed I/O
|
|
% mlr --from infile.dat put 'tee | "gzip > ./taps/data-".$a."-".$b.".gz", $*'
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Interoperate with other data-processing tools using standard pipes
|
|
% mlr --from infile.dat put -q '@v=$*; dump | "jq .[]"'
|
|
</pre>
|
|
</div>
|
|
|
|
<div class="pokipanel">
|
|
<pre>
|
|
# Tap/trace
|
|
% mlr --from infile.dat put '(NR % 1000 == 0) { print > stderr, "Checkpoint ".NR}'
|
|
</pre>
|
|
</div>
|
|
|
|
</body>
|
|
</html>
|