mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-19 01:15:21 +00:00
305 lines
13 KiB
HTML
305 lines
13 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<html lang="en">
|
|
|
|
<!-- PAGE GENERATED FROM template.html and content-for-originality.html BY poki. -->
|
|
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
|
|
<head>
|
|
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
|
|
<meta name="description" content="Miller documentation"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
|
|
<meta name="keywords"
|
|
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
|
|
|
|
<title> How original is Miller? </title>
|
|
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
|
|
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
|
|
</head>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
|
|
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
|
|
</script>
|
|
<script type="text/javascript">
|
|
try {
|
|
var pageTracker = _gat._getTracker("UA-15651652-1");
|
|
pageTracker._trackPageview();
|
|
} catch(err) {}
|
|
</script>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
|
|
function vis_expand_all_navbar_sections() {
|
|
//console.log('vis_expand_all_navbar_sections ENTER');
|
|
vis_expand_all_with_prefix('navbar_section_toggle_');
|
|
//console.log('vis_expand_all_navbar_sections EXIT');
|
|
}
|
|
function vis_collapse_all_navbar_sections() {
|
|
//console.log('vis_collapse_all_navbar_sections ENTER');
|
|
vis_collapse_all_with_prefix('navbar_section_toggle_');
|
|
//console.log('vis_collapse_all_navbar_sections EXIT');
|
|
}
|
|
|
|
function vis_expand_all_body_sections() {
|
|
//console.log('vis_expand_all_body_sections ENTER');
|
|
vis_expand_all_with_prefix('body_section_toggle_');
|
|
//console.log('vis_expand_all_body_sections EXIT');
|
|
}
|
|
function vis_collapse_all_body_sections() {
|
|
//console.log('vis_collapse_all_body_sections ENTER');
|
|
vis_collapse_all_with_prefix('body_section_toggle_');
|
|
//console.log('vis_collapse_all_body_sections EXIT');
|
|
}
|
|
|
|
function vis_expand_all_with_prefix(prefix) {
|
|
//console.log('vis_expand_all_with_prefix ENTER', prefix);
|
|
var divs = document.getElementsByTagName('div');
|
|
console.log("DIVS", divs);
|
|
for(var i = 0; i < divs.length; i++) {
|
|
const div = divs[i];
|
|
if (div.id.startsWith(prefix)) {
|
|
vis_expand_div(div);
|
|
}
|
|
}
|
|
//console.log('vis_expand_all_with_prefix EXIT', prefix);
|
|
}
|
|
function vis_collapse_all_with_prefix(prefix) {
|
|
var divs = document.getElementsByTagName('div');
|
|
for(var i = 0; i < divs.length; i++){
|
|
const div = divs[i];
|
|
if (div.id.startsWith(prefix)) {
|
|
vis_collapse_div(div);
|
|
}
|
|
}
|
|
}
|
|
|
|
function vis_toggle_by_name(divName) {
|
|
vis_toggle_div(document.getElementById(divName));
|
|
}
|
|
function vis_expand_by_name(divName) {
|
|
vis_expand_div(document.getElementById(divName));
|
|
}
|
|
function vis_collapse_by_name(divName) {
|
|
vis_collapse_div(document.getElementById(divName));
|
|
}
|
|
|
|
function vis_toggle_div(div) {
|
|
if (div != null) {
|
|
var state = div.style.display;
|
|
if (state == 'block') {
|
|
div.style.display = 'none';
|
|
} else {
|
|
div.style.display = 'block';
|
|
}
|
|
}
|
|
}
|
|
function vis_expand_div(div) {
|
|
if (div != null) {
|
|
div.style.display = 'block';
|
|
}
|
|
}
|
|
function vis_collapse_div(div) {
|
|
if (div != null) {
|
|
div.style.display = 'none';
|
|
}
|
|
}
|
|
|
|
</script>
|
|
|
|
<body bgcolor="#ffffff">
|
|
|
|
<!-- ================================================================ -->
|
|
|
|
<!-- navbar -->
|
|
<div class="pokinav">
|
|
<center><titleinbody>Miller</titleinbody></center>
|
|
|
|
<!-- NAVBAR GENERATED FROM template.html BY poki -->
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
|
|
href="javascript:;"
|
|
>
|
|
Overview
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
|
|
href="javascript:;"
|
|
>
|
|
Using
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
|
|
href="javascript:;"
|
|
>
|
|
Reference
|
|
</button>
|
|
<button
|
|
style="font-weight:bold;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
|
|
href="javascript:;"
|
|
>
|
|
Background
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
|
|
href="javascript:;"
|
|
>
|
|
Repository
|
|
</button>
|
|
<div id="navbar_section_toggle_overview" style="display: none;text-align: left">
|
|
<br/><a href="index.html">About</a>
|
|
<br/><a href="10-min.html">10-minute intro</a>
|
|
<br/><a href="file-formats.html">File formats</a>
|
|
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
|
|
<br/><a href="record-heterogeneity.html">Record-hetrogeneity</a>
|
|
<br/><a href="internationalization.html">Internationalization</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_using" style="display: none;text-align: left">
|
|
<br/><a href="faq.html">FAQ</a>
|
|
<br/><a href="10-min.html">10-minute intro</a>
|
|
<br/><a href="data-sharing.html">Mixing with other languages</a>
|
|
<br/><a href="cookbook.html">Cookbook part 1</a>
|
|
<br/><a href="cookbook2.html">Cookbook part 2</a>
|
|
<br/><a href="cookbook3.html">Cookbook part 3</a>
|
|
<br/><a href="data-examples.html">Data-diving examples</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
|
|
<br/><a href="reference.html">Main reference</a>
|
|
<br/><a href="reference-verbs.html">Verbs reference</a>
|
|
<br/><a href="reference-dsl.html">DSL reference</a>
|
|
<br/><a href="manpage.html">Manpage</a>
|
|
<br/><a href="release-docs.html">Documents by release</a>
|
|
<br/><a href="build.html">Installation</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_background" style="display: block;text-align: left">
|
|
<br/><a href="why.html">Why?</a>
|
|
<br/><a href="whyc.html">Why C?</a>
|
|
<br/><a href="etymology.html">Why call it Miller?</a>
|
|
<br/><a href="originality.html"><b>How original is Miller?</b></a>
|
|
<br/><a href="performance.html">Performance</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
|
|
<br/><a href="contact.html">Contact information</a>
|
|
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- page body -->
|
|
<p/>
|
|
|
|
<!-- BODY COPIED FROM content-for-originality.html BY poki -->
|
|
<div class="pokitoc">
|
|
<center><titleinbody>How original is Miller?</titleinbody></center>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> It isn’t. Miller is one of many, many participants in the
|
|
online-analytical-processing culture. Other key participants include
|
|
<code>awk</code>, SQL, spreadsheets, etc. etc. etc. Far from being an original
|
|
concept, Miller explicitly strives to imitate several existing tools:
|
|
|
|
<p/>
|
|
<boldmaroon>Unix toolkit</boldmaroon>: Intentional similarities as described in
|
|
<a href="feature-comparison.html">Unix-toolkit context</a>.
|
|
|
|
<p/>Recipes abound for command-line data analysis using the Unix toolkit. Here are just a couple of my favorites:
|
|
<ul>
|
|
<li/> <a href="http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line">http://en.wikibooks.org/wiki/Ad_Hoc_Data_Analysis_From_The_Unix_Command_Line</a>
|
|
<li/> <a href="http://www.gregreda.com/2013/07/15/unix-commands-for-data-science">http://www.gregreda.com/2013/07/15/unix-commands-for-data-science</a>
|
|
<li/> <a href="https://github.com/dbohdan/structured-text-tools">https://github.com/dbohdan/structured-text-tools</a>
|
|
</ul>
|
|
|
|
<p/> <boldmaroon>RecordStream</boldmaroon>: Miller owes particular inspiration
|
|
to <a href="https://github.com/benbernard/RecordStream">RecordStream</a>. The
|
|
key difference is that RecordStream is a Perl-based tool for manipulating JSON
|
|
(including requiring it to separately manipulate other formats such as CSV into
|
|
and out of JSON), while Miller is fast C which handles its formats natively.
|
|
The similarities include the <code>sort</code>, <code>stats1</code> (analog of
|
|
RecordStream’s <code>collate</code>), and <code>delta</code> operations, as well
|
|
as <code>filter</code> and <code>put</code>, and pretty-print formatting.
|
|
|
|
<p/> <boldmaroon>stats_m</boldmaroon>: A third source of lineage is my Python
|
|
<a href="https://github.com/johnkerl/scripts-math/tree/master/stats">stats_m</a>
|
|
module. This includes simple single-pass algorithms which form Miller’s
|
|
<code>stats1</code> and <code>stats2</code> subcommands.
|
|
|
|
<p/> <boldmaroon>SQL</boldmaroon>: Fourthly, Miller’s <code>group-by</code> command
|
|
name is from SQL, as is the term <code>aggregate</code>.
|
|
|
|
<p/> <boldmaroon>Added value</boldmaroon>:
|
|
Miller’s added values include:
|
|
<ul>
|
|
<li> Name-indexing, compared to the Unix toolkit’s positional indexing.
|
|
<li> Raw speed, compared to <code>awk</code>, RecordStream, <code>stats_m</code>, or various other kinds of Python/Ruby/etc. scripts one can easily create.
|
|
<li> Compact keystroking for many common tasks, with a decent amount of flexibility.
|
|
<li> Ability to handle text files on the Unix pipe, without need for creating database tables, compared to SQL databases.
|
|
<li> Various file formats, and on-the-fly format conversion.
|
|
</ul>
|
|
|
|
<p/><boldmaroon>jq</boldmaroon>: Miller does for name-indexed text what
|
|
<a href="http://stedolan.github.io/jq/">jq</a> does for JSON. If you’re
|
|
not already familiar with <code>jq</code>, please <a href="http://stedolan.github.io/jq/">check it out!</a>.
|
|
|
|
<p/><boldmaroon>What about similar tools?</boldmaroon>
|
|
Here’s a comprehensive list:
|
|
<a href="https://github.com/dbohdan/structured-text-tools">https://github.com/dbohdan/structured-text-tools</a>.
|
|
It doesn’t mention <a href="https://github.com/turicas/rows">rows</a> so here’s a plug for that as well.
|
|
As it turns out, I learned about most of these after writing Miller.
|
|
|
|
<p/><boldmaroon>What about DOTADIW?</boldmaroon> One of the key points of the
|
|
<a href="http://en.wikipedia.org/wiki/Unix_philosophy">Unix philosophy</a> is
|
|
that a tool should do one thing and do it well. Hence <code>sort</code> and
|
|
<code>cut</code> do just one thing. Why does Miller put <code>awk</code>-like
|
|
processing, a few SQL-like operations, and statistical reduction all into one
|
|
tool (see also <a href="reference.html">Main reference</a>)? This is a fair
|
|
question. First note that many standard tools, such as <code>awk</code> and
|
|
<code>perl</code>, do quite a few things — as does <code>jq</code>. But I could
|
|
have pushed for putting format awareness and name-indexing options into
|
|
<code>cut</code>, <code>awk</code>, and so on (so you could do <code>cut -f
|
|
hostname,uptime</code> or <code>awk '{sum += $x*$y}END{print sum}'</code>). Patching
|
|
<code>cut</code>, <code>sort</code>, etc. on multiple operating systems is a
|
|
non-starter in terms of uptake. Moreover, it makes sense for me to have Miller
|
|
be a tool which collects together format-aware record-stream processing into
|
|
one place, with good reuse of Miller-internal library code for its various
|
|
features.
|
|
|
|
<p/><boldmaroon>Why not use Perl/Python/Ruby etc.?</boldmaroon> Maybe you
|
|
should. With those tools you’ll get far more expressive power, and
|
|
sufficiently quick turnaround time for small-to-medium-sized data. Using
|
|
Miller you’ll get something less than a complete programming language,
|
|
but which is fast, with moderate amounts of flexibility and much less
|
|
keystroking.
|
|
|
|
<p/>When I was first developing Miller I made a survey of several languages.
|
|
Using low-level implementation languages like C, Go, Rust, and Nim, I’d
|
|
need to create my own domain-specific language (DSL) which would always be less
|
|
featured than a full programming language, but I’d get better
|
|
performance. Using high-level interpreted languages such as Perl/Python/Ruby
|
|
I’d get the language’s <code>eval</code> for free and I wouldn’t
|
|
need a DSL; Miller would have mainly been a set of format-specific I/O hooks.
|
|
If I’d gotten good enough performance from the latter I’d have done
|
|
it without question and Miller would be far more flexible. But C won the
|
|
performance criteria by a landslide so we have Miller in C with a custom DSL.
|
|
|
|
<p/> <boldmaroon>No, really, why one more command-line data-manipulation
|
|
tool?</boldmaroon> I wrote Miller because I was frustrated with tools like
|
|
<code>grep</code>, <code>sed</code>, and so on being <i>line-aware</i> without being
|
|
<i>format-aware</i>. The single most poignant example I can think of is seeing
|
|
people grep data lines out of their CSV files and sadly losing their header
|
|
lines. While some lighter-than-SQL processing is very nice to have, at core I
|
|
wanted the format-awareness of <a
|
|
href="https://github.com/benbernard/RecordStream">RecordStream</a> combined
|
|
with the raw speed of the Unix toolkit. Miller does precisely that.
|
|
|
|
</body>
|
|
</html>
|