mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 08:55:41 +00:00
248 lines
8.9 KiB
HTML
248 lines
8.9 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<html lang="en">
|
|
|
|
<!-- PAGE GENERATED FROM template.html and content-for-performance.html BY poki. -->
|
|
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
|
|
<head>
|
|
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
|
|
<meta name="description" content="Miller documentation"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
|
|
<meta name="keywords"
|
|
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
|
|
|
|
<title> Performance </title>
|
|
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
|
|
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
|
|
</head>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
|
|
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
|
|
</script>
|
|
<script type="text/javascript">
|
|
try {
|
|
var pageTracker = _gat._getTracker("UA-15651652-1");
|
|
pageTracker._trackPageview();
|
|
} catch(err) {}
|
|
</script>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
|
|
function vis_expand_all_navbar_sections() {
|
|
vis_expand_all_with_prefix('navbar_section_toggle_');
|
|
}
|
|
function vis_collapse_all_navbar_sections() {
|
|
vis_collapse_all_with_prefix('navbar_section_toggle_');
|
|
}
|
|
|
|
function vis_expand_all_body_sections() {
|
|
vis_expand_all_with_prefix('body_section_toggle_');
|
|
}
|
|
function vis_collapse_all_body_sections() {
|
|
vis_collapse_all_with_prefix('body_section_toggle_');
|
|
}
|
|
|
|
function vis_expand_all_with_prefix(prefix) {
|
|
var divs = document.getElementsByTagName('div');
|
|
for(var i = 0; i < divs.length; i++) {
|
|
const div = divs[i];
|
|
if (div.id.startsWith(prefix)) {
|
|
vis_expand_div(div);
|
|
}
|
|
}
|
|
}
|
|
function vis_collapse_all_with_prefix(prefix) {
|
|
var divs = document.getElementsByTagName('div');
|
|
for(var i = 0; i < divs.length; i++){
|
|
const div = divs[i];
|
|
if (div.id.startsWith(prefix)) {
|
|
vis_collapse_div(div);
|
|
}
|
|
}
|
|
}
|
|
|
|
function vis_toggle_by_name(divName) {
|
|
vis_toggle_div(document.getElementById(divName));
|
|
}
|
|
function vis_expand_by_name(divName) {
|
|
vis_expand_div(document.getElementById(divName));
|
|
}
|
|
function vis_collapse_by_name(divName) {
|
|
vis_collapse_div(document.getElementById(divName));
|
|
}
|
|
|
|
function vis_toggle_div(div) {
|
|
if (div != null) {
|
|
var state = div.style.display;
|
|
if (state == 'block') {
|
|
div.style.display = 'none';
|
|
} else {
|
|
div.style.display = 'block';
|
|
}
|
|
}
|
|
}
|
|
function vis_expand_div(div) {
|
|
if (div != null) {
|
|
div.style.display = 'block';
|
|
}
|
|
}
|
|
function vis_collapse_div(div) {
|
|
if (div != null) {
|
|
div.style.display = 'none';
|
|
}
|
|
}
|
|
|
|
</script>
|
|
|
|
<body bgcolor="#ffffff">
|
|
|
|
<!-- ================================================================ -->
|
|
|
|
<!-- navbar -->
|
|
<div class="pokinav">
|
|
<center><titleinbody>Miller</titleinbody></center>
|
|
|
|
<!-- NAVBAR GENERATED FROM template.html BY poki -->
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
|
|
href="javascript:;"
|
|
>
|
|
Overview
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
|
|
href="javascript:;"
|
|
>
|
|
Using
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
|
|
href="javascript:;"
|
|
>
|
|
Reference
|
|
</button>
|
|
<button
|
|
style="font-weight:bold;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
|
|
href="javascript:;"
|
|
>
|
|
Background
|
|
</button>
|
|
<button
|
|
style="font-weight:normal;font-size:100%;color:maroon;border:0"
|
|
padding=0
|
|
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
|
|
href="javascript:;"
|
|
>
|
|
Repository
|
|
</button>
|
|
<div id="navbar_section_toggle_overview" style="display: none;text-align: left">
|
|
<br/><a href="index.html">About</a>
|
|
<br/><a href="10-min.html">10-minute intro</a>
|
|
<br/><a href="file-formats.html">File formats</a>
|
|
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
|
|
<br/><a href="record-heterogeneity.html">Record-hetrogeneity</a>
|
|
<br/><a href="internationalization.html">Internationalization</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_using" style="display: none;text-align: left">
|
|
<br/><a href="faq.html">FAQ</a>
|
|
<br/><a href="data-sharing.html">Mixing with other languages</a>
|
|
<br/><a href="cookbook.html">Cookbook part 1</a>
|
|
<br/><a href="cookbook2.html">Cookbook part 2</a>
|
|
<br/><a href="cookbook3.html">Cookbook part 3</a>
|
|
<br/><a href="data-examples.html">Data-diving examples</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
|
|
<br/><a href="reference.html">Main reference</a>
|
|
<br/><a href="reference-verbs.html">Verbs reference</a>
|
|
<br/><a href="reference-dsl.html">DSL reference</a>
|
|
<br/><a href="manpage.html">Manpage</a>
|
|
<br/><a href="release-docs.html">Documents by release</a>
|
|
<br/><a href="build.html">Installation</a>
|
|
</div>
|
|
<div id="navbar_section_toggle_background" style="display: block;text-align: left">
|
|
<br/><a href="why.html">Why?</a>
|
|
<br/><a href="whyc.html">Why C?</a>
|
|
<br/><a href="whyc-details.html">Why C: details</a>
|
|
<br/><a href="etymology.html">Why call it Miller?</a>
|
|
<br/><a href="originality.html">How original is Miller?</a>
|
|
<br/><a href="performance.html"><b>Performance</b></a>
|
|
</div>
|
|
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
|
|
<br/><a href="contact.html">Contact information</a>
|
|
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- page body -->
|
|
<p/>
|
|
|
|
<!-- BODY COPIED FROM content-for-performance.html BY poki -->
|
|
<div class="pokitoc">
|
|
<center><titleinbody>Performance</titleinbody></center>
|
|
• <a href="#Disclaimer">Disclaimer</a><br/>
|
|
• <a href="#Summary">Summary</a><br/>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_expand_all_body_sections();" href="javascript:;">Expand all sections</button>
|
|
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_collapse_all_body_sections();" href="javascript:;">Collapse all sections</button>
|
|
|
|
<a id="Disclaimer"/><h1>Disclaimer</h1>
|
|
|
|
In a previous version of this page (see <a
|
|
href="http://johnkerl.org/miller-releases/miller-5.1.0/doc/performance.html">here</a>)
|
|
I compared Miller to some items in the Unix toolkit in terms of run time. But
|
|
such comparisons are very much not apples-to-apples:
|
|
|
|
<ul>
|
|
|
|
<li/> Miller’s principal strength is that it handles <b>key-value data in
|
|
various formats</b> while the system tools <b>do not</b>. So if you time
|
|
<code>mlr sort</code> on a CSV file against system <code>sort</code>, it's not relevant
|
|
to say which is faster by how many percent — Miller will respect the
|
|
header line, leaving it in place, while the system sort will move it, sorting
|
|
it along with all the other header lines. This would be comparing the run times
|
|
of two programs produce different outputs. Likewise, <code>awk</code>
|
|
doesn’t respect header lines, although you can code up some CSV-handling
|
|
using <code>if (NR==1) { ... } else { ... }</code>. And that’s just CSV: I
|
|
don’t know any simple way to get <code>sort</code>, <code>awk</code>, etc. to
|
|
handle DKVP, JSON, etc. — which is the main rreason I wrote Miller.
|
|
|
|
<li/> <b>Implementations differ by platform</b>: one <code>awk</code> may be
|
|
fundamentally faster than another, and <code>mawk</code> has a very efficient
|
|
bytecode implementation — which handles positionally indexed data
|
|
far faster than Miller does.
|
|
|
|
<li/> The system <code>sort</code> command will, on some systems, handle
|
|
too-large-for-RAM datasets by spilling to disk; Miller (as of version 5.2.0,
|
|
mid-2017) does not. Miller sorts are always stable; GNU supports stable and
|
|
unstable variants.
|
|
|
|
<li/> Etc.
|
|
|
|
</ul>
|
|
|
|
<a id="Summary"/><h1>Summary</h1>
|
|
|
|
<p/> Miller can do many kinds of processing on key-value-pair data using
|
|
elapsed time roughly of the same order of magnitude as items in the Unix
|
|
toolkit can handle positionally indexed data. Specific results vary widely
|
|
by platform, implementation details, multi-core use (or not). Lastly,
|
|
specific special-purpose non-record-aware processing will run far faster
|
|
in <code>grep</code>, <code>sed</code>, etc.
|
|
|
|
</div>
|
|
|
|
</body>
|
|
</html>
|