mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-18 00:45:47 +00:00
1012 lines
32 KiB
HTML
1012 lines
32 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<html lang="en">
|
|
|
|
<!-- PAGE GENERATED FROM template.html and content-for-file-formats.html BY poki. -->
|
|
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
|
|
<head>
|
|
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
|
|
<meta name="description" content="Miller documentation"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
|
|
<meta name="keywords"
|
|
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
|
|
|
|
<title> File formats </title>
|
|
<link rel="stylesheet" type="text/css" href="css/miller.css"/>
|
|
<link rel="stylesheet" type="text/css" href="css/poki-callbacks.css"/>
|
|
</head>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
|
|
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
|
|
</script>
|
|
<script type="text/javascript">
|
|
try {
|
|
var pageTracker = _gat._getTracker("UA-15651652-1");
|
|
pageTracker._trackPageview();
|
|
} catch(err) {}
|
|
</script>
|
|
|
|
<!-- ================================================================ -->
|
|
<script type="text/javascript">
|
|
function toggle_div(div) {
|
|
if (div != null) {
|
|
if (div.id.startsWith("section_toggle_")) {
|
|
var state = div.style.display;
|
|
if (state == "block") {
|
|
div.style.display = "none";
|
|
} else {
|
|
div.style.display = "block";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
function expand_div(div) {
|
|
if (div != null) {
|
|
if (div.id.startsWith("section_toggle_")) {
|
|
div.style.display = "block";
|
|
}
|
|
}
|
|
}
|
|
function collapse_div(div) {
|
|
if (div != null) {
|
|
if (div.id.startsWith("section_toggle_")) {
|
|
div.style.display = "none";
|
|
}
|
|
}
|
|
}
|
|
|
|
function toggle_by_name(divName) {
|
|
toggle_div(document.getElementById(divName));
|
|
}
|
|
function expand_by_name(divName) {
|
|
expand_div(document.getElementById(divName));
|
|
}
|
|
function collapse_by_name(divName) {
|
|
collapse_div(document.getElementById(divName));
|
|
}
|
|
|
|
function expand_all() {
|
|
var divs = document.getElementsByTagName("div");
|
|
for(var i = 0; i < divs.length; i++) {
|
|
expand_div(divs[i]);
|
|
}
|
|
}
|
|
function collapse_all() {
|
|
var divs = document.getElementsByTagName("div");
|
|
for(var i = 0; i < divs.length; i++){
|
|
collapse_div(divs[i]);
|
|
}
|
|
}
|
|
</script>
|
|
|
|
<!--
|
|
The background image is from a screenshot of a Google search for "data analysis
|
|
tools", lightened and sepia-toned. Over this was placed a Mac Terminal app with
|
|
very light-grey font and translucent background, in which a few statistical
|
|
Miller commands were run with pretty-print-tabular output format.
|
|
<body background="pix/sepia-overlay.jpg">
|
|
-->
|
|
<body bgcolor="#ffffff">
|
|
|
|
<!-- ================================================================ -->
|
|
<table width="100%">
|
|
<tr>
|
|
|
|
<!-- navbar -->
|
|
<td width="15%">
|
|
<!--
|
|
<img src="pix/mlr.jpg" />
|
|
<img style="border-width:1px; color:black;" src="pix/mlr.jpg" />
|
|
-->
|
|
|
|
<div class="pokinav">
|
|
<center><titleinbody>Miller</titleinbody></center>
|
|
|
|
<!-- PAGE LIST GENERATED FROM template.html BY poki -->
|
|
<br/><b>Overview:</b>
|
|
<br/>• <a href="index.html">About Miller</a>
|
|
<br/>• <a href="10-min.html">Miller in 10 minutes</a>
|
|
<br/>• <a href="file-formats.html"><b>File formats</b></a>
|
|
<br/>• <a href="feature-comparison.html">Miller features in the context of the Unix toolkit</a>
|
|
<br/>• <a href="record-heterogeneity.html">Record-heterogeneity</a>
|
|
<br/>• <a href="internationalization.html">Internationalization</a>
|
|
<br/><b>Using Miller:</b>
|
|
<br/>• <a href="faq.html">FAQ</a>
|
|
<br/>• <a href="cookbook.html">Cookbook part 1</a>
|
|
<br/>• <a href="cookbook2.html">Cookbook part 2</a>
|
|
<br/>• <a href="cookbook3.html">Cookbook part 3</a>
|
|
<br/>• <a href="data-examples.html">Data-diving examples</a>
|
|
<br/>• <a href="manpage.html">Manpage</a>
|
|
<br/>• <a href="reference.html">Reference</a>
|
|
<br/>• <a href="reference-verbs.html">Reference: Verbs</a>
|
|
<br/>• <a href="reference-dsl.html">Reference: DSL</a>
|
|
<br/>• <a href="release-docs.html">Documents by release</a>
|
|
<br/>• <a href="build.html">Installation, portability, dependencies, and testing</a>
|
|
<br/><b>Background:</b>
|
|
<br/>• <a href="why.html">Why?</a>
|
|
<br/>• <a href="whyc.html">Why C?</a>
|
|
<br/>• <a href="etymology.html">Why call it Miller?</a>
|
|
<br/>• <a href="originality.html">How original is Miller?</a>
|
|
<br/>• <a href="performance.html">Performance</a>
|
|
<br/><b>Repository:</b>
|
|
<br/>• <a href="to-do.html">Things to do</a>
|
|
<br/>• <a href="contact.html">Contact information</a>
|
|
<br/>• <a href="https://github.com/johnkerl/miller">GitHub repo</a>
|
|
<br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/>
|
|
<br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/> <br/>
|
|
<br/> <br/> <br/> <br/> <br/> <br/>
|
|
</div>
|
|
</td>
|
|
|
|
<!-- page body -->
|
|
<td>
|
|
<!--
|
|
This is a visually gorgeous feature (here & in the CSS): it allows for
|
|
independent scroll of the nav and body panels. In particular the nav
|
|
stays on-screen as you scroll the body.
|
|
|
|
However, two problems:
|
|
|
|
(1) In Firefox & Chrome both I get janky end-of-body scrolls: there is
|
|
more content but I can't scroll down to it unless I repeatedly retry the
|
|
scrolldown. Which is weird.
|
|
|
|
(2) Worse, only the first page renders in PDF (again, Firefox & Chrome).
|
|
|
|
For now I'm disabling this separate-scroll feature. A frontender, I am
|
|
not ... maybe someday I'll find a config which gets *all* the features
|
|
I want; for now, it's a tradeoff.
|
|
-->
|
|
|
|
<!-- Implementation details: one bit is right here:
|
|
|
|
div style="overflow-y:scroll;height:1500px"
|
|
|
|
and the other bit is in css/poki-callbacks.css:
|
|
|
|
.pokinav {
|
|
display: inline-block;
|
|
background: #e8d9bc;
|
|
border: 1;
|
|
box-shadow: 0px 0px 3px 3px #C9C9C9;
|
|
margin: 10px;
|
|
padding-top: 10px;
|
|
padding-bottom: 10px;
|
|
padding-left: 10px;
|
|
padding-right: 10px;
|
|
overflow-y: scroll; < - - - - - - here
|
|
height: 1500px;
|
|
}
|
|
|
|
-->
|
|
<div>
|
|
<center> <titleinbody> File formats </titleinbody> </center>
|
|
<p/>
|
|
|
|
<!-- BODY COPIED FROM content-for-file-formats.html BY poki -->
|
|
<div class="pokitoc">
|
|
<center><b>Contents:</b></center>
|
|
• <a href="#Overview">Overview</a><br/>
|
|
• <a href="#Examples">Examples</a><br/>
|
|
• <a href="#CSV/TSV/etc.">CSV/TSV/etc.</a><br/>
|
|
• <a href="#DKVP:_Key-value_pairs">DKVP: Key-value pairs</a><br/>
|
|
• <a href="#NIDX:_Index-numbered_(toolkit_style)">NIDX: Index-numbered (toolkit style)</a><br/>
|
|
• <a href="#Tabular_JSON">Tabular JSON</a><br/>
|
|
• <a href="#Single-level_JSON_objects">Single-level JSON objects</a><br/>
|
|
• <a href="#Nested_JSON_objects">Nested JSON objects</a><br/>
|
|
• <a href="#Arrays">Arrays</a><br/>
|
|
• <a href="#Formatting_JSON_options">Formatting JSON options</a><br/>
|
|
• <a href="#JSON_non-streaming">JSON non-streaming</a><br/>
|
|
• <a href="#PPRINT:_Pretty-printed_tabular">PPRINT: Pretty-printed tabular</a><br/>
|
|
• <a href="#XTAB:_Vertical_tabular">XTAB: Vertical tabular</a><br/>
|
|
• <a href="#Markdown_tabular">Markdown tabular</a><br/>
|
|
• <a href="#Data-conversion_keystroke-savers">Data-conversion keystroke-savers</a><br/>
|
|
• <a href="#Autodetect_of_line_endings">Autodetect of line endings</a><br/>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>
|
|
<button style="font-weight:bold;color:maroon;border:0" onclick="expand_all();" href="javascript:;">Expand all sections</button>
|
|
<button style="font-weight:bold;color:maroon;border:0" onclick="collapse_all();" href="javascript:;">Collapse all sections</button>
|
|
|
|
<a id="Overview"/><h1>Overview</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_overview');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_overview" style="display: block">
|
|
|
|
<p/>Miller handles name-indexed data using several formats: some you probably
|
|
know by name, such as CSV, TSV, and JSON — and other formats you’re
|
|
likely already seeing and using in your structured data.
|
|
|
|
</div>
|
|
<a id="Examples"/><h1>Examples</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_examples');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_examples" style="display: block">
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --usage-data-format-examples
|
|
DKVP: delimited key-value pairs (Miller default format)
|
|
+---------------------+
|
|
| apple=1,bat=2,cog=3 | Record 1: "apple" => "1", "bat" => "2", "cog" => "3"
|
|
| dish=7,egg=8,flint | Record 2: "dish" => "7", "egg" => "8", "3" => "flint"
|
|
+---------------------+
|
|
|
|
NIDX: implicitly numerically indexed (Unix-toolkit style)
|
|
+---------------------+
|
|
| the quick brown | Record 1: "1" => "the", "2" => "quick", "3" => "brown"
|
|
| fox jumped | Record 2: "1" => "fox", "2" => "jumped"
|
|
+---------------------+
|
|
|
|
CSV/CSV-lite: comma-separated values with separate header line
|
|
+---------------------+
|
|
| apple,bat,cog |
|
|
| 1,2,3 | Record 1: "apple => "1", "bat" => "2", "cog" => "3"
|
|
| 4,5,6 | Record 2: "apple" => "4", "bat" => "5", "cog" => "6"
|
|
+---------------------+
|
|
|
|
Tabular JSON: nested objects are supported, although arrays within them are not:
|
|
+---------------------+
|
|
| { |
|
|
| "apple": 1, | Record 1: "apple" => "1", "bat" => "2", "cog" => "3"
|
|
| "bat": 2, |
|
|
| "cog": 3 |
|
|
| } |
|
|
| { |
|
|
| "dish": { | Record 2: "dish:egg" => "7", "dish:flint" => "8", "garlic" => ""
|
|
| "egg": 7, |
|
|
| "flint": 8 |
|
|
| }, |
|
|
| "garlic": "" |
|
|
| } |
|
|
+---------------------+
|
|
|
|
PPRINT: pretty-printed tabular
|
|
+---------------------+
|
|
| apple bat cog |
|
|
| 1 2 3 | Record 1: "apple => "1", "bat" => "2", "cog" => "3"
|
|
| 4 5 6 | Record 2: "apple" => "4", "bat" => "5", "cog" => "6"
|
|
+---------------------+
|
|
|
|
XTAB: pretty-printed transposed tabular
|
|
+---------------------+
|
|
| apple 1 | Record 1: "apple" => "1", "bat" => "2", "cog" => "3"
|
|
| bat 2 |
|
|
| cog 3 |
|
|
| |
|
|
| dish 7 | Record 2: "dish" => "7", "egg" => "8"
|
|
| egg 8 |
|
|
+---------------------+
|
|
|
|
Markdown tabular (supported for output only):
|
|
+-----------------------+
|
|
| | apple | bat | cog | |
|
|
| | --- | --- | --- | |
|
|
| | 1 | 2 | 3 | | Record 1: "apple => "1", "bat" => "2", "cog" => "3"
|
|
| | 4 | 5 | 6 | | Record 2: "apple" => "4", "bat" => "5", "cog" => "6"
|
|
+-----------------------+
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
</div>
|
|
<a id="CSV/TSV/etc."/><h1>CSV/TSV/etc.</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_csv_tsv_etc');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_csv_tsv_etc" style="display: block">
|
|
|
|
<p/>
|
|
When <tt>mlr</tt> is invoked with the <tt>--csv</tt> or <tt>--csvlite</tt> option,
|
|
key names are found on the first record and values are taken from subsequent
|
|
records. This includes the case of CSV-formatted files. See
|
|
<a href="record-heterogeneity.html">Record-heterogeneity</a> for how Miller handles
|
|
changes of field names within a single data stream.
|
|
|
|
<p/> Miller has record separator <tt>RS</tt> and field separator <tt>FS</tt>,
|
|
just as <tt>awk</tt> does. For TSV, use <tt>--fs tab</tt>; to convert TSV to
|
|
CSV, use <tt>--ifs tab --ofs comma</tt>, etc. (See also
|
|
<a href="reference.html#Record/field/pair_separators">Reference</a>.)
|
|
|
|
<p/> The following are synonymous pairs:
|
|
<ul>
|
|
<li/> <tt>--tsv</tt> and <tt>--csv --fs tab</tt>
|
|
<li/> <tt>--itsv</tt> and <tt>--icsv --ifs tab</tt>
|
|
<li/> <tt>--otsv</tt> and <tt>--ocsv --ofs tab</tt>
|
|
<li/> <tt>--tsvlite</tt> and <tt>--csvlite --fs tab</tt>
|
|
<li/> <tt>--itsvlite</tt> and <tt>--icsvlite --ifs tab</tt>
|
|
<li/> <tt>--otsvlite</tt> and <tt>--ocsvlite --ofs tab</tt>
|
|
</ul>
|
|
|
|
<p/>Miller’s <tt>--csv</tt> flag supports RFC-4180 CSV (<a href="https://tools.ietf.org/html/rfc4180">
|
|
https://tools.ietf.org/html/rfc4180</a>). This includes CRLF line-terminators by default, regardless
|
|
of platform.
|
|
|
|
<p/>Here are the differences between CSV and CSV-lite:
|
|
|
|
<ul>
|
|
|
|
<li/>CSV supports <a href="https://tools.ietf.org/html/rfc4180">
|
|
RFC-4180</a>)-style double-quoting, including the ability to have commas and/or
|
|
LF/CRLF line-endings contained within an input field; CSV-lite does not.
|
|
|
|
<li/>CSV does not allow heterogeneous data; CSV-lite does (see also <a
|
|
href="record-heterogeneity.html">here</a>).
|
|
|
|
<li/>The CSV-lite input-reading code is fractionally more efficient than the CSV
|
|
input-reader.
|
|
|
|
</ul>
|
|
|
|
<p/>Here are things they have in common:
|
|
|
|
<ul>
|
|
|
|
<li/>The ability to specify record/field separators other than the default,
|
|
e.g. CR-LF vs. LF, or tab instead of comma for TSV, and so on.
|
|
|
|
<li/>The <tt>--implicit-csv-header</tt> flag for input and the
|
|
<tt>--headerless-csv-output</tt> flag for output.
|
|
|
|
</ul>
|
|
|
|
</div>
|
|
<a id="DKVP:_Key-value_pairs"/><h1>DKVP: Key-value pairs</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_dkvp');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_dkvp" style="display: block">
|
|
|
|
<p/>
|
|
Miller’s default file format is DKVP, for <b>delimited key-value pairs</b>. Example:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr cat data/small
|
|
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
|
|
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
|
|
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
|
|
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
|
|
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
Such data are easy to generate, e.g. in Ruby with
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
puts "host=#{hostname},seconds=#{t2-t1},message=#{msg}"
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
puts mymap.collect{|k,v| "#{k}=#{v}"}.join(',')
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
or <tt>print</tt> statements in various languages, e.g.
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
echo "type=3,user=$USER,date=$date\n";
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
logger.log("type=3,user=$USER,date=$date\n");
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>Fields lacking an IPS will have positional index (starting at 1) used as
|
|
the key, as in NIDX format. For example, <tt>dish=7,egg=8,flint</tt> is parsed
|
|
as <tt>"dish" => "7", "egg" => "8", "3" => "flint"</tt> and
|
|
<tt>dish,egg,flint</tt> is parsed as <tt>"1" => "dish", "2" => "egg", "3"
|
|
=> "flint"</tt>.
|
|
|
|
<p/> As discussed in <a href="record-heterogeneity.html">Record-heterogeneity</a>,
|
|
Miller handles changes of field names within the same data stream. But using
|
|
DKVP format this is particularly natural. One of my favorite use-cases for
|
|
Miller is in application/server logs, where I log all sorts of lines such as
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
resource=/path/to/file,loadsec=0.45,ok=true
|
|
record_count=100, resource=/path/to/file
|
|
resource=/some/other/path,loadsec=0.97,ok=false
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
etc. and I just log them as needed. Then later, I can use <tt>grep</tt>, <tt>mlr --opprint group-like</tt>, etc.
|
|
to analyze my logs.
|
|
|
|
<p/>See <a href="reference.html">Reference</a> regarding how to specify separators other than
|
|
the default equals-sign and comma.
|
|
|
|
</div>
|
|
<a id="NIDX:_Index-numbered_(toolkit_style)"/><h1>NIDX: Index-numbered (toolkit style)</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_nidx');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_nidx" style="display: block">
|
|
|
|
With <tt>--inidx --ifs ' ' --repifs</tt>, Miller splits lines on whitespace and
|
|
assigns integer field names starting with 1. This recapitulates Unix-toolkit
|
|
behavior.
|
|
|
|
<p/> Example with index-numbered output:
|
|
|
|
<table><tr> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/small
|
|
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
|
|
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
|
|
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
|
|
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
|
|
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --onidx --ofs ' ' cat data/small
|
|
pan pan 1 0.3467901443380824 0.7268028627434533
|
|
eks pan 2 0.7586799647899636 0.5221511083334797
|
|
wye wye 3 0.20460330576630303 0.33831852551664776
|
|
eks wye 4 0.38139939387114097 0.13418874328430463
|
|
wye pan 5 0.5732889198020006 0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> </tr></table>
|
|
|
|
<p/> Example with index-numbered input:
|
|
|
|
<table><tr> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/mydata.txt
|
|
oh say can you see
|
|
by the dawn's
|
|
early light
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --inidx --ifs ' ' --odkvp cat data/mydata.txt
|
|
1=oh,2=say,3=can,4=you,5=see
|
|
1=by,2=the,3=dawn's
|
|
1=early,2=light
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> </tr></table>
|
|
|
|
<p/> Example with index-numbered input and output:
|
|
|
|
<table><tr> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/mydata.txt
|
|
oh say can you see
|
|
by the dawn's
|
|
early light
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --nidx --fs ' ' --repifs cut -f 2,3 data/mydata.txt
|
|
say can
|
|
the dawn's
|
|
light
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> </tr></table>
|
|
|
|
</div>
|
|
<a id="Tabular_JSON"/><h1>Tabular JSON</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_json');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_json" style="display: block">
|
|
|
|
<p/>JSON is a format which supports arbitrarily deep nesting of
|
|
“objects” (hashmaps) and “arrays” (lists), while Miller
|
|
is a tool for handling <boldmaroon>tabular data</boldmaroon> only. This means
|
|
Miller cannot (and should not) handle arbitrary JSON. (Check out <a
|
|
href="http://stedolan.github.io/jq/">jq</a>.)
|
|
|
|
<p/>But if you have tabular data represented in JSON then Miller can handle that for you.
|
|
|
|
<a id="Single-level_JSON_objects"/><h2>Single-level JSON objects</h2>
|
|
|
|
An <boldmaroon>array of single-level objects</boldmaroon> is, quite simply,
|
|
<boldmaroon>a table:</boldmaroon>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --json head -n 2 then cut -f color,shape data/json-example-1.json
|
|
{ "color": "yellow", "shape": "triangle" }
|
|
{ "color": "red", "shape": "square" }
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --json --jvstack head -n 2 then cut -f color,u,v data/json-example-1.json
|
|
{
|
|
"color": "yellow",
|
|
"u": 0.6321695890307647,
|
|
"v": 0.9887207810889004
|
|
}
|
|
{
|
|
"color": "red",
|
|
"u": 0.21966833570651523,
|
|
"v": 0.001257332190235938
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --ijson --opprint stats1 -a mean,stddev,count -f u -g shape data/json-example-1.json
|
|
shape u_mean u_stddev u_count
|
|
triangle 0.583995 0.131184 3
|
|
square 0.409355 0.365428 4
|
|
circle 0.366013 0.209094 3
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Nested_JSON_objects"/><h2>Nested JSON objects</h2>
|
|
|
|
<p/>Additionally, Miller can <boldmaroon>tabularize nested objects by concatentating keys:</boldmaroon>
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --json --jvstack head -n 2 data/json-example-2.json
|
|
{
|
|
"flag": 1,
|
|
"i": 11,
|
|
"attributes": {
|
|
"color": "yellow",
|
|
"shape": "triangle"
|
|
},
|
|
"values": {
|
|
"u": 0.632170,
|
|
"v": 0.988721,
|
|
"w": 0.436498,
|
|
"x": 5.798188
|
|
}
|
|
}
|
|
{
|
|
"flag": 1,
|
|
"i": 15,
|
|
"attributes": {
|
|
"color": "red",
|
|
"shape": "square"
|
|
},
|
|
"values": {
|
|
"u": 0.219668,
|
|
"v": 0.001257,
|
|
"w": 0.792778,
|
|
"x": 2.944117
|
|
}
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --ijson --opprint head -n 4 data/json-example-2.json
|
|
flag i attributes:color attributes:shape values:u values:v values:w values:x
|
|
1 11 yellow triangle 0.632170 0.988721 0.436498 5.798188
|
|
1 15 red square 0.219668 0.001257 0.792778 2.944117
|
|
1 16 red circle 0.209017 0.290052 0.138103 5.065034
|
|
0 48 red square 0.956274 0.746720 0.775542 7.117831
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/>Note in particular that as far as Miller’s <tt>put</tt> and <tt>filter</tt>, as well as other
|
|
I/O formats, are concerned, these are simply field names with colons in them:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --json --jvstack head -n 1 then put '${values:uv} = ${values:u} * ${values:v}' data/json-example-2.json
|
|
{
|
|
"flag": 1,
|
|
"i": 11,
|
|
"attributes": {
|
|
"color": "yellow",
|
|
"shape": "triangle"
|
|
},
|
|
"values": {
|
|
"u": 0.632170,
|
|
"v": 0.988721,
|
|
"w": 0.436498,
|
|
"x": 5.798188,
|
|
"uv": 0.625040
|
|
}
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<a id="Arrays"/><h2>Arrays</h2>
|
|
|
|
<p/>Arrays aren’t supported in Miller’s
|
|
<tt>put</tt>/<tt>filter</tt> DSL. By default, JSON arrays are read in as
|
|
integer-keyed maps.
|
|
|
|
<p/> Suppose you have arrays like this in our input data:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ cat data/json-example-3.json
|
|
{
|
|
"label": "orange",
|
|
"values": [12.2, 13.8, 17.2]
|
|
}
|
|
{
|
|
"label": "purple",
|
|
"values": [27.0, 32.4]
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> Then integer indices (starting from 0 and counting up) are used as map keys:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --ijson --oxtab cat data/json-example-3.json
|
|
label orange
|
|
values:0 12.2
|
|
values:1 13.8
|
|
values:2 17.2
|
|
|
|
label purple
|
|
values:0 27.0
|
|
values:1 32.4
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> When the data are written back out as JSON, field names are re-expanded as above,
|
|
but what were arrays on input are now maps on output:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --json --jvstack cat data/json-example-3.json
|
|
{
|
|
"label": "orange",
|
|
"values": {
|
|
"0": 12.2,
|
|
"1": 13.8,
|
|
"2": 17.2
|
|
}
|
|
}
|
|
{
|
|
"label": "purple",
|
|
"values": {
|
|
"0": 27.0,
|
|
"1": 32.4
|
|
}
|
|
}
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
<p/> This is non-ideal, but it allows Miller (5.x release being latest as of
|
|
this writing) to handle JSON arrays at all.
|
|
|
|
<p/>You might also use <tt>mlr --json-skip-arrays-on-input</tt> or <tt>mlr
|
|
--json-fatal-arrays-on-input</tt>.
|
|
|
|
To truly handle JSON, please use a JSON-processing tool such as <a
|
|
href="http://stedolan.github.io/jq/">jq</a>.
|
|
|
|
<a id="Formatting_JSON_options"/><h2>Formatting JSON options</h2>
|
|
|
|
<p/>JSON isn’t a parameterized format, so <tt>RS</tt>, <tt>FS</tt>,
|
|
<tt>PS</tt> aren’t specifiable. Nonetheless, you can do the following:
|
|
|
|
<ul>
|
|
|
|
<li/>Use <tt>--jvstack</tt> to pretty-print JSON objects with multi-line
|
|
(vertically stacked) spacing. By defaulty, each Miller record (JSON object) is
|
|
one per line.
|
|
|
|
<li/>Use <tt>--jlistwrap</tt> to print the sequence of JSON objects wrapped in
|
|
an outermost <tt>[</tt> and <tt>]</tt>. By default, these aren’t printed.
|
|
|
|
<li/>Use <tt>--jquoteall</tt> to double-quote all object values. By default,
|
|
integers, floating-point numbers, and booleans <tt>true</tt> and <tt>false</tt>
|
|
are not double-quoted when they appear as JSON-object keys.
|
|
|
|
<li/>Use <tt>--jflatsep yourstringhere</tt> to specify the string used for
|
|
key concatenation: this defaults to a single colon.
|
|
|
|
</ul>
|
|
|
|
<p/>Again, please see <a href="http://stedolan.github.io/jq/">jq</a> for a
|
|
truly powerful, JSON-specific tool.
|
|
|
|
<a id="JSON_non-streaming"/><h2>JSON non-streaming</h2>
|
|
|
|
<p/>The JSON parser Miller uses does not return until all input is parsed: in
|
|
particular this means that, unlike for other file formats, Miller does not (at
|
|
present) handle JSON files in <tt>tail -f</tt> contexts.
|
|
|
|
</div>
|
|
<a id="PPRINT:_Pretty-printed_tabular"/><h1>PPRINT: Pretty-printed tabular</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_pprint');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_pprint" style="display: block">
|
|
|
|
<p/>
|
|
Miller’s pretty-print format is like CSV, but column-aligned. For example, compare
|
|
|
|
<table><tr><td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --ocsv cat data/small
|
|
a,b,i,x,y
|
|
pan,pan,1,0.3467901443380824,0.7268028627434533
|
|
eks,pan,2,0.7586799647899636,0.5221511083334797
|
|
wye,wye,3,0.20460330576630303,0.33831852551664776
|
|
eks,wye,4,0.38139939387114097,0.13418874328430463
|
|
wye,pan,5,0.5732889198020006,0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td>
|
|
<td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint cat data/small
|
|
a b i x y
|
|
pan pan 1 0.3467901443380824 0.7268028627434533
|
|
eks pan 2 0.7586799647899636 0.5221511083334797
|
|
wye wye 3 0.20460330576630303 0.33831852551664776
|
|
eks wye 4 0.38139939387114097 0.13418874328430463
|
|
wye pan 5 0.5732889198020006 0.8636244699032729
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td></tr></table>
|
|
|
|
Note that while Miller is a line-at-a-time processor and retains input lines in
|
|
memory only where necessary (e.g. for sort), pretty-print output requires it to
|
|
accumulate all input lines (so that it can compute maximum column widths)
|
|
before producing any output. This has two consequences: (a) pretty-print output
|
|
won’t work on <tt>tail -f</tt> contexts, where Miller will be waiting for
|
|
an end-of-file marker which never arrives; (b) pretty-print output for large
|
|
files is constrained by available machine memory.
|
|
|
|
<p/> See <a href="record-heterogeneity.html">Record-heterogeneity</a> for how Miller
|
|
handles changes of field names within a single data stream.
|
|
|
|
<p/> For output only (this isn’t supported in the input-scanner as of 5.0.0)
|
|
you can use <tt>--barred</tt> with pprint output format:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --opprint --barred cat data/small
|
|
+-----+-----+---+---------------------+---------------------+
|
|
| a | b | i | x | y |
|
|
+-----+-----+---+---------------------+---------------------+
|
|
| pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
|
|
| eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
|
|
| wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
|
|
| eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
|
|
| wye | pan | 5 | 0.5732889198020006 | 0.8636244699032729 |
|
|
+-----+-----+---+---------------------+---------------------+
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
</div>
|
|
<a id="XTAB:_Vertical_tabular"/><h1>XTAB: Vertical tabular</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_xtab');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_xtab" style="display: block">
|
|
|
|
<p/>This is perhaps most useful for looking a very wide and/or multi-column
|
|
data which causes line-wraps on the screen (but see also <a
|
|
href="https://github.com/twosigma/ngrid">https://github.com/twosigma/ngrid</a>
|
|
for an entirely different, very powerful option). Namely:
|
|
|
|
<table><tr> <td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ grep -v '^#' /etc/passwd | head -n 6 | mlr --nidx --fs : --opprint cat
|
|
1 2 3 4 5 6 7
|
|
nobody * -2 -2 Unprivileged User /var/empty /usr/bin/false
|
|
root * 0 0 System Administrator /var/root /bin/sh
|
|
daemon * 1 1 System Services /var/root /usr/bin/false
|
|
_uucp * 4 4 Unix to Unix Copy Protocol /var/spool/uucp /usr/sbin/uucico
|
|
_taskgated * 13 13 Task Gate Daemon /var/empty /usr/bin/false
|
|
_networkd * 24 24 Network Services /var/networkd /usr/bin/false
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td></tr> <tr><td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ grep -v '^#' /etc/passwd | head -n 2 | mlr --nidx --fs : --oxtab cat
|
|
1 nobody
|
|
2 *
|
|
3 -2
|
|
4 -2
|
|
5 Unprivileged User
|
|
6 /var/empty
|
|
7 /usr/bin/false
|
|
|
|
1 root
|
|
2 *
|
|
3 0
|
|
4 0
|
|
5 System Administrator
|
|
6 /var/root
|
|
7 /bin/sh
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td></tr> <tr><td>
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ grep -v '^#' /etc/passwd | head -n 2 | \
|
|
mlr --nidx --fs : --ojson --jvstack --jlistwrap label name,password,uid,gid,gecos,home_dir,shell
|
|
[
|
|
{
|
|
"name": "nobody",
|
|
"password": "*",
|
|
"uid": -2,
|
|
"gid": -2,
|
|
"gecos": "Unprivileged User",
|
|
"home_dir": "/var/empty",
|
|
"shell": "/usr/bin/false"
|
|
}
|
|
,{
|
|
"name": "root",
|
|
"password": "*",
|
|
"uid": 0,
|
|
"gid": 0,
|
|
"gecos": "System Administrator",
|
|
"home_dir": "/var/root",
|
|
"shell": "/bin/sh"
|
|
}
|
|
]
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
</td> </tr></table>
|
|
|
|
</div>
|
|
<a id="Markdown_tabular"/><h1>Markdown tabular</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_markdown_tabular');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_markdown_tabular" style="display: block">
|
|
|
|
<p/>Markdown format looks like this:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --omd cat data/small
|
|
| a | b | i | x | y |
|
|
| --- | --- | --- | --- | --- |
|
|
| pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
|
|
| eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
|
|
| wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
|
|
| eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
|
|
| wye | pan | 5 | 0.5732889198020006 | 0.8636244699032729 |
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
which renders like this when dropped into various web tools (e.g. github comments):
|
|
|
|
<p/>
|
|
<img src="pix/omd.png"/>
|
|
|
|
<p/> As of Miller 4.3.0, markdown format is supported only for output, not input.
|
|
|
|
</div>
|
|
<a id="Data-conversion_keystroke-savers"/><h1>Data-conversion keystroke-savers</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_x2y');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_x2y" style="display: block">
|
|
|
|
<p/>While you can do format conversion using <tt>mlr --icsv --ojson cat myfile.csv</tt>,
|
|
there are also keystroke-savers for this purpose, such as <tt>mlr --c2j cat myfile.csv</tt>.
|
|
For a complete list:
|
|
|
|
<p/>
|
|
<div class="pokipanel">
|
|
<pre>
|
|
$ mlr --usage-format-conversion-keystroke-saver-options
|
|
As keystroke-savers for format-conversion you may use the following:
|
|
--c2t --c2d --c2n --c2j --c2x --c2p --c2m
|
|
--t2c --t2d --t2n --t2j --t2x --t2p --t2m
|
|
--d2c --d2t --d2n --d2j --d2x --d2p --d2m
|
|
--n2c --n2t --n2d --n2j --n2x --n2p --n2m
|
|
--j2c --j2t --j2d --j2n --j2x --j2p --j2m
|
|
--x2c --x2t --x2d --x2n --x2j --x2p --x2m
|
|
--p2c --p2t --p2d --p2n --p2j --p2x --p2m
|
|
The letters c t d n j x p m refer to formats CSV, TSV, DKVP, NIDX, JSON, XTAB,
|
|
PPRINT, and markdown, respectively. Note that markdown format is available for
|
|
output only.
|
|
</pre>
|
|
</div>
|
|
<p/>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
<a id="Autodetect_of_line_endings"/><h1>Autodetect of line endings</h1>
|
|
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_autodetect_line_endings');" href="javascript:;">Toggle section visibility</button>
|
|
<div id="section_toggle_autodetect_line_endings" style="display: block">
|
|
|
|
<p/> Default line endings (<tt>--irs</tt> and <tt>--ors</tt>) are <tt>'auto'</tt>
|
|
which means <boldmaroon>autodetect from the input file format</boldmaroon>, as
|
|
long as the input file(s) have lines ending in either LF (also known as
|
|
linefeed, <tt>'\n'</tt>, <tt>0x0a</tt>, Unix-style) or CRLF (also known as
|
|
carriage-return/linefeed pairs, <tt>'\r\n'</tt>, <tt>0x0d 0x0a</tt>, Windows
|
|
style).
|
|
|
|
<p/> <boldmaroon>If both IRS and ORS are auto (which is the default) then LF input will
|
|
lead to LF output and CRLF input will lead to CRLF output, regardless of the
|
|
platform you’re running on</boldmaroon>.
|
|
|
|
<p/> The line-ending autodetector triggers on the first line ending detected in
|
|
the input stream. E.g. if you specify a CRLF-terminated file on the command
|
|
line followed by an LF-terminated file then autodetected line endings will be
|
|
CRLF.
|
|
|
|
<p/> If you use <tt>--ors {something else}</tt> with (default or explicitly
|
|
specified) <tt>--irs auto</tt> then line endings are autodetected on input and
|
|
set to what you specify on output.
|
|
|
|
<p/> If you use <tt>--irs {something else}</tt> with (default or explicitly
|
|
specified) <tt>--ors auto</tt> then the output line endings used are LF on
|
|
Unix/Linux/BSD/MacOSX, and CRLF on Windows.
|
|
|
|
<p/> See also
|
|
<a href="reference.html#Record/field/pair_separators">Reference</a> for
|
|
more information about record/field/pair separators.
|
|
|
|
</div>
|
|
</div>
|
|
</td>
|
|
|
|
</table>
|
|
</body>
|
|
</html>
|