Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-20 18:10:07 +00:00
Code Issues Projects Releases Wiki Activity
miller/doc/file-formats.html
johnkerl e5c43cb4da fix 
missing data-sharing.html link
2019-09-17 22:39:56 -04:00

1118 lines
36 KiB
HTML
Raw Blame History

<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<!-- PAGE GENERATED FROM template.html and content-for-file-formats.html BY poki. -->
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
<head>
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
<meta name="description" content="Miller documentation"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
<meta name="keywords"
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
<title> File formats </title>
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
</head>
<!-- ================================================================ -->
<script type="text/javascript">
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type="text/javascript">
try {
var pageTracker = _gat._getTracker("UA-15651652-1");
pageTracker._trackPageview();
} catch(err) {}
</script>
<!-- ================================================================ -->
<script type="text/javascript">
function vis_expand_all_navbar_sections() {
//console.log('vis_expand_all_navbar_sections ENTER');
vis_expand_all_with_prefix('navbar_section_toggle_');
//console.log('vis_expand_all_navbar_sections EXIT');
}
function vis_collapse_all_navbar_sections() {
//console.log('vis_collapse_all_navbar_sections ENTER');
vis_collapse_all_with_prefix('navbar_section_toggle_');
//console.log('vis_collapse_all_navbar_sections EXIT');
}
function vis_expand_all_body_sections() {
//console.log('vis_expand_all_body_sections ENTER');
vis_expand_all_with_prefix('body_section_toggle_');
//console.log('vis_expand_all_body_sections EXIT');
}
function vis_collapse_all_body_sections() {
//console.log('vis_collapse_all_body_sections ENTER');
vis_collapse_all_with_prefix('body_section_toggle_');
//console.log('vis_collapse_all_body_sections EXIT');
}
function vis_expand_all_with_prefix(prefix) {
//console.log('vis_expand_all_with_prefix ENTER', prefix);
var divs = document.getElementsByTagName('div');
console.log("DIVS", divs);
for(var i = 0; i < divs.length; i++) {
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_expand_div(div);
}
}
//console.log('vis_expand_all_with_prefix EXIT', prefix);
}
function vis_collapse_all_with_prefix(prefix) {
var divs = document.getElementsByTagName('div');
for(var i = 0; i < divs.length; i++){
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_collapse_div(div);
}
}
}
function vis_toggle_by_name(divName) {
vis_toggle_div(document.getElementById(divName));
}
function vis_expand_by_name(divName) {
vis_expand_div(document.getElementById(divName));
}
function vis_collapse_by_name(divName) {
vis_collapse_div(document.getElementById(divName));
}
function vis_toggle_div(div) {
if (div != null) {
var state = div.style.display;
if (state == 'block') {
div.style.display = 'none';
} else {
div.style.display = 'block';
}
}
}
function vis_expand_div(div) {
if (div != null) {
div.style.display = 'block';
}
}
function vis_collapse_div(div) {
if (div != null) {
div.style.display = 'none';
}
}
</script>
<body bgcolor="#ffffff">
<!-- ================================================================ -->
<!-- navbar -->
<div class="pokinav">
<center><titleinbody>Miller</titleinbody></center>
<!-- NAVBAR GENERATED FROM template.html BY poki -->
<button
style="font-weight:bold;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
href="javascript:;"
>
Overview
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
href="javascript:;"
>
Using
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
href="javascript:;"
>
Reference
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
href="javascript:;"
>
Background
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
href="javascript:;"
>
Repository
</button>
<div id="navbar_section_toggle_overview" style="display: block;text-align: left">
<br/><a href="index.html">About</a>
<br/><a href="10-min.html">10-minute intro</a>
<br/><a href="file-formats.html"><b>File formats</b></a>
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
<br/><a href="record-heterogeneity.html">Record-hetrogeneity</a>
<br/><a href="internationalization.html">Internationalization</a>
</div>
<div id="navbar_section_toggle_using" style="display: none;text-align: left">
<br/><a href="faq.html">FAQ</a>
<br/><a href="10-min.html">10-minute intro</a>
<br/><a href="data-sharing.html">Mixing with other languages</a>
<br/><a href="cookbook.html">Cookbook part 1</a>
<br/><a href="cookbook2.html">Cookbook part 2</a>
<br/><a href="cookbook3.html">Cookbook part 3</a>
<br/><a href="data-examples.html">Data-diving examples</a>
</div>
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
<br/><a href="reference.html">Main reference</a>
<br/><a href="reference-verbs.html">Verbs reference</a>
<br/><a href="reference-dsl.html">DSL reference</a>
<br/><a href="manpage.html">Manpage</a>
<br/><a href="release-docs.html">Documents by release</a>
<br/><a href="build.html">Installation</a>
</div>
<div id="navbar_section_toggle_background" style="display: none;text-align: left">
<br/><a href="why.html">Why?</a>
<br/><a href="whyc.html">Why C?</a>
<br/><a href="etymology.html">Why call it Miller?</a>
<br/><a href="originality.html">How original is Miller?</a>
<br/><a href="performance.html">Performance</a>
</div>
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
<br/><a href="contact.html">Contact information</a>
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
</div>
</div>
<!-- page body -->
<p/>
<!-- BODY COPIED FROM content-for-file-formats.html BY poki -->
<div class="pokitoc">
<center><titleinbody>File formats</titleinbody></center>
&bull;&nbsp;<a href="#Overview">Overview</a><br/>
&bull;&nbsp;<a href="#Examples">Examples</a><br/>
&bull;&nbsp;<a href="#CSV/TSV/ASV/USV/etc.">CSV/TSV/ASV/USV/etc.</a><br/>
&bull;&nbsp;<a href="#DKVP:_Key-value_pairs">DKVP: Key-value pairs</a><br/>
&bull;&nbsp;<a href="#NIDX:_Index-numbered_(toolkit_style)">NIDX: Index-numbered (toolkit style)</a><br/>
&bull;&nbsp;<a href="#Tabular_JSON">Tabular JSON</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#Single-level_JSON_objects">Single-level JSON objects</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#Nested_JSON_objects">Nested JSON objects</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#Arrays">Arrays</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#Formatting_JSON_options">Formatting JSON options</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#JSON_non-streaming">JSON non-streaming</a><br/>
&bull;&nbsp;<a href="#PPRINT:_Pretty-printed_tabular">PPRINT: Pretty-printed tabular</a><br/>
&bull;&nbsp;<a href="#XTAB:_Vertical_tabular">XTAB: Vertical tabular</a><br/>
&bull;&nbsp;<a href="#Markdown_tabular">Markdown tabular</a><br/>
&bull;&nbsp;<a href="#Data-conversion_keystroke-savers">Data-conversion keystroke-savers</a><br/>
&bull;&nbsp;<a href="#Autodetect_of_line_endings">Autodetect of line endings</a><br/>
&bull;&nbsp;<a href="#Comments_in_data">Comments in data</a><br/>
</div>
<p/>
<p/>
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_expand_all_body_sections();" href="javascript:;">Expand all sections</button>
<button style="font-weight:bold;color:maroon;border:0" onclick="vis_collapse_all_body_sections();" href="javascript:;">Collapse all sections</button>
<a id="Overview"/><h1>Overview</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_overview');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_overview" style="display: block">
<p/>Miller handles name-indexed data using several formats: some you probably
know by name, such as CSV, TSV, and JSON &mdash; and other formats you&rsquo;re
likely already seeing and using in your structured data. Additionally, Miller gives
you the option of including comments within your data.
</div>
<a id="Examples"/><h1>Examples</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_examples');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_examples" style="display: block">
<p/>
<div class="pokipanel">
<pre>
$ mlr --usage-data-format-examples
DKVP: delimited key-value pairs (Miller default format)
+---------------------+
| apple=1,bat=2,cog=3 | Record 1: "apple" =&gt; "1", "bat" =&gt; "2", "cog" =&gt; "3"
| dish=7,egg=8,flint | Record 2: "dish" =&gt; "7", "egg" =&gt; "8", "3" =&gt; "flint"
+---------------------+
NIDX: implicitly numerically indexed (Unix-toolkit style)
+---------------------+
| the quick brown | Record 1: "1" =&gt; "the", "2" =&gt; "quick", "3" =&gt; "brown"
| fox jumped | Record 2: "1" =&gt; "fox", "2" =&gt; "jumped"
+---------------------+
CSV/CSV-lite: comma-separated values with separate header line
+---------------------+
| apple,bat,cog |
| 1,2,3 | Record 1: "apple =&gt; "1", "bat" =&gt; "2", "cog" =&gt; "3"
| 4,5,6 | Record 2: "apple" =&gt; "4", "bat" =&gt; "5", "cog" =&gt; "6"
+---------------------+
Tabular JSON: nested objects are supported, although arrays within them are not:
+---------------------+
| { |
| "apple": 1, | Record 1: "apple" =&gt; "1", "bat" =&gt; "2", "cog" =&gt; "3"
| "bat": 2, |
| "cog": 3 |
| } |
| { |
| "dish": { | Record 2: "dish:egg" =&gt; "7", "dish:flint" =&gt; "8", "garlic" =&gt; ""
| "egg": 7, |
| "flint": 8 |
| }, |
| "garlic": "" |
| } |
+---------------------+
PPRINT: pretty-printed tabular
+---------------------+
| apple bat cog |
| 1 2 3 | Record 1: "apple =&gt; "1", "bat" =&gt; "2", "cog" =&gt; "3"
| 4 5 6 | Record 2: "apple" =&gt; "4", "bat" =&gt; "5", "cog" =&gt; "6"
+---------------------+
XTAB: pretty-printed transposed tabular
+---------------------+
| apple 1 | Record 1: "apple" =&gt; "1", "bat" =&gt; "2", "cog" =&gt; "3"
| bat 2 |
| cog 3 |
| |
| dish 7 | Record 2: "dish" =&gt; "7", "egg" =&gt; "8"
| egg 8 |
+---------------------+
Markdown tabular (supported for output only):
+-----------------------+
| | apple | bat | cog | |
| | --- | --- | --- | |
| | 1 | 2 | 3 | | Record 1: "apple =&gt; "1", "bat" =&gt; "2", "cog" =&gt; "3"
| | 4 | 5 | 6 | | Record 2: "apple" =&gt; "4", "bat" =&gt; "5", "cog" =&gt; "6"
+-----------------------+
</pre>
</div>
<p/>
</div>
<a id="CSV/TSV/ASV/USV/etc."/><h1>CSV/TSV/ASV/USV/etc.</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_csv_tsv_etc');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_csv_tsv_etc" style="display: block">
<p/>
When <code>mlr</code> is invoked with the <code>--csv</code> or <code>--csvlite</code> option,
key names are found on the first record and values are taken from subsequent
records. This includes the case of CSV-formatted files. See
<a href="record-heterogeneity.html">Record-hetrogeneity</a> for how Miller handles
changes of field names within a single data stream.
<p/> Miller has record separator <code>RS</code> and field separator <code>FS</code>,
just as <code>awk</code> does. For TSV, use <code>--fs tab</code>; to convert TSV to
CSV, use <code>--ifs tab --ofs comma</code>, etc. (See also
<a href="reference.html#Record/field/pair_separators">Main reference</a>.)
<p/> <b>TSV (tab-separated values):</b> the following are synonymous pairs:
<ul>
<li/> <code>--tsv</code> and <code>--csv --fs tab</code>
<li/> <code>--itsv</code> and <code>--icsv --ifs tab</code>
<li/> <code>--otsv</code> and <code>--ocsv --ofs tab</code>
<li/> <code>--tsvlite</code> and <code>--csvlite --fs tab</code>
<li/> <code>--itsvlite</code> and <code>--icsvlite --ifs tab</code>
<li/> <code>--otsvlite</code> and <code>--ocsvlite --ofs tab</code>
</ul>
<p/> <b>ASV (ASCII-separated values):</b> the flags
<code>--asv</code>,
<code>--iasv</code>,
<code>--oasv</code>,
<code>--asvlite</code>,
<code>--iasvlite</code>, and
<code>--oasvlite</code> are analogous except they use ASCII FS and RS 0x1f and 0x1e, respectively.
<p/> <b>USV (Unicode-separated values):</b> likewise, the flags
<code>--usv</code>,
<code>--iusv</code>,
<code>--ousv</code>,
<code>--usvlite</code>,
<code>--iusvlite</code>, and
<code>--ousvlite</code> use Unicode FS and RS U+241F (UTF-8 0x0xe2909f) and U+241E (UTF-8 0xe2909e), respectively.
<p/>Miller&rsquo;s <code>--csv</code> flag supports RFC-4180 CSV (<a href="https://tools.ietf.org/html/rfc4180">
https://tools.ietf.org/html/rfc4180</a>). This includes CRLF line-terminators by default, regardless
of platform.
<p/>Here are the differences between CSV and CSV-lite:
<ul>
<li/>CSV supports <a href="https://tools.ietf.org/html/rfc4180">
RFC-4180</a>)-style double-quoting, including the ability to have commas and/or
LF/CRLF line-endings contained within an input field; CSV-lite does not.
<li/>CSV does not allow heterogeneous data; CSV-lite does (see also <a
href="record-heterogeneity.html">here</a>).
<li/>The CSV-lite input-reading code is fractionally more efficient than the CSV
input-reader.
</ul>
<p/>Here are things they have in common:
<ul>
<li/>The ability to specify record/field separators other than the default,
e.g. CR-LF vs. LF, or tab instead of comma for TSV, and so on.
<li/>The <code>--implicit-csv-header</code> flag for input and the
<code>--headerless-csv-output</code> flag for output.
</ul>
</div>
<a id="DKVP:_Key-value_pairs"/><h1>DKVP: Key-value pairs</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_dkvp');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_dkvp" style="display: block">
<p/>
Miller&rsquo;s default file format is DKVP, for <b>delimited key-value pairs</b>. Example:
<p/>
<div class="pokipanel">
<pre>
$ mlr cat data/small
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
</pre>
</div>
<p/>
Such data are easy to generate, e.g. in Ruby with
<p/>
<div class="pokipanel">
<pre>
puts "host=#{hostname},seconds=#{t2-t1},message=#{msg}"
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
puts mymap.collect{|k,v| "#{k}=#{v}"}.join(',')
</pre>
</div>
<p/>
or <code>print</code> statements in various languages, e.g.
<p/>
<div class="pokipanel">
<pre>
echo "type=3,user=$USER,date=$date\n";
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
logger.log("type=3,user=$USER,date=$date\n");
</pre>
</div>
<p/>
<p/>Fields lacking an IPS will have positional index (starting at 1) used as
the key, as in NIDX format. For example, <code>dish=7,egg=8,flint</code> is parsed
as <code>"dish" =&gt; "7", "egg" =&gt; "8", "3" =&gt; "flint"</code> and
<code>dish,egg,flint</code> is parsed as <code>"1" =&gt; "dish", "2" =&gt; "egg", "3"
=&gt; "flint"</code>.
<p/> As discussed in <a href="record-heterogeneity.html">Record-hetrogeneity</a>,
Miller handles changes of field names within the same data stream. But using
DKVP format this is particularly natural. One of my favorite use-cases for
Miller is in application/server logs, where I log all sorts of lines such as
<p/>
<div class="pokipanel">
<pre>
resource=/path/to/file,loadsec=0.45,ok=true
record_count=100, resource=/path/to/file
resource=/some/other/path,loadsec=0.97,ok=false
</pre>
</div>
<p/>
etc. and I just log them as needed. Then later, I can use <code>grep</code>, <code>mlr --opprint group-like</code>, etc.
to analyze my logs.
<p/>See <a href="reference.html">Main reference</a> regarding how to specify separators other than
the default equals-sign and comma.
</div>
<a id="NIDX:_Index-numbered_(toolkit_style)"/><h1>NIDX: Index-numbered (toolkit style)</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_nidx');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_nidx" style="display: block">
With <code>--inidx --ifs ' ' --repifs</code>, Miller splits lines on whitespace and
assigns integer field names starting with 1. This recapitulates Unix-toolkit
behavior.
<p/> Example with index-numbered output:
<table><tr> <td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/small
a=pan,b=pan,i=1,x=0.3467901443380824,y=0.7268028627434533
a=eks,b=pan,i=2,x=0.7586799647899636,y=0.5221511083334797
a=wye,b=wye,i=3,x=0.20460330576630303,y=0.33831852551664776
a=eks,b=wye,i=4,x=0.38139939387114097,y=0.13418874328430463
a=wye,b=pan,i=5,x=0.5732889198020006,y=0.8636244699032729
</pre>
</div>
<p/>
</td> <td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --onidx --ofs ' ' cat data/small
pan pan 1 0.3467901443380824 0.7268028627434533
eks pan 2 0.7586799647899636 0.5221511083334797
wye wye 3 0.20460330576630303 0.33831852551664776
eks wye 4 0.38139939387114097 0.13418874328430463
wye pan 5 0.5732889198020006 0.8636244699032729
</pre>
</div>
<p/>
</td> </tr></table>
<p/> Example with index-numbered input:
<table><tr> <td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/mydata.txt
oh say can you see
by the dawn's
early light
</pre>
</div>
<p/>
</td> <td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --inidx --ifs ' ' --odkvp cat data/mydata.txt
1=oh,2=say,3=can,4=you,5=see
1=by,2=the,3=dawn's
1=early,2=light
</pre>
</div>
<p/>
</td> </tr></table>
<p/> Example with index-numbered input and output:
<table><tr> <td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/mydata.txt
oh say can you see
by the dawn's
early light
</pre>
</div>
<p/>
</td> <td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --nidx --fs ' ' --repifs cut -f 2,3 data/mydata.txt
say can
the dawn's
light
</pre>
</div>
<p/>
</td> </tr></table>
</div>
<a id="Tabular_JSON"/><h1>Tabular JSON</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_json');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_json" style="display: block">
<p/>JSON is a format which supports arbitrarily deep nesting of
&ldquo;objects&rdquo; (hashmaps) and &ldquo;arrays&rdquo; (lists), while Miller
is a tool for handling <boldmaroon>tabular data</boldmaroon> only. This means
Miller cannot (and should not) handle arbitrary JSON. (Check out <a
href="http://stedolan.github.io/jq/">jq</a>.)
<p/>But if you have tabular data represented in JSON then Miller can handle that for you.
<a id="Single-level_JSON_objects"/><h2>Single-level JSON objects</h2>
An <boldmaroon>array of single-level objects</boldmaroon> is, quite simply,
<boldmaroon>a table:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --json head -n 2 then cut -f color,shape data/json-example-1.json
{ "color": "yellow", "shape": "triangle" }
{ "color": "red", "shape": "square" }
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --json --jvstack head -n 2 then cut -f color,u,v data/json-example-1.json
{
"color": "yellow",
"u": 0.6321695890307647,
"v": 0.9887207810889004
}
{
"color": "red",
"u": 0.21966833570651523,
"v": 0.001257332190235938
}
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --ijson --opprint stats1 -a mean,stddev,count -f u -g shape data/json-example-1.json
shape u_mean u_stddev u_count
triangle 0.583995 0.131184 3
square 0.409355 0.365428 4
circle 0.366013 0.209094 3
</pre>
</div>
<p/>
<a id="Nested_JSON_objects"/><h2>Nested JSON objects</h2>
<p/>Additionally, Miller can <boldmaroon>tabularize nested objects by concatentating keys:</boldmaroon>
<p/>
<div class="pokipanel">
<pre>
$ mlr --json --jvstack head -n 2 data/json-example-2.json
{
"flag": 1,
"i": 11,
"attributes": {
"color": "yellow",
"shape": "triangle"
},
"values": {
"u": 0.632170,
"v": 0.988721,
"w": 0.436498,
"x": 5.798188
}
}
{
"flag": 1,
"i": 15,
"attributes": {
"color": "red",
"shape": "square"
},
"values": {
"u": 0.219668,
"v": 0.001257,
"w": 0.792778,
"x": 2.944117
}
}
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --ijson --opprint head -n 4 data/json-example-2.json
flag i attributes:color attributes:shape values:u values:v values:w values:x
1 11 yellow triangle 0.632170 0.988721 0.436498 5.798188
1 15 red square 0.219668 0.001257 0.792778 2.944117
1 16 red circle 0.209017 0.290052 0.138103 5.065034
0 48 red square 0.956274 0.746720 0.775542 7.117831
</pre>
</div>
<p/>
<p/>Note in particular that as far as Miller&rsquo;s <code>put</code> and <code>filter</code>, as well as other
I/O formats, are concerned, these are simply field names with colons in them:
<p/>
<div class="pokipanel">
<pre>
$ mlr --json --jvstack head -n 1 then put '${values:uv} = ${values:u} * ${values:v}' data/json-example-2.json
{
"flag": 1,
"i": 11,
"attributes": {
"color": "yellow",
"shape": "triangle"
},
"values": {
"u": 0.632170,
"v": 0.988721,
"w": 0.436498,
"x": 5.798188,
"uv": 0.625040
}
}
</pre>
</div>
<p/>
<a id="Arrays"/><h2>Arrays</h2>
<p/>Arrays aren&rsquo;t supported in Miller&rsquo;s
<code>put</code>/<code>filter</code> DSL. By default, JSON arrays are read in as
integer-keyed maps.
<p/> Suppose you have arrays like this in our input data:
<p/>
<div class="pokipanel">
<pre>
$ cat data/json-example-3.json
{
"label": "orange",
"values": [12.2, 13.8, 17.2]
}
{
"label": "purple",
"values": [27.0, 32.4]
}
</pre>
</div>
<p/>
<p/> Then integer indices (starting from 0 and counting up) are used as map keys:
<p/>
<div class="pokipanel">
<pre>
$ mlr --ijson --oxtab cat data/json-example-3.json
label orange
values:0 12.2
values:1 13.8
values:2 17.2
label purple
values:0 27.0
values:1 32.4
</pre>
</div>
<p/>
<p/> When the data are written back out as JSON, field names are re-expanded as above,
but what were arrays on input are now maps on output:
<p/>
<div class="pokipanel">
<pre>
$ mlr --json --jvstack cat data/json-example-3.json
{
"label": "orange",
"values": {
"0": 12.2,
"1": 13.8,
"2": 17.2
}
}
{
"label": "purple",
"values": {
"0": 27.0,
"1": 32.4
}
}
</pre>
</div>
<p/>
<p/> This is non-ideal, but it allows Miller (5.x release being latest as of
this writing) to handle JSON arrays at all.
<p/>You might also use <code>mlr --json-skip-arrays-on-input</code> or <code>mlr
--json-fatal-arrays-on-input</code>.
To truly handle JSON, please use a JSON-processing tool such as <a
href="http://stedolan.github.io/jq/">jq</a>.
<a id="Formatting_JSON_options"/><h2>Formatting JSON options</h2>
<p/>JSON isn&rsquo;t a parameterized format, so <code>RS</code>, <code>FS</code>,
<code>PS</code> aren&rsquo;t specifiable. Nonetheless, you can do the following:
<ul>
<li/>Use <code>--jvstack</code> to pretty-print JSON objects with multi-line
(vertically stacked) spacing. By defaulty, each Miller record (JSON object) is
one per line.
<li/>Use <code>--jlistwrap</code> to print the sequence of JSON objects wrapped in
an outermost <code>[</code> and <code>]</code>. By default, these aren&rsquo;t printed.
<li/>Use <code>--jquoteall</code> to double-quote all object values. By default,
integers, floating-point numbers, and booleans <code>true</code> and <code>false</code>
are not double-quoted when they appear as JSON-object keys.
<li/>Use <code>--jflatsep yourstringhere</code> to specify the string used for
key concatenation: this defaults to a single colon.
<li/>Use <code>--jofmt</code> to force Miller to apply the global <code>--ofmt</code>
to floating-point values. First note: please use sprintf-style codes for
double precision, e.g. ending in <code>%lf</code>, <code>%le</code>, or <code>%lg</code>.
Miller floats are double-precision so behavior using <code>%f</code>, <code>%d</code>,
etc. is undefined. Second note: <code>0.123</code> is valid JSON; <code>.123</code> is
not. Thus this feature allows you to emit JSON which may be unparseable by
other tools.
</ul>
<p/>Again, please see <a href="http://stedolan.github.io/jq/">jq</a> for a
truly powerful, JSON-specific tool.
<a id="JSON_non-streaming"/><h2>JSON non-streaming</h2>
<p/>The JSON parser Miller uses does not return until all input is parsed: in
particular this means that, unlike for other file formats, Miller does not (at
present) handle JSON files in <code>tail -f</code> contexts.
</div>
<a id="PPRINT:_Pretty-printed_tabular"/><h1>PPRINT: Pretty-printed tabular</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_pprint');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_pprint" style="display: block">
<p/>
Miller&rsquo;s pretty-print format is like CSV, but column-aligned. For example, compare
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --ocsv cat data/small
a,b,i,x,y
pan,pan,1,0.3467901443380824,0.7268028627434533
eks,pan,2,0.7586799647899636,0.5221511083334797
wye,wye,3,0.20460330576630303,0.33831852551664776
eks,wye,4,0.38139939387114097,0.13418874328430463
wye,pan,5,0.5732889198020006,0.8636244699032729
</pre>
</div>
<p/>
</td>
<td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint cat data/small
a b i x y
pan pan 1 0.3467901443380824 0.7268028627434533
eks pan 2 0.7586799647899636 0.5221511083334797
wye wye 3 0.20460330576630303 0.33831852551664776
eks wye 4 0.38139939387114097 0.13418874328430463
wye pan 5 0.5732889198020006 0.8636244699032729
</pre>
</div>
<p/>
</td></tr></table>
Note that while Miller is a line-at-a-time processor and retains input lines in
memory only where necessary (e.g. for sort), pretty-print output requires it to
accumulate all input lines (so that it can compute maximum column widths)
before producing any output. This has two consequences: (a) pretty-print output
won&rsquo;t work on <code>tail -f</code> contexts, where Miller will be waiting for
an end-of-file marker which never arrives; (b) pretty-print output for large
files is constrained by available machine memory.
<p/> See <a href="record-heterogeneity.html">Record-hetrogeneity</a> for how Miller
handles changes of field names within a single data stream.
<p/> For output only (this isn&rsquo;t supported in the input-scanner as of 5.0.0)
you can use <code>--barred</code> with pprint output format:
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint --barred cat data/small
+-----+-----+---+---------------------+---------------------+
| a | b | i | x | y |
+-----+-----+---+---------------------+---------------------+
| pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
| eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
| wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
| eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
| wye | pan | 5 | 0.5732889198020006 | 0.8636244699032729 |
+-----+-----+---+---------------------+---------------------+
</pre>
</div>
<p/>
</div>
<a id="XTAB:_Vertical_tabular"/><h1>XTAB: Vertical tabular</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_xtab');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_xtab" style="display: block">
<p/>This is perhaps most useful for looking a very wide and/or multi-column
data which causes line-wraps on the screen (but see also <a
href="https://github.com/twosigma/ngrid">https://github.com/twosigma/ngrid</a>
for an entirely different, very powerful option). Namely:
<table><tr> <td>
<p/>
<div class="pokipanel">
<pre>
$ grep -v '^#' /etc/passwd | head -n 6 | mlr --nidx --fs : --opprint cat
1 2 3 4 5 6 7
nobody * -2 -2 Unprivileged User /var/empty /usr/bin/false
root * 0 0 System Administrator /var/root /bin/sh
daemon * 1 1 System Services /var/root /usr/bin/false
_uucp * 4 4 Unix to Unix Copy Protocol /var/spool/uucp /usr/sbin/uucico
_taskgated * 13 13 Task Gate Daemon /var/empty /usr/bin/false
_networkd * 24 24 Network Services /var/networkd /usr/bin/false
</pre>
</div>
<p/>
</td></tr> <tr><td>
<p/>
<div class="pokipanel">
<pre>
$ grep -v '^#' /etc/passwd | head -n 2 | mlr --nidx --fs : --oxtab cat
1 nobody
2 *
3 -2
4 -2
5 Unprivileged User
6 /var/empty
7 /usr/bin/false
1 root
2 *
3 0
4 0
5 System Administrator
6 /var/root
7 /bin/sh
</pre>
</div>
<p/>
</td></tr> <tr><td>
<p/>
<div class="pokipanel">
<pre>
$ grep -v '^#' /etc/passwd | head -n 2 | \
mlr --nidx --fs : --ojson --jvstack --jlistwrap label name,password,uid,gid,gecos,home_dir,shell
[
{
"name": "nobody",
"password": "*",
"uid": -2,
"gid": -2,
"gecos": "Unprivileged User",
"home_dir": "/var/empty",
"shell": "/usr/bin/false"
}
,{
"name": "root",
"password": "*",
"uid": 0,
"gid": 0,
"gecos": "System Administrator",
"home_dir": "/var/root",
"shell": "/bin/sh"
}
]
</pre>
</div>
<p/>
</td> </tr></table>
</div>
<a id="Markdown_tabular"/><h1>Markdown tabular</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_markdown_tabular');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_markdown_tabular" style="display: block">
<p/>Markdown format looks like this:
<p/>
<div class="pokipanel">
<pre>
$ mlr --omd cat data/small
| a | b | i | x | y |
| --- | --- | --- | --- | --- |
| pan | pan | 1 | 0.3467901443380824 | 0.7268028627434533 |
| eks | pan | 2 | 0.7586799647899636 | 0.5221511083334797 |
| wye | wye | 3 | 0.20460330576630303 | 0.33831852551664776 |
| eks | wye | 4 | 0.38139939387114097 | 0.13418874328430463 |
| wye | pan | 5 | 0.5732889198020006 | 0.8636244699032729 |
</pre>
</div>
<p/>
which renders like this when dropped into various web tools (e.g. github comments):
<p/>
<img src="pix/omd.png"/>
<p/> As of Miller 4.3.0, markdown format is supported only for output, not input.
</div>
<a id="Data-conversion_keystroke-savers"/><h1>Data-conversion keystroke-savers</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_x2y');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_x2y" style="display: block">
<p/>While you can do format conversion using <code>mlr --icsv --ojson cat myfile.csv</code>,
there are also keystroke-savers for this purpose, such as <code>mlr --c2j cat myfile.csv</code>.
For a complete list:
<p/>
<div class="pokipanel">
<pre>
$ mlr --usage-format-conversion-keystroke-saver-options
As keystroke-savers for format-conversion you may use the following:
--c2t --c2d --c2n --c2j --c2x --c2p --c2m
--t2c --t2d --t2n --t2j --t2x --t2p --t2m
--d2c --d2t --d2n --d2j --d2x --d2p --d2m
--n2c --n2t --n2d --n2j --n2x --n2p --n2m
--j2c --j2t --j2d --j2n --j2x --j2p --j2m
--x2c --x2t --x2d --x2n --x2j --x2p --x2m
--p2c --p2t --p2d --p2n --p2j --p2x --p2m
The letters c t d n j x p m refer to formats CSV, TSV, DKVP, NIDX, JSON, XTAB,
PPRINT, and markdown, respectively. Note that markdown format is available for
output only.
</pre>
</div>
<p/>
</div>
</div>
<a id="Autodetect_of_line_endings"/><h1>Autodetect of line endings</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_autodetect_line_endings');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_autodetect_line_endings" style="display: block">
<p/> Default line endings (<code>--irs</code> and <code>--ors</code>) are <code>'auto'</code>
which means <boldmaroon>autodetect from the input file format</boldmaroon>, as
long as the input file(s) have lines ending in either LF (also known as
linefeed, <code>'\n'</code>, <code>0x0a</code>, Unix-style) or CRLF (also known as
carriage-return/linefeed pairs, <code>'\r\n'</code>, <code>0x0d 0x0a</code>, Windows
style).
<p/> <boldmaroon>If both IRS and ORS are auto (which is the default) then LF input will
lead to LF output and CRLF input will lead to CRLF output, regardless of the
platform you&rsquo;re running on</boldmaroon>.
<p/> The line-ending autodetector triggers on the first line ending detected in
the input stream. E.g. if you specify a CRLF-terminated file on the command
line followed by an LF-terminated file then autodetected line endings will be
CRLF.
<p/> If you use <code>--ors {something else}</code> with (default or explicitly
specified) <code>--irs auto</code> then line endings are autodetected on input and
set to what you specify on output.
<p/> If you use <code>--irs {something else}</code> with (default or explicitly
specified) <code>--ors auto</code> then the output line endings used are LF on
Unix/Linux/BSD/MacOSX, and CRLF on Windows.
<p/> See also
<a href="reference.html#Record/field/pair_separators">Main reference</a> for
more information about record/field/pair separators.
</div>
<a id="Comments_in_data"/><h1>Comments in data</h1>
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="vis_toggle_by_name('body_section_toggle_comments_in_data');" href="javascript:;">Toggle section visibility</button>
<div id="body_section_toggle_comments_in_data" style="display: block">
<p/>You can include comments within your data files, and either have them ignored, or passed directly
through to the standard output as soon as they are encountered:
<p/>
<div class="pokipanel">
<pre>
$ mlr --usage-comments-in-data
--skip-comments Ignore commented lines (prefixed by "#")
within the input.
--skip-comments-with {string} Ignore commented lines within input, with
specified prefix.
--pass-comments Immediately print commented lines (prefixed by "#")
within the input.
--pass-comments-with {string} Immediately print commented lines within input, with
specified prefix.
Notes:
* Comments are only honored at the start of a line.
* In the absence of any of the above four options, comments are data like
any other text.
* When pass-comments is used, comment lines are written to standard output
immediately upon being read; they are not part of the record stream.
Results may be counterintuitive. A suggestion is to place comments at the
start of data files.
</pre>
</div>
<p/>
<p/>Examples:
<p/>
<div class="pokipanel">
<pre>
$ cat data/budget.csv
# Asana -- here are the budget figures you asked for!
type,quantity
purple,456.78
green,678.12
orange,123.45
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --skip-comments --icsv --opprint sort -nr quantity data/budget.csv
type quantity
green 678.12
purple 456.78
orange 123.45
</pre>
</div>
<p/>
<p/>
<div class="pokipanel">
<pre>
$ mlr --pass-comments --icsv --opprint sort -nr quantity data/budget.csv
# Asana -- here are the budget figures you asked for!
type quantity
green 678.12
purple 456.78
orange 123.45
</pre>
</div>
<p/>
</div>
</body>
</html>
Reference in a new issue View git blame Copy permalink