Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-22 15:37:59 +00:00
Code Issues Projects Releases Wiki Activity
miller/doc/record-heterogeneity.html
johnkerl e5c43cb4da fix 
missing data-sharing.html link
2019-09-17 22:39:56 -04:00

473 lines
13 KiB
HTML
Raw Blame History

<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<!-- PAGE GENERATED FROM template.html and content-for-record-heterogeneity.html BY poki. -->
<!-- PLEASE MAKE CHANGES THERE AND THEN RE-RUN poki. -->
<head>
<meta http-equiv="Content-type" content="text/html;charset=UTF-8"/>
<meta name="description" content="Miller documentation"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> <!-- mobile-friendly -->
<meta name="keywords"
content="John Kerl, Kerl, Miller, miller, mlr, OLAP, data analysis software, regression, correlation, variance, data tools, " />
<title> Record-hetrogeneity </title>
<link rel="stylesheet" type="text/css" href="css2/miller.css"/>
<link rel="stylesheet" type="text/css" href="css2/poki-callbacks.css"/>
</head>
<!-- ================================================================ -->
<script type="text/javascript">
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type="text/javascript">
try {
var pageTracker = _gat._getTracker("UA-15651652-1");
pageTracker._trackPageview();
} catch(err) {}
</script>
<!-- ================================================================ -->
<script type="text/javascript">
function vis_expand_all_navbar_sections() {
//console.log('vis_expand_all_navbar_sections ENTER');
vis_expand_all_with_prefix('navbar_section_toggle_');
//console.log('vis_expand_all_navbar_sections EXIT');
}
function vis_collapse_all_navbar_sections() {
//console.log('vis_collapse_all_navbar_sections ENTER');
vis_collapse_all_with_prefix('navbar_section_toggle_');
//console.log('vis_collapse_all_navbar_sections EXIT');
}
function vis_expand_all_body_sections() {
//console.log('vis_expand_all_body_sections ENTER');
vis_expand_all_with_prefix('body_section_toggle_');
//console.log('vis_expand_all_body_sections EXIT');
}
function vis_collapse_all_body_sections() {
//console.log('vis_collapse_all_body_sections ENTER');
vis_collapse_all_with_prefix('body_section_toggle_');
//console.log('vis_collapse_all_body_sections EXIT');
}
function vis_expand_all_with_prefix(prefix) {
//console.log('vis_expand_all_with_prefix ENTER', prefix);
var divs = document.getElementsByTagName('div');
console.log("DIVS", divs);
for(var i = 0; i < divs.length; i++) {
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_expand_div(div);
}
}
//console.log('vis_expand_all_with_prefix EXIT', prefix);
}
function vis_collapse_all_with_prefix(prefix) {
var divs = document.getElementsByTagName('div');
for(var i = 0; i < divs.length; i++){
const div = divs[i];
if (div.id.startsWith(prefix)) {
vis_collapse_div(div);
}
}
}
function vis_toggle_by_name(divName) {
vis_toggle_div(document.getElementById(divName));
}
function vis_expand_by_name(divName) {
vis_expand_div(document.getElementById(divName));
}
function vis_collapse_by_name(divName) {
vis_collapse_div(document.getElementById(divName));
}
function vis_toggle_div(div) {
if (div != null) {
var state = div.style.display;
if (state == 'block') {
div.style.display = 'none';
} else {
div.style.display = 'block';
}
}
}
function vis_expand_div(div) {
if (div != null) {
div.style.display = 'block';
}
}
function vis_collapse_div(div) {
if (div != null) {
div.style.display = 'none';
}
}
</script>
<body bgcolor="#ffffff">
<!-- ================================================================ -->
<!-- navbar -->
<div class="pokinav">
<center><titleinbody>Miller</titleinbody></center>
<!-- NAVBAR GENERATED FROM template.html BY poki -->
<button
style="font-weight:bold;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_overview');"
href="javascript:;"
>
Overview
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_using');"
href="javascript:;"
>
Using
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_reference');"
href="javascript:;"
>
Reference
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_background');"
href="javascript:;"
>
Background
</button>
<button
style="font-weight:normal;font-size:100%;color:maroon;border:0"
padding=0
onclick="vis_collapse_all_navbar_sections(); vis_toggle_by_name('navbar_section_toggle_repository');"
href="javascript:;"
>
Repository
</button>
<div id="navbar_section_toggle_overview" style="display: block;text-align: left">
<br/><a href="index.html">About</a>
<br/><a href="10-min.html">10-minute intro</a>
<br/><a href="file-formats.html">File formats</a>
<br/><a href="feature-comparison.html">Unix-toolkit context</a>
<br/><a href="record-heterogeneity.html"><b>Record-hetrogeneity</b></a>
<br/><a href="internationalization.html">Internationalization</a>
</div>
<div id="navbar_section_toggle_using" style="display: none;text-align: left">
<br/><a href="faq.html">FAQ</a>
<br/><a href="10-min.html">10-minute intro</a>
<br/><a href="data-sharing.html">Mixing with other languages</a>
<br/><a href="cookbook.html">Cookbook part 1</a>
<br/><a href="cookbook2.html">Cookbook part 2</a>
<br/><a href="cookbook3.html">Cookbook part 3</a>
<br/><a href="data-examples.html">Data-diving examples</a>
</div>
<div id="navbar_section_toggle_reference" style="display: none;text-align: left">
<br/><a href="reference.html">Main reference</a>
<br/><a href="reference-verbs.html">Verbs reference</a>
<br/><a href="reference-dsl.html">DSL reference</a>
<br/><a href="manpage.html">Manpage</a>
<br/><a href="release-docs.html">Documents by release</a>
<br/><a href="build.html">Installation</a>
</div>
<div id="navbar_section_toggle_background" style="display: none;text-align: left">
<br/><a href="why.html">Why?</a>
<br/><a href="whyc.html">Why C?</a>
<br/><a href="etymology.html">Why call it Miller?</a>
<br/><a href="originality.html">How original is Miller?</a>
<br/><a href="performance.html">Performance</a>
</div>
<div id="navbar_section_toggle_repository" style="display: none;text-align: left">
<br/><a href="contact.html">Contact information</a>
<br/><a href="https://github.com/johnkerl/miller">https://github.com/johnkerl/miller</a>
</div>
</div>
<!-- page body -->
<p/>
<!-- BODY COPIED FROM content-for-record-heterogeneity.html BY poki -->
<div class="pokitoc">
<center><titleinbody>Record-hetrogeneity</titleinbody></center>
&bull;&nbsp;<a href="#For_I/O">For I/O</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#CSV_and_pretty-print">CSV and pretty-print</a><br/>
&nbsp;&nbsp;&nbsp;&nbsp;&bull;&nbsp;<a href="#Key-value-pair,_vertical-tabular,_and_index-numbered_formats">Key-value-pair, vertical-tabular, and index-numbered formats</a><br/>
&bull;&nbsp;<a href="#For_processing">For processing</a><br/>
</div>
<p/>
We think of CSV tables as rectangular: if there are 17 columns in the header then there are 17 columns for every row, else the data have a formatting error.
<p/>But heterogeneous data abound (today&rsquo;s no-SQL databases for example). Miller handles this.
<a id="For_I/O"/><h1>For I/O</h1>
<a id="CSV_and_pretty-print"/><h2>CSV and pretty-print</h2>
Miller simply prints a newline and a new header when there is a schema change. When there is no schema change, you get CSV per se as a special case. Likewise, Miller reads heterogeneous CSV or pretty-print input the same way. The difference between CSV and CSV-lite is that the former is RFC4180-compliant, while the latter readily handles heterogeneous data (which is non-compliant). For example:
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/het.dkvp
resource=/path/to/file,loadsec=0.45,ok=true
record_count=100,resource=/path/to/file
resource=/path/to/second/file,loadsec=0.32,ok=true
record_count=150,resource=/path/to/second/file
resource=/some/other/path,loadsec=0.97,ok=false
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --ocsvlite cat data/het.dkvp
resource,loadsec,ok
/path/to/file,0.45,true
record_count,resource
100,/path/to/file
resource,loadsec,ok
/path/to/second/file,0.32,true
record_count,resource
150,/path/to/second/file
resource,loadsec,ok
/some/other/path,0.97,false
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint cat data/het.dkvp
resource loadsec ok
/path/to/file 0.45 true
record_count resource
100 /path/to/file
resource loadsec ok
/path/to/second/file 0.32 true
record_count resource
150 /path/to/second/file
resource loadsec ok
/some/other/path 0.97 false
</pre>
</div>
<p/>
</td></tr></table>
<p/> Miller handles explicit header changes as just shown. If your CSV input contains ragged data &mdash; if there are implicit header changes &mdash; you can use <code>--allow-ragged-csv-input</code> (or keystroke-saver <code>--ragged</code>). For too-short data lines, values are filled with empty string; for too-long data lines, missing field names are replaced with positional indices (counting up from 1, not 0), as follows:
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/ragged.csv
a,b,c
1,2,3
4,5
6,7,8,9
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --icsv --oxtab --allow-ragged-csv-input cat data/ragged.csv
a 1
b 2
c 3
a 4
b 5
c
a 6
b 7
c 8
4 9
</pre>
</div>
<p/>
</td></tr></table>
<p/> You may also find Miller&rsquo;s <code>group-like</code> feature handy (see also
<a href="reference.html">Main reference</a>):
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --ocsvlite group-like data/het.dkvp
resource,loadsec,ok
/path/to/file,0.45,true
/path/to/second/file,0.32,true
/some/other/path,0.97,false
record_count,resource
100,/path/to/file
150,/path/to/second/file
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --opprint group-like data/het.dkvp
resource loadsec ok
/path/to/file 0.45 true
/path/to/second/file 0.32 true
/some/other/path 0.97 false
record_count resource
100 /path/to/file
150 /path/to/second/file
</pre>
</div>
<p/>
</td></tr></table>
<a id="Key-value-pair,_vertical-tabular,_and_index-numbered_formats"/><h2>Key-value-pair, vertical-tabular, and index-numbered formats</h2>
For these formats, record-heterogeneity comes naturally:
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/het.dkvp
resource=/path/to/file,loadsec=0.45,ok=true
record_count=100,resource=/path/to/file
resource=/path/to/second/file,loadsec=0.32,ok=true
record_count=150,resource=/path/to/second/file
resource=/some/other/path,loadsec=0.97,ok=false
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --onidx --ofs ' ' cat data/het.dkvp
/path/to/file 0.45 true
100 /path/to/file
/path/to/second/file 0.32 true
150 /path/to/second/file
/some/other/path 0.97 false
</pre>
</div>
<p/>
</td> </tr><tr> <td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab cat data/het.dkvp
resource /path/to/file
loadsec 0.45
ok true
record_count 100
resource /path/to/file
resource /path/to/second/file
loadsec 0.32
ok true
record_count 150
resource /path/to/second/file
resource /some/other/path
loadsec 0.97
ok false
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr --oxtab group-like data/het.dkvp
resource /path/to/file
loadsec 0.45
ok true
resource /path/to/second/file
loadsec 0.32
ok true
resource /some/other/path
loadsec 0.97
ok false
record_count 100
resource /path/to/file
record_count 150
resource /path/to/second/file
</pre>
</div>
<p/>
</td></tr></table>
<a id="For_processing"/><h1>For processing</h1>
<p/> Miller operates on specified fields and takes the rest along: for example, if you are sorting on the
<code>count</code> field then all records in the input stream must have a <code>count</code> field but the other fields can
vary, and moreover the sorted-on field name(s) don&rsquo;t need to be in the same position on each line:
<table><tr><td>
<p/>
<div class="pokipanel">
<pre>
$ cat data/sort-het.dkvp
count=500,color=green
count=600
status=ok,count=250,hours=0.22
status=ok,count=200,hours=3.4
count=300,color=blue
count=100,color=green
count=450
</pre>
</div>
<p/>
</td><td>
<p/>
<div class="pokipanel">
<pre>
$ mlr sort -n count data/sort-het.dkvp
count=100,color=green
status=ok,count=200,hours=3.4
status=ok,count=250,hours=0.22
count=300,color=blue
count=450
count=500,color=green
count=600
</pre>
</div>
<p/>
</td></tr></table>
</body>
</html>
Reference in a new issue View git blame Copy permalink