mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-25 17:04:01 +00:00
feature-count example @ cookbook
This commit is contained in:
John Kerl
parent
0e79bce0e4
commit
ecc4eab44a
6 changed files with 145 additions and 2 deletions
|
|
@ -1334,7 +1334,7 @@ int cli_handle_reader_options(char** argv, int argc, int *pargi, cli_reader_opts
|
|||
preader_opts->allow_repeat_ifs = TRUE;
|
||||
argi += 1;
|
||||
|
||||
} else if (streq(argv[argi], "--json-skip-arrays-on-input")) { // xxx to mlr -h
|
||||
} else if (streq(argv[argi], "--json-skip-arrays-on-input")) {
|
||||
preader_opts->json_skip_arrays_on_input = TRUE;
|
||||
argi += 1;
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ BUGFIXES
|
|||
- incl. json-skip-arrays-on-input
|
||||
|
||||
* feature-counting script to cookbook
|
||||
* json_encode / json_decode functions? (map->string and string->map)
|
||||
? mlrfunc docsection?
|
||||
|
||||
! windows port
|
||||
|
|
@ -170,6 +169,8 @@ COMPARES:
|
|||
================================================================
|
||||
5.2.0 IDEAS:
|
||||
|
||||
* json_encode / json_decode functions? (map->string and string->map)
|
||||
|
||||
* strptime/strftime:
|
||||
- strptime w/ fractional seconds. maybe %OS?
|
||||
- strftime: %OS1-%OS9?
|
||||
|
|
|
|||
|
|
@ -287,6 +287,27 @@ or, more simply,
|
|||
|
||||
POKI_INCLUDE_AND_RUN_ESCAPED(data/ragged-csv-2.sh)HERE
|
||||
|
||||
<!-- ================================================================ -->
|
||||
</div>
|
||||
<h1>Feature-counting</h1>
|
||||
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_feature_counting');" href="javascript:;">Toggle section visibility</button>
|
||||
<div id="section_toggle_feature_counting" style="display: block">
|
||||
|
||||
<p/>Suppose you have some heterogeneous data like this:
|
||||
|
||||
POKI_INCLUDE_ESCAPED(data/features.json)HERE
|
||||
|
||||
<p/> A reasonable question to ask is, how many occurrences of each field are
|
||||
there? And, what percentage of total row count has each of them? Since the
|
||||
denominator of the percentage is not known until the end, this is a two-pass
|
||||
algorithm:
|
||||
|
||||
POKI_INCLUDE_ESCAPED(data/feature-count.mlr)HERE
|
||||
|
||||
<p/> Then
|
||||
POKI_RUN_COMMAND{{mlr --json put -q -f data/feature-count.mlr data/features.json}}HERE
|
||||
POKI_RUN_COMMAND{{mlr --ijson --opprint put -q -f data/feature-count.mlr data/features.json}}HERE
|
||||
|
||||
<!-- ================================================================ -->
|
||||
</div>
|
||||
<h1>Unsparsing</h1>
|
||||
|
|
|
|||
|
|
@ -205,6 +205,7 @@ Miller commands were run with pretty-print-tabular output format.
|
|||
• <a href="#Two-pass_algorithms:_records_having_max_value">Two-pass algorithms: records having max value</a><br/>
|
||||
• <a href="#Rectangularizing_data">Rectangularizing data</a><br/>
|
||||
• <a href="#Regularizing_ragged_CSV">Regularizing ragged CSV</a><br/>
|
||||
• <a href="#Feature-counting">Feature-counting</a><br/>
|
||||
• <a href="#Unsparsing">Unsparsing</a><br/>
|
||||
• <a href="#Parsing_log-file_output">Parsing log-file output</a><br/>
|
||||
• <a href="#Memoization_with_out-of-stream_variables">Memoization with out-of-stream variables</a><br/>
|
||||
|
|
@ -1150,6 +1151,101 @@ a,b,c
|
|||
</div>
|
||||
<p/>
|
||||
|
||||
<!-- ================================================================ -->
|
||||
</div>
|
||||
<a id="Feature-counting"/><h1>Feature-counting</h1>
|
||||
<button style="font-weight:bold;color:maroon;border:0" padding=0 onclick="toggle_by_name('section_toggle_feature_counting');" href="javascript:;">Toggle section visibility</button>
|
||||
<div id="section_toggle_feature_counting" style="display: block">
|
||||
|
||||
<p/>Suppose you have some heterogeneous data like this:
|
||||
|
||||
<p/>
|
||||
<div class="pokipanel">
|
||||
<pre>
|
||||
{ "qoh": 29874, "rate": 1.68, "latency": 0.02 }
|
||||
{ "name": "alice", "uid": 572 }
|
||||
{ "qoh": 1227, "rate": 1.01, "latency": 0.07 }
|
||||
{ "qoh": 13458, "rate": 1.72, "latency": 0.04 }
|
||||
{ "qoh": 56782, "rate": 1.64, "latency": 0.02 }
|
||||
{ "qoh": 23512, "rate": 1.71, "latency": 0.03 }
|
||||
{ "qoh": 9876, "rate": 1.89, "latency": 0.08 }
|
||||
{ "name": "bill", "uid": 684 }
|
||||
{ "name": "chuck", "uid": 908 }
|
||||
{ "name": "dottie", "uid": 440 }
|
||||
{ "qoh": 0, "rate": 0.40, "latency": 0.01 }
|
||||
{ "qoh": 5438, "rate": 1.56, "latency": 0.17 }
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
||||
<p/> A reasonable question to ask is, how many occurrences of each field are
|
||||
there? And, what percentage of total row count has each of them? Since the
|
||||
denominator of the percentage is not known until the end, this is a two-pass
|
||||
algorithm:
|
||||
|
||||
<p/>
|
||||
<div class="pokipanel">
|
||||
<pre>
|
||||
for (key in $*) {
|
||||
@key_counts[key] += 1;
|
||||
}
|
||||
@record_count += 1;
|
||||
|
||||
end {
|
||||
for (key in @key_counts) {
|
||||
@key_fraction[key] = @key_counts[key] / @record_count
|
||||
}
|
||||
emit @record_count;
|
||||
emit @key_counts, "key";
|
||||
emit @key_fraction,"key"
|
||||
}
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
||||
<p/> Then
|
||||
<p/>
|
||||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ mlr --json put -q -f data/feature-count.mlr data/features.json
|
||||
{ "record_count": 12 }
|
||||
{ "key": "qoh", "key_counts": 8 }
|
||||
{ "key": "rate", "key_counts": 8 }
|
||||
{ "key": "latency", "key_counts": 8 }
|
||||
{ "key": "name", "key_counts": 4 }
|
||||
{ "key": "uid", "key_counts": 4 }
|
||||
{ "key": "qoh", "key_fraction": 0.666667 }
|
||||
{ "key": "rate", "key_fraction": 0.666667 }
|
||||
{ "key": "latency", "key_fraction": 0.666667 }
|
||||
{ "key": "name", "key_fraction": 0.333333 }
|
||||
{ "key": "uid", "key_fraction": 0.333333 }
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
<p/>
|
||||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ mlr --ijson --opprint put -q -f data/feature-count.mlr data/features.json
|
||||
record_count
|
||||
12
|
||||
|
||||
key key_counts
|
||||
qoh 8
|
||||
rate 8
|
||||
latency 8
|
||||
name 4
|
||||
uid 4
|
||||
|
||||
key key_fraction
|
||||
qoh 0.666667
|
||||
rate 0.666667
|
||||
latency 0.666667
|
||||
name 0.333333
|
||||
uid 0.333333
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
||||
<!-- ================================================================ -->
|
||||
</div>
|
||||
<a id="Unsparsing"/><h1>Unsparsing</h1>
|
||||
|
|
|
|||
13
doc/data/feature-count.mlr
Normal file
13
doc/data/feature-count.mlr
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
for (key in $*) {
|
||||
@key_counts[key] += 1;
|
||||
}
|
||||
@record_count += 1;
|
||||
|
||||
end {
|
||||
for (key in @key_counts) {
|
||||
@key_fraction[key] = @key_counts[key] / @record_count
|
||||
}
|
||||
emit @record_count;
|
||||
emit @key_counts, "key";
|
||||
emit @key_fraction,"key"
|
||||
}
|
||||
12
doc/data/features.json
Normal file
12
doc/data/features.json
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
{ "qoh": 29874, "rate": 1.68, "latency": 0.02 }
|
||||
{ "name": "alice", "uid": 572 }
|
||||
{ "qoh": 1227, "rate": 1.01, "latency": 0.07 }
|
||||
{ "qoh": 13458, "rate": 1.72, "latency": 0.04 }
|
||||
{ "qoh": 56782, "rate": 1.64, "latency": 0.02 }
|
||||
{ "qoh": 23512, "rate": 1.71, "latency": 0.03 }
|
||||
{ "qoh": 9876, "rate": 1.89, "latency": 0.08 }
|
||||
{ "name": "bill", "uid": 684 }
|
||||
{ "name": "chuck", "uid": 908 }
|
||||
{ "name": "dottie", "uid": 440 }
|
||||
{ "qoh": 0, "rate": 0.40, "latency": 0.01 }
|
||||
{ "qoh": 5438, "rate": 1.56, "latency": 0.17 }
|
||||
Loading…
Add table
Add a link
Reference in a new issue