mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-28 10:13:59 +00:00
shrink data-file sizes
This commit is contained in:
John Kerl
parent
9ba1f46b2a
commit
f827177aaf
4 changed files with 106 additions and 90041 deletions
|
|
@ -22,19 +22,6 @@
|
|||
|
||||
// ----------------------------------------------------------------
|
||||
#define STRING_BUILDER_INIT_SIZE 1024
|
||||
#define TERMIND_RS 0x1111
|
||||
#define TERMIND_FS 0x2222
|
||||
#define TERMIND_EOF 0x3333
|
||||
|
||||
//typedef struct _field_wrapper_t {
|
||||
// char* contents;
|
||||
// int termind;
|
||||
//} field_wrapper_t;
|
||||
|
||||
//typedef struct _record_wrapper_t {
|
||||
// slls_t* contents;
|
||||
// int at_eof;
|
||||
//} record_wrapper_t;
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
typedef struct _lrec_reader_csvex_state_t {
|
||||
|
|
|
|||
|
|
@ -324,7 +324,7 @@ CITRUS COUNTY 0.989943 0.965940 138635.818880
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ wc -l data/colored-shapes.dkvp
|
||||
100000 data/colored-shapes.dkvp
|
||||
10078 data/colored-shapes.dkvp
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -332,13 +332,13 @@ $ wc -l data/colored-shapes.dkvp
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ head -n 6 data/colored-shapes.dkvp | mlr --opprint cat
|
||||
color shape flag i u v w x
|
||||
green circle 1 1 0.3333681345949695 0.21376251997142037 0.500764608334147 5.335248996137616
|
||||
red square 0 2 0.9698237361677843 0.49622821725976396 0.8040548802590068 4.860269966304398
|
||||
yellow circle 1 3 0.9879523154172577 0.8487346180375205 0.6423100990686053 5.277708912462331
|
||||
red circle 0 4 0.4622496635675988 0.5046956653096718 0.13094461569540355 5.370828092344023
|
||||
blue circle 1 5 0.9992738066015238 0.08847422779789516 0.020449531719886935 4.712152492660485
|
||||
purple triangle 0 6 0.983871885308542 0.5390716976565088 0.8211678541134113 3.873918555931081
|
||||
color shape flag i u v w x
|
||||
yellow triangle 1 11 0.6321695890307647 0.9887207810889004 0.4364983936735774 5.7981881667050565
|
||||
red square 1 15 0.21966833570651523 0.001257332190235938 0.7927778364718627 2.944117399716207
|
||||
red circle 1 16 0.20901671281497636 0.29005231936593445 0.13810280912907674 5.065034003400998
|
||||
red square 0 48 0.9562743938458542 0.7467203085342884 0.7755423050923582 7.117831369597269
|
||||
purple triangle 0 51 0.4355354501763202 0.8591292672156728 0.8122903963006748 5.753094629505863
|
||||
red square 0 64 0.2015510269821953 0.9531098083420033 0.7719912015786777 5.612050466474166
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -350,16 +350,16 @@ Here it looks reasonable that <tt>u</tt> is unit-uniform; something’s up w
|
|||
<pre>
|
||||
$ mlr --oxtab stats1 -a min,mean,max -f flag,u,v data/colored-shapes.dkvp | creach 3
|
||||
flag_min 0.000000
|
||||
flag_mean 0.397960
|
||||
flag_mean 0.398889
|
||||
flag_max 1.000000
|
||||
|
||||
u_min 0.000010
|
||||
u_mean 0.501086
|
||||
u_max 0.999983
|
||||
u_min 0.000044
|
||||
u_mean 0.498326
|
||||
u_max 0.999969
|
||||
|
||||
v_min -0.096195
|
||||
v_mean 0.500016
|
||||
v_max 1.095540
|
||||
v_min -0.092709
|
||||
v_mean 0.497787
|
||||
v_max 1.072500
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -369,18 +369,18 @@ v_max 1.095540
|
|||
<pre>
|
||||
$ mlr --opprint histogram -f flag,u,v --lo -0.1 --hi 1.1 --nbins 12 data/colored-shapes.dkvp
|
||||
bin_lo bin_hi flag_count u_count v_count
|
||||
-0.100000 0.000000 60204 0 287
|
||||
0.000000 0.100000 0 10089 9613
|
||||
0.100000 0.200000 0 9844 10022
|
||||
0.200000 0.300000 0 9976 10115
|
||||
0.300000 0.400000 0 9923 10027
|
||||
0.400000 0.500000 0 9918 9988
|
||||
0.500000 0.600000 0 10017 9934
|
||||
0.600000 0.700000 0 9998 9968
|
||||
0.700000 0.800000 0 10202 10124
|
||||
0.800000 0.900000 0 10003 9970
|
||||
0.900000 1.000000 39796 10030 9650
|
||||
1.000000 1.100000 0 0 302
|
||||
-0.100000 0.000000 6058 0 36
|
||||
0.000000 0.100000 0 1062 988
|
||||
0.100000 0.200000 0 985 1003
|
||||
0.200000 0.300000 0 1024 1014
|
||||
0.300000 0.400000 0 1002 991
|
||||
0.400000 0.500000 0 989 1041
|
||||
0.500000 0.600000 0 1001 1016
|
||||
0.600000 0.700000 0 972 962
|
||||
0.700000 0.800000 0 1035 1070
|
||||
0.800000 0.900000 0 995 993
|
||||
0.900000 1.000000 4020 1013 939
|
||||
1.000000 1.100000 0 0 25
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -394,12 +394,12 @@ Bernoulli probablities from the data-generator script:
|
|||
<pre>
|
||||
$ mlr --opprint stats1 -a min,mean,max -f flag,u,v -g color then sort -f color data/colored-shapes.dkvp
|
||||
color flag_min flag_mean flag_max u_min u_mean u_max v_min v_mean v_max
|
||||
blue 0.000000 0.599086 1.000000 0.000044 0.503581 0.999976 0.000057 0.496972 0.999870
|
||||
green 0.000000 0.207102 1.000000 0.000010 0.504358 0.999983 0.000079 0.498956 0.999889
|
||||
orange 0.000000 0.505079 1.000000 0.000464 0.496712 0.999574 0.000185 0.493220 0.999967
|
||||
purple 0.000000 0.098957 1.000000 0.000037 0.498311 0.999693 0.000011 0.502892 0.999993
|
||||
red 0.000000 0.297820 1.000000 0.000019 0.499904 0.999946 -0.096195 0.500635 1.095540
|
||||
yellow 0.000000 0.898270 1.000000 0.000327 0.502877 0.999923 0.000138 0.501022 0.999983
|
||||
blue 0.000000 0.584354 1.000000 0.000044 0.517717 0.999969 0.001489 0.491056 0.999576
|
||||
green 0.000000 0.209197 1.000000 0.000488 0.504861 0.999936 0.000501 0.499085 0.999676
|
||||
orange 0.000000 0.521452 1.000000 0.001235 0.490532 0.998885 0.002449 0.487764 0.998475
|
||||
purple 0.000000 0.090193 1.000000 0.000266 0.494005 0.999647 0.000364 0.497051 0.999975
|
||||
red 0.000000 0.303167 1.000000 0.000671 0.492560 0.999882 -0.092709 0.496535 1.072500
|
||||
yellow 0.000000 0.892427 1.000000 0.001300 0.497129 0.999923 0.000711 0.510627 0.999919
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -408,9 +408,9 @@ yellow 0.000000 0.898270 1.000000 0.000327 0.502877 0.999923 0.000138 0.501022
|
|||
<pre>
|
||||
$ mlr --opprint stats1 -a min,mean,max -f flag,u,v -g shape then sort -f shape data/colored-shapes.dkvp
|
||||
shape flag_min flag_mean flag_max u_min u_mean u_max v_min v_mean v_max
|
||||
circle 0.000000 0.398115 1.000000 0.000037 0.502212 0.999976 -0.096195 0.496996 1.095540
|
||||
square 0.000000 0.397398 1.000000 0.000037 0.499935 0.999970 0.000088 0.500389 0.999975
|
||||
triangle 0.000000 0.398542 1.000000 0.000010 0.501676 0.999983 0.000010 0.501804 0.999995
|
||||
circle 0.000000 0.399846 1.000000 0.000044 0.498555 0.999923 -0.092709 0.495524 1.072500
|
||||
square 0.000000 0.396112 1.000000 0.000188 0.499385 0.999969 0.000089 0.496538 0.999975
|
||||
triangle 0.000000 0.401542 1.000000 0.000881 0.496859 0.999661 0.000717 0.501050 0.999995
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -420,8 +420,8 @@ triangle 0.000000 0.398542 1.000000 0.000010 0.501676 0.999983 0.000010 0.5018
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ mlr --opprint --right stats2 -a corr -f u,v,w,x data/colored-shapes.dkvp
|
||||
u_v_corr w_x_corr
|
||||
0.115078 0.000020
|
||||
u_v_corr w_x_corr
|
||||
0.133418 -0.011320
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -430,24 +430,24 @@ u_v_corr w_x_corr
|
|||
<pre>
|
||||
$ mlr --opprint --right stats2 -a corr -f u,v,w,x -g color,shape then sort -nr u_v_corr data/colored-shapes.dkvp
|
||||
color shape u_v_corr w_x_corr
|
||||
red circle 0.980343 0.004277
|
||||
blue triangle 0.029301 0.017138
|
||||
orange circle 0.021871 -0.040626
|
||||
purple circle 0.021701 0.037687
|
||||
green triangle 0.010633 -0.025945
|
||||
red square 0.006553 -0.007587
|
||||
orange square 0.005342 0.042003
|
||||
yellow square 0.001905 -0.002849
|
||||
orange triangle 0.000620 -0.013529
|
||||
red triangle -0.000349 0.003179
|
||||
yellow triangle -0.001735 0.032633
|
||||
yellow circle -0.002083 -0.012606
|
||||
purple triangle -0.004290 0.006341
|
||||
green square -0.006983 0.001269
|
||||
purple square -0.011754 -0.003372
|
||||
blue circle -0.017771 0.014975
|
||||
blue square -0.019537 -0.006952
|
||||
green circle -0.034313 -0.049415
|
||||
red circle 0.980798 -0.018565
|
||||
orange square 0.176858 -0.071044
|
||||
green circle 0.057644 0.011795
|
||||
red square 0.055745 -0.000680
|
||||
yellow triangle 0.044573 0.024605
|
||||
yellow square 0.043792 -0.044623
|
||||
purple circle 0.035874 0.134112
|
||||
blue square 0.032412 -0.053508
|
||||
blue triangle 0.015356 -0.000608
|
||||
orange circle 0.010519 -0.162795
|
||||
red triangle 0.008098 0.012486
|
||||
purple triangle 0.005155 -0.045058
|
||||
purple square -0.025680 0.057694
|
||||
green square -0.025776 -0.003265
|
||||
orange triangle -0.030457 -0.131870
|
||||
yellow circle -0.064773 0.073695
|
||||
blue circle -0.102348 -0.030529
|
||||
green triangle -0.109018 -0.048488
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -275,7 +275,7 @@ Other options:
|
|||
Output of one verb may be chained as input to another using "then", e.g.
|
||||
mlr stats1 -a min,mean,max -f flag,u,v -g color then sort -f color
|
||||
Please see http://johnkerl.org/miller/doc and/or http://github.com/johnkerl/miller for more information.
|
||||
This is Miller version >= v2.0.0.
|
||||
This is Miller version >= v2.1.1.
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -1726,12 +1726,12 @@ zee 0.504242 0.502997
|
|||
<pre>
|
||||
$ mlr --opprint stats1 -a p50,p99 -f u,v -g color then put '$ur=$u_p99/$u_p50;$vr=$v_p99/$v_p50' data/colored-shapes.dkvp
|
||||
color u_p50 u_p99 v_p50 v_p99 ur vr
|
||||
green 0.505848 0.990764 0.500787 0.989642 1.958620 1.976174
|
||||
red 0.500564 0.989587 0.500415 0.996280 1.976944 1.990908
|
||||
yellow 0.506272 0.989401 0.502912 0.989215 1.954287 1.966974
|
||||
blue 0.506270 0.990783 0.490120 0.990957 1.957025 2.021866
|
||||
purple 0.500051 0.990108 0.505464 0.990086 1.980014 1.958767
|
||||
orange 0.488821 0.991091 0.491891 0.991451 2.027513 2.015591
|
||||
yellow 0.501019 0.989046 0.520630 0.987034 1.974069 1.895845
|
||||
red 0.485038 0.990054 0.492586 0.994444 2.041189 2.018823
|
||||
purple 0.501319 0.988893 0.504571 0.988287 1.972582 1.958668
|
||||
green 0.502015 0.990764 0.505359 0.990175 1.973574 1.959350
|
||||
blue 0.525226 0.992655 0.485170 0.993873 1.889958 2.048505
|
||||
orange 0.483548 0.993635 0.480913 0.989102 2.054884 2.056717
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -1741,9 +1741,9 @@ orange 0.488821 0.991091 0.491891 0.991451 2.027513 2.015591
|
|||
<pre>
|
||||
$ mlr --opprint count-distinct -f shape then sort -nr count data/colored-shapes.dkvp
|
||||
shape count
|
||||
square 41588
|
||||
triangle 33477
|
||||
circle 24935
|
||||
square 4115
|
||||
triangle 3372
|
||||
circle 2591
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -1753,9 +1753,9 @@ circle 24935
|
|||
<pre>
|
||||
$ mlr --opprint stats1 -a mode -f color -g shape data/colored-shapes.dkvp
|
||||
shape color_mode
|
||||
circle red
|
||||
square red
|
||||
triangle red
|
||||
square red
|
||||
circle red
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -2090,11 +2090,11 @@ Passes through the last n records, optionally by category.
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ mlr --opprint tail -n 4 data/colored-shapes.dkvp
|
||||
color shape flag i u v w x
|
||||
yellow circle 1 99997 0.5228034832314841 0.7478634261534541 0.49477944033468396 6.085638633037881
|
||||
red triangle 0 99998 0.8566019561040149 0.5583785393850178 0.4993735796215503 6.393409471109115
|
||||
yellow triangle 1 99999 0.5369350176939407 0.5197619334387739 0.5064468446479313 3.2682256831629695
|
||||
green square 0 100000 0.0277485352321325 0.5303062901341336 0.5274344049261097 5.806843329974349
|
||||
color shape flag i u v w x
|
||||
blue square 1 99974 0.6189062525431605 0.2637962404841453 0.5311465405784674 6.210738209085753
|
||||
blue triangle 0 99976 0.008110504040268474 0.8267274952432482 0.4732962944898885 6.146956761817328
|
||||
yellow triangle 0 99990 0.3839424618160777 0.55952913620132 0.5113763011485609 4.307973891915119
|
||||
yellow circle 1 99994 0.764950884927175 0.25284227383991364 0.49969878539567425 5.013809741826425
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -2103,10 +2103,10 @@ green square 0 100000 0.0277485352321325 0.5303062901341336 0.527434404926
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ mlr --opprint tail -n 1 -g shape data/colored-shapes.dkvp
|
||||
color shape flag i u v w x
|
||||
yellow circle 1 99997 0.5228034832314841 0.7478634261534541 0.49477944033468396 6.085638633037881
|
||||
green square 0 100000 0.0277485352321325 0.5303062901341336 0.5274344049261097 5.806843329974349
|
||||
yellow triangle 1 99999 0.5369350176939407 0.5197619334387739 0.5064468446479313 3.2682256831629695
|
||||
color shape flag i u v w x
|
||||
yellow triangle 0 99990 0.3839424618160777 0.55952913620132 0.5113763011485609 4.307973891915119
|
||||
blue square 1 99974 0.6189062525431605 0.2637962404841453 0.5311465405784674 6.210738209085753
|
||||
yellow circle 1 99994 0.764950884927175 0.25284227383991364 0.49969878539567425 5.013809741826425
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -2189,7 +2189,7 @@ Prints distinct values for specified field names. With -c, same as count-distinc
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ wc -l data/colored-shapes.dkvp
|
||||
100000 data/colored-shapes.dkvp
|
||||
10078 data/colored-shapes.dkvp
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
@ -2198,21 +2198,21 @@ $ wc -l data/colored-shapes.dkvp
|
|||
<div class="pokipanel">
|
||||
<pre>
|
||||
$ mlr uniq -g color,shape data/colored-shapes.dkvp
|
||||
color=green,shape=circle
|
||||
color=red,shape=square
|
||||
color=yellow,shape=circle
|
||||
color=red,shape=circle
|
||||
color=blue,shape=circle
|
||||
color=purple,shape=triangle
|
||||
color=blue,shape=triangle
|
||||
color=green,shape=square
|
||||
color=red,shape=triangle
|
||||
color=yellow,shape=triangle
|
||||
color=red,shape=square
|
||||
color=red,shape=circle
|
||||
color=purple,shape=triangle
|
||||
color=yellow,shape=circle
|
||||
color=purple,shape=square
|
||||
color=blue,shape=square
|
||||
color=yellow,shape=square
|
||||
color=red,shape=triangle
|
||||
color=green,shape=triangle
|
||||
color=green,shape=square
|
||||
color=blue,shape=circle
|
||||
color=blue,shape=triangle
|
||||
color=purple,shape=circle
|
||||
color=blue,shape=square
|
||||
color=green,shape=circle
|
||||
color=orange,shape=triangle
|
||||
color=orange,shape=square
|
||||
color=orange,shape=circle
|
||||
|
|
@ -2225,24 +2225,24 @@ color=orange,shape=circle
|
|||
<pre>
|
||||
$ mlr --opprint uniq -g color,shape -c then sort -f color,shape data/colored-shapes.dkvp
|
||||
color shape count
|
||||
blue circle 3578
|
||||
blue square 6016
|
||||
blue triangle 4843
|
||||
green circle 2832
|
||||
green square 4678
|
||||
green triangle 3924
|
||||
orange circle 705
|
||||
orange square 1196
|
||||
orange triangle 954
|
||||
purple circle 2861
|
||||
purple square 4808
|
||||
purple triangle 3841
|
||||
red circle 11477
|
||||
red square 19051
|
||||
red triangle 15248
|
||||
yellow circle 3482
|
||||
yellow square 5839
|
||||
yellow triangle 4667
|
||||
blue circle 384
|
||||
blue square 589
|
||||
blue triangle 497
|
||||
green circle 287
|
||||
green square 454
|
||||
green triangle 368
|
||||
orange circle 68
|
||||
orange square 128
|
||||
orange triangle 107
|
||||
purple circle 289
|
||||
purple square 481
|
||||
purple triangle 372
|
||||
red circle 1207
|
||||
red square 1874
|
||||
red triangle 1560
|
||||
yellow circle 356
|
||||
yellow square 589
|
||||
yellow triangle 468
|
||||
</pre>
|
||||
</div>
|
||||
<p/>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue