Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 00:45:47 +00:00
Code Issues Projects Releases Wiki Activity

experimental/getlines.c

Browse source
This commit is contained in:
John Kerl 2020-01-26 10:37:13 -05:00
parent dff72699b2
commit bf7ceff5c9
10 changed files with 25 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

49
c/experimental/getlines.c
View file

@ -5,7 +5,6 @@
#include "lib/mlrutil.h"
#include "lib/mlrdatetime.h"
#include "input/file_reader_stdio.h"
#include "input/file_reader_mmap.h"
#include "input/lrec_readers.h"
#include "lib/string_builder.h"
#include "input/byte_readers.h"
@ -259,47 +258,6 @@ static int read_file_fgetc_psb(char* filename, int do_write) {
return bc;
}
// ================================================================
static char* read_line_mmap_psb(file_reader_mmap_state_t* ph, string_builder_t* psb, char* irs) {
char *p = ph->sol;
while (TRUE) {
if (p == ph->eof) {
ph->sol = p;
if (sb_is_empty(psb))
return NULL;
else
return sb_finish(psb);
} else if (*p == irs[0]) {
ph->sol = p+1;
return sb_finish(psb);
} else {
sb_append_char(psb, *p);
p++;
}
}
}
static int read_file_mmap_psb(char* filename, int do_write) {
file_reader_mmap_state_t* ph = file_reader_mmap_open(NULL, filename);
string_builder_t* psb = sb_alloc(STRING_BUILDER_INIT_SIZE);
char* irs = "\n";
int bc = 0;
while (TRUE) {
char* line = read_line_mmap_psb(ph, psb, irs);
if (line == NULL)
break;
if (do_write) {
fputs(line, stdout);
fputc('\n', stdout);
}
bc += strlen(line);
}
sb_free(psb);
file_reader_mmap_close(ph, NULL);
return bc;
}
// ================================================================
#define IRS_TOKEN 11
#define EOF_TOKEN 22
@ -431,13 +389,6 @@ int main(int argc, char** argv) {
printf("type=fgetc_psb,t=%.6lf,n=%d\n", t, bc);
fflush(stdout);
s = get_systime();
bc = read_file_mmap_psb(filename, do_write);
e = get_systime();
t = e - s;
printf("type=mmap_psb,t=%.6lf,n=%d\n", t, bc);
fflush(stdout);
s = get_systime();
bc = read_file_pfr_psb(filename, do_write);
e = get_systime();

2
doc/cookbook.html
View file

@ -858,7 +858,7 @@ date,qoh
<div class="pokipanel">
<pre>
$ wc -l data/miss-date.csv
1372 data/miss-date.csv
1372 data/miss-date.csv
</pre>
</div>
<p/>

2
doc/data-examples.html
View file

@ -208,7 +208,7 @@ St. Johns - - -
<div class="pokipanel">
<pre>
$ wc -l data/colored-shapes.dkvp
10078 data/colored-shapes.dkvp
10078 data/colored-shapes.dkvp
</pre>
</div>
<p/>

32
doc/data-sharing.html
View file

@ -206,11 +206,11 @@ Run as-is:
<div class="pokipanel">
<pre>
$ python polyglot-dkvp-io/example.py &lt; data/small
a=pan,b=pan,i=1,y=0.7268028627434533,ab=panpan,iy=1.7268028627434533,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=eks,b=pan,i=2,y=0.5221511083334797,ab=ekspan,iy=2.5221511083334796,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=wye,b=wye,i=3,y=0.33831852551664776,ab=wyewye,iy=3.3383185255166477,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=eks,b=wye,i=4,y=0.13418874328430463,ab=ekswye,iy=4.134188743284304,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=wye,b=pan,i=5,y=0.8636244699032729,ab=wyepan,iy=5.863624469903273,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=pan,b=pan,i=1,y=0.726802862743,ab=panpan,iy=1.72680286274,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=eks,b=pan,i=2,y=0.522151108333,ab=ekspan,iy=2.52215110833,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=wye,b=wye,i=3,y=0.338318525517,ab=wyewye,iy=3.33831852552,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=eks,b=wye,i=4,y=0.134188743284,ab=ekswye,iy=4.13418874328,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
a=wye,b=pan,i=5,y=0.863624469903,ab=wyepan,iy=5.8636244699,ta=str,tb=str,ti=int,ty=float,tab=str,tiy=float
</pre>
</div>
<p/>
@ -221,12 +221,12 @@ Run as-is, then pipe to Miller for pretty-printing:
<div class="pokipanel">
<pre>
$ python polyglot-dkvp-io/example.py &lt; data/small | mlr --opprint cat
a b i y ab iy ta tb ti ty tab tiy
pan pan 1 0.7268028627434533 panpan 1.7268028627434533 str str int float str float
eks pan 2 0.5221511083334797 ekspan 2.5221511083334796 str str int float str float
wye wye 3 0.33831852551664776 wyewye 3.3383185255166477 str str int float str float
eks wye 4 0.13418874328430463 ekswye 4.134188743284304 str str int float str float
wye pan 5 0.8636244699032729 wyepan 5.863624469903273 str str int float str float
a b i y ab iy ta tb ti ty tab tiy
pan pan 1 0.726802862743 panpan 1.72680286274 str str int float str float
eks pan 2 0.522151108333 ekspan 2.52215110833 str str int float str float
wye wye 3 0.338318525517 wyewye 3.33831852552 str str int float str float
eks wye 4 0.134188743284 ekswye 4.13418874328 str str int float str float
wye pan 5 0.863624469903 wyepan 5.8636244699 str str int float str float
</pre>
</div>
<p/>
@ -417,11 +417,11 @@ wye pan 5 0.5732889198020006 0.8636244699032729 {5}
<pre>
$ mlr --opprint put '$o = system("echo -n ".$a."| sha1sum")' data/small
a b i x y o
pan pan 1 0.3467901443380824 0.7268028627434533 f29c748220331c273ef16d5115f6ecd799947f13 -
eks pan 2 0.7586799647899636 0.5221511083334797 456d988ecb3bf1b75f057fc6e9fe70db464e9388 -
wye wye 3 0.20460330576630303 0.33831852551664776 eab0de043d67f441c7fd1e335f0ca38708e6ebf7 -
eks wye 4 0.38139939387114097 0.13418874328430463 456d988ecb3bf1b75f057fc6e9fe70db464e9388 -
wye pan 5 0.5732889198020006 0.8636244699032729 eab0de043d67f441c7fd1e335f0ca38708e6ebf7 -
pan pan 1 0.3467901443380824 0.7268028627434533 bd2bd8216b9cb4aa5a12daa6cfc98eef2ee20e56 -
eks pan 2 0.7586799647899636 0.5221511083334797 16191338e81a46c7d127f5c8899f5c92e3cd38e3 -
wye wye 3 0.20460330576630303 0.33831852551664776 14ba3c3e96a2474ab6dc7409ebf9d6b9cc3d84f0 -
eks wye 4 0.38139939387114097 0.13418874328430463 16191338e81a46c7d127f5c8899f5c92e3cd38e3 -
wye pan 5 0.5732889198020006 0.8636244699032729 14ba3c3e96a2474ab6dc7409ebf9d6b9cc3d84f0 -
</pre>
</div>
<p/>

12
doc/manpage.html
View file

@ -289,14 +289,6 @@ OPTIONS
-p is a keystroke-saver for --nidx --fs space --repifs
--mmap --no-mmap --mmap-below {n} Use mmap for files whenever possible, never, or
for files less than n bytes in size. Default is for
files less than 4294967296 bytes in size.
'Whenever possible' means always except for when reading
standard input which is not mmappable. If you don't know
what this means, don't worry about it -- it's a minor
performance optimization.
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
@ -818,8 +810,6 @@ VERBS
--ips {pair-separator character}
--repifs
--repips
--mmap
--no-mmap
Please use "mlr --usage-separator-options" for information on specifying separators.
Please see http://johnkerl.org/miller/doc/reference.html for more information
including examples.
@ -2372,7 +2362,7 @@ SEE ALSO
2020-01-05 MILLER(1)
2020-01-26 MILLER(1)
</pre>
</div>
<p/>

12
doc/manpage.txt
View file

@ -215,14 +215,6 @@ OPTIONS
-p is a keystroke-saver for --nidx --fs space --repifs
--mmap --no-mmap --mmap-below {n} Use mmap for files whenever possible, never, or
for files less than n bytes in size. Default is for
files less than 4294967296 bytes in size.
'Whenever possible' means always except for when reading
standard input which is not mmappable. If you don't know
what this means, don't worry about it -- it's a minor
performance optimization.
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
@ -744,8 +736,6 @@ VERBS
--ips {pair-separator character}
--repifs
--repips
--mmap
--no-mmap
Please use "mlr --usage-separator-options" for information on specifying separators.
Please see http://johnkerl.org/miller/doc/reference.html for more information
including examples.
@ -2298,4 +2288,4 @@ SEE ALSO
2020-01-05 MILLER(1)
2020-01-26 MILLER(1)

14
doc/mlr.1
View file

@ -2,12 +2,12 @@
.\" Title: mlr
.\" Author: [see the "AUTHOR" section]
.\" Generator: ./mkman.rb
.\" Date: 2020-01-05
.\" Date: 2020-01-26
.\" Manual: \ \&
.\" Source: \ \&
.\" Language: English
.\"
.TH "MILLER" "1" "2020-01-05" "\ \&" "\ \&"
.TH "MILLER" "1" "2020-01-26" "\ \&" "\ \&"
.\" -----------------------------------------------------------------
.\" * Portability definitions
.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -270,14 +270,6 @@ Please use "mlr --help-function {function name}" for function-specific help.
-p is a keystroke-saver for --nidx --fs space --repifs
--mmap --no-mmap --mmap-below {n} Use mmap for files whenever possible, never, or
for files less than n bytes in size. Default is for
files less than 4294967296 bytes in size.
'Whenever possible' means always except for when reading
standard input which is not mmappable. If you don't know
what this means, don't worry about it -- it's a minor
performance optimization.
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.
@ -987,8 +979,6 @@ the main "mlr --help" for more information on syntax for these arguments.
--ips {pair-separator character}
--repifs
--repips
--mmap
--no-mmap
Please use "mlr --usage-separator-options" for information on specifying separators.
Please see http://johnkerl.org/miller/doc/reference.html for more information
including examples.

2
doc/reference-dsl.html
View file

@ -2025,7 +2025,7 @@ subsequent assignments are done unconditionally:
<pre>
$ mlr put '$x &gt; 0.0; $y = log10($x); $z = sqrt($y)' data/put-gating-example-1.dkvp
x=-1,y=nan,z=nan
x=0,y=-inf,z=nan
x=0,y=-inf,z=-nan
x=1,y=0.000000,z=0.000000
x=2,y=0.301030,z=0.548662
x=3,y=0.477121,z=0.690740

6
doc/reference-verbs.html
View file

@ -1738,8 +1738,6 @@ the main "mlr --help" for more information on syntax for these arguments.
--ips {pair-separator character}
--repifs
--repips
--mmap
--no-mmap
Please use "mlr --usage-separator-options" for information on specifying separators.
Please see http://johnkerl.org/miller/doc/reference.html for more information
including examples.
@ -3958,7 +3956,7 @@ Options:
<div class="pokipanel">
<pre>
$ wc -l data/colored-shapes.dkvp
10078 data/colored-shapes.dkvp
10078 data/colored-shapes.dkvp
</pre>
</div>
<p/>
@ -4130,7 +4128,7 @@ color=purple,shape=square,flag=0
<div class="pokipanel">
<pre>
$ wc -l data/repeats.dkvp
57 data/repeats.dkvp
57 data/repeats.dkvp
</pre>
</div>
<p/>

8
doc/reference.html
View file

@ -1416,14 +1416,6 @@ Data-format options, for input, output, or both:
-p is a keystroke-saver for --nidx --fs space --repifs
--mmap --no-mmap --mmap-below {n} Use mmap for files whenever possible, never, or
for files less than n bytes in size. Default is for
files less than 4294967296 bytes in size.
'Whenever possible' means always except for when reading
standard input which is not mmappable. If you don't know
what this means, don't worry about it -- it's a minor
performance optimization.
Examples: --csv for CSV-formatted input and output; --idkvp --opprint for
DKVP-formatted input and pretty-printed output.