mirror of
https://github.com/johnkerl/miller.git
synced 2026-07-20 01:46:26 +00:00
new line-readers -> getlines performance comparator
This commit is contained in:
John Kerl
parent
ba297100bd
commit
21aaaba495
3 changed files with 87 additions and 114 deletions
|
|
@ -28,76 +28,71 @@ static FILE* fopen_or_die(char* filename) {
|
|||
return fp;
|
||||
}
|
||||
|
||||
//// ================================================================
|
||||
//static int read_file_mlr_get_line(char* filename, int do_write) {
|
||||
// FILE* fp = fopen_or_die(filename);
|
||||
// int bc = 0;
|
||||
// while (1) {
|
||||
// char* line = mlr_get_cline(fp, '\n');
|
||||
// if (line == NULL)
|
||||
// break;
|
||||
// bc += strlen(line);
|
||||
// if (do_write) {
|
||||
// fputs(line, stdout);
|
||||
// fputc('\n', stdout);
|
||||
// }
|
||||
// free(line);
|
||||
// }
|
||||
// fclose(fp);
|
||||
// return bc;
|
||||
//}
|
||||
// ================================================================
|
||||
static int read_file_mlr_alloc_read_line_single_delimiter_no_autodetect(char* filename, int do_write) {
|
||||
FILE* fp = fopen_or_die(filename);
|
||||
int bc = 0;
|
||||
size_t line_length = MLR_ALLOC_READ_LINE_INITIAL_SIZE;
|
||||
while (1) {
|
||||
char* line = mlr_alloc_read_line_single_delimiter(
|
||||
fp, '\n', &line_length, FALSE, NULL);
|
||||
if (line == NULL)
|
||||
break;
|
||||
bc += strlen(line);
|
||||
if (do_write) {
|
||||
fputs(line, stdout);
|
||||
fputc('\n', stdout);
|
||||
}
|
||||
free(line);
|
||||
}
|
||||
fclose(fp);
|
||||
return bc;
|
||||
}
|
||||
|
||||
//// ================================================================
|
||||
//static int read_file_mlr_getsdelim(char* filename, int do_write) {
|
||||
// FILE* fp = fopen_or_die(filename);
|
||||
// char* irs = "\r\n";
|
||||
// int irslen = strlen(irs);
|
||||
// int bc = 0;
|
||||
// while (1) {
|
||||
// char* line = mlr_get_sline(fp, irs, irslen);
|
||||
// if (line == NULL)
|
||||
// break;
|
||||
// //bc += linelen; // available by API, but make a fair comparison
|
||||
// bc += strlen(line);
|
||||
// if (do_write) {
|
||||
// fputs(line, stdout);
|
||||
// fputc('\n', stdout);
|
||||
// }
|
||||
// free(line);
|
||||
// }
|
||||
// fclose(fp);
|
||||
// return bc;
|
||||
//}
|
||||
// ================================================================
|
||||
static int read_file_mlr_alloc_read_line_single_delimiter_with_autodetect(char* filename, int do_write) {
|
||||
FILE* fp = fopen_or_die(filename);
|
||||
int bc = 0;
|
||||
size_t line_length = MLR_ALLOC_READ_LINE_INITIAL_SIZE;
|
||||
context_t ctx;
|
||||
context_init_from_first_file_name(&ctx, "fake-file-name");
|
||||
|
||||
//// ================================================================
|
||||
//static int popen_file_mlr_getsdelim(char* reader, char* filename, int do_write) {
|
||||
// char* command = mlr_malloc_or_die(strlen(reader) + 1 + strlen(filename) + 1);
|
||||
// strcpy(command, reader);
|
||||
// strcat(command, " ");
|
||||
// strcat(command, filename);
|
||||
// FILE* fp = popen(command, "r");
|
||||
// if (fp == NULL) {
|
||||
// perror("popen");
|
||||
// exit(1);
|
||||
// }
|
||||
// char* irs = "\r\n";
|
||||
// int irslen = strlen(irs);
|
||||
// int bc = 0;
|
||||
// while (1) {
|
||||
// char* line = mlr_get_sline(fp, irs, irslen);
|
||||
// if (line == NULL)
|
||||
// break;
|
||||
// //bc += linelen; // available by API, but make a fair comparison
|
||||
// bc += strlen(line);
|
||||
// if (do_write) {
|
||||
// fputs(line, stdout);
|
||||
// fputc('\n', stdout);
|
||||
// }
|
||||
// free(line);
|
||||
// }
|
||||
// pclose(fp);
|
||||
// return bc;
|
||||
//}
|
||||
while (1) {
|
||||
char* line = mlr_alloc_read_line_single_delimiter(
|
||||
fp, '\n', &line_length, TRUE, &ctx);
|
||||
if (line == NULL)
|
||||
break;
|
||||
bc += strlen(line);
|
||||
if (do_write) {
|
||||
fputs(line, stdout);
|
||||
fputc('\n', stdout);
|
||||
}
|
||||
free(line);
|
||||
}
|
||||
fclose(fp);
|
||||
return bc;
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
static int read_file_mlr_alloc_read_line_multiple_delimiter(char* filename, int do_write) {
|
||||
FILE* fp = fopen_or_die(filename);
|
||||
int bc = 0;
|
||||
size_t line_length = MLR_ALLOC_READ_LINE_INITIAL_SIZE;
|
||||
|
||||
while (1) {
|
||||
char* line = mlr_alloc_read_line_multiple_delimiter(fp, "\n", 1, &line_length);
|
||||
if (line == NULL)
|
||||
break;
|
||||
bc += strlen(line);
|
||||
if (do_write) {
|
||||
fputs(line, stdout);
|
||||
fputc('\n', stdout);
|
||||
}
|
||||
free(line);
|
||||
}
|
||||
fclose(fp);
|
||||
return bc;
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
static char* read_line_fgetc(FILE* fp, char* irs) {
|
||||
|
|
@ -387,26 +382,26 @@ int main(int argc, char** argv) {
|
|||
|
||||
for (int i = 0; i < nreps; i++) {
|
||||
|
||||
// s = get_systime();
|
||||
// bc = read_file_mlr_get_line(filename, do_write);
|
||||
// e = get_systime();
|
||||
// t = e - s;
|
||||
// printf("type=getdelim,t=%.6lf,n=%d\n", t, bc);
|
||||
// fflush(stdout);
|
||||
s = get_systime();
|
||||
bc = read_file_mlr_alloc_read_line_single_delimiter_no_autodetect(filename, do_write);
|
||||
e = get_systime();
|
||||
t = e - s;
|
||||
printf("type=single_delim_no_auto,t=%.6lf,n=%d\n", t, bc);
|
||||
fflush(stdout);
|
||||
|
||||
// s = get_systime();
|
||||
// bc = read_file_mlr_getsdelim(filename, do_write);
|
||||
// e = get_systime();
|
||||
// t = e - s;
|
||||
// printf("type=mlr_getsdelim,t=%.6lf,n=%d\n", t, bc);
|
||||
// fflush(stdout);
|
||||
s = get_systime();
|
||||
bc = read_file_mlr_alloc_read_line_single_delimiter_with_autodetect(filename, do_write);
|
||||
e = get_systime();
|
||||
t = e - s;
|
||||
printf("type=single_delim_with_auto,t=%.6lf,n=%d\n", t, bc);
|
||||
fflush(stdout);
|
||||
|
||||
// s = get_systime();
|
||||
// bc = popen_file_mlr_getsdelim("zcat -cf < ", filename, do_write);
|
||||
// e = get_systime();
|
||||
// t = e - s;
|
||||
// printf("type=mlr_popen_getsdelim,t=%.6lf,n=%d\n", t, bc);
|
||||
// fflush(stdout);
|
||||
s = get_systime();
|
||||
bc = read_file_mlr_alloc_read_line_multiple_delimiter(filename, do_write);
|
||||
e = get_systime();
|
||||
t = e - s;
|
||||
printf("type=multiple_delim,t=%.6lf,n=%d\n", t, bc);
|
||||
fflush(stdout);
|
||||
|
||||
s = get_systime();
|
||||
bc = read_file_fgetc_fixed_len(filename, do_write);
|
||||
|
|
|
|||
|
|
@ -8,19 +8,10 @@
|
|||
// * The caller should free the return value.
|
||||
// * The line-terminator is not returned as part of the string.
|
||||
// * Null is returned at EOF.
|
||||
|
||||
// xxx type up comments:
|
||||
// * in delimiter (single/multiple)
|
||||
// * in fp
|
||||
// * -in do_auto_line_term- separate variant
|
||||
// * -inout pctx- separate variant
|
||||
// * out line
|
||||
// * out reached eof
|
||||
// * inout strlen (old/new). DEFAULT_SIZE @ first call
|
||||
// * inout linecap (old/new) DEFAULT_SIZE @ first call
|
||||
//
|
||||
// reuse linecap on subsequent calls. power of two above last readlen.
|
||||
// work autodetect deeper into the callstack
|
||||
// * Simiar to getdelim but customized for Miller: in particular, support for autodetected line endings (LF/CRLF).
|
||||
// Also, exists on Windows MSYS2 where there isn't a getdelim.
|
||||
// * Line-length reuses previous length for initial buffer-size allocation. Pass MLR_ALLOC_READ_LINE_INITIAL_SIZE
|
||||
// on first call. On subsequent calls, buffer-size allocations will adapt to the file's line-lengths.
|
||||
|
||||
#define MLR_ALLOC_READ_LINE_INITIAL_SIZE 128
|
||||
|
||||
|
|
|
|||
17
c/todo.txt
17
c/todo.txt
|
|
@ -38,18 +38,8 @@ MSYS2 TO-DO
|
|||
|
||||
! msys2 / pacman -S / etc @ appveyor
|
||||
|
||||
* getdelim replacement:
|
||||
- comments @ header file
|
||||
k push it into the stdio lrec readers
|
||||
k rid of local_getdelim
|
||||
! experimental/getlines !
|
||||
|
||||
k get_unlocked / getc:
|
||||
k @ mlr_arch.h: ifdef MSYS2 define getc_unlocked(x) getc(x)
|
||||
|
||||
k mman/mmap ifdef out entirely with default --no-mmap.
|
||||
k @ mlr_arch.h: ifdef MSYS2 undef ENABLE_MMAP
|
||||
k @ callsites: ifdef ENABLE_MMAP
|
||||
! MLR_ON_MSYS2: needs autocreate from ./configure or some such
|
||||
! -lpcreposix at Makefile.am: needs autocreate from ./configure or some such
|
||||
|
||||
* setenv w/o gmtime_r
|
||||
- @ mlr_arch.h: ifdef MSYS2 undef ENABLE_SETENV
|
||||
|
|
@ -61,9 +51,6 @@ k mman/mmap ifdef out entirely with default --no-mmap.
|
|||
- @ mlr_arch_unixish: as now
|
||||
- @ mlr_arch_msys2: do it that new way. or fatal until then ... or "unimpl" retval.
|
||||
|
||||
* -lpcreposix
|
||||
- found how to do this in ./configure, Makefile.am w/ ifdef, something ...
|
||||
|
||||
* needs-DLL issue:
|
||||
- find out how to fix at ./configure / msys2 build
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue