diff --git a/.gitignore b/.gitignore index 7abc26d40..dba812f4a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,29 @@ c/dsls/filter_dsl_parse.out c/dsls/pdm c/dsls/fdm tags +*.la +*.lo +*~ +.deps/ +.libs/ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache/ +compile +config.guess +config.h +config.h.in +config.log +config.status +config.sub +configure +depcomp +install-sh +libtool +ltmain.sh +m4/ +missing +stamp-h1 +test_join_bucket_keeper +test_lrec diff --git a/Makefile b/Makefile deleted file mode 100644 index ae03fe752..000000000 --- a/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -all: c -devall: c install doc -# TODO: the install target exists to put most-recent mlr executable in the -# path to be picked up by the mlr-execs in the docs dir. better would be to -# export PATH here with ./c at its head. -c: .always - make -C c top -doc: .always - cd doc && poki -install: .always - make -C c install -clean: .always - make -C c clean -.always: - @true diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 000000000..32fe30751 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,4 @@ +# not GPL, thus no COPYING file +AUTOMAKE_OPTIONS=dist-xz foreign + +SUBDIRS=c # doc diff --git a/c/Makefile b/c/Makefile.am similarity index 65% rename from c/Makefile rename to c/Makefile.am index 871de17de..d4d5a67cd 100644 --- a/c/Makefile +++ b/c/Makefile.am @@ -1,53 +1,20 @@ -# ================================================================ -CCOMP=gcc -CFLAGS=-std=gnu99 -IFLAGS=-I. -WFLAGS=-Wall -Werror -LFLAGS=-lm +SUBDIRS= lib cli stream input mapping output containers dsls -# You can do -# -# make -e INSTALLDIR=/usr/local/bin -# -# pending https://github.com/johnkerl/miller/issues/9 -INSTALLDIR=/usr/local/bin - -CC=$(CCOMP) $(CFLAGS) $(IFLAGS) $(WFLAGS) -O3 -#CCDEBUG=$(CCOMP) -g -O1 $(CFLAGS) $(IFLAGS) $(WFLAGS) -CCDEBUG=$(CCOMP) -g $(CFLAGS) $(IFLAGS) $(WFLAGS) - -NON_DSL_SRCS = *.c cli/*.c lib/*.c containers/*.c stream/*.c input/*.c mapping/*.c output/*.c -PDSL_OBJS = ./dsls/put_dsl_parse.o ./dsls/put_dsl_lexer.o ./dsls/put_dsl_wrapper.o -FDSL_OBJS = ./dsls/filter_dsl_parse.o ./dsls/filter_dsl_lexer.o ./dsls/filter_dsl_wrapper.o - -TEST_LREC_SRCS = lib/mlrutil.c lib/mlr_globals.c lib/string_builder.c \ -containers/lrec.c containers/header_keeper.c containers/sllv.c \ -containers/slls.c containers/lhmslv.c \ -input/file_reader_mmap.c input/file_reader_stdio.c \ -input/lrec_reader_stdio_csv.c \ -input/lrec_reader_mmap_csvlite.c input/lrec_reader_stdio_csvlite.c \ -input/peek_file_reader.c \ -input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \ -input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ -input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ -containers/test_lrec.c - -TEST_JOIN_BUCKET_KEEPER_SRCS = \ -lib/mlrutil.c lib/mlr_globals.c lib/string_builder.c \ -mapping/context.c \ -containers/lrec.c \ -containers/sllv.c containers/slls.c containers/lhmslv.c containers/hss.c containers/mixutil.c \ -containers/header_keeper.c \ -containers/join_bucket_keeper.c \ -input/lrec_reader_in_memory.c input/lrec_readers.c \ -input/lrec_reader_stdio_csv.c \ -input/lrec_reader_mmap_csvlite.c input/lrec_reader_stdio_csvlite.c \ -input/lrec_reader_mmap_dkvp.c input/lrec_reader_stdio_dkvp.c \ -input/lrec_reader_mmap_nidx.c input/lrec_reader_stdio_nidx.c \ -input/lrec_reader_mmap_xtab.c input/lrec_reader_stdio_xtab.c \ -input/file_reader_mmap.c input/file_reader_stdio.c \ -input/peek_file_reader.c \ -containers/test_join_bucket_keeper.c +AM_CPPFLAGS= -I${srcdir} +AM_CFLAGS= -Wall -Werror -std=gnu99 -O3 +bin_PROGRAMS= mlr +mlr_SOURCES= mlrmain.c +mlr_LDADD= \ + cli/libcli.la \ + containers/libcontainers.la \ + stream/libstream.la \ + input/libinput.la \ + mapping/libmapping.la \ + output/liboutput.la \ + lib/libmlr.la \ + dsls/libfdsl.la \ + dsls/libpdsl.la \ + -lm # ================================================================ # User-make: creates the executable and runs unit & regression tests (without @@ -62,26 +29,10 @@ top: mlr tests # It requires ctags and valgrind in addition to flex and the C compiler. dev: tags mlr dev-tests -install: mlr tests - cp mlr $(INSTALLDIR) -installhome: mlr tests - cp mlr $(HOME)/bin - # ================================================================ tags: .always ctags -R . -mlr: .always dsls - $(CC) $(NON_DSL_SRCS) $(PDSL_OBJS) $(FDSL_OBJS) $(LFLAGS) -o mlr - -dsls: .always - make -C dsls put_dsl_parse.o - make -C dsls put_dsl_lexer.o - make -C dsls put_dsl_wrapper.o - make -C dsls filter_dsl_parse.o - make -C dsls filter_dsl_lexer.o - make -C dsls filter_dsl_wrapper.o - # ---------------------------------------------------------------- # Other executable variants @@ -127,7 +78,7 @@ dev-tests: dev-unit-test reg-test # Unfortunately --error-exitcode=1 doesn't work well since there are # valgrind-detected errors in stdlibs. :( dev-unit-test: test-mlrutil test-lrec test-string-builder test-join-bucket-keeper - #valgrind --leak-check=full --error-exitcode=1 ./a.out +# valgrind --leak-check=full --error-exitcode=1 ./a.out valgrind --leak-check=full ./test-mlrutil valgrind --leak-check=full ./test-lrec valgrind --leak-check=full ./test-string-builder @@ -208,9 +159,6 @@ termcvt: tools/termcvt.c $(CCDEBUG) tools/termcvt.c -o termcvt # ================================================================ -clean: - @rm -vf mlr mlrd mlrg mlrp tester - @make -C dsls clean perfclean profclean: @rm -vf gmon.out perf.data perf.data.old .always: diff --git a/c/cli/Makefile.am b/c/cli/Makefile.am new file mode 100644 index 000000000..48ca1b2d0 --- /dev/null +++ b/c/cli/Makefile.am @@ -0,0 +1,8 @@ +noinst_LTLIBRARIES= libcli.la +libcli_la_CFLAGS= -std=gnu99 +libcli_la_CPPFLAGS= -I${srcdir}/../ +libcli_la_SOURCES= \ + argparse.c \ + argparse.h \ + mlrcli.c \ + mlrcli.h diff --git a/c/containers/Makefile.am b/c/containers/Makefile.am new file mode 100644 index 000000000..301419374 --- /dev/null +++ b/c/containers/Makefile.am @@ -0,0 +1,51 @@ +noinst_LTLIBRARIES= libcontainers.la +libcontainers_la_SOURCES= \ + dheap.c \ + dheap.h \ + header_keeper.c \ + header_keeper.h \ + hss.c \ + hss.h \ + join_bucket_keeper.c \ + join_bucket_keeper.h \ + lhms2v.c \ + lhms2v.h \ + lhmsi.c \ + lhmsi.h \ + lhmslv.c \ + lhmslv.h \ + lhmss.c \ + lhmss.h \ + lhmsv.c \ + lhmsv.h \ + lrec.c \ + lrec.h \ + mixutil.c \ + mixutil.h \ + mlr_dsl_ast.c \ + mlr_dsl_ast.h \ + percentile_keeper.c \ + percentile_keeper.h \ + slls.c \ + slls.h \ + sllv.c \ + sllv.h \ + top_keeper.c \ + top_keeper.h +libcontainers_la_LIBADD= \ + ../lib/libmlr.la \ + ../mapping/libmapping.la + +noinst_PROGRAMS= test_join_bucket_keeper test_lrec +test_join_bucket_keeper_LDADD= ../input/libinput.la \ + ../mapping/libmapping.la \ + ../lib/libmlr.la \ + libcontainers.la +test_join_bucket_keeper_CPPFLAGS= -D__TEST_JOIN_BUCKET_KEEPER_MAIN__ ${AM_CPPFLAGS} +test_lrec_LDADD= ../input/libinput.la \ + ../lib/libmlr.la \ + libcontainers.la +test_lrec_CPPFLAGS= -D__TEST_LREC_MAIN__ ${AM_CPPFLAGS} + +AM_CPPFLAGS= -I${srcdir}/../ +AM_CFLAGS= -std=gnu99 diff --git a/c/dsls/Makefile b/c/dsls/Makefile deleted file mode 100644 index 18d390152..000000000 --- a/c/dsls/Makefile +++ /dev/null @@ -1,85 +0,0 @@ -top: pdm fdm -CCOMP=gcc - -CFLAGS=-I.. -O3 -#CFLAGS=-I.. -g - -# ---------------------------------------------------------------- -pdm: put_dsl_parse.o put_dsl_lexer.o put_dsl_wrapper.o ../containers/mlr_dsl_ast.c ../containers/mlr_dsl_ast.h - $(CCOMP) $(CFLAGS) -std=gnu99 -D __PUT_DSL_MAIN__ ../containers/sllv.c ../containers/slls.c ../lib/mlrutil.c ../containers/lrec.c ../containers/header_keeper.c put_dsl_parse.c put_dsl_lexer.c ../containers/mlr_dsl_ast.c put_dsl_wrapper.c -o pdm -lm - -put_dsl_wrapper.o: put_dsl_wrapper.c put_dsl_wrapper.h - $(CCOMP) -Wall $(CFLAGS) -c -std=gnu99 put_dsl_wrapper.c - -put_dsl_parse.o: put_dsl_parse.c put_dsl_parse.h - $(CCOMP) $(CFLAGS) -c -std=gnu99 put_dsl_parse.c -put_dsl_parse.c put_dsl_parse.h: put_dsl_parse.y lemon - ./lemon put_dsl_parse.y - mv put_dsl_parse.c put_dsl_parse.c.tmp - sed \ - -e 's/ParseTrace/put_dsl_ParseTrace/g' \ - -e 's/ParseTokenName/put_dsl_ParseTokenName/g' \ - -e 's/lemon_parser_alloc/put_dsl_lemon_parser_alloc/g' \ - -e 's/lemon_parser_free/put_dsl_lemon_parser_free/g' \ - -e 's/lemon_parser_parse_token/put_dsl_lemon_parser_parse_token/g' \ - -e 's/yy_destructor/put_dsl_yy_destructor/g' \ - put_dsl_parse.c.tmp > put_dsl_parse.c - rm -f put_dsl_parse.c.tmp - - -# muldef: -# ParseTrace -# ParseTokenName -# lemon_parser_alloc -# lemon_parser_free -# lemon_parser_parse_token -# yy_destructor - -put_dsl_lexer.o: put_dsl_lexer.c put_dsl_lexer.h - $(CCOMP) $(CFLAGS) -c -std=gnu99 put_dsl_lexer.c -put_dsl_lexer.c put_dsl_lexer.h: put_dsl_lexer.l - flex --prefix=put_dsl_lexer_ --outfile=put_dsl_lexer.c --header-file=put_dsl_lexer.h put_dsl_lexer.l - -# ---------------------------------------------------------------- -fdm: filter_dsl_parse.o filter_dsl_lexer.o filter_dsl_wrapper.o ../containers/mlr_dsl_ast.c ../containers/mlr_dsl_ast.h - $(CCOMP) $(CFLAGS) -std=gnu99 -D __FILTER_DSL_MAIN__ ../containers/sllv.c ../containers/slls.c ../lib/mlrutil.c ../containers/lrec.c ../containers/header_keeper.c filter_dsl_parse.c filter_dsl_lexer.c ..//containers/mlr_dsl_ast.c filter_dsl_wrapper.c -o fdm -lm - -filter_dsl_wrapper.o: filter_dsl_wrapper.c filter_dsl_wrapper.h - $(CCOMP) -Wall $(CFLAGS) -c -std=gnu99 filter_dsl_wrapper.c - -filter_dsl_parse.o: filter_dsl_parse.c filter_dsl_parse.h - $(CCOMP) $(CFLAGS) -c -std=gnu99 filter_dsl_parse.c -filter_dsl_parse.c filter_dsl_parse.h: filter_dsl_parse.y lemon - ./lemon filter_dsl_parse.y - mv filter_dsl_parse.c filter_dsl_parse.c.tmp - sed \ - -e 's/ParseTrace/filter_dsl_ParseTrace/g' \ - -e 's/ParseTokenName/filter_dsl_ParseTokenName/g' \ - -e 's/lemon_parser_alloc/filter_dsl_lemon_parser_alloc/g' \ - -e 's/lemon_parser_free/filter_dsl_lemon_parser_free/g' \ - -e 's/lemon_parser_parse_token/filter_dsl_lemon_parser_parse_token/g' \ - -e 's/yy_destructor/filter_dsl_yy_destructor/g' \ - filter_dsl_parse.c.tmp > filter_dsl_parse.c - rm -f filter_dsl_parse.c.tmp - -filter_dsl_lexer.o: filter_dsl_lexer.c filter_dsl_lexer.h - $(CCOMP) $(CFLAGS) -c -std=gnu99 filter_dsl_lexer.c -filter_dsl_lexer.c filter_dsl_lexer.h: filter_dsl_lexer.l - flex --prefix=filter_dsl_lexer_ --outfile=filter_dsl_lexer.c --header-file=filter_dsl_lexer.h filter_dsl_lexer.l - -# ---------------------------------------------------------------- -lemon: lemon.c lempar.c - $(CCOMP) -o lemon lemon.c - -# ---------------------------------------------------------------- -clean: - rm -f pdm fdm - rm -f *.o - rm -f put_dsl_parse.c put_dsl_parse.h put_dsl_parse.out - rm -f put_dsl_lexer.c put_dsl_lexer.h - rm -f filter_dsl_parse.c filter_dsl_parse.h filter_dsl_parse.out - rm -f filter_dsl_lexer.c filter_dsl_lexer.h - rm -f lemon - -.always: - @/bin/true diff --git a/c/dsls/Makefile.am b/c/dsls/Makefile.am new file mode 100644 index 000000000..2a0686626 --- /dev/null +++ b/c/dsls/Makefile.am @@ -0,0 +1,62 @@ +AM_CPPFLAGS= -I${srcdir} +AM_CFLAGS= -std=gnu99 + +noinst_PROGRAMS= lemon pdm fdm +lemon_SOURCES= lemon.c + +noinst_LTLIBRARIES= libfdsl.la libpdsl.la +libfdsl_la_SOURCES= filter_dsl_parse.c filter_dsl_lexer.c filter_dsl_wrapper.c +libpdsl_la_SOURCES= put_dsl_parse.c put_dsl_lexer.c put_dsl_wrapper.c +BUILT_SOURCES= put_dsl_parse.c filter_dsl_parse.c +CLEANFILES= ${BUILT_SOURCES} put_dsl_parse.out filter_dsl_parse.out +EXTRA_DIST= \ + filter_dsl_lexer.l \ + filter_dsl_parse.y \ + put_dsl_lexer.l \ + put_dsl_parse.y + +fdm_SOURCES= filter_dsl_parse.c filter_dsl_lexer.c filter_dsl_wrapper.c +fdm_LDADD= ../containers/libcontainers.la ../lib/libmlr.la libfdsl.la -lm +fdm_CFLAGS= -D __FILTER_DSL_MAIN__ ${AM_CFLAGS} +pdm_SOURCES= put_dsl_parse.c put_dsl_lexer.c put_dsl_wrapper.c +pdm_LDADD= ../containers/libcontainers.la ../lib/libmlr.la libpdsl.la -lm +pdm_CFLAGS= -D __PUT_DSL_MAIN__ ${AM_CFLAGS} + +put_dsl_parse.c put_dsl_parse.h: put_dsl_parse.y lemon + ./lemon put_dsl_parse.y +# lemon does not re-generate the file if it exists and is the same +# this causes problems with the dependency above + touch put_dsl_parse.h + mv put_dsl_parse.c put_dsl_parse.c.tmp + sed \ + -e 's/ParseTrace/put_dsl_ParseTrace/g' \ + -e 's/ParseTokenName/put_dsl_ParseTokenName/g' \ + -e 's/lemon_parser_alloc/put_dsl_lemon_parser_alloc/g' \ + -e 's/lemon_parser_free/put_dsl_lemon_parser_free/g' \ + -e 's/lemon_parser_parse_token/put_dsl_lemon_parser_parse_token/g' \ + -e 's/yy_destructor/put_dsl_yy_destructor/g' \ + put_dsl_parse.c.tmp > put_dsl_parse.c + rm -f put_dsl_parse.c.tmp + + +put_dsl_lexer.c put_dsl_lexer.h: put_dsl_lexer.l + flex --prefix=put_dsl_lexer_ --outfile=put_dsl_lexer.c --header-file=put_dsl_lexer.h put_dsl_lexer.l + +filter_dsl_parse.c filter_dsl_parse.h: filter_dsl_parse.y lemon + ./lemon filter_dsl_parse.y +# lemon does not re-generate the file if it exists and is the same +# this causes problems with the dependency above + touch filter_dsl_parse.h + mv filter_dsl_parse.c filter_dsl_parse.c.tmp + sed \ + -e 's/ParseTrace/filter_dsl_ParseTrace/g' \ + -e 's/ParseTokenName/filter_dsl_ParseTokenName/g' \ + -e 's/lemon_parser_alloc/filter_dsl_lemon_parser_alloc/g' \ + -e 's/lemon_parser_free/filter_dsl_lemon_parser_free/g' \ + -e 's/lemon_parser_parse_token/filter_dsl_lemon_parser_parse_token/g' \ + -e 's/yy_destructor/filter_dsl_yy_destructor/g' \ + filter_dsl_parse.c.tmp > filter_dsl_parse.c + rm -f filter_dsl_parse.c.tmp + +filter_dsl_lexer.c filter_dsl_lexer.h: filter_dsl_lexer.l + flex --prefix=filter_dsl_lexer_ --outfile=filter_dsl_lexer.c --header-file=filter_dsl_lexer.h filter_dsl_lexer.l diff --git a/c/input/Makefile.am b/c/input/Makefile.am new file mode 100644 index 000000000..78c9afb3f --- /dev/null +++ b/c/input/Makefile.am @@ -0,0 +1,24 @@ +noinst_LTLIBRARIES= libinput.la +libinput_la_SOURCES= \ + file_reader_mmap.c \ + file_reader_mmap.h \ + file_reader_stdio.c \ + file_reader_stdio.h \ + lrec_reader.h \ + lrec_reader_in_memory.c \ + lrec_reader_mmap_csvlite.c \ + lrec_reader_mmap_dkvp.c \ + lrec_reader_mmap_nidx.c \ + lrec_reader_mmap_xtab.c \ + lrec_reader_stdio_csv.c \ + lrec_reader_stdio_csvlite.c \ + lrec_reader_stdio_dkvp.c \ + lrec_reader_stdio_nidx.c \ + lrec_reader_stdio_xtab.c \ + lrec_readers.c \ + lrec_readers.h \ + peek_file_reader.c \ + peek_file_reader.h +libinput_la_LIBADD= ../lib/libmlr.la +libinput_la_CPPFLAGS= -I${srcdir}/../ +libinput_la_CFLAGS= -std=gnu99 diff --git a/c/lib/Makefile.am b/c/lib/Makefile.am new file mode 100644 index 000000000..f665fb669 --- /dev/null +++ b/c/lib/Makefile.am @@ -0,0 +1,17 @@ +noinst_LTLIBRARIES= libmlr.la +libmlr_la_SOURCES= minunit.h \ + mlr_globals.c \ + mlr_globals.h \ + mlrmath.c \ + mlrmath.h \ + mlrstat.c \ + mlrstat.h \ + mlrutil.c \ + mlrutil.h \ + mtrand.c \ + mtrand.h \ + string_builder.c \ + string_builder.h +libmlr_la_CPPFLAGS= -I${srcdir}/../ +libmlr_la_CFLAGS= -std=gnu99 +#noinst_PROGRAMS= test_mlrutil test_string_builder diff --git a/c/mapping/Makefile.am b/c/mapping/Makefile.am new file mode 100644 index 000000000..b4acaed45 --- /dev/null +++ b/c/mapping/Makefile.am @@ -0,0 +1,36 @@ +noinst_LTLIBRARIES= libmapping.la +libmapping_la_SOURCES= \ + context.c \ + context.h \ + lrec_evaluator.h \ + lrec_evaluators.c \ + lrec_evaluators.h \ + mapper.h \ + mapper_cat.c \ + mapper_check.c \ + mapper_cut.c \ + mapper_filter.c \ + mapper_group_like.c \ + mapper_having_fields.c \ + mapper_head.c \ + mapper_histogram.c \ + mapper_join.c \ + mapper_label.c \ + mapper_put.c \ + mapper_regularize.c \ + mapper_rename.c \ + mapper_reorder.c \ + mapper_sort.c \ + mapper_stats1.c \ + mapper_stats2.c \ + mapper_step.c \ + mapper_tac.c \ + mapper_tail.c \ + mapper_top.c \ + mapper_uniq.c \ + mappers.h \ + mlr_val.c \ + mlr_val.h +libmapping_la_LIBADD= ../lib/libmlr.la ../cli/libcli.la ../input/libinput.la +libmapping_la_CPPFLAGS= -I${srcdir}/../ +libmapping_la_CFLAGS= -std=gnu99 diff --git a/c/output/Makefile.am b/c/output/Makefile.am new file mode 100644 index 000000000..c4b21bec1 --- /dev/null +++ b/c/output/Makefile.am @@ -0,0 +1,13 @@ +noinst_LTLIBRARIES= liboutput.la +liboutput_la_SOURCES= \ + lrec_writer.h \ + lrec_writer_csv.c \ + lrec_writer_csvlite.c \ + lrec_writer_dkvp.c \ + lrec_writer_nidx.c \ + lrec_writer_pprint.c \ + lrec_writer_xtab.c \ + lrec_writers.h +liboutput_la_LIBADD= ../lib/libmlr.la +liboutput_la_CPPFLAGS= -I${srcdir}/../ +liboutput_la_CFLAGS= -std=gnu99 diff --git a/c/stream/Makefile.am b/c/stream/Makefile.am new file mode 100644 index 000000000..3afa2a494 --- /dev/null +++ b/c/stream/Makefile.am @@ -0,0 +1,4 @@ +noinst_LTLIBRARIES= libstream.la +libstream_la_SOURCES= stream.c stream.h +libstream_la_CPPFLAGS= -I${srcdir}/../ +libstream_la_CFLAGS= -std=gnu99 diff --git a/configure.ac b/configure.ac new file mode 100644 index 000000000..cf897fc81 --- /dev/null +++ b/configure.ac @@ -0,0 +1,25 @@ +AC_PREREQ([2.60]) +AC_INIT([mlr],[2.0.0],[email@address.missing]) +AC_CONFIG_SRCDIR([c/mlrmain.c]) +AC_CONFIG_HEADERS([config.h]) +AC_CONFIG_MACRO_DIR([m4]) +AM_INIT_AUTOMAKE + +AC_CANONICAL_HOST +AC_PROG_CC +AM_PROG_LEX +AM_PROG_CC_C_O +AC_EXEEXT +LT_INIT + +AC_CONFIG_FILES([Makefile + c/cli/Makefile + c/containers/Makefile + c/dsls/Makefile + c/input/Makefile + c/lib/Makefile + c/mapping/Makefile + c/output/Makefile + c/stream/Makefile + c/Makefile]) +AC_OUTPUT