Merge pull request #67 from 0-wiz-0/master

autoconf/automake support

.gitignore

@@ -1,6 +1,7 @@
 mlr
 mlrd
 mlrg
+mlrk
 mlrp
 mlr-unit-tester
 test-argparse
@@ -19,6 +20,7 @@ a.out
 catc
 catc0
 catm
+gmon.out
 *.o
 .*.swp
 .*.swo
@@ -38,3 +40,37 @@ c/dsls/fdm
 c/test/output
 c/output/out
 tags
+*.la
+*.lo
+*.log
+*.trs
+*~
+.deps/
+.libs/
+Makefile
+Makefile.in
+aclocal.m4
+autom4te.cache/
+config.h
+config.h.in
+config.log
+config.status
+configure
+libtool
+m4/
+autotools/
+stamp-h1
+ap
+doc/miller.1
+getl
+lemon_prepared
+lrim
+mlr-[0-9.]*.tar.*
+test_byte_readers
+test_join_bucket_keeper
+test_lrec
+test_mlrutil
+test_multiple_containers
+test_parse_trie
+test_peek_file_reader
+test_string_builder

Makefile.am

@@ -0,0 +1,6 @@
+# not GPL, thus no COPYING file
+AUTOMAKE_OPTIONS=foreign
+
+EXTRA_DIST= LICENSE.txt README.md
+
+SUBDIRS=c doc

c/Makefile.am

@@ -0,0 +1,41 @@
+SUBDIRS=	lib cli stream input mapping output containers dsls experimental tools . test
+
+AM_CPPFLAGS=	-I${srcdir}
+AM_CFLAGS=	-Wall -Werror -std=gnu99 -O3
+bin_PROGRAMS=	mlr mlrg mlrp
+mlr_SOURCES=	mlrmain.c
+mlr_LDADD=	\
+		cli/libcli.la \
+		containers/libcontainers.la \
+		stream/libstream.la \
+		input/libinput.la \
+		mapping/libmapping.la \
+		output/liboutput.la \
+		lib/libmlr.la \
+		dsls/libfdsl.la \
+		dsls/libpdsl.la \
+		-lm
+
+# Other executable variants
+# Debug version
+mlrg_CFLAGS=	-std=gnu99 -g
+mlrg_LDADD=	${mlr_LDADD}
+mlrg_SOURCES=	${mlr_SOURCES}
+
+# Profile version. Usage:
+# * make mlrp
+# * mlrp {arguments>
+# * gprof mlrp gmon.out > myfile.txt
+# Note: works on Linux; not on OSX.
+mlrp_CFLAGS=	-std=gnu99 -g -pg
+mlrp_LDADD=	${mlr_LDADD}
+mlrp_SOURCES=	${mlr_SOURCES}
+
+# Run this after unit-test expected output has changed, and is verified to be
+# OK. (Example: after adding new test cases in test/run.)
+regtest-copy:
+	cp output/out test/expected
+
+# ================================================================
+perfclean profclean:
+	@rm -vf gmon.out perf.data perf.data.old

c/cli/Makefile.am

@@ -0,0 +1,15 @@
+noinst_LTLIBRARIES=	libcli.la
+libcli_la_SOURCES=	\
+			argparse.c \
+			argparse.h \
+			mlrcli.c \
+			mlrcli.h
+
+# TODO: causes circular dependency
+#noinst_PROGRAMS=	ap
+#ap_CPPFLAGS=		-D__AP_MAIN__ ${AM_CPPFLAGS}
+#ap_LDADD=		../containers/libcontainers.la ../lib/libmlr.la
+#ap_SOURCES=		argparse.c
+
+AM_CFLAGS=	-std=gnu99
+AM_CPPFLAGS=	-I${srcdir}/../

c/cli/mlrcli.c

@@ -2,7 +2,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "mlrvers.h"
 #include "lib/mlrutil.h"
 #include "lib/mtrand.h"
 #include "containers/slls.h"
@@ -15,6 +14,8 @@
 #include "cli/mlrcli.h"
 #include "cli/argparse.h"
 
+#include "config.h"
+
 // ----------------------------------------------------------------
 static mapper_setup_t* mapper_lookup_table[] = {
 	&mapper_cat_setup,
@@ -259,7 +260,7 @@ static void main_usage(char* argv0, int exit_code) {
 	fprintf(o, "Output of one verb may be chained as input to another using \"then\", e.g.\n");
 	fprintf(o, "  %s stats1 -a min,mean,max -f flag,u,v -g color then sort -f color\n", argv0);
 	fprintf(o, "Please see http://johnkerl.org/miller/doc and/or http://github.com/johnkerl/miller for more information.\n");
-	fprintf(o, "This is Miller version >= %s.\n", MLR_VERSION);
+	fprintf(o, "This is Miller version >= %s.\n", PACKAGE_VERSION);
 
 	exit(exit_code);
 }

c/containers/Makefile.am

@@ -0,0 +1,64 @@
+noinst_LTLIBRARIES=	libcontainers.la
+libcontainers_la_SOURCES=	\
+			dheap.c \
+			dheap.h \
+			header_keeper.c \
+			header_keeper.h \
+			hss.c \
+			hss.h \
+			join_bucket_keeper.c \
+			join_bucket_keeper.h \
+			lhms2v.c \
+			lhms2v.h \
+			lhmsi.c \
+			lhmsi.h \
+			lhmslv.c \
+			lhmslv.h \
+			lhmss.c \
+			lhmss.h \
+			lhmsv.c \
+			lhmsv.h \
+			lrec.c \
+			lrec.h \
+			mixutil.c \
+			mixutil.h \
+			mlr_dsl_ast.c \
+			mlr_dsl_ast.h \
+			parse_trie.c \
+			parse_trie.h \
+			percentile_keeper.c \
+			percentile_keeper.h \
+			slls.c \
+			slls.h \
+			sllv.c \
+			sllv.h \
+			top_keeper.c \
+			top_keeper.h
+libcontainers_la_LIBADD=	\
+			../lib/libmlr.la \
+			../mapping/libmapping.la
+
+TESTS=			$(check_PROGRAMS)
+check_PROGRAMS=		test_join_bucket_keeper test_lrec test_multiple_containers test_parse_trie
+
+test_join_bucket_keeper_CPPFLAGS=	-D__TEST_JOIN_BUCKET_KEEPER_MAIN__ ${AM_CPPFLAGS}
+test_join_bucket_keeper_LDADD=		../input/libinput.la \
+					../mapping/libmapping.la \
+					../lib/libmlr.la \
+					libcontainers.la
+
+test_lrec_CPPFLAGS=		-D__TEST_LREC_MAIN__ ${AM_CPPFLAGS}
+test_lrec_LDADD=		../input/libinput.la \
+				../lib/libmlr.la \
+				libcontainers.la
+
+test_multiple_containers_CPPFLAGS=	-D__TEST_MULTIPLE_CONTAINERS_MAIN__ ${AM_CPPFLAGS}
+test_multiple_containers_LDADD=		../input/libinput.la \
+					../lib/libmlr.la \
+					libcontainers.la
+
+test_parse_trie_CPPFLAGS=	-D__TEST_PARSE_TRIE_MAIN__ ${AM_CPPFLAGS}
+test_parse_trie_LDADD=		libcontainers.la
+
+AM_CPPFLAGS=	-I${srcdir}/../
+AM_CFLAGS=	-std=gnu99

c/dsls/Makefile.am

@@ -0,0 +1,75 @@
+AM_CPPFLAGS=	-I${srcdir}
+AM_CFLAGS=	-std=gnu99
+
+noinst_PROGRAMS=	lemon pdm fdm
+lemon_SOURCES=		lemon.c
+
+noinst_LTLIBRARIES=	libfdsl.la libpdsl.la
+# if we don't want to distribute the generated sources, move them to a nodist_libfoo_la_SOURCES variable
+libfdsl_la_SOURCES=	filter_dsl_lexer.c filter_dsl_lexer.h filter_dsl_parse.c filter_dsl_parse.h filter_dsl_wrapper.c
+libpdsl_la_SOURCES=	put_dsl_lexer.c put_dsl_lexer.h put_dsl_parse.c put_dsl_parse.h put_dsl_wrapper.c
+CLEANFILES=		filter_dsl_lexer.c filter_dsl_lexer.h filter_dsl_parse.c filter_dsl_parse.h filter_dsl_parse.out \
+			put_dsl_lexer.c put_dsl_lexer.h put_dsl_parse.c put_dsl_parse.h put_dsl_parse.out
+EXTRA_DIST=		\
+			filter_dsl_lexer.l \
+			filter_dsl_parse.y \
+			lempar.c \
+			put_dsl_lexer.l \
+			put_dsl_parse.y
+
+fdm_SOURCES=		filter_dsl_parse.c filter_dsl_parse.h filter_dsl_lexer.c filter_dsl_lexer.h filter_dsl_wrapper.c filter_dsl_wrapper.h
+fdm_LDADD=		../containers/libcontainers.la ../lib/libmlr.la libfdsl.la -lm
+fdm_CFLAGS=		-D __FILTER_DSL_MAIN__ ${AM_CFLAGS}
+pdm_SOURCES=		put_dsl_parse.c put_dsl_parse.h put_dsl_lexer.c put_dsl_lexer.h put_dsl_wrapper.c put_dsl_wrapper.h
+pdm_LDADD=		../containers/libcontainers.la ../lib/libmlr.la libpdsl.la -lm
+pdm_CFLAGS=		-D __PUT_DSL_MAIN__ ${AM_CFLAGS}
+
+put_dsl_wrapper.c: put_dsl_lexer.h
+put_dsl_parse.h: put_dsl_parse.c
+put_dsl_parse.c: put_dsl_parse.y lempar.c lemon
+	./lemon put_dsl_parse.y
+# lemon does not re-generate the file if it exists and is the same
+# this causes problems with the dependency above
+	touch put_dsl_parse.h
+	mv put_dsl_parse.c put_dsl_parse.c.tmp
+	sed \
+			-e 's/ParseTrace/put_dsl_ParseTrace/g' \
+			-e 's/ParseTokenName/put_dsl_ParseTokenName/g' \
+			-e 's/lemon_parser_alloc/put_dsl_lemon_parser_alloc/g' \
+			-e 's/lemon_parser_free/put_dsl_lemon_parser_free/g' \
+			-e 's/lemon_parser_parse_token/put_dsl_lemon_parser_parse_token/g' \
+			-e 's/yy_destructor/put_dsl_yy_destructor/g' \
+		put_dsl_parse.c.tmp > put_dsl_parse.c
+	rm -f put_dsl_parse.c.tmp
+
+
+put_dsl_lexer.h: put_dsl_lexer.c
+put_dsl_lexer.c: put_dsl_lexer.l put_dsl_parse.h
+	flex --prefix=put_dsl_lexer_ --outfile=put_dsl_lexer.c --header-file=put_dsl_lexer.h ${srcdir}/put_dsl_lexer.l
+
+filter_dsl_wrapper.c: filter_dsl_lexer.h
+filter_dsl_parse.h: filter_dsl_parse.c
+filter_dsl_parse.c: filter_dsl_parse.y lempar.c lemon
+	./lemon filter_dsl_parse.y
+# lemon does not re-generate the file if it exists and is the same
+# this causes problems with the dependency above
+	touch filter_dsl_parse.h
+	mv filter_dsl_parse.c filter_dsl_parse.c.tmp
+	sed \
+			-e 's/ParseTrace/filter_dsl_ParseTrace/g' \
+			-e 's/ParseTokenName/filter_dsl_ParseTokenName/g' \
+			-e 's/lemon_parser_alloc/filter_dsl_lemon_parser_alloc/g' \
+			-e 's/lemon_parser_free/filter_dsl_lemon_parser_free/g' \
+			-e 's/lemon_parser_parse_token/filter_dsl_lemon_parser_parse_token/g' \
+			-e 's/yy_destructor/filter_dsl_yy_destructor/g' \
+		filter_dsl_parse.c.tmp > filter_dsl_parse.c
+	rm -f filter_dsl_parse.c.tmp
+
+filter_dsl_lexer.h: filter_dsl_lexer.c
+filter_dsl_lexer.c: filter_dsl_lexer.l filter_dsl_parse.c
+	flex --prefix=filter_dsl_lexer_ --outfile=filter_dsl_lexer.c --header-file=filter_dsl_lexer.h ${srcdir}/filter_dsl_lexer.l
+
+distclean-local:
+	if [ "${srcdir}" != "." ]; then \
+		rm -f filter_dsl_parse.y lempar.c put_dsl_parse.y; \
+	fi

c/dsls/lemon.c

@@ -11,6 +11,7 @@
 #include <string.h>
 #include <ctype.h>
 #include <stdlib.h>
+#include <unistd.h>
 
 #ifndef __WIN32__
 #   if defined(_WIN32) || defined(WIN32)
@@ -2274,7 +2275,7 @@ static void parseonetoken(struct pstate *psp)
 ** macros.  This routine looks for "%ifdef" and "%ifndef" and "%endif" and
 ** comments them out.  Text in between is also commented out as appropriate.
 */
-static preprocess_input(char *z){
+static int preprocess_input(char *z){
 	int i, j, k, n;
 	int exclude = 0;
 	int start;
@@ -2755,7 +2756,6 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask)
 	char *pathlist;
 	char *path,*cp;
 	char c;
-	extern int access();
 
 #ifdef __WIN32__
 	cp = strrchr(argv0,'\\');

c/experimental/Makefile.am

@@ -0,0 +1,7 @@
+# TODO: replace the interesting content with unit tests; jettison the rest
+noinst_PROGRAMS=	getl
+AM_CFLAGS=		-std=gnu99
+AM_CPPFLAGS=		-I${srcdir}/../
+
+getl_SOURCES=	getlines.c
+getl_LDADD=	../lib/libmlr.la ../input/libinput.la ../containers/libcontainers.la

c/input/Makefile.am

@@ -0,0 +1,51 @@
+noinst_LTLIBRARIES=	libinput.la
+libinput_la_SOURCES=	\
+			byte_reader.h \
+			byte_readers.h \
+			file_reader_mmap.c \
+			file_reader_mmap.h \
+			file_reader_stdio.c \
+			file_reader_stdio.h \
+			line_readers.c \
+			line_readers.h \
+			lrec_reader.h \
+			lrec_reader_csv.c \
+			lrec_reader_in_memory.c \
+			lrec_reader_mmap_csvlite.c \
+			lrec_reader_mmap_dkvp.c \
+			lrec_reader_mmap_nidx.c \
+			lrec_reader_mmap_xtab.c \
+			lrec_reader_stdio_csvlite.c \
+			lrec_reader_stdio_dkvp.c \
+			lrec_reader_stdio_nidx.c \
+			lrec_reader_stdio_xtab.c \
+			lrec_readers.c \
+			lrec_readers.h \
+			mmap_byte_reader.c \
+			peek_file_reader.c \
+			peek_file_reader.h \
+			stdio_byte_reader.c \
+			string_byte_reader.c
+libinput_la_LIBADD=	../lib/libmlr.la
+libinput_la_CPPFLAGS=	-I${srcdir}/../
+
+TESTS=			$(check_PROGRAMS)
+check_PROGRAMS=		test_byte_readers test_peek_file_reader
+# TODO: replace the interesting content with unit tests; jettison the rest
+# TODO: causes circular dependency
+#noinst_PROGRAMS=	lrim
+
+#lrim_SOURCES=		lrec_reader_in_memory.c
+#lrim_CPPFLAGS=		-D__LREC_READER_IN_MEMORY_MAIN__ ${AM_CPPFLAGS}
+#lrim_LDADD=		../containers/libcontainers.la ../lib/libmlr.la # -lm
+
+test_byte_readers_CPPFLAGS=	-D__TEST_BYTE_READERS_MAIN__ ${AM_CPPFLAGS}
+test_byte_readers_LDADD=	libinput.la \
+				../lib/libmlr.la
+
+test_peek_file_reader_CPPFLAGS=	-D__TEST_PEEK_FILE_READER_MAIN__ ${AM_CPPFLAGS}
+test_peek_file_reader_LDADD=	libinput.la
+
+
+AM_CPPFLAGS=	-I${srcdir}/../
+AM_CFLAGS=	-std=gnu99

c/lib/Makefile.am

@@ -0,0 +1,29 @@
+noinst_LTLIBRARIES=	libmlr.la
+libmlr_la_SOURCES=	minunit.h \
+			mlr_globals.c \
+			mlr_globals.h \
+			mlrmath.c \
+			mlrmath.h \
+			mlrstat.c \
+			mlrstat.h \
+			mlrutil.c \
+			mlrutil.h \
+			mtrand.c \
+			mtrand.h \
+			string_builder.c \
+			string_builder.h \
+			mlr_test_util.c \
+			mlr_test_util.h
+
+TESTS=		$(check_PROGRAMS)
+check_PROGRAMS=	test_mlrutil test_string_builder
+
+test_mlrutil_LDADD=		libmlr.la
+test_mlrutil_CPPFLAGS=		-D__TEST_MLRUTIL_MAIN__ ${AM_CPPFLAGS}
+
+test_string_builder_LDADD=	libmlr.la
+test_string_builder_CPPFLAGS=	-D__TEST_STRING_BUILDER_MAIN__ ${AM_CPPFLAGS}
+
+
+AM_CPPFLAGS=	-I${srcdir}/../
+AM_CFLAGS=	-std=gnu99

c/mapping/Makefile.am

@@ -0,0 +1,44 @@
+noinst_LTLIBRARIES=	libmapping.la
+libmapping_la_SOURCES=	\
+			context.c \
+			context.h \
+			lrec_evaluator.h \
+			lrec_evaluators.c \
+			lrec_evaluators.h \
+			mapper.h \
+			mapper_cat.c \
+			mapper_check.c \
+			mapper_cut.c \
+			mapper_filter.c \
+			mapper_group_like.c \
+			mapper_having_fields.c \
+			mapper_head.c \
+			mapper_histogram.c \
+			mapper_join.c \
+			mapper_label.c \
+			mapper_put.c \
+			mapper_regularize.c \
+			mapper_rename.c \
+			mapper_reorder.c \
+			mapper_sort.c \
+			mapper_stats1.c \
+			mapper_stats2.c \
+			mapper_step.c \
+			mapper_tac.c \
+			mapper_tail.c \
+			mapper_top.c \
+			mapper_uniq.c \
+			mappers.h \
+			mlr_val.c \
+			mlr_val.h
+libmapping_la_LIBADD=	../lib/libmlr.la ../cli/libcli.la ../input/libinput.la
+
+# TODO: replace the interesting content with unit tests; jettison the rest
+# TODO: Does not build
+#noinst_PROGRAMS=	evl
+#evl_SOURCES=		lrec_evaluators.c
+#evl_LDADD=		libmapping.la ../containers/libcontainers.la ../lib/libmlr.la
+#evl_CPPFLAGS=		-D__LREC_EVALUATORS_MAIN__ ${AM_CPPFLAGS}
+
+AM_CPPFLAGS=	-I${srcdir}/../
+AM_CFLAGS=	-std=gnu99

c/output/Makefile.am

@@ -0,0 +1,13 @@
+noinst_LTLIBRARIES=	liboutput.la
+liboutput_la_SOURCES=	\
+			lrec_writer.h \
+			lrec_writer_csv.c \
+			lrec_writer_csvlite.c \
+			lrec_writer_dkvp.c \
+			lrec_writer_nidx.c \
+			lrec_writer_pprint.c \
+			lrec_writer_xtab.c \
+			lrec_writers.h
+liboutput_la_LIBADD=	../lib/libmlr.la
+liboutput_la_CPPFLAGS=	-I${srcdir}/../
+liboutput_la_CFLAGS=	-std=gnu99

c/stream/Makefile.am

@@ -0,0 +1,4 @@
+noinst_LTLIBRARIES=	libstream.la
+libstream_la_SOURCES=	stream.c stream.h
+libstream_la_CPPFLAGS=	-I${srcdir}/../
+libstream_la_CFLAGS=	-std=gnu99

c/test/Makefile.am

@@ -0,0 +1,8 @@
+SUBDIRS=	input expected
+EXTRA_DIST=	run README.md
+
+TESTS=		run
+
+# created by 'run'
+clean-local:
+	rm -rf output

c/test/expected/Makefile.am

@@ -0,0 +1 @@
+EXTRA_DIST= out

c/test/input/Makefile.am

@@ -0,0 +1,52 @@
+SUBDIRS=	rfc-csv
+EXTRA_DIST=	\
+		a.csv \
+		a.pprint \
+		abixy \
+		abixy-wide \
+		b.csv \
+		b.pprint \
+		c.csv \
+		c.pprint \
+		d.csv \
+		d.pprint \
+		dots.pprint \
+		dots.xtab \
+		double-ps.dkvp \
+		e.csv \
+		e.pprint \
+		f.csv \
+		f.pprint \
+		fsec2xhms \
+		g.csv \
+		g.pprint \
+		gmt2sec \
+		het.csv \
+		het.dkvp \
+		joina.dkvp \
+		joinb.dkvp \
+		minmax.dkvp \
+		missings.dkvp \
+		multi-sep.csvl \
+		multi-sep.dkvp \
+		null-fields.csv \
+		null-fields.nidx \
+		regularize.dkvp \
+		rfc-csv \
+		sec2gmt \
+		sec2xhms \
+		short \
+		small \
+		space-pad.dkvp \
+		space-pad.nidx \
+		space-pad.pprint \
+		truncated.csv \
+		truncated.dkvp \
+		truncated.nidx \
+		truncated.pprint \
+		truncated.xtab \
+		truncated.xtab-crlf \
+		utf8-1.csv \
+		utf8-2.csv \
+		utf8-align.dkvp \
+		utf8-align.nidx

c/test/input/rfc-csv/Makefile.am

@@ -0,0 +1,10 @@
+EXTRA_DIST=	\
+		modify-defaults.csv \
+		narrow-truncated.csv \
+		narrow.csv \
+		quoted-comma-truncated.csv \
+		quoted-comma.csv \
+		quoted-crlf-truncated.csv \
+		quoted-crlf.csv \
+		simple-truncated.csv \
+		simple.csv

c/tools/Makefile.am

@@ -0,0 +1,3 @@
+# TODO: replace the interesting content with unit tests; jettison the rest
+noinst_PROGRAMS=	termcvt
+AM_CFLAGS=		-std=gnu99

configure.ac

@@ -0,0 +1,41 @@
+AC_PREREQ([2.60])
+# Manually increment on updates to https://github.com/johnkerl/miller/releases
+AC_INIT([mlr],[2.2.0])
+AC_CONFIG_SRCDIR([c/mlrmain.c])
+AC_CONFIG_HEADERS([config.h])
+AC_CONFIG_AUX_DIR([autotools])
+AC_CONFIG_MACRO_DIR([m4])
+AM_INIT_AUTOMAKE
+
+AC_CANONICAL_HOST
+AC_PROG_CC
+AM_PROG_AR
+AM_PROG_LEX
+AC_EXEEXT
+LT_INIT
+
+
+# TODO: better source handling for lemon sources?
+# perhaps lemon can be improved to survive being called from the build dir
+AC_CONFIG_LINKS([c/dsls/filter_dsl_parse.y:c/dsls/filter_dsl_parse.y])
+AC_CONFIG_LINKS([c/dsls/lempar.c:c/dsls/lempar.c])
+AC_CONFIG_LINKS([c/dsls/put_dsl_parse.y:c/dsls/put_dsl_parse.y])
+
+AC_CONFIG_FILES([Makefile
+		c/cli/Makefile
+		c/containers/Makefile
+		c/dsls/Makefile
+		c/experimental/Makefile
+		c/input/Makefile
+		c/lib/Makefile
+		c/mapping/Makefile
+		c/output/Makefile
+		c/stream/Makefile
+		c/test/Makefile
+		c/test/expected/Makefile
+		c/test/input/Makefile
+		c/test/input/rfc-csv/Makefile
+		c/tools/Makefile
+		c/Makefile
+		doc/Makefile])
+AC_OUTPUT

doc/Makefile.am

@@ -0,0 +1,7 @@
+EXTRA_DIST=	miller.1.txt miller.1
+man1_MANS=	miller.1
+
+MAINTAINERCLEANFILES=	miller.1
+
+miller.1: miller.1.txt
+	a2x -d manpage -f manpage miller.1.txt