From 950337fde14bf8ab8f0bf0b4af18d48719da05d2 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Sun, 7 Nov 2021 23:41:15 -0500 Subject: [PATCH] Simplify release & dev build processes (#735) --- Makefile | 22 ++++++++++++++++++- docs/Makefile | 6 +++++ docs/mkdocs-readme.md.not | 13 +++++------ docs/src/Makefile | 31 ++++++-------------------- docs/src/build.md | 4 ++-- docs/src/build.md.in | 4 ++-- docs/src/contributing.md | 7 +++--- docs/src/contributing.md.in | 7 +++--- docs/src/make.bat | 35 ------------------------------ docs/src/manpage.md | 2 +- docs/src/manpage.txt | 2 +- go/src/cli/flag_types.go | 8 +++---- go/src/input/record_reader_csv.go | 2 +- go/src/output/record_writer_csv.go | 6 ++++- man/manpage.txt | 2 +- man/mlr.1 | 4 ++-- 16 files changed, 64 insertions(+), 91 deletions(-) create mode 100644 docs/Makefile delete mode 100644 docs/src/make.bat diff --git a/Makefile b/Makefile index 79f7cff9b..31d441116 100644 --- a/Makefile +++ b/Makefile @@ -12,5 +12,25 @@ install: make -C go install make -C man install +# For developers before pushing to GitHub. +# +# These steps are done in a particular order: +# go: +# * builds the mlr executable +# man: +# * creates manpage mlr.1 and manpage.txt using mlr from the $PATH +# * copies the latter to docs/src +# docs: +# * turns *.md.in into *.md (live code samples), using mlr from the $PATH +# * note the man/manpage.txt becomes some of the HTML content +# * turns *.md into docs/site HTML and CSS files +precommit: + make -C go fmt + make -C go build + make -C go check + make -C man build + make -C docs + echo DONE + # Go does its own dependency management, outside of make. -.PHONY: build +.PHONY: build precommit diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..acc03b3d3 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,6 @@ +# Turn src/*.md.in into respective src/*.md. Then, generate static HTML into ./site. +build: + make -C src genmds + mkdocs build + +.PHONY: build diff --git a/docs/mkdocs-readme.md.not b/docs/mkdocs-readme.md.not index 7052a267d..876c421bd 100644 --- a/docs/mkdocs-readme.md.not +++ b/docs/mkdocs-readme.md.not @@ -16,19 +16,16 @@ * `docs/src` has `*.md.in` files containing markdown as well as directives for auto-generating code samples. * A `genmds` script reads `docs/src/*.md.in` and writes `docs/src/*.md`. * The `mkdocs build` tools reads `docs/src/*.md` and writes HTML files in `docs/site`. + * Running `make` within the `docs` directory handles both of those steps. * Quick-editing loop: * In one terminal, cd to this directory and leave `mkdocs serve` running. * In another terminal, cd to the `src` subdirectory of `docs` and edit `*.md.in`. - * Run `genmds` to re-create all the `*.md` files, or `genmds foo.md.in` to just re-create the `foo.md.in` file you just edited. + * Run `genmds` to re-create all the `*.md` files, or `genmds foo.md.in` to just re-create the `foo.md.in` file you just edited, or (simplest) just `make` within the `docs/src` subdirectory. * In your browser, visit http://127.0.0.1:8000 - * This doesn't write HTML in `docs/site`; HTML is served up directly in the browser -- this is nice for previewing interactive edits + * This doesn't write HTML in `docs/site`; HTML is served up directly in the browser -- this is nice for previewing interactive edits. * For-publish editing loop: - * Leave one terminal open as a place you will run `mkdocs build` - * In one terminal, cd to the `src` subdirectory of `docs` and edit `*.md.in`. - * Generate `src/*.md` from `src/*.md.in`, and then from that generate the `site/*/*.html`: - * Run `genmds` to re-create all the `*.md` files, or `genmds foo.md.in` to just re-create the `foo.md.in` file you just edited. - * In the first terminal, run `mkdocs build` which will populate the `site` directory. - * See also [./regen.sh](./regen.sh) which combines the `genmds` and `mkdocs build` steps. + * cd to the `src` subdirectory of `docs` and edit `*.md.in`. + * `make -C ..` * This does write HTML in `docs/site`. * In your browser, visit `file:///your/path/to/miller/docs/site/index.html` * Link-checking: diff --git a/docs/src/Makefile b/docs/src/Makefile index 97b18a775..ef0637382 100644 --- a/docs/src/Makefile +++ b/docs/src/Makefile @@ -1,28 +1,11 @@ -# Minimal makefile for Sphinx documentation -# -# Note: run this after make in the ../c directory and make in the ../man directory -# since ../c/mlr is used to autogenerate ../man/manpage.txt which is used in this directory. -# See also https://miller.readthedocs.io/en/latest/build.html#creating-a-new-release-for-developers +# Miller docs have lots of text generated by running the Miller executable. We +# have source files named like foo.md.in which are writable; the ./genmds +# script turns each into its respective foo.md file. -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build +FILES=$(wildcard *.md) -# Respective MANPATH entries would include /usr/local/share/man or $HOME/man. -INSTALLDIR=/usr/local/share/man/man1 -INSTALLHOME=$(HOME)/man/man1 +genmds: $(FILES) -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +%.md: %.md.in + @./genmds $< -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - ./genmds - $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/src/build.md b/docs/src/build.md index 8c543aaa6..fecaa313a 100644 --- a/docs/src/build.md +++ b/docs/src/build.md @@ -69,8 +69,8 @@ In this example I am using version 6.1.0 to 6.2.0; of course that will change fo * Update version found in `mlr --version` and `man mlr`: * Edit `go/src/version/version.go` from `6.1.0-dev` to `6.2.0`. - * `sh build-go-src-test-man-doc.sh` - * The ordering in this script is important: the first build creates `mlr`; the second runs `mlr` to create `manpage.txt`; the third includes `manpage.txt` into one of its outputs. + * Run `make precommit` in the Miller repo base directory + * The ordering in this makefile rule is important: the first build creates `mlr`; the second runs `mlr` to create `manpage.txt`; the third includes `manpage.txt` into one of its outputs. * Commit and push. * Create the release tarball and SRPM: diff --git a/docs/src/build.md.in b/docs/src/build.md.in index 557ea6914..0faac146a 100644 --- a/docs/src/build.md.in +++ b/docs/src/build.md.in @@ -53,8 +53,8 @@ In this example I am using version 6.1.0 to 6.2.0; of course that will change fo * Update version found in `mlr --version` and `man mlr`: * Edit `go/src/version/version.go` from `6.1.0-dev` to `6.2.0`. - * `sh build-go-src-test-man-doc.sh` - * The ordering in this script is important: the first build creates `mlr`; the second runs `mlr` to create `manpage.txt`; the third includes `manpage.txt` into one of its outputs. + * Run `make precommit` in the Miller repo base directory + * The ordering in this makefile rule is important: the first build creates `mlr`; the second runs `mlr` to create `manpage.txt`; the third includes `manpage.txt` into one of its outputs. * Commit and push. * Create the release tarball and SRPM: diff --git a/docs/src/contributing.md b/docs/src/contributing.md index 737dc3cfe..ef9103cdf 100644 --- a/docs/src/contributing.md +++ b/docs/src/contributing.md @@ -48,7 +48,6 @@ PRs which pass regression test ([https://github.com/johnkerl/miller/blob/main/go Much of Miller's documentation is autogenerated from code. With the `miller/go` directory in your `$PATH` (so it will find the modified `mlr` executable if -you've modified any source code), please do `sh build-go-src-test-man-doc.sh` -in the `miller` directory. This runs source build, unit test, regression test, -manual-page autogen, document autogen, and document static-build all in the -correct order. +you've modified any source code), please do `make precommit` in the Miller base +directory. This runs source build, unit test, regression test, manual-page +autogen, document autogen, and document static-build all in the correct order. diff --git a/docs/src/contributing.md.in b/docs/src/contributing.md.in index f2c964948..73791470d 100644 --- a/docs/src/contributing.md.in +++ b/docs/src/contributing.md.in @@ -32,7 +32,6 @@ PRs which pass regression test ([https://github.com/johnkerl/miller/blob/main/go Much of Miller's documentation is autogenerated from code. With the `miller/go` directory in your `$PATH` (so it will find the modified `mlr` executable if -you've modified any source code), please do `sh build-go-src-test-man-doc.sh` -in the `miller` directory. This runs source build, unit test, regression test, -manual-page autogen, document autogen, and document static-build all in the -correct order. +you've modified any source code), please do `make precommit` in the Miller base +directory. This runs source build, unit test, regression test, manual-page +autogen, document autogen, and document static-build all in the correct order. diff --git a/docs/src/make.bat b/docs/src/make.bat deleted file mode 100644 index 2119f5109..000000000 --- a/docs/src/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/src/manpage.md b/docs/src/manpage.md index 631ec3588..075375e98 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -2975,5 +2975,5 @@ SEE ALSO - 2021-11-06 MILLER(1) + 2021-11-08 MILLER(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 085790da2..7b214ec60 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -2954,4 +2954,4 @@ SEE ALSO - 2021-11-06 MILLER(1) + 2021-11-08 MILLER(1) diff --git a/go/src/cli/flag_types.go b/go/src/cli/flag_types.go index 0428fa9ce..e64a83e8e 100644 --- a/go/src/cli/flag_types.go +++ b/go/src/cli/flag_types.go @@ -107,10 +107,10 @@ type Flag struct { // * Any code bits should be marked with backticks. These look OK for // on-line help / manpage, and render marvelously for webdocs which // take markdown. - // * After changing flags you can run `sh build-go-src-test-man-doc.sh` - // followed by `git diff` to see how the output looks. See also - // the README.md files in the docs and man directories for how - // to look at the autogenned docs pre-commit. + // * After changing flags you can run `make precommit` in the Miller + // repo base directory followed by `git diff` to see how the output + // looks. See also the README.md files in the docs and man directories + // for how to look at the autogenned docs pre-commit. help string // A function for parsing the command line, as described above. diff --git a/go/src/input/record_reader_csv.go b/go/src/input/record_reader_csv.go index 36b629094..6e1c7beb3 100644 --- a/go/src/input/record_reader_csv.go +++ b/go/src/input/record_reader_csv.go @@ -23,7 +23,7 @@ type RecordReaderCSV struct { // ---------------------------------------------------------------- func NewRecordReaderCSV(readerOptions *cli.TReaderOptions) (*RecordReaderCSV, error) { if readerOptions.IRS != "\n" { - return nil, errors.New("CSV IRS can only be newline; LF vs CR/LF is autodetected.") + return nil, errors.New("CSV IRS cannot be altered; LF vs CR/LF is autodetected.") } if len(readerOptions.IFS) != 1 { return nil, errors.New("CSV IFS can only be a single character") diff --git a/go/src/output/record_writer_csv.go b/go/src/output/record_writer_csv.go index 4e210c0c2..d62e4c429 100644 --- a/go/src/output/record_writer_csv.go +++ b/go/src/output/record_writer_csv.go @@ -13,6 +13,7 @@ import ( type RecordWriterCSV struct { writerOptions *cli.TWriterOptions + ofs0 byte // Go's CSV library only lets its 'Comma' be a single character csvWriter *csv.Writer // For reporting schema changes: we print a newline and the new header lastJoinedHeader *string @@ -21,8 +22,11 @@ type RecordWriterCSV struct { } func NewRecordWriterCSV(writerOptions *cli.TWriterOptions) (*RecordWriterCSV, error) { + if len(writerOptions.OFS) != 1 { + return nil, errors.New("CSV OFS can only be a single character") + } if writerOptions.ORS != "\n" { - return nil, errors.New("CSV ORS can only be newline") + return nil, errors.New("CSV ORS cannot be altered") } return &RecordWriterCSV{ writerOptions: writerOptions, diff --git a/man/manpage.txt b/man/manpage.txt index 085790da2..7b214ec60 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -2954,4 +2954,4 @@ SEE ALSO - 2021-11-06 MILLER(1) + 2021-11-08 MILLER(1) diff --git a/man/mlr.1 b/man/mlr.1 index 61fd0858f..20d057552 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2021-11-06 +.\" Date: 2021-11-08 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2021-11-06" "\ \&" "\ \&" +.TH "MILLER" "1" "2021-11-08" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~