diff --git a/docs/src/data/sample2.rec b/docs/src/data/sample2.rec
new file mode 100644
index 000000000..0bf31e4ec
--- /dev/null
+++ b/docs/src/data/sample2.rec
@@ -0,0 +1,11 @@
+# A record descriptor whose %doc field has no value of its own -- just a
+# bare colon -- with the actual text supplied entirely by the "+"
+# continuation line that follows.
+# Source: https://www.gnu.org/software/recutils/manual/html_node/A-Little-Example.html
+
+%rec: Book
+%doc:
++ A book in my personal collection.
+
+Title: GNU Emacs Manual
+Author: Richard M. Stallman
diff --git a/docs/src/file-formats.md b/docs/src/file-formats.md
index c7349ce15..f5966b714 100644
--- a/docs/src/file-formats.md
+++ b/docs/src/file-formats.md
@@ -994,6 +994,43 @@ Since `#`-prefixed lines are, like every other Miller format, treated as data un
Miller has no notion of recutils' record-descriptor/schema records (lines starting with `%rec:` which declare field types, mandatory fields, keys, and so on) -- those are read and written as ordinary records, with no special interpretation, since Miller is a schema-less stream processor.
+A field can also be given an empty value of its own -- just `Name:` with nothing after the colon, not even a space -- with its real value supplied entirely by the `+` continuation line(s) that follow. This is how GNU recutils' own `%doc` record-descriptor field is commonly written:
+
+
+cat data/sample2.rec
+
+
+# A record descriptor whose %doc field has no value of its own -- just a
+# bare colon -- with the actual text supplied entirely by the "+"
+# continuation line that follows.
+# Source: https://www.gnu.org/software/recutils/manual/html_node/A-Little-Example.html
+
+%rec: Book
+%doc:
++ A book in my personal collection.
+
+Title: GNU Emacs Manual
+Author: Richard M. Stallman
+
+
+
+mlr --skip-comments -i recutils -o json cat data/sample2.rec
+
+
+[
+{
+ "%rec": "Book",
+ "%doc": "A book in my personal collection."
+},
+{
+ "Title": "GNU Emacs Manual",
+ "Author": "Richard M. Stallman"
+}
+]
+
+
+Here the continuation becomes the field's value outright -- unlike the non-empty-value case above, there is no leading embedded newline.
+
Use `--irecutils`/`--orecutils`/`--recutils` (or `-i recutils`/`-o recutils`) for recutils input/output/both, analogously to `--idcf`/`--odcf`/`--dcf`.
Note: a field value whose last line ends in a literal `\` is ambiguous with an in-progress backslash-continuation on write/re-read, since recutils has no in-value backslash-escaping mechanism. This is a limitation of the recutils format itself, not specific to Miller.
diff --git a/docs/src/file-formats.md.in b/docs/src/file-formats.md.in
index 1599fafc5..cbe854cf0 100644
--- a/docs/src/file-formats.md.in
+++ b/docs/src/file-formats.md.in
@@ -480,6 +480,18 @@ GENMD-EOF
Miller has no notion of recutils' record-descriptor/schema records (lines starting with `%rec:` which declare field types, mandatory fields, keys, and so on) -- those are read and written as ordinary records, with no special interpretation, since Miller is a schema-less stream processor.
+A field can also be given an empty value of its own -- just `Name:` with nothing after the colon, not even a space -- with its real value supplied entirely by the `+` continuation line(s) that follow. This is how GNU recutils' own `%doc` record-descriptor field is commonly written:
+
+GENMD-RUN-COMMAND
+cat data/sample2.rec
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --skip-comments -i recutils -o json cat data/sample2.rec
+GENMD-EOF
+
+Here the continuation becomes the field's value outright -- unlike the non-empty-value case above, there is no leading embedded newline.
+
Use `--irecutils`/`--orecutils`/`--recutils` (or `-i recutils`/`-o recutils`) for recutils input/output/both, analogously to `--idcf`/`--odcf`/`--dcf`.
Note: a field value whose last line ends in a literal `\` is ambiguous with an in-progress backslash-continuation on write/re-read, since recutils has no in-value backslash-escaping mechanism. This is a limitation of the recutils format itself, not specific to Miller.
diff --git a/pkg/input/record_reader_rec.go b/pkg/input/record_reader_rec.go
index f8d05db7e..fab913acf 100644
--- a/pkg/input/record_reader_rec.go
+++ b/pkg/input/record_reader_rec.go
@@ -149,21 +149,21 @@ func (reader *RecordReaderREC) getRecordBatch(
}
// recordFromRECLines converts one blank-line-delimited stanza's raw lines
-// into a Miller record. This happens in three passes, in this order:
+// into a Miller record. This happens in two passes, in this order:
//
// 1. Backslash-newline logical-line joining: a line ending in a literal "\"
// is joined directly (no separator inserted) with the next physical
// line.
-// 2. "+"-continuation folding: a line starting with "+" continues the
-// immediately preceding field's value with an embedded "\n"; a single
-// leading space after the "+" is stripped.
-// 3. Splitting each remaining logical line on the first occurrence of
-// ": " into a field name and value.
+// 2. Field parsing with "+"-continuation folding: each remaining logical
+// line is either a "+"-continuation, which is folded into the
+// immediately preceding field's value with an embedded "\n" (a single
+// leading space after the "+" is stripped), or a "Name:" / "Name: value"
+// field line.
//
// Malformed input (a "+"-continuation with no preceding field, or a line
-// with no ": " separator) is a hard error -- recutils' own spec requires
-// exactly ": " as the field separator, and there is no leniency flag for
-// this format (unlike e.g. CSV's ragged-input handling).
+// with no ":" separator) is a hard error -- recutils' own spec requires a
+// colon as the field separator, and there is no leniency flag for this
+// format (unlike e.g. CSV's ragged-input handling).
func (reader *RecordReaderREC) recordFromRECLines(
stanza []string,
) (*mlrval.Mlrmap, error) {
@@ -172,20 +172,13 @@ func (reader *RecordReaderREC) recordFromRECLines(
joinedLines := joinRECBackslashContinuations(stanza)
- fieldLines, err := foldRECPlusContinuations(joinedLines)
+ fields, err := parseRECFields(joinedLines)
if err != nil {
return nil, err
}
- for _, line := range fieldLines {
- key, value, found := strings.Cut(line, ": ")
- if !found {
- return nil, fmt.Errorf(
- "mlr: recutils: missing \": \" field separator in line %q",
- line,
- )
- }
- reader.recordArena.PutDeferred(record, key, value, dedupeFieldNames)
+ for _, field := range fields {
+ reader.recordArena.PutDeferred(record, field.key, field.value, dedupeFieldNames)
}
return record, nil
@@ -224,28 +217,68 @@ func joinRECBackslashContinuations(lines []string) []string {
return joined
}
-// foldRECPlusContinuations implements recutils' "+"-continuation: a line
-// starting with "+" continues the immediately preceding field's value with
-// an embedded "\n". A single leading space after the "+" is stripped (a
-// bare "+" folds in an empty continuation line).
-func foldRECPlusContinuations(lines []string) ([]string, error) {
- folded := make([]string, 0, len(lines))
+// tRECField is one field's key/value pair, parsed out of a rec-format
+// stanza line, prior to being placed into a Miller record.
+type tRECField struct {
+ key string
+ value string
+}
+
+// parseRECFields splits a stanza's backslash-joined lines into ordered
+// key/value fields, folding "+"-continuation lines into the value of the
+// immediately preceding field as it goes. Folding must happen at the
+// key/value level, not the raw-line level: a field's value may itself be
+// empty -- a bare "Name:" with nothing after the colon, its actual value
+// supplied entirely by the "+" lines that follow -- and folding raw lines
+// together would destroy the ":" separator needed to parse that field out.
+// A single leading space after the "+" is stripped (a bare "+" folds in an
+// empty continuation line). When the preceding value is empty, the
+// continuation becomes the value outright rather than being appended after
+// a spurious leading "\n".
+func parseRECFields(lines []string) ([]tRECField, error) {
+ fields := make([]tRECField, 0, len(lines))
for _, line := range lines {
- if strings.HasPrefix(line, "+") {
- if len(folded) == 0 {
+ if rest, ok := strings.CutPrefix(line, "+"); ok {
+ if len(fields) == 0 {
return nil, fmt.Errorf(
"mlr: recutils: continuation line %q has no preceding field in this record",
line,
)
}
- continuation := strings.TrimPrefix(line, "+")
- continuation = strings.TrimPrefix(continuation, " ")
- folded[len(folded)-1] = folded[len(folded)-1] + "\n" + continuation
- } else {
- folded = append(folded, line)
+ continuation := strings.TrimPrefix(rest, " ")
+ last := &fields[len(fields)-1]
+ if last.value == "" {
+ last.value = continuation
+ } else {
+ last.value = last.value + "\n" + continuation
+ }
+ continue
}
+
+ key, rest, found := strings.Cut(line, ":")
+ if !found {
+ return nil, fmt.Errorf(
+ "mlr: recutils: missing \":\" field separator in line %q",
+ line,
+ )
+ }
+
+ var value string
+ switch {
+ case rest == "":
+ value = ""
+ case strings.HasPrefix(rest, " "):
+ value = rest[1:]
+ default:
+ return nil, fmt.Errorf(
+ "mlr: recutils: missing \": \" field separator in line %q",
+ line,
+ )
+ }
+
+ fields = append(fields, tRECField{key: key, value: value})
}
- return folded, nil
+ return fields, nil
}
diff --git a/pkg/input/record_reader_rec_test.go b/pkg/input/record_reader_rec_test.go
index b8c0d722a..0a670ef65 100644
--- a/pkg/input/record_reader_rec_test.go
+++ b/pkg/input/record_reader_rec_test.go
@@ -127,8 +127,11 @@ func TestRecordFromRECLinesMissingColonSpaceIsError(t *testing.T) {
func TestRecordFromRECLinesMissingColonSpaceBareColonIsError(t *testing.T) {
reader := newTestRECReader(t)
- // Per the recutils spec, the separator is exactly ": " (colon-space) --
- // a bare colon with no following space does not count.
+ // Per the recutils spec, a non-empty value must be separated from the
+ // field name by ": " (colon-space) -- a colon directly followed by
+ // value text with no separating space is malformed. (A colon followed
+ // by nothing at all is a different, valid case: an empty value -- see
+ // TestRecordFromRECLinesBareColonEmptyValue.)
record, err := reader.recordFromRECLines([]string{
"Foo:bar",
})
@@ -153,11 +156,46 @@ func TestJoinRECBackslashContinuations(t *testing.T) {
)
}
-func TestFoldRECPlusContinuations(t *testing.T) {
- folded, err := foldRECPlusContinuations([]string{"a: 1", "+ 2", "+3"})
+func TestParseRECFields(t *testing.T) {
+ fields, err := parseRECFields([]string{"a: 1", "+ 2", "+3"})
assert.Nil(t, err)
- assert.Equal(t, []string{"a: 1\n2\n3"}, folded)
+ assert.Equal(t, []tRECField{{key: "a", value: "1\n2\n3"}}, fields)
- _, err = foldRECPlusContinuations([]string{"+ orphan"})
+ _, err = parseRECFields([]string{"+ orphan"})
assert.NotNil(t, err)
}
+
+func TestRecordFromRECLinesBareColonEmptyValue(t *testing.T) {
+ reader := newTestRECReader(t)
+
+ // Per the recutils spec, a field's value may be empty: a colon with
+ // nothing after it (not even a space) is valid, distinct from the
+ // "Foo:bar" case above which is missing the required separator space.
+ record, err := reader.recordFromRECLines([]string{
+ "notes:",
+ })
+ assert.Nil(t, err)
+ assert.NotNil(t, record)
+ assert.Equal(t, int64(1), record.FieldCount)
+ assert.Equal(t, "notes", record.Head.Key)
+ assert.Equal(t, "", record.Head.Value.String())
+}
+
+func TestRecordFromRECLinesBareColonPlusContinuation(t *testing.T) {
+ reader := newTestRECReader(t)
+
+ // This is the idiomatic recutils "%doc:" pattern from the official
+ // manual's tutorial example: an empty-valued field whose actual value
+ // is supplied entirely by the "+"-continuation lines that follow. The
+ // continuation becomes the value outright, with no spurious leading
+ // "\n" from folding onto an empty base value.
+ record, err := reader.recordFromRECLines([]string{
+ "%doc:",
+ "+ A book in my personal collection.",
+ })
+ assert.Nil(t, err)
+ assert.NotNil(t, record)
+ assert.Equal(t, int64(1), record.FieldCount)
+ assert.Equal(t, "%doc", record.Head.Key)
+ assert.Equal(t, "A book in my personal collection.", record.Head.Value.String())
+}
diff --git a/test/cases/io-recutils/0006/cmd b/test/cases/io-recutils/0006/cmd
new file mode 100644
index 000000000..0b128ab6d
--- /dev/null
+++ b/test/cases/io-recutils/0006/cmd
@@ -0,0 +1 @@
+mlr --skip-comments --irecutils --ojson cat test/input/books.rec
diff --git a/test/cases/io-recutils/0006/experr b/test/cases/io-recutils/0006/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-recutils/0006/expout b/test/cases/io-recutils/0006/expout
new file mode 100644
index 000000000..4557cde66
--- /dev/null
+++ b/test/cases/io-recutils/0006/expout
@@ -0,0 +1,34 @@
+[
+{
+ "%rec": "Book",
+ "%mandatory": "Title",
+ "%type": "Location enum loaned home unknown",
+ "%doc": "A book in my personal collection."
+},
+{
+ "Title": "GNU Emacs Manual",
+ "Author": "Richard M. Stallman",
+ "Publisher": "FSF",
+ "Location": "home"
+},
+{
+ "Title": "The Colour of Magic",
+ "Author": "Terry Pratchett",
+ "Location": "loaned"
+},
+{
+ "Title": "Mio Cid",
+ "Author": "Anonymous",
+ "Location": "home"
+},
+{
+ "Title": "chapters.gnu.org administration guide",
+ "Author": "Nacho Gonzalez",
+ "Author_2": "Jose E. Marchesi",
+ "Location": "unknown"
+},
+{
+ "Title": "Yeelong User Manual",
+ "Location": "home"
+}
+]
diff --git a/test/cases/io-recutils/0007/cmd b/test/cases/io-recutils/0007/cmd
new file mode 100644
index 000000000..9f7e59965
--- /dev/null
+++ b/test/cases/io-recutils/0007/cmd
@@ -0,0 +1 @@
+mlr --skip-comments --irecutils --orecutils cat test/input/books.rec
diff --git a/test/cases/io-recutils/0007/experr b/test/cases/io-recutils/0007/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/io-recutils/0007/expout b/test/cases/io-recutils/0007/expout
new file mode 100644
index 000000000..cde65fa56
--- /dev/null
+++ b/test/cases/io-recutils/0007/expout
@@ -0,0 +1,26 @@
+%rec: Book
+%mandatory: Title
+%type: Location enum loaned home unknown
+%doc: A book in my personal collection.
+
+Title: GNU Emacs Manual
+Author: Richard M. Stallman
+Publisher: FSF
+Location: home
+
+Title: The Colour of Magic
+Author: Terry Pratchett
+Location: loaned
+
+Title: Mio Cid
+Author: Anonymous
+Location: home
+
+Title: chapters.gnu.org administration guide
+Author: Nacho Gonzalez
+Author_2: Jose E. Marchesi
+Location: unknown
+
+Title: Yeelong User Manual
+Location: home
+
diff --git a/test/input/books.rec b/test/input/books.rec
new file mode 100644
index 000000000..3f233d7ed
--- /dev/null
+++ b/test/input/books.rec
@@ -0,0 +1,31 @@
+# -*- mode: rec -*-
+# Source: https://www.gnu.org/software/recutils/manual/html_node/A-Little-Example.html
+
+%rec: Book
+%mandatory: Title
+%type: Location enum loaned home unknown
+%doc:
++ A book in my personal collection.
+
+Title: GNU Emacs Manual
+Author: Richard M. Stallman
+Publisher: FSF
+Location: home
+
+Title: The Colour of Magic
+Author: Terry Pratchett
+Location: loaned
+
+Title: Mio Cid
+Author: Anonymous
+Location: home
+
+Title: chapters.gnu.org administration guide
+Author: Nacho Gonzalez
+Author: Jose E. Marchesi
+Location: unknown
+
+Title: Yeelong User Manual
+Location: home
+
+# End of books.rec