diff --git a/docs/src/data/join-ignore-empty-left.csv b/docs/src/data/join-ignore-empty-left.csv
new file mode 100644
index 000000000..171fd1d5c
--- /dev/null
+++ b/docs/src/data/join-ignore-empty-left.csv
@@ -0,0 +1,6 @@
+id,code
+3,0000ff
+2,00ff00
+4,ff0000
+,ffffff
+,000000
diff --git a/docs/src/data/join-ignore-empty-right.csv b/docs/src/data/join-ignore-empty-right.csv
new file mode 100644
index 000000000..badd61587
--- /dev/null
+++ b/docs/src/data/join-ignore-empty-right.csv
@@ -0,0 +1,5 @@
+id,color
+4,red
+2,green
+,white
+,black
diff --git a/docs/src/manpage.md b/docs/src/manpage.md
index 6b1d1b71a..782050585 100644
--- a/docs/src/manpage.md
+++ b/docs/src/manpage.md
@@ -1559,6 +1559,13 @@ This is simply a copy of what you should see on running `man mlr` at a command p
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+ --ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt
index 3415b2b84..feb351a26 100644
--- a/docs/src/manpage.txt
+++ b/docs/src/manpage.txt
@@ -1538,6 +1538,13 @@
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+ --ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/docs/src/reference-verbs.md b/docs/src/reference-verbs.md
index 7680eddbb..30c2ede20 100644
--- a/docs/src/reference-verbs.md
+++ b/docs/src/reference-verbs.md
@@ -1940,6 +1940,13 @@ Options:
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+--ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
@@ -2135,6 +2142,55 @@ left_a left_b left_c right_a right_b right_c
1 4 5 1 4 5
+By default, records with an empty-string value in a join field are joined just like any other value -- so two records which are both missing an ID, say, will be paired with one another even though that's rarely what's wanted:
+
+
+mlr --csv cat data/join-ignore-empty-left.csv
+
+
+id,code
+3,0000ff
+2,00ff00
+4,ff0000
+,ffffff
+,000000
+
+
+
+mlr --csv cat data/join-ignore-empty-right.csv
+
+
+id,color
+4,red
+2,green
+,white
+,black
+
+
+
+mlr --csv join -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+
+
+id,code,color
+4,ff0000,red
+2,00ff00,green
+,ffffff,white
+,000000,white
+,ffffff,black
+,000000,black
+
+
+Use `--ignore-empty` to instead treat an empty-string join-field value as if the field were absent, on both the left and right files. Such records are never paired -- not even with one another:
+
+
+mlr --csv join --ignore-empty -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+
+
+id,code,color
+4,ff0000,red
+2,00ff00,green
+
+
## json-parse
diff --git a/docs/src/reference-verbs.md.in b/docs/src/reference-verbs.md.in
index 59d274ea1..4282bd80c 100644
--- a/docs/src/reference-verbs.md.in
+++ b/docs/src/reference-verbs.md.in
@@ -669,6 +669,26 @@ GENMD-RUN-COMMAND
mlr --csvlite --opprint join -j "" --lp left_ --rp right_ -f data/self-join.csv data/self-join.csv
GENMD-EOF
+By default, records with an empty-string value in a join field are joined just like any other value -- so two records which are both missing an ID, say, will be paired with one another even though that's rarely what's wanted:
+
+GENMD-RUN-COMMAND
+mlr --csv cat data/join-ignore-empty-left.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --csv cat data/join-ignore-empty-right.csv
+GENMD-EOF
+
+GENMD-RUN-COMMAND
+mlr --csv join -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+GENMD-EOF
+
+Use `--ignore-empty` to instead treat an empty-string join-field value as if the field were absent, on both the left and right files. Such records are never paired -- not even with one another:
+
+GENMD-RUN-COMMAND
+mlr --csv join --ignore-empty -j id -f data/join-ignore-empty-left.csv data/join-ignore-empty-right.csv
+GENMD-EOF
+
## json-parse
GENMD-RUN-COMMAND
diff --git a/man/manpage.txt b/man/manpage.txt
index 3415b2b84..feb351a26 100644
--- a/man/manpage.txt
+++ b/man/manpage.txt
@@ -1538,6 +1538,13 @@
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+ --ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/man/mlr.1 b/man/mlr.1
index 621f3d71a..e246cde7f 100644
--- a/man/mlr.1
+++ b/man/mlr.1
@@ -1912,6 +1912,13 @@ Options:
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+--ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/pkg/transformers/join.go b/pkg/transformers/join.go
index 8f23e09eb..2a5c45b0d 100644
--- a/pkg/transformers/join.go
+++ b/pkg/transformers/join.go
@@ -26,6 +26,7 @@ var joinOptions = []OptionSpec{
{Flag: "--np", Type: "bool", Desc: "Do not emit paired records."},
{Flag: "--ul", Type: "bool", Desc: "Emit unpaired records from the left file."},
{Flag: "--ur", Type: "bool", Desc: "Emit unpaired records from the right file(s)."},
+ {Flag: "--ignore-empty", Type: "bool", Desc: "Treat records with empty-string values in any join-field as if that join-field were absent, on both the left and right files. Such records are never paired -- not even with one another -- and are treated as unpaired, subject to --np/--ul/--ur as usual."},
{Flag: "-s", Aliases: []string{"--sorted-input"}, Type: "bool", Desc: "Require sorted input: records must be sorted lexically by their join-field names, else not all records will be paired. The only likely use case for this is with a left file which is too big to fit into system memory otherwise."},
{Flag: "-u", Type: "bool", Desc: "Enable unsorted input. (This is the default even without -u.) In this case, the entire left file will be loaded into memory."},
{Flag: "--prepipe", Arg: "{command}", Type: "string", Desc: "Shell command to prepipe the left-file input through. As in main input options; see mlr --help for details. If you wish to use a prepipe command for the main input as well as here, it must be specified there as well as here."},
@@ -54,10 +55,11 @@ type tJoinOptions struct {
leftJoinFieldNames []string
rightJoinFieldNames []string
- allowUnsortedInput bool
- emitPairables bool
- emitLeftUnpairables bool
- emitRightUnpairables bool
+ allowUnsortedInput bool
+ emitPairables bool
+ emitLeftUnpairables bool
+ emitRightUnpairables bool
+ ignoreEmptyJoinFields bool
leftFileName string
prepipe string
@@ -77,10 +79,11 @@ func newJoinOptions() *tJoinOptions {
leftJoinFieldNames: nil,
rightJoinFieldNames: nil,
- allowUnsortedInput: true,
- emitPairables: true,
- emitLeftUnpairables: false,
- emitRightUnpairables: false,
+ allowUnsortedInput: true,
+ emitPairables: true,
+ emitLeftUnpairables: false,
+ emitRightUnpairables: false,
+ ignoreEmptyJoinFields: false,
leftFileName: "",
prepipe: "",
@@ -222,6 +225,9 @@ func transformerJoinParseCLI(
case "--ur":
opts.emitRightUnpairables = true
+ case "--ignore-empty":
+ opts.ignoreEmptyJoinFields = true
+
case "-u":
opts.allowUnsortedInput = true
@@ -290,6 +296,18 @@ func transformerJoinParseCLI(
return transformer, nil
}
+// anyValueIsEmpty returns true if any of the given values is present but
+// empty-string (mlrval "void"). Used for --ignore-empty, which treats such
+// join-field values as though the field were absent altogether.
+func anyValueIsEmpty(values []*mlrval.Mlrval) bool {
+ for _, value := range values {
+ if value != nil && value.IsVoid() {
+ return true
+ }
+ }
+ return false
+}
+
type TransformerJoin struct {
opts *tJoinOptions
@@ -352,6 +370,7 @@ func NewTransformerJoin(
&opts.joinFlagOptions.ReaderOptions,
opts.leftJoinFieldNames,
tr.leftKeepFieldNameSet,
+ opts.ignoreEmptyJoinFields,
)
if err != nil {
return nil, err
@@ -400,6 +419,12 @@ func (tr *TransformerJoin) transformHalfStreaming(
groupingKey, hasAllJoinKeys := inrec.GetSelectedValuesJoined(
tr.opts.rightJoinFieldNames,
)
+ if hasAllJoinKeys && tr.opts.ignoreEmptyJoinFields {
+ rightFieldValues, _ := inrec.GetSelectedValues(tr.opts.rightJoinFieldNames)
+ if anyValueIsEmpty(rightFieldValues) {
+ hasAllJoinKeys = false
+ }
+ }
if hasAllJoinKeys {
leftBucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey)
if leftBucket == nil {
@@ -447,6 +472,9 @@ func (tr *TransformerJoin) transformDoublyStreaming(
rightFieldValues, hasAllJoinKeys := rightRec.ReferenceSelectedValues(
tr.opts.rightJoinFieldNames,
)
+ if hasAllJoinKeys && tr.opts.ignoreEmptyJoinFields && anyValueIsEmpty(rightFieldValues) {
+ hasAllJoinKeys = false
+ }
if hasAllJoinKeys {
var err error
isPaired, err = keeper.FindJoinBucket(rightFieldValues)
@@ -569,6 +597,9 @@ func (tr *TransformerJoin) ingestLeftFile() error {
groupingKey, leftFieldValues, ok := leftrec.GetSelectedValuesAndJoined(
tr.opts.leftJoinFieldNames,
)
+ if ok && tr.opts.ignoreEmptyJoinFields && anyValueIsEmpty(leftFieldValues) {
+ ok = false
+ }
if ok {
bucket := tr.leftBucketsByJoinFieldValues.Get(groupingKey)
if bucket == nil { // New key-field-value: new bucket and hash-map entry
diff --git a/pkg/transformers/utils/join_bucket_keeper.go b/pkg/transformers/utils/join_bucket_keeper.go
index 0aba766b8..d069a0ced 100644
--- a/pkg/transformers/utils/join_bucket_keeper.go
+++ b/pkg/transformers/utils/join_bucket_keeper.go
@@ -124,8 +124,9 @@ type JoinBucketKeeper struct {
// TODO: merge with leof flag
recordReaderDone bool
- leftJoinFieldNames []string
- leftKeepFieldNameSet map[string]bool
+ leftJoinFieldNames []string
+ leftKeepFieldNameSet map[string]bool
+ ignoreEmptyJoinFields bool
// Given a left-file of the following form (with left-join-field name "L"):
// +-----+
@@ -157,6 +158,7 @@ func NewJoinBucketKeeper(
joinReaderOptions *cli.TReaderOptions,
leftJoinFieldNames []string,
leftKeepFieldNameSet map[string]bool,
+ ignoreEmptyJoinFields bool,
) (*JoinBucketKeeper, error) {
// Instantiate the record-reader
@@ -187,8 +189,9 @@ func NewJoinBucketKeeper(
errorChannel: errorChannel,
recordReaderDone: false,
- leftJoinFieldNames: leftJoinFieldNames,
- leftKeepFieldNameSet: leftKeepFieldNameSet,
+ leftJoinFieldNames: leftJoinFieldNames,
+ leftKeepFieldNameSet: leftKeepFieldNameSet,
+ ignoreEmptyJoinFields: ignoreEmptyJoinFields,
JoinBucket: NewJoinBucket(nil),
peekRecordAndContext: nil,
@@ -337,7 +340,7 @@ func (keeper *JoinBucketKeeper) prepareForFirstJoinBucket() error {
if keeper.peekRecordAndContext == nil { // left EOF
break
}
- if keeper.peekRecordAndContext.Record.HasSelectedKeys(keeper.leftJoinFieldNames) {
+ if recordHasJoinKeys(keeper.peekRecordAndContext.Record, keeper.leftJoinFieldNames, keeper.ignoreEmptyJoinFields) {
break
}
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
@@ -412,7 +415,7 @@ func (keeper *JoinBucketKeeper) prepareForNewJoinBucket(
}
peekRec := keeper.peekRecordAndContext.Record
- if peekRec.HasSelectedKeys(keeper.leftJoinFieldNames) {
+ if recordHasJoinKeys(peekRec, keeper.leftJoinFieldNames, keeper.ignoreEmptyJoinFields) {
break
}
keeper.leftUnpaireds = append(keeper.leftUnpaireds, keeper.peekRecordAndContext)
@@ -485,6 +488,9 @@ func (keeper *JoinBucketKeeper) fillNextJoinBucket() error {
peekFieldValues, hasAllJoinKeys := peekRec.ReferenceSelectedValues(
keeper.leftJoinFieldNames,
)
+ if hasAllJoinKeys && keeper.ignoreEmptyJoinFields && valuesContainVoid(peekFieldValues) {
+ hasAllJoinKeys = false
+ }
if hasAllJoinKeys {
cmp := compareLexically(
@@ -595,6 +601,37 @@ func moveRecordsAndContexts(
*source = (*source)[:0]
}
+// recordHasJoinKeys reports whether rec has all of the given field names. If
+// ignoreEmptyJoinFields is set, a field holding an empty-string value counts
+// as absent, same as for --ignore-empty on the right-hand side of the join.
+func recordHasJoinKeys(
+ rec *mlrval.Mlrmap,
+ fieldNames []string,
+ ignoreEmptyJoinFields bool,
+) bool {
+ for _, fieldName := range fieldNames {
+ value := rec.Get(fieldName)
+ if value == nil {
+ return false
+ }
+ if ignoreEmptyJoinFields && value.IsVoid() {
+ return false
+ }
+ }
+ return true
+}
+
+// valuesContainVoid returns true if any of the given values is present but
+// empty-string (mlrval "void").
+func valuesContainVoid(values []*mlrval.Mlrval) bool {
+ for _, value := range values {
+ if value != nil && value.IsVoid() {
+ return true
+ }
+ }
+ return false
+}
+
// Returns -1, 0, 1 as left <, ==, > right, using lexical comparison only (even
// for numerical values).
diff --git a/test/cases/cli-help/0001/expout b/test/cases/cli-help/0001/expout
index d513c22ac..51548e827 100644
--- a/test/cases/cli-help/0001/expout
+++ b/test/cases/cli-help/0001/expout
@@ -562,6 +562,13 @@ Options:
--ul Emit unpaired records from the left file.
--ur Emit unpaired records from the right
file(s).
+--ignore-empty Treat records with empty-string values in
+ any join-field as if that join-field were
+ absent, on both the left and right files.
+ Such records are never paired -- not even
+ with one another -- and are treated as
+ unpaired, subject to --np/--ul/--ur as
+ usual.
-s|--sorted-input Require sorted input: records must be
sorted lexically by their join-field names,
else not all records will be paired. The
diff --git a/test/cases/verb-join/ignore-empty-basic/cmd b/test/cases/verb-join/ignore-empty-basic/cmd
new file mode 100644
index 000000000..4d73970e6
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-basic/cmd
@@ -0,0 +1 @@
+mlr --csv join --ignore-empty -j id -f test/input/join-1194-left.csv test/input/join-1194-right.csv
diff --git a/test/cases/verb-join/ignore-empty-basic/experr b/test/cases/verb-join/ignore-empty-basic/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/ignore-empty-basic/expout b/test/cases/verb-join/ignore-empty-basic/expout
new file mode 100644
index 000000000..9e8af04d2
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-basic/expout
@@ -0,0 +1,3 @@
+id,code,color
+4,ff0000,red
+2,00ff00,green
diff --git a/test/cases/verb-join/ignore-empty-sorted/cmd b/test/cases/verb-join/ignore-empty-sorted/cmd
new file mode 100644
index 000000000..920da6a1b
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-sorted/cmd
@@ -0,0 +1 @@
+mlr --csv join -s --ignore-empty -j id -f test/input/join-1194-left-sorted.csv test/input/join-1194-right-sorted.csv
diff --git a/test/cases/verb-join/ignore-empty-sorted/experr b/test/cases/verb-join/ignore-empty-sorted/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/ignore-empty-sorted/expout b/test/cases/verb-join/ignore-empty-sorted/expout
new file mode 100644
index 000000000..157c4fc20
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-sorted/expout
@@ -0,0 +1,3 @@
+id,code,color
+2,00ff00,green
+4,ff0000,red
diff --git a/test/cases/verb-join/ignore-empty-unpaired/cmd b/test/cases/verb-join/ignore-empty-unpaired/cmd
new file mode 100644
index 000000000..d805bb4b1
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-unpaired/cmd
@@ -0,0 +1 @@
+mlr --icsv --ojson join --ignore-empty --ul --ur -j id -f test/input/join-1194-left.csv test/input/join-1194-right.csv
diff --git a/test/cases/verb-join/ignore-empty-unpaired/experr b/test/cases/verb-join/ignore-empty-unpaired/experr
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/cases/verb-join/ignore-empty-unpaired/expout b/test/cases/verb-join/ignore-empty-unpaired/expout
new file mode 100644
index 000000000..738200530
--- /dev/null
+++ b/test/cases/verb-join/ignore-empty-unpaired/expout
@@ -0,0 +1,32 @@
+[
+{
+ "id": 4,
+ "code": "ff0000",
+ "color": "red"
+},
+{
+ "id": 2,
+ "code": "00ff00",
+ "color": "green"
+},
+{
+ "id": "",
+ "color": "white"
+},
+{
+ "id": "",
+ "color": "black"
+},
+{
+ "id": 3,
+ "code": "0000ff"
+},
+{
+ "id": "",
+ "code": "ffffff"
+},
+{
+ "id": "",
+ "code": "000000"
+}
+]
diff --git a/test/input/join-1194-left-sorted.csv b/test/input/join-1194-left-sorted.csv
new file mode 100644
index 000000000..bebe0ddbd
--- /dev/null
+++ b/test/input/join-1194-left-sorted.csv
@@ -0,0 +1,6 @@
+id,code
+,ffffff
+,000000
+2,00ff00
+3,0000ff
+4,ff0000
diff --git a/test/input/join-1194-left.csv b/test/input/join-1194-left.csv
new file mode 100644
index 000000000..171fd1d5c
--- /dev/null
+++ b/test/input/join-1194-left.csv
@@ -0,0 +1,6 @@
+id,code
+3,0000ff
+2,00ff00
+4,ff0000
+,ffffff
+,000000
diff --git a/test/input/join-1194-right-sorted.csv b/test/input/join-1194-right-sorted.csv
new file mode 100644
index 000000000..abbaf64a6
--- /dev/null
+++ b/test/input/join-1194-right-sorted.csv
@@ -0,0 +1,5 @@
+id,color
+,white
+,black
+2,green
+4,red
diff --git a/test/input/join-1194-right.csv b/test/input/join-1194-right.csv
new file mode 100644
index 000000000..badd61587
--- /dev/null
+++ b/test/input/join-1194-right.csv
@@ -0,0 +1,5 @@
+id,color
+4,red
+2,green
+,white
+,black