diff --git a/go/regtest/cases/io-uri-schemes/0001/cmd b/go/regtest/cases/io-uri-schemes/0001/cmd new file mode 100644 index 000000000..abbab01fc --- /dev/null +++ b/go/regtest/cases/io-uri-schemes/0001/cmd @@ -0,0 +1 @@ +mlr --icsv --opprint cat file://regtest/input/abixy.csv diff --git a/go/regtest/cases/io-uri-schemes/0001/experr b/go/regtest/cases/io-uri-schemes/0001/experr new file mode 100644 index 000000000..e69de29bb diff --git a/go/regtest/cases/io-uri-schemes/0001/expout b/go/regtest/cases/io-uri-schemes/0001/expout new file mode 100644 index 000000000..c894dcf82 --- /dev/null +++ b/go/regtest/cases/io-uri-schemes/0001/expout @@ -0,0 +1,11 @@ +a b i x y +pan pan 1 0.3467901443380824 0.7268028627434533 +eks pan 2 0.7586799647899636 0.5221511083334797 +wye wye 3 0.20460330576630303 0.33831852551664776 +eks wye 4 0.38139939387114097 0.13418874328430463 +wye pan 5 0.5732889198020006 0.8636244699032729 +zee pan 6 0.5271261600918548 0.49322128674835697 +eks zee 7 0.6117840605678454 0.1878849191181694 +zee wye 8 0.5985540091064224 0.976181385699006 +hat wye 9 0.03144187646093577 0.7495507603507059 +pan wye 10 0.5026260055412137 0.9526183602969864 diff --git a/go/src/lib/file-readers.go b/go/src/lib/file-readers.go index c2f9cfe9f..157cf7738 100644 --- a/go/src/lib/file-readers.go +++ b/go/src/lib/file-readers.go @@ -25,6 +25,7 @@ import ( "compress/gzip" "compress/zlib" "io" + "net/http" "os" "strings" ) @@ -53,7 +54,7 @@ func OpenFileForRead( if prepipe != "" { return openPrepipedHandleForRead(filename, prepipe, prepipeIsRaw) } else { - handle, err := os.Open(filename) + handle, err := PathToHandle(filename) if err != nil { return nil, err } @@ -61,6 +62,28 @@ func OpenFileForRead( } } +// PathToHandle maps various back-ends to a stream. As of 2021-07-07, the +// following URI schemes are supported: +// * https://... and http://... +// * file://... +// * plain disk files +func PathToHandle( + path string, +) (io.ReadCloser, error) { + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + resp, err := http.Get(path) + if err != nil { + return nil, err + } + handle := resp.Body + return handle, err + } else if strings.HasPrefix(path, "file://") { + return os.Open(strings.Replace(path, "file://", "", 1)) + } else { + return os.Open(path) + } +} + // OpenStdin: if prepipe is non-empty, popens "{prepipe}" and returns a handle // to that where prepipe is nominally things like "gunzip", "cat", etc. // Otherwise, delegates to an in-process reader which can natively handle @@ -125,7 +148,7 @@ func escapeFileNameForPopen(filename string) string { // TODO: comment func openEncodedHandleForRead( - handle *os.File, + handle io.ReadCloser, encoding TFileInputEncoding, filename string, ) (io.ReadCloser, error) { @@ -160,11 +183,11 @@ func openEncodedHandleForRead( // ---------------------------------------------------------------- // BZip2ReadCloser remedies the fact that bzip2.NewReader does not implement io.ReadCloser. type BZip2ReadCloser struct { - originalHandle *os.File + originalHandle io.ReadCloser bzip2Handle io.Reader } -func NewBZip2ReadCloser(handle *os.File) *BZip2ReadCloser { +func NewBZip2ReadCloser(handle io.ReadCloser) *BZip2ReadCloser { return &BZip2ReadCloser{ originalHandle: handle, bzip2Handle: bzip2.NewReader(handle),