Support https://, http://, and file:// URI schemes for input (#598)

* Allow http/https as input sources

* Regtest data

go/regtest/cases/io-uri-schemes/0001/cmd

@@ -0,0 +1 @@
+mlr --icsv --opprint cat file://regtest/input/abixy.csv

go/regtest/cases/io-uri-schemes/0001/expout

@@ -0,0 +1,11 @@
+a   b   i  x                   y
+pan pan 1  0.3467901443380824  0.7268028627434533
+eks pan 2  0.7586799647899636  0.5221511083334797
+wye wye 3  0.20460330576630303 0.33831852551664776
+eks wye 4  0.38139939387114097 0.13418874328430463
+wye pan 5  0.5732889198020006  0.8636244699032729
+zee pan 6  0.5271261600918548  0.49322128674835697
+eks zee 7  0.6117840605678454  0.1878849191181694
+zee wye 8  0.5985540091064224  0.976181385699006
+hat wye 9  0.03144187646093577 0.7495507603507059
+pan wye 10 0.5026260055412137  0.9526183602969864

go/src/lib/file-readers.go

@@ -25,6 +25,7 @@ import (
 	"compress/gzip"
 	"compress/zlib"
 	"io"
+	"net/http"
 	"os"
 	"strings"
 )
@@ -53,7 +54,7 @@ func OpenFileForRead(
 	if prepipe != "" {
 		return openPrepipedHandleForRead(filename, prepipe, prepipeIsRaw)
 	} else {
-		handle, err := os.Open(filename)
+		handle, err := PathToHandle(filename)
 		if err != nil {
 			return nil, err
 		}
@@ -61,6 +62,28 @@ func OpenFileForRead(
 	}
 }
 
+// PathToHandle maps various back-ends to a stream. As of 2021-07-07, the
+// following URI schemes are supported:
+// * https://... and http://...
+// * file://...
+// * plain disk files
+func PathToHandle(
+	path string,
+) (io.ReadCloser, error) {
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		resp, err := http.Get(path)
+		if err != nil {
+			return nil, err
+		}
+		handle := resp.Body
+		return handle, err
+	} else if strings.HasPrefix(path, "file://") {
+		return os.Open(strings.Replace(path, "file://", "", 1))
+	} else {
+		return os.Open(path)
+	}
+}
+
 // OpenStdin: if prepipe is non-empty, popens "{prepipe}" and returns a handle
 // to that where prepipe is nominally things like "gunzip", "cat", etc.
 // Otherwise, delegates to an in-process reader which can natively handle
@@ -125,7 +148,7 @@ func escapeFileNameForPopen(filename string) string {
 
 // TODO: comment
 func openEncodedHandleForRead(
-	handle *os.File,
+	handle io.ReadCloser,
 	encoding TFileInputEncoding,
 	filename string,
 ) (io.ReadCloser, error) {
@@ -160,11 +183,11 @@ func openEncodedHandleForRead(
 // ----------------------------------------------------------------
 // BZip2ReadCloser remedies the fact that bzip2.NewReader does not implement io.ReadCloser.
 type BZip2ReadCloser struct {
-	originalHandle *os.File
+	originalHandle io.ReadCloser
 	bzip2Handle    io.Reader
 }
 
-func NewBZip2ReadCloser(handle *os.File) *BZip2ReadCloser {
+func NewBZip2ReadCloser(handle io.ReadCloser) *BZip2ReadCloser {
 	return &BZip2ReadCloser{
 		originalHandle: handle,
 		bzip2Handle:    bzip2.NewReader(handle),