// Logic for executing regression tests via 'mlr regtest'. // // Why: Basically we're re-implementing a shell, or at least some basic // elements of one. In a previous version of Miller, regression-testing was // done with a Bash script -- not much more than an expected-output file and a // diff check at the end. However, as of Miller 6 we now have full support for // Windows and we want to run a continuous-integration job (via GitHub Actions // as of late 2021) on every commit, for Linux, Mac, and Windows. Given the // lack of bash for the latter, we have the following Go logic instead. // // Directory structure: This uses ./test/cases/ (by default) as test-data // storage. Arbitrary subdirectory structure can be used. Directory trees // must end in a 'case directory' containing the following: // // * A 'cmd' file with a Miller shell command in it. // * Also 'expout' and 'experr' for expected stdout/stdout from the command. // * Optionally, a 'mlr' script if the test uses one. // * Optionally, an 'env' file with environment variables to be set before the // case and unset after. // * Optionally a case-local 'input' file; many cases use shared/common data // in test/input/. // * The cmd file can refer to '${CASEDIR}' which is expanded at runtime to // the case directory path, so cases can refer to their 'input' and 'mlr' // files. // // Example: // // $ ls test/cases/dsl-redirects/0109/* // test/cases/dsl-redirects/0109/cmd // test/cases/dsl-redirects/0109/experr // test/cases/dsl-redirects/0109/expout // test/cases/dsl-redirects/0109/mlr // // $ cat test/cases/dsl-redirects//0109/cmd // mlr head -n 4 then put -q -f ${CASEDIR}/mlr test/input/abixy // // $ cat test/cases/dsl-redirects//0109/experr // NR=1,a=pan,b=pan // NR=1,a=pan,b=pan // NR=2,a=eks,b=pan // NR=1,a=pan,b=pan // NR=2,a=eks,b=pan // NR=3,a=wye,b=wye // NR=1,a=pan,b=pan // NR=2,a=eks,b=pan // NR=3,a=wye,b=wye // NR=4,a=eks,b=wye // // $ cat test/cases/dsl-redirects//0109/expout // // $cat test/cases/dsl-redirects//0109/mlr // @a[NR]=$a; @b[NR]=$b; emit > stderr, (@a, @b), "NR" package regtest import ( "fmt" "os" "path/filepath" "runtime" "sort" "strings" "github.com/johnkerl/miller/v6/pkg/colorizer" "github.com/johnkerl/miller/v6/pkg/lib" ) const CmdName = "cmd" const MlrName = "mlr" const EnvName = "env" const ScriptName = "mlr" const PreCopyName = "precopy" const ExpectedStdoutName = "expout" const ExpectedStderrName = "experr" const PostCompareName = "postcmp" const ShouldFailName = "should-fail" const MajorSeparator = "================================================================" const MinorSeparator = "----------------------------------------------------------------" // Don't unset MLR_PASS_COLOR or MLR_FAIL_COLOR -- if people want to change the // output-coloring used by this regression-tester, we should let them. We // should only unset environment variables which can cause tests to fail. var envVarsToUnset = []string{ "MLRRC", "MLR_KEY_COLOR", "MLR_VALUE_COLOR", "MLR_REPL_PS1", "MLR_REPL_PS2", "MLR_REPL_PS1_COLOR", "MLR_REPL_PS2_COLOR", } type stringPair struct { first string second string } type RegTester struct { exeName string verbosityLevel int plainMode bool doPopulate bool directoryPassCount int directoryFailCount int casePassCount int caseFailCount int failDirNames []string failCaseNames []string firstNFailsToShow int } func NewRegTester( exeName string, doPopulate bool, verbosityLevel int, plainMode bool, firstNFailsToShow int, ) *RegTester { return &RegTester{ exeName: exeName, doPopulate: doPopulate, verbosityLevel: verbosityLevel, plainMode: plainMode, directoryPassCount: 0, directoryFailCount: 0, casePassCount: 0, caseFailCount: 0, failDirNames: []string{}, failCaseNames: []string{}, firstNFailsToShow: firstNFailsToShow, } } func (rt *RegTester) resetCounts() { rt.directoryPassCount = 0 rt.directoryFailCount = 0 rt.casePassCount = 0 rt.caseFailCount = 0 } // Top-level entrypoint for the rt. See the usage function in entry.go // for semantics. func (rt *RegTester) Execute( casePaths []string, ) bool { // Don't let the current user's settings affect expected results for _, name := range envVarsToUnset { _ = os.Unsetenv(name) } // If there is an accessible .mlrrc file, we don't want it to be read for the regression test. _ = os.Setenv("MLRRC", "__none__") // This is important for multi-platform regression testing, wherein default floating-point // output format has varying numbers of decimal places between the platform where // the expected results were generated, and the platform where the actual values are being // computed. For regression-test we OFMT from an environment variable. _ = os.Setenv("MLR_OFMT", "%.8f") rt.resetCounts() lib.InternalCodingErrorIf(len(casePaths) == 0) sort.Strings(casePaths) if !rt.plainMode { fmt.Println("REGRESSION TEST:") for _, path := range casePaths { fmt.Printf(" %s\n", path) } fmt.Printf("Using executable: %s\n", rt.exeName) fmt.Println() } for _, path := range casePaths { rt.executeSinglePath(path) } if len(rt.failCaseNames) > 0 && rt.firstNFailsToShow > 0 { fmt.Println() fmt.Println("RERUNS OF FIRST FAILED CASE FILES:") verbosityLevel := 3 i := 0 for _, e := range rt.failCaseNames { rt.executeSingleCmdFile(e, verbosityLevel) i++ if i >= rt.firstNFailsToShow { break } } } if !rt.plainMode && len(rt.failDirNames) > 0 { fmt.Println() fmt.Println("FAILED CASE DIRECTORIES:") for _, e := range rt.failDirNames { fmt.Printf(" %s/\n", e) } } if !rt.plainMode { fmt.Println() fmt.Printf("NUMBER OF CASES PASSED %d\n", rt.casePassCount) fmt.Printf("NUMBER OF CASES FAILED %d\n", rt.caseFailCount) fmt.Printf("NUMBER OF CASE-DIRECTORIES PASSED %d\n", rt.directoryPassCount) fmt.Printf("NUMBER OF CASE-DIRECTORIES FAILED %d\n", rt.directoryFailCount) fmt.Println() } // Directory count may be zero if we were invoked with all paths on the // command line being .cmd files. if rt.casePassCount > 0 && rt.caseFailCount == 0 { if !rt.plainMode { fmt.Printf("%s overall\n", colorizer.MaybeColorizePass("PASS", true)) } return true } if !rt.plainMode { fmt.Printf("%s overall\n", colorizer.MaybeColorizeFail("FAIL", true)) } return false } // Recursively invoked routine to process either a single .cmd file, or a // directory of such, or a directory of directories. func (rt *RegTester) executeSinglePath( path string, ) bool { handle, err := os.Stat(path) if err != nil { fmt.Printf("%s: %v\n", path, err) return false } mode := handle.Mode() if mode.IsDir() { passed, hasCaseSubdirectories := rt.executeSingleDirectory(path) if hasCaseSubdirectories { if passed { rt.directoryPassCount++ } else { rt.directoryFailCount++ rt.failDirNames = append(rt.failDirNames, path) } } return passed } else if mode.IsRegular() { basename := filepath.Base(path) if basename == CmdName { passed := rt.executeSingleCmdFile(path, rt.verbosityLevel) if passed { rt.casePassCount++ } else { rt.caseFailCount++ rt.failCaseNames = append(rt.failCaseNames, path) } return passed } return true // No .cmd files directly inside } fmt.Printf("%s: neither directory nor regular file.\n", path) return false // fall-through } func (rt *RegTester) executeSingleDirectory( dirName string, ) (bool, bool) { passed := true // TODO: comment fileNames, hasCaseSubdirectories := rt.hasCaseSubdirectories(dirName) if !rt.plainMode { if hasCaseSubdirectories && rt.verbosityLevel >= 2 { fmt.Printf("%s BEGIN %s/\n", MajorSeparator, dirName) } } for _, name := range fileNames { path := dirName + "/" + name ok := rt.executeSinglePath(path) if !ok { passed = false } } // Only print if there are .cmd files directly in this directory. // Otherwise it's just a directory-of-directories and we don't need to // multiply announce. if hasCaseSubdirectories { if passed { if !rt.plainMode { fmt.Printf("%s %s\n", colorizer.MaybeColorizePass("PASS", true), dirName) } } else { if !rt.plainMode { fmt.Printf("%s %s\n", colorizer.MaybeColorizeFail("FAIL", true), dirName) } } } if !rt.plainMode { if !hasCaseSubdirectories && rt.verbosityLevel >= 2 { fmt.Printf("%s END %s/\n", MajorSeparator, dirName) fmt.Println() } } return passed, hasCaseSubdirectories } // Sees if a directory contains a single test case. If so, we don't want to // print a banner at start and end at the default verbosity level. In order to // manage print volume, we only want to print for directories-of-cases (next // level up). // TODO: don't print container-of-containers, via entries.any.isCaseDirectory // or somesuch. func (rt *RegTester) hasCaseSubdirectories( dirName string, ) ([]string, bool) { f, err := os.Open(dirName) if err != nil { fmt.Printf("%s: %v\n", dirName, err) os.Exit(1) } defer func() { _ = f.Close() }() names, err := f.Readdirnames(-1) if err != nil { fmt.Printf("%s: %v\n", dirName, err) os.Exit(1) } sort.Strings(names) for _, name := range names { path := dirName + string(filepath.Separator) + name if rt.isCaseDirectory(path) { return names, true } } return names, false } func (rt *RegTester) isCaseDirectory( dirName string, ) bool { cmdFilePath := dirName + string(filepath.Separator) + CmdName return rt.FileExists(cmdFilePath) } // This is the main regression-test logic for a single .cmd file (a single mlr // invocation) and its associated supporting files. func (rt *RegTester) executeSingleCmdFile( cmdFilePath string, verbosityLevel int, ) bool { if strings.Contains(cmdFilePath, "non-windows") { // These are tests which cannot pass on Windows. Please see the README.md file // within test/cases/... for details. if runtime.GOOS == "windows" && os.Getenv("MSYSTEM") != "" { if verbosityLevel >= 1 { fmt.Printf("%s %s\n", colorizer.MaybeColorizePass("skip", true), cmdFilePath) } } return true } if verbosityLevel >= 2 { fmt.Printf("%s begin %s\n", MinorSeparator, cmdFilePath) defer fmt.Printf("%s end %s\n", MinorSeparator, cmdFilePath) } // Given 'test/cases/foo/0038.cmd', get 'test/cases/foo' and '0038'. // Various support files use syntax ${CASEDIR} within them so they're // relocatable, but we need to expand those in order to execute the test // case. // Using backslash on Windows works well in *almost* all cases. However, // there are annoying issues with making all regression-test cases // relocatable using the ${CASEDIR} substitution. This is fine in `cmd` // files, but for those (relatively few) cases which need casedir access // within the `mlr` files -- namely for redirected emit/emitp/tee/dump // within `mlr put` -- the substitution becomes unwieldy. So, here we // simply use forward slashes, trusting in all modern Windows systems to // handle this for regression-test cases. (Note: this is only for // regression testing which is nominally done by a developer, or in GitHub // Actions for Continuous Integration. End users using Miller don't // typically touch this part, and Miller in general -- outside here -- // doesn't rewrite backslashes to slashes on Windows.) caseDir := filepath.Dir(cmdFilePath) caseDir = strings.ReplaceAll(caseDir, "\\", "/") // Not slash := string(filepath.Separator) slash := "/" cmd, err := rt.loadFile(cmdFilePath, caseDir) if err != nil { if verbosityLevel >= 2 { fmt.Printf("%s: %v\n", cmdFilePath, err) } return false } mlrFileName := caseDir + slash + MlrName envFileName := caseDir + slash + EnvName preCopyFileName := caseDir + slash + PreCopyName expectedStdoutFileName := caseDir + slash + ExpectedStdoutName expectedStderrFileName := caseDir + slash + ExpectedStderrName expectFailFileName := caseDir + slash + ShouldFailName postCompareFileName := caseDir + slash + PostCompareName if verbosityLevel >= 2 { fmt.Println("Command:") fmt.Println(cmd) mlr, err := rt.loadFile(mlrFileName, caseDir) if err == nil { fmt.Println("Miller DSL script:") fmt.Println(mlr) } } if rt.plainMode { fmt.Println("----------------------------------------------------------------") fmt.Println(cmd) scriptFileName := caseDir + slash + ScriptName scriptContents, err := rt.loadFile(scriptFileName, caseDir) if err == nil { fmt.Println(scriptContents) } } // The .env needn't exist (most test cases don't have one) in which case // the returned map will be empty. envKeyValuePairs, err := rt.loadEnvFile(envFileName, caseDir) if err != nil { fmt.Println(err) return false } // The .precopy needn't exist (most test cases don't have one) in which case // the returned map will be empty. preCopySrcDestPairs, err := rt.loadStringPairFile(preCopyFileName, caseDir) if err != nil { fmt.Println(err) return false } passed := true // Set any case-specific environment variables before running the case. for pe := envKeyValuePairs.Head; pe != nil; pe = pe.Next { key := pe.Key value := pe.Value if verbosityLevel >= 3 { fmt.Printf("SETENV %s=%s\n", key, value) } _ = os.Setenv(key, value) } // This is so 'mlr' files can find the case-directory if they need it -- // typically, for redirected emit/dump/etc statements where we want // the redirected-to files to be written into the case-directory. _ = os.Setenv("CASEDIR", caseDir) // Copy any files requested by the test. (Most don't; some do, e.g. those // which test the write-in-place logic of mlr -I.) for _, pair := range preCopySrcDestPairs { src := pair.first dst := pair.second if verbosityLevel >= 3 { fmt.Printf("%s: copy %s to %s\n", cmdFilePath, src, dst) } err := rt.copyFile(src, dst, caseDir) if err != nil { fmt.Printf("%s: %v\n", dst, err) passed = false } } // **************************************************************** // HERE IS WHERE WE RUN THE MILLER COMMAND LINE FOR THE TEST CASE actualStdout, actualStderr, actualExitCode := RunMillerCommand(rt.exeName, cmd) // **************************************************************** // Unset any case-specific environment variables after running the case. // This is important since the setenv is done in the current process, // and we don't want to affect subsequent test cases. for pe := envKeyValuePairs.Head; pe != nil; pe = pe.Next { key := pe.Key if verbosityLevel >= 3 { fmt.Printf("UNSETENV %s\n", key) } _ = os.Setenv(key, "") } _ = os.Setenv("CASEDIR", "") // The .postcmp needn't exist (most test cases don't have one) in which case // the returned map will be empty. postCompareExpectedActualPairs, err := rt.loadStringPairFile(postCompareFileName, caseDir) if err != nil { fmt.Println(err) return false } if rt.doPopulate { // Populate mode, not verify mode: write out the actual // stdout/stderr/exit-code to disk as expected values for subsequent // runs. // TODO: temp replace-all for CR/LF to LF. Will need re-work once auto-detect is ported. actualStdout = strings.ReplaceAll(actualStdout, "\r\n", "\n") actualStderr = strings.ReplaceAll(actualStderr, "\r\n", "\n") // Write the .expout file err = rt.storeFile(expectedStdoutFileName, actualStdout) if err != nil { fmt.Printf("%s: %v\n", expectedStdoutFileName, err) passed = false } else { if rt.verbosityLevel >= 1 { fmt.Printf("wrote %s\n", expectedStdoutFileName) } } // Write the .experr file err = rt.storeFile(expectedStderrFileName, actualStderr) if err != nil { fmt.Printf("%s: %v\n", expectedStderrFileName, err) passed = false } else { if rt.verbosityLevel >= 1 { fmt.Printf("wrote %s\n", expectedStdoutFileName) } } // Write the .should-fail file if actualExitCode == 0 { // Remove it, if it exists. _ = os.Remove(expectFailFileName) } else { err = rt.storeFile(expectFailFileName, "") if err != nil { fmt.Printf("%s: %v\n", expectedStderrFileName, err) passed = false } else { if rt.verbosityLevel >= 1 { fmt.Printf("wrote %s\n", expectedStdoutFileName) } } } for _, pair := range postCompareExpectedActualPairs { expectedFileName := pair.first actualFileName := pair.second err := rt.copyFile(actualFileName, expectedFileName, caseDir) if err != nil { fmt.Printf("Could not copy %s to %s: %v\n", actualFileName, expectedFileName, err) passed = false } if verbosityLevel >= 3 { fmt.Printf("Copied %s to %s: %v\n", actualFileName, expectedFileName, err) } } } else { // Verify mode, not populate mode: check actuals against expecteds // Load the .expout file expectedStdout, err := rt.loadFile(expectedStdoutFileName, caseDir) if err != nil { if verbosityLevel >= 2 { fmt.Printf("%s: %v\n", expectedStdoutFileName, err) } return false } // Load the .experr file expectedStderr, err := rt.loadFile(expectedStderrFileName, caseDir) if err != nil { if verbosityLevel >= 2 { fmt.Printf("%s: %v\n", expectedStderrFileName, err) } return false } // Load the .should-fail file expectedExitCode := 0 if rt.FileExists(expectFailFileName) { expectedExitCode = 1 } if rt.plainMode { fmt.Print(actualStdout) fmt.Print(actualStderr) fmt.Println() } if verbosityLevel >= 3 { fmt.Printf("actualStdout [%d]:\n", len(actualStdout)) fmt.Println(actualStdout) fmt.Printf("expectedStdout [%d]:\n", len(expectedStdout)) fmt.Println(expectedStdout) fmt.Printf("actualStderr [%d]:\n", len(actualStderr)) fmt.Println(actualStderr) fmt.Printf("expectedStderr [%d]:\n", len(expectedStderr)) fmt.Println(expectedStderr) fmt.Println("actualExitCode:") fmt.Println(actualExitCode) fmt.Println("expectedExitCode:") fmt.Println(expectedExitCode) fmt.Println() } // TODO: temp replace-all for CR/LF to LF. Will need re-work once auto-detect is ported. actualStdout = strings.ReplaceAll(actualStdout, "\r\n", "\n") actualStderr = strings.ReplaceAll(actualStderr, "\r\n", "\n") expectedStdout = strings.ReplaceAll(expectedStdout, "\r\n", "\n") expectedStderr = strings.ReplaceAll(expectedStderr, "\r\n", "\n") // Compare stdout to .expout if actualStdout != expectedStdout { if verbosityLevel >= 2 { fmt.Printf( "%s: stdout does not match expected %s\n", cmdFilePath, expectedStdoutFileName, ) } if verbosityLevel >= 3 { fmt.Println(RunDiffCommandOnStrings(expectedStdout, actualStdout)) } passed = false } // Compare stderr to .experr if actualStderr != expectedStderr { if verbosityLevel >= 2 { fmt.Printf( "%s: stderr does not match expected %s\n", cmdFilePath, expectedStderrFileName, ) } if verbosityLevel >= 3 { fmt.Println(RunDiffCommandOnStrings(expectedStderr, actualStderr)) } passed = false } // Compare exit code if actualExitCode != expectedExitCode { if verbosityLevel >= 2 { fmt.Printf( "%s: exit code %d does not match expected %d\n", cmdFilePath, actualExitCode, expectedExitCode, ) } passed = false } // Compare any additional output files. Most test cases don't have // these (just stdout/stderr), but some do: for example, those which // test the tee verb/function. for _, pair := range postCompareExpectedActualPairs { expectedFileName := pair.first actualFileName := pair.second ok, expectedContents, actualContents, err := rt.compareFiles(expectedFileName, actualFileName, caseDir) if err != nil { fmt.Printf("%s: %v\n", cmdFilePath, err) passed = false } else if !ok { if verbosityLevel >= 2 { fmt.Printf( "%s: %s does not match %s\n", cmdFilePath, expectedFileName, actualFileName, ) } if verbosityLevel >= 3 { fmt.Println() fmt.Printf("%s:\n", expectedFileName) fmt.Print(expectedContents) fmt.Println() fmt.Printf("%s:\n", actualFileName) fmt.Print(actualContents) fmt.Println() fmt.Println(RunDiffCommandOnFilenames(expectedFileName, actualFileName)) } passed = false } else { if verbosityLevel >= 2 { fmt.Printf( "%s: %s matches %s\n", cmdFilePath, expectedFileName, actualFileName, ) } } } // Clean up any requested file-copies so that we're git-clean after the regression-test run. for _, pair := range preCopySrcDestPairs { dst := pair.second _ = os.Remove(dst) if verbosityLevel >= 3 { fmt.Printf("%s: clean up %s\n", cmdFilePath, dst) } } // Clean up any extra output files so that we're git-clean after the regression-test run. for _, pair := range postCompareExpectedActualPairs { actualFileName := pair.second _ = os.Remove(actualFileName) if verbosityLevel >= 3 { fmt.Printf("%s: clean up %s\n", cmdFilePath, actualFileName) } } } if verbosityLevel >= 1 { if passed { fmt.Printf("%s %s\n", colorizer.MaybeColorizePass("pass", true), cmdFilePath) } else { fmt.Printf("%s %s\n", colorizer.MaybeColorizeFail("fail", true), cmdFilePath) } } return passed } func (rt *RegTester) FileExists(fileName string) bool { fileInfo, err := os.Stat(fileName) if err != nil { return false } return !fileInfo.IsDir() } func (rt *RegTester) loadFile( fileName string, caseDir string, ) (string, error) { byteContents, err := os.ReadFile(fileName) if err != nil { return "", err } contents := string(byteContents) contents = strings.ReplaceAll(contents, "${CASEDIR}", caseDir) contents = strings.ReplaceAll(contents, "${PATHSEP}", string(os.PathSeparator)) contents = strings.ReplaceAll(contents, "${MLR}", rt.exeName) return contents, nil } func (rt *RegTester) storeFile( fileName string, contents string, ) error { err := os.WriteFile(fileName, []byte(contents), 0o666) if err != nil { return err } return nil } func (rt *RegTester) copyFile( src string, dst string, caseDir string, ) error { contents, err := rt.loadFile(src, caseDir) if err != nil { return err } err = rt.storeFile(dst, contents) if err != nil { return err } return nil } func (rt *RegTester) compareFiles( expectedFileName string, actualFileName string, caseDir string, ) (bool, string, string, error) { expectedContents, err := rt.loadFile(expectedFileName, caseDir) if err != nil { return false, "", "", err } actualContents, err := rt.loadFile(actualFileName, caseDir) if err != nil { return false, "", "", err } // TODO: maybe rethink later with autoterm expectedContents = strings.ReplaceAll(expectedContents, "\r\n", "\n") actualContents = strings.ReplaceAll(actualContents, "\r\n", "\n") return expectedContents == actualContents, expectedContents, actualContents, nil } func (rt *RegTester) loadEnvFile( filename string, caseDir string, ) (*lib.OrderedMap[string], error) { // If the file doesn't exist that's the normal case -- most cases do not // have a .env file. _, err := os.Stat(filename) if os.IsNotExist(err) { return lib.NewOrderedMap[string](), nil } // If the file does exist and isn't loadable, that's an error. contents, err := rt.loadFile(filename, caseDir) if err != nil { return nil, err } keyValuePairs := lib.NewOrderedMap[string]() lines := strings.SplitSeq(contents, "\n") for line := range lines { line = strings.TrimSuffix(line, "\r") if line == "" { continue } fields := strings.SplitN(line, "=", 2) if len(fields) != 2 { return nil, fmt.Errorf( `mlr: could not parse line "%s" from file "%s"`, line, filename, ) } keyValuePairs.Put(fields[0], fields[1]) } return keyValuePairs, nil } func (rt *RegTester) loadStringPairFile( filename string, caseDir string, ) ([]stringPair, error) { pairs := []stringPair{} // If the file doesn't exist that's the normal case -- most cases do not // have a .precopy or .postcmp file. _, err := os.Stat(filename) if os.IsNotExist(err) { return pairs, nil } // If the file does exist and isn't loadable, that's an error. contents, err := rt.loadFile(filename, caseDir) if err != nil { return nil, err } pairs = []stringPair{} lines := strings.SplitSeq(contents, "\n") for line := range lines { line = strings.TrimSuffix(line, "\r") if line == "" { continue } fields := strings.SplitN(line, " ", 2) // TODO: split on multi-space if len(fields) != 2 { return nil, fmt.Errorf( `mlr: could not parse line "%s" from file "%s"`, line, filename, ) } pair := stringPair{first: fields[0], second: fields[1]} pairs = append(pairs, pair) } return pairs, nil }