feat: ignore_difs key to ignore diffs in tests
This commit is contained in:
@@ -23,6 +23,7 @@ var (
|
|||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
TestDir string
|
TestDir string
|
||||||
|
IgnoreDiffs []string
|
||||||
Sandbox map[string]string
|
Sandbox map[string]string
|
||||||
Mounts []string
|
Mounts []string
|
||||||
Paths []string
|
Paths []string
|
||||||
@@ -35,6 +36,7 @@ type tomlConfig struct {
|
|||||||
|
|
||||||
type tomlMireConfig struct {
|
type tomlMireConfig struct {
|
||||||
TestDir string `toml:"test_dir"`
|
TestDir string `toml:"test_dir"`
|
||||||
|
IgnoreDiffs []string `toml:"ignore_diffs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type tomlSandboxConfig struct {
|
type tomlSandboxConfig struct {
|
||||||
@@ -70,6 +72,13 @@ func ReadConfig(path string) (Config, error) {
|
|||||||
if raw.Mire.TestDir == "" {
|
if raw.Mire.TestDir == "" {
|
||||||
return Config{}, fmt.Errorf("failed to read %s: empty mire.test_dir", path)
|
return Config{}, fmt.Errorf("failed to read %s: empty mire.test_dir", path)
|
||||||
}
|
}
|
||||||
|
if !meta.IsDefined("mire", "ignore_diffs") {
|
||||||
|
return Config{}, fmt.Errorf("failed to read %s: missing required mire.ignore_diffs", path)
|
||||||
|
}
|
||||||
|
ignoreDiffs, err := validateIgnoreDiffs(path, raw.Mire.IgnoreDiffs)
|
||||||
|
if err != nil {
|
||||||
|
return Config{}, err
|
||||||
|
}
|
||||||
if !meta.IsDefined("sandbox") {
|
if !meta.IsDefined("sandbox") {
|
||||||
return Config{}, fmt.Errorf("failed to read %s: missing [sandbox] config", path)
|
return Config{}, fmt.Errorf("failed to read %s: missing [sandbox] config", path)
|
||||||
}
|
}
|
||||||
@@ -90,6 +99,7 @@ func ReadConfig(path string) (Config, error) {
|
|||||||
|
|
||||||
return Config{
|
return Config{
|
||||||
TestDir: raw.Mire.TestDir,
|
TestDir: raw.Mire.TestDir,
|
||||||
|
IgnoreDiffs: ignoreDiffs,
|
||||||
Sandbox: sandbox,
|
Sandbox: sandbox,
|
||||||
Mounts: mounts,
|
Mounts: mounts,
|
||||||
Paths: paths,
|
Paths: paths,
|
||||||
@@ -170,6 +180,16 @@ func validateSandbox(path string, sandbox map[string]string, mounts, paths []str
|
|||||||
return validated, validatedMounts, validatedPaths, nil
|
return validated, validatedMounts, validatedPaths, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateIgnoreDiffs(path string, ignoreDiffs []string) ([]string, error) {
|
||||||
|
for i, pattern := range ignoreDiffs {
|
||||||
|
if _, err := regexp.Compile(pattern); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read %s: invalid mire.ignore_diffs[%d] regex %q: %v", path, i, pattern, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cloneStrings(ignoreDiffs), nil
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeMounts(configPath string, mounts []string) ([]string, error) {
|
func normalizeMounts(configPath string, mounts []string) ([]string, error) {
|
||||||
normalized := make([]string, 0, len(mounts))
|
normalized := make([]string, 0, len(mounts))
|
||||||
for _, mount := range mounts {
|
for _, mount := range mounts {
|
||||||
@@ -225,13 +245,13 @@ func cloneSandbox(sandbox map[string]string) map[string]string {
|
|||||||
return cloned
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneMounts(mounts []string) []string {
|
func cloneStrings(values []string) []string {
|
||||||
if len(mounts) == 0 {
|
if len(values) == 0 {
|
||||||
return []string{}
|
return []string{}
|
||||||
}
|
}
|
||||||
|
|
||||||
cloned := make([]string, len(mounts))
|
cloned := make([]string, len(values))
|
||||||
copy(cloned, mounts)
|
copy(cloned, values)
|
||||||
|
|
||||||
return cloned
|
return cloned
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,16 +13,17 @@ func TestReadConfig(t *testing.T) {
|
|||||||
name string
|
name string
|
||||||
content string
|
content string
|
||||||
wantDir string
|
wantDir string
|
||||||
|
wantIgnoreDiffs []string
|
||||||
wantSandbox map[string]string
|
wantSandbox map[string]string
|
||||||
wantMounts []string
|
wantMounts []string
|
||||||
wantPaths []string
|
wantPaths []string
|
||||||
wantErr string
|
wantErr string
|
||||||
wantMissing bool
|
wantMissing bool
|
||||||
setup func(t *testing.T, dir string) (string, []string, []string)
|
setup func(t *testing.T, dir string) (string, []string, []string, []string)
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "with test dir and sandbox",
|
name: "with test dir and sandbox",
|
||||||
setup: func(t *testing.T, dir string) (string, []string, []string) {
|
setup: func(t *testing.T, dir string) (string, []string, []string, []string) {
|
||||||
hostData := filepath.Join(dir, "host-data")
|
hostData := filepath.Join(dir, "host-data")
|
||||||
hostCache := filepath.Join(dir, "host-cache")
|
hostCache := filepath.Join(dir, "host-cache")
|
||||||
for _, path := range []string{hostData, hostCache} {
|
for _, path := range []string{hostData, hostCache} {
|
||||||
@@ -30,11 +31,16 @@ func TestReadConfig(t *testing.T) {
|
|||||||
t.Fatalf("MkdirAll(%q) error = %v", path, err)
|
t.Fatalf("MkdirAll(%q) error = %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = [\"" + hostData + ":/sandbox/data\", \"" + hostCache + ":/sandbox/cache\"]\npaths = []\nkey_word = \"value\"\n",
|
return "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = [\"^ts=.*$\", \"^id=.*$\"]\n\n[sandbox]\nhome = \"/home/test\"\nmounts = [\"" + hostData + ":/sandbox/data\", \"" + hostCache + ":/sandbox/cache\"]\npaths = []\nkey_word = \"value\"\n",
|
||||||
[]string{hostData + ":/sandbox/data", hostCache + ":/sandbox/cache"},
|
[]string{hostData + ":/sandbox/data", hostCache + ":/sandbox/cache"},
|
||||||
nil
|
nil,
|
||||||
|
[]string{"^ts=.*$", "^id=.*$"}
|
||||||
},
|
},
|
||||||
wantDir: "custom/suite",
|
wantDir: "custom/suite",
|
||||||
|
wantIgnoreDiffs: []string{
|
||||||
|
"^ts=.*$",
|
||||||
|
"^id=.*$",
|
||||||
|
},
|
||||||
wantSandbox: map[string]string{
|
wantSandbox: map[string]string{
|
||||||
"home": "/home/test",
|
"home": "/home/test",
|
||||||
"key_word": "value",
|
"key_word": "value",
|
||||||
@@ -55,6 +61,11 @@ func TestReadConfig(t *testing.T) {
|
|||||||
content: "[mire]\n",
|
content: "[mire]\n",
|
||||||
wantErr: "missing required mire.test_dir",
|
wantErr: "missing required mire.test_dir",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "without ignore diffs",
|
||||||
|
content: "[mire]\ntest_dir = \"custom/suite\"\n",
|
||||||
|
wantErr: "missing required mire.ignore_diffs",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "empty test dir",
|
name: "empty test dir",
|
||||||
content: "[mire]\ntest_dir = \"\"\n",
|
content: "[mire]\ntest_dir = \"\"\n",
|
||||||
@@ -62,44 +73,59 @@ func TestReadConfig(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "without sandbox table",
|
name: "without sandbox table",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n",
|
||||||
wantErr: "missing [sandbox] config",
|
wantErr: "missing [sandbox] config",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "without required home",
|
name: "without required home",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nmounts = []\npaths = []\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nmounts = []\npaths = []\n",
|
||||||
wantErr: "missing required sandbox.home",
|
wantErr: "missing required sandbox.home",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "without required mounts",
|
name: "without required mounts",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\npaths = []\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\npaths = []\n",
|
||||||
wantErr: "missing required sandbox.mounts",
|
wantErr: "missing required sandbox.mounts",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "empty required home",
|
name: "empty required home",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"\"\nmounts = []\npaths = []\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"\"\nmounts = []\npaths = []\n",
|
||||||
wantErr: "empty sandbox.home",
|
wantErr: "empty sandbox.home",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "relative home",
|
name: "relative home",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"home/test\"\nmounts = []\npaths = []\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"home/test\"\nmounts = []\npaths = []\n",
|
||||||
wantErr: "sandbox.home must be an absolute path",
|
wantErr: "sandbox.home must be an absolute path",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid sandbox key",
|
name: "invalid sandbox key",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\nKeyWord = \"value\"\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\nKeyWord = \"value\"\n",
|
||||||
wantErr: "invalid sandbox key",
|
wantErr: "invalid sandbox key",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "non string sandbox value",
|
name: "non string sandbox value",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\nkey_word = 1\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\nkey_word = 1\n",
|
||||||
wantErr: "sandbox.key_word must be a string",
|
wantErr: "sandbox.key_word must be a string",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "mounts wrong type",
|
name: "mounts wrong type",
|
||||||
content: "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = \"oops\"\npaths = []\n",
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\nmounts = \"oops\"\npaths = []\n",
|
||||||
wantErr: "failed to read",
|
wantErr: "failed to read",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "ignore diffs wrong type",
|
||||||
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = \"oops\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\n",
|
||||||
|
wantErr: "failed to read",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ignore diff entry not string",
|
||||||
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = [1]\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\n",
|
||||||
|
wantErr: "failed to read",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ignore diff regex invalid",
|
||||||
|
content: "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = [\"(\"]\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\n",
|
||||||
|
wantErr: "invalid mire.ignore_diffs[0] regex",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "invalid toml",
|
name: "invalid toml",
|
||||||
content: "[mire]\ntest_dir = [\n",
|
content: "[mire]\ntest_dir = [\n",
|
||||||
@@ -111,14 +137,15 @@ func TestReadConfig(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "normalizes relative mount host path",
|
name: "normalizes relative mount host path",
|
||||||
setup: func(t *testing.T, dir string) (string, []string, []string) {
|
setup: func(t *testing.T, dir string) (string, []string, []string, []string) {
|
||||||
hostBuild := filepath.Join(dir, "build")
|
hostBuild := filepath.Join(dir, "build")
|
||||||
if err := os.MkdirAll(hostBuild, 0o755); err != nil {
|
if err := os.MkdirAll(hostBuild, 0o755); err != nil {
|
||||||
t.Fatalf("MkdirAll(%q) error = %v", hostBuild, err)
|
t.Fatalf("MkdirAll(%q) error = %v", hostBuild, err)
|
||||||
}
|
}
|
||||||
return "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = [\"./build:/sandbox/build\"]\npaths = []\n",
|
return "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\nmounts = [\"./build:/sandbox/build\"]\npaths = []\n",
|
||||||
[]string{hostBuild + ":/sandbox/build"},
|
[]string{hostBuild + ":/sandbox/build"},
|
||||||
nil
|
nil,
|
||||||
|
[]string{}
|
||||||
},
|
},
|
||||||
wantDir: "custom/suite",
|
wantDir: "custom/suite",
|
||||||
wantSandbox: map[string]string{
|
wantSandbox: map[string]string{
|
||||||
@@ -127,7 +154,7 @@ func TestReadConfig(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "normalizes relative paths host path",
|
name: "normalizes relative paths host path",
|
||||||
setup: func(t *testing.T, dir string) (string, []string, []string) {
|
setup: func(t *testing.T, dir string) (string, []string, []string, []string) {
|
||||||
hostTool := filepath.Join(dir, "build", "mend")
|
hostTool := filepath.Join(dir, "build", "mend")
|
||||||
if err := os.MkdirAll(filepath.Dir(hostTool), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(hostTool), 0o755); err != nil {
|
||||||
t.Fatalf("MkdirAll(%q) error = %v", filepath.Dir(hostTool), err)
|
t.Fatalf("MkdirAll(%q) error = %v", filepath.Dir(hostTool), err)
|
||||||
@@ -135,9 +162,10 @@ func TestReadConfig(t *testing.T) {
|
|||||||
if err := os.WriteFile(hostTool, []byte("tool"), 0o644); err != nil {
|
if err := os.WriteFile(hostTool, []byte("tool"), 0o644); err != nil {
|
||||||
t.Fatalf("WriteFile(%q) error = %v", hostTool, err)
|
t.Fatalf("WriteFile(%q) error = %v", hostTool, err)
|
||||||
}
|
}
|
||||||
return "[mire]\ntest_dir = \"custom/suite\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = [\"./build/mend\"]\n",
|
return "[mire]\ntest_dir = \"custom/suite\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = [\"./build/mend\"]\n",
|
||||||
nil,
|
nil,
|
||||||
[]string{hostTool}
|
[]string{hostTool},
|
||||||
|
[]string{}
|
||||||
},
|
},
|
||||||
wantDir: "custom/suite",
|
wantDir: "custom/suite",
|
||||||
wantSandbox: map[string]string{
|
wantSandbox: map[string]string{
|
||||||
@@ -152,9 +180,10 @@ func TestReadConfig(t *testing.T) {
|
|||||||
path := filepath.Join(dir, "mire.toml")
|
path := filepath.Join(dir, "mire.toml")
|
||||||
wantMounts := tt.wantMounts
|
wantMounts := tt.wantMounts
|
||||||
wantPaths := tt.wantPaths
|
wantPaths := tt.wantPaths
|
||||||
|
wantIgnoreDiffs := tt.wantIgnoreDiffs
|
||||||
content := tt.content
|
content := tt.content
|
||||||
if tt.setup != nil {
|
if tt.setup != nil {
|
||||||
content, wantMounts, wantPaths = tt.setup(t, dir)
|
content, wantMounts, wantPaths, wantIgnoreDiffs = tt.setup(t, dir)
|
||||||
}
|
}
|
||||||
if !tt.wantMissing {
|
if !tt.wantMissing {
|
||||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
@@ -184,6 +213,14 @@ func TestReadConfig(t *testing.T) {
|
|||||||
if got.TestDir != tt.wantDir {
|
if got.TestDir != tt.wantDir {
|
||||||
t.Fatalf("ReadConfig() TestDir = %q, want %q", got.TestDir, tt.wantDir)
|
t.Fatalf("ReadConfig() TestDir = %q, want %q", got.TestDir, tt.wantDir)
|
||||||
}
|
}
|
||||||
|
if len(got.IgnoreDiffs) != len(wantIgnoreDiffs) {
|
||||||
|
t.Fatalf("ReadConfig() IgnoreDiffs = %#v, want %#v", got.IgnoreDiffs, wantIgnoreDiffs)
|
||||||
|
}
|
||||||
|
for i, want := range wantIgnoreDiffs {
|
||||||
|
if got.IgnoreDiffs[i] != want {
|
||||||
|
t.Fatalf("ReadConfig() IgnoreDiffs[%d] = %q, want %q", i, got.IgnoreDiffs[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(got.Sandbox) != len(tt.wantSandbox) {
|
if len(got.Sandbox) != len(tt.wantSandbox) {
|
||||||
t.Fatalf("ReadConfig() Sandbox = %#v, want %#v", got.Sandbox, tt.wantSandbox)
|
t.Fatalf("ReadConfig() Sandbox = %#v, want %#v", got.Sandbox, tt.wantSandbox)
|
||||||
}
|
}
|
||||||
@@ -226,6 +263,9 @@ func TestWriteDefaultConfig(t *testing.T) {
|
|||||||
if got.TestDir != "e2e" {
|
if got.TestDir != "e2e" {
|
||||||
t.Fatalf("ReadConfig() TestDir = %q, want %q", got.TestDir, "e2e")
|
t.Fatalf("ReadConfig() TestDir = %q, want %q", got.TestDir, "e2e")
|
||||||
}
|
}
|
||||||
|
if len(got.IgnoreDiffs) != 0 {
|
||||||
|
t.Fatalf("ReadConfig() IgnoreDiffs = %#v, want empty", got.IgnoreDiffs)
|
||||||
|
}
|
||||||
if got.Sandbox["home"] != DefaultVisibleHome {
|
if got.Sandbox["home"] != DefaultVisibleHome {
|
||||||
t.Fatalf("ReadConfig() Sandbox[home] = %q, want %q", got.Sandbox["home"], DefaultVisibleHome)
|
t.Fatalf("ReadConfig() Sandbox[home] = %q, want %q", got.Sandbox["home"], DefaultVisibleHome)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
[mire]
|
[mire]
|
||||||
# which folder to strore tests in
|
# which folder to strore tests in
|
||||||
test_dir = "e2e"
|
test_dir = "e2e"
|
||||||
|
# regexes for differing lines to ignore during replay comparison
|
||||||
|
ignore_diffs = []
|
||||||
|
|
||||||
[sandbox]
|
[sandbox]
|
||||||
# home is where mire would drop you by default on record
|
# home is where mire would drop you by default on record
|
||||||
|
|||||||
+47
-5
@@ -3,6 +3,7 @@ package mire
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"mire/internal/output"
|
"mire/internal/output"
|
||||||
@@ -22,15 +23,21 @@ type mismatchDetails struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeScenarioMismatch(w io.Writer, mismatch *testMismatchError) {
|
func writeScenarioMismatch(w io.Writer, mismatch *testMismatchError) {
|
||||||
details := firstMismatchingLine(mismatch.expected, mismatch.actual)
|
details := mismatch.details
|
||||||
|
|
||||||
fmt.Fprintf(w, "%s\n%s\n", italicLabel("Expected", output.Pass), formatMismatchLine(details.expected))
|
fmt.Fprintf(w, "%s\n%s\n", italicLabel("Expected", output.Pass), formatMismatchLine(details.expected))
|
||||||
fmt.Fprintf(w, "%s\n%s\n", italicLabel("Actual", output.Fail), formatMismatchLine(details.actual))
|
fmt.Fprintf(w, "%s\n%s\n", italicLabel("Actual", output.Fail), formatMismatchLine(details.actual))
|
||||||
}
|
}
|
||||||
|
|
||||||
func firstMismatchingLine(expected, actual []byte) mismatchDetails {
|
func firstMismatchingLine(expected, actual []byte) mismatchDetails {
|
||||||
|
_, details := compareOutput(expected, actual, nil)
|
||||||
|
return details
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareOutput(expected, actual []byte, ignoreDiffs []string) (bool, mismatchDetails) {
|
||||||
expectedLines := splitMismatchLines(expected)
|
expectedLines := splitMismatchLines(expected)
|
||||||
actualLines := splitMismatchLines(actual)
|
actualLines := splitMismatchLines(actual)
|
||||||
|
patterns := compileIgnoreDiffs(ignoreDiffs)
|
||||||
|
|
||||||
maxLines := len(expectedLines)
|
maxLines := len(expectedLines)
|
||||||
if len(actualLines) > maxLines {
|
if len(actualLines) > maxLines {
|
||||||
@@ -40,21 +47,56 @@ func firstMismatchingLine(expected, actual []byte) mismatchDetails {
|
|||||||
for i := 0; i < maxLines; i++ {
|
for i := 0; i < maxLines; i++ {
|
||||||
expectedLine := mismatchLineAt(expectedLines, i)
|
expectedLine := mismatchLineAt(expectedLines, i)
|
||||||
actualLine := mismatchLineAt(actualLines, i)
|
actualLine := mismatchLineAt(actualLines, i)
|
||||||
if expectedLine != actualLine {
|
if expectedLine == actualLine {
|
||||||
return mismatchDetails{
|
continue
|
||||||
|
}
|
||||||
|
if shouldIgnoreMismatch(expectedLine, actualLine, patterns) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return false, mismatchDetails{
|
||||||
lineNumber: i + 1,
|
lineNumber: i + 1,
|
||||||
expected: expectedLine,
|
expected: expectedLine,
|
||||||
actual: actualLine,
|
actual: actualLine,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return mismatchDetails{
|
return true, mismatchDetails{
|
||||||
expected: mismatchLineAt(expectedLines, len(expectedLines)-1),
|
expected: mismatchLineAt(expectedLines, len(expectedLines)-1),
|
||||||
actual: mismatchLineAt(actualLines, len(actualLines)-1),
|
actual: mismatchLineAt(actualLines, len(actualLines)-1),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func shouldIgnoreMismatch(expected, actual mismatchLine, patterns []*regexp.Regexp) bool {
|
||||||
|
if !expected.present || !actual.present || len(patterns) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return lineMatchesAnyPattern(expected.text, patterns) && lineMatchesAnyPattern(actual.text, patterns)
|
||||||
|
}
|
||||||
|
|
||||||
|
func compileIgnoreDiffs(ignoreDiffs []string) []*regexp.Regexp {
|
||||||
|
if len(ignoreDiffs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
patterns := make([]*regexp.Regexp, 0, len(ignoreDiffs))
|
||||||
|
for _, ignoreDiff := range ignoreDiffs {
|
||||||
|
patterns = append(patterns, regexp.MustCompile(ignoreDiff))
|
||||||
|
}
|
||||||
|
|
||||||
|
return patterns
|
||||||
|
}
|
||||||
|
|
||||||
|
func lineMatchesAnyPattern(line string, patterns []*regexp.Regexp) bool {
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
if pattern.MatchString(line) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func splitMismatchLines(data []byte) []string {
|
func splitMismatchLines(data []byte) []string {
|
||||||
if len(data) == 0 {
|
if len(data) == 0 {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -40,6 +40,75 @@ func TestFirstMismatchingLineMarksMissingLine(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompareOutputMatchesEqualOutput(t *testing.T) {
|
||||||
|
matched, details := compareOutput([]byte("alpha\n"), []byte("alpha\n"), nil)
|
||||||
|
if !matched {
|
||||||
|
t.Fatalf("compareOutput() matched = false, want true with details %#v", details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareOutputSkipsIgnoredMismatches(t *testing.T) {
|
||||||
|
matched, details := compareOutput(
|
||||||
|
[]byte("ts=111\nid=foo\nstable\n"),
|
||||||
|
[]byte("ts=222\nid=bar\nstable\n"),
|
||||||
|
[]string{`^ts=\d+$`, `^id=\w+$`},
|
||||||
|
)
|
||||||
|
if !matched {
|
||||||
|
t.Fatalf("compareOutput() matched = false, want true with details %#v", details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareOutputReturnsFirstNonIgnoredMismatch(t *testing.T) {
|
||||||
|
matched, details := compareOutput(
|
||||||
|
[]byte("ts=111\nstable\nfinal=a\n"),
|
||||||
|
[]byte("ts=222\nstable\nfinal=b\n"),
|
||||||
|
[]string{`^ts=\d+$`},
|
||||||
|
)
|
||||||
|
if matched {
|
||||||
|
t.Fatal("compareOutput() matched = true, want false")
|
||||||
|
}
|
||||||
|
if details.lineNumber != 3 {
|
||||||
|
t.Fatalf("lineNumber = %d, want 3", details.lineNumber)
|
||||||
|
}
|
||||||
|
if details.expected != (mismatchLine{text: "final=a", present: true}) {
|
||||||
|
t.Fatalf("expected line = %#v, want first non-ignored mismatch", details.expected)
|
||||||
|
}
|
||||||
|
if details.actual != (mismatchLine{text: "final=b", present: true}) {
|
||||||
|
t.Fatalf("actual line = %#v, want first non-ignored mismatch", details.actual)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareOutputDoesNotIgnoreOneSidedMatch(t *testing.T) {
|
||||||
|
matched, details := compareOutput(
|
||||||
|
[]byte("ts=111\n"),
|
||||||
|
[]byte("value=222\n"),
|
||||||
|
[]string{`^ts=\d+$`},
|
||||||
|
)
|
||||||
|
if matched {
|
||||||
|
t.Fatal("compareOutput() matched = true, want false")
|
||||||
|
}
|
||||||
|
if details.lineNumber != 1 {
|
||||||
|
t.Fatalf("lineNumber = %d, want 1", details.lineNumber)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareOutputDoesNotIgnoreMissingLine(t *testing.T) {
|
||||||
|
matched, details := compareOutput(
|
||||||
|
[]byte("ts=111\n"),
|
||||||
|
[]byte(""),
|
||||||
|
[]string{`^ts=\d+$`},
|
||||||
|
)
|
||||||
|
if matched {
|
||||||
|
t.Fatal("compareOutput() matched = true, want false")
|
||||||
|
}
|
||||||
|
if details.lineNumber != 1 {
|
||||||
|
t.Fatalf("lineNumber = %d, want 1", details.lineNumber)
|
||||||
|
}
|
||||||
|
if details.actual.present {
|
||||||
|
t.Fatalf("actual line = %#v, want missing line", details.actual)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWriteScenarioMismatchPrintsOnlyFirstMismatch(t *testing.T) {
|
func TestWriteScenarioMismatchPrintsOnlyFirstMismatch(t *testing.T) {
|
||||||
t.Setenv("NO_COLOR", "1")
|
t.Setenv("NO_COLOR", "1")
|
||||||
|
|
||||||
@@ -47,6 +116,11 @@ func TestWriteScenarioMismatchPrintsOnlyFirstMismatch(t *testing.T) {
|
|||||||
writeScenarioMismatch(&buf, &testMismatchError{
|
writeScenarioMismatch(&buf, &testMismatchError{
|
||||||
expected: []byte("$ echo x\nx\n$ \nexit\n"),
|
expected: []byte("$ echo x\nx\n$ \nexit\n"),
|
||||||
actual: []byte("$ echo a\na\n$ \nexit\n"),
|
actual: []byte("$ echo a\na\n$ \nexit\n"),
|
||||||
|
details: mismatchDetails{
|
||||||
|
lineNumber: 1,
|
||||||
|
expected: mismatchLine{text: "$ echo x", present: true},
|
||||||
|
actual: mismatchLine{text: "$ echo a", present: true},
|
||||||
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
got := buf.String()
|
got := buf.String()
|
||||||
@@ -78,6 +152,11 @@ func TestWriteScenarioMismatchFormatsMissingLine(t *testing.T) {
|
|||||||
writeScenarioMismatch(&buf, &testMismatchError{
|
writeScenarioMismatch(&buf, &testMismatchError{
|
||||||
expected: []byte("alpha\nbeta\n"),
|
expected: []byte("alpha\nbeta\n"),
|
||||||
actual: []byte("alpha\n"),
|
actual: []byte("alpha\n"),
|
||||||
|
details: mismatchDetails{
|
||||||
|
lineNumber: 2,
|
||||||
|
expected: mismatchLine{text: "beta", present: true},
|
||||||
|
actual: mismatchLine{},
|
||||||
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
got := buf.String()
|
got := buf.String()
|
||||||
@@ -96,6 +175,11 @@ func TestWriteScenarioMismatchItalicizesLabels(t *testing.T) {
|
|||||||
writeScenarioMismatch(&buf, &testMismatchError{
|
writeScenarioMismatch(&buf, &testMismatchError{
|
||||||
expected: []byte("alpha\n"),
|
expected: []byte("alpha\n"),
|
||||||
actual: []byte("beta\n"),
|
actual: []byte("beta\n"),
|
||||||
|
details: mismatchDetails{
|
||||||
|
lineNumber: 1,
|
||||||
|
expected: mismatchLine{text: "alpha", present: true},
|
||||||
|
actual: mismatchLine{text: "beta", present: true},
|
||||||
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
got := buf.String()
|
got := buf.String()
|
||||||
@@ -105,7 +189,7 @@ func TestWriteScenarioMismatchItalicizesLabels(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTestMismatchErrorIncludesLineNumber(t *testing.T) {
|
func TestTestMismatchErrorIncludesLineNumber(t *testing.T) {
|
||||||
err := &testMismatchError{lineNumber: 7}
|
err := &testMismatchError{details: mismatchDetails{lineNumber: 7}}
|
||||||
if got := err.Error(); got != "output differed at line 7" {
|
if got := err.Error(); got != "output differed at line 7" {
|
||||||
t.Fatalf("Error() = %q, want %q", got, "output differed at line 7")
|
t.Fatalf("Error() = %q, want %q", got, "output differed at line 7")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
type replaySuite struct {
|
type replaySuite struct {
|
||||||
shellPath string
|
shellPath string
|
||||||
scenarios []testScenario
|
scenarios []testScenario
|
||||||
|
ignoreDiffs []string
|
||||||
sandboxConfig map[string]string
|
sandboxConfig map[string]string
|
||||||
mounts []string
|
mounts []string
|
||||||
paths []string
|
paths []string
|
||||||
@@ -57,6 +58,7 @@ func loadReplaySuite(path string) (replaySuite, error) {
|
|||||||
return replaySuite{
|
return replaySuite{
|
||||||
shellPath: shellPath,
|
shellPath: shellPath,
|
||||||
scenarios: scenarios,
|
scenarios: scenarios,
|
||||||
|
ignoreDiffs: cfg.IgnoreDiffs,
|
||||||
sandboxConfig: cfg.Sandbox,
|
sandboxConfig: cfg.Sandbox,
|
||||||
mounts: cfg.Mounts,
|
mounts: cfg.Mounts,
|
||||||
paths: cfg.Paths,
|
paths: cfg.Paths,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package mire
|
package mire
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -41,7 +40,7 @@ type testFixtureFiles struct {
|
|||||||
type testMismatchError struct {
|
type testMismatchError struct {
|
||||||
expected []byte
|
expected []byte
|
||||||
actual []byte
|
actual []byte
|
||||||
lineNumber int
|
details mismatchDetails
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -50,8 +49,8 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (e *testMismatchError) Error() string {
|
func (e *testMismatchError) Error() string {
|
||||||
if e.lineNumber > 0 {
|
if e.details.lineNumber > 0 {
|
||||||
return fmt.Sprintf("output differed at line %d", e.lineNumber)
|
return fmt.Sprintf("output differed at line %d", e.details.lineNumber)
|
||||||
}
|
}
|
||||||
|
|
||||||
return "output differed"
|
return "output differed"
|
||||||
@@ -76,7 +75,7 @@ func runTests(path string, tio testIO) error {
|
|||||||
output.Fprintf(tio.out, "%s %s\n", output.Label("RUN", output.Info), scenario.relPath)
|
output.Fprintf(tio.out, "%s %s\n", output.Label("RUN", output.Info), scenario.relPath)
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
if err := replayScenario(scenario, suite.shellPath, suite.sandboxConfig, suite.mounts, suite.paths); err != nil {
|
if err := replayScenario(scenario, suite.shellPath, suite.ignoreDiffs, suite.sandboxConfig, suite.mounts, suite.paths); err != nil {
|
||||||
elapsed := time.Since(start)
|
elapsed := time.Since(start)
|
||||||
summary.failed++
|
summary.failed++
|
||||||
output.Fprintf(tio.out, "%s %s (%s): %v\n", output.Label("FAIL", output.Fail), scenario.relPath, formatElapsed(elapsed), err)
|
output.Fprintf(tio.out, "%s %s (%s): %v\n", output.Label("FAIL", output.Fail), scenario.relPath, formatElapsed(elapsed), err)
|
||||||
@@ -209,7 +208,7 @@ func discoverTestScenarios(discoveryRoot, displayRoot string) ([]testScenario, e
|
|||||||
return scenarios, nil
|
return scenarios, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func replayScenario(scenario testScenario, shellPath string, sandboxConfig map[string]string, mounts, paths []string) error {
|
func replayScenario(scenario testScenario, shellPath string, ignoreDiffs []string, sandboxConfig map[string]string, mounts, paths []string) error {
|
||||||
got, err := replayScenarioOutput(scenario, shellPath, sandboxConfig, mounts, paths)
|
got, err := replayScenarioOutput(scenario, shellPath, sandboxConfig, mounts, paths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -220,12 +219,12 @@ func replayScenario(scenario testScenario, shellPath string, sandboxConfig map[s
|
|||||||
return fmt.Errorf("failed to read expected output: %v", err)
|
return fmt.Errorf("failed to read expected output: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !bytes.Equal(got, want) {
|
matched, details := compareOutput(want, got, ignoreDiffs)
|
||||||
details := firstMismatchingLine(want, got)
|
if !matched {
|
||||||
return &testMismatchError{
|
return &testMismatchError{
|
||||||
expected: want,
|
expected: want,
|
||||||
actual: got,
|
actual: got,
|
||||||
lineNumber: details.lineNumber,
|
details: details,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -105,7 +105,28 @@ func TestReplayScenarioUsesRecordedInputAndKeepsReplayOutputQuiet(t *testing.T)
|
|||||||
inPath: filepath.Join(scenarioDir, "in"),
|
inPath: filepath.Join(scenarioDir, "in"),
|
||||||
outPath: filepath.Join(scenarioDir, "out"),
|
outPath: filepath.Join(scenarioDir, "out"),
|
||||||
setupScripts: nil,
|
setupScripts: nil,
|
||||||
}, shellPath, defaultSandboxConfig(), nil, nil)
|
}, shellPath, nil, defaultSandboxConfig(), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("replayScenario() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplayScenarioIgnoresConfiguredDiffLines(t *testing.T) {
|
||||||
|
testutil.AddFakeRecordDependencies(t, "bwrap", "bash")
|
||||||
|
|
||||||
|
testDir := filepath.Join(t.TempDir(), "e2e")
|
||||||
|
shellPath := filepath.Join(testDir, "shell.sh")
|
||||||
|
mustWriteRecordShell(t, testDir)
|
||||||
|
scenarioDir := filepath.Join(testDir, "suite", "spec")
|
||||||
|
testutil.WriteScenarioFixtures(t, scenarioDir, "ts=222\nexit\n", "ts=111\r\nexit\r\n")
|
||||||
|
|
||||||
|
err := replayScenario(testScenario{
|
||||||
|
dir: scenarioDir,
|
||||||
|
relPath: filepath.Join("suite", "spec"),
|
||||||
|
inPath: filepath.Join(scenarioDir, "in"),
|
||||||
|
outPath: filepath.Join(scenarioDir, "out"),
|
||||||
|
setupScripts: nil,
|
||||||
|
}, shellPath, []string{`^ts=\d+$`}, defaultSandboxConfig(), nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("replayScenario() error = %v", err)
|
t.Fatalf("replayScenario() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -131,7 +152,7 @@ func TestReplayScenarioWaitsForPromptReadyMarkerBeforeSendingInput(t *testing.T)
|
|||||||
inPath: filepath.Join(scenarioDir, "in"),
|
inPath: filepath.Join(scenarioDir, "in"),
|
||||||
outPath: filepath.Join(scenarioDir, "out"),
|
outPath: filepath.Join(scenarioDir, "out"),
|
||||||
setupScripts: nil,
|
setupScripts: nil,
|
||||||
}, shellPath, defaultSandboxConfig(), nil, nil)
|
}, shellPath, nil, defaultSandboxConfig(), nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("replayScenario() error = %v", err)
|
t.Fatalf("replayScenario() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -153,7 +174,7 @@ func TestReplayScenarioFailsWhenCompareMarkerMissing(t *testing.T) {
|
|||||||
inPath: filepath.Join(scenarioDir, "in"),
|
inPath: filepath.Join(scenarioDir, "in"),
|
||||||
outPath: filepath.Join(scenarioDir, "out"),
|
outPath: filepath.Join(scenarioDir, "out"),
|
||||||
setupScripts: nil,
|
setupScripts: nil,
|
||||||
}, shellPath, defaultSandboxConfig(), nil, nil)
|
}, shellPath, nil, defaultSandboxConfig(), nil, nil)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("replayScenario() error = nil, want error")
|
t.Fatal("replayScenario() error = nil, want error")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -444,5 +444,5 @@ func MustGitInit(t *testing.T, dir string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validConfigContent(testDir string) string {
|
func validConfigContent(testDir string) string {
|
||||||
return "[mire]\ntest_dir = \"" + testDir + "\"\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\n"
|
return "[mire]\ntest_dir = \"" + testDir + "\"\nignore_diffs = []\n\n[sandbox]\nhome = \"/home/test\"\nmounts = []\npaths = []\n"
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user