feat: flag to fail instead of skip if ./tests cases are missing

.github/workflows/go.yml

@@ -46,9 +46,12 @@ jobs:
         run:
           | # Upstream flakes are race conditions exacerbated by concurrent tests
           go list ./... | grep -P "${FLAKY_REGEX}" | xargs -n 1 go test -short;
-      - name: Run non-flaky tests concurrently
+      - name: Run ./tests directory tests
         run: |
-          go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}");
+          go test -short ./tests -libevm.fail_instead_of_skip;
+      - name: Run non-flaky tests concurrently
+        run: | # The ./tests directory is run separetely to allow use of a specific flag
+          go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}" | grep -Pv "ava-labs/libevm/tests$");
 
   go_test_tooling:
     runs-on: ubuntu-latest

tests/block_test.go

@@ -64,7 +64,7 @@ func TestBlockchain(t *testing.T) {
 // TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests.
 func TestExecutionSpecBlocktests(t *testing.T) {
 	if !common.FileExist(executionSpecBlockchainTestDir) {
-		t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir)
+		failOrSkip(t, "directory %s does not exist", executionSpecBlockchainTestDir)
 	}
 	bt := new(testMatcher)
 

tests/init_test.go

@@ -197,7 +197,7 @@ func (tm *testMatcher) walk(t *testing.T, dir string, runTest interface{}) {
 	dirinfo, err := os.Stat(dir)
 	if os.IsNotExist(err) || !dirinfo.IsDir() {
 		fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the tests submodule?\n", dir)
-		t.Skip("missing test files")
+		failOrSkip(t, "missing test files")
 	}
 	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		name := filepath.ToSlash(strings.TrimPrefix(path, dir+string(filepath.Separator)))

tests/noskip.libevm_test.go

@@ -0,0 +1,48 @@
+// Copyright 2026 the libevm authors.
+//
+// The libevm additions to go-ethereum are free software: you can redistribute
+// them and/or modify them under the terms of the GNU Lesser General Public License
+// as published by the Free Software Foundation, either version 3 of the License,
+// or (at your option) any later version.
+//
+// The libevm additions are distributed in the hope that they will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+// General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see
+// <http://www.gnu.org/licenses/>.
+
+package tests
+
+import (
+	"flag"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	flag.BoolVar(
+		&failInsteadOfSkip,
+		"libevm.fail_instead_of_skip",
+		false,
+		"If true and test cases are missing then respective tests fail instead of skipping",
+	)
+	flag.Parse()
+
+	m.Run()
+}
+
+var failInsteadOfSkip bool
+
+// failOrSkip propagates its arguments to Skipf or Fatalf, depending on the
+// value of [failInsteadOfSkip], defaulting to skipping for backwards
+// compatibility. See [TestMain] for the respective flag.
+func failOrSkip(tb testing.TB, format string, args ...any) {
+	tb.Helper()
+	fn := tb.Skipf
+	if failInsteadOfSkip {
+		fn = tb.Fatalf
+	}
+	fn(format, args...)
+}

tests/state_test.go

@@ -89,7 +89,7 @@ func TestLegacyState(t *testing.T) {
 // TestExecutionSpecState runs the test fixtures from execution-spec-tests.
 func TestExecutionSpecState(t *testing.T) {
 	if !common.FileExist(executionSpecStateTestDir) {
-		t.Skipf("directory %s does not exist", executionSpecStateTestDir)
+		failOrSkip(t, "directory %s does not exist", executionSpecStateTestDir)
 	}
 	st := new(testMatcher)
 
@@ -198,7 +198,7 @@ func BenchmarkEVM(b *testing.B) {
 	dirinfo, err := os.Stat(dir)
 	if os.IsNotExist(err) || !dirinfo.IsDir() {
 		fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the evm-benchmarks submodule?\n", dir)
-		b.Skip("missing test files")
+		failOrSkip(b, "missing test files")
 	}
 	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if info.IsDir() {