From 8aa3608a3c8114f5235e1554cb7f04c9c7ae78fb Mon Sep 17 00:00:00 2001
From: Karl Seguin <k@openmymind.io>
Date: Tue, 22 Apr 2025 21:05:12 +0800
Subject: [PATCH] Don't [try] to run non-tests

Currently, we treat every .html file in tests/wpt as-if it's a test. That
isn't always the case. wpt has a manifest tool to generate a manifest (in
JSON) of the tests, we should probably use that (but it's quite large).

This PR filters out two groups. First, everything in resources/ appears to
be things to _run_ the tests, not actual tests.

Second, any file without a "testharness.js" doesn't appear to be a test
either. Note that WPT's own manifest generator looks at this:
https://github.com/web-platform-tests/wpt/blob/43a06153617aa58f877f9e5873c0a02290b1c2a8/tools/manifest/sourcefile.py#L676
(which is used later in the file to determine the type of file).
---
 src/main_wpt.zig |  4 ++++
 src/wpt/run.zig  | 18 +++++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index eeabaf12..5f16b954 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -148,6 +148,10 @@ pub fn main() !void {
             }
             failures += 1;
             continue;
+        } orelse {
+            // This test should _not_ have been run.
+            run -= 1;
+            continue;
         };
 
         const suite = try Suite.init(alloc, tc, true, res);
diff --git a/src/wpt/run.zig b/src/wpt/run.zig
index 6823aaf9..fcca32d0 100644
--- a/src/wpt/run.zig
+++ b/src/wpt/run.zig
@@ -30,7 +30,7 @@ const polyfill = @import("../browser/polyfill/polyfill.zig");
 // runWPT parses the given HTML file, starts a js env and run the first script
 // tags containing javascript sources.
 // It loads first the js libs files.
-pub fn run(arena: Allocator, comptime dir: []const u8, f: []const u8, loader: *FileLoader, err_msg: *?[]const u8) ![]const u8 {
+pub fn run(arena: Allocator, comptime dir: []const u8, f: []const u8, loader: *FileLoader, err_msg: *?[]const u8) !?[]const u8 {
     // document
     const html = blk: {
         const file = try std.fs.cwd().openFile(f, .{});
@@ -38,6 +38,12 @@ pub fn run(arena: Allocator, comptime dir: []const u8, f: []const u8, loader: *F
         break :blk try file.readToEndAlloc(arena, 128 * 1024);
     };
 
+    if (std.mem.indexOf(u8, html, "testharness.js") == null) {
+        // This isn't a test. A lot of files are helpers/content for tests to
+        // make use of.
+        return null;
+    }
+
     const dirname = fspath.dirname(f[dir.len..]) orelse unreachable;
 
     var runner = try @import("../testing.zig").jsRunner(arena, .{
@@ -115,11 +121,11 @@ pub fn run(arena: Allocator, comptime dir: []const u8, f: []const u8, loader: *F
 
     // return the detailed result.
     const res = try runner.eval("report.log", "report", err_msg);
-    return res.toString(arena);
+    return try res.toString(arena);
 }
 
 // browse the path to find the tests list.
-pub fn find(allocator: std.mem.Allocator, comptime path: []const u8, list: *std.ArrayList([]const u8)) !void {
+pub fn find(allocator: Allocator, comptime path: []const u8, list: *std.ArrayList([]const u8)) !void {
     var dir = try std.fs.cwd().openDir(path, .{ .iterate = true, .no_follow = true });
     defer dir.close();
 
@@ -130,6 +136,12 @@ pub fn find(allocator: std.mem.Allocator, comptime path: []const u8, list: *std.
         if (entry.kind != .file) {
             continue;
         }
+
+        if (std.mem.startsWith(u8, entry.path, "resources/")) {
+            // resources for running the tests themselves, not actual tests
+            continue;
+        }
+
         if (!std.mem.endsWith(u8, entry.basename, ".html") and !std.mem.endsWith(u8, entry.basename, ".htm")) {
             continue;
         }