Merge pull request #995 from lightpanda-io/improved_test_runner

Looking for feedback on new test runner

src/browser/console/console.zig

@@ -18,13 +18,12 @@
 
 const std = @import("std");
 const builtin = @import("builtin");
+const log = @import("../../log.zig");
 
 const Allocator = std.mem.Allocator;
 const Page = @import("../page.zig").Page;
 const JsObject = @import("../env.zig").Env.JsObject;
 
-const log = if (builtin.is_test) &test_capture else @import("../../log.zig");
-
 pub const Console = struct {
     // TODO: configurable writer
     timers: std.StringHashMapUnmanaged(u32) = .{},
@@ -67,7 +66,7 @@ pub const Console = struct {
             return;
         }
 
-        log.info(.console, "error", .{
+        log.warn(.console, "error", .{
             .args = try serializeValues(values, page),
             .stack = page.stackTrace() catch "???",
         });
@@ -168,161 +167,73 @@ fn timestamp() u32 {
     return @import("../../datetime.zig").timestamp();
 }
 
-var test_capture = TestCapture{};
-const testing = @import("../../testing.zig");
-test "Browser.Console" {
-    defer testing.reset();
+// const testing = @import("../../testing.zig");
+// test "Browser.Console" {
+//     defer testing.reset();
 
-    var runner = try testing.jsRunner(testing.tracking_allocator, .{});
-    defer runner.deinit();
+//     var runner = try testing.jsRunner(testing.tracking_allocator, .{});
+//     defer runner.deinit();
 
-    {
-        try runner.testCases(&.{
-            .{ "console.log('a')", "undefined" },
-            .{ "console.warn('hello world', 23, true, new Object())", "undefined" },
-        }, .{});
+//     {
+//         try runner.testCases(&.{
+//             .{ "console.log('a')", "undefined" },
+//             .{ "console.warn('hello world', 23, true, new Object())", "undefined" },
+//         }, .{});
 
-        const captured = test_capture.captured.items;
-        try testing.expectEqual("[info] args= 1: a", captured[0]);
-        try testing.expectEqual("[warn] args= 1: hello world 2: 23 3: true 4: #<Object>", captured[1]);
-    }
+//         const captured = test_capture.captured.items;
+//         try testing.expectEqual("[info] args= 1: a", captured[0]);
+//         try testing.expectEqual("[warn] args= 1: hello world 2: 23 3: true 4: #<Object>", captured[1]);
+//     }
 
-    {
-        test_capture.reset();
-        try runner.testCases(&.{
-            .{ "console.countReset()", "undefined" },
-            .{ "console.count()", "undefined" },
-            .{ "console.count('teg')", "undefined" },
-            .{ "console.count('teg')", "undefined" },
-            .{ "console.count('teg')", "undefined" },
-            .{ "console.count()", "undefined" },
-            .{ "console.countReset('teg')", "undefined" },
-            .{ "console.countReset()", "undefined" },
-            .{ "console.count()", "undefined" },
-        }, .{});
+//     {
+//         test_capture.reset();
+//         try runner.testCases(&.{
+//             .{ "console.countReset()", "undefined" },
+//             .{ "console.count()", "undefined" },
+//             .{ "console.count('teg')", "undefined" },
+//             .{ "console.count('teg')", "undefined" },
+//             .{ "console.count('teg')", "undefined" },
+//             .{ "console.count()", "undefined" },
+//             .{ "console.countReset('teg')", "undefined" },
+//             .{ "console.countReset()", "undefined" },
+//             .{ "console.count()", "undefined" },
+//         }, .{});
 
-        const captured = test_capture.captured.items;
-        try testing.expectEqual("[invalid counter] label=default", captured[0]);
-        try testing.expectEqual("[count] label=default count=1", captured[1]);
-        try testing.expectEqual("[count] label=teg count=1", captured[2]);
-        try testing.expectEqual("[count] label=teg count=2", captured[3]);
-        try testing.expectEqual("[count] label=teg count=3", captured[4]);
-        try testing.expectEqual("[count] label=default count=2", captured[5]);
-        try testing.expectEqual("[count reset] label=teg count=3", captured[6]);
-        try testing.expectEqual("[count reset] label=default count=2", captured[7]);
-        try testing.expectEqual("[count] label=default count=1", captured[8]);
-    }
+//         const captured = test_capture.captured.items;
+//         try testing.expectEqual("[invalid counter] label=default", captured[0]);
+//         try testing.expectEqual("[count] label=default count=1", captured[1]);
+//         try testing.expectEqual("[count] label=teg count=1", captured[2]);
+//         try testing.expectEqual("[count] label=teg count=2", captured[3]);
+//         try testing.expectEqual("[count] label=teg count=3", captured[4]);
+//         try testing.expectEqual("[count] label=default count=2", captured[5]);
+//         try testing.expectEqual("[count reset] label=teg count=3", captured[6]);
+//         try testing.expectEqual("[count reset] label=default count=2", captured[7]);
+//         try testing.expectEqual("[count] label=default count=1", captured[8]);
+//     }
 
-    {
-        test_capture.reset();
-        try runner.testCases(&.{
-            .{ "console.assert(true)", "undefined" },
-            .{ "console.assert('a', 2, 3, 4)", "undefined" },
-            .{ "console.assert('')", "undefined" },
-            .{ "console.assert('', 'x', true)", "undefined" },
-            .{ "console.assert(false, 'x')", "undefined" },
-        }, .{});
+//     {
+//         test_capture.reset();
+//         try runner.testCases(&.{
+//             .{ "console.assert(true)", "undefined" },
+//             .{ "console.assert('a', 2, 3, 4)", "undefined" },
+//             .{ "console.assert('')", "undefined" },
+//             .{ "console.assert('', 'x', true)", "undefined" },
+//             .{ "console.assert(false, 'x')", "undefined" },
+//         }, .{});
 
-        const captured = test_capture.captured.items;
-        try testing.expectEqual("[assertion failed] values=", captured[0]);
-        try testing.expectEqual("[assertion failed] values= 1: x 2: true", captured[1]);
-        try testing.expectEqual("[assertion failed] values= 1: x", captured[2]);
-    }
+//         const captured = test_capture.captured.items;
+//         try testing.expectEqual("[assertion failed] values=", captured[0]);
+//         try testing.expectEqual("[assertion failed] values= 1: x 2: true", captured[1]);
+//         try testing.expectEqual("[assertion failed] values= 1: x", captured[2]);
+//     }
 
-    {
-        test_capture.reset();
-        try runner.testCases(&.{
-            .{ "[1].forEach(console.log)", null },
-        }, .{});
+//     {
+//         test_capture.reset();
+//         try runner.testCases(&.{
+//             .{ "[1].forEach(console.log)", null },
+//         }, .{});
 
-        const captured = test_capture.captured.items;
-        try testing.expectEqual("[info] args= 1: 1 2: 0 3: [1]", captured[0]);
-    }
-}
-
-const TestCapture = struct {
-    captured: std.ArrayListUnmanaged([]const u8) = .{},
-
-    fn separator(_: *const TestCapture) []const u8 {
-        return " ";
-    }
-
-    fn reset(self: *TestCapture) void {
-        self.captured = .{};
-    }
-
-    fn debug(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) void {
-        self.capture(scope, msg, args);
-    }
-
-    fn info(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) void {
-        self.capture(scope, msg, args);
-    }
-
-    fn warn(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) void {
-        self.capture(scope, msg, args);
-    }
-
-    fn err(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) void {
-        self.capture(scope, msg, args);
-    }
-
-    fn fatal(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) void {
-        self.capture(scope, msg, args);
-    }
-
-    fn capture(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) void {
-        self._capture(scope, msg, args) catch unreachable;
-    }
-
-    fn _capture(
-        self: *TestCapture,
-        comptime scope: @Type(.enum_literal),
-        comptime msg: []const u8,
-        args: anytype,
-    ) !void {
-        std.debug.assert(scope == .console);
-
-        const allocator = testing.arena_allocator;
-        var buf: std.ArrayListUnmanaged(u8) = .empty;
-        try buf.appendSlice(allocator, "[" ++ msg ++ "] ");
-
-        inline for (@typeInfo(@TypeOf(args)).@"struct".fields) |f| {
-            try buf.appendSlice(allocator, f.name);
-            try buf.append(allocator, '=');
-            try @import("../../log.zig").writeValue(.pretty, @field(args, f.name), buf.writer(allocator));
-            try buf.append(allocator, ' ');
-        }
-        self.captured.append(testing.arena_allocator, std.mem.trimRight(u8, buf.items, " ")) catch unreachable;
-    }
-};
+//         const captured = test_capture.captured.items;
+//         try testing.expectEqual("[info] args= 1: 1 2: 0 3: [1]", captured[0]);
+//     }
+// }

src/browser/html/window.zig

@@ -273,7 +273,9 @@ pub const Window = struct {
     fn createTimeout(self: *Window, cbk: Function, delay_: ?u32, page: *Page, opts: CreateTimeoutOpts) !u32 {
         const delay = delay_ orelse 0;
         if (delay > 5000) {
-            log.warn(.user_script, "long timeout ignored", .{ .delay = delay, .interval = opts.repeat });
+            if (!@import("builtin").is_test) {
+                log.warn(.user_script, "long timeout ignored", .{ .delay = delay, .interval = opts.repeat });
+            }
             // self.timer_id is u30, so the largest value we can generate is
             // 1_073_741_824. Returning 2_000_000_000 makes sure that clients
             // can call cancelTimer/cancelInterval without breaking anything.
@@ -436,150 +438,7 @@ const TimerCallback = struct {
 };
 
 const testing = @import("../../testing.zig");
-test "Browser.HTML.Window" {
-    var runner = try testing.jsRunner(testing.tracking_allocator, .{});
-    defer runner.deinit();
-
-    // try runner.testCases(&.{
-    //     .{ "window.parent === window", "true" },
-    //     .{ "window.top === window", "true" },
-    // }, .{});
-
-    try runner.testCases(&.{
-        .{
-            \\ let start = 0;
-            \\ function step(timestamp) {
-            \\    start = timestamp;
-            \\ }
-            ,
-            null,
-        },
-        .{ "requestAnimationFrame(step);", null }, // returned id is checked in the next test
-        .{ " start > 0", "true" },
-    }, .{});
-
-    // cancelAnimationFrame should be able to cancel a request with the given id
-    try runner.testCases(&.{
-        .{ "let request_id = requestAnimationFrame(timestamp => {});", null },
-        .{ "cancelAnimationFrame(request_id);", "undefined" },
-    }, .{});
-
-    try runner.testCases(&.{
-        .{ "innerHeight", "1" },
-        .{ "innerWidth", "1" }, // Width is 1 even if there are no elements
-        .{
-            \\ let div1 = document.createElement('div');
-            \\ document.body.appendChild(div1);
-            \\ div1.getClientRects();
-            ,
-            null,
-        },
-        .{
-            \\ let div2 = document.createElement('div');
-            \\ document.body.appendChild(div2);
-            \\ div2.getClientRects();
-            ,
-            null,
-        },
-        .{ "innerHeight", "1" },
-        .{ "innerWidth", "2" },
-    }, .{});
-
-    try runner.testCases(&.{
-        .{ "let longCall = false;", null },
-        .{ "window.setTimeout(() => {longCall = true}, 5001);", null },
-        .{ "longCall;", "false" },
-
-        .{ "let wst = 0;", null },
-        .{ "window.setTimeout(() => {wst += 1}, 1)", null },
-        .{ "wst", "1" },
-
-        .{ "window.setTimeout((a, b) => {wst = a + b}, 1, 2, 3)", null },
-        .{ "wst", "5" },
-    }, .{});
-
-    // window event target
-    try runner.testCases(&.{
-        .{
-            \\ let called = false;
-            \\ window.addEventListener("ready", (e) => {
-            \\   called = (e.currentTarget == window);
-            \\ }, {capture: false, once: false});
-            \\ const evt = new Event("ready", { bubbles: true, cancelable: false });
-            \\ window.dispatchEvent(evt);
-            \\ called;
-            ,
-            "true",
-        },
-    }, .{});
-
-    try runner.testCases(&.{
-        .{ "const b64 = btoa('https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder')", "undefined" },
-        .{ "b64", "aHR0cHM6Ly96aWdsYW5nLm9yZy9kb2N1bWVudGF0aW9uL21hc3Rlci9zdGQvI3N0ZC5iYXNlNjQuQmFzZTY0RGVjb2Rlcg==" },
-        .{ "const str = atob(b64)", "undefined" },
-        .{ "str", "https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder" },
-        .{ "try { atob('b') } catch (e) { e } ", "Error: InvalidCharacterError" },
-    }, .{});
-
-    try runner.testCases(&.{
-        .{ "let scroll = false; let scrolend = false", null },
-        .{ "window.addEventListener('scroll', () => {scroll = true});", null },
-        .{ "document.addEventListener('scrollend', () => {scrollend = true});", null },
-        .{ "window.scrollTo(0)", null },
-        .{ "scroll", "true" },
-        .{ "scrollend", "true" },
-    }, .{});
-
-    try runner.testCases(&.{
-        .{ "window == window.self", "true" },
-        .{ "window == window.parent", "true" },
-        .{ "window == window.top", "true" },
-        .{ "window == window.frames", "true" },
-        .{ "window.frames.length", "0" },
-    }, .{});
-
-    try runner.testCases(&.{
-        .{ "var qm = false; window.queueMicrotask(() => {qm = true });", null },
-        .{ "qm", "true" },
-    }, .{});
-
-    {
-        try runner.testCases(&.{
-            .{
-                \\ let dcl = false;
-                \\ window.addEventListener('DOMContentLoaded', (e) => {
-                \\   dcl = e.target == document;
-                \\ });
-                ,
-                null,
-            },
-        }, .{});
-        try runner.dispatchDOMContentLoaded();
-        try runner.testCases(&.{
-            .{ "dcl", "true" },
-        }, .{});
-    }
-}
-
-test "Browser.HTML.Window.frames" {
-    var runner = try testing.jsRunner(testing.tracking_allocator, .{ .html = 
-        \\<body>
-        \\ <iframe
-        \\   src="https://httpbin.io/html"
-        \\   title="iframea">
-        \\ </iframe>
-        \\ <iframe
-        \\   src="https://httpbin.io/html"
-        \\   title="iframeb">
-        \\ </iframe>
-        \\</body>
-    });
-
-    defer runner.deinit();
-
-    try runner.testCases(&.{
-        .{ "frames.length", "2" },
-        .{ "try { frames[1] } catch (e) { e }", "Error: TODO" }, // TODO fixme
-        .{ "frames[3]", "undefined" },
-    }, .{});
+test "Browser: Window" {
+    try testing.newRunner("window/window.html");
+    try testing.newRunner("window/frames.html");
 }

src/browser/tests/testing.js

@@ -0,0 +1,144 @@
+// Note: this code tries to make sure that we don't fail to execute a <script>
+// block without reporting an error. In other words, if the test passes, you
+// should be confident that the code actually ran.
+// We do a couple things to ensure this.
+// 1 - We make sure that ever script with an id has at least 1 assertion called
+// 2 - We add an onerror handler to every script and, on error, fail.
+//
+// This is pretty straightforward, with the only complexity coming from "eventually"
+// assertions, which are assertions we lazily check in `getStatus()`. We
+// do this because, by the time `getStatus()`, `page.wait()` will have been called
+// and any timer (setTimeout, requestAnimation, MutationObserver, etc...) will
+// have been evaluated. Test which use/test these behavior will use `eventually`.
+(() => {
+  function expectEqual(expected, actual) {
+    _recordExecution();
+    if (expected !== actual) {
+      testing._status = 'fail';
+      let msg = `expected: ${JSON.stringify(expected)}, got: ${JSON.stringify(actual)}`;
+
+      console.warn(
+        `id: ${testing._captured?.script_id || document.currentScript.id}`,
+        `msg: ${msg}`,
+        `stack: ${testing._captured?.stack || new Error().stack}`,
+      );
+    }
+  }
+
+  function expectError(expected, fn) {
+    withError((err) => {
+      expectEqual(expected, err.toString());
+    }, fn);
+  }
+
+  function withError(cb, fn) {
+    try{
+      fn();
+    } catch (err) {
+      cb(err);
+      return;
+    }
+    expectEqual('an error', null);
+  }
+
+  function getStatus() {
+    // if we're already in a fail state, return fail, nothing can recover this
+    if (testing._status === 'fail') return 'fail';
+
+    // run any eventually's that we've captured
+    for (const ev of testing._eventually) {
+      testing._captured = ev[1];
+      ev[0]();
+      testing._captured = null;
+    }
+
+    // Again, if we're in a fail state, we can immediately fail
+    if (testing._status === 'fail') return 'fail';
+
+    // make sure that any <script id=xyz></script> tags we have have had at least
+    // 1 assertion. This helps ensure that if a script tag fails to execute,
+    // we'll report an error, even if no assertions failed.
+    const scripts = document.getElementsByTagName('script');
+    for (script of scripts) {
+      const id = script.id;
+      if (!id) {
+        continue;
+      }
+      if (!testing._executed_scripts[id]) {
+        console.warn(`Failed to execute any expectations for <script id="${id}">...</script>`),
+        testing._status = 'fail';
+      }
+    }
+
+    return testing._status;
+  }
+
+  // Set expectations to happen at some point in the future. Necessary for
+  // testing callbacks which will only be executed after page.wait is called.
+  function eventually(fn) {
+    // capture the current state (script id, stack) so that, when we do run this
+    // we can display more meaningful details on failure.
+    testing._eventually.push([fn, {
+      script_id: document.currentScript.id,
+      stack: new Error().stack,
+    }]);
+
+    _registerErrorCallback();
+  }
+
+  function _recordExecution() {
+    if (testing._status === 'fail') {
+      return;
+    }
+    testing._status = 'ok';
+
+    const script_id = testing._captured?.script_id || document.currentScript.id;
+    testing._executed_scripts[script_id] = true;
+    _registerErrorCallback();
+  }
+
+  function _registerErrorCallback() {
+    const script = document.currentScript;
+    if (!script) {
+      // can be null if we're executing an eventually assertion, but that's ok
+      // because the errorCallback would have been registered for this script
+      // already
+      return;
+    }
+
+    if (script.onerror) {
+      // already registered
+      return;
+    }
+
+    script.onerror = function(err, b) {
+      testing._status = 'fail';
+      console.warn(
+        `id: ${script.id}`,
+        `msg: There was an error executing the <script id=${script.id}>...</script>.\n      There should be a eval error printed above this.`,
+      );
+    }
+  }
+
+  window.testing = {
+    _status: 'empty',
+    _eventually: [],
+    _executed_scripts: {},
+    _captured: null,
+    getStatus: getStatus,
+    eventually: eventually,
+    expectEqual: expectEqual,
+    expectError: expectError,
+    withError: withError,
+  };
+
+  // Helper, so you can do $(sel) in a test
+  window.$ = function(sel) {
+    return document.querySelector(sel);
+  }
+
+  // Helper, so you can do $$(sel) in a test
+  window.$$ = function(sel) {
+    return document.querySelectorAll(sel);
+  }
+})();

src/browser/tests/window/frames.html

@@ -0,0 +1,12 @@
+<script src="../testing.js"></script>
+
+<body>
+ <iframe src="https://httpbin.io/html" title="iframea"></iframe>
+ <iframe src="https://httpbin.io/html" title="iframeb"></iframe>
+</body>
+
+<script id=frames>
+  testing.expectEqual(2, frames.length);
+  testing.expectEqual(undefined, frames[3])
+  testing.expectError('Error: TODO', () => { frames[1] });
+</script>

src/browser/tests/window/window.html

@@ -0,0 +1,105 @@
+<script src="../testing.js"></script>
+<body></body>
+<script id=aliases>
+  testing.expectEqual(window, window.self);
+  testing.expectEqual(window, window.parent);
+  testing.expectEqual(window, window.top);
+  testing.expectEqual(window, window.frames);
+  testing.expectEqual(0, window.frames.length);
+</script>
+
+<script id=request_animation>
+  let start = 0;
+  function step(timestamp) {
+    start = timestamp;
+  }
+  requestAnimationFrame(step);
+  testing.eventually(() => testing.expectEqual(true, start > 0));
+
+  let request_id = requestAnimationFrame(() => {
+    start = 0;
+  });
+  cancelAnimationFrame(request_id);
+  testing.eventually(() => testing.expectEqual(true, start > 0));
+</script>
+
+<script id=dimensions>
+  testing.expectEqual(1, innerHeight);
+  // Width is 1 even if there are no elements
+  testing.expectEqual(1, innerWidth);
+
+  let div1 = document.createElement('div');
+  document.body.appendChild(div1);
+  div1.getClientRects()
+
+  let div2 = document.createElement('div');
+  document.body.appendChild(div2);
+  div2.getClientRects();
+
+  testing.expectEqual(1, innerHeight);
+  testing.expectEqual(2, innerWidth);
+</script>
+
+<script id=setTimeout>
+  let longCall = false;
+  window.setTimeout(() => {longCall = true}, 5001);
+  testing.eventually(() => testing.expectEqual(false, longCall));
+
+  let wst1 = 0;
+  window.setTimeout(() => {wst1 += 1}, 1);
+  testing.eventually(() => testing.expectEqual(1, wst1));
+
+  let wst2 = 1;
+  window.setTimeout((a, b) => {wst2 = a + b}, 1, 2, 3);
+  testing.eventually(() => testing.expectEqual(5, wst2));
+</script>
+
+<script id=eventTarget>
+  let called = false;
+
+  window.addEventListener("ready", (e) => {
+    called = (e.currentTarget == window);
+  }, {capture: false, once: false});
+
+  const evt = new Event("ready", { bubbles: true, cancelable: false });
+  window.dispatchEvent(evt);
+  testing.expectEqual(true, called);
+</script>
+
+<script id=btoa_atob>
+  const b64 = btoa('https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder')
+  testing.expectEqual('aHR0cHM6Ly96aWdsYW5nLm9yZy9kb2N1bWVudGF0aW9uL21hc3Rlci9zdGQvI3N0ZC5iYXNlNjQuQmFzZTY0RGVjb2Rlcg==', b64);
+
+  const str = atob(b64)
+  testing.expectEqual('https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder', str);
+
+  testing.expectError('Error: InvalidCharacterError', () => {
+    atob('b');
+  });
+</script>
+
+<script id=scroll>
+  let scroll = false;
+  let scrolend = false
+  window.addEventListener('scroll', () => {scroll = true});
+  document.addEventListener('scrollend', () => {scrollend = true});
+  window.scrollTo(0);
+
+  testing.expectEqual(true, scroll);
+  testing.expectEqual(true, scrollend);
+</script>
+
+<script id=queueMicroTask>
+  var qm = false;
+  window.queueMicrotask(() => {qm = true });
+  testing.eventually(() => testing.expectEqual(true, qm));
+</script>
+
+<script id=DOMContentLoaded>
+  let dcl = false;
+  window.queueMicrotask(() => {qm = true });
+  window.addEventListener('DOMContentLoaded', (e) => {
+    dcl = e.target == document;
+  });
+  testing.eventually(() => testing.expectEqual(true, dcl));
+</script>

src/log.zig

@@ -348,6 +348,9 @@ fn elapsed() struct { time: f64, unit: []const u8 } {
 
 const testing = @import("testing.zig");
 test "log: data" {
+    opts.format = .logfmt;
+    defer opts.format = .pretty;
+
     var aw = std.Io.Writer.Allocating.init(testing.allocator);
     defer aw.deinit();
 
@@ -384,6 +387,9 @@ test "log: data" {
 }
 
 test "log: string escape" {
+    opts.format = .logfmt;
+    defer opts.format = .pretty;
+
     var aw = std.Io.Writer.Allocating.init(testing.allocator);
     defer aw.deinit();
 

src/main.zig

@@ -725,8 +725,8 @@ var test_cdp_server: ?Server = null;
 var test_http_server: ?TestHTTPServer = null;
 
 test "tests:beforeAll" {
-    log.opts.level = .err;
-    log.opts.format = .logfmt;
+    log.opts.level = .warn;
+    log.opts.format = .pretty;
     try testing.setup();
     var wg: std.Thread.WaitGroup = .{};
     wg.startMany(2);
@@ -796,5 +796,12 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
         });
     }
 
+    if (std.mem.startsWith(u8, path, "/src/browser/tests/")) {
+        // strip off leading / so that it's relative to CWD
+        return TestHTTPServer.sendFile(req, path[1..]);
+    }
+
+    std.debug.print("TestHTTPServer was asked to serve an unknown file: {s}\n", .{path});
+
     unreachable;
 }

src/testing.zig

@@ -35,10 +35,13 @@ pub var arena_instance = std.heap.ArenaAllocator.init(std.heap.c_allocator);
 pub const arena_allocator = arena_instance.allocator();
 
 pub fn reset() void {
-    _ = arena_instance.reset(.{ .retain_capacity = {} });
+    _ = arena_instance.reset(.retain_capacity);
 }
 
 const App = @import("app.zig").App;
+const Env = @import("browser/env.zig").Env;
+const Browser = @import("browser/browser.zig").Browser;
+const Session = @import("browser/session.zig").Session;
 const parser = @import("browser/netsurf.zig");
 
 // Merged std.testing.expectEqual and std.testing.expectString
@@ -362,9 +365,7 @@ fn isJsonValue(a: std.json.Value, b: std.json.Value) bool {
 pub const tracking_allocator = @import("root").tracking_allocator.allocator();
 pub const JsRunner = struct {
     const URL = @import("url.zig").URL;
-    const Env = @import("browser/env.zig").Env;
     const Page = @import("browser/page.zig").Page;
-    const Browser = @import("browser/browser.zig").Browser;
 
     page: *Page,
     browser: *Browser,
@@ -485,13 +486,60 @@ pub fn jsRunner(alloc: Allocator, opts: RunnerOpts) !JsRunner {
 
 var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init;
 pub var test_app: *App = undefined;
+pub var test_browser: Browser = undefined;
+pub var test_session: *Session = undefined;
 
 pub fn setup() !void {
     test_app = try App.init(gpa.allocator(), .{
         .run_mode = .serve,
         .tls_verify_host = false,
     });
+    errdefer test_app.deinit();
+
+    test_browser = try Browser.init(test_app);
+    errdefer test_browser.deinit();
+
+    test_session = try test_browser.newSession();
 }
 pub fn shutdown() void {
+    test_browser.deinit();
     test_app.deinit();
 }
+
+pub fn newRunner(file: []const u8) !void {
+    defer _ = arena_instance.reset(.retain_capacity);
+    const page = try test_session.createPage();
+    defer test_session.removePage();
+
+    const js_context = page.main_context;
+    var try_catch: Env.TryCatch = undefined;
+    try_catch.init(js_context);
+    defer try_catch.deinit();
+
+    const url = try std.fmt.allocPrint(arena_allocator, "http://localhost:9582/src/browser/tests/{s}", .{file});
+    try page.navigate(url, .{});
+    page.wait(2);
+
+    const value = js_context.exec("testing.getStatus()", "testing.getStatus()") catch |err| {
+        const msg = try_catch.err(arena_allocator) catch @errorName(err) orelse "unknown";
+        std.debug.print("{s}: test failure\nError: {s}\n", .{ file, msg });
+        return err;
+    };
+
+    const status = try value.toString(arena_allocator);
+    if (std.mem.eql(u8, status, "ok")) {
+        return;
+    }
+
+    if (std.mem.eql(u8, status, "empty")) {
+        std.debug.print("{s}: No testing assertions were made\n", .{file});
+        return error.NoTestingAssertions;
+    }
+
+    if (std.mem.eql(u8, status, "fail")) {
+        return error.TestFail;
+    }
+
+    std.debug.print("{s}: Invalid test status: '{s}'\n", .{ file, status });
+    return error.TestFail;
+}