From c40704d2f3b51b99bc76d06dc3d45ba753b79c98 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 1 Sep 2025 18:09:12 +0800 Subject: [PATCH] Prototype new test runner Follows up on https://github.com/lightpanda-io/browser/pull/994 and replaces the jsRunner with a new page.navigation-based test runner. Currently only implemented for the Window tests, looking for feedback and converting every existing test will take time - so for a while, newRunner (to be renamed) will sit side-by-side with jsRunner. In addition to the benefits outlined in 994, largely around code simplicity and putting more of the actual code under tests, I think our WebAPI tests particularly benefit from: 1 - No need to recompile when modifying the html tests 2 - Much better assertions, e.g. you can assert that something is actually an array, not just a string representation of an array 3 - Ability to test some edge cases (e.g. dynamic script loading) I've put some effort into testing.js to make sure that, if the encapsulating zig test passes, it's because it actually passed, not because it didn't run. For the time being, console tests are removed. I think it's more useful to have access to the console within tests, than it is to test the console (which is just a wrapper around log, which is both tested and heavily used). --- src/browser/console/console.zig | 215 ++++++++------------------- src/browser/html/window.zig | 149 +------------------ src/browser/tests/testing.js | 144 ++++++++++++++++++ src/browser/tests/window/frames.html | 10 ++ src/browser/tests/window/window.html | 104 +++++++++++++ src/log.zig | 6 + src/main.zig | 11 +- src/testing.zig | 66 +++++++- 8 files changed, 402 insertions(+), 303 deletions(-) create mode 100644 src/browser/tests/testing.js create mode 100644 src/browser/tests/window/frames.html create mode 100644 src/browser/tests/window/window.html diff --git a/src/browser/console/console.zig b/src/browser/console/console.zig index cd4cb443..f9b2ebda 100644 --- a/src/browser/console/console.zig +++ b/src/browser/console/console.zig @@ -18,13 +18,12 @@ const std = @import("std"); const builtin = @import("builtin"); +const log = @import("../../log.zig"); const Allocator = std.mem.Allocator; const Page = @import("../page.zig").Page; const JsObject = @import("../env.zig").Env.JsObject; -const log = if (builtin.is_test) &test_capture else @import("../../log.zig"); - pub const Console = struct { // TODO: configurable writer timers: std.StringHashMapUnmanaged(u32) = .{}, @@ -67,7 +66,7 @@ pub const Console = struct { return; } - log.info(.console, "error", .{ + log.warn(.console, "error", .{ .args = try serializeValues(values, page), .stack = page.stackTrace() catch "???", }); @@ -168,161 +167,73 @@ fn timestamp() u32 { return @import("../../datetime.zig").timestamp(); } -var test_capture = TestCapture{}; -const testing = @import("../../testing.zig"); -test "Browser.Console" { - defer testing.reset(); +// const testing = @import("../../testing.zig"); +// test "Browser.Console" { +// defer testing.reset(); - var runner = try testing.jsRunner(testing.tracking_allocator, .{}); - defer runner.deinit(); +// var runner = try testing.jsRunner(testing.tracking_allocator, .{}); +// defer runner.deinit(); - { - try runner.testCases(&.{ - .{ "console.log('a')", "undefined" }, - .{ "console.warn('hello world', 23, true, new Object())", "undefined" }, - }, .{}); +// { +// try runner.testCases(&.{ +// .{ "console.log('a')", "undefined" }, +// .{ "console.warn('hello world', 23, true, new Object())", "undefined" }, +// }, .{}); - const captured = test_capture.captured.items; - try testing.expectEqual("[info] args= 1: a", captured[0]); - try testing.expectEqual("[warn] args= 1: hello world 2: 23 3: true 4: #", captured[1]); - } +// const captured = test_capture.captured.items; +// try testing.expectEqual("[info] args= 1: a", captured[0]); +// try testing.expectEqual("[warn] args= 1: hello world 2: 23 3: true 4: #", captured[1]); +// } - { - test_capture.reset(); - try runner.testCases(&.{ - .{ "console.countReset()", "undefined" }, - .{ "console.count()", "undefined" }, - .{ "console.count('teg')", "undefined" }, - .{ "console.count('teg')", "undefined" }, - .{ "console.count('teg')", "undefined" }, - .{ "console.count()", "undefined" }, - .{ "console.countReset('teg')", "undefined" }, - .{ "console.countReset()", "undefined" }, - .{ "console.count()", "undefined" }, - }, .{}); +// { +// test_capture.reset(); +// try runner.testCases(&.{ +// .{ "console.countReset()", "undefined" }, +// .{ "console.count()", "undefined" }, +// .{ "console.count('teg')", "undefined" }, +// .{ "console.count('teg')", "undefined" }, +// .{ "console.count('teg')", "undefined" }, +// .{ "console.count()", "undefined" }, +// .{ "console.countReset('teg')", "undefined" }, +// .{ "console.countReset()", "undefined" }, +// .{ "console.count()", "undefined" }, +// }, .{}); - const captured = test_capture.captured.items; - try testing.expectEqual("[invalid counter] label=default", captured[0]); - try testing.expectEqual("[count] label=default count=1", captured[1]); - try testing.expectEqual("[count] label=teg count=1", captured[2]); - try testing.expectEqual("[count] label=teg count=2", captured[3]); - try testing.expectEqual("[count] label=teg count=3", captured[4]); - try testing.expectEqual("[count] label=default count=2", captured[5]); - try testing.expectEqual("[count reset] label=teg count=3", captured[6]); - try testing.expectEqual("[count reset] label=default count=2", captured[7]); - try testing.expectEqual("[count] label=default count=1", captured[8]); - } +// const captured = test_capture.captured.items; +// try testing.expectEqual("[invalid counter] label=default", captured[0]); +// try testing.expectEqual("[count] label=default count=1", captured[1]); +// try testing.expectEqual("[count] label=teg count=1", captured[2]); +// try testing.expectEqual("[count] label=teg count=2", captured[3]); +// try testing.expectEqual("[count] label=teg count=3", captured[4]); +// try testing.expectEqual("[count] label=default count=2", captured[5]); +// try testing.expectEqual("[count reset] label=teg count=3", captured[6]); +// try testing.expectEqual("[count reset] label=default count=2", captured[7]); +// try testing.expectEqual("[count] label=default count=1", captured[8]); +// } - { - test_capture.reset(); - try runner.testCases(&.{ - .{ "console.assert(true)", "undefined" }, - .{ "console.assert('a', 2, 3, 4)", "undefined" }, - .{ "console.assert('')", "undefined" }, - .{ "console.assert('', 'x', true)", "undefined" }, - .{ "console.assert(false, 'x')", "undefined" }, - }, .{}); +// { +// test_capture.reset(); +// try runner.testCases(&.{ +// .{ "console.assert(true)", "undefined" }, +// .{ "console.assert('a', 2, 3, 4)", "undefined" }, +// .{ "console.assert('')", "undefined" }, +// .{ "console.assert('', 'x', true)", "undefined" }, +// .{ "console.assert(false, 'x')", "undefined" }, +// }, .{}); - const captured = test_capture.captured.items; - try testing.expectEqual("[assertion failed] values=", captured[0]); - try testing.expectEqual("[assertion failed] values= 1: x 2: true", captured[1]); - try testing.expectEqual("[assertion failed] values= 1: x", captured[2]); - } +// const captured = test_capture.captured.items; +// try testing.expectEqual("[assertion failed] values=", captured[0]); +// try testing.expectEqual("[assertion failed] values= 1: x 2: true", captured[1]); +// try testing.expectEqual("[assertion failed] values= 1: x", captured[2]); +// } - { - test_capture.reset(); - try runner.testCases(&.{ - .{ "[1].forEach(console.log)", null }, - }, .{}); +// { +// test_capture.reset(); +// try runner.testCases(&.{ +// .{ "[1].forEach(console.log)", null }, +// }, .{}); - const captured = test_capture.captured.items; - try testing.expectEqual("[info] args= 1: 1 2: 0 3: [1]", captured[0]); - } -} - -const TestCapture = struct { - captured: std.ArrayListUnmanaged([]const u8) = .{}, - - fn separator(_: *const TestCapture) []const u8 { - return " "; - } - - fn reset(self: *TestCapture) void { - self.captured = .{}; - } - - fn debug( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) void { - self.capture(scope, msg, args); - } - - fn info( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) void { - self.capture(scope, msg, args); - } - - fn warn( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) void { - self.capture(scope, msg, args); - } - - fn err( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) void { - self.capture(scope, msg, args); - } - - fn fatal( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) void { - self.capture(scope, msg, args); - } - - fn capture( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) void { - self._capture(scope, msg, args) catch unreachable; - } - - fn _capture( - self: *TestCapture, - comptime scope: @Type(.enum_literal), - comptime msg: []const u8, - args: anytype, - ) !void { - std.debug.assert(scope == .console); - - const allocator = testing.arena_allocator; - var buf: std.ArrayListUnmanaged(u8) = .empty; - try buf.appendSlice(allocator, "[" ++ msg ++ "] "); - - inline for (@typeInfo(@TypeOf(args)).@"struct".fields) |f| { - try buf.appendSlice(allocator, f.name); - try buf.append(allocator, '='); - try @import("../../log.zig").writeValue(.pretty, @field(args, f.name), buf.writer(allocator)); - try buf.append(allocator, ' '); - } - self.captured.append(testing.arena_allocator, std.mem.trimRight(u8, buf.items, " ")) catch unreachable; - } -}; +// const captured = test_capture.captured.items; +// try testing.expectEqual("[info] args= 1: 1 2: 0 3: [1]", captured[0]); +// } +// } diff --git a/src/browser/html/window.zig b/src/browser/html/window.zig index 203d3b8c..e9528eb6 100644 --- a/src/browser/html/window.zig +++ b/src/browser/html/window.zig @@ -436,150 +436,7 @@ const TimerCallback = struct { }; const testing = @import("../../testing.zig"); -test "Browser.HTML.Window" { - var runner = try testing.jsRunner(testing.tracking_allocator, .{}); - defer runner.deinit(); - - // try runner.testCases(&.{ - // .{ "window.parent === window", "true" }, - // .{ "window.top === window", "true" }, - // }, .{}); - - try runner.testCases(&.{ - .{ - \\ let start = 0; - \\ function step(timestamp) { - \\ start = timestamp; - \\ } - , - null, - }, - .{ "requestAnimationFrame(step);", null }, // returned id is checked in the next test - .{ " start > 0", "true" }, - }, .{}); - - // cancelAnimationFrame should be able to cancel a request with the given id - try runner.testCases(&.{ - .{ "let request_id = requestAnimationFrame(timestamp => {});", null }, - .{ "cancelAnimationFrame(request_id);", "undefined" }, - }, .{}); - - try runner.testCases(&.{ - .{ "innerHeight", "1" }, - .{ "innerWidth", "1" }, // Width is 1 even if there are no elements - .{ - \\ let div1 = document.createElement('div'); - \\ document.body.appendChild(div1); - \\ div1.getClientRects(); - , - null, - }, - .{ - \\ let div2 = document.createElement('div'); - \\ document.body.appendChild(div2); - \\ div2.getClientRects(); - , - null, - }, - .{ "innerHeight", "1" }, - .{ "innerWidth", "2" }, - }, .{}); - - try runner.testCases(&.{ - .{ "let longCall = false;", null }, - .{ "window.setTimeout(() => {longCall = true}, 5001);", null }, - .{ "longCall;", "false" }, - - .{ "let wst = 0;", null }, - .{ "window.setTimeout(() => {wst += 1}, 1)", null }, - .{ "wst", "1" }, - - .{ "window.setTimeout((a, b) => {wst = a + b}, 1, 2, 3)", null }, - .{ "wst", "5" }, - }, .{}); - - // window event target - try runner.testCases(&.{ - .{ - \\ let called = false; - \\ window.addEventListener("ready", (e) => { - \\ called = (e.currentTarget == window); - \\ }, {capture: false, once: false}); - \\ const evt = new Event("ready", { bubbles: true, cancelable: false }); - \\ window.dispatchEvent(evt); - \\ called; - , - "true", - }, - }, .{}); - - try runner.testCases(&.{ - .{ "const b64 = btoa('https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder')", "undefined" }, - .{ "b64", "aHR0cHM6Ly96aWdsYW5nLm9yZy9kb2N1bWVudGF0aW9uL21hc3Rlci9zdGQvI3N0ZC5iYXNlNjQuQmFzZTY0RGVjb2Rlcg==" }, - .{ "const str = atob(b64)", "undefined" }, - .{ "str", "https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder" }, - .{ "try { atob('b') } catch (e) { e } ", "Error: InvalidCharacterError" }, - }, .{}); - - try runner.testCases(&.{ - .{ "let scroll = false; let scrolend = false", null }, - .{ "window.addEventListener('scroll', () => {scroll = true});", null }, - .{ "document.addEventListener('scrollend', () => {scrollend = true});", null }, - .{ "window.scrollTo(0)", null }, - .{ "scroll", "true" }, - .{ "scrollend", "true" }, - }, .{}); - - try runner.testCases(&.{ - .{ "window == window.self", "true" }, - .{ "window == window.parent", "true" }, - .{ "window == window.top", "true" }, - .{ "window == window.frames", "true" }, - .{ "window.frames.length", "0" }, - }, .{}); - - try runner.testCases(&.{ - .{ "var qm = false; window.queueMicrotask(() => {qm = true });", null }, - .{ "qm", "true" }, - }, .{}); - - { - try runner.testCases(&.{ - .{ - \\ let dcl = false; - \\ window.addEventListener('DOMContentLoaded', (e) => { - \\ dcl = e.target == document; - \\ }); - , - null, - }, - }, .{}); - try runner.dispatchDOMContentLoaded(); - try runner.testCases(&.{ - .{ "dcl", "true" }, - }, .{}); - } -} - -test "Browser.HTML.Window.frames" { - var runner = try testing.jsRunner(testing.tracking_allocator, .{ .html = - \\ - \\ - \\ - \\ - }); - - defer runner.deinit(); - - try runner.testCases(&.{ - .{ "frames.length", "2" }, - .{ "try { frames[1] } catch (e) { e }", "Error: TODO" }, // TODO fixme - .{ "frames[3]", "undefined" }, - }, .{}); +test "Browser: Window" { + try testing.newRunner("window/window.html"); + try testing.newRunner("window/frames.html"); } diff --git a/src/browser/tests/testing.js b/src/browser/tests/testing.js new file mode 100644 index 00000000..2befecbf --- /dev/null +++ b/src/browser/tests/testing.js @@ -0,0 +1,144 @@ +// Note: this code tries to make sure that we don't fail to execute a tags we have have had at least + // 1 assertion. This helps ensure that if a script tag fails to execute, + // we'll report an error, even if no assertions failed. + const scripts = document.getElementsByTagName('script'); + for (script of scripts) { + const id = script.id; + if (!id) { + continue; + } + if (!testing._executed_scripts[id]) { + console.warn(`Failed to execute any expectations for `), + testing._status = 'fail'; + } + } + + return testing._status; + } + + // Set expectations to happen at some point in the future. Necessary for + // testing callbacks which will only be executed after page.wait is called. + function eventually(fn) { + // capture the current state (script id, stack) so that, when we do run this + // we can display more meaningful details on failure. + testing._eventually.push([fn, { + script_id: document.currentScript.id, + stack: new Error().stack, + }]); + + _registerErrorCallback(); + } + + function _recordExecution() { + if (testing._status === 'fail') { + return; + } + testing._status = 'ok'; + + const script_id = testing._captured?.script_id || document.currentScript.id; + testing._executed_scripts[script_id] = true; + _registerErrorCallback(); + } + + function _registerErrorCallback() { + const script = document.currentScript; + if (!script) { + // can be null if we're executing an eventually assertion, but that's ok + // because the errorCallback would have been registered for this script + // already + return; + } + + if (script.onerror) { + // already registered + return; + } + + script.onerror = function(err, b) { + testing._status = 'fail'; + console.warn( + `id: ${script.id}`, + `msg: There was an error executing the .\n There should be a eval error printed above this.`, + ); + } + } + + window.testing = { + _status: 'empty', + _eventually: [], + _executed_scripts: {}, + _captured: null, + getStatus: getStatus, + eventually: eventually, + expectEqual: expectEqual, + expectError: expectError, + withError: withError, + }; + + // Helper, so you can do $(sel) in a test + window.$ = function(sel) { + return document.querySelector(sel); + } + + // Helper, so you can do $$(sel) in a test + window.$$ = function(sel) { + return document.querySelectorAll(sel); + } +})(); diff --git a/src/browser/tests/window/frames.html b/src/browser/tests/window/frames.html new file mode 100644 index 00000000..a48df7a5 --- /dev/null +++ b/src/browser/tests/window/frames.html @@ -0,0 +1,10 @@ + + + + + + diff --git a/src/browser/tests/window/window.html b/src/browser/tests/window/window.html new file mode 100644 index 00000000..20aae1b5 --- /dev/null +++ b/src/browser/tests/window/window.html @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/log.zig b/src/log.zig index abeaeadd..24ae9ff8 100644 --- a/src/log.zig +++ b/src/log.zig @@ -348,6 +348,9 @@ fn elapsed() struct { time: f64, unit: []const u8 } { const testing = @import("testing.zig"); test "log: data" { + opts.format = .logfmt; + defer opts.format = .pretty; + var aw = std.Io.Writer.Allocating.init(testing.allocator); defer aw.deinit(); @@ -384,6 +387,9 @@ test "log: data" { } test "log: string escape" { + opts.format = .logfmt; + defer opts.format = .pretty; + var aw = std.Io.Writer.Allocating.init(testing.allocator); defer aw.deinit(); diff --git a/src/main.zig b/src/main.zig index 35bcf7df..ab88dede 100644 --- a/src/main.zig +++ b/src/main.zig @@ -725,8 +725,8 @@ var test_cdp_server: ?Server = null; var test_http_server: ?TestHTTPServer = null; test "tests:beforeAll" { - log.opts.level = .err; - log.opts.format = .logfmt; + log.opts.level = .warn; + log.opts.format = .pretty; try testing.setup(); var wg: std.Thread.WaitGroup = .{}; wg.startMany(2); @@ -796,5 +796,12 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void { }); } + if (std.mem.startsWith(u8, path, "/src/browser/tests/")) { + // strip off leading / so that it's relative to CWD + return TestHTTPServer.sendFile(req, path[1..]); + } + + std.debug.print("TestHTTPServer was asked to serve an unknown file: {s}\n", .{path}); + unreachable; } diff --git a/src/testing.zig b/src/testing.zig index 81269001..0264474e 100644 --- a/src/testing.zig +++ b/src/testing.zig @@ -35,10 +35,13 @@ pub var arena_instance = std.heap.ArenaAllocator.init(std.heap.c_allocator); pub const arena_allocator = arena_instance.allocator(); pub fn reset() void { - _ = arena_instance.reset(.{ .retain_capacity = {} }); + _ = arena_instance.reset(.retain_capacity); } const App = @import("app.zig").App; +const Env = @import("browser/env.zig").Env; +const Browser = @import("browser/browser.zig").Browser; +const Session = @import("browser/session.zig").Session; const parser = @import("browser/netsurf.zig"); // Merged std.testing.expectEqual and std.testing.expectString @@ -362,9 +365,7 @@ fn isJsonValue(a: std.json.Value, b: std.json.Value) bool { pub const tracking_allocator = @import("root").tracking_allocator.allocator(); pub const JsRunner = struct { const URL = @import("url.zig").URL; - const Env = @import("browser/env.zig").Env; const Page = @import("browser/page.zig").Page; - const Browser = @import("browser/browser.zig").Browser; page: *Page, browser: *Browser, @@ -485,13 +486,72 @@ pub fn jsRunner(alloc: Allocator, opts: RunnerOpts) !JsRunner { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; pub var test_app: *App = undefined; +pub var test_browser: Browser = undefined; +pub var test_session: *Session = undefined; pub fn setup() !void { test_app = try App.init(gpa.allocator(), .{ .run_mode = .serve, .tls_verify_host = false, }); + errdefer test_app.deinit(); + + test_browser = try Browser.init(test_app); + errdefer test_browser.deinit(); + + test_session = try test_browser.newSession(); } pub fn shutdown() void { + test_browser.deinit(); test_app.deinit(); } + +pub fn newRunner(file: []const u8) !void { + defer _ = arena_instance.reset(.retain_capacity); + const page = try test_session.createPage(); + defer test_session.removePage(); + + const js_context = page.main_context; + var try_catch: Env.TryCatch = undefined; + try_catch.init(js_context); + defer try_catch.deinit(); + + // if you want to make a lot of changes to testing.js, but don't want to reload + // every time, use this to dynamically load it during development + const content = try std.fs.cwd().readFileAlloc(arena_allocator, "src/browser/tests/testing.js", 1024 * 32); + + // const content = @embedFile("browser/tests/testing.js"); + + js_context.eval(content, "testing.js") catch |err| { + const msg = try_catch.err(arena_allocator) catch @errorName(err) orelse "unknown"; + std.debug.print("Failed to setup testing.js: {s}\n", .{msg}); + return err; + }; + + const url = try std.fmt.allocPrint(arena_allocator, "http://localhost:9582/src/browser/tests/{s}", .{file}); + try page.navigate(url, .{}); + page.wait(2); + + const value = js_context.exec("testing.getStatus()", "testing.getStatus()") catch |err| { + const msg = try_catch.err(arena_allocator) catch @errorName(err) orelse "unknown"; + std.debug.print("{s}: test failure\nError: {s}\n", .{ file, msg }); + return err; + }; + + const status = try value.toString(arena_allocator); + if (std.mem.eql(u8, status, "ok")) { + return; + } + + if (std.mem.eql(u8, status, "empty")) { + std.debug.print("{s}: No testing assertions were made\n", .{file}); + return error.NoTestingAssertions; + } + + if (std.mem.eql(u8, status, "fail")) { + return error.TestFail; + } + + std.debug.print("{s}: Invalid test status: '{s}'\n", .{ file, status }); + return error.TestFail; +}