public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2025-10-28 22:53:28 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Start unifying test and code

Browse Source 
Depends on https://github.com/lightpanda-io/browser/pull/993

There's currently 3 ways to execute a page:
1 - page.navigate (as used in both the 'fetch' and 'serve' commands)
2 - jsRunner as used in unit tests
3 - main_wpt as used in the WPT runner

Both jsRunner and main_wpt replicate the page.navigate code, but in their own
hack-ish way. main_wpt re-implements the DOM walking in order to extract and
execute <script> tags, as well as the needed page lifecycle events.

This PR replaces the existing main_wpt loader with a call to page.navigate. To
support this, a test HTTP server was added. (The test HTTP server is extracted
from the existing unit test test server, and re-used between the two).

There are benefits to this approach:
1 - The code is simpler
2 - More of the actual code and flow is tested
3 - There's 1 way to do things (page.navigate)
4 - Having an HTTP server might unlock some WPT tests

Technically, we're replacing file IO with network IO i.e. http requests). This
has potential downsides:
1 - The tests might be more brittle
2 - The tests might be slower

I think we need to run it for a while to see if we get flaky behavior.

The goal for following PRs is to bring this unification to the jsRunner.
This commit is contained in:
Karl Seguin
2025-08-31 19:25:17 +08:00
parent 6c41245c73
commit 7d46e8fe80
6 changed files with 265 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
src/TestHTTPServer.zig Normal file
View File

@@ -0,0 +1,117 @@
const std = @import("std");
const TestHTTPServer = @This();
shutdown: bool,
listener: ?std.net.Server,
handler: Handler,
const Handler = *const fn (req: *std.http.Server.Request) anyerror!void;
pub fn init(handler: Handler) TestHTTPServer {
return .{
.shutdown = true,
.listener = null,
.handler = handler,
};
}
pub fn deinit(self: *TestHTTPServer) void {
self.shutdown = true;
if (self.listener) |*listener| {
listener.deinit();
}
}
pub fn run(self: *TestHTTPServer, wg: *std.Thread.WaitGroup) !void {
const address = try std.net.Address.parseIp("127.0.0.1", 9582);
self.listener = try address.listen(.{ .reuse_address = true });
var listener = &self.listener.?;
wg.finish();
while (true) {
const conn = listener.accept() catch |err| {
if (self.shutdown) {
return;
}
return err;
};
const thrd = try std.Thread.spawn(.{}, handleConnection, .{ self, conn });
thrd.detach();
}
}
fn handleConnection(self: *TestHTTPServer, conn: std.net.Server.Connection) !void {
defer conn.stream.close();
var req_buf: [2048]u8 = undefined;
var conn_reader = conn.stream.reader(&req_buf);
var conn_writer = conn.stream.writer(&req_buf);
var http_server = std.http.Server.init(conn_reader.interface(), &conn_writer.interface);
while (true) {
var req = http_server.receiveHead() catch |err| switch (err) {
error.ReadFailed => continue,
error.HttpConnectionClosing => continue,
else => {
std.debug.print("Test HTTP Server error: {}\n", .{err});
return err;
},
};
self.handler(&req) catch |err| {
std.debug.print("test http error '{s}': {}\n", .{ req.head.target, err });
try req.respond("server error", .{ .status = .internal_server_error });
return;
};
}
}
pub fn sendFile(req: *std.http.Server.Request, file_path: []const u8) !void {
var file = std.fs.cwd().openFile(file_path, .{}) catch |err| switch (err) {
error.FileNotFound => return req.respond("server error", .{ .status = .not_found }),
else => return err,
};
const stat = try file.stat();
var send_buffer: [4096]u8 = undefined;
var res = try req.respondStreaming(&send_buffer, .{
.content_length = stat.size,
.respond_options = .{
.extra_headers = &.{
.{ .name = "content-type", .value = getContentType(file_path) },
},
},
});
var read_buffer: [4096]u8 = undefined;
var reader = file.reader(&read_buffer);
_ = try res.writer.sendFileAll(&reader, .unlimited);
try res.writer.flush();
try res.end();
}
fn getContentType(file_path: []const u8) []const u8 {
if (std.mem.endsWith(u8, file_path, ".js")) {
return "application/json";
}
if (std.mem.endsWith(u8, file_path, ".html")) {
return "text/html";
}
if (std.mem.endsWith(u8, file_path, ".htm")) {
return "text/html";
}
if (std.mem.endsWith(u8, file_path, ".xml")) {
// some wpt tests do this
return "text/xml";
}
std.debug.print("TestHTTPServer asked to serve an unknown file type: {s}\n", .{file_path});
return "text/html";
}

24
src/browser/ScriptManager.zig
View File

@@ -403,14 +403,7 @@ fn startCallback(transfer: *Http.Transfer) !void {
fn headerCallback(transfer: *Http.Transfer) !void {
const script: *PendingScript = @ptrCast(@alignCast(transfer.ctx));
script.headerCallback(transfer) catch |err| {
log.err(.http, "SM.headerCallback", .{
.err = err,
.transfer = transfer,
.status = transfer.response_header.?.status,
});
return err;
};
script.headerCallback(transfer);
}
fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
@@ -463,18 +456,23 @@ const PendingScript = struct {
log.debug(.http, "script fetch start", .{ .req = transfer });
}
fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) void {
const header = &transfer.response_header.?;
if (header.status != 200) {
log.info(.http, "script header", .{
.req = transfer,
.status = header.status,
.content_type = header.contentType(),
});
return;
}
log.debug(.http, "script header", .{
.req = transfer,
.status = header.status,
.content_type = header.contentType(),
});
if (header.status != 200) {
return error.InvalidStatusCode;
}
// If this isn't true, then we'll likely leak memory. If you don't
// set `CURLOPT_SUPPRESS_CONNECT_HEADERS` and CONNECT to a proxy, this
// will fail. This assertion exists to catch incorrect assumptions about

3
src/browser/xhr/xhr.zig
View File

@@ -785,8 +785,7 @@ test "Browser.XHR.XMLHttpRequest" {
.{ "req.statusText", "OK" },
.{ "req.getResponseHeader('Content-Type')", "text/html; charset=utf-8" },
.{ "req.getAllResponseHeaders()", "content-length: 100\r\n" ++
"Content-Type: text/html; charset=utf-8\r\n" ++
"Connection: Close\r\n" },
"Content-Type: text/html; charset=utf-8\r\n" },
.{ "req.responseText.length", "100" },
.{ "req.response.length == req.responseText.length", "true" },
.{ "req.responseXML instanceof Document", "true" },

94
src/main.zig
View File

@@ -719,23 +719,26 @@ test {
std.testing.refAllDecls(@This());
}
const TestHTTPServer = @import("TestHTTPServer.zig");
var test_cdp_server: ?Server = null;
var test_http_server: ?TestHTTPServer = null;
test "tests:beforeAll" {
log.opts.level = .err;
log.opts.format = .logfmt;
try testing.setup();
var wg: std.Thread.WaitGroup = .{};
wg.startMany(2);
{
const thread = try std.Thread.spawn(.{}, serveHTTP, .{&wg});
const thread = try std.Thread.spawn(.{}, serveCDP, .{&wg});
thread.detach();
}
test_http_server = TestHTTPServer.init(testHTTPHandler);
{
const thread = try std.Thread.spawn(.{}, serveCDP, .{&wg});
const thread = try std.Thread.spawn(.{}, TestHTTPServer.run, .{ &test_http_server.?, &wg });
thread.detach();
}
@@ -748,59 +751,10 @@ test "tests:afterAll" {
if (test_cdp_server) |*server| {
server.deinit();
}
testing.shutdown();
}
fn serveHTTP(wg: *std.Thread.WaitGroup) !void {
const address = try std.net.Address.parseIp("127.0.0.1", 9582);
var listener = try address.listen(.{ .reuse_address = true });
defer listener.deinit();
wg.finish();
var buf: [1024]u8 = undefined;
while (true) {
var conn = try listener.accept();
defer conn.stream.close();
var conn_reader = conn.stream.reader(&buf);
var conn_writer = conn.stream.writer(&buf);
var http_server = std.http.Server.init(conn_reader.interface(), &conn_writer.interface);
var request = http_server.receiveHead() catch |err| switch (err) {
error.HttpConnectionClosing => continue,
else => {
std.debug.print("Test HTTP Server error: {}\n", .{err});
return err;
},
};
const path = request.head.target;
if (std.mem.eql(u8, path, "/loader")) {
try request.respond("Hello!", .{
.extra_headers = &.{.{ .name = "Connection", .value = "close" }},
});
} else if (std.mem.eql(u8, path, "/xhr")) {
try request.respond("1234567890" ** 10, .{
.extra_headers = &.{
.{ .name = "Content-Type", .value = "text/html; charset=utf-8" },
.{ .name = "Connection", .value = "Close" },
},
});
} else if (std.mem.eql(u8, path, "/xhr/json")) {
try request.respond("{\"over\":\"9000!!!\"}", .{
.extra_headers = &.{
.{ .name = "Content-Type", .value = "application/json" },
.{ .name = "Connection", .value = "Close" },
},
});
} else {
// should not have an unknown path
unreachable;
}
if (test_http_server) |*server| {
server.deinit();
}
testing.shutdown();
}
fn serveCDP(wg: *std.Thread.WaitGroup) !void {
@@ -816,3 +770,31 @@ fn serveCDP(wg: *std.Thread.WaitGroup) !void {
return err;
};
}
fn testHTTPHandler(req: *std.http.Server.Request) !void {
const path = req.head.target;
if (std.mem.eql(u8, path, "/loader")) {
return req.respond("Hello!", .{
.extra_headers = &.{.{ .name = "Connection", .value = "close" }},
});
}
if (std.mem.eql(u8, path, "/xhr")) {
return req.respond("1234567890" ** 10, .{
.extra_headers = &.{
.{ .name = "Content-Type", .value = "text/html; charset=utf-8" },
},
});
}
if (std.mem.eql(u8, path, "/xhr/json")) {
return req.respond("{\"over\":\"9000!!!\"}", .{
.extra_headers = &.{
.{ .name = "Content-Type", .value = "application/json" },
},
});
}
unreachable;
}

234
src/main_wpt.zig
View File

@@ -22,24 +22,34 @@ const log = @import("log.zig");
const Allocator = std.mem.Allocator;
const ArenaAllocator = std.heap.ArenaAllocator;
const App = @import("app.zig").App;
const Env = @import("browser/env.zig").Env;
const Platform = @import("runtime/js.zig").Platform;
const Browser = @import("browser/browser.zig").Browser;
const Session = @import("browser/session.zig").Session;
const TestHTTPServer = @import("TestHTTPServer.zig");
const parser = @import("browser/netsurf.zig");
const polyfill = @import("browser/polyfill/polyfill.zig");
const WPT_DIR = "tests/wpt";
// TODO For now the WPT tests run is specific to WPT.
// It manually load js framwork libs, and run the first script w/ js content in
// the HTML page.
// Once lightpanda will have the html loader, it would be useful to refactor
// this test to use it.
pub fn main() !void {
var gpa: std.heap.DebugAllocator(.{}) = .init;
defer _ = gpa.deinit();
const allocator = gpa.allocator();
log.opts.level = .warn;
log.opts.level = .err;
var http_server = TestHTTPServer.init(httpHandler);
defer http_server.deinit();
{
var wg: std.Thread.WaitGroup = .{};
wg.startMany(1);
var thrd = try std.Thread.spawn(.{}, TestHTTPServer.run, .{ &http_server, &wg });
thrd.detach();
wg.wait();
}
// An arena for the runner itself, lives for the duration of the the process
var ra = ArenaAllocator.init(allocator);
@@ -48,29 +58,33 @@ pub fn main() !void {
const cmd = try parseArgs(runner_arena);
try @import("testing.zig").setup();
defer @import("testing.zig").shutdown();
// prepare libraries to load on each test case.
var loader = FileLoader.init(runner_arena, WPT_DIR);
var it = try TestIterator.init(runner_arena, WPT_DIR, cmd.filters);
var it = try TestIterator.init(allocator, WPT_DIR, cmd.filters);
defer it.deinit();
var writer = try Writer.init(runner_arena, cmd.format);
var writer = try Writer.init(allocator, cmd.format);
defer writer.deinit();
// An arena for running each tests. Is reset after every test.
var test_arena = ArenaAllocator.init(allocator);
defer test_arena.deinit();
var app = try App.init(allocator, .{
.run_mode = .fetch,
});
defer app.deinit();
var browser = try Browser.init(app);
defer browser.deinit();
const session = try browser.newSession();
while (try it.next()) |test_file| {
defer _ = test_arena.reset(.{ .retain_capacity = {} });
defer _ = test_arena.reset(.retain_capacity);
var err_out: ?[]const u8 = null;
const result = run(
test_arena.allocator(),
session,
test_file,
&loader,
&err_out,
) catch |err| blk: {
if (err_out == null) {
@@ -78,13 +92,6 @@ pub fn main() !void {
}
break :blk null;
};
if (result == null and err_out == null) {
// We sometimes pass a non-test to `run` (we don't know it's a non
// test, we need to open the contents of the test file to find out
// and that's in run).
continue;
}
try writer.process(test_file, result, err_out);
}
try writer.finalize();
@@ -92,102 +99,41 @@ pub fn main() !void {
fn run(
arena: Allocator,
session: *Session,
test_file: []const u8,
loader: *FileLoader,
err_out: *?[]const u8,
) !?[]const u8 {
// document
const html = blk: {
const full_path = try std.fs.path.join(arena, &.{ WPT_DIR, test_file });
const file = try std.fs.cwd().openFile(full_path, .{});
defer file.close();
break :blk try file.readToEndAlloc(arena, 128 * 1024);
};
) ![]const u8 {
const page = try session.createPage();
defer session.removePage();
if (std.mem.indexOf(u8, html, "testharness.js") == null) {
// This isn't a test. A lot of files are helpers/content for tests to
// make use of.
return null;
}
const url = try std.fmt.allocPrint(arena, "http://localhost:9582/{s}", .{test_file});
try page.navigate(url, .{});
// this returns null for the success.html test in the root of tests/wpt
const dirname = std.fs.path.dirname(test_file) orelse "";
page.wait(2);
var runner = try @import("testing.zig").jsRunner(arena, .{
.url = "http://127.0.0.1",
.html = html,
});
defer runner.deinit();
defer if (err_out.*) |eo| {
// the error might be owned by the runner, we'll dupe it with our
// own arena so that it can be returned out of this function.
err_out.* = arena.dupe(u8, eo) catch "failed to dupe error";
};
try polyfill.preload(arena, runner.page.main_context);
// loop over the scripts.
const doc = parser.documentHTMLToDocument(runner.page.window.document);
const scripts = try parser.documentGetElementsByTagName(doc, "script");
const script_count = try parser.nodeListLength(scripts);
for (0..script_count) |i| {
const s = (try parser.nodeListItem(scripts, @intCast(i))).?;
// If the script contains an src attribute, load it.
if (try parser.elementGetAttribute(@as(*parser.Element, @ptrCast(s)), "src")) |src| {
var path = src;
if (!std.mem.startsWith(u8, src, "/")) {
path = try std.fs.path.join(arena, &.{ "/", dirname, path });
}
const script_source = loader.get(path) catch |err| {
err_out.* = std.fmt.allocPrint(arena, "{s} - {s}", .{ @errorName(err), path }) catch null;
return err;
};
try runner.exec(script_source, src, err_out);
}
// If the script as a source text, execute it.
const src = try parser.nodeTextContent(s) orelse continue;
try runner.exec(src, null, err_out);
}
{
// Mark tests as ready to run.
const loadevt = try parser.eventCreate();
defer parser.eventDestroy(loadevt);
try parser.eventInit(loadevt, "load", .{});
_ = try parser.eventTargetDispatchEvent(
parser.toEventTarget(@TypeOf(runner.page.window), &runner.page.window),
loadevt,
);
}
{
// wait for all async executions
var try_catch: Env.TryCatch = undefined;
try_catch.init(runner.page.main_context);
defer try_catch.deinit();
runner.page.wait(2);
if (try_catch.hasCaught()) {
err_out.* = (try try_catch.err(arena)) orelse "unknwon error";
}
}
const js_context = page.main_context;
var try_catch: Env.TryCatch = undefined;
try_catch.init(js_context);
defer try_catch.deinit();
// Check the final test status.
try runner.exec("report.status", "teststatus", err_out);
js_context.eval("report.status", "teststatus") catch |err| {
err_out.* = try_catch.err(arena) catch @errorName(err) orelse "unknown";
return err;
};
// return the detailed result.
const res = try runner.eval("report.log", "report", err_out);
const value = js_context.exec("report.log", "report") catch |err| {
err_out.* = try_catch.err(arena) catch @errorName(err) orelse "unknown";
return err;
};
return try res.toString(arena);
return value.toString(arena);
}
const Writer = struct {
format: Format,
arena: Allocator,
allocator: Allocator,
pass_count: usize = 0,
fail_count: usize = 0,
case_pass_count: usize = 0,
@@ -201,7 +147,7 @@ const Writer = struct {
summary,
};
fn init(arena: Allocator, format: Format) !Writer {
fn init(allocator: Allocator, format: Format) !Writer {
const out = std.fs.File.stdout();
var writer = out.writer(&.{});
@@ -210,12 +156,16 @@ const Writer = struct {
}
return .{
.arena = arena,
.format = format,
.writer = writer,
.allocator = allocator,
};
}
fn deinit(self: *Writer) void {
self.cases.deinit(self.allocator);
}
fn finalize(self: *Writer) !void {
var writer = &self.writer.interface;
if (self.format == .json) {
@@ -303,7 +253,7 @@ const Writer = struct {
case_fail_count += 1;
}
try cases.append(self.arena, .{
try cases.append(self.allocator, .{
.name = case_name,
.pass = case_pass,
.message = case_message,
@@ -396,22 +346,26 @@ const TestIterator = struct {
dir: Dir,
walker: Dir.Walker,
filters: [][]const u8,
read_arena: ArenaAllocator,
const Dir = std.fs.Dir;
fn init(arena: Allocator, root: []const u8, filters: [][]const u8) !TestIterator {
fn init(allocator: Allocator, root: []const u8, filters: [][]const u8) !TestIterator {
var dir = try std.fs.cwd().openDir(root, .{ .iterate = true, .no_follow = true });
errdefer dir.close();
return .{
.dir = dir,
.filters = filters,
.walker = try dir.walk(arena),
.walker = try dir.walk(allocator),
.read_arena = ArenaAllocator.init(allocator),
};
}
fn deinit(self: *TestIterator) void {
self.walker.deinit();
self.dir.close();
self.read_arena.deinit();
}
fn next(self: *TestIterator) !?[]const u8 {
@@ -436,6 +390,25 @@ const TestIterator = struct {
}
}
{
defer _ = self.read_arena.reset(.retain_capacity);
// We need to read the file's content to see if there's a
// "testharness.js" in it. If there isn't, it isn't a test.
// Shame we have to do this.
const arena = self.read_arena.allocator();
const full_path = try std.fs.path.join(arena, &.{ WPT_DIR, path });
const file = try std.fs.cwd().openFile(full_path, .{});
defer file.close();
const html = try file.readToEndAlloc(arena, 128 * 1024);
if (std.mem.indexOf(u8, html, "testharness.js") == null) {
// This isn't a test. A lot of files are helpers/content for tests to
// make use of.
continue :NEXT;
}
}
return path;
}
@@ -456,35 +429,16 @@ const Test = struct {
cases: []Case,
};
pub const FileLoader = struct {
path: []const u8,
arena: Allocator,
files: std.StringHashMapUnmanaged([]const u8),
fn httpHandler(req: *std.http.Server.Request) !void {
const path = req.head.target;
pub fn init(arena: Allocator, path: []const u8) FileLoader {
return .{
.path = path,
.files = .{},
.arena = arena,
};
}
pub fn get(self: *FileLoader, name: []const u8) ![]const u8 {
const gop = try self.files.getOrPut(self.arena, name);
if (gop.found_existing == false) {
gop.key_ptr.* = try self.arena.dupe(u8, name);
gop.value_ptr.* = self.load(name) catch |err| {
_ = self.files.remove(name);
return err;
};
}
return gop.value_ptr.*;
if (std.mem.eql(u8, path, "/")) {
// There's 1 test that does an XHR request to this, and it just seems
// to want a 200 success.
return req.respond("Hello!", .{});
}
fn load(self: *FileLoader, name: []const u8) ![]const u8 {
const filename = try std.fs.path.join(self.arena, &.{ self.path, name });
var file = try std.fs.cwd().openFile(filename, .{});
defer file.close();
return file.readToEndAlloc(self.arena, 4 * 1024 * 1024);
}
};
var buf: [1024]u8 = undefined;
const file_path = try std.fmt.bufPrint(&buf, WPT_DIR ++ "{s}", .{path});
return TestHTTPServer.sendFile(req, file_path);
}

8
src/runtime/js.zig
View File

@@ -1516,12 +1516,12 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
}
const op = js_obj.getInternalField(0).castTo(v8.External).get();
const toa: *TaggedAnyOpaque = @ptrCast(@alignCast(op));
const tao: *TaggedAnyOpaque = @ptrCast(@alignCast(op));
const expected_type_index = @field(TYPE_LOOKUP, type_name);
var type_index = toa.index;
var type_index = tao.index;
if (type_index == expected_type_index) {
return @ptrCast(@alignCast(toa.ptr));
return @ptrCast(@alignCast(tao.ptr));
}
const meta_lookup = self.meta_lookup;
@@ -1533,7 +1533,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
// ...unless, the proto is behind a pointer, then total_offset will
// get reset to 0, and our base_ptr will move to the address
// referenced by the proto field.
var base_ptr: usize = @intFromPtr(toa.ptr);
var base_ptr: usize = @intFromPtr(tao.ptr);
// search through the prototype tree
while (true) {