From b55c72c2ed1aef026a86a943de548e20b8882647 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Tue, 28 Nov 2023 14:09:15 +0100
Subject: [PATCH 01/14] wpt: add json option

---
 src/main_wpt.zig | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 972f951b..0f8ae0e4 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -58,6 +58,15 @@ const FileLoader = struct {
     }
 };
 
+const usage =
+    \\usage: {s} [options] [test filter]
+    \\  Run the Web Test Platform.
+    \\
+    \\  -h, --help       Print this help message and exit.
+    \\  --json           result is formatted in JSON.
+    \\
+;
+
 // TODO For now the WPT tests run is specific to WPT.
 // It manually load js framwork libs, and run the first script w/ js content in
 // the HTML page.
@@ -74,10 +83,28 @@ pub fn main() !void {
     defer _ = gpa.deinit();
     const alloc = gpa.allocator();
 
-    const args = try std.process.argsAlloc(alloc);
-    defer std.process.argsFree(alloc, args);
+    var args = try std.process.argsWithAllocator(alloc);
+    defer args.deinit();
 
-    const filter = args[1..];
+    // get the exec name.
+    const execname = args.next().?;
+
+    var json = false;
+
+    var filter = std.ArrayList([]const u8).init(alloc);
+    defer filter.deinit();
+
+    while (args.next()) |arg| {
+        if (std.mem.eql(u8, "-h", arg) or std.mem.eql(u8, "--help", arg)) {
+            try std.io.getStdErr().writer().print(usage, .{execname});
+            std.os.exit(0);
+        }
+        if (std.mem.eql(u8, "--json", arg)) {
+            json = true;
+            continue;
+        }
+        try filter.append(arg[0..]);
+    }
 
     // initialize VM JS lib.
     const vm = jsruntime.VM.init();
@@ -100,9 +127,9 @@ pub fn main() !void {
     var run: usize = 0;
     var failures: usize = 0;
     for (list.items) |tc| {
-        if (filter.len > 0) {
+        if (filter.items.len > 0) {
             var match = false;
-            for (filter) |f| {
+            for (filter.items) |f| {
                 if (std.mem.startsWith(u8, tc, f)) {
                     match = true;
                     break;

From 74b25161a3f607c0771d1b180a375e0ffceb84db Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Tue, 28 Nov 2023 14:11:13 +0100
Subject: [PATCH 02/14] wpt: change js output format

---
 tests/wpt/resources/testharnessreport.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/wpt/resources/testharnessreport.js b/tests/wpt/resources/testharnessreport.js
index bb916b06..58c36ac7 100644
--- a/tests/wpt/resources/testharnessreport.js
+++ b/tests/wpt/resources/testharnessreport.js
@@ -10,7 +10,7 @@ var report = {
 add_completion_callback(function (tests, status) {
   // report the tests global status.
   // TODO the status.status is always OK even if a test fail.
-  // I ingore the global status for now, but I build one with the tests results.
+  // I ignore the global status for now, but I build one with the tests results.
   //report.status = status.status;
 
   var status = "Pass";
@@ -18,9 +18,9 @@ add_completion_callback(function (tests, status) {
   var log = "";
   for (var i = 0; i < tests.length; i++) {
     const test = tests[i];
-    log += test.name+": "+test.format_status();
+    log += test.name+"\t"+test.format_status()+"\t";
     if (test.message != null) {
-      log += " " + test.message;
+      log +=  test.message;
     }
     log += "\n";
 

From 486927c889b906239815c8c6b2c804de91ab66c7 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Tue, 28 Nov 2023 17:15:04 +0100
Subject: [PATCH 03/14] wpt: refacto results report

---
 src/main_wpt.zig                         |  25 ++-
 src/wpt/testcase.zig                     | 248 +++++++++++++++++++++++
 tests/wpt/resources/testharnessreport.js |   4 +-
 3 files changed, 265 insertions(+), 12 deletions(-)
 create mode 100644 src/wpt/testcase.zig

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 0f8ae0e4..36831cbc 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -2,6 +2,7 @@ const std = @import("std");
 
 const parser = @import("netsurf.zig");
 const jsruntime = @import("jsruntime");
+const wpt = @import("wpt/testcase.zig");
 
 const TPL = jsruntime.TPL;
 const Env = jsruntime.Env;
@@ -159,22 +160,26 @@ pub fn main() !void {
         };
         // no need to call res.deinit() thanks to the arena allocator.
 
-        if (!res.success) {
-            std.debug.print("FAIL\t{s}\n{s}\n", .{ tc, res.stack orelse res.result });
+        const suite = try wpt.Suite.init(arena.allocator(), tc, res.success, res.result, res.stack);
+        defer suite.deinit();
+
+        if (!suite.pass) {
+            std.debug.print("Fail\t{s}\n{s}\n", .{ suite.name, suite.fmtMessage() });
             failures += 1;
-            continue;
-        }
-        if (!std.mem.eql(u8, res.result, "Pass")) {
-            std.debug.print("FAIL\t{s}\n{s}\n", .{ tc, res.stack orelse res.result });
-            failures += 1;
-            continue;
+        } else {
+            std.debug.print("Pass\t{s}\n", .{suite.name});
         }
 
-        std.debug.print("PASS\t{s}\n", .{tc});
+        // display details
+        if (suite.cases) |cases| {
+            for (cases) |case| {
+                std.debug.print("\t{s}\t{s}\t{s}\n", .{ case.fmtStatus(), case.name, case.fmtMessage() });
+            }
+        }
     }
 
     if (failures > 0) {
-        std.debug.print("{d}/{d} tests failures\n", .{ failures, run });
+        std.debug.print("{d}/{d} tests suites failures\n", .{ failures, run });
         std.os.exit(1);
     }
 }
diff --git a/src/wpt/testcase.zig b/src/wpt/testcase.zig
new file mode 100644
index 00000000..c1a7e56e
--- /dev/null
+++ b/src/wpt/testcase.zig
@@ -0,0 +1,248 @@
+const std = @import("std");
+const testing = std.testing;
+
+pub const Case = struct {
+    pass: bool,
+    name: []const u8,
+
+    message: ?[]const u8,
+
+    fn init(alloc: std.mem.Allocator, name: []const u8, status: []const u8, message: []const u8) !Case {
+        var case = Case{
+            .pass = std.mem.eql(u8, "Pass", status),
+            .name = try alloc.dupe(u8, name),
+            .message = null,
+        };
+
+        if (message.len > 0) {
+            case.message = try alloc.dupe(u8, message);
+        }
+
+        return case;
+    }
+
+    fn deinit(self: Case, alloc: std.mem.Allocator) void {
+        alloc.free(self.name);
+
+        if (self.message) |msg| {
+            alloc.free(msg);
+        }
+    }
+
+    pub fn fmtStatus(self: Case) []const u8 {
+        if (self.pass) {
+            return "Pass";
+        }
+        return "Fail";
+    }
+
+    pub fn fmtMessage(self: Case) []const u8 {
+        if (self.message) |v| {
+            return v;
+        }
+        return "";
+    }
+};
+
+pub const Suite = struct {
+    alloc: std.mem.Allocator,
+    pass: bool,
+    name: []const u8,
+    message: ?[]const u8,
+    stack: ?[]const u8,
+    cases: ?[]Case,
+
+    // caller owns the wpt.Suite.
+    // owner must call deinit().
+    pub fn init(alloc: std.mem.Allocator, name: []const u8, pass: bool, res: []const u8, stack: ?[]const u8) !Suite {
+        var suite = Suite{
+            .alloc = alloc,
+            .pass = false,
+            .name = try alloc.dupe(u8, name),
+            .message = null,
+            .stack = null,
+            .cases = null,
+        };
+
+        // handle JS error.
+        if (!pass) {
+            suite.message = try alloc.dupe(u8, res);
+            if (stack) |st| {
+                suite.stack = try alloc.dupe(u8, st);
+            }
+
+            return suite;
+        }
+
+        // no JS error, let's try to parse the result.
+        suite.pass = true;
+
+        // special case: the result contains only "Pass" message
+        if (std.mem.eql(u8, "Pass", res)) {
+            return suite;
+        }
+
+        var cases = std.ArrayList(Case).init(alloc);
+        defer cases.deinit();
+
+        var lines = std.mem.splitScalar(u8, res, '\n');
+        while (lines.next()) |line| {
+            if (line.len == 0) {
+                break;
+            }
+            var fields = std.mem.splitScalar(u8, line, '|');
+            var ff: [3][]const u8 = .{ "", "", "" };
+            var i: u8 = 0;
+            while (fields.next()) |field| {
+                if (i >= 3) {
+                    suite.pass = false;
+                    suite.message = try alloc.dupe(u8, res);
+                    return suite;
+                }
+
+                ff[i] = field;
+                i += 1;
+            }
+
+            // invalid output format
+            if (i != 2 and i != 3) {
+                suite.pass = false;
+                suite.message = try alloc.dupe(u8, res);
+                return suite;
+            }
+
+            const case = try Case.init(alloc, ff[0], ff[1], ff[2]);
+            if (!case.pass) {
+                suite.pass = false;
+            }
+
+            try cases.append(case);
+        }
+
+        suite.cases = try cases.toOwnedSlice();
+
+        return suite;
+    }
+
+    pub fn deinit(self: Suite) void {
+        self.alloc.free(self.name);
+
+        if (self.stack) |stack| {
+            self.alloc.free(stack);
+        }
+
+        if (self.message) |res| {
+            self.alloc.free(res);
+        }
+
+        if (self.cases) |cases| {
+            for (cases) |case| {
+                case.deinit(self.alloc);
+            }
+            self.alloc.free(cases);
+        }
+    }
+
+    pub fn fmtMessage(self: Suite) []const u8 {
+        if (self.message) |v| {
+            return v;
+        }
+        if (self.stack) |v| {
+            return v;
+        }
+        return "";
+    }
+};
+
+test "success test case" {
+    const alloc = testing.allocator;
+
+    const Res = struct {
+        pass: bool,
+        result: []const u8,
+    };
+
+    const res = Res{
+        .pass = true,
+        .result =
+        \\Empty string as a name for Document.getElementsByTagName|Pass
+        \\Empty string as a name for Element.getElementsByTagName|Pass
+        \\
+        ,
+    };
+
+    const suite = Suite.init(alloc, "foo", res.pass, res.result, null) catch unreachable; // TODO
+    defer suite.deinit();
+
+    try testing.expect(suite.pass == true);
+    try testing.expect(suite.cases != null);
+    try testing.expect(suite.cases.?.len == 2);
+    try testing.expect(suite.cases.?[0].pass == true);
+    try testing.expect(suite.cases.?[1].pass == true);
+}
+
+test "failed test case" {
+    const alloc = testing.allocator;
+
+    const Res = struct {
+        pass: bool,
+        result: []const u8,
+    };
+
+    const res = Res{
+        .pass = true,
+        .result =
+        \\Empty string as a name for Document.getElementsByTagName|Pass
+        \\Empty string as a name for Element.getElementsByTagName|Fail|div.getElementsByTagName is not a function
+        \\
+        ,
+    };
+
+    const suite = Suite.init(alloc, "foo", res.pass, res.result, null) catch unreachable; // TODO
+    defer suite.deinit();
+
+    try testing.expect(suite.pass == false);
+    try testing.expect(suite.cases != null);
+    try testing.expect(suite.cases.?.len == 2);
+    try testing.expect(suite.cases.?[0].pass == true);
+    try testing.expect(suite.cases.?[1].pass == false);
+}
+
+test "invalid result" {
+    const alloc = testing.allocator;
+
+    const Res = struct {
+        pass: bool,
+        result: []const u8,
+    };
+
+    const res = Res{
+        .pass = true,
+        .result =
+        \\this is|an|invalid|result
+        ,
+    };
+
+    const suite = Suite.init(alloc, "foo", res.pass, res.result, null) catch unreachable; // TODO
+    defer suite.deinit();
+
+    try testing.expect(suite.pass == false);
+    try testing.expect(suite.message != null);
+    try testing.expect(std.mem.eql(u8, res.result, suite.message.?));
+    try testing.expect(suite.cases == null);
+
+    const res2 = Res{
+        .pass = true,
+        .result =
+        \\this is an invalid result.
+        ,
+    };
+
+    const suite2 = Suite.init(alloc, "foo", res2.pass, res2.result, null) catch unreachable; // TODO
+    defer suite2.deinit();
+
+    try testing.expect(suite2.pass == false);
+    try testing.expect(suite2.message != null);
+    try testing.expect(std.mem.eql(u8, res2.result, suite2.message.?));
+    try testing.expect(suite2.cases == null);
+}
diff --git a/tests/wpt/resources/testharnessreport.js b/tests/wpt/resources/testharnessreport.js
index 58c36ac7..c9817431 100644
--- a/tests/wpt/resources/testharnessreport.js
+++ b/tests/wpt/resources/testharnessreport.js
@@ -18,9 +18,9 @@ add_completion_callback(function (tests, status) {
   var log = "";
   for (var i = 0; i < tests.length; i++) {
     const test = tests[i];
-    log += test.name+"\t"+test.format_status()+"\t";
+    log += test.name+"|"+test.format_status();
     if (test.message != null) {
-      log +=  test.message;
+      log +=  "|"+test.message;
     }
     log += "\n";
 

From b9c0d9e20e8469e17ea37691729f9ce1ad204162 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Wed, 29 Nov 2023 11:52:55 +0100
Subject: [PATCH 04/14] wpt: remove new lines from report

---
 tests/wpt/resources/testharnessreport.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/wpt/resources/testharnessreport.js b/tests/wpt/resources/testharnessreport.js
index c9817431..38a17de0 100644
--- a/tests/wpt/resources/testharnessreport.js
+++ b/tests/wpt/resources/testharnessreport.js
@@ -20,7 +20,7 @@ add_completion_callback(function (tests, status) {
     const test = tests[i];
     log += test.name+"|"+test.format_status();
     if (test.message != null) {
-      log +=  "|"+test.message;
+      log +=  "|"+test.message.replaceAll("\n"," ");
     }
     log += "\n";
 

From d7ddec3ba73362916dbf0c32bf215f5793019929 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Wed, 29 Nov 2023 14:04:56 +0100
Subject: [PATCH 05/14] wpt: move code into wpt/ module

---
 src/main_wpt.zig       | 207 ++---------------------------------------
 src/wpt/fileloader.zig |  46 +++++++++
 src/wpt/run.zig        | 158 +++++++++++++++++++++++++++++++
 3 files changed, 210 insertions(+), 201 deletions(-)
 create mode 100644 src/wpt/fileloader.zig
 create mode 100644 src/wpt/run.zig

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 36831cbc..fd7f1fb8 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -1,64 +1,16 @@
 const std = @import("std");
 
-const parser = @import("netsurf.zig");
 const jsruntime = @import("jsruntime");
-const wpt = @import("wpt/testcase.zig");
 
-const TPL = jsruntime.TPL;
-const Env = jsruntime.Env;
-const Loop = jsruntime.Loop;
+const Suite = @import("wpt/testcase.zig").Suite;
+const FileLoader = @import("wpt/fileloader.zig").FileLoader;
+const wpt = @import("wpt/run.zig");
 
 const DOM = @import("dom.zig");
 const HTMLElem = @import("html/elements.zig");
 
-const fspath = std.fs.path;
-
 const wpt_dir = "tests/wpt";
 
-// FileLoader loads files content from the filesystem.
-const FileLoader = struct {
-    const FilesMap = std.StringHashMap([]const u8);
-
-    files: FilesMap,
-    path: []const u8,
-    alloc: std.mem.Allocator,
-
-    fn init(alloc: std.mem.Allocator, path: []const u8) FileLoader {
-        const files = FilesMap.init(alloc);
-
-        return FileLoader{
-            .path = path,
-            .alloc = alloc,
-            .files = files,
-        };
-    }
-    fn get(self: *FileLoader, name: []const u8) ![]const u8 {
-        if (!self.files.contains(name)) {
-            try self.load(name);
-        }
-        return self.files.get(name).?;
-    }
-    fn load(self: *FileLoader, name: []const u8) !void {
-        const filename = try fspath.join(self.alloc, &.{ self.path, name });
-        defer self.alloc.free(filename);
-        var file = try std.fs.cwd().openFile(filename, .{});
-        defer file.close();
-
-        const file_size = try file.getEndPos();
-        const content = try file.readToEndAlloc(self.alloc, file_size);
-        const namedup = try self.alloc.dupe(u8, name);
-        try self.files.put(namedup, content);
-    }
-    fn deinit(self: *FileLoader) void {
-        var iter = self.files.iterator();
-        while (iter.next()) |entry| {
-            self.alloc.free(entry.key_ptr.*);
-            self.alloc.free(entry.value_ptr.*);
-        }
-        self.files.deinit();
-    }
-};
-
 const usage =
     \\usage: {s} [options] [test filter]
     \\  Run the Web Test Platform.
@@ -117,7 +69,7 @@ pub fn main() !void {
 
     // browse the dir to get the tests dynamically.
     var list = std.ArrayList([]const u8).init(alloc);
-    try findWPTTests(alloc, wpt_dir, &list);
+    try wpt.find(alloc, wpt_dir, &list);
     defer {
         for (list.items) |tc| {
             alloc.free(tc);
@@ -153,14 +105,14 @@ pub fn main() !void {
 
         // TODO I don't use testing.expect here b/c I want to execute all the
         // tests. And testing.expect stops running test in the first failure.
-        const res = runWPT(&arena, apis, tc, &loader) catch |err| {
+        const res = wpt.run(&arena, apis, wpt_dir, tc, &loader) catch |err| {
             std.debug.print("FAIL\t{s}\n{any}\n", .{ tc, err });
             failures += 1;
             continue;
         };
         // no need to call res.deinit() thanks to the arena allocator.
 
-        const suite = try wpt.Suite.init(arena.allocator(), tc, res.success, res.result, res.stack);
+        const suite = try Suite.init(arena.allocator(), tc, res.success, res.result, res.stack);
         defer suite.deinit();
 
         if (!suite.pass) {
@@ -183,150 +135,3 @@ pub fn main() !void {
         std.os.exit(1);
     }
 }
-
-// runWPT parses the given HTML file, starts a js env and run the first script
-// tags containing javascript sources.
-// It loads first the js libs files.
-fn runWPT(arena: *std.heap.ArenaAllocator, comptime apis: []jsruntime.API, f: []const u8, loader: *FileLoader) !jsruntime.JSResult {
-    const alloc = arena.allocator();
-
-    // document
-    const html_doc = try parser.documentHTMLParseFromFileAlloc(alloc, f);
-    const doc = parser.documentHTMLToDocument(html_doc);
-
-    const dirname = fspath.dirname(f[wpt_dir.len..]) orelse unreachable;
-
-    // create JS env
-    var loop = try Loop.init(alloc);
-    defer loop.deinit();
-    var js_env = try Env.init(arena, &loop);
-    defer js_env.deinit();
-
-    // load APIs in JS env
-    var tpls: [apis.len]TPL = undefined;
-    try js_env.load(apis, &tpls);
-
-    // start JS env
-    try js_env.start(alloc, apis);
-    defer js_env.stop();
-
-    // add document object
-    try js_env.addObject(apis, html_doc, "document");
-
-    // alias global as self and window
-    try js_env.attachObject(try js_env.getGlobal(), "self", null);
-    try js_env.attachObject(try js_env.getGlobal(), "window", null);
-
-    // thanks to the arena, we don't need to deinit res.
-    var res: jsruntime.JSResult = undefined;
-
-    const init =
-        \\window.listeners = [];
-        \\window.document = document;
-        \\window.parent = window;
-        \\window.addEventListener = function (type, listener, options) {
-        \\  window.listeners.push({type: type, listener: listener, options: options});
-        \\};
-        \\window.dispatchEvent = function (event) {
-        \\  len = window.listeners.length;
-        \\  for (var i = 0; i < len; i++) {
-        \\      if (window.listeners[i].type == event.target) {
-        \\          window.listeners[i].listener(event);
-        \\      }
-        \\  }
-        \\  return true;
-        \\};
-        \\window.removeEventListener = function () {};
-        \\
-        \\console = [];
-        \\console.log = function () {
-        \\  console.push(...arguments);
-        \\};
-    ;
-    res = try evalJS(js_env, alloc, init, "init");
-    if (!res.success) {
-        return res;
-    }
-
-    // loop hover the scripts.
-    const scripts = try parser.documentGetElementsByTagName(doc, "script");
-    const slen = try parser.nodeListLength(scripts);
-    for (0..slen) |i| {
-        const s = (try parser.nodeListItem(scripts, @intCast(i))).?;
-
-        // If the script contains an src attribute, load it.
-        if (try parser.elementGetAttribute(@as(*parser.Element, @ptrCast(s)), "src")) |src| {
-            var path = src;
-            if (!std.mem.startsWith(u8, src, "/")) {
-                // no need to free path, thanks to the arena.
-                path = try fspath.join(alloc, &.{ "/", dirname, path });
-            }
-
-            res = try evalJS(js_env, alloc, try loader.get(path), src);
-            if (!res.success) {
-                return res;
-            }
-        }
-
-        // If the script as a source text, execute it.
-        const src = try parser.nodeTextContent(s) orelse continue;
-        res = try evalJS(js_env, alloc, src, "");
-
-        // return the first failure.
-        if (!res.success) {
-            return res;
-        }
-    }
-
-    // Mark tests as ready to run.
-    res = try evalJS(js_env, alloc, "window.dispatchEvent({target: 'load'});", "ready");
-    if (!res.success) {
-        return res;
-    }
-
-    // display console logs
-    res = try evalJS(js_env, alloc, "console.join(', ');", "console");
-    if (res.result.len > 0) {
-        std.debug.print("-- CONSOLE LOG\n{s}\n--\n", .{res.result});
-    }
-
-    // Check the final test status.
-    res = try evalJS(js_env, alloc, "report.status;", "teststatus");
-    if (!res.success) {
-        return res;
-    }
-
-    // If the test failed, return detailed logs intead of the simple status.
-    if (!std.mem.eql(u8, res.result, "Pass")) {
-        return try evalJS(js_env, alloc, "report.log", "teststatus");
-    }
-
-    // return the final result.
-    return res;
-}
-
-fn evalJS(env: jsruntime.Env, alloc: std.mem.Allocator, script: []const u8, name: ?[]const u8) !jsruntime.JSResult {
-    var res = jsruntime.JSResult{};
-    try env.run(alloc, script, name, &res, null);
-    return res;
-}
-
-// browse the path to find the tests list.
-fn findWPTTests(allocator: std.mem.Allocator, path: []const u8, list: *std.ArrayList([]const u8)) !void {
-    var dir = try std.fs.cwd().openIterableDir(path, .{ .no_follow = true });
-    defer dir.close();
-
-    var walker = try dir.walk(allocator);
-    defer walker.deinit();
-
-    while (try walker.next()) |entry| {
-        if (entry.kind != .file) {
-            continue;
-        }
-        if (!std.mem.endsWith(u8, entry.basename, ".html")) {
-            continue;
-        }
-
-        try list.append(try fspath.join(allocator, &.{ path, entry.path }));
-    }
-}
diff --git a/src/wpt/fileloader.zig b/src/wpt/fileloader.zig
new file mode 100644
index 00000000..41958185
--- /dev/null
+++ b/src/wpt/fileloader.zig
@@ -0,0 +1,46 @@
+const std = @import("std");
+const fspath = std.fs.path;
+
+// FileLoader loads files content from the filesystem.
+pub const FileLoader = struct {
+    const FilesMap = std.StringHashMap([]const u8);
+
+    files: FilesMap,
+    path: []const u8,
+    alloc: std.mem.Allocator,
+
+    pub fn init(alloc: std.mem.Allocator, path: []const u8) FileLoader {
+        const files = FilesMap.init(alloc);
+
+        return FileLoader{
+            .path = path,
+            .alloc = alloc,
+            .files = files,
+        };
+    }
+    pub fn get(self: *FileLoader, name: []const u8) ![]const u8 {
+        if (!self.files.contains(name)) {
+            try self.load(name);
+        }
+        return self.files.get(name).?;
+    }
+    pub fn load(self: *FileLoader, name: []const u8) !void {
+        const filename = try fspath.join(self.alloc, &.{ self.path, name });
+        defer self.alloc.free(filename);
+        var file = try std.fs.cwd().openFile(filename, .{});
+        defer file.close();
+
+        const file_size = try file.getEndPos();
+        const content = try file.readToEndAlloc(self.alloc, file_size);
+        const namedup = try self.alloc.dupe(u8, name);
+        try self.files.put(namedup, content);
+    }
+    pub fn deinit(self: *FileLoader) void {
+        var iter = self.files.iterator();
+        while (iter.next()) |entry| {
+            self.alloc.free(entry.key_ptr.*);
+            self.alloc.free(entry.value_ptr.*);
+        }
+        self.files.deinit();
+    }
+};
diff --git a/src/wpt/run.zig b/src/wpt/run.zig
new file mode 100644
index 00000000..a7c77077
--- /dev/null
+++ b/src/wpt/run.zig
@@ -0,0 +1,158 @@
+const std = @import("std");
+const fspath = std.fs.path;
+
+const FileLoader = @import("fileloader.zig").FileLoader;
+
+const parser = @import("../netsurf.zig");
+
+const jsruntime = @import("jsruntime");
+const Loop = jsruntime.Loop;
+const Env = jsruntime.Env;
+const TPL = jsruntime.TPL;
+
+// runWPT parses the given HTML file, starts a js env and run the first script
+// tags containing javascript sources.
+// It loads first the js libs files.
+pub fn run(arena: *std.heap.ArenaAllocator, comptime apis: []jsruntime.API, comptime dir: []const u8, f: []const u8, loader: *FileLoader) !jsruntime.JSResult {
+    const alloc = arena.allocator();
+
+    // document
+    const html_doc = try parser.documentHTMLParseFromFileAlloc(alloc, f);
+    const doc = parser.documentHTMLToDocument(html_doc);
+
+    const dirname = fspath.dirname(f[dir.len..]) orelse unreachable;
+
+    // create JS env
+    var loop = try Loop.init(alloc);
+    defer loop.deinit();
+    var js_env = try Env.init(arena, &loop);
+    defer js_env.deinit();
+
+    // load APIs in JS env
+    var tpls: [apis.len]TPL = undefined;
+    try js_env.load(apis, &tpls);
+
+    // start JS env
+    try js_env.start(alloc, apis);
+    defer js_env.stop();
+
+    // add document object
+    try js_env.addObject(apis, html_doc, "document");
+
+    // alias global as self and window
+    try js_env.attachObject(try js_env.getGlobal(), "self", null);
+    try js_env.attachObject(try js_env.getGlobal(), "window", null);
+
+    // thanks to the arena, we don't need to deinit res.
+    var res: jsruntime.JSResult = undefined;
+
+    const init =
+        \\window.listeners = [];
+        \\window.document = document;
+        \\window.parent = window;
+        \\window.addEventListener = function (type, listener, options) {
+        \\  window.listeners.push({type: type, listener: listener, options: options});
+        \\};
+        \\window.dispatchEvent = function (event) {
+        \\  len = window.listeners.length;
+        \\  for (var i = 0; i < len; i++) {
+        \\      if (window.listeners[i].type == event.target) {
+        \\          window.listeners[i].listener(event);
+        \\      }
+        \\  }
+        \\  return true;
+        \\};
+        \\window.removeEventListener = function () {};
+        \\
+        \\console = [];
+        \\console.log = function () {
+        \\  console.push(...arguments);
+        \\};
+    ;
+    res = try evalJS(js_env, alloc, init, "init");
+    if (!res.success) {
+        return res;
+    }
+
+    // loop hover the scripts.
+    const scripts = try parser.documentGetElementsByTagName(doc, "script");
+    const slen = try parser.nodeListLength(scripts);
+    for (0..slen) |i| {
+        const s = (try parser.nodeListItem(scripts, @intCast(i))).?;
+
+        // If the script contains an src attribute, load it.
+        if (try parser.elementGetAttribute(@as(*parser.Element, @ptrCast(s)), "src")) |src| {
+            var path = src;
+            if (!std.mem.startsWith(u8, src, "/")) {
+                // no need to free path, thanks to the arena.
+                path = try fspath.join(alloc, &.{ "/", dirname, path });
+            }
+
+            res = try evalJS(js_env, alloc, try loader.get(path), src);
+            if (!res.success) {
+                return res;
+            }
+        }
+
+        // If the script as a source text, execute it.
+        const src = try parser.nodeTextContent(s) orelse continue;
+        res = try evalJS(js_env, alloc, src, "");
+
+        // return the first failure.
+        if (!res.success) {
+            return res;
+        }
+    }
+
+    // Mark tests as ready to run.
+    res = try evalJS(js_env, alloc, "window.dispatchEvent({target: 'load'});", "ready");
+    if (!res.success) {
+        return res;
+    }
+
+    // display console logs
+    res = try evalJS(js_env, alloc, "console.join(', ');", "console");
+    if (res.result.len > 0) {
+        std.debug.print("-- CONSOLE LOG\n{s}\n--\n", .{res.result});
+    }
+
+    // Check the final test status.
+    res = try evalJS(js_env, alloc, "report.status;", "teststatus");
+    if (!res.success) {
+        return res;
+    }
+
+    // If the test failed, return detailed logs intead of the simple status.
+    if (!std.mem.eql(u8, res.result, "Pass")) {
+        return try evalJS(js_env, alloc, "report.log", "teststatus");
+    }
+
+    // return the final result.
+    return res;
+}
+
+fn evalJS(env: jsruntime.Env, alloc: std.mem.Allocator, script: []const u8, name: ?[]const u8) !jsruntime.JSResult {
+    var res = jsruntime.JSResult{};
+    try env.run(alloc, script, name, &res, null);
+    return res;
+}
+
+// browse the path to find the tests list.
+pub fn find(allocator: std.mem.Allocator, comptime path: []const u8, list: *std.ArrayList([]const u8)) !void {
+    var dir = try std.fs.cwd().openIterableDir(path, .{ .no_follow = true });
+    defer dir.close();
+
+    var walker = try dir.walk(allocator);
+    defer walker.deinit();
+
+    while (try walker.next()) |entry| {
+        if (entry.kind != .file) {
+            continue;
+        }
+        if (!std.mem.endsWith(u8, entry.basename, ".html")) {
+            continue;
+        }
+
+        try list.append(try fspath.join(allocator, &.{ path, entry.path }));
+    }
+}

From 49318b18df2a03435fd70669506c922348873f10 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Wed, 29 Nov 2023 14:22:47 +0100
Subject: [PATCH 06/14] wpt: use gpa allocator

---
 src/main_wpt.zig | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index fd7f1fb8..94dfefdb 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -103,8 +103,6 @@ pub fn main() !void {
         var arena = std.heap.ArenaAllocator.init(alloc);
         defer arena.deinit();
 
-        // TODO I don't use testing.expect here b/c I want to execute all the
-        // tests. And testing.expect stops running test in the first failure.
         const res = wpt.run(&arena, apis, wpt_dir, tc, &loader) catch |err| {
             std.debug.print("FAIL\t{s}\n{any}\n", .{ tc, err });
             failures += 1;
@@ -112,7 +110,7 @@ pub fn main() !void {
         };
         // no need to call res.deinit() thanks to the arena allocator.
 
-        const suite = try Suite.init(arena.allocator(), tc, res.success, res.result, res.stack);
+        const suite = try Suite.init(alloc, tc, res.success, res.result, res.stack);
         defer suite.deinit();
 
         if (!suite.pass) {

From 62721e672f002a66574203691c13693a96393bce Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Wed, 29 Nov 2023 15:36:29 +0100
Subject: [PATCH 07/14] wpt: add json output

---
 src/main_wpt.zig | 77 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 74 insertions(+), 3 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 94dfefdb..b4ed2031 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -77,6 +77,14 @@ pub fn main() !void {
         list.deinit();
     }
 
+    var results = std.ArrayList(Suite).init(alloc);
+    defer {
+        for (results.items) |suite| {
+            suite.deinit();
+        }
+        results.deinit();
+    }
+
     var run: usize = 0;
     var failures: usize = 0;
     for (list.items) |tc| {
@@ -104,14 +112,23 @@ pub fn main() !void {
         defer arena.deinit();
 
         const res = wpt.run(&arena, apis, wpt_dir, tc, &loader) catch |err| {
-            std.debug.print("FAIL\t{s}\n{any}\n", .{ tc, err });
+            const suite = try Suite.init(alloc, tc, false, @errorName(err), null);
+            try results.append(suite);
+
+            if (!json) {
+                std.debug.print("FAIL\t{s}\t{}\n", .{ tc, err });
+            }
             failures += 1;
             continue;
         };
         // no need to call res.deinit() thanks to the arena allocator.
 
         const suite = try Suite.init(alloc, tc, res.success, res.result, res.stack);
-        defer suite.deinit();
+        try results.append(suite);
+
+        if (json) {
+            continue;
+        }
 
         if (!suite.pass) {
             std.debug.print("Fail\t{s}\n{s}\n", .{ suite.name, suite.fmtMessage() });
@@ -128,7 +145,61 @@ pub fn main() !void {
         }
     }
 
-    if (failures > 0) {
+    if (json) {
+        const Case = struct {
+            pass: bool,
+            name: []const u8,
+            message: ?[]const u8,
+        };
+        const Test = struct {
+            pass: bool,
+            crash: bool = false, // TODO
+            name: []const u8,
+            cases: []Case,
+        };
+
+        var output = std.ArrayList(Test).init(alloc);
+        defer output.deinit();
+
+        for (results.items) |suite| {
+            var cases = std.ArrayList(Case).init(alloc);
+            defer cases.deinit();
+
+            if (suite.cases) |scases| {
+                for (scases) |case| {
+                    try cases.append(Case{
+                        .pass = case.pass,
+                        .name = case.name,
+                        .message = case.message,
+                    });
+                }
+            } else {
+                // no cases, generate a fake one
+                try cases.append(Case{
+                    .pass = suite.pass,
+                    .name = suite.name,
+                    .message = suite.stack orelse suite.message,
+                });
+            }
+
+            try output.append(Test{
+                .pass = suite.pass,
+                .name = suite.name,
+                .cases = try cases.toOwnedSlice(),
+            });
+        }
+
+        defer {
+            for (output.items) |suite| {
+                alloc.free(suite.cases);
+            }
+        }
+
+        try std.json.stringify(output.items, .{ .whitespace = .indent_2 }, std.io.getStdOut().writer());
+        std.os.exit(0);
+    }
+
+    if (!json and failures > 0) {
         std.debug.print("{d}/{d} tests suites failures\n", .{ failures, run });
         std.os.exit(1);
     }

From 6e0491e3b03e519a300aaf2cff1f122f6d2237b8 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Wed, 29 Nov 2023 15:41:32 +0100
Subject: [PATCH 08/14] wpt: remove output

---
 src/main_wpt.zig | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index b4ed2031..b6394a91 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -30,8 +30,6 @@ pub fn main() !void {
     // generate APIs
     const apis = comptime jsruntime.compile(DOM.Interfaces);
 
-    std.debug.print("Running WPT test suite\n", .{});
-
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
     defer _ = gpa.deinit();
     const alloc = gpa.allocator();

From 8b8d7b3617d59f8d6b8fbaa1ca223bdf7d817919 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Thu, 30 Nov 2023 15:29:16 +0100
Subject: [PATCH 09/14] wpt: add safe test run in separate process

---
 src/main_wpt.zig | 91 ++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 84 insertions(+), 7 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index b6394a91..e7ff8f4d 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -17,6 +17,8 @@ const usage =
     \\
     \\  -h, --help       Print this help message and exit.
     \\  --json           result is formatted in JSON.
+    \\  --safe           each test is run in a separate process.
+    \\  --summary        print a summary result. Incompatible w/ --json
     \\
 ;
 
@@ -41,6 +43,8 @@ pub fn main() !void {
     const execname = args.next().?;
 
     var json = false;
+    var safe = false;
+    var summary = false;
 
     var filter = std.ArrayList([]const u8).init(alloc);
     defer filter.deinit();
@@ -54,16 +58,26 @@ pub fn main() !void {
             json = true;
             continue;
         }
+        if (std.mem.eql(u8, "--safe", arg)) {
+            safe = true;
+            continue;
+        }
+        if (std.mem.eql(u8, "--summary", arg)) {
+            summary = true;
+            continue;
+        }
         try filter.append(arg[0..]);
     }
 
-    // initialize VM JS lib.
-    const vm = jsruntime.VM.init();
-    defer vm.deinit();
-
-    // prepare libraries to load on each test case.
-    var loader = FileLoader.init(alloc, wpt_dir);
-    defer loader.deinit();
+    // both json and summary are incompatible.
+    if (summary and json) {
+        try std.io.getStdErr().writer().print("--json and --summary are incompatible\n", .{});
+        std.os.exit(1);
+    }
+    // summary is available in safe mode only.
+    if (summary) {
+        safe = true;
+    }
 
     // browse the dir to get the tests dynamically.
     var list = std.ArrayList([]const u8).init(alloc);
@@ -75,6 +89,61 @@ pub fn main() !void {
         list.deinit();
     }
 
+    if (safe) {
+        for (list.items) |tc| {
+            // TODO use std.ChildProcess.run after next zig upgrade.
+            var child = std.ChildProcess.init(&.{ execname, tc }, alloc);
+            child.stdin_behavior = .Ignore;
+            child.stdout_behavior = .Pipe;
+            child.stderr_behavior = .Pipe;
+
+            var stdout = std.ArrayList(u8).init(alloc);
+            var stderr = std.ArrayList(u8).init(alloc);
+            defer {
+                stdout.deinit();
+                stderr.deinit();
+            }
+
+            try child.spawn();
+            try child.collectOutput(&stdout, &stderr, 1024 * 1024);
+            const term = try child.wait();
+
+            const Result = enum {
+                pass,
+                fail,
+                crash,
+            };
+
+            var result: Result = undefined;
+            switch (term) {
+                .Exited => |v| {
+                    if (v == 0) {
+                        result = .pass;
+                    } else {
+                        result = .fail;
+                    }
+                },
+                .Signal => result = .crash,
+                .Stopped => result = .crash,
+                .Unknown => result = .crash,
+            }
+
+            if (summary) {
+                switch (result) {
+                    .pass => std.debug.print("Pass", .{}),
+                    .fail => std.debug.print("Fail", .{}),
+                    .crash => std.debug.print("Crash", .{}),
+                }
+                std.debug.print("\t{s}\n", .{tc});
+                continue;
+            }
+
+            // std.debug.print("{s}\n", .{stderr.items});
+        }
+
+        return;
+    }
+
     var results = std.ArrayList(Suite).init(alloc);
     defer {
         for (results.items) |suite| {
@@ -83,6 +152,14 @@ pub fn main() !void {
         results.deinit();
     }
 
+    // initialize VM JS lib.
+    const vm = jsruntime.VM.init();
+    defer vm.deinit();
+
+    // prepare libraries to load on each test case.
+    var loader = FileLoader.init(alloc, wpt_dir);
+    defer loader.deinit();
+
     var run: usize = 0;
     var failures: usize = 0;
     for (list.items) |tc| {

From 07270fbb19f6bdc2e42ca0d1e69b9f7257c00d14 Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Thu, 30 Nov 2023 15:41:43 +0100
Subject: [PATCH 10/14] wpt: extract filter on its own func

---
 src/main_wpt.zig | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index e7ff8f4d..5fc8f815 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -91,6 +91,10 @@ pub fn main() !void {
 
     if (safe) {
         for (list.items) |tc| {
+            if (!shouldRun(filter.items, tc)) {
+                continue;
+            }
+
             // TODO use std.ChildProcess.run after next zig upgrade.
             var child = std.ChildProcess.init(&.{ execname, tc }, alloc);
             child.stdin_behavior = .Ignore;
@@ -138,7 +142,7 @@ pub fn main() !void {
                 continue;
             }
 
-            // std.debug.print("{s}\n", .{stderr.items});
+            std.debug.print("{s}\n", .{stderr.items});
         }
 
         return;
@@ -163,21 +167,8 @@ pub fn main() !void {
     var run: usize = 0;
     var failures: usize = 0;
     for (list.items) |tc| {
-        if (filter.items.len > 0) {
-            var match = false;
-            for (filter.items) |f| {
-                if (std.mem.startsWith(u8, tc, f)) {
-                    match = true;
-                    break;
-                }
-                if (std.mem.endsWith(u8, tc, f)) {
-                    match = true;
-                    break;
-                }
-            }
-            if (!match) {
-                continue;
-            }
+        if (!shouldRun(filter.items, tc)) {
+            continue;
         }
 
         run += 1;
@@ -279,3 +270,19 @@ pub fn main() !void {
         std.os.exit(1);
     }
 }
+
+fn shouldRun(filter: [][]const u8, tc: []const u8) bool {
+    if (filter.len == 0) {
+        return true;
+    }
+
+    for (filter) |f| {
+        if (std.mem.startsWith(u8, tc, f)) {
+            return true;
+        }
+        if (std.mem.endsWith(u8, tc, f)) {
+            return true;
+        }
+    }
+    return false;
+}

From 29f5f15c504559d34d58c34410462709f0391ecf Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Thu, 30 Nov 2023 15:46:32 +0100
Subject: [PATCH 11/14] wpt: move safe run in its own func

---
 src/main_wpt.zig | 120 +++++++++++++++++++++++++----------------------
 1 file changed, 64 insertions(+), 56 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 5fc8f815..227560c6 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -90,62 +90,7 @@ pub fn main() !void {
     }
 
     if (safe) {
-        for (list.items) |tc| {
-            if (!shouldRun(filter.items, tc)) {
-                continue;
-            }
-
-            // TODO use std.ChildProcess.run after next zig upgrade.
-            var child = std.ChildProcess.init(&.{ execname, tc }, alloc);
-            child.stdin_behavior = .Ignore;
-            child.stdout_behavior = .Pipe;
-            child.stderr_behavior = .Pipe;
-
-            var stdout = std.ArrayList(u8).init(alloc);
-            var stderr = std.ArrayList(u8).init(alloc);
-            defer {
-                stdout.deinit();
-                stderr.deinit();
-            }
-
-            try child.spawn();
-            try child.collectOutput(&stdout, &stderr, 1024 * 1024);
-            const term = try child.wait();
-
-            const Result = enum {
-                pass,
-                fail,
-                crash,
-            };
-
-            var result: Result = undefined;
-            switch (term) {
-                .Exited => |v| {
-                    if (v == 0) {
-                        result = .pass;
-                    } else {
-                        result = .fail;
-                    }
-                },
-                .Signal => result = .crash,
-                .Stopped => result = .crash,
-                .Unknown => result = .crash,
-            }
-
-            if (summary) {
-                switch (result) {
-                    .pass => std.debug.print("Pass", .{}),
-                    .fail => std.debug.print("Fail", .{}),
-                    .crash => std.debug.print("Crash", .{}),
-                }
-                std.debug.print("\t{s}\n", .{tc});
-                continue;
-            }
-
-            std.debug.print("{s}\n", .{stderr.items});
-        }
-
-        return;
+        return try runSafe(alloc, execname, summary, list.items, filter.items);
     }
 
     var results = std.ArrayList(Suite).init(alloc);
@@ -286,3 +231,66 @@ fn shouldRun(filter: [][]const u8, tc: []const u8) bool {
     }
     return false;
 }
+
+fn runSafe(
+    alloc: std.mem.Allocator,
+    execname: []const u8,
+    summary: bool,
+    testcases: [][]const u8,
+    filter: [][]const u8,
+) !void {
+    const Result = enum {
+        pass,
+        fail,
+        crash,
+    };
+
+    for (testcases) |tc| {
+        if (!shouldRun(filter, tc)) {
+            continue;
+        }
+
+        // TODO use std.ChildProcess.run after next zig upgrade.
+        var child = std.ChildProcess.init(&.{ execname, tc }, alloc);
+        child.stdin_behavior = .Ignore;
+        child.stdout_behavior = .Pipe;
+        child.stderr_behavior = .Pipe;
+
+        var stdout = std.ArrayList(u8).init(alloc);
+        var stderr = std.ArrayList(u8).init(alloc);
+        defer {
+            stdout.deinit();
+            stderr.deinit();
+        }
+
+        try child.spawn();
+        try child.collectOutput(&stdout, &stderr, 1024 * 1024);
+        const term = try child.wait();
+
+        var result: Result = undefined;
+        switch (term) {
+            .Exited => |v| {
+                if (v == 0) {
+                    result = .pass;
+                } else {
+                    result = .fail;
+                }
+            },
+            .Signal => result = .crash,
+            .Stopped => result = .crash,
+            .Unknown => result = .crash,
+        }
+
+        if (summary) {
+            switch (result) {
+                .pass => std.debug.print("Pass", .{}),
+                .fail => std.debug.print("Fail", .{}),
+                .crash => std.debug.print("Crash", .{}),
+            }
+            std.debug.print("\t{s}\n", .{tc});
+            continue;
+        }
+
+        std.debug.print("{s}\n", .{stderr.items});
+    }
+}

From 73e80a567842cb1c5e7661f6207238c5bf57c30c Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Thu, 30 Nov 2023 15:58:51 +0100
Subject: [PATCH 12/14] wpt: use enum for output

---
 src/main_wpt.zig | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 227560c6..32177747 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -42,9 +42,8 @@ pub fn main() !void {
     // get the exec name.
     const execname = args.next().?;
 
-    var json = false;
+    var out: Out = .text;
     var safe = false;
-    var summary = false;
 
     var filter = std.ArrayList([]const u8).init(alloc);
     defer filter.deinit();
@@ -55,7 +54,7 @@ pub fn main() !void {
             std.os.exit(0);
         }
         if (std.mem.eql(u8, "--json", arg)) {
-            json = true;
+            out = .json;
             continue;
         }
         if (std.mem.eql(u8, "--safe", arg)) {
@@ -63,19 +62,14 @@ pub fn main() !void {
             continue;
         }
         if (std.mem.eql(u8, "--summary", arg)) {
-            summary = true;
+            out = .summary;
             continue;
         }
         try filter.append(arg[0..]);
     }
 
-    // both json and summary are incompatible.
-    if (summary and json) {
-        try std.io.getStdErr().writer().print("--json and --summary are incompatible\n", .{});
-        std.os.exit(1);
-    }
     // summary is available in safe mode only.
-    if (summary) {
+    if (out == .summary) {
         safe = true;
     }
 
@@ -90,7 +84,7 @@ pub fn main() !void {
     }
 
     if (safe) {
-        return try runSafe(alloc, execname, summary, list.items, filter.items);
+        return try runSafe(alloc, execname, out, list.items, filter.items);
     }
 
     var results = std.ArrayList(Suite).init(alloc);
@@ -126,7 +120,7 @@ pub fn main() !void {
             const suite = try Suite.init(alloc, tc, false, @errorName(err), null);
             try results.append(suite);
 
-            if (!json) {
+            if (out == .text) {
                 std.debug.print("FAIL\t{s}\t{}\n", .{ tc, err });
             }
             failures += 1;
@@ -137,7 +131,7 @@ pub fn main() !void {
         const suite = try Suite.init(alloc, tc, res.success, res.result, res.stack);
         try results.append(suite);
 
-        if (json) {
+        if (out == .json) {
             continue;
         }
 
@@ -156,7 +150,7 @@ pub fn main() !void {
         }
     }
 
-    if (json) {
+    if (out == .json) {
         const Case = struct {
             pass: bool,
             name: []const u8,
@@ -210,7 +204,7 @@ pub fn main() !void {
         std.os.exit(0);
     }
 
-    if (!json and failures > 0) {
+    if (out == .text and failures > 0) {
         std.debug.print("{d}/{d} tests suites failures\n", .{ failures, run });
         std.os.exit(1);
     }
@@ -232,10 +226,16 @@ fn shouldRun(filter: [][]const u8, tc: []const u8) bool {
     return false;
 }
 
+const Out = enum {
+    json,
+    summary,
+    text,
+};
+
 fn runSafe(
     alloc: std.mem.Allocator,
     execname: []const u8,
-    summary: bool,
+    out: Out,
     testcases: [][]const u8,
     filter: [][]const u8,
 ) !void {
@@ -281,7 +281,7 @@ fn runSafe(
             .Unknown => result = .crash,
         }
 
-        if (summary) {
+        if (out == .summary) {
             switch (result) {
                 .pass => std.debug.print("Pass", .{}),
                 .fail => std.debug.print("Fail", .{}),

From ee8628acfd47e356cb78f5388cd32144c4d56d1d Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Thu, 30 Nov 2023 17:24:09 +0100
Subject: [PATCH 13/14] wpt: handle json outpout w/ safe run

---
 src/main_wpt.zig | 90 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 66 insertions(+), 24 deletions(-)

diff --git a/src/main_wpt.zig b/src/main_wpt.zig
index 32177747..f6d56244 100644
--- a/src/main_wpt.zig
+++ b/src/main_wpt.zig
@@ -22,6 +22,13 @@ const usage =
     \\
 ;
 
+// Out list all the ouputs handled by WPT.
+const Out = enum {
+    json,
+    summary,
+    text,
+};
+
 // TODO For now the WPT tests run is specific to WPT.
 // It manually load js framwork libs, and run the first script w/ js content in
 // the HTML page.
@@ -151,18 +158,6 @@ pub fn main() !void {
     }
 
     if (out == .json) {
-        const Case = struct {
-            pass: bool,
-            name: []const u8,
-            message: ?[]const u8,
-        };
-        const Test = struct {
-            pass: bool,
-            crash: bool = false, // TODO
-            name: []const u8,
-            cases: []Case,
-        };
-
         var output = std.ArrayList(Test).init(alloc);
         defer output.deinit();
 
@@ -210,6 +205,20 @@ pub fn main() !void {
     }
 }
 
+// struct used for JSON output.
+const Case = struct {
+    pass: bool,
+    name: []const u8,
+    message: ?[]const u8,
+};
+const Test = struct {
+    pass: bool,
+    crash: bool = false,
+    name: []const u8,
+    cases: []Case,
+};
+
+// shouldRun return true if the test should be run accroding to the given filters.
 fn shouldRun(filter: [][]const u8, tc: []const u8) bool {
     if (filter.len == 0) {
         return true;
@@ -226,42 +235,50 @@ fn shouldRun(filter: [][]const u8, tc: []const u8) bool {
     return false;
 }
 
-const Out = enum {
-    json,
-    summary,
-    text,
-};
-
+// runSafe rune each test cae in a separate child process to detect crashes.
 fn runSafe(
-    alloc: std.mem.Allocator,
+    allocator: std.mem.Allocator,
     execname: []const u8,
     out: Out,
     testcases: [][]const u8,
     filter: [][]const u8,
 ) !void {
+    var arena = std.heap.ArenaAllocator.init(allocator);
+    defer arena.deinit();
+
+    const alloc = arena.allocator();
+
     const Result = enum {
         pass,
         fail,
         crash,
     };
 
+    var argv = try std.ArrayList([]const u8).initCapacity(alloc, 3);
+    defer argv.deinit();
+    argv.appendAssumeCapacity(execname);
+    if (out == .json) {
+        argv.appendAssumeCapacity("--json");
+    }
+
+    var output = std.ArrayList(Test).init(alloc);
+
     for (testcases) |tc| {
         if (!shouldRun(filter, tc)) {
             continue;
         }
 
+        argv.appendAssumeCapacity(tc);
+        defer _ = argv.pop();
+
         // TODO use std.ChildProcess.run after next zig upgrade.
-        var child = std.ChildProcess.init(&.{ execname, tc }, alloc);
+        var child = std.ChildProcess.init(argv.items, alloc);
         child.stdin_behavior = .Ignore;
         child.stdout_behavior = .Pipe;
         child.stderr_behavior = .Pipe;
 
         var stdout = std.ArrayList(u8).init(alloc);
         var stderr = std.ArrayList(u8).init(alloc);
-        defer {
-            stdout.deinit();
-            stderr.deinit();
-        }
 
         try child.spawn();
         try child.collectOutput(&stdout, &stderr, 1024 * 1024);
@@ -291,6 +308,31 @@ fn runSafe(
             continue;
         }
 
+        if (out == .json) {
+            if (result == .crash) {
+                var cases = [_]Case{.{
+                    .pass = false,
+                    .name = "crash",
+                    .message = stderr.items,
+                }};
+                try output.append(Test{
+                    .pass = false,
+                    .crash = true,
+                    .name = tc,
+                    .cases = cases[0..1],
+                });
+                continue;
+            }
+
+            const jp = try std.json.parseFromSlice([]Test, alloc, stdout.items, .{});
+            try output.appendSlice(jp.value);
+            continue;
+        }
+
         std.debug.print("{s}\n", .{stderr.items});
     }
+
+    if (out == .json) {
+        try std.json.stringify(output.items, .{ .whitespace = .indent_2 }, std.io.getStdOut().writer());
+    }
 }

From 843c404b493d05a8d28845af6952172368b015bb Mon Sep 17 00:00:00 2001
From: Pierre Tachoire <pierre@lightpanda.io>
Date: Thu, 30 Nov 2023 17:25:02 +0100
Subject: [PATCH 14/14] ci: use safe run for WPT ci tests

---
 .github/workflows/wpt.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml
index dbd2f6c1..9d25847d 100644
--- a/.github/workflows/wpt.yml
+++ b/.github/workflows/wpt.yml
@@ -71,7 +71,7 @@ jobs:
           ln -s /usr/local/lib/netsurf/lib vendor/netsurf/lib
           ln -s /usr/local/lib/netsurf/include vendor/netsurf/include
 
-      - run: zig build wpt -Dengine=v8
+      - run: zig build wpt -Dengine=v8 -- --safe --summary
 
         # For now WPT tests doesn't pass at all.
         # We accept then to continue the job on failure.