From 486927c889b906239815c8c6b2c804de91ab66c7 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 28 Nov 2023 17:15:04 +0100 Subject: [PATCH] wpt: refacto results report --- src/main_wpt.zig | 25 ++- src/wpt/testcase.zig | 248 +++++++++++++++++++++++ tests/wpt/resources/testharnessreport.js | 4 +- 3 files changed, 265 insertions(+), 12 deletions(-) create mode 100644 src/wpt/testcase.zig diff --git a/src/main_wpt.zig b/src/main_wpt.zig index 0f8ae0e4..36831cbc 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -2,6 +2,7 @@ const std = @import("std"); const parser = @import("netsurf.zig"); const jsruntime = @import("jsruntime"); +const wpt = @import("wpt/testcase.zig"); const TPL = jsruntime.TPL; const Env = jsruntime.Env; @@ -159,22 +160,26 @@ pub fn main() !void { }; // no need to call res.deinit() thanks to the arena allocator. - if (!res.success) { - std.debug.print("FAIL\t{s}\n{s}\n", .{ tc, res.stack orelse res.result }); + const suite = try wpt.Suite.init(arena.allocator(), tc, res.success, res.result, res.stack); + defer suite.deinit(); + + if (!suite.pass) { + std.debug.print("Fail\t{s}\n{s}\n", .{ suite.name, suite.fmtMessage() }); failures += 1; - continue; - } - if (!std.mem.eql(u8, res.result, "Pass")) { - std.debug.print("FAIL\t{s}\n{s}\n", .{ tc, res.stack orelse res.result }); - failures += 1; - continue; + } else { + std.debug.print("Pass\t{s}\n", .{suite.name}); } - std.debug.print("PASS\t{s}\n", .{tc}); + // display details + if (suite.cases) |cases| { + for (cases) |case| { + std.debug.print("\t{s}\t{s}\t{s}\n", .{ case.fmtStatus(), case.name, case.fmtMessage() }); + } + } } if (failures > 0) { - std.debug.print("{d}/{d} tests failures\n", .{ failures, run }); + std.debug.print("{d}/{d} tests suites failures\n", .{ failures, run }); std.os.exit(1); } } diff --git a/src/wpt/testcase.zig b/src/wpt/testcase.zig new file mode 100644 index 00000000..c1a7e56e --- /dev/null +++ b/src/wpt/testcase.zig @@ -0,0 +1,248 @@ +const std = @import("std"); +const testing = std.testing; + +pub const Case = struct { + pass: bool, + name: []const u8, + + message: ?[]const u8, + + fn init(alloc: std.mem.Allocator, name: []const u8, status: []const u8, message: []const u8) !Case { + var case = Case{ + .pass = std.mem.eql(u8, "Pass", status), + .name = try alloc.dupe(u8, name), + .message = null, + }; + + if (message.len > 0) { + case.message = try alloc.dupe(u8, message); + } + + return case; + } + + fn deinit(self: Case, alloc: std.mem.Allocator) void { + alloc.free(self.name); + + if (self.message) |msg| { + alloc.free(msg); + } + } + + pub fn fmtStatus(self: Case) []const u8 { + if (self.pass) { + return "Pass"; + } + return "Fail"; + } + + pub fn fmtMessage(self: Case) []const u8 { + if (self.message) |v| { + return v; + } + return ""; + } +}; + +pub const Suite = struct { + alloc: std.mem.Allocator, + pass: bool, + name: []const u8, + message: ?[]const u8, + stack: ?[]const u8, + cases: ?[]Case, + + // caller owns the wpt.Suite. + // owner must call deinit(). + pub fn init(alloc: std.mem.Allocator, name: []const u8, pass: bool, res: []const u8, stack: ?[]const u8) !Suite { + var suite = Suite{ + .alloc = alloc, + .pass = false, + .name = try alloc.dupe(u8, name), + .message = null, + .stack = null, + .cases = null, + }; + + // handle JS error. + if (!pass) { + suite.message = try alloc.dupe(u8, res); + if (stack) |st| { + suite.stack = try alloc.dupe(u8, st); + } + + return suite; + } + + // no JS error, let's try to parse the result. + suite.pass = true; + + // special case: the result contains only "Pass" message + if (std.mem.eql(u8, "Pass", res)) { + return suite; + } + + var cases = std.ArrayList(Case).init(alloc); + defer cases.deinit(); + + var lines = std.mem.splitScalar(u8, res, '\n'); + while (lines.next()) |line| { + if (line.len == 0) { + break; + } + var fields = std.mem.splitScalar(u8, line, '|'); + var ff: [3][]const u8 = .{ "", "", "" }; + var i: u8 = 0; + while (fields.next()) |field| { + if (i >= 3) { + suite.pass = false; + suite.message = try alloc.dupe(u8, res); + return suite; + } + + ff[i] = field; + i += 1; + } + + // invalid output format + if (i != 2 and i != 3) { + suite.pass = false; + suite.message = try alloc.dupe(u8, res); + return suite; + } + + const case = try Case.init(alloc, ff[0], ff[1], ff[2]); + if (!case.pass) { + suite.pass = false; + } + + try cases.append(case); + } + + suite.cases = try cases.toOwnedSlice(); + + return suite; + } + + pub fn deinit(self: Suite) void { + self.alloc.free(self.name); + + if (self.stack) |stack| { + self.alloc.free(stack); + } + + if (self.message) |res| { + self.alloc.free(res); + } + + if (self.cases) |cases| { + for (cases) |case| { + case.deinit(self.alloc); + } + self.alloc.free(cases); + } + } + + pub fn fmtMessage(self: Suite) []const u8 { + if (self.message) |v| { + return v; + } + if (self.stack) |v| { + return v; + } + return ""; + } +}; + +test "success test case" { + const alloc = testing.allocator; + + const Res = struct { + pass: bool, + result: []const u8, + }; + + const res = Res{ + .pass = true, + .result = + \\Empty string as a name for Document.getElementsByTagName|Pass + \\Empty string as a name for Element.getElementsByTagName|Pass + \\ + , + }; + + const suite = Suite.init(alloc, "foo", res.pass, res.result, null) catch unreachable; // TODO + defer suite.deinit(); + + try testing.expect(suite.pass == true); + try testing.expect(suite.cases != null); + try testing.expect(suite.cases.?.len == 2); + try testing.expect(suite.cases.?[0].pass == true); + try testing.expect(suite.cases.?[1].pass == true); +} + +test "failed test case" { + const alloc = testing.allocator; + + const Res = struct { + pass: bool, + result: []const u8, + }; + + const res = Res{ + .pass = true, + .result = + \\Empty string as a name for Document.getElementsByTagName|Pass + \\Empty string as a name for Element.getElementsByTagName|Fail|div.getElementsByTagName is not a function + \\ + , + }; + + const suite = Suite.init(alloc, "foo", res.pass, res.result, null) catch unreachable; // TODO + defer suite.deinit(); + + try testing.expect(suite.pass == false); + try testing.expect(suite.cases != null); + try testing.expect(suite.cases.?.len == 2); + try testing.expect(suite.cases.?[0].pass == true); + try testing.expect(suite.cases.?[1].pass == false); +} + +test "invalid result" { + const alloc = testing.allocator; + + const Res = struct { + pass: bool, + result: []const u8, + }; + + const res = Res{ + .pass = true, + .result = + \\this is|an|invalid|result + , + }; + + const suite = Suite.init(alloc, "foo", res.pass, res.result, null) catch unreachable; // TODO + defer suite.deinit(); + + try testing.expect(suite.pass == false); + try testing.expect(suite.message != null); + try testing.expect(std.mem.eql(u8, res.result, suite.message.?)); + try testing.expect(suite.cases == null); + + const res2 = Res{ + .pass = true, + .result = + \\this is an invalid result. + , + }; + + const suite2 = Suite.init(alloc, "foo", res2.pass, res2.result, null) catch unreachable; // TODO + defer suite2.deinit(); + + try testing.expect(suite2.pass == false); + try testing.expect(suite2.message != null); + try testing.expect(std.mem.eql(u8, res2.result, suite2.message.?)); + try testing.expect(suite2.cases == null); +} diff --git a/tests/wpt/resources/testharnessreport.js b/tests/wpt/resources/testharnessreport.js index 58c36ac7..c9817431 100644 --- a/tests/wpt/resources/testharnessreport.js +++ b/tests/wpt/resources/testharnessreport.js @@ -18,9 +18,9 @@ add_completion_callback(function (tests, status) { var log = ""; for (var i = 0; i < tests.length; i++) { const test = tests[i]; - log += test.name+"\t"+test.format_status()+"\t"; + log += test.name+"|"+test.format_status(); if (test.message != null) { - log += test.message; + log += "|"+test.message; } log += "\n";