Merge pull request #1341 from lightpanda-io/test-bench

tests: re-enable metrics JSON output

.github/workflows/zig-test.yml

@@ -103,7 +103,7 @@ jobs:
       - uses: ./.github/actions/install
 
       - name: zig build test
-        run: zig build  -Dprebuilt_v8_path=v8/libc_v8.a test -- --json > bench.json
+        run: METRICS=true zig build -Dprebuilt_v8_path=v8/libc_v8.a test > bench.json
 
       - name: write commit
         run: |

src/test_runner.zig

@@ -25,7 +25,8 @@ const BORDER = "=" ** 80;
 
 // use in custom panic handler
 var current_test: ?[]const u8 = null;
-pub var ran_webapi_test = false;
+pub var v8_peak_memory: usize = 0;
+pub var tracking_allocator: Allocator = undefined;
 
 var RUNNER: *Runner = undefined;
 
@@ -37,12 +38,14 @@ pub fn main() !void {
     var arena = std.heap.ArenaAllocator.init(gpa.allocator());
     defer arena.deinit();
 
+    var ta = TrackingAllocator.init(gpa.allocator());
+    tracking_allocator = ta.allocator();
+
     const allocator = fba.allocator();
 
     const env = Env.init(allocator);
-    defer env.deinit(allocator);
 
-    var runner = Runner.init(allocator, arena.allocator(), env);
+    var runner = Runner.init(allocator, arena.allocator(), &ta, env);
     RUNNER = &runner;
     try runner.run();
 }
@@ -50,13 +53,15 @@ pub fn main() !void {
 const Runner = struct {
     env: Env,
     allocator: Allocator,
+    ta: *TrackingAllocator,
 
     // per-test arena, used for collecting substests
     arena: Allocator,
     subtests: std.ArrayListUnmanaged([]const u8),
 
-    fn init(allocator: Allocator, arena: Allocator, env: Env) Runner {
+    fn init(allocator: Allocator, arena: Allocator, ta: *TrackingAllocator, env: Env) Runner {
         return .{
+            .ta = ta,
             .env = env,
             .arena = arena,
             .subtests = .empty,
@@ -72,6 +77,8 @@ const Runner = struct {
         var fail: usize = 0;
         var skip: usize = 0;
         var leak: usize = 0;
+        // track all tests duration, excluding setup and teardown.
+        var ns_duration: u64 = 0;
 
         Printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line
 
@@ -138,6 +145,7 @@ const Runner = struct {
             }
 
             const ns_taken = slowest.endTiming(friendly_name, is_unnamed_test);
+            ns_duration += ns_taken;
 
             if (std.testing.allocator_instance.deinit() == .leak) {
                 leak += 1;
@@ -204,6 +212,28 @@ const Runner = struct {
         Printer.fmt("\n", .{});
         try slowest.display();
         Printer.fmt("\n", .{});
+
+        // stats
+        if (self.env.metrics) {
+            var stdout = std.fs.File.stdout();
+            var writer = stdout.writer(&.{});
+            const stats = self.ta.stats();
+            try std.json.Stringify.value(&.{
+                .{ .name = "browser", .bench = .{
+                    .duration = ns_duration,
+                    .alloc_nb = stats.allocation_count,
+                    .realloc_nb = stats.reallocation_count,
+                    .alloc_size = stats.allocated_bytes,
+                } },
+                .{ .name = "v8", .bench = .{
+                    .duration = ns_duration,
+                    .alloc_nb = 0,
+                    .realloc_nb = 0,
+                    .alloc_size = v8_peak_memory,
+                } },
+            }, .{ .whitespace = .indent_2 }, &writer.interface);
+        }
+
         std.posix.exit(if (fail == 0) 0 else 1);
     }
 };
@@ -323,6 +353,7 @@ const Env = struct {
     fail_first: bool,
     filter: ?[]const u8,
     subfilter: ?[]const u8,
+    metrics: bool,
 
     fn init(allocator: Allocator) Env {
         const full_filter = readEnv(allocator, "TEST_FILTER");
@@ -332,6 +363,7 @@ const Env = struct {
             .fail_first = readEnvBool(allocator, "TEST_FAIL_FIRST", false),
             .filter = filter,
             .subfilter = subfilter,
+            .metrics = readEnvBool(allocator, "METRICS", false),
         };
     }
 
@@ -403,3 +435,90 @@ fn isSetup(t: std.builtin.TestFn) bool {
 fn isTeardown(t: std.builtin.TestFn) bool {
     return std.mem.endsWith(u8, t.name, "tests:afterAll");
 }
+
+pub const TrackingAllocator = struct {
+    parent_allocator: Allocator,
+    free_count: usize = 0,
+    allocated_bytes: usize = 0,
+    allocation_count: usize = 0,
+    reallocation_count: usize = 0,
+
+    const Stats = struct {
+        allocated_bytes: usize,
+        allocation_count: usize,
+        reallocation_count: usize,
+    };
+
+    fn init(parent_allocator: Allocator) TrackingAllocator {
+        return .{
+            .parent_allocator = parent_allocator,
+        };
+    }
+
+    pub fn stats(self: *const TrackingAllocator) Stats {
+        return .{
+            .allocated_bytes = self.allocated_bytes,
+            .allocation_count = self.allocation_count,
+            .reallocation_count = self.reallocation_count,
+        };
+    }
+
+    pub fn allocator(self: *TrackingAllocator) Allocator {
+        return .{ .ptr = self, .vtable = &.{
+            .alloc = alloc,
+            .resize = resize,
+            .free = free,
+            .remap = remap,
+        } };
+    }
+
+    fn alloc(
+        ctx: *anyopaque,
+        len: usize,
+        alignment: std.mem.Alignment,
+        return_address: usize,
+    ) ?[*]u8 {
+        const self: *TrackingAllocator = @ptrCast(@alignCast(ctx));
+        const result = self.parent_allocator.rawAlloc(len, alignment, return_address);
+        self.allocation_count += 1;
+        self.allocated_bytes += len;
+        return result;
+    }
+
+    fn resize(
+        ctx: *anyopaque,
+        old_mem: []u8,
+        alignment: std.mem.Alignment,
+        new_len: usize,
+        ra: usize,
+    ) bool {
+        const self: *TrackingAllocator = @ptrCast(@alignCast(ctx));
+        const result = self.parent_allocator.rawResize(old_mem, alignment, new_len, ra);
+        self.reallocation_count += 1; // TODO: only if result is not null?
+        return result;
+    }
+
+    fn free(
+        ctx: *anyopaque,
+        old_mem: []u8,
+        alignment: std.mem.Alignment,
+        ra: usize,
+    ) void {
+        const self: *TrackingAllocator = @ptrCast(@alignCast(ctx));
+        self.parent_allocator.rawFree(old_mem, alignment, ra);
+        self.free_count += 1;
+    }
+
+    fn remap(
+        ctx: *anyopaque,
+        memory: []u8,
+        alignment: std.mem.Alignment,
+        new_len: usize,
+        ret_addr: usize,
+    ) ?[*]u8 {
+        const self: *TrackingAllocator = @ptrCast(@alignCast(ctx));
+        const result = self.parent_allocator.rawRemap(memory, alignment, new_len, ret_addr);
+        self.reallocation_count += 1; // TODO: only if result is not null?
+        return result;
+    }
+};

src/testing.zig

@@ -326,7 +326,6 @@ fn isJsonValue(a: std.json.Value, b: std.json.Value) bool {
     }
 }
 
-var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init;
 pub var test_app: *App = undefined;
 pub var test_browser: Browser = undefined;
 pub var test_session: *Session = undefined;
@@ -446,7 +445,7 @@ test "tests:beforeAll" {
     log.opts.level = .warn;
     log.opts.format = .pretty;
 
-    test_app = try App.init(gpa.allocator(), .{
+    test_app = try App.init(@import("root").tracking_allocator, .{
         .run_mode = .serve,
         .tls_verify_host = false,
         .user_agent = "User-Agent: Lightpanda/1.0 internal-tester",
@@ -485,6 +484,8 @@ test "tests:afterAll" {
         server.deinit();
     }
 
+    @import("root").v8_peak_memory = test_browser.env.isolate.getHeapStatistics().total_physical_size;
+
     test_browser.deinit();
     test_app.deinit();
 }