Use per-cdp connection HttpClient

src/Server.zig

@@ -30,6 +30,8 @@ const log = @import("log.zig");
 const App = @import("App.zig");
 const Config = @import("Config.zig");
 const CDP = @import("cdp/cdp.zig").CDP;
+const Http = @import("http/Http.zig");
+const HttpClient = @import("http/Client.zig");
 
 const Server = @This();
 
@@ -147,19 +149,22 @@ fn readLoop(self: *Server, socket: posix.socket_t, timeout_ms: u32) !void {
     client.* = try Client.init(socket, self);
     defer client.deinit();
 
-    var http = &self.app.http;
+    client.http.cdp_client = .{
-    http.addCDPClient(.{
+        .socket = client.socket,
-        .socket = socket,
         .ctx = client,
         .blocking_read_start = Client.blockingReadStart,
         .blocking_read = Client.blockingRead,
         .blocking_read_end = Client.blockingReadStop,
-    });
+    };
-    defer http.removeCDPClient();
+    defer client.http.cdp_client = null;
 
     lp.assert(client.mode == .http, "Server.readLoop invalid mode", .{});
     while (true) {
-        if (http.poll(timeout_ms) != .cdp_socket) {
+        const status = client.http.tick(timeout_ms) catch |err| {
+            log.err(.app, "http tick", .{ .err = err });
+            return;
+        };
+        if (status != .cdp_socket) {
             log.info(.app, "CDP timeout", .{});
             return;
         }
@@ -186,7 +191,11 @@ fn readLoop(self: *Server, socket: posix.socket_t, timeout_ms: u32) !void {
                 ms_remaining = timeout_ms;
             },
             .no_page => {
-                if (http.poll(ms_remaining) != .cdp_socket) {
+                const status = client.http.tick(ms_remaining) catch |err| {
+                    log.err(.app, "http tick", .{ .err = err });
+                    return;
+                };
+                if (status != .cdp_socket) {
                     log.info(.app, "CDP timeout", .{});
                     return;
                 }
@@ -217,6 +226,7 @@ pub const Client = struct {
     },
 
     server: *Server,
+    http: *HttpClient,
     reader: Reader(true),
     socket: posix.socket_t,
     socket_flags: usize,
@@ -240,9 +250,13 @@ pub const Client = struct {
         var reader = try Reader(true).init(server.allocator);
         errdefer reader.deinit();
 
+        const http = try server.app.http.createClient(server.allocator);
+        errdefer http.deinit();
+
         return .{
             .socket = socket,
             .server = server,
+            .http = http,
             .reader = reader,
             .mode = .{ .http = {} },
             .socket_flags = socket_flags,
@@ -471,7 +485,7 @@ pub const Client = struct {
             break :blk res;
         };
 
-        self.mode = .{ .cdp = try CDP.init(self.server.app, self) };
+        self.mode = .{ .cdp = try CDP.init(self.server.app, self.http, self) };
         return self.send(response);
     }
 

src/browser/Browser.zig

@@ -24,9 +24,9 @@ const ArenaAllocator = std.heap.ArenaAllocator;
 const js = @import("js/js.zig");
 const log = @import("../log.zig");
 const App = @import("../App.zig");
+const HttpClient = @import("../http/Client.zig");
 
 const ArenaPool = App.ArenaPool;
-const HttpClient = App.Http.Client;
 
 const IS_DEBUG = @import("builtin").mode == .Debug;
 
@@ -47,6 +47,7 @@ http_client: *HttpClient,
 
 const InitOpts = struct {
     env: js.Env.InitOpts = .{},
+    http_client: *HttpClient,
 };
 
 pub fn init(app: *App, opts: InitOpts) !Browser {
@@ -61,7 +62,7 @@ pub fn init(app: *App, opts: InitOpts) !Browser {
         .session = null,
         .allocator = allocator,
         .arena_pool = &app.arena_pool,
-        .http_client = app.http.client,
+        .http_client = opts.http_client,
     };
 }
 

src/cdp/cdp.zig

@@ -28,6 +28,7 @@ const js = @import("../browser/js/js.zig");
 const App = @import("../App.zig");
 const Browser = @import("../browser/Browser.zig");
 const Session = @import("../browser/Session.zig");
+const HttpClient = @import("../http/Client.zig");
 const Page = @import("../browser/Page.zig");
 const Incrementing = @import("../id.zig").Incrementing;
 const Notification = @import("../Notification.zig");
@@ -84,10 +85,11 @@ pub fn CDPT(comptime TypeProvider: type) type {
 
         const Self = @This();
 
-        pub fn init(app: *App, client: TypeProvider.Client) !Self {
+        pub fn init(app: *App, http_client: *HttpClient, client: TypeProvider.Client) !Self {
             const allocator = app.allocator;
             const browser = try Browser.init(app, .{
                 .env = .{ .with_inspector = true },
+                .http_client = http_client,
             });
             errdefer browser.deinit();
 

src/cdp/testing.zig

@@ -85,7 +85,7 @@ const TestContext = struct {
             self.client = Client.init(self.arena.allocator());
             // Don't use the arena here. We want to detect leaks in CDP.
             // The arena is only for test-specific stuff
-            self.cdp_ = TestCDP.init(base.test_app, &self.client.?) catch unreachable;
+            self.cdp_ = TestCDP.init(base.test_app, base.test_http, &self.client.?) catch unreachable;
         }
         return &self.cdp_.?;
     }

src/http/Http.zig

@@ -17,8 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const lp = @import("lightpanda");
-const Config = @import("../Config.zig");
 
 pub const c = @cImport({
     @cInclude("curl/curl.h");
@@ -28,6 +26,8 @@ pub const ENABLE_DEBUG = false;
 pub const Client = @import("Client.zig");
 pub const Transfer = Client.Transfer;
 
+const lp = @import("lightpanda");
+const Config = @import("../Config.zig");
 const log = @import("../log.zig");
 const errors = @import("errors.zig");
 const RobotStore = @import("../browser/Robots.zig").RobotStore;
@@ -42,10 +42,11 @@ const ArenaAllocator = std.heap.ArenaAllocator;
 // once for all http connections is a win.
 const Http = @This();
 
-config: *const Config,
-client: *Client,
-ca_blob: ?c.curl_blob,
 arena: ArenaAllocator,
+allocator: Allocator,
+config: *const Config,
+ca_blob: ?c.curl_blob,
+robot_store: *RobotStore,
 
 pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Config) !Http {
     try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL));
@@ -60,40 +61,29 @@ pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Confi
 
     var ca_blob: ?c.curl_blob = null;
     if (config.tlsVerifyHost()) {
-        ca_blob = try loadCerts(allocator, arena.allocator());
+        ca_blob = try loadCerts(allocator);
     }
 
-    var client = try Client.init(allocator, ca_blob, robot_store, config);
-    errdefer client.deinit();
-
     return .{
         .arena = arena,
-        .client = client,
+        .allocator = allocator,
-        .ca_blob = ca_blob,
         .config = config,
+        .ca_blob = ca_blob,
+        .robot_store = robot_store,
     };
 }
 
 pub fn deinit(self: *Http) void {
-    self.client.deinit();
+    if (self.ca_blob) |ca_blob| {
+        const data: [*]u8 = @ptrCast(ca_blob.data);
+        self.allocator.free(data[0..ca_blob.len]);
+    }
     c.curl_global_cleanup();
     self.arena.deinit();
 }
 
-pub fn poll(self: *Http, timeout_ms: u32) Client.PerformStatus {
+pub fn createClient(self: *Http, allocator: Allocator) !*Client {
-    return self.client.tick(timeout_ms) catch |err| {
+    return Client.init(allocator, self.ca_blob, self.robot_store, self.config);
-        log.err(.app, "http poll", .{ .err = err });
-        return .normal;
-    };
-}
-
-pub fn addCDPClient(self: *Http, cdp_client: Client.CDPClient) void {
-    lp.assert(self.client.cdp_client == null, "Http addCDPClient existing", .{});
-    self.client.cdp_client = cdp_client;
-}
-
-pub fn removeCDPClient(self: *Http) void {
-    self.client.cdp_client = null;
 }
 
 pub fn newConnection(self: *Http) !Connection {
@@ -351,7 +341,7 @@ pub const Method = enum(u8) {
 // This whole rescan + decode is really just needed for MacOS. On Linux
 // bundle.rescan does find the .pem file(s) which could be in a few different
 // places, so it's still useful, just not efficient.
-fn loadCerts(allocator: Allocator, arena: Allocator) !c.curl_blob {
+fn loadCerts(allocator: Allocator) !c.curl_blob {
     var bundle: std.crypto.Certificate.Bundle = .{};
     try bundle.rescan(allocator);
     defer bundle.deinit(allocator);
@@ -374,8 +364,9 @@ fn loadCerts(allocator: Allocator, arena: Allocator) !c.curl_blob {
         (bundle.map.count() * 75) + // start / end per certificate + extra, just in case
         (encoded_size / 64) // newline per 64 characters
     ;
-    try arr.ensureTotalCapacity(arena, buffer_size);
+    try arr.ensureTotalCapacity(allocator, buffer_size);
-    var writer = arr.writer(arena);
+    errdefer arr.deinit(allocator);
+    var writer = arr.writer(allocator);
 
     var it = bundle.map.valueIterator();
     while (it.next()) |index| {
@@ -388,11 +379,16 @@ fn loadCerts(allocator: Allocator, arena: Allocator) !c.curl_blob {
     }
 
     // Final encoding should not be larger than our initial size estimate
-    lp.assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estiate = buffer_size, .len = arr.items.len });
+    lp.assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estimate = buffer_size, .len = arr.items.len });
+
+    // Allocate exactly the size needed and copy the data
+    const result = try allocator.dupe(u8, arr.items);
+    // Free the original oversized allocation
+    arr.deinit(allocator);
 
     return .{
-        .len = arr.items.len,
+        .len = result.len,
-        .data = arr.items.ptr,
+        .data = result.ptr,
         .flags = 0,
     };
 }

src/lightpanda.zig

@@ -39,10 +39,13 @@ pub const FetchOpts = struct {
     writer: ?*std.Io.Writer = null,
 };
 pub fn fetch(app: *App, url: [:0]const u8, opts: FetchOpts) !void {
+    const http_client = try app.http.createClient(app.allocator);
+    defer http_client.deinit();
+
     const notification = try Notification.init(app.allocator);
     defer notification.deinit();
 
-    var browser = try Browser.init(app, .{});
+    var browser = try Browser.init(app, .{ .http_client = http_client });
     defer browser.deinit();
 
     var session = try browser.newSession(notification);

src/main_legacy_test.zig

@@ -46,17 +46,24 @@ pub fn main() !void {
     var test_arena = std.heap.ArenaAllocator.init(allocator);
     defer test_arena.deinit();
 
-    var browser = try lp.Browser.init(app, .{});
+    const http_client = try app.http.createClient(allocator);
-    const notification = try lp.Notification.init(app.allocator);
+    defer http_client.deinit();
-    defer notification.deinit();
+
+    var browser = try lp.Browser.init(app, .{ .http_client = http_client });
     defer browser.deinit();
 
+    const notification = try lp.Notification.init(allocator);
+    defer notification.deinit();
+
     const session = try browser.newSession(notification);
+    defer session.deinit();
 
     var dir = try std.fs.cwd().openDir("src/browser/tests/legacy/", .{ .iterate = true, .no_follow = true });
     defer dir.close();
+
     var walker = try dir.walk(allocator);
     defer walker.deinit();
+
     while (try walker.next()) |entry| {
         _ = test_arena.reset(.retain_capacity);
         if (entry.kind != .file) {

src/main_wpt.zig

@@ -69,7 +69,10 @@ pub fn main() !void {
     var app = try lp.App.init(allocator, &config);
     defer app.deinit();
 
-    var browser = try lp.Browser.init(app, .{});
+    const http_client = try app.http.createClient(allocator);
+    defer http_client.deinit();
+
+    var browser = try lp.Browser.init(app, .{ .http_client = http_client });
     defer browser.deinit();
 
     // An arena for running each tests. Is reset after every test.

src/testing.zig

@@ -39,6 +39,7 @@ pub fn reset() void {
 const App = @import("App.zig");
 const js = @import("browser/js/js.zig");
 const Config = @import("Config.zig");
+const Client = @import("http/Client.zig");
 const Page = @import("browser/Page.zig");
 const Browser = @import("browser/Browser.zig");
 const Session = @import("browser/Session.zig");
@@ -334,6 +335,7 @@ fn isJsonValue(a: std.json.Value, b: std.json.Value) bool {
 }
 
 pub var test_app: *App = undefined;
+pub var test_http: *Client = undefined;
 pub var test_browser: Browser = undefined;
 pub var test_notification: *Notification = undefined;
 pub var test_session: *Session = undefined;
@@ -472,7 +474,10 @@ test "tests:beforeAll" {
     test_app = try App.init(test_allocator, &test_config);
     errdefer test_app.deinit();
 
-    test_browser = try Browser.init(test_app, .{});
+    test_http = try test_app.http.createClient(test_allocator);
+    errdefer test_http.deinit();
+
+    test_browser = try Browser.init(test_app, .{ .http_client = test_http });
     errdefer test_browser.deinit();
 
     // Create notification for testing
@@ -519,6 +524,7 @@ test "tests:afterAll" {
 
     test_notification.deinit();
     test_browser.deinit();
+    test_http.deinit();
     test_app.deinit();
     test_config.deinit(@import("root").tracking_allocator);
 }