Re-enable telemetry

Start work on supporting navigation events (clicks, form submission).

src/http/Client.zig

@@ -16,271 +16,307 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-pub const c = @cImport({
-    @cInclude("curl/curl.h");
-});
-
-const ENABLE_DEBUG = false;
-
 const std = @import("std");
 const log = @import("../log.zig");
 const builtin = @import("builtin");
-const errors = @import("errors.zig");
+const Http = @import("Http.zig");
+
+const c = Http.c;
 
 const Allocator = std.mem.Allocator;
-const ArenaAllocator = std.heap.ArenaAllocator;
 
-pub fn init() !void {
-    try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL));
-    if (comptime ENABLE_DEBUG) {
-        std.debug.print("curl version: {s}\n\n", .{c.curl_version()});
-    }
-}
+const errorCheck = Http.errorCheck;
+const errorMCheck = Http.errorMCheck;
 
-pub fn deinit() void {
-    c.curl_global_cleanup();
-}
+pub const Method = Http.Method;
 
-pub const Client = struct {
-    active: usize,
-    multi: *c.CURLM,
-    handles: Handles,
-    queue: RequestQueue,
-    allocator: Allocator,
-    transfer_pool: std.heap.MemoryPool(Transfer),
-    queue_node_pool: std.heap.MemoryPool(RequestQueue.Node),
-    //@newhttp
-    http_proxy: ?std.Uri = null,
+// This is loosely tied to a browser Page. Loading all the <scripts>, doing
+// XHR requests, and loading imports all happens through here. Sine the app
+// currently supports 1 browser and 1 page at-a-time, we only have 1 Client and
+// re-use it from page to page. This allows us better re-use of the various
+// buffers/caches (including keepalive connections) that libcurl has.
+//
+// The app has other secondary http needs, like telemetry. While we want to
+// share some things (namely the ca blob, and maybe some configuration
+// (TODO: ??? should proxy settings be global ???)), we're able to do call
+// client.abort() to abort the transfers being made by a page, without impacting
+// those other http requests.
+pub const Client = @This();
 
-    const RequestQueue = std.DoublyLinkedList(Request);
+active: usize,
+multi: *c.CURLM,
+handles: Handles,
+queue: RequestQueue,
+allocator: Allocator,
+transfer_pool: std.heap.MemoryPool(Transfer),
+queue_node_pool: std.heap.MemoryPool(RequestQueue.Node),
+//@newhttp
+http_proxy: ?std.Uri = null,
 
-    const Opts = struct {
-        timeout_ms: u31 = 0,
-        max_redirects: u8 = 10,
-        connect_timeout_ms: u31 = 5000,
-        max_concurrent_transfers: u8 = 5,
+const RequestQueue = std.DoublyLinkedList(Request);
+
+pub fn init(allocator: Allocator, ca_blob: c.curl_blob, opts: Http.Opts) !*Client {
+    var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
+    errdefer transfer_pool.deinit();
+
+    var queue_node_pool = std.heap.MemoryPool(RequestQueue.Node).init(allocator);
+    errdefer queue_node_pool.deinit();
+
+    const client = try allocator.create(Client);
+    errdefer allocator.destroy(client);
+
+    const multi = c.curl_multi_init() orelse return error.FailedToInitializeMulti;
+    errdefer _ = c.curl_multi_cleanup(multi);
+
+    var handles = try Handles.init(allocator, client, ca_blob, opts);
+    errdefer handles.deinit(allocator, multi);
+
+    client.* = .{
+        .queue = .{},
+        .active = 0,
+        .multi = multi,
+        .handles = handles,
+        .allocator = allocator,
+        .transfer_pool = transfer_pool,
+        .queue_node_pool = queue_node_pool,
     };
-    pub fn init(allocator: Allocator, opts: Opts) !*Client {
-        var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
-        errdefer transfer_pool.deinit();
 
-        var queue_node_pool = std.heap.MemoryPool(RequestQueue.Node).init(allocator);
-        errdefer queue_node_pool.deinit();
+    return client;
+}
 
-        const client = try allocator.create(Client);
-        errdefer allocator.destroy(client);
+pub fn deinit(self: *Client) void {
+    self.handles.deinit(self.allocator, self.multi);
+    _ = c.curl_multi_cleanup(self.multi);
 
-        var handles = try Handles.init(allocator, client, opts);
-        errdefer handles.deinit(allocator);
+    self.transfer_pool.deinit();
+    self.queue_node_pool.deinit();
+    self.allocator.destroy(self);
+}
 
-        const multi = c.curl_multi_init() orelse return error.FailedToInitializeMulti;
-        errdefer _ = c.curl_multi_cleanup(multi);
+pub fn abort(self: *Client) void {
+    self.handles.abort(self.multi);
 
-        client.* = .{
-            .queue = .{},
-            .active = 0,
-            .multi = multi,
-            .handles = handles,
-            .allocator = allocator,
-            .transfer_pool = transfer_pool,
-            .queue_node_pool = queue_node_pool,
-        };
-        return client;
+    var n = self.queue.first;
+    while (n) |node| {
+        n = node.next;
+        self.queue_node_pool.destroy(node);
     }
+    self.queue = .{};
+    self.active = 0;
 
-    pub fn deinit(self: *Client) void {
-        self.handles.deinit(self.allocator);
-        _ = c.curl_multi_cleanup(self.multi);
-
-        self.transfer_pool.deinit();
-        self.queue_node_pool.deinit();
-        self.allocator.destroy(self);
-    }
-
-    pub fn tick(self: *Client, timeout_ms: usize) !void {
-        var handles = &self.handles.available;
-        while (true) {
-            if (handles.first == null) {
-                break;
-            }
-            const queue_node = self.queue.popFirst() orelse break;
-
-            defer self.queue_node_pool.destroy(queue_node);
-
-            const handle = handles.popFirst().?.data;
-            try self.makeRequest(handle, queue_node.data);
-        }
-
-        try self.perform(@intCast(timeout_ms));
-    }
-
-    pub fn request(self: *Client, req: Request) !void {
-        if (self.handles.getFreeHandle()) |handle| {
-            return self.makeRequest(handle, req);
-        }
-
-        const node = try self.queue_node_pool.create();
-        node.data = req;
-        self.queue.append(node);
-    }
-
-    fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
-        const easy = handle.easy;
-
-        const header_list = blk: {
-            errdefer self.handles.release(handle);
-            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_URL, req.url.ptr));
-            switch (req.method) {
-                .GET => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPGET, @as(c_long, 1))),
-                .POST => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPPOST, @as(c_long, 1))),
-                .PUT => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "put")),
-                .DELETE => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "delete")),
-                .HEAD => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "head")),
-                .OPTIONS => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "options")),
-            }
-
-            const header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0");
-            errdefer c.curl_slist_free_all(header_list);
-
-            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list));
-
-            break :blk header_list;
-        };
-
-        {
-            errdefer self.handles.release(handle);
-
-            const transfer = try self.transfer_pool.create();
-            transfer.* = .{
-                .id = 0,
-                .req = req,
-                .ctx = req.ctx,
-                .handle = handle,
-                ._request_header_list = header_list,
-            };
-            errdefer self.transfer_pool.destroy(transfer);
-            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer));
-
-            try errorMCheck(c.curl_multi_add_handle(self.multi, easy));
-            if (req.start_callback) |cb| {
-                cb(transfer) catch |err| {
-                    try errorMCheck(c.curl_multi_remove_handle(self.multi, easy));
-                    return err;
-                };
-            }
-        }
-
-        self.active += 1;
-        return self.perform(0);
-    }
-
-    fn perform(self: *Client, timeout_ms: c_int) !void {
-        const multi = self.multi;
-
+    // Maybe a bit of overkill
+    // We can remove some (all?) of these once we're confident its right.
+    std.debug.assert(self.handles.in_use.first == null);
+    std.debug.assert(self.handles.available.len == self.handles.handles.len);
+    if (builtin.mode == .Debug) {
         var running: c_int = undefined;
-        try errorMCheck(c.curl_multi_perform(multi, &running));
+        std.debug.assert(c.curl_multi_perform(self.multi, &running) == c.CURLE_OK);
+        std.debug.assert(running == 0);
+    }
+}
 
-        if (running > 0 and timeout_ms > 0) {
-            try errorMCheck(c.curl_multi_poll(multi, null, 0, timeout_ms, null));
+pub fn tick(self: *Client, timeout_ms: usize) !void {
+    var handles = &self.handles.available;
+    while (true) {
+        if (handles.first == null) {
+            break;
+        }
+        const queue_node = self.queue.popFirst() orelse break;
+
+        defer self.queue_node_pool.destroy(queue_node);
+
+        const handle = handles.popFirst().?.data;
+        try self.makeRequest(handle, queue_node.data);
+    }
+
+    try self.perform(@intCast(timeout_ms));
+}
+
+pub fn request(self: *Client, req: Request) !void {
+    if (self.handles.getFreeHandle()) |handle| {
+        return self.makeRequest(handle, req);
+    }
+
+    const node = try self.queue_node_pool.create();
+    node.data = req;
+    self.queue.append(node);
+}
+
+fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
+    const easy = handle.easy;
+
+    const header_list = blk: {
+        errdefer self.handles.release(handle);
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_URL, req.url.ptr));
+
+        try Http.setMethod(easy, req.method);
+        if (req.body) |b| {
+            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, b.ptr));
+            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(b.len))));
         }
 
-        while (true) {
-            var remaining: c_int = undefined;
-            const msg: *c.CURLMsg = c.curl_multi_info_read(multi, &remaining) orelse break;
-            if (msg.msg == c.CURLMSG_DONE) {
-                self.active -= 1;
-                const easy = msg.easy_handle.?;
-                const transfer = try Transfer.fromEasy(easy);
-                defer {
-                    self.handles.release(transfer.handle);
-                    transfer.deinit();
-                    self.transfer_pool.destroy(transfer);
-                }
+        var header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0");
+        errdefer c.curl_slist_free_all(header_list);
 
-                if (errorCheck(msg.data.result)) {
-                    transfer.req.done_callback(transfer) catch |err| transfer.onError(err);
-                } else |err| {
-                    transfer.onError(err);
-                }
+        if (req.content_type) |ct| {
+            header_list = c.curl_slist_append(header_list, ct);
+        }
 
-                try errorMCheck(c.curl_multi_remove_handle(multi, easy));
-            }
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list));
 
-            if (remaining == 0) {
-                break;
-            }
+        break :blk header_list;
+    };
+
+    {
+        errdefer self.handles.release(handle);
+
+        const transfer = try self.transfer_pool.create();
+        transfer.* = .{
+            .id = 0,
+            .req = req,
+            .ctx = req.ctx,
+            .handle = handle,
+            ._request_header_list = header_list,
+        };
+        errdefer self.transfer_pool.destroy(transfer);
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer));
+
+        try errorMCheck(c.curl_multi_add_handle(self.multi, easy));
+        if (req.start_callback) |cb| {
+            cb(transfer) catch |err| {
+                try errorMCheck(c.curl_multi_remove_handle(self.multi, easy));
+                return err;
+            };
         }
     }
-};
+
+    self.active += 1;
+    return self.perform(0);
+}
+
+fn perform(self: *Client, timeout_ms: c_int) !void {
+    const multi = self.multi;
+
+    var running: c_int = undefined;
+    try errorMCheck(c.curl_multi_perform(multi, &running));
+
+    if (running > 0 and timeout_ms > 0) {
+        try errorMCheck(c.curl_multi_poll(multi, null, 0, timeout_ms, null));
+    }
+
+    while (true) {
+        var remaining: c_int = undefined;
+        const msg: *c.CURLMsg = c.curl_multi_info_read(multi, &remaining) orelse break;
+        if (msg.msg == c.CURLMSG_DONE) {
+            const easy = msg.easy_handle.?;
+
+            const transfer = try Transfer.fromEasy(easy);
+
+            const ctx = transfer.ctx;
+            const done_callback = transfer.req.done_callback;
+            const error_callback = transfer.req.error_callback;
+            // release it ASAP so that it's avaiable (since some done_callbacks
+            // will load more resources).
+            self.endTransfer(transfer);
+
+            if (errorCheck(msg.data.result)) {
+                done_callback(ctx) catch |err| error_callback(ctx, err);
+            } else |err| {
+                error_callback(ctx, err);
+            }
+        }
+
+        if (remaining == 0) {
+            break;
+        }
+    }
+}
+
+fn endTransfer(self: *Client, transfer: *Transfer) void {
+    const handle = transfer.handle;
+
+    transfer.deinit();
+    self.transfer_pool.destroy(transfer);
+
+    errorMCheck(c.curl_multi_remove_handle(self.multi, handle.easy)) catch |err| {
+        log.fatal(.http, "Failed to abort", .{ .err = err });
+    };
+
+    self.handles.release(handle);
+    self.active -= 1;
+}
 
 const Handles = struct {
     handles: []Handle,
-    available: FreeList,
-    cert_arena: ArenaAllocator,
+    in_use: HandleList,
+    available: HandleList,
 
-    const FreeList = std.DoublyLinkedList(*Handle);
+    const HandleList = std.DoublyLinkedList(*Handle);
 
-    fn init(allocator: Allocator, client: *Client, opts: Client.Opts) !Handles {
+    fn init(allocator: Allocator, client: *Client, ca_blob: c.curl_blob, opts: Http.Opts) !Handles {
         const count = opts.max_concurrent_transfers;
         std.debug.assert(count > 0);
 
         const handles = try allocator.alloc(Handle, count);
         errdefer allocator.free(handles);
 
-        var initialized_count: usize = 0;
-        errdefer cleanup(allocator, handles[0..initialized_count]);
-
-        var cert_arena = ArenaAllocator.init(allocator);
-        errdefer cert_arena.deinit();
-        const ca_blob = try @import("ca_certs.zig").load(allocator, cert_arena.allocator());
-
-        var available: FreeList = .{};
+        var available: HandleList = .{};
         for (0..count) |i| {
-            const node = try allocator.create(FreeList.Node);
-            errdefer allocator.destroy(node);
+            const easy = c.curl_easy_init() orelse return error.FailedToInitializeEasy;
+            errdefer _ = c.curl_easy_cleanup(easy);
 
             handles[i] = .{
-                .node = node,
+                .easy = easy,
                 .client = client,
-                .easy = undefined,
+                .node = undefined,
             };
-            try handles[i].init(ca_blob, opts);
-            initialized_count += 1;
+            try handles[i].configure(ca_blob, opts);
 
-            node.data = &handles[i];
-            available.append(node);
+            handles[i].node.data = &handles[i];
+            available.append(&handles[i].node);
         }
 
         return .{
+            .in_use = .{},
             .handles = handles,
             .available = available,
-            .cert_arena = cert_arena,
         };
     }
 
-    fn deinit(self: *Handles, allocator: Allocator) void {
-        cleanup(allocator, self.handles);
+    fn deinit(self: *Handles, allocator: Allocator, multi: *c.CURLM) void {
+        self.abort(multi);
+        for (self.handles) |*h| {
+            _ = c.curl_easy_cleanup(h.easy);
+        }
         allocator.free(self.handles);
-        self.cert_arena.deinit();
     }
 
-    // Done line this so that cleanup can be called from init with a partial state
-    fn cleanup(allocator: Allocator, handles: []Handle) void {
-        for (handles) |*h| {
-            _ = c.curl_easy_cleanup(h.easy);
-            allocator.destroy(h.node);
+    fn abort(self: *Handles, multi: *c.CURLM) void {
+        while (self.in_use.first) |node| {
+            const handle = node.data;
+            errorMCheck(c.curl_multi_remove_handle(multi, handle.easy)) catch |err| {
+                log.err(.http, "remove handle", .{ .err = err });
+            };
+            self.release(handle);
         }
     }
 
     fn getFreeHandle(self: *Handles) ?*Handle {
-        if (self.available.popFirst()) |handle| {
-            return handle.data;
+        if (self.available.popFirst()) |node| {
+            node.prev = null;
+            node.next = null;
+            self.in_use.append(node);
+            return node.data;
         }
         return null;
     }
 
     fn release(self: *Handles, handle: *Handle) void {
-        self.available.append(handle.node);
+        const node = &handle.node;
+        self.in_use.remove(node);
+        node.prev = null;
+        node.next = null;
+        self.available.append(node);
     }
 };
 
@@ -289,17 +325,13 @@ const Handles = struct {
 const Handle = struct {
     easy: *c.CURL,
     client: *Client,
-    node: *Handles.FreeList.Node,
+    node: Handles.HandleList.Node,
     error_buffer: [c.CURL_ERROR_SIZE:0]u8 = undefined,
 
     // Is called by Handles when already partially initialized. Done like this
     // so that we have a stable pointer to error_buffer.
-    fn init(self: *Handle, ca_blob: c.curl_blob, opts: Client.Opts) !void {
-        const easy = c.curl_easy_init() orelse return error.FailedToInitializeEasy;
-        errdefer _ = c.curl_easy_cleanup(easy);
-
-        self.easy = easy;
-
+    fn configure(self: *Handle, ca_blob: c.curl_blob, opts: Http.Opts) !void {
+        const easy = self.easy;
         try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_ERRORBUFFER, &self.error_buffer));
 
         // timeouts
@@ -323,7 +355,7 @@ const Handle = struct {
         try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CAINFO_BLOB, ca_blob));
 
         // debug
-        if (comptime ENABLE_DEBUG) {
+        if (comptime Http.ENABLE_DEBUG) {
             try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_VERBOSE, @as(c_long, 1)));
         }
     }
@@ -332,6 +364,9 @@ const Handle = struct {
 pub const Request = struct {
     method: Method,
     url: [:0]const u8,
+    body: ?[]const u8 = null,
+    content_type: ?[:0]const u8 = null,
+
     // arbitrary data that can be associated with this request
     ctx: *anyopaque = undefined,
 
@@ -339,8 +374,8 @@ pub const Request = struct {
     header_callback: ?*const fn (req: *Transfer, header: []const u8) anyerror!void = null,
     header_done_callback: *const fn (req: *Transfer) anyerror!void,
     data_callback: *const fn (req: *Transfer, data: []const u8) anyerror!void,
-    done_callback: *const fn (req: *Transfer) anyerror!void,
-    error_callback: *const fn (req: *Transfer, err: anyerror) void,
+    done_callback: *const fn (ctx: *anyopaque) anyerror!void,
+    error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
 };
 
 pub const Transfer = struct {
@@ -368,14 +403,10 @@ pub const Transfer = struct {
         return writer.print("[{d}] {s} {s}", .{ self.id, @tagName(req.method), req.url });
     }
 
-    fn onError(self: *Transfer, err: anyerror) void {
-        self.req.error_callback(self, err);
-    }
-
     pub fn setBody(self: *Transfer, body: []const u8) !void {
         const easy = self.handle.easy;
-        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(body.len))));
         try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, body.ptr));
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(body.len))));
     }
 
     pub fn addHeader(self: *Transfer, value: [:0]const u8) !void {
@@ -383,12 +414,7 @@ pub const Transfer = struct {
     }
 
     pub fn abort(self: *Transfer) void {
-        var client = self.handle.client;
-        errorMCheck(c.curl_multi_remove_handle(client.multi, self.handle.easy)) catch |err| {
-            log.err(.http, "Failed to abort", .{ .err = err });
-        };
-        client.active -= 1;
-        self.deinit();
+        self.handle.client.endTransfer(self);
     }
 
     fn headerCallback(buffer: [*]const u8, header_count: usize, buf_len: usize, data: *anyopaque) callconv(.c) usize {
@@ -410,7 +436,7 @@ pub const Transfer = struct {
                 if (transfer._redirecting) {
                     return buf_len;
                 }
-                transfer.onError(error.InvalidResponseLine);
+                log.debug(.http, "invalid response line", .{ .line = header });
                 return 0;
             }
             const version_start: usize = if (header[5] == '2') 7 else 9;
@@ -421,7 +447,7 @@ pub const Transfer = struct {
             std.debug.assert(version_end < 13);
 
             const status = std.fmt.parseInt(u16, header[version_start..version_end], 10) catch {
-                transfer.onError(error.InvalidResponseStatus);
+                log.debug(.http, "invalid status code", .{ .line = header });
                 return 0;
             };
 
@@ -433,7 +459,7 @@ pub const Transfer = struct {
 
             var url: [*c]u8 = undefined;
             errorCheck(c.curl_easy_getinfo(handle.easy, c.CURLINFO_EFFECTIVE_URL, &url)) catch |err| {
-                transfer.onError(err);
+                log.err(.http, "failed to get URL", .{ .err = err });
                 return 0;
             };
 
@@ -511,41 +537,3 @@ pub const Header = struct {
         return self._content_type[0..self._content_type_len];
     }
 };
-
-fn errorCheck(code: c.CURLcode) errors.Error!void {
-    if (code == c.CURLE_OK) {
-        return;
-    }
-    return errors.fromCode(code);
-}
-
-fn errorMCheck(code: c.CURLMcode) errors.Multi!void {
-    if (code == c.CURLM_OK) {
-        return;
-    }
-    if (code == c.CURLM_CALL_MULTI_PERFORM) {
-        // should we can client.perform() here?
-        // or just wait until the next time we naturally call it?
-        return;
-    }
-    return errors.fromMCode(code);
-}
-
-pub const Method = enum {
-    GET,
-    PUT,
-    POST,
-    DELETE,
-    HEAD,
-    OPTIONS,
-};
-
-pub const ProxyType = enum {
-    forward,
-    connect,
-};
-
-pub const ProxyAuth = union(enum) {
-    basic: struct { user_pass: []const u8 },
-    bearer: struct { token: []const u8 },
-};

src/http/Http.zig

@@ -0,0 +1,269 @@
+// Copyright (C) 2023-2025  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+
+pub const c = @cImport({
+    @cInclude("curl/curl.h");
+});
+const errors = @import("errors.zig");
+const Client = @import("Client.zig");
+
+const Allocator = std.mem.Allocator;
+const ArenaAllocator = std.heap.ArenaAllocator;
+
+pub const ENABLE_DEBUG = false;
+
+// Client.zig does the bulk of the work and is loosely tied to a browser Page.
+// But we still need something above Client.zig for the "utility" http stuff
+// we need to do, like telemetry. The most important thing we want from this
+// is to be able to share the ca_blob, which can be quite large - loading it
+// once for all http connections is a win.
+const Http = @This();
+
+opts: Opts,
+client: *Client,
+ca_blob: ?c.curl_blob,
+cert_arena: ArenaAllocator,
+
+pub fn init(allocator: Allocator, opts: Opts) !Http {
+    try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL));
+    errdefer c.curl_global_cleanup();
+
+    if (comptime ENABLE_DEBUG) {
+        std.debug.print("curl version: {s}\n\n", .{c.curl_version()});
+    }
+
+    var cert_arena = ArenaAllocator.init(allocator);
+    errdefer cert_arena.deinit();
+    const ca_blob = try loadCerts(allocator, cert_arena.allocator());
+
+    var client = try Client.init(allocator, ca_blob, opts);
+    errdefer client.deinit();
+
+    return .{
+        .opts = opts,
+        .client = client,
+        .ca_blob = ca_blob,
+        .cert_arena = cert_arena,
+    };
+}
+
+pub fn deinit(self: *Http) void {
+    self.client.deinit();
+    c.curl_global_cleanup();
+    self.cert_arena.deinit();
+}
+
+pub fn newConnection(self: *Http) !Connection {
+    return Connection.init(self.ca_blob, self.opts);
+}
+
+pub const Connection = struct {
+    easy: *c.CURL,
+
+    // Is called by Handles when already partially initialized. Done like this
+    // so that we have a stable pointer to error_buffer.
+    pub fn init(ca_blob_: ?c.curl_blob, opts: Opts) !Connection {
+        const easy = c.curl_easy_init() orelse return error.FailedToInitializeEasy;
+        errdefer _ = c.curl_easy_cleanup(easy);
+
+        // timeouts
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_TIMEOUT_MS, @as(c_long, @intCast(opts.timeout_ms))));
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CONNECTTIMEOUT_MS, @as(c_long, @intCast(opts.connect_timeout_ms))));
+
+        // redirect behavior
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_MAXREDIRS, @as(c_long, @intCast(opts.max_redirects))));
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_FOLLOWLOCATION, @as(c_long, 2)));
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_REDIR_PROTOCOLS_STR, "HTTP,HTTPS")); // remove FTP and FTPS from the default
+
+        // tls
+        // try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYHOST, @as(c_long, 0)));
+        // try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYPEER, @as(c_long, 0)));
+        if (ca_blob_) |ca_blob| {
+            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CAINFO_BLOB, ca_blob));
+        }
+
+        // debug
+        if (comptime Http.ENABLE_DEBUG) {
+            try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_VERBOSE, @as(c_long, 1)));
+        }
+
+        return .{
+            .easy = easy,
+        };
+    }
+
+    pub fn deinit(self: *const Connection) void {
+        c.curl_easy_cleanup(self.easy);
+    }
+
+    pub fn setURL(self: *const Connection, url: [:0]const u8) !void {
+        try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_URL, url.ptr));
+    }
+
+    pub fn setMethod(self: *const Connection, method: Method) !void {
+        try Http.setMethod(self.easy, method);
+    }
+
+    pub fn setBody(self: *const Connection, body: []const u8) !void {
+        const easy = self.easy;
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(body.len))));
+        try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, body.ptr));
+    }
+
+    pub fn request(self: *const Connection) !u16 {
+        try errorCheck(c.curl_easy_perform(self.easy));
+        var http_code: c_long = undefined;
+        try errorCheck(c.curl_easy_getinfo(self.easy, c.CURLINFO_RESPONSE_CODE, &http_code));
+        if (http_code < 0 or http_code > std.math.maxInt(u16)) {
+            return 0;
+        }
+        return @intCast(http_code);
+    }
+};
+
+// used by Connection and Handle
+pub fn setMethod(easy: *c.CURL, method: Method) !void {
+    switch (method) {
+        .GET => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPGET, @as(c_long, 1))),
+        .POST => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPPOST, @as(c_long, 1))),
+        .PUT => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "put")),
+        .DELETE => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "delete")),
+        .HEAD => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "head")),
+        .OPTIONS => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "options")),
+    }
+}
+
+pub fn errorCheck(code: c.CURLcode) errors.Error!void {
+    if (code == c.CURLE_OK) {
+        return;
+    }
+    return errors.fromCode(code);
+}
+
+pub fn errorMCheck(code: c.CURLMcode) errors.Multi!void {
+    if (code == c.CURLM_OK) {
+        return;
+    }
+    if (code == c.CURLM_CALL_MULTI_PERFORM) {
+        // should we can client.perform() here?
+        // or just wait until the next time we naturally call it?
+        return;
+    }
+    return errors.fromMCode(code);
+}
+
+pub const Opts = struct {
+    timeout_ms: u31 = 0,
+    max_redirects: u8 = 10,
+    connect_timeout_ms: u31 = 5000,
+    max_concurrent_transfers: u8 = 5,
+};
+
+pub const Method = enum {
+    GET,
+    PUT,
+    POST,
+    DELETE,
+    HEAD,
+    OPTIONS,
+};
+
+pub const ProxyType = enum {
+    forward,
+    connect,
+};
+
+pub const ProxyAuth = union(enum) {
+    basic: struct { user_pass: []const u8 },
+    bearer: struct { token: []const u8 },
+};
+
+// TODO: on BSD / Linux, we could just read the PEM file directly.
+// This whole rescan + decode is really just needed for MacOS. On Linux
+// bundle.rescan does find the .pem file(s) which could be in a few different
+// places, so it's still useful, just not efficient.
+fn loadCerts(allocator: Allocator, arena: Allocator) !c.curl_blob {
+    var bundle: std.crypto.Certificate.Bundle = .{};
+    try bundle.rescan(allocator);
+    defer bundle.deinit(allocator);
+
+    var it = bundle.map.valueIterator();
+    const bytes = bundle.bytes.items;
+
+    const encoder = std.base64.standard.Encoder;
+    var arr: std.ArrayListUnmanaged(u8) = .empty;
+
+    const encoded_size = encoder.calcSize(bytes.len);
+    const buffer_size = encoded_size +
+        (bundle.map.count() * 75) + // start / end per certificate + extra, just in case
+        (encoded_size / 64) // newline per 64 characters
+    ;
+    try arr.ensureTotalCapacity(arena, buffer_size);
+    var writer = arr.writer(arena);
+
+    while (it.next()) |index| {
+        const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*);
+
+        try writer.writeAll("-----BEGIN CERTIFICATE-----\n");
+        var line_writer = LineWriter{ .inner = writer };
+        try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]);
+        try writer.writeAll("\n-----END CERTIFICATE-----\n");
+    }
+
+    // Final encoding should not be larger than our initial size estimate
+    std.debug.assert(buffer_size > arr.items.len);
+
+    return .{
+        .len = arr.items.len,
+        .data = arr.items.ptr,
+        .flags = 0,
+    };
+}
+
+// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is
+// what Zig has), with lines wrapped at 64 characters and with a basic header
+// and footer
+const LineWriter = struct {
+    col: usize = 0,
+    inner: std.ArrayListUnmanaged(u8).Writer,
+
+    pub fn writeAll(self: *LineWriter, data: []const u8) !void {
+        var writer = self.inner;
+
+        var col = self.col;
+        const len = 64 - col;
+
+        var remain = data;
+        if (remain.len > len) {
+            col = 0;
+            try writer.writeAll(data[0..len]);
+            try writer.writeByte('\n');
+            remain = data[len..];
+        }
+
+        while (remain.len > 64) {
+            try writer.writeAll(remain[0..64]);
+            try writer.writeByte('\n');
+            remain = data[len..];
+        }
+        try writer.writeAll(remain);
+        self.col = col + remain.len;
+    }
+};

src/http/ca_certs.zig

@@ -1,93 +0,0 @@
-// Copyright (C) 2023-2025  Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <francis@lightpanda.io>
-// Pierre Tachoire <pierre@lightpanda.io>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const c = @import("client.zig").c;
-
-const Allocator = std.mem.Allocator;
-
-// TODO: on BSD / Linux, we could just read the PEM file directly.
-// This whole rescan + decode is really just needed for MacOS. On Linux
-// bundle.rescan does find the .pem file(s) which could be in a few different
-// places, so it's still useful, just not efficient.
-pub fn load(allocator: Allocator, arena: Allocator) !c.curl_blob {
-    var bundle: std.crypto.Certificate.Bundle = .{};
-    try bundle.rescan(allocator);
-    defer bundle.deinit(allocator);
-
-    var it = bundle.map.valueIterator();
-    const bytes = bundle.bytes.items;
-
-    const encoder = std.base64.standard.Encoder;
-    var arr: std.ArrayListUnmanaged(u8) = .empty;
-
-    const encoded_size = encoder.calcSize(bytes.len);
-    const buffer_size = encoded_size +
-        (bundle.map.count() * 75) + // start / end per certificate + extra, just in case
-        (encoded_size / 64) // newline per 64 characters
-    ;
-    try arr.ensureTotalCapacity(arena, buffer_size);
-    var writer = arr.writer(arena);
-
-    while (it.next()) |index| {
-        const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*);
-
-        try writer.writeAll("-----BEGIN CERTIFICATE-----\n");
-        var line_writer = LineWriter{ .inner = writer };
-        try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]);
-        try writer.writeAll("\n-----END CERTIFICATE-----\n");
-    }
-
-    // Final encoding should not be larger than our initial size estimate
-    std.debug.assert(buffer_size > arr.items.len);
-
-    return .{
-        .len = arr.items.len,
-        .data = arr.items.ptr,
-        .flags = 0,
-    };
-}
-
-// Wraps lines @ 64 columns
-const LineWriter = struct {
-    col: usize = 0,
-    inner: std.ArrayListUnmanaged(u8).Writer,
-
-    pub fn writeAll(self: *LineWriter, data: []const u8) !void {
-        var writer = self.inner;
-
-        var col = self.col;
-        const len = 64 - col;
-
-        var remain = data;
-        if (remain.len > len) {
-            col = 0;
-            try writer.writeAll(data[0..len]);
-            try writer.writeByte('\n');
-            remain = data[len..];
-        }
-
-        while (remain.len > 64) {
-            try writer.writeAll(remain[0..64]);
-            try writer.writeByte('\n');
-            remain = data[len..];
-        }
-        try writer.writeAll(remain);
-        self.col = col + remain.len;
-    }
-};

src/http/errors.zig

@@ -17,7 +17,7 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const c = @import("client.zig").c;
+const c = @import("Http.zig").c;
 
 pub const Error = error{
     UnsupportedProtocol,