Merge pull request #1889 from lightpanda-io/wp/mrdimidium/refactor-redirects

Rework header/data callbacks in HttpClient

.github/workflows/nightly.yml

@@ -7,7 +7,7 @@ env:
   AWS_REGION: ${{ vars.NIGHTLY_BUILD_AWS_REGION }}
 
   RELEASE: ${{ github.ref_type == 'tag' && github.ref_name || 'nightly' }}
-  VERSION_FLAG: ${{ github.ref_type == 'tag' && format('-Dversion_string={0}', github.ref_name) || format('-Dpre_version={0}', 'nightly') }}
+  VERSION_FLAG: ${{ github.ref_type == 'tag' && format('-Dversion={0}', github.ref_name) || '-Dversion=nightly' }}
 
 on:
   push:

README.md

@@ -170,6 +170,7 @@ You may still encounter errors or crashes. Please open an issue with specifics i
 
 Here are the key features we have implemented:
 
+- [ ] CORS [#2015](https://github.com/lightpanda-io/browser/issues/2015)
 - [x] HTTP loader ([Libcurl](https://curl.se/libcurl/))
 - [x] HTML parser ([html5ever](https://github.com/servo/html5ever))
 - [x] DOM tree

build.zig

@@ -719,39 +719,45 @@ fn buildCurl(
     return lib;
 }
 
-/// Returns `MAJOR.MINOR.PATCH-dev` when `git describe` fails.
+/// Resolves the semantic version of the build.
+///
+/// The base version is read from `build.zig.zon`. This can be overridden
+/// using the `-Dversion` command-line flag:
+/// - If the flag contains a full semantic version (e.g., `1.2.3`), it replaces
+///   the base version entirely.
+/// - If the flag contains a simple string (e.g., `nightly`), it replaces only
+///   the pre-release tag of the base version (e.g., `1.0.0-dev` -> `1.0.0-nightly`).
+///
+/// For versions that have a pre-release tag and no explicit build metadata,
+/// this function automatically enriches the version with the git commit count
+/// and short hash (e.g., `1.0.0-dev.5243+dbe45229`).
 fn resolveVersion(b: *std.Build) std.SemanticVersion {
-    const version_string = b.option([]const u8, "version_string", "Override the version of this build");
-    if (version_string) |semver_string| {
-        return std.SemanticVersion.parse(semver_string) catch |err| {
-            std.debug.panic("Expected -Dversion-string={s} to be a semantic version: {}", .{ semver_string, err });
-        };
+    const opt_version = b.option([]const u8, "version", "Override the version of this build");
+
+    const version = if (opt_version) |v|
+        std.SemanticVersion.parse(v) catch blk: {
+            var fallback = lightpanda_version;
+            fallback.pre = v;
+            break :blk fallback;
         }
+    else
+        lightpanda_version;
 
-    const pre_version = b.option([]const u8, "pre_version", "Override the pre version of this build");
-    const pre = blk: {
-        if (pre_version) |pre| {
-            break :blk pre;
-        }
-
-        break :blk lightpanda_version.pre;
-    };
-
-    // If it's a stable release (no pre or build metadata in build.zig.zon), use it as is
-    if (pre == null and lightpanda_version.build == null) return lightpanda_version;
+    // Only enrich versions that have a pre-release field and no explicit build metadata.
+    if (version.pre == null or version.build != null) return version;
 
     // For dev/nightly versions, calculate the commit count and hash
-    const git_hash_raw = runGit(b, &.{ "rev-parse", "--short", "HEAD" }) catch return lightpanda_version;
+    const git_hash_raw = runGit(b, &.{ "rev-parse", "--short", "HEAD" }) catch return version;
     const commit_hash = std.mem.trim(u8, git_hash_raw, " \n\r");
 
-    const git_count_raw = runGit(b, &.{ "rev-list", "--count", "HEAD" }) catch return lightpanda_version;
+    const git_count_raw = runGit(b, &.{ "rev-list", "--count", "HEAD" }) catch return version;
     const commit_count = std.mem.trim(u8, git_count_raw, " \n\r");
 
     return .{
-        .major = lightpanda_version.major,
-        .minor = lightpanda_version.minor,
-        .patch = lightpanda_version.patch,
-        .pre = b.fmt("{s}.{s}", .{ pre.?, commit_count }),
+        .major = version.major,
+        .minor = version.minor,
+        .patch = version.patch,
+        .pre = b.fmt("{s}.{s}", .{ version.pre.?, commit_count }),
         .build = commit_hash,
     };
 }

src/browser/Page.zig

@@ -381,12 +381,9 @@ pub fn getTitle(self: *Page) !?[]const u8 {
     return null;
 }
 
-// Add comon headers for a request:
-// * cookies
+// Add common headers for a request:
 // * referer
-pub fn headersForRequest(self: *Page, temp: Allocator, url: [:0]const u8, headers: *HttpClient.Headers) !void {
-    try self.requestCookie(.{}).headersForRequest(temp, url, headers);
-
+pub fn headersForRequest(self: *Page, headers: *HttpClient.Headers) !void {
     // Build the referer
     const referer = blk: {
         if (self.referer_header == null) {
@@ -541,8 +538,6 @@ pub fn navigate(self: *Page, request_url: [:0]const u8, opts: NavigateOpts) !voi
     if (opts.header) |hdr| {
         try headers.add(hdr);
     }
-    try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, self.url, &headers);
-
     // We dispatch page_navigate event before sending the request.
     // It ensures the event page_navigated is not dispatched before this one.
     session.notification.dispatch(.page_navigate, &.{
@@ -569,6 +564,7 @@ pub fn navigate(self: *Page, request_url: [:0]const u8, opts: NavigateOpts) !voi
         .headers = headers,
         .body = opts.body,
         .cookie_jar = &session.cookie_jar,
+        .cookie_origin = self.url,
         .resource_type = .document,
         .notification = self._session.notification,
         .header_callback = pageHeaderDoneCallback,
@@ -1032,6 +1028,7 @@ fn pageDoneCallback(ctx: *anyopaque) !void {
             });
 
             parser.parse(html);
+            self._parse_state = .complete;
             self.documentIsComplete();
         },
         else => unreachable,
@@ -3550,19 +3547,6 @@ pub fn insertText(self: *Page, v: []const u8) !void {
     }
 }
 
-const RequestCookieOpts = struct {
-    is_http: bool = true,
-    is_navigation: bool = false,
-};
-pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie {
-    return .{
-        .jar = &self._session.cookie_jar,
-        .origin = self.url,
-        .is_http = opts.is_http,
-        .is_navigation = opts.is_navigation,
-    };
-}
-
 fn asUint(comptime string: anytype) std.meta.Int(
     .unsigned,
     @bitSizeOf(@TypeOf(string.*)) - 8, // (- 8) to exclude sentinel 0

src/browser/ScriptManager.zig

@@ -136,9 +136,9 @@ fn clearList(list: *std.DoublyLinkedList) void {
     }
 }
 
-fn getHeaders(self: *ScriptManager, arena: Allocator, url: [:0]const u8) !net_http.Headers {
+fn getHeaders(self: *ScriptManager) !net_http.Headers {
     var headers = try self.client.newHeaders();
-    try self.page.headersForRequest(arena, url, &headers);
+    try self.page.headersForRequest(&headers);
     return headers;
 }
 
@@ -278,9 +278,10 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
             .ctx = script,
             .method = .GET,
             .frame_id = page._frame_id,
-            .headers = try self.getHeaders(arena, url),
+            .headers = try self.getHeaders(),
             .blocking = is_blocking,
             .cookie_jar = &page._session.cookie_jar,
+            .cookie_origin = page.url,
             .resource_type = .script,
             .notification = page._session.notification,
             .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
@@ -403,8 +404,9 @@ pub fn preloadImport(self: *ScriptManager, url: [:0]const u8, referrer: []const
         .ctx = script,
         .method = .GET,
         .frame_id = page._frame_id,
-        .headers = try self.getHeaders(arena, url),
+        .headers = try self.getHeaders(),
         .cookie_jar = &page._session.cookie_jar,
+        .cookie_origin = page.url,
         .resource_type = .script,
         .notification = page._session.notification,
         .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
@@ -506,10 +508,11 @@ pub fn getAsyncImport(self: *ScriptManager, url: [:0]const u8, cb: ImportAsync.C
         .url = url,
         .method = .GET,
         .frame_id = page._frame_id,
-        .headers = try self.getHeaders(arena, url),
+        .headers = try self.getHeaders(),
         .ctx = script,
         .resource_type = .script,
         .cookie_jar = &page._session.cookie_jar,
+        .cookie_origin = page.url,
         .notification = page._session.notification,
         .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
         .header_callback = Script.headerCallback,
@@ -652,7 +655,6 @@ pub const Script = struct {
     debug_transfer_aborted: bool = false,
     debug_transfer_bytes_received: usize = 0,
     debug_transfer_notified_fail: bool = false,
-    debug_transfer_redirecting: bool = false,
     debug_transfer_intercept_state: u8 = 0,
     debug_transfer_auth_challenge: bool = false,
     debug_transfer_easy_id: usize = 0,
@@ -728,7 +730,6 @@ pub const Script = struct {
                 .a3 = self.debug_transfer_aborted,
                 .a4 = self.debug_transfer_bytes_received,
                 .a5 = self.debug_transfer_notified_fail,
-                .a6 = self.debug_transfer_redirecting,
                 .a7 = self.debug_transfer_intercept_state,
                 .a8 = self.debug_transfer_auth_challenge,
                 .a9 = self.debug_transfer_easy_id,
@@ -737,10 +738,9 @@ pub const Script = struct {
                 .b3 = transfer.aborted,
                 .b4 = transfer.bytes_received,
                 .b5 = transfer._notified_fail,
-                .b6 = transfer._redirecting,
                 .b7 = @intFromEnum(transfer._intercept_state),
                 .b8 = transfer._auth_challenge != null,
-                .b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0,
+                .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
             });
             self.header_callback_called = true;
             self.debug_transfer_id = transfer.id;
@@ -748,10 +748,9 @@ pub const Script = struct {
             self.debug_transfer_aborted = transfer.aborted;
             self.debug_transfer_bytes_received = transfer.bytes_received;
             self.debug_transfer_notified_fail = transfer._notified_fail;
-            self.debug_transfer_redirecting = transfer._redirecting;
             self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
             self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
-            self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0;
+            self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
         }
 
         lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity });

src/browser/webapi/net/Fetch.zig

@@ -80,7 +80,7 @@ pub fn init(input: Input, options: ?InitOpts, page: *Page) !js.Promise {
     if (request._headers) |h| {
         try h.populateHttpHeader(page.call_arena, &headers);
     }
-    try page.headersForRequest(page.arena, request._url, &headers);
+    try page.headersForRequest(&headers);
 
     if (comptime IS_DEBUG) {
         log.debug(.http, "fetch", .{ .url = request._url });
@@ -95,6 +95,7 @@ pub fn init(input: Input, options: ?InitOpts, page: *Page) !js.Promise {
         .headers = headers,
         .resource_type = .fetch,
         .cookie_jar = &page._session.cookie_jar,
+        .cookie_origin = page.url,
         .notification = page._session.notification,
         .start_callback = httpStartCallback,
         .header_callback = httpHeaderDoneCallback,

src/browser/webapi/net/XMLHttpRequest.zig

@@ -224,7 +224,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?[]const u8) !void {
 
     try self._request_headers.populateHttpHeader(page.call_arena, &headers);
     if (cookie_support) {
-        try page.headersForRequest(self._arena, self._url, &headers);
+        try page.headersForRequest(&headers);
     }
 
     try http_client.request(.{
@@ -235,6 +235,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?[]const u8) !void {
         .frame_id = page._frame_id,
         .body = self._request_body,
         .cookie_jar = if (cookie_support) &page._session.cookie_jar else null,
+        .cookie_origin = page.url,
         .resource_type = .xhr,
         .notification = page._session.notification,
         .start_callback = httpStartCallback,

src/cdp/CDP.zig

@@ -468,8 +468,8 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             if (self.http_proxy_changed) {
                 // has to be called after browser.closeSession, since it won't
                 // work if there are active connections.
-                browser.http_client.restoreOriginalProxy() catch |err| {
-                    log.warn(.http, "restoreOriginalProxy", .{ .err = err });
+                browser.http_client.changeProxy(null) catch |err| {
+                    log.warn(.http, "changeProxy", .{ .err = err });
                 };
             }
             self.intercept_state.deinit();

src/cdp/domains/network.zig

@@ -351,6 +351,10 @@ pub const TransferAsRequestWriter = struct {
                 try jws.objectField(hdr.name);
                 try jws.write(hdr.value);
             }
+            if (try transfer.getCookieString()) |cookies| {
+                try jws.objectField("Cookie");
+                try jws.write(cookies[0 .. cookies.len - 1]);
+            }
             try jws.endObject();
         }
         try jws.endObject();

src/network/Runtime.zig

@@ -461,7 +461,7 @@ fn drainQueue(self: *Runtime) void {
             self.releaseConnection(conn);
             continue;
         };
-        libcurl.curl_multi_add_handle(multi, conn.easy) catch |err| {
+        libcurl.curl_multi_add_handle(multi, conn._easy) catch |err| {
             lp.log.err(.app, "curl multi add", .{ .err = err });
             self.releaseConnection(conn);
         };
@@ -565,7 +565,7 @@ pub fn getConnection(self: *Runtime) ?*net_http.Connection {
 }
 
 pub fn releaseConnection(self: *Runtime, conn: *net_http.Connection) void {
-    conn.reset() catch |err| {
+    conn.reset(self.config, self.ca_blob) catch |err| {
         lp.assert(false, "couldn't reset curl easy", .{ .err = err });
     };
 

src/network/http.zig

@@ -54,14 +54,13 @@ pub const Header = struct {
 
 pub const Headers = struct {
     headers: ?*libcurl.CurlSList,
-    cookies: ?[*c]const u8,
 
     pub fn init(user_agent: [:0]const u8) !Headers {
         const header_list = libcurl.curl_slist_append(null, user_agent);
         if (header_list == null) {
             return error.OutOfMemory;
         }
-        return .{ .headers = header_list, .cookies = null };
+        return .{ .headers = header_list };
     }
 
     pub fn deinit(self: *const Headers) void {
@@ -92,20 +91,14 @@ pub const Headers = struct {
     pub fn iterator(self: *Headers) Iterator {
         return .{
             .header = self.headers,
-            .cookies = self.cookies,
         };
     }
 
     const Iterator = struct {
         header: [*c]libcurl.CurlSList,
-        cookies: ?[*c]const u8,
 
         pub fn next(self: *Iterator) ?Header {
-            const h = self.header orelse {
-                const cookies = self.cookies orelse return null;
-                self.cookies = null;
-                return .{ .name = "Cookie", .value = std.mem.span(@as([*:0]const u8, cookies)) };
-            };
+            const h = self.header orelse return null;
 
             self.header = h.*.next;
             return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));
@@ -132,7 +125,7 @@ pub const HeaderIterator = union(enum) {
         prev: ?*libcurl.CurlHeader = null,
 
         pub fn next(self: *CurlHeaderIterator) ?Header {
-            const h = libcurl.curl_easy_nextheader(self.conn.easy, .header, -1, self.prev) orelse return null;
+            const h = libcurl.curl_easy_nextheader(self.conn._easy, .header, -1, self.prev) orelse return null;
             self.prev = h;
 
             const header = h.*;
@@ -164,33 +157,24 @@ const HeaderValue = struct {
 };
 
 pub const AuthChallenge = struct {
+    const Source = enum { server, proxy };
+    const Scheme = enum { basic, digest };
+
     status: u16,
-    source: ?enum { server, proxy },
-    scheme: ?enum { basic, digest },
+    source: ?Source,
+    scheme: ?Scheme,
     realm: ?[]const u8,
 
-    pub fn parse(status: u16, header: []const u8) !AuthChallenge {
+    pub fn parse(status: u16, source: Source, value: []const u8) !AuthChallenge {
         var ac: AuthChallenge = .{
             .status = status,
-            .source = null,
+            .source = source,
             .realm = null,
             .scheme = null,
         };
 
-        const sep = std.mem.indexOfPos(u8, header, 0, ": ") orelse return error.InvalidHeader;
-        const hname = header[0..sep];
-        const hvalue = header[sep + 2 ..];
-
-        if (std.ascii.eqlIgnoreCase("WWW-Authenticate", hname)) {
-            ac.source = .server;
-        } else if (std.ascii.eqlIgnoreCase("Proxy-Authenticate", hname)) {
-            ac.source = .proxy;
-        } else {
-            return error.InvalidAuthChallenge;
-        }
-
-        const pos = std.mem.indexOfPos(u8, std.mem.trim(u8, hvalue, std.ascii.whitespace[0..]), 0, " ") orelse hvalue.len;
-        const _scheme = hvalue[0..pos];
+        const pos = std.mem.indexOfPos(u8, std.mem.trim(u8, value, std.ascii.whitespace[0..]), 0, " ") orelse value.len;
+        const _scheme = value[0..pos];
         if (std.ascii.eqlIgnoreCase(_scheme, "basic")) {
             ac.scheme = .basic;
         } else if (std.ascii.eqlIgnoreCase(_scheme, "digest")) {
@@ -226,77 +210,28 @@ pub const ResponseHead = struct {
 };
 
 pub const Connection = struct {
-    easy: *libcurl.Curl,
+    _easy: *libcurl.Curl,
     node: std.DoublyLinkedList.Node = .{},
 
     pub fn init(
-        ca_blob_: ?libcurl.CurlBlob,
+        ca_blob: ?libcurl.CurlBlob,
         config: *const Config,
     ) !Connection {
         const easy = libcurl.curl_easy_init() orelse return error.FailedToInitializeEasy;
-        errdefer libcurl.curl_easy_cleanup(easy);
 
-        // timeouts
-        try libcurl.curl_easy_setopt(easy, .timeout_ms, config.httpTimeout());
-        try libcurl.curl_easy_setopt(easy, .connect_timeout_ms, config.httpConnectTimeout());
+        const self = Connection{ ._easy = easy };
+        errdefer self.deinit();
 
-        // redirect behavior
-        try libcurl.curl_easy_setopt(easy, .max_redirs, config.httpMaxRedirects());
-        try libcurl.curl_easy_setopt(easy, .follow_location, 2);
-        try libcurl.curl_easy_setopt(easy, .redir_protocols_str, "HTTP,HTTPS"); // remove FTP and FTPS from the default
-
-        // proxy
-        const http_proxy = config.httpProxy();
-        if (http_proxy) |proxy| {
-            try libcurl.curl_easy_setopt(easy, .proxy, proxy.ptr);
-        }
-
-        // tls
-        if (ca_blob_) |ca_blob| {
-            try libcurl.curl_easy_setopt(easy, .ca_info_blob, ca_blob);
-            if (http_proxy != null) {
-                try libcurl.curl_easy_setopt(easy, .proxy_ca_info_blob, ca_blob);
-            }
-        } else {
-            assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{});
-
-            try libcurl.curl_easy_setopt(easy, .ssl_verify_host, false);
-            try libcurl.curl_easy_setopt(easy, .ssl_verify_peer, false);
-
-            if (http_proxy != null) {
-                try libcurl.curl_easy_setopt(easy, .proxy_ssl_verify_host, false);
-                try libcurl.curl_easy_setopt(easy, .proxy_ssl_verify_peer, false);
-            }
-        }
-
-        // compression, don't remove this. CloudFront will send gzip content
-        // even if we don't support it, and then it won't be decompressed.
-        // empty string means: use whatever's available
-        try libcurl.curl_easy_setopt(easy, .accept_encoding, "");
-
-        // debug
-        if (comptime ENABLE_DEBUG) {
-            try libcurl.curl_easy_setopt(easy, .verbose, true);
-
-            // Sometimes the default debug output hides some useful data. You can
-            // uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to
-            // get more control over the data (specifically, the `CURLINFO_TEXT`
-            // can include useful data).
-
-            // try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback);
-        }
-
-        return .{
-            .easy = easy,
-        };
+        try self.reset(config, ca_blob);
+        return self;
     }
 
     pub fn deinit(self: *const Connection) void {
-        libcurl.curl_easy_cleanup(self.easy);
+        libcurl.curl_easy_cleanup(self._easy);
     }
 
     pub fn setURL(self: *const Connection, url: [:0]const u8) !void {
-        try libcurl.curl_easy_setopt(self.easy, .url, url.ptr);
+        try libcurl.curl_easy_setopt(self._easy, .url, url.ptr);
     }
 
     // a libcurl request has 2 methods. The first is the method that
@@ -319,7 +254,7 @@ pub const Connection = struct {
     // can infer that based on the presence of the body, but we also reset it
     // to be safe);
     pub fn setMethod(self: *const Connection, method: Method) !void {
-        const easy = self.easy;
+        const easy = self._easy;
         const m: [:0]const u8 = switch (method) {
             .GET => "GET",
             .POST => "POST",
@@ -334,56 +269,97 @@ pub const Connection = struct {
     }
 
     pub fn setBody(self: *const Connection, body: []const u8) !void {
-        const easy = self.easy;
+        const easy = self._easy;
         try libcurl.curl_easy_setopt(easy, .post, true);
         try libcurl.curl_easy_setopt(easy, .post_field_size, body.len);
         try libcurl.curl_easy_setopt(easy, .copy_post_fields, body.ptr);
     }
 
     pub fn setGetMode(self: *const Connection) !void {
-        try libcurl.curl_easy_setopt(self.easy, .http_get, true);
+        try libcurl.curl_easy_setopt(self._easy, .http_get, true);
     }
 
     pub fn setHeaders(self: *const Connection, headers: *Headers) !void {
-        try libcurl.curl_easy_setopt(self.easy, .http_header, headers.headers);
+        try libcurl.curl_easy_setopt(self._easy, .http_header, headers.headers);
     }
 
     pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void {
-        try libcurl.curl_easy_setopt(self.easy, .cookie, cookies);
+        try libcurl.curl_easy_setopt(self._easy, .cookie, cookies);
     }
 
     pub fn setPrivate(self: *const Connection, ptr: *anyopaque) !void {
-        try libcurl.curl_easy_setopt(self.easy, .private, ptr);
+        try libcurl.curl_easy_setopt(self._easy, .private, ptr);
     }
 
     pub fn setProxyCredentials(self: *const Connection, creds: [:0]const u8) !void {
-        try libcurl.curl_easy_setopt(self.easy, .proxy_user_pwd, creds.ptr);
+        try libcurl.curl_easy_setopt(self._easy, .proxy_user_pwd, creds.ptr);
     }
 
     pub fn setCredentials(self: *const Connection, creds: [:0]const u8) !void {
-        try libcurl.curl_easy_setopt(self.easy, .user_pwd, creds.ptr);
+        try libcurl.curl_easy_setopt(self._easy, .user_pwd, creds.ptr);
     }
 
     pub fn setCallbacks(
-        self: *const Connection,
-        comptime header_cb: libcurl.CurlHeaderFunction,
+        self: *Connection,
         comptime data_cb: libcurl.CurlWriteFunction,
     ) !void {
-        try libcurl.curl_easy_setopt(self.easy, .header_data, self.easy);
-        try libcurl.curl_easy_setopt(self.easy, .header_function, header_cb);
-        try libcurl.curl_easy_setopt(self.easy, .write_data, self.easy);
-        try libcurl.curl_easy_setopt(self.easy, .write_function, data_cb);
+        try libcurl.curl_easy_setopt(self._easy, .write_data, self);
+        try libcurl.curl_easy_setopt(self._easy, .write_function, data_cb);
     }
 
-    pub fn reset(self: *const Connection) !void {
-        try libcurl.curl_easy_setopt(self.easy, .proxy, null);
-        try libcurl.curl_easy_setopt(self.easy, .http_header, null);
+    pub fn reset(
+        self: *const Connection,
+        config: *const Config,
+        ca_blob: ?libcurl.CurlBlob,
+    ) !void {
+        libcurl.curl_easy_reset(self._easy);
 
-        try libcurl.curl_easy_setopt(self.easy, .header_data, null);
-        try libcurl.curl_easy_setopt(self.easy, .header_function, null);
+        // timeouts
+        try libcurl.curl_easy_setopt(self._easy, .timeout_ms, config.httpTimeout());
+        try libcurl.curl_easy_setopt(self._easy, .connect_timeout_ms, config.httpConnectTimeout());
 
-        try libcurl.curl_easy_setopt(self.easy, .write_data, null);
-        try libcurl.curl_easy_setopt(self.easy, .write_function, discardBody);
+        // compression, don't remove this. CloudFront will send gzip content
+        // even if we don't support it, and then it won't be decompressed.
+        // empty string means: use whatever's available
+        try libcurl.curl_easy_setopt(self._easy, .accept_encoding, "");
+
+        // proxy
+        const http_proxy = config.httpProxy();
+        if (http_proxy) |proxy| {
+            try libcurl.curl_easy_setopt(self._easy, .proxy, proxy.ptr);
+        } else {
+            try libcurl.curl_easy_setopt(self._easy, .proxy, null);
+        }
+
+        // tls
+        if (ca_blob) |ca| {
+            try libcurl.curl_easy_setopt(self._easy, .ca_info_blob, ca);
+            if (http_proxy != null) {
+                try libcurl.curl_easy_setopt(self._easy, .proxy_ca_info_blob, ca);
+            }
+        } else {
+            assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{});
+
+            try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);
+            try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);
+
+            if (http_proxy != null) {
+                try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, false);
+                try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, false);
+            }
+        }
+
+        // debug
+        if (comptime ENABLE_DEBUG) {
+            try libcurl.curl_easy_setopt(self._easy, .verbose, true);
+
+            // Sometimes the default debug output hides some useful data. You can
+            // uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to
+            // get more control over the data (specifically, the `CURLINFO_TEXT`
+            // can include useful data).
+
+            // try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback);
+        }
     }
 
     fn discardBody(_: [*]const u8, count: usize, len: usize, _: ?*anyopaque) usize {
@@ -391,27 +367,31 @@ pub const Connection = struct {
     }
 
     pub fn setProxy(self: *const Connection, proxy: ?[:0]const u8) !void {
-        try libcurl.curl_easy_setopt(self.easy, .proxy, if (proxy) |p| p.ptr else null);
+        try libcurl.curl_easy_setopt(self._easy, .proxy, if (proxy) |p| p.ptr else null);
+    }
+
+    pub fn setFollowLocation(self: *const Connection, follow: bool) !void {
+        try libcurl.curl_easy_setopt(self._easy, .follow_location, @as(c_long, if (follow) 2 else 0));
     }
 
     pub fn setTlsVerify(self: *const Connection, verify: bool, use_proxy: bool) !void {
-        try libcurl.curl_easy_setopt(self.easy, .ssl_verify_host, verify);
-        try libcurl.curl_easy_setopt(self.easy, .ssl_verify_peer, verify);
+        try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, verify);
+        try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, verify);
         if (use_proxy) {
-            try libcurl.curl_easy_setopt(self.easy, .proxy_ssl_verify_host, verify);
-            try libcurl.curl_easy_setopt(self.easy, .proxy_ssl_verify_peer, verify);
+            try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, verify);
+            try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, verify);
         }
     }
 
     pub fn getEffectiveUrl(self: *const Connection) ![*c]const u8 {
         var url: [*c]u8 = undefined;
-        try libcurl.curl_easy_getinfo(self.easy, .effective_url, &url);
+        try libcurl.curl_easy_getinfo(self._easy, .effective_url, &url);
         return url;
     }
 
     pub fn getResponseCode(self: *const Connection) !u16 {
         var status: c_long = undefined;
-        try libcurl.curl_easy_getinfo(self.easy, .response_code, &status);
+        try libcurl.curl_easy_getinfo(self._easy, .response_code, &status);
         if (status < 0 or status > std.math.maxInt(u16)) {
             return 0;
         }
@@ -420,13 +400,13 @@ pub const Connection = struct {
 
     pub fn getRedirectCount(self: *const Connection) !u32 {
         var count: c_long = undefined;
-        try libcurl.curl_easy_getinfo(self.easy, .redirect_count, &count);
+        try libcurl.curl_easy_getinfo(self._easy, .redirect_count, &count);
         return @intCast(count);
     }
 
     pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
         var hdr: ?*libcurl.CurlHeader = null;
-        libcurl.curl_easy_header(self.easy, name, index, .header, -1, &hdr) catch |err| {
+        libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| {
             // ErrorHeader includes OutOfMemory — rare but real errors from curl internals.
             // Logged and returned as null since callers don't expect errors.
             log.err(.http, "get response header", .{
@@ -444,7 +424,7 @@ pub const Connection = struct {
 
     pub fn getPrivate(self: *const Connection) !*anyopaque {
         var private: *anyopaque = undefined;
-        try libcurl.curl_easy_getinfo(self.easy, .private, &private);
+        try libcurl.curl_easy_getinfo(self._easy, .private, &private);
         return private;
     }
 
@@ -461,12 +441,7 @@ pub const Connection = struct {
         try self.secretHeaders(&header_list, http_headers);
         try self.setHeaders(&header_list);
 
-        // Add cookies.
-        if (header_list.cookies) |cookies| {
-            try self.setCookies(cookies);
-        }
-
-        try libcurl.curl_easy_perform(self.easy);
+        try libcurl.curl_easy_perform(self._easy);
         return self.getResponseCode();
     }
 };
@@ -488,11 +463,11 @@ pub const Handles = struct {
     }
 
     pub fn add(self: *Handles, conn: *const Connection) !void {
-        try libcurl.curl_multi_add_handle(self.multi, conn.easy);
+        try libcurl.curl_multi_add_handle(self.multi, conn._easy);
     }
 
     pub fn remove(self: *Handles, conn: *const Connection) !void {
-        try libcurl.curl_multi_remove_handle(self.multi, conn.easy);
+        try libcurl.curl_multi_remove_handle(self.multi, conn._easy);
     }
 
     pub fn perform(self: *Handles) !c_int {
@@ -515,7 +490,7 @@ pub const Handles = struct {
         const msg = libcurl.curl_multi_info_read(self.multi, &messages_count) orelse return null;
         return switch (msg.data) {
             .done => |err| .{
-                .conn = .{ .easy = msg.easy_handle },
+                .conn = .{ ._easy = msg.easy_handle },
                 .err = err,
             },
             else => unreachable,

src/sys/libcurl.zig

@@ -516,6 +516,10 @@ pub fn curl_easy_cleanup(easy: *Curl) void {
     c.curl_easy_cleanup(easy);
 }
 
+pub fn curl_easy_reset(easy: *Curl) void {
+    c.curl_easy_reset(easy);
+}
+
 pub fn curl_easy_perform(easy: *Curl) Error!void {
     try errorCheck(c.curl_easy_perform(easy));
 }