cookie support

src/browser/ScriptManager.zig

@@ -222,6 +222,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
         .url = remote_url.?,
         .ctx = pending_script,
         .method = .GET,
+        .cookie = page.requestCookie(.{}),
         .start_callback = if (log.enabled(.http, .debug)) startCallback else null,
         .header_done_callback = headerCallback,
         .data_callback = dataCallback,
@@ -274,6 +275,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
         .url = url,
         .method = .GET,
         .ctx = &blocking,
+        .cookie = self.page.requestCookie(.{}),
         .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null,
         .header_done_callback = Blocking.headerCallback,
         .data_callback = Blocking.dataCallback,

src/browser/html/document.zig

@@ -85,7 +85,10 @@ pub const HTMLDocument = struct {
 
     pub fn get_cookie(_: *parser.DocumentHTML, page: *Page) ![]const u8 {
         var buf: std.ArrayListUnmanaged(u8) = .{};
-        try page.cookie_jar.forRequest(&page.url.uri, buf.writer(page.arena), .{ .navigation = true, .is_http = false });
+        try page.cookie_jar.forRequest(&page.url.uri, buf.writer(page.arena), .{
+            .is_http = false,
+            .is_navigation = true,
+        });
         return buf.items;
     }
 

src/browser/page.zig

@@ -426,6 +426,19 @@ pub const Page = struct {
         return arr.items;
     }
 
+    const RequestCookieOpts = struct {
+        is_http: bool = true,
+        is_navigation: bool = false,
+    };
+    pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie {
+        return .{
+            .jar = self.cookie_jar,
+            .origin = &self.url.uri,
+            .is_http = opts.is_http,
+            .is_navigation = opts.is_navigation,
+        };
+    }
+
     // spec reference: https://html.spec.whatwg.org/#document-lifecycle
     pub fn navigate(self: *Page, request_url: []const u8, opts: NavigateOpts) !void {
         if (self.mode != .pre) {
@@ -453,17 +466,22 @@ pub const Page = struct {
         }
 
         const owned_url = try self.arena.dupeZ(u8, request_url);
+        self.url = try URL.parse(owned_url, null);
 
-        try self.http_client.request(.{
+        self.http_client.request(.{
             .ctx = self,
             .url = owned_url,
             .method = opts.method,
             .body = opts.body,
+            .cookie = self.requestCookie(.{ .is_navigation = true }),
             .header_done_callback = pageHeaderDoneCallback,
             .data_callback = pageDataCallback,
             .done_callback = pageDoneCallback,
             .error_callback = pageErrorCallback,
-        });
+        }) catch |err| {
+            log.err(.http, "navigate request", .{ .url = owned_url, .err = err });
+            return err;
+        };
 
         self.session.browser.notification.dispatch(.page_navigate, &.{
             .opts = opts,

src/browser/storage/cookie.zig

@@ -10,8 +10,8 @@ const public_suffix_list = @import("../../data/public_suffix_list.zig").lookup;
 pub const LookupOpts = struct {
     request_time: ?i64 = null,
     origin_uri: ?*const Uri = null,
-    navigation: bool = true,
     is_http: bool,
+    is_navigation: bool = true,
 };
 
 pub const Jar = struct {
@@ -91,7 +91,7 @@ pub const Jar = struct {
 
         var first = true;
         for (self.cookies.items) |*cookie| {
-            if (!cookie.appliesTo(&target, same_site, opts.navigation, opts.is_http)) continue;
+            if (!cookie.appliesTo(&target, same_site, opts.is_navigation, opts.is_http)) continue;
 
             // we have a match!
             if (first) {
@@ -103,18 +103,15 @@ pub const Jar = struct {
         }
     }
 
-    // @newhttp
+    pub fn populateFromResponse(self: *Jar, uri: *const Uri, set_cookie: []const u8) !void {
-    // pub fn populateFromResponse(self: *Jar, uri: *const Uri, header: *const http.ResponseHeader) !void {
+        const c = Cookie.parse(self.allocator, uri, set_cookie) catch |err| {
-    //     const now = std.time.timestamp();
+            log.warn(.web_api, "cookie parse failed", .{ .raw = set_cookie, .err = err });
-    //     var it = header.iterate("set-cookie");
+            return;
-    //     while (it.next()) |set_cookie| {
+        };
-    //         const c = Cookie.parse(self.allocator, uri, set_cookie) catch |err| {
+
-    //             log.warn(.web_api, "cookie parse failed", .{ .raw = set_cookie, .err = err });
+        const now = std.time.timestamp();
-    //             continue;
+        try self.add(c, now);
-    //         };
+    }
-    //         try self.add(c, now);
-    //     }
-    // }
 
     fn writeCookie(cookie: *const Cookie, writer: anytype) !void {
         if (cookie.name.len > 0) {
@@ -429,7 +426,7 @@ pub const Cookie = struct {
         return .{ name, value, rest };
     }
 
-    pub fn appliesTo(self: *const Cookie, url: *const PreparedUri, same_site: bool, navigation: bool, is_http: bool) bool {
+    pub fn appliesTo(self: *const Cookie, url: *const PreparedUri, same_site: bool, is_navigation: bool, is_http: bool) bool {
         if (self.http_only and is_http == false) {
             // http only cookies can be accessed from Javascript
             return false;
@@ -448,7 +445,7 @@ pub const Cookie = struct {
             // and cookie.same_site == .lax
             switch (self.same_site) {
                 .strict => return false,
-                .lax => if (navigation == false) return false,
+                .lax => if (is_navigation == false) return false,
                 .none => {},
             }
         }
@@ -619,7 +616,7 @@ test "Jar: forRequest" {
 
     // nothing fancy here
     try expectCookies("global1=1; global2=2", &jar, test_uri, .{ .is_http = true });
-    try expectCookies("global1=1; global2=2", &jar, test_uri, .{ .origin_uri = &test_uri, .navigation = false, .is_http = true });
+    try expectCookies("global1=1; global2=2", &jar, test_uri, .{ .origin_uri = &test_uri, .is_navigation = false, .is_http = true });
 
     // We have a cookie where Domain=lightpanda.io
     // This should _not_ match xyxlightpanda.io
@@ -685,22 +682,22 @@ test "Jar: forRequest" {
     // non-navigational cross domain, insecure
     try expectCookies("", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{
         .origin_uri = &(try std.Uri.parse("https://example.com/")),
-        .navigation = false,
         .is_http = true,
+        .is_navigation = false,
     });
 
     // non-navigational cross domain, secure
     try expectCookies("sitenone=6", &jar, try std.Uri.parse("https://lightpanda.io/x/"), .{
         .origin_uri = &(try std.Uri.parse("https://example.com/")),
-        .navigation = false,
         .is_http = true,
+        .is_navigation = false,
     });
 
     // non-navigational same origin
     try expectCookies("global1=1; global2=2; sitelax=7; sitestrict=8", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{
         .origin_uri = &(try std.Uri.parse("https://lightpanda.io/")),
-        .navigation = false,
         .is_http = true,
+        .is_navigation = false,
     });
 
     // exact domain match + suffix

src/browser/xhr/xhr.zig

@@ -81,7 +81,6 @@ pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     arena: Allocator,
     transfer: ?*HttpClient.Transfer = null,
-    cookie_jar: *CookieJar,
     err: ?anyerror = null,
     last_dispatch: i64 = 0,
     send_flag: bool = false,
@@ -169,7 +168,6 @@ pub const XMLHttpRequest = struct {
             .headers = .{},
             .method = undefined,
             .state = .unsent,
-            .cookie_jar = page.cookie_jar,
         };
     }
 
@@ -378,6 +376,7 @@ pub const XMLHttpRequest = struct {
             .method = self.method,
             .body = self.request_body,
             .content_type = "Content-Type: text/plain; charset=UTF-8", // @newhttp TODO
+            .cookie = page.requestCookie(.{}),
             .start_callback = httpStartCallback,
             .header_callback = httpHeaderCallback,
             .header_done_callback = httpHeaderDoneCallback,
@@ -395,20 +394,6 @@ pub const XMLHttpRequest = struct {
         }
 
         log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" });
-
-        // @newhttp
-        // {
-        //     var arr: std.ArrayListUnmanaged(u8) = .{};
-        //     try self.cookie_jar.forRequest(&self.url.?.uri, arr.writer(self.arena), .{
-        //         .navigation = false,
-        //         .origin_uri = &self.origin_url.uri,
-        //         .is_http = true,
-        //     });
-
-        //     if (arr.items.len > 0) {
-        //         try request.addHeader("Cookie", arr.items, .{});
-        //     }
-        // }
         self.transfer = transfer;
     }
 
@@ -445,9 +430,6 @@ pub const XMLHttpRequest = struct {
 
         self.state = .loading;
         self.dispatchEvt("readystatechange");
-
-        // @newhttp
-        // try self.cookie_jar.populateFromResponse(self.request.?.request_uri, &header);
     }
 
     fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {

src/data/public_suffix_list.zig

@@ -8,8 +8,7 @@ pub fn lookup(value: []const u8) bool {
 const public_suffix_list = std.StaticStringMap(void).initComptime(entries);
 
 const entries: []const struct { []const u8, void } =
-    // @newhttp
+    if (builtin.is_test) &.{
-    if (builtin.is_test or true) &.{
         .{ "api.gov.uk", {} },
         .{ "gov.uk", {} },
     } else &.{

src/http/Client.zig

@@ -24,6 +24,7 @@ const Http = @import("Http.zig");
 const c = Http.c;
 
 const Allocator = std.mem.Allocator;
+const ArenaAllocator = std.heap.ArenaAllocator;
 
 const errorCheck = Http.errorCheck;
 const errorMCheck = Http.errorMCheck;
@@ -43,18 +44,49 @@ pub const Method = Http.Method;
 // those other http requests.
 pub const Client = @This();
 
+// count of active requests
 active: usize,
+
+// curl has 2 APIs: easy and multi. Multi is like a combination of some I/O block
+// (e.g. epoll) and a bunch of pools. You add/remove easys to the multiple and
+// then poll the multi.
 multi: *c.CURLM,
+
+// Our easy handles. Although the multi contains buffer pools and connections
+// pools, re-using the easys is still recommended. This acts as our own pool
+// of easys.
 handles: Handles,
+
+// When handles has no more available easys, requests get queued.
 queue: RequestQueue,
-allocator: Allocator,
+
-transfer_pool: std.heap.MemoryPool(Transfer),
+// Memory pool for Queue nodes.
 queue_node_pool: std.heap.MemoryPool(RequestQueue.Node),
+
+// The main app allocator
+allocator: Allocator,
+
+// Once we have a handle/easy to process a request with, we create a Transfer
+// which contains the Request as well as any state we need to process the
+// request. These wil come and go with each request.
+transfer_pool: std.heap.MemoryPool(Transfer),
+
 //@newhttp
 http_proxy: ?std.Uri = null,
+
+// see ScriptManager.blockingGet
 blocking: Handle,
+
+// Boolean to check that we don't make a blocking request while an existing
+// blocking request is already being processed.
 blocking_active: if (builtin.mode == .Debug) bool else void = if (builtin.mode == .Debug) false else {},
 
+// The only place this is meant to be used is in `makeRequest` BEFORE `perform`
+// is called. It is used to generate our Cookie header. It can be used for other
+// purposes, but keep in mind that, while single-threaded, calls like makeRequest
+// can result in makeRequest being re-called (from a doneCallback).
+arena: ArenaAllocator,
+
 const RequestQueue = std.DoublyLinkedList(Request);
 
 pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Client {
@@ -85,6 +117,7 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie
         .allocator = allocator,
         .transfer_pool = transfer_pool,
         .queue_node_pool = queue_node_pool,
+        .arena = ArenaAllocator.init(allocator),
     };
 
     return client;
@@ -99,6 +132,7 @@ pub fn deinit(self: *Client) void {
 
     self.transfer_pool.deinit();
     self.queue_node_pool.deinit();
+    self.arena.deinit();
     self.allocator.destroy(self);
 }
 
@@ -176,6 +210,13 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
     const conn = handle.conn;
     const easy = conn.easy;
 
+    // we need this for cookies
+    const uri = std.Uri.parse(req.url) catch |err| {
+        self.handles.release(handle);
+        log.warn(.http, "invalid url", .{ .err = err, .url = req.url });
+        return;
+    };
+
     const header_list = blk: {
         errdefer self.handles.release(handle);
         try conn.setMethod(req.method);
@@ -192,6 +233,23 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
             header_list = c.curl_slist_append(header_list, ct);
         }
 
+        {
+            const COOKIE_HEADER = "Cookie: ";
+            const aa = self.arena.allocator();
+            defer _ = self.arena.reset(.{ .retain_with_limit = 2048 });
+
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            try arr.appendSlice(aa, COOKIE_HEADER);
+            try req.cookie.forRequest(&uri, arr.writer(aa));
+
+            if (arr.items.len > COOKIE_HEADER.len) {
+                try arr.append(aa, 0); //null terminate
+
+                // copies the value
+                header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr));
+            }
+        }
+
         try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list));
 
         break :blk header_list;
@@ -203,12 +261,14 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void {
         const transfer = try self.transfer_pool.create();
         transfer.* = .{
             .id = 0,
+            .uri = uri,
             .req = req,
             .ctx = req.ctx,
             .handle = handle,
             ._request_header_list = header_list,
         };
         errdefer self.transfer_pool.destroy(transfer);
+
         try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer));
 
         try errorMCheck(c.curl_multi_add_handle(self.multi, easy));
@@ -370,11 +430,27 @@ const Handle = struct {
     }
 };
 
+pub const RequestCookie = struct {
+    is_http: bool,
+    is_navigation: bool,
+    origin: *const std.Uri,
+    jar: *@import("../browser/storage/cookie.zig").Jar,
+
+    fn forRequest(self: *const RequestCookie, uri: *const std.Uri, writer: anytype) !void {
+        return self.jar.forRequest(uri, writer, .{
+            .is_http = self.is_http,
+            .is_navigation = self.is_navigation,
+            .origin_uri = self.origin,
+        });
+    }
+};
+
 pub const Request = struct {
     method: Method,
     url: [:0]const u8,
     body: ?[]const u8 = null,
     content_type: ?[:0]const u8 = null,
+    cookie: RequestCookie,
 
     // arbitrary data that can be associated with this request
     ctx: *anyopaque = undefined,
@@ -391,6 +467,7 @@ pub const Transfer = struct {
     id: usize,
     req: Request,
     ctx: *anyopaque,
+    uri: std.Uri, // used for setting/getting the cookie
 
     // We'll store the response header here
     response_header: ?Header = null,
@@ -479,10 +556,10 @@ pub const Transfer = struct {
             return buf_len;
         }
 
-        const CONTENT_TYPE_LEN = "content-type:".len;
-
         var hdr = &transfer.response_header.?;
+
         if (hdr._content_type_len == 0) {
+            const CONTENT_TYPE_LEN = "content-type:".len;
             if (buf_len > CONTENT_TYPE_LEN) {
                 if (std.ascii.eqlIgnoreCase(header[0..CONTENT_TYPE_LEN], "content-type:")) {
                     const value = std.mem.trimLeft(u8, header[CONTENT_TYPE_LEN..], " ");
@@ -493,6 +570,18 @@ pub const Transfer = struct {
             }
         }
 
+        {
+            const SET_COOKIE_LEN = "set-cookie:".len;
+            if (buf_len > SET_COOKIE_LEN) {
+                if (std.ascii.eqlIgnoreCase(header[0..SET_COOKIE_LEN], "set-cookie:")) {
+                    const value = std.mem.trimLeft(u8, header[SET_COOKIE_LEN..], " ");
+                    transfer.req.cookie.jar.populateFromResponse(&transfer.uri, value) catch |err| {
+                        log.err(.http, "set cookie", .{ .err = err, .req = transfer });
+                    };
+                }
+            }
+        }
+
         if (buf_len == 2) {
             transfer.req.header_done_callback(transfer) catch |err| {
                 log.err(.http, "header_done_callback", .{ .err = err, .req = transfer });