add cookies to XHR requests

src/browser/browser.zig

@@ -380,7 +380,7 @@ pub const Page = struct {
 
         var response = try request.sendSync(.{});
         const header = response.header;
-        try self.processHTTPResponse(self.uri, &header);
+        try self.session.cookie_jar.populateFromResponse(self.uri, &header);
 
         log.info("GET {any} {d}", .{ self.uri, header.status });
 
@@ -445,7 +445,9 @@ pub const Page = struct {
 
         // replace the user context document with the new one.
         try session.env.setUserContext(.{
+            .uri = self.uri,
             .document = html_doc,
+            .cookie_jar = @ptrCast(&self.session.cookie_jar),
             .http_client = @ptrCast(self.session.http_client),
         });
 
@@ -621,14 +623,14 @@ pub const Page = struct {
         const u = try std.Uri.resolve_inplace(self.uri, res_src, &b);
 
         var request = try self.newHTTPRequest(.GET, u, .{
-            .origin = self.uri,
+            .origin_uri = self.uri,
             .navigation = false,
         });
         defer request.deinit();
 
         var response = try request.sendSync(.{});
         var header = response.header;
-        try self.processHTTPResponse(u, &header);
+        try self.session.cookie_jar.populateFromResponse(u, &header);
 
         log.info("fetch {any}: {d}", .{ u, header.status });
 
@@ -657,46 +659,21 @@ pub const Page = struct {
         try s.eval(arena, &self.session.env, body);
     }
 
-    const RequestOpts = struct {
-        origin: ?std.Uri = null,
-        navigation: bool = true,
-    };
-    fn newHTTPRequest(self: *const Page, method: http.Request.Method, uri: std.Uri, opts: RequestOpts) !http.Request {
+    fn newHTTPRequest(self: *const Page, method: http.Request.Method, uri: std.Uri, opts: storage.cookie.LookupOpts) !http.Request {
         const session = self.session;
         var request = try session.http_client.request(method, uri);
         errdefer request.deinit();
 
-        var cookies = try session.cookie_jar.forRequest(
-            self.arena,
-            std.time.timestamp(),
-            opts.origin,
-            uri,
-            opts.navigation,
-        );
-        defer cookies.deinit(self.arena);
+        var arr: std.ArrayListUnmanaged(u8) = .{};
+        try session.cookie_jar.forRequest(uri, arr.writer(self.arena), opts);
 
-        if (cookies.len() > 0) {
-            var arr: std.ArrayListUnmanaged(u8) = .{};
-            try cookies.write(arr.writer(self.arena));
+        if (arr.items.len > 0) {
             try request.addHeader("Cookie", arr.items, .{});
         }
 
         return request;
     }
 
-    fn processHTTPResponse(self: *const Page, uri: std.Uri, header: *const http.ResponseHeader) !void {
-        const session = self.session;
-        const now = std.time.timestamp();
-        var it = header.iterate("set-cookie");
-        while (it.next()) |set_cookie| {
-            const c = storage.Cookie.parse(self.arena, uri, set_cookie) catch |err| {
-                log.warn("Couldn't parse cookie '{s}': {}\n", .{ set_cookie, err });
-                continue;
-            };
-            try session.cookie_jar.add(c, now);
-        }
-    }
-
     const Script = struct {
         element: *parser.Element,
         kind: Kind,

src/main_tests.zig

@@ -28,8 +28,7 @@ const apiweb = @import("apiweb.zig");
 const Window = @import("html/window.zig").Window;
 const xhr = @import("xhr/xhr.zig");
 const storage = @import("storage/storage.zig");
-const url = @import("url/url.zig");
-const URL = url.URL;
+const URL = @import("url/url.zig").URL;
 const urlquery = @import("url/query.zig");
 const Location = @import("html/location.zig").Location;
 
@@ -54,7 +53,7 @@ const EventTestExecFn = @import("events/event.zig").testExecFn;
 const XHRTestExecFn = xhr.testExecFn;
 const ProgressEventTestExecFn = @import("xhr/progress_event.zig").testExecFn;
 const StorageTestExecFn = storage.testExecFn;
-const URLTestExecFn = url.testExecFn;
+const URLTestExecFn = @import("url/url.zig").testExecFn;
 const HTMLElementTestExecFn = @import("html/elements.zig").testExecFn;
 const MutationObserverTestExecFn = @import("dom/mutation_observer.zig").testExecFn;
 
@@ -91,16 +90,23 @@ fn testExecFn(
     var http_client = try @import("http/client.zig").Client.init(alloc, 5, .{});
     defer http_client.deinit();
 
-    try js_env.setUserContext(.{
-        .document = doc,
-        .http_client = &http_client,
-    });
-
     // alias global as self and window
     var window = Window.create(null, null);
 
-    var u = try URL.constructor(alloc, "https://lightpanda.io/opensource-browser/", null);
+    const url = "https://lightpanda.io/opensource-browser/";
+    var u = try URL.constructor(alloc, url, null);
     defer u.deinit(alloc);
+
+    var cookie_jar = storage.CookieJar.init(alloc);
+    defer cookie_jar.deinit();
+
+    try js_env.setUserContext(.{
+        .uri = try std.Uri.parse(url),
+        .document = doc,
+        .cookie_jar = &cookie_jar,
+        .http_client = &http_client,
+    });
+
     var location = Location{ .url = &u };
     try window.replaceLocation(&location);
 

src/storage/cookie.zig

@@ -3,9 +3,14 @@ const Uri = std.Uri;
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
 
+const http = @import("../http/client.zig");
 const DateTime = @import("../datetime.zig").DateTime;
 const public_suffix_list = @import("../data/public_suffix_list.zig").lookup;
 
+const log = std.log.scoped(.cookie);
+
+pub const LookupOpts = struct { request_time: ?i64 = null, origin_uri: ?Uri = null, navigation: bool = true };
+
 pub const Jar = struct {
     allocator: Allocator,
     cookies: std.ArrayListUnmanaged(Cookie),
@@ -51,24 +56,19 @@ pub const Jar = struct {
         }
     }
 
-    pub fn forRequest(
-        self: *Jar,
-        allocator: Allocator,
-        request_time: i64,
-        origin_uri: ?Uri,
-        target_uri: Uri,
-        navigation: bool,
-    ) !CookieList {
+    pub fn forRequest(self: *Jar, target_uri: Uri, writer: anytype, opts: LookupOpts) !void {
         const target_path = target_uri.path.percent_encoded;
         const target_host = (target_uri.host orelse return error.InvalidURI).percent_encoded;
 
-        const same_site = try areSameSite(origin_uri, target_host);
+        const same_site = try areSameSite(opts.origin_uri, target_host);
         const is_secure = std.mem.eql(u8, target_uri.scheme, "https");
 
-        var matching: std.ArrayListUnmanaged(*const Cookie) = .{};
-
         var i: usize = 0;
         var cookies = self.cookies.items;
+        const navigation = opts.navigation;
+        const request_time = opts.request_time orelse std.time.timestamp();
+
+        var first = true;
         while (i < cookies.len) {
             const cookie = &cookies[i];
 
@@ -138,10 +138,35 @@ pub const Jar = struct {
                 }
             }
             // we have a match!
-            try matching.append(allocator, cookie);
+            if (first) {
+                first = false;
+            } else {
+                try writer.writeAll(", ");
+            }
+            try writeCookie(cookie, writer);
         }
+    }
 
-        return .{ ._cookies = matching };
+    pub fn populateFromResponse(self: *Jar, uri: Uri, header: *const http.ResponseHeader) !void {
+        const now = std.time.timestamp();
+        var it = header.iterate("set-cookie");
+        while (it.next()) |set_cookie| {
+            const c = Cookie.parse(self.allocator, uri, set_cookie) catch |err| {
+                log.warn("Couldn't parse cookie '{s}': {}\n", .{ set_cookie, err });
+                continue;
+            };
+            try self.add(c, now);
+        }
+    }
+
+    fn writeCookie(cookie: *const Cookie, writer: anytype) !void {
+        if (cookie.name.len > 0) {
+            try writer.writeAll(cookie.name);
+            try writer.writeByte('=');
+        }
+        if (cookie.value.len > 0) {
+            try writer.writeAll(cookie.value);
+        }
     }
 };
 
@@ -503,20 +528,11 @@ test "Jar: add" {
 
 test "Jar: forRequest" {
     const expectCookies = struct {
-        fn expect(expected: []const []const u8, list: *CookieList) !void {
-            defer list.deinit(testing.allocator);
-            const acutal_cookies = list._cookies.items;
-
-            try testing.expectEqual(expected.len, acutal_cookies.len);
-            LOOP: for (expected) |e| {
-                for (acutal_cookies) |c| {
-                    if (std.mem.eql(u8, e, c.name)) {
-                        continue :LOOP;
-                    }
-                }
-                std.debug.print("Cookie '{s}' not found", .{e});
-                return error.CookieNotFound;
-            }
+        fn expect(expected: []const u8, jar: *Jar, target_uri: Uri, opts: LookupOpts) !void {
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            defer arr.deinit(testing.allocator);
+            try jar.forRequest(target_uri, arr.writer(testing.allocator), opts);
+            try testing.expectEqual(expected, arr.items);
         }
     }.expect;
 
@@ -529,8 +545,7 @@ test "Jar: forRequest" {
 
     {
         // test with no cookies
-        var matches = try jar.forRequest(testing.allocator, now, test_uri, test_uri, true);
-        try expectCookies(&.{}, &matches);
+        try expectCookies("", &jar, test_uri, .{});
     }
 
     try jar.add(try Cookie.parse(testing.allocator, test_uri, "global1=1"), now);
@@ -543,212 +558,100 @@ test "Jar: forRequest" {
     try jar.add(try Cookie.parse(testing.allocator, test_uri, "sitestrict=8;SameSite=Strict;Path=/x/"), now);
     try jar.add(try Cookie.parse(testing.allocator, test_uri_2, "domain1=9;domain=test.lightpanda.io"), now);
 
-    {
-        // nothing fancy here
-        var matches = try jar.forRequest(testing.allocator, now, test_uri, test_uri, true);
-        try expectCookies(&.{ "global1", "global2" }, &matches);
-    }
+    // nothing fancy here
+    try expectCookies("global1=1, global2=2", &jar, test_uri, .{});
+    try expectCookies("global1=1, global2=2", &jar, test_uri, .{ .origin_uri = test_uri, .navigation = false });
 
-    {
-        // We have a cookie where Domain=lightpanda.io
-        // This should _not_ match xyxlightpanda.io
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://anothersitelightpanda.io/"),
-            true,
-        );
-        try expectCookies(&.{}, &matches);
-    }
+    // We have a cookie where Domain=lightpanda.io
+    // This should _not_ match xyxlightpanda.io
+    try expectCookies("", &jar, try std.Uri.parse("http://anothersitelightpanda.io/"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // matching path without trailing /
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://lightpanda.io/about"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2", "path1" }, &matches);
-    }
+    // matching path without trailing /
+    try expectCookies("global1=1, global2=2, path1=3", &jar, try std.Uri.parse("http://lightpanda.io/about"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // incomplete prefix path
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://lightpanda.io/abou"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2" }, &matches);
-    }
+    // incomplete prefix path
+    try expectCookies("global1=1, global2=2", &jar, try std.Uri.parse("http://lightpanda.io/abou"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // path doesn't match
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://lightpanda.io/aboutus"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2" }, &matches);
-    }
+    // path doesn't match
+    try expectCookies("global1=1, global2=2", &jar, try std.Uri.parse("http://lightpanda.io/aboutus"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // path doesn't match cookie directory
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://lightpanda.io/docs"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2" }, &matches);
-    }
+    // path doesn't match cookie directory
+    try expectCookies("global1=1, global2=2", &jar, try std.Uri.parse("http://lightpanda.io/docs"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // exact directory match
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://lightpanda.io/docs/"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2", "path2" }, &matches);
-    }
+    // exact directory match
+    try expectCookies("global1=1, global2=2, path2=4", &jar, try std.Uri.parse("http://lightpanda.io/docs/"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // sub directory match
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://lightpanda.io/docs/more"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2", "path2" }, &matches);
-    }
+    // sub directory match
+    try expectCookies("global1=1, global2=2, path2=4", &jar, try std.Uri.parse("http://lightpanda.io/docs/more"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // secure
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("https://lightpanda.io/"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2", "secure" }, &matches);
-    }
+    // secure
+    try expectCookies("global1=1, global2=2, secure=5", &jar, try std.Uri.parse("https://lightpanda.io/"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // navigational cross domain, secure
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            try std.Uri.parse("https://example.com/"),
-            try std.Uri.parse("https://lightpanda.io/x/"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2", "sitenone", "sitelax", "secure" }, &matches);
-    }
+    // navigational cross domain, secure
+    try expectCookies("global1=1, global2=2, secure=5, sitenone=6, sitelax=7", &jar, try std.Uri.parse("https://lightpanda.io/x/"), .{
+        .origin_uri = try std.Uri.parse("https://example.com/"),
+    });
 
-    {
-        // navigational cross domain, insecure
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            try std.Uri.parse("http://example.com/"),
-            try std.Uri.parse("http://lightpanda.io/x/"),
-            true,
-        );
-        try expectCookies(&.{ "global1", "global2", "sitelax" }, &matches);
-    }
+    // navigational cross domain, insecure
+    try expectCookies("global1=1, global2=2, sitelax=7", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{
+        .origin_uri = try std.Uri.parse("https://example.com/"),
+    });
 
-    {
-        // non-navigational cross domain, insecure
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            try std.Uri.parse("http://example.com/"),
-            try std.Uri.parse("http://lightpanda.io/x/"),
-            false,
-        );
-        try expectCookies(&.{}, &matches);
-    }
+    // non-navigational cross domain, insecure
+    try expectCookies("", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{
+        .origin_uri = try std.Uri.parse("https://example.com/"),
+        .navigation = false,
+    });
 
-    {
-        // non-navigational cross domain, secure
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            try std.Uri.parse("https://example.com/"),
-            try std.Uri.parse("https://lightpanda.io/x/"),
-            false,
-        );
-        try expectCookies(&.{"sitenone"}, &matches);
-    }
+    // non-navigational cross domain, secure
+    try expectCookies("sitenone=6", &jar, try std.Uri.parse("https://lightpanda.io/x/"), .{
+        .origin_uri = try std.Uri.parse("https://example.com/"),
+        .navigation = false,
+    });
 
-    {
-        // non-navigational same origin
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            try std.Uri.parse("http://lightpanda.io/"),
-            try std.Uri.parse("http://lightpanda.io/x/"),
-            false,
-        );
-        try expectCookies(&.{ "global1", "global2", "sitelax", "sitestrict" }, &matches);
-    }
+    // non-navigational same origin
+    try expectCookies("global1=1, global2=2, sitelax=7, sitestrict=8", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{
+        .origin_uri = try std.Uri.parse("https://lightpanda.io/"),
+        .navigation = false,
+    });
 
-    {
-        // exact domain match + suffix
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://test.lightpanda.io/"),
-            true,
-        );
-        try expectCookies(&.{ "global2", "domain1" }, &matches);
-    }
+    // exact domain match + suffix
+    try expectCookies("global2=2, domain1=9", &jar, try std.Uri.parse("http://test.lightpanda.io/"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // domain suffix match + suffix
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://1.test.lightpanda.io/"),
-            true,
-        );
-        try expectCookies(&.{ "global2", "domain1" }, &matches);
-    }
+    // domain suffix match + suffix
+    try expectCookies("global2=2, domain1=9", &jar, try std.Uri.parse("http://1.test.lightpanda.io/"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // non-matching domain
-        var matches = try jar.forRequest(
-            testing.allocator,
-            now,
-            test_uri,
-            try std.Uri.parse("http://other.lightpanda.io/"),
-            true,
-        );
-        try expectCookies(&.{"global2"}, &matches);
-    }
+    // non-matching domain
+    try expectCookies("global2=2", &jar, try std.Uri.parse("http://other.lightpanda.io/"), .{
+        .origin_uri = test_uri,
+    });
 
-    {
-        // cookie has expired
-        const l = jar.cookies.items.len;
-        var matches = try jar.forRequest(testing.allocator, now + 100, test_uri, test_uri, true);
-        try expectCookies(&.{"global1"}, &matches);
-        try testing.expectEqual(l - 1, jar.cookies.items.len);
-    }
+    const l = jar.cookies.items.len;
+    try expectCookies("global1=1", &jar, test_uri, .{
+        .request_time = now + 100,
+        .origin_uri = test_uri,
+    });
+    try testing.expectEqual(l - 1, jar.cookies.items.len);
 
     // If you add more cases after this point, note that the above test removes
     // the 'global2' cookie

src/storage/storage.zig

@@ -25,7 +25,7 @@ const DOMError = @import("netsurf").DOMError;
 
 const log = std.log.scoped(.storage);
 
-const cookie = @import("cookie.zig");
+pub const cookie = @import("cookie.zig");
 pub const Cookie = cookie.Cookie;
 pub const CookieJar = cookie.Jar;
 

src/user_context.zig

@@ -1,8 +1,11 @@
 const std = @import("std");
 const parser = @import("netsurf");
+const storage = @import("storage/storage.zig");
 const Client = @import("http/client.zig").Client;
 
 pub const UserContext = struct {
-    document: *parser.DocumentHTML,
     http_client: *Client,
+    uri: std.Uri,
+    document: *parser.DocumentHTML,
+    cookie_jar: *storage.CookieJar,
 };

src/xhr/xhr.zig

@@ -35,6 +35,7 @@ const http = @import("../http/client.zig");
 
 const parser = @import("netsurf");
 
+const CookieJar = @import("../storage/storage.zig").CookieJar;
 const UserContext = @import("../user_context.zig").UserContext;
 
 const log = std.log.scoped(.xhr);
@@ -110,6 +111,10 @@ pub const XMLHttpRequest = struct {
     err: ?anyerror = null,
     last_dispatch: i64 = 0,
 
+    cookie_jar: *CookieJar,
+    // the URI of the page where this request is originating from
+    origin_uri: std.Uri,
+
     // TODO uncomment this field causes casting issue with
     // XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but
     // not sure. see
@@ -289,7 +294,9 @@ pub const XMLHttpRequest = struct {
             .url = null,
             .uri = undefined,
             .state = .unsent,
+            .origin_uri = userctx.uri,
             .client = userctx.http_client,
+            .cookie_jar = userctx.cookie_jar,
         };
     }
 
@@ -481,6 +488,18 @@ pub const XMLHttpRequest = struct {
             try request.addHeader(hdr.name, hdr.value, .{});
         }
 
+        {
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            try self.cookie_jar.forRequest(self.uri, arr.writer(alloc), .{
+                .navigation = false,
+                .origin_uri = self.origin_uri,
+            });
+
+            if (arr.items.len > 0) {
+                try request.addHeader("Cookie", arr.items, .{});
+            }
+        }
+
         //  The body argument provides the request body, if any, and is ignored
         //  if the request method is GET or HEAD.
         //  https://xhr.spec.whatwg.org/#the-send()-method
@@ -526,6 +545,8 @@ pub const XMLHttpRequest = struct {
 
             self.state = .loading;
             self.dispatchEvt("readystatechange");
+
+            try self.cookie_jar.populateFromResponse(self.uri, &header);
         }
 
         if (progress.data) |data| {