Merge pull request #1265 from lightpanda-io/network-event

cdp: improve network's events

src/browser/mime.zig

@@ -24,6 +24,7 @@ pub const Mime = struct {
     // IANA defines max. charset value length as 40.
     // We keep 41 for null-termination since HTML parser expects in this format.
     charset: [41]u8 = default_charset,
+    charset_len: usize = 5,
 
     /// String "UTF-8" continued by null characters.
     pub const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36;
@@ -53,9 +54,25 @@ pub const Mime = struct {
         other: struct { type: []const u8, sub_type: []const u8 },
     };
 
+    pub fn contentTypeString(mime: *const Mime) []const u8 {
+        return switch (mime.content_type) {
+            .text_xml => "text/xml",
+            .text_html => "text/html",
+            .text_javascript => "application/javascript",
+            .text_plain => "text/plain",
+            .text_css => "text/css",
+            .application_json => "application/json",
+            else => "",
+        };
+    }
+
     /// Returns the null-terminated charset value.
-    pub fn charsetString(mime: *const Mime) [:0]const u8 {
-        return @ptrCast(&mime.charset);
+    pub fn charsetStringZ(mime: *const Mime) [:0]const u8 {
+        return mime.charset[0..mime.charset_len :0];
+    }
+
+    pub fn charsetString(mime: *const Mime) []const u8 {
+        return mime.charset[0..mime.charset_len];
     }
 
     /// Removes quotes of value if quotes are given.
@@ -99,6 +116,7 @@ pub const Mime = struct {
         const params = trimLeft(normalized[type_len..]);
 
         var charset: [41]u8 = undefined;
+        var charset_len: usize = undefined;
 
         var it = std.mem.splitScalar(u8, params, ';');
         while (it.next()) |attr| {
@@ -124,6 +142,7 @@ pub const Mime = struct {
                     @memcpy(charset[0..attribute_value.len], attribute_value);
                     // Null-terminate right after attribute value.
                     charset[attribute_value.len] = 0;
+                    charset_len = attribute_value.len;
                 },
             }
         }
@@ -131,6 +150,7 @@ pub const Mime = struct {
         return .{
             .params = params,
             .charset = charset,
+            .charset_len = charset_len,
             .content_type = content_type,
         };
     }
@@ -511,9 +531,9 @@ fn expect(expected: Expectation, input: []const u8) !void {
 
     if (expected.charset) |ec| {
         // We remove the null characters for testing purposes here.
-        try testing.expectEqual(ec, actual.charsetString()[0..ec.len]);
+        try testing.expectEqual(ec, actual.charsetString());
     } else {
         const m: Mime = .unknown;
-        try testing.expectEqual(m.charsetString(), actual.charsetString());
+        try testing.expectEqual(m.charsetStringZ(), actual.charsetStringZ());
     }
 }

src/browser/page.zig

@@ -83,6 +83,8 @@ pub const Page = struct {
 
     // indicates intention to navigate to another page on the next loop execution.
     delayed_navigation: bool = false,
+    req_id: ?usize = null,
+    navigated_options: ?NavigatedOpts = null,
 
     state_pool: *std.heap.MemoryPool(State),
 
@@ -546,11 +548,14 @@ pub const Page = struct {
             try self.reset();
         }
 
+        const req_id = self.http_client.nextReqId();
+
         log.info(.http, "navigate", .{
             .url = request_url,
             .method = opts.method,
             .reason = opts.reason,
             .body = opts.body != null,
+            .req_id = req_id,
         });
 
         // if the url is about:blank, we load an empty HTML document in the
@@ -568,22 +573,39 @@ pub const Page = struct {
             self.documentIsComplete();
 
             self.session.browser.notification.dispatch(.page_navigate, &.{
+                .req_id = req_id,
                 .opts = opts,
                 .url = request_url,
                 .timestamp = timestamp(),
             });
 
             self.session.browser.notification.dispatch(.page_navigated, &.{
+                .req_id = req_id,
+                .opts = .{
+                    .cdp_id = opts.cdp_id,
+                    .reason = opts.reason,
+                    .method = opts.method,
+                },
                 .url = request_url,
                 .timestamp = timestamp(),
             });
 
+            // force next request id manually b/c we won't create a real req.
+            _ = self.http_client.incrReqId();
+
             return;
         }
 
         const owned_url = try self.arena.dupeZ(u8, request_url);
         self.url = try URL.parse(owned_url, null);
 
+        self.req_id = req_id;
+        self.navigated_options = .{
+            .cdp_id = opts.cdp_id,
+            .reason = opts.reason,
+            .method = opts.method,
+        };
+
         var headers = try self.http_client.newHeaders();
         if (opts.header) |hdr| try headers.add(hdr);
         try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers);
@@ -591,6 +613,7 @@ pub const Page = struct {
         // We dispatch page_navigate event before sending the request.
         // It ensures the event page_navigated is not dispatched before this one.
         self.session.browser.notification.dispatch(.page_navigate, &.{
+            .req_id = req_id,
             .opts = opts,
             .url = owned_url,
             .timestamp = timestamp(),
@@ -656,7 +679,11 @@ pub const Page = struct {
             log.err(.browser, "document is complete", .{ .err = err });
         };
 
+        std.debug.assert(self.req_id != null);
+        std.debug.assert(self.navigated_options != null);
         self.session.browser.notification.dispatch(.page_navigated, &.{
+            .req_id = self.req_id.?,
+            .opts = self.navigated_options.?,
             .url = self.url.raw,
             .timestamp = timestamp(),
         });
@@ -713,14 +740,14 @@ pub const Page = struct {
             log.debug(.http, "navigate first chunk", .{ .content_type = mime.content_type, .len = data.len });
 
             self.mode = switch (mime.content_type) {
-                .text_html => .{ .html = try parser.Parser.init(mime.charsetString()) },
+                .text_html => .{ .html = try parser.Parser.init(mime.charsetStringZ()) },
 
                 .application_json,
                 .text_javascript,
                 .text_css,
                 .text_plain,
                 => blk: {
-                    var p = try parser.Parser.init(mime.charsetString());
+                    var p = try parser.Parser.init(mime.charsetStringZ());
                     try p.process("<html><head><meta charset=\"utf-8\"></head><body><pre>");
                     break :blk .{ .text = p };
                 },
@@ -1264,6 +1291,12 @@ pub const NavigateOpts = struct {
     force: bool = false,
 };
 
+pub const NavigatedOpts = struct {
+    cdp_id: ?i64 = null,
+    reason: NavigateReason = .address_bar,
+    method: Http.Method = .GET,
+};
+
 const IdleNotification = union(enum) {
     // hasn't started yet.
     init,

src/browser/xhr/xhr.zig

@@ -678,7 +678,7 @@ pub const XMLHttpRequest = struct {
         }
 
         var fbs = std.io.fixedBufferStream(self.response_bytes.items);
-        const doc = parser.documentHTMLParse(fbs.reader(), mime.charsetString()) catch {
+        const doc = parser.documentHTMLParse(fbs.reader(), mime.charsetStringZ()) catch {
             self.response_obj = .{ .Failure = {} };
             return;
         };

src/cdp/cdp.zig

@@ -551,7 +551,8 @@ pub fn BrowserContext(comptime CDP_T: type) type {
 
         pub fn onPageNavigated(ctx: *anyopaque, msg: *const Notification.PageNavigated) !void {
             const self: *Self = @ptrCast(@alignCast(ctx));
-            return @import("domains/page.zig").pageNavigated(self, msg);
+            defer self.resetNotificationArena();
+            return @import("domains/page.zig").pageNavigated(self.notification_arena, self, msg);
         }
 
         pub fn onPageNetworkIdle(ctx: *anyopaque, msg: *const Notification.PageNetworkIdle) !void {

src/cdp/domains/network.zig

@@ -22,6 +22,7 @@ const Allocator = std.mem.Allocator;
 const CdpStorage = @import("storage.zig");
 const Transfer = @import("../../http/Client.zig").Transfer;
 const Notification = @import("../../notification.zig").Notification;
+const Mime = @import("../../browser/mime.zig").Mime;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
@@ -242,14 +243,18 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, msg: *const Notification.
     }
 
     const transfer = msg.transfer;
+    const loader_id = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id});
     // We're missing a bunch of fields, but, for now, this seems like enough
     try bc.cdp.sendEvent("Network.requestWillBeSent", .{
-        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
+        .requestId = loader_id,
         .frameId = target_id,
-        .loaderId = bc.loader_id,
-        .documentUrl = DocumentUrlWriter.init(&page.url.uri),
+        .loaderId = loader_id,
+        .type = msg.transfer.req.resource_type.string(),
+        .documentURL = DocumentUrlWriter.init(&page.url.uri),
         .request = TransferAsRequestWriter.init(transfer),
         .initiator = .{ .type = "other" },
+        .redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo
+        .hasUserGesture = false,
     }, .{ .session_id = session_id });
 }
 
@@ -259,12 +264,16 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: anytype, msg: *const Notific
     const session_id = bc.session_id orelse return;
     const target_id = bc.target_id orelse unreachable;
 
+    const transfer = msg.transfer;
+    const loader_id = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id});
+
     // We're missing a bunch of fields, but, for now, this seems like enough
     try bc.cdp.sendEvent("Network.responseReceived", .{
-        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{msg.transfer.id}),
-        .loaderId = bc.loader_id,
+        .requestId = loader_id,
         .frameId = target_id,
+        .loaderId = loader_id,
         .response = TransferAsResponseWriter.init(arena, msg.transfer),
+        .hasExtraInfo = false, // TODO change after adding Network.responseReceivedExtraInfo
     }, .{ .session_id = session_id });
 }
 
@@ -392,6 +401,20 @@ const TransferAsResponseWriter = struct {
             try jws.write(@as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown");
         }
 
+        {
+            const mime: Mime = blk: {
+                if (transfer.response_header.?.contentType()) |ct| {
+                    break :blk try Mime.parse(ct);
+                }
+                break :blk .unknown;
+            };
+
+            try jws.objectField("mimeType");
+            try jws.write(mime.contentTypeString());
+            try jws.objectField("charset");
+            try jws.write(mime.charsetString());
+        }
+
         {
             // chromedp doesn't like having duplicate header names. It's pretty
             // common to get these from a server (e.g. for Cache-Control), but

src/cdp/domains/page.zig

@@ -176,7 +176,6 @@ fn navigate(cmd: anytype) !void {
     }
 
     var page = bc.session.currentPage() orelse return error.PageNotLoaded;
-    bc.loader_id = bc.cdp.loader_id_gen.next();
 
     try page.navigate(params.url, .{
         .reason = .address_bar,
@@ -189,8 +188,7 @@ pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.Pa
     // things, but no session.
     const session_id = bc.session_id orelse return;
 
-    bc.loader_id = bc.cdp.loader_id_gen.next();
-    const loader_id = bc.loader_id;
+    const loader_id = try std.fmt.allocPrint(arena, "REQ-{d}", .{event.req_id});
     const target_id = bc.target_id orelse unreachable;
 
     bc.reset();
@@ -234,6 +232,30 @@ pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.Pa
     try cdp.sendEvent("Page.frameStartedLoading", .{
         .frameId = target_id,
     }, .{ .session_id = session_id });
+}
+
+pub fn pageRemove(bc: anytype) !void {
+    // The main page is going to be removed, we need to remove contexts from other worlds first.
+    for (bc.isolated_worlds.items) |*isolated_world| {
+        try isolated_world.removeContext();
+    }
+}
+
+pub fn pageCreated(bc: anytype, page: *Page) !void {
+    for (bc.isolated_worlds.items) |*isolated_world| {
+        try isolated_world.createContextAndLoadPolyfills(bc.arena, page);
+    }
+}
+
+pub fn pageNavigated(arena: Allocator, bc: anytype, event: *const Notification.PageNavigated) !void {
+    // detachTarget could be called, in which case, we still have a page doing
+    // things, but no session.
+    const session_id = bc.session_id orelse return;
+    const loader_id = try std.fmt.allocPrint(arena, "REQ-{d}", .{event.req_id});
+    const target_id = bc.target_id orelse unreachable;
+    const timestamp = event.timestamp;
+
+    var cdp = bc.cdp;
 
     // Drivers are sensitive to the order of events. Some more than others.
     // The result for the Page.navigate seems like it _must_ come after
@@ -260,6 +282,17 @@ pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.Pa
         }, .{ .session_id = session_id });
     }
 
+    const reason_: ?[]const u8 = switch (event.opts.reason) {
+        .anchor => "anchorClick",
+        .script, .history, .navigation => "scriptInitiated",
+        .form => switch (event.opts.method) {
+            .GET => "formSubmissionGet",
+            .POST => "formSubmissionPost",
+            else => unreachable,
+        },
+        .address_bar => null,
+    };
+
     if (reason_ != null) {
         try cdp.sendEvent("Page.frameClearedScheduledNavigation", .{
             .frameId = target_id,
@@ -293,37 +326,14 @@ pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.Pa
             false,
         );
     }
-}
 
-pub fn pageRemove(bc: anytype) !void {
-    // The main page is going to be removed, we need to remove contexts from other worlds first.
-    for (bc.isolated_worlds.items) |*isolated_world| {
-        try isolated_world.removeContext();
-    }
-}
-
-pub fn pageCreated(bc: anytype, page: *Page) !void {
-    for (bc.isolated_worlds.items) |*isolated_world| {
-        try isolated_world.createContextAndLoadPolyfills(bc.arena, page);
-    }
-}
-
-pub fn pageNavigated(bc: anytype, event: *const Notification.PageNavigated) !void {
-    // detachTarget could be called, in which case, we still have a page doing
-    // things, but no session.
-    const session_id = bc.session_id orelse return;
-    const loader_id = bc.loader_id;
-    const target_id = bc.target_id orelse unreachable;
-    const timestamp = event.timestamp;
-
-    var cdp = bc.cdp;
     // frameNavigated event
     try cdp.sendEvent("Page.frameNavigated", .{
         .type = "Navigation",
         .frame = Frame{
             .id = target_id,
             .url = event.url,
-            .loaderId = bc.loader_id,
+            .loaderId = loader_id,
             .securityOrigin = bc.security_origin,
             .secureContextType = bc.secure_context_type,
         },

src/http/Client.zig

@@ -261,6 +261,16 @@ pub fn fulfillTransfer(self: *Client, transfer: *Transfer, status: u16, headers:
     return transfer.fulfill(status, headers, body);
 }
 
+pub fn nextReqId(self: *Client) usize {
+    return self.next_request_id + 1;
+}
+
+pub fn incrReqId(self: *Client) usize {
+    const id = self.next_request_id + 1;
+    self.next_request_id = id;
+    return id;
+}
+
 fn makeTransfer(self: *Client, req: Request) !*Transfer {
     errdefer req.headers.deinit();
 
@@ -273,8 +283,7 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer {
     const transfer = try self.transfer_pool.create();
     errdefer self.transfer_pool.destroy(transfer);
 
-    const id = self.next_request_id + 1;
-    self.next_request_id = id;
+    const id = self.incrReqId();
     transfer.* = .{
         .arena = ArenaAllocator.init(self.allocator),
         .id = id,
@@ -679,6 +688,19 @@ pub const Request = struct {
         xhr,
         script,
         fetch,
+
+        // Allowed Values: Document, Stylesheet, Image, Media, Font, Script,
+        // TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, Manifest,
+        // SignedExchange, Ping, CSPViolationReport, Preflight, FedCM, Other
+        // https://chromedevtools.github.io/devtools-protocol/tot/Network/#type-ResourceType
+        pub fn string(self: ResourceType) []const u8 {
+            return switch (self) {
+                .document => "Document",
+                .xhr => "XHR",
+                .script => "Script",
+                .fetch => "Fetch",
+            };
+        }
     };
 };
 

src/notification.zig

@@ -90,14 +90,17 @@ pub const Notification = struct {
     pub const PageRemove = struct {};
 
     pub const PageNavigate = struct {
+        req_id: usize,
         timestamp: u32,
         url: []const u8,
         opts: page.NavigateOpts,
     };
 
     pub const PageNavigated = struct {
+        req_id: usize,
         timestamp: u32,
         url: []const u8,
+        opts: page.NavigatedOpts,
     };
 
     pub const PageNetworkIdle = struct {
@@ -296,6 +299,7 @@ test "Notification" {
 
     // noop
     notifier.dispatch(.page_navigate, &.{
+        .req_id = 1,
         .timestamp = 4,
         .url = undefined,
         .opts = .{},
@@ -305,6 +309,7 @@ test "Notification" {
 
     try notifier.register(.page_navigate, &tc, TestClient.pageNavigate);
     notifier.dispatch(.page_navigate, &.{
+        .req_id = 1,
         .timestamp = 4,
         .url = undefined,
         .opts = .{},
@@ -313,6 +318,7 @@ test "Notification" {
 
     notifier.unregisterAll(&tc);
     notifier.dispatch(.page_navigate, &.{
+        .req_id = 1,
         .timestamp = 10,
         .url = undefined,
         .opts = .{},
@@ -322,21 +328,23 @@ test "Notification" {
     try notifier.register(.page_navigate, &tc, TestClient.pageNavigate);
     try notifier.register(.page_navigated, &tc, TestClient.pageNavigated);
     notifier.dispatch(.page_navigate, &.{
+        .req_id = 1,
         .timestamp = 10,
         .url = undefined,
         .opts = .{},
     });
-    notifier.dispatch(.page_navigated, &.{ .timestamp = 6, .url = undefined });
+    notifier.dispatch(.page_navigated, &.{ .req_id = 1, .timestamp = 6, .url = undefined, .opts = .{} });
     try testing.expectEqual(14, tc.page_navigate);
     try testing.expectEqual(6, tc.page_navigated);
 
     notifier.unregisterAll(&tc);
     notifier.dispatch(.page_navigate, &.{
+        .req_id = 1,
         .timestamp = 100,
         .url = undefined,
         .opts = .{},
     });
-    notifier.dispatch(.page_navigated, &.{ .timestamp = 100, .url = undefined });
+    notifier.dispatch(.page_navigated, &.{ .req_id = 1, .timestamp = 100, .url = undefined, .opts = .{} });
     try testing.expectEqual(14, tc.page_navigate);
     try testing.expectEqual(6, tc.page_navigated);
 
@@ -344,27 +352,27 @@ test "Notification" {
         // unregister
         try notifier.register(.page_navigate, &tc, TestClient.pageNavigate);
         try notifier.register(.page_navigated, &tc, TestClient.pageNavigated);
-        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
-        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        notifier.dispatch(.page_navigate, &.{ .req_id = 1, .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .req_id = 1, .timestamp = 1000, .url = undefined, .opts = .{} });
         try testing.expectEqual(114, tc.page_navigate);
         try testing.expectEqual(1006, tc.page_navigated);
 
         notifier.unregister(.page_navigate, &tc);
-        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
-        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        notifier.dispatch(.page_navigate, &.{ .req_id = 1, .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .req_id = 1, .timestamp = 1000, .url = undefined, .opts = .{} });
         try testing.expectEqual(114, tc.page_navigate);
         try testing.expectEqual(2006, tc.page_navigated);
 
         notifier.unregister(.page_navigated, &tc);
-        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
-        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        notifier.dispatch(.page_navigate, &.{ .req_id = 1, .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .req_id = 1, .timestamp = 1000, .url = undefined, .opts = .{} });
         try testing.expectEqual(114, tc.page_navigate);
         try testing.expectEqual(2006, tc.page_navigated);
 
         // already unregistered, try anyways
         notifier.unregister(.page_navigated, &tc);
-        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
-        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        notifier.dispatch(.page_navigate, &.{ .req_id = 1, .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .req_id = 1, .timestamp = 1000, .url = undefined, .opts = .{} });
         try testing.expectEqual(114, tc.page_navigate);
         try testing.expectEqual(2006, tc.page_navigated);
     }