Merge pull request #1946 from lightpanda-io/cdp-response-body

Encode non-utf8 Network.getResponseBody in base64

src/browser/Mime.zig

@@ -386,6 +386,14 @@ pub fn isHTML(self: *const Mime) bool {
     return self.content_type == .text_html;
 }
 
+pub fn isText(mime: *const Mime) bool {
+    return switch (mime.content_type) {
+        .text_xml, .text_html, .text_javascript, .text_plain, .text_css => true,
+        .application_json => true,
+        else => false,
+    };
+}
+
 // we expect value to be lowercase
 fn parseContentType(value: []const u8) !struct { ContentType, usize } {
     const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len;

src/cdp/cdp.zig

@@ -33,6 +33,7 @@ const Page = @import("../browser/Page.zig");
 const Incrementing = @import("id.zig").Incrementing;
 const Notification = @import("../Notification.zig");
 const InterceptState = @import("domains/fetch.zig").InterceptState;
+const Mime = @import("../browser/Mime.zig");
 
 pub const URL_BASE = "chrome://newtab/";
 
@@ -315,6 +316,11 @@ pub fn BrowserContext(comptime CDP_T: type) type {
     const Node = @import("Node.zig");
     const AXNode = @import("AXNode.zig");
 
+    const CapturedResponse = struct {
+        must_encode: bool,
+        data: std.ArrayList(u8),
+    };
+
     return struct {
         id: []const u8,
         cdp: *CDP_T,
@@ -375,7 +381,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         // ever streamed. So if CDP is the only thing that needs bodies in
         // memory for an arbitrary amount of time, then that's where we're going
         // to store the,
-        captured_responses: std.AutoHashMapUnmanaged(usize, std.ArrayList(u8)),
+        captured_responses: std.AutoHashMapUnmanaged(usize, CapturedResponse),
 
         notification: *Notification,
 
@@ -628,6 +634,35 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         pub fn onHttpResponseHeadersDone(ctx: *anyopaque, msg: *const Notification.ResponseHeaderDone) !void {
             const self: *Self = @ptrCast(@alignCast(ctx));
             defer self.resetNotificationArena();
+
+            const arena = self.page_arena;
+
+            // Prepare the captured response value.
+            const id = msg.transfer.id;
+            const gop = try self.captured_responses.getOrPut(arena, id);
+            if (!gop.found_existing) {
+                gop.value_ptr.* = .{
+                    .data = .empty,
+                    // Encode the data in base64 by default, but don't encode
+                    // for well known content-type.
+                    .must_encode = blk: {
+                        const transfer = msg.transfer;
+                        if (transfer.response_header.?.contentType()) |ct| {
+                            const mime = try Mime.parse(ct);
+
+                            if (!mime.isText()) {
+                                break :blk true;
+                            }
+
+                            if (std.mem.eql(u8, "UTF-8", mime.charsetString())) {
+                                break :blk false;
+                            }
+                        }
+                        break :blk true;
+                    },
+                };
+            }
+
             return @import("domains/network.zig").httpResponseHeaderDone(self.notification_arena, self, msg);
         }
 
@@ -641,11 +676,9 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             const arena = self.page_arena;
 
             const id = msg.transfer.id;
-            const gop = try self.captured_responses.getOrPut(arena, id);
-            if (!gop.found_existing) {
-                gop.value_ptr.* = .{};
-            }
-            try gop.value_ptr.appendSlice(arena, try arena.dupe(u8, msg.data));
+            const resp = self.captured_responses.getPtr(id) orelse lp.assert(false, "onHttpResponseData missinf captured response", .{});
+
+            return resp.data.appendSlice(arena, msg.data);
         }
 
         pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {

src/cdp/domains/network.zig

@@ -208,12 +208,23 @@ fn getResponseBody(cmd: anytype) !void {
 
     const request_id = try idFromRequestId(params.requestId);
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-    const buf = bc.captured_responses.getPtr(request_id) orelse return error.RequestNotFound;
+    const resp = bc.captured_responses.getPtr(request_id) orelse return error.RequestNotFound;
 
-    try cmd.sendResult(.{
-        .body = buf.items,
+    if (!resp.must_encode) {
+        return cmd.sendResult(.{
+            .body = resp.data.items,
             .base64Encoded = false,
         }, .{});
+    }
+
+    const encoded_len = std.base64.standard.Encoder.calcSize(resp.data.items.len);
+    const encoded = try cmd.arena.alloc(u8, encoded_len);
+    _ = std.base64.standard.Encoder.encode(encoded, resp.data.items);
+
+    return cmd.sendResult(.{
+        .body = encoded,
+        .base64Encoded = true,
+    }, .{});
 }
 
 pub fn httpRequestFail(bc: anytype, msg: *const Notification.RequestFail) !void {