Merge pull request #495 from lightpanda-io/cdp_node

Add CDP Node Registry

src/cdp/Node.zig

@@ -0,0 +1,385 @@
+// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const parser = @import("netsurf");
+const Allocator = std.mem.Allocator;
+
+pub const Id = u32;
+
+const Node = @This();
+
+id: Id,
+parent_id: ?Id = null,
+node_type: u32,
+backend_node_id: Id,
+node_name: []const u8,
+local_name: []const u8,
+node_value: []const u8,
+child_node_count: u32,
+children: []const *Node,
+document_url: ?[]const u8,
+base_url: ?[]const u8,
+xml_version: []const u8,
+compatibility_mode: CompatibilityMode,
+is_scrollable: bool,
+_node: *parser.Node,
+
+const CompatibilityMode = enum {
+    NoQuirksMode,
+};
+
+pub fn jsonStringify(self: *const Node, writer: anytype) !void {
+    try writer.beginObject();
+    try writer.objectField("nodeId");
+    try writer.write(self.id);
+
+    try writer.objectField("parentId");
+    try writer.write(self.parent_id);
+
+    try writer.objectField("backendNodeId");
+    try writer.write(self.backend_node_id);
+
+    try writer.objectField("nodeType");
+    try writer.write(self.node_type);
+
+    try writer.objectField("nodeName");
+    try writer.write(self.node_name);
+
+    try writer.objectField("localName");
+    try writer.write(self.local_name);
+
+    try writer.objectField("nodeValue");
+    try writer.write(self.node_value);
+
+    try writer.objectField("childNodeCount");
+    try writer.write(self.child_node_count);
+
+    try writer.objectField("children");
+    try writer.write(self.children);
+
+    try writer.objectField("documentURL");
+    try writer.write(self.document_url);
+
+    try writer.objectField("baseURL");
+    try writer.write(self.base_url);
+
+    try writer.objectField("xmlVersion");
+    try writer.write(self.xml_version);
+
+    try writer.objectField("compatibilityMode");
+    try writer.write(self.compatibility_mode);
+
+    try writer.objectField("isScrollable");
+    try writer.write(self.is_scrollable);
+    try writer.endObject();
+}
+
+// Whenever we send a node to the client, we register it here for future lookup.
+// We maintain a node -> id and id -> node lookup.
+pub const Registry = struct {
+    node_id: u32,
+    allocator: Allocator,
+    node_pool: std.heap.MemoryPool(Node),
+    lookup_by_id: std.AutoHashMapUnmanaged(Id, *Node),
+    lookup_by_node: std.HashMapUnmanaged(*parser.Node, *Node, NodeContext, std.hash_map.default_max_load_percentage),
+
+    pub fn init(allocator: Allocator) Registry {
+        return .{
+            .node_id = 0,
+            .allocator = allocator,
+            .lookup_by_id = .{},
+            .lookup_by_node = .{},
+            .node_pool = std.heap.MemoryPool(Node).init(allocator),
+        };
+    }
+
+    pub fn deinit(self: *Registry) void {
+        const allocator = self.allocator;
+        self.lookup_by_id.deinit(allocator);
+        self.lookup_by_node.deinit(allocator);
+        self.node_pool.deinit();
+    }
+
+    pub fn reset(self: *Registry) void {
+        self.lookup_by_id.clearRetainingCapacity();
+        self.lookup_by_node.clearRetainingCapacity();
+        _ = self.node_pool.reset(.{ .retain_capacity = {} });
+    }
+
+    pub fn register(self: *Registry, n: *parser.Node) !*Node {
+        const node_lookup_gop = try self.lookup_by_node.getOrPut(self.allocator, n);
+        if (node_lookup_gop.found_existing) {
+            return node_lookup_gop.value_ptr.*;
+        }
+
+        // on error, we're probably going to abort the entire browser context
+        // but, just in case, let's try to keep things tidy.
+        errdefer _ = self.lookup_by_node.remove(n);
+
+        const children = try parser.nodeGetChildNodes(n);
+        const children_count = try parser.nodeListLength(children);
+
+        const id = self.node_id;
+        defer self.node_id = id + 1;
+
+        const node = try self.node_pool.create();
+        errdefer self.node_pool.destroy(node);
+
+        node.* = .{
+            ._node = n,
+            .id = id,
+            .parent_id = null, // TODO
+            .backend_node_id = id, // ??
+            .node_name = try parser.nodeName(n),
+            .local_name = try parser.nodeLocalName(n),
+            .node_value = try parser.nodeValue(n) orelse "",
+            .node_type = @intFromEnum(try parser.nodeType(n)),
+            .child_node_count = children_count,
+            .children = &.{}, // TODO
+            .document_url = null,
+            .base_url = null,
+            .xml_version = "",
+            .compatibility_mode = .NoQuirksMode,
+            .is_scrollable = false,
+        };
+
+        // if (try parser.nodeParentNode(n)) |pn| {
+        //     _  = pn;
+        //     // TODO
+        // }
+
+        node_lookup_gop.value_ptr.* = node;
+        try self.lookup_by_id.putNoClobber(self.allocator, id, node);
+        return node;
+    }
+};
+
+const NodeContext = struct {
+    pub fn hash(_: NodeContext, n: *parser.Node) u64 {
+        return std.hash.Wyhash.hash(0, std.mem.asBytes(&@intFromPtr(n)));
+    }
+
+    pub fn eql(_: NodeContext, a: *parser.Node, b: *parser.Node) bool {
+        return @intFromPtr(a) == @intFromPtr(b);
+    }
+};
+
+// Searches are a 3 step process:
+// 1 - Dom.performSearch
+// 2 - Dom.getSearchResults
+// 3 - Dom.discardSearchResults
+//
+// For a given browser context, we can have multiple active searches. I.e.
+// performSearch could be called multiple times without getSearchResults or
+// discardSearchResults being called. We keep these active searches in the
+// browser context's node_search_list, which is a SearchList. Since we don't
+// expect many active searches (mostly just 1), a list is fine to scan through.
+pub const Search = struct {
+    name: []const u8,
+    node_ids: []const Id,
+
+    pub const List = struct {
+        registry: *Registry,
+        search_id: u16 = 0,
+        arena: std.heap.ArenaAllocator,
+        searches: std.ArrayListUnmanaged(Search) = .{},
+
+        pub fn init(allocator: Allocator, registry: *Registry) List {
+            return .{
+                .registry = registry,
+                .arena = std.heap.ArenaAllocator.init(allocator),
+            };
+        }
+
+        pub fn deinit(self: *List) void {
+            self.arena.deinit();
+        }
+
+        pub fn reset(self: *List) void {
+            self.search_id = 0;
+            self.searches = .{};
+            _ = self.arena.reset(.{ .retain_with_limit = 4096 });
+        }
+
+        pub fn create(self: *List, nodes: []const *parser.Node) !Search {
+            const id = self.search_id;
+            defer self.search_id = id +% 1;
+
+            const arena = self.arena.allocator();
+
+            const name = switch (id) {
+                0 => "0",
+                1 => "1",
+                2 => "2",
+                3 => "3",
+                4 => "4",
+                5 => "5",
+                6 => "6",
+                7 => "7",
+                8 => "8",
+                9 => "9",
+                else => try std.fmt.allocPrint(arena, "{d}", .{id}),
+            };
+
+            var registry = self.registry;
+            const node_ids = try arena.alloc(Id, nodes.len);
+            for (nodes, node_ids) |node, *node_id| {
+                node_id.* = (try registry.register(node)).id;
+            }
+
+            const search = Search{
+                .name = name,
+                .node_ids = node_ids,
+            };
+            try self.searches.append(arena, search);
+            return search;
+        }
+
+        pub fn remove(self: *List, name: []const u8) void {
+            for (self.searches.items, 0..) |search, i| {
+                if (std.mem.eql(u8, name, search.name)) {
+                    _ = self.searches.swapRemove(i);
+                    return;
+                }
+            }
+        }
+
+        pub fn get(self: *const List, name: []const u8) ?Search {
+            for (self.searches.items) |search| {
+                if (std.mem.eql(u8, name, search.name)) {
+                    return search;
+                }
+            }
+            return null;
+        }
+    };
+};
+
+const testing = @import("testing.zig");
+test "CDP Node: Registry register" {
+    var registry = Registry.init(testing.allocator);
+    defer registry.deinit();
+
+    try testing.expectEqual(0, registry.lookup_by_id.count());
+    try testing.expectEqual(0, registry.lookup_by_node.count());
+
+    var doc = try testing.Document.init("<a id=a1>link1</a><div id=d2><p>other</p></div>");
+    defer doc.deinit();
+
+    {
+        const n = (try doc.querySelector("#a1")).?;
+        const node = try registry.register(n);
+        const n1b = registry.lookup_by_id.get(0).?;
+        const n1c = registry.lookup_by_node.get(node._node).?;
+        try testing.expectEqual(node, n1b);
+        try testing.expectEqual(node, n1c);
+
+        try testing.expectEqual(0, node.id);
+        try testing.expectEqual(null, node.parent_id);
+        try testing.expectEqual(1, node.node_type);
+        try testing.expectEqual(0, node.backend_node_id);
+        try testing.expectEqual("A", node.node_name);
+        try testing.expectEqual("a", node.local_name);
+        try testing.expectEqual("", node.node_value);
+        try testing.expectEqual(1, node.child_node_count);
+        try testing.expectEqual(0, node.children.len);
+        try testing.expectEqual(null, node.document_url);
+        try testing.expectEqual(null, node.base_url);
+        try testing.expectEqual("", node.xml_version);
+        try testing.expectEqual(.NoQuirksMode, node.compatibility_mode);
+        try testing.expectEqual(false, node.is_scrollable);
+        try testing.expectEqual(n, node._node);
+    }
+
+    {
+        const n = (try doc.querySelector("p")).?;
+        const node = try registry.register(n);
+        const n1b = registry.lookup_by_id.get(1).?;
+        const n1c = registry.lookup_by_node.get(node._node).?;
+        try testing.expectEqual(node, n1b);
+        try testing.expectEqual(node, n1c);
+
+        try testing.expectEqual(1, node.id);
+        try testing.expectEqual(null, node.parent_id);
+        try testing.expectEqual(1, node.node_type);
+        try testing.expectEqual(1, node.backend_node_id);
+        try testing.expectEqual("P", node.node_name);
+        try testing.expectEqual("p", node.local_name);
+        try testing.expectEqual("", node.node_value);
+        try testing.expectEqual(1, node.child_node_count);
+        try testing.expectEqual(0, node.children.len);
+        try testing.expectEqual(null, node.document_url);
+        try testing.expectEqual(null, node.base_url);
+        try testing.expectEqual("", node.xml_version);
+        try testing.expectEqual(.NoQuirksMode, node.compatibility_mode);
+        try testing.expectEqual(false, node.is_scrollable);
+        try testing.expectEqual(n, node._node);
+    }
+}
+
+test "CDP Node: search list" {
+    var registry = Registry.init(testing.allocator);
+    defer registry.deinit();
+
+    var search_list = Search.List.init(testing.allocator, &registry);
+    defer search_list.deinit();
+
+    {
+        // empty search list, noops
+        search_list.remove("0");
+        try testing.expectEqual(null, search_list.get("0"));
+    }
+
+    {
+        // empty nodes
+        const s1 = try search_list.create(&.{});
+        try testing.expectEqual("0", s1.name);
+        try testing.expectEqual(0, s1.node_ids.len);
+
+        const s2 = search_list.get("0").?;
+        try testing.expectEqual("0", s2.name);
+        try testing.expectEqual(0, s2.node_ids.len);
+
+        search_list.remove("0");
+        try testing.expectEqual(null, search_list.get("0"));
+    }
+
+    {
+        var doc = try testing.Document.init("<a id=a1></a><a id=a2></a>");
+        defer doc.deinit();
+
+        const s1 = try search_list.create(try doc.querySelectorAll("a"));
+        try testing.expectEqual("1", s1.name);
+        try testing.expectEqualSlices(u32, &.{ 0, 1 }, s1.node_ids);
+
+        try testing.expectEqual(2, registry.lookup_by_id.count());
+        try testing.expectEqual(2, registry.lookup_by_node.count());
+
+        const s2 = try search_list.create(try doc.querySelectorAll("#a1"));
+        try testing.expectEqual("2", s2.name);
+        try testing.expectEqualSlices(u32, &.{0}, s2.node_ids);
+
+        const s3 = try search_list.create(try doc.querySelectorAll("#a2"));
+        try testing.expectEqual("3", s3.name);
+        try testing.expectEqualSlices(u32, &.{1}, s3.node_ids);
+
+        try testing.expectEqual(2, registry.lookup_by_id.count());
+        try testing.expectEqual(2, registry.lookup_by_node.count());
+    }
+}

src/cdp/cdp.zig

@@ -29,10 +29,6 @@ const log = std.log.scoped(.cdp);
 pub const URL_BASE = "chrome://newtab/";
 pub const LOADER_ID = "LOADERID24DD2FD56CF1EF33C965C79C";
 
-pub const TimestampEvent = struct {
-    timestamp: f64,
-};
-
 pub const CDP = CDPT(struct {
     const Client = *@import("../server.zig").Client;
     const Browser = @import("../browser/browser.zig").Browser;
@@ -176,40 +172,40 @@ pub fn CDPT(comptime TypeProvider: type) type {
 
             switch (domain.len) {
                 3 => switch (@as(u24, @bitCast(domain[0..3].*))) {
-                    asUint("DOM") => return @import("dom.zig").processMessage(command),
+                    asUint("DOM") => return @import("domains/dom.zig").processMessage(command),
-                    asUint("Log") => return @import("log.zig").processMessage(command),
+                    asUint("Log") => return @import("domains/log.zig").processMessage(command),
-                    asUint("CSS") => return @import("css.zig").processMessage(command),
+                    asUint("CSS") => return @import("domains/css.zig").processMessage(command),
                     else => {},
                 },
                 4 => switch (@as(u32, @bitCast(domain[0..4].*))) {
-                    asUint("Page") => return @import("page.zig").processMessage(command),
+                    asUint("Page") => return @import("domains/page.zig").processMessage(command),
                     else => {},
                 },
                 5 => switch (@as(u40, @bitCast(domain[0..5].*))) {
-                    asUint("Fetch") => return @import("fetch.zig").processMessage(command),
+                    asUint("Fetch") => return @import("domains/fetch.zig").processMessage(command),
                     else => {},
                 },
                 6 => switch (@as(u48, @bitCast(domain[0..6].*))) {
-                    asUint("Target") => return @import("target.zig").processMessage(command),
+                    asUint("Target") => return @import("domains/target.zig").processMessage(command),
                     else => {},
                 },
                 7 => switch (@as(u56, @bitCast(domain[0..7].*))) {
-                    asUint("Browser") => return @import("browser.zig").processMessage(command),
+                    asUint("Browser") => return @import("domains/browser.zig").processMessage(command),
-                    asUint("Runtime") => return @import("runtime.zig").processMessage(command),
+                    asUint("Runtime") => return @import("domains/runtime.zig").processMessage(command),
-                    asUint("Network") => return @import("network.zig").processMessage(command),
+                    asUint("Network") => return @import("domains/network.zig").processMessage(command),
                     else => {},
                 },
                 8 => switch (@as(u64, @bitCast(domain[0..8].*))) {
-                    asUint("Security") => return @import("security.zig").processMessage(command),
+                    asUint("Security") => return @import("domains/security.zig").processMessage(command),
                     else => {},
                 },
                 9 => switch (@as(u72, @bitCast(domain[0..9].*))) {
-                    asUint("Emulation") => return @import("emulation.zig").processMessage(command),
+                    asUint("Emulation") => return @import("domains/emulation.zig").processMessage(command),
-                    asUint("Inspector") => return @import("inspector.zig").processMessage(command),
+                    asUint("Inspector") => return @import("domains/inspector.zig").processMessage(command),
                     else => {},
                 },
                 11 => switch (@as(u88, @bitCast(domain[0..11].*))) {
-                    asUint("Performance") => return @import("performance.zig").processMessage(command),
+                    asUint("Performance") => return @import("domains/performance.zig").processMessage(command),
                     else => {},
                 },
                 else => {},
@@ -258,7 +254,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
 }
 
 pub fn BrowserContext(comptime CDP_T: type) type {
-    const dom = @import("dom.zig");
+    const Node = @import("Node.zig");
 
     return struct {
         id: []const u8,
@@ -291,12 +287,17 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         security_origin: []const u8,
         page_life_cycle_events: bool,
         secure_context_type: []const u8,
-        node_list: dom.NodeList,
+        node_registry: Node.Registry,
-        node_search_list: dom.NodeSearchList,
+        node_search_list: Node.Search.List,
 
         const Self = @This();
 
         fn init(self: *Self, id: []const u8, cdp: *CDP_T) !void {
+            const allocator = cdp.allocator;
+
+            var registry = Node.Registry.init(allocator);
+            errdefer registry.deinit();
+
             self.* = .{
                 .id = id,
                 .cdp = cdp,
@@ -308,27 +309,20 @@ pub fn BrowserContext(comptime CDP_T: type) type {
                 .loader_id = LOADER_ID,
                 .session = try cdp.browser.newSession(self),
                 .page_life_cycle_events = false, // TODO; Target based value
-                .node_list = dom.NodeList.init(cdp.allocator),
+                .node_registry = registry,
-                .node_search_list = dom.NodeSearchList.init(cdp.allocator),
+                .node_search_list = undefined,
             };
+            self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
         }
 
         pub fn deinit(self: *Self) void {
-            self.node_list.deinit();
+            self.node_registry.deinit();
-            for (self.node_search_list.items) |*s| {
-                s.deinit();
-            }
             self.node_search_list.deinit();
         }
 
         pub fn reset(self: *Self) void {
-            self.node_list.reset();
+            self.node_registry.reset();
-
+            self.node_search_list.reset();
-            // deinit all node searches.
-            for (self.node_search_list.items) |*s| {
-                s.deinit();
-            }
-            self.node_search_list.clearAndFree();
         }
 
         pub fn onInspectorResponse(ctx: *anyopaque, _: u32, msg: []const u8) void {

src/cdp/dom.zig

@@ -1,259 +0,0 @@
-// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <francis@lightpanda.io>
-// Pierre Tachoire <pierre@lightpanda.io>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const cdp = @import("cdp.zig");
-const css = @import("../dom/css.zig");
-
-const parser = @import("netsurf");
-
-pub fn processMessage(cmd: anytype) !void {
-    const action = std.meta.stringToEnum(enum {
-        enable,
-        getDocument,
-        performSearch,
-        getSearchResults,
-        discardSearchResults,
-    }, cmd.input.action) orelse return error.UnknownMethod;
-
-    switch (action) {
-        .enable => return cmd.sendResult(null, .{}),
-        .getDocument => return getDocument(cmd),
-        .performSearch => return performSearch(cmd),
-        .getSearchResults => return getSearchResults(cmd),
-        .discardSearchResults => return discardSearchResults(cmd),
-    }
-}
-
-// NodeList references tree nodes with an array id.
-pub const NodeList = struct {
-    coll: List,
-
-    const List = std.ArrayList(*parser.Node);
-
-    pub fn init(alloc: std.mem.Allocator) NodeList {
-        return .{
-            .coll = List.init(alloc),
-        };
-    }
-
-    pub fn deinit(self: *NodeList) void {
-        self.coll.deinit();
-    }
-
-    pub fn reset(self: *NodeList) void {
-        self.coll.clearAndFree();
-    }
-
-    pub fn set(self: *NodeList, node: *parser.Node) !NodeId {
-        const coll = &self.coll;
-        for (coll.items, 0..) |n, i| {
-            if (n == node) {
-                return @intCast(i);
-            }
-        }
-
-        try coll.append(node);
-        return @intCast(coll.items.len);
-    }
-};
-
-const NodeId = u32;
-
-const Node = struct {
-    nodeId: NodeId,
-    parentId: ?NodeId = null,
-    backendNodeId: NodeId,
-    nodeType: u32,
-    nodeName: []const u8 = "",
-    localName: []const u8 = "",
-    nodeValue: []const u8 = "",
-    childNodeCount: ?u32 = null,
-    children: ?[]const Node = null,
-    documentURL: ?[]const u8 = null,
-    baseURL: ?[]const u8 = null,
-    xmlVersion: []const u8 = "",
-    compatibilityMode: []const u8 = "NoQuirksMode",
-    isScrollable: bool = false,
-
-    fn init(n: *parser.Node, nlist: *NodeList) !Node {
-        const id = try nlist.set(n);
-        return .{
-            .nodeId = id,
-            .backendNodeId = id,
-            .nodeType = @intFromEnum(try parser.nodeType(n)),
-            .nodeName = try parser.nodeName(n),
-            .localName = try parser.nodeLocalName(n),
-            .nodeValue = try parser.nodeValue(n) orelse "",
-        };
-    }
-
-    fn initChildren(
-        self: *Node,
-        alloc: std.mem.Allocator,
-        n: *parser.Node,
-        nlist: *NodeList,
-    ) !std.ArrayList(Node) {
-        const children = try parser.nodeGetChildNodes(n);
-        const ln = try parser.nodeListLength(children);
-        self.childNodeCount = ln;
-
-        var list = try std.ArrayList(Node).initCapacity(alloc, ln);
-
-        for (0..ln) |i| {
-            const child = try parser.nodeListItem(children, @intCast(i)) orelse continue;
-            list.appendAssumeCapacity(try Node.init(child, nlist));
-        }
-
-        self.children = list.items;
-
-        return list;
-    }
-};
-
-// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getDocument
-fn getDocument(cmd: anytype) !void {
-    // const params = (try cmd.params(struct {
-    //     depth: ?u32 = null,
-    //     pierce: ?bool = null,
-    // })) orelse return error.InvalidParams;
-
-    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
-    const doc = page.doc orelse return error.DocumentNotLoaded;
-
-    const node = parser.documentToNode(doc);
-    var n = try Node.init(node, &bc.node_list);
-    _ = try n.initChildren(cmd.arena, node, &bc.node_list);
-
-    return cmd.sendResult(.{
-        .root = n,
-    }, .{});
-}
-
-pub const NodeSearch = struct {
-    coll: List,
-    name: []u8,
-    alloc: std.mem.Allocator,
-
-    var count: u8 = 0;
-
-    const List = std.ArrayListUnmanaged(NodeId);
-
-    pub fn initCapacity(alloc: std.mem.Allocator, ln: usize) !NodeSearch {
-        count += 1;
-
-        return .{
-            .alloc = alloc,
-            .coll = try List.initCapacity(alloc, ln),
-            .name = try std.fmt.allocPrint(alloc, "{d}", .{count}),
-        };
-    }
-
-    pub fn deinit(self: *NodeSearch) void {
-        self.coll.deinit(self.alloc);
-        self.alloc.free(self.name);
-    }
-
-    pub fn append(self: *NodeSearch, id: NodeId) !void {
-        try self.coll.append(self.alloc, id);
-    }
-};
-pub const NodeSearchList = std.ArrayList(NodeSearch);
-
-// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-performSearch
-fn performSearch(cmd: anytype) !void {
-    const params = (try cmd.params(struct {
-        query: []const u8,
-        includeUserAgentShadowDOM: ?bool = null,
-    })) orelse return error.InvalidParams;
-
-    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
-    const doc = page.doc orelse return error.DocumentNotLoaded;
-
-    const list = try css.querySelectorAll(cmd.cdp.allocator, parser.documentToNode(doc), params.query);
-    const ln = list.nodes.items.len;
-    var ns = try NodeSearch.initCapacity(cmd.cdp.allocator, ln);
-
-    for (list.nodes.items) |n| {
-        const id = try bc.node_list.set(n);
-        try ns.append(id);
-    }
-
-    try bc.node_search_list.append(ns);
-
-    return cmd.sendResult(.{
-        .searchId = ns.name,
-        .resultCount = @as(u32, @intCast(ln)),
-    }, .{});
-}
-
-// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-discardSearchResults
-fn discardSearchResults(cmd: anytype) !void {
-    const params = (try cmd.params(struct {
-        searchId: []const u8,
-    })) orelse return error.InvalidParams;
-
-    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-
-    // retrieve the search from context
-    for (bc.node_search_list.items, 0..) |*s, i| {
-        if (!std.mem.eql(u8, s.name, params.searchId)) continue;
-
-        s.deinit();
-        _ = bc.node_search_list.swapRemove(i);
-        break;
-    }
-
-    return cmd.sendResult(null, .{});
-}
-
-// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getSearchResults
-fn getSearchResults(cmd: anytype) !void {
-    const params = (try cmd.params(struct {
-        searchId: []const u8,
-        fromIndex: u32,
-        toIndex: u32,
-    })) orelse return error.InvalidParams;
-
-    if (params.fromIndex >= params.toIndex) {
-        return error.BadIndices;
-    }
-
-    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-
-    // retrieve the search from context
-    var ns: ?*const NodeSearch = undefined;
-    for (bc.node_search_list.items) |s| {
-        if (!std.mem.eql(u8, s.name, params.searchId)) continue;
-        ns = &s;
-        break;
-    }
-
-    if (ns == null) {
-        return error.searchResultNotFound;
-    }
-
-    const items = ns.?.coll.items;
-
-    if (params.fromIndex >= items.len) return error.BadFromIndex;
-    if (params.toIndex > items.len) return error.BadToIndex;
-
-    return cmd.sendResult(.{ .nodeIds = ns.?.coll.items[params.fromIndex..params.toIndex] }, .{});
-}

src/cdp/domains/browser.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 // TODO: hard coded data
 const PROTOCOL_VERSION = "1.3";
@@ -81,7 +80,7 @@ fn setWindowBounds(cmd: anytype) !void {
     return cmd.sendResult(null, .{});
 }
 
-const testing = @import("testing.zig");
+const testing = @import("../testing.zig");
 test "cdp.browser: getVersion" {
     var ctx = testing.context();
     defer ctx.deinit();

src/cdp/domains/css.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/dom.zig

@@ -0,0 +1,184 @@
+// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const parser = @import("netsurf");
+const Node = @import("../Node.zig");
+const css = @import("../../dom/css.zig");
+
+pub fn processMessage(cmd: anytype) !void {
+    const action = std.meta.stringToEnum(enum {
+        enable,
+        getDocument,
+        performSearch,
+        getSearchResults,
+        discardSearchResults,
+    }, cmd.input.action) orelse return error.UnknownMethod;
+
+    switch (action) {
+        .enable => return cmd.sendResult(null, .{}),
+        .getDocument => return getDocument(cmd),
+        .performSearch => return performSearch(cmd),
+        .getSearchResults => return getSearchResults(cmd),
+        .discardSearchResults => return discardSearchResults(cmd),
+    }
+}
+
+// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getDocument
+fn getDocument(cmd: anytype) !void {
+    // const params = (try cmd.params(struct {
+    //     depth: ?u32 = null,
+    //     pierce: ?bool = null,
+    // })) orelse return error.InvalidParams;
+
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
+    const doc = page.doc orelse return error.DocumentNotLoaded;
+
+    const node = try bc.node_registry.register(parser.documentToNode(doc));
+    return cmd.sendResult(.{
+        .root = node,
+    }, .{});
+}
+
+// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-performSearch
+fn performSearch(cmd: anytype) !void {
+    const params = (try cmd.params(struct {
+        query: []const u8,
+        includeUserAgentShadowDOM: ?bool = null,
+    })) orelse return error.InvalidParams;
+
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
+    const doc = page.doc orelse return error.DocumentNotLoaded;
+
+    const allocator = cmd.cdp.allocator;
+    var list = try css.querySelectorAll(allocator, parser.documentToNode(doc), params.query);
+    defer list.deinit(allocator);
+
+    const search = try bc.node_search_list.create(list.nodes.items);
+
+    return cmd.sendResult(.{
+        .searchId = search.name,
+        .resultCount = @as(u32, @intCast(search.node_ids.len)),
+    }, .{});
+}
+
+// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-discardSearchResults
+fn discardSearchResults(cmd: anytype) !void {
+    const params = (try cmd.params(struct {
+        searchId: []const u8,
+    })) orelse return error.InvalidParams;
+
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+
+    bc.node_search_list.remove(params.searchId);
+    return cmd.sendResult(null, .{});
+}
+
+// https://chromedevtools.github.io/devtools-protocol/tot/DOM/#method-getSearchResults
+fn getSearchResults(cmd: anytype) !void {
+    const params = (try cmd.params(struct {
+        searchId: []const u8,
+        fromIndex: u32,
+        toIndex: u32,
+    })) orelse return error.InvalidParams;
+
+    if (params.fromIndex >= params.toIndex) {
+        return error.BadIndices;
+    }
+
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+
+    const search = bc.node_search_list.get(params.searchId) orelse {
+        return error.SearchResultNotFound;
+    };
+
+    const node_ids = search.node_ids;
+
+    if (params.fromIndex >= node_ids.len) return error.BadFromIndex;
+    if (params.toIndex > node_ids.len) return error.BadToIndex;
+
+    return cmd.sendResult(.{ .nodeIds = node_ids[params.fromIndex..params.toIndex] }, .{});
+}
+
+const testing = @import("../testing.zig");
+test "cdp.dom: getSearchResults unknown search id" {
+    var ctx = testing.context();
+    defer ctx.deinit();
+
+    try testing.expectError(error.BrowserContextNotLoaded, ctx.processMessage(.{
+        .id = 8,
+        .method = "DOM.getSearchResults",
+        .params = .{ .searchId = "Nope", .fromIndex = 0, .toIndex = 10 },
+    }));
+}
+
+test "cdp.dom: search flow" {
+    var ctx = testing.context();
+    defer ctx.deinit();
+
+    _ = try ctx.loadBrowserContext(.{ .id = "BID-A", .html = "<p>1</p> <p>2</p>" });
+
+    try ctx.processMessage(.{
+        .id = 12,
+        .method = "DOM.performSearch",
+        .params = .{ .query = "p" },
+    });
+    try ctx.expectSentResult(.{ .searchId = "0", .resultCount = 2 }, .{ .id = 12 });
+
+    {
+        // getSearchResults
+        try ctx.processMessage(.{
+            .id = 13,
+            .method = "DOM.getSearchResults",
+            .params = .{ .searchId = "0", .fromIndex = 0, .toIndex = 2 },
+        });
+        try ctx.expectSentResult(.{ .nodeIds = &.{ 0, 1 } }, .{ .id = 13 });
+
+        // different fromIndex
+        try ctx.processMessage(.{
+            .id = 14,
+            .method = "DOM.getSearchResults",
+            .params = .{ .searchId = "0", .fromIndex = 1, .toIndex = 2 },
+        });
+        try ctx.expectSentResult(.{ .nodeIds = &.{1} }, .{ .id = 14 });
+
+        // different toIndex
+        try ctx.processMessage(.{
+            .id = 15,
+            .method = "DOM.getSearchResults",
+            .params = .{ .searchId = "0", .fromIndex = 0, .toIndex = 1 },
+        });
+        try ctx.expectSentResult(.{ .nodeIds = &.{0} }, .{ .id = 15 });
+    }
+
+    try ctx.processMessage(.{
+        .id = 16,
+        .method = "DOM.discardSearchResults",
+        .params = .{ .searchId = "0" },
+    });
+    try ctx.expectSentResult(null, .{ .id = 16 });
+
+    // make sure the delete actually did something
+    try testing.expectError(error.SearchResultNotFound, ctx.processMessage(.{
+        .id = 17,
+        .method = "DOM.getSearchResults",
+        .params = .{ .searchId = "0", .fromIndex = 0, .toIndex = 1 },
+    }));
+}

src/cdp/domains/emulation.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 const Runtime = @import("runtime.zig");
 
 pub fn processMessage(cmd: anytype) !void {

src/cdp/domains/fetch.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/inspector.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/log.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/network.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/page.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 const runtime = @import("runtime.zig");
 
 pub fn processMessage(cmd: anytype) !void {
@@ -230,7 +229,7 @@ fn navigate(cmd: anytype) !void {
     // TODO: partially hard coded
     try cmd.sendEvent(
         "Page.domContentEventFired",
-        cdp.TimestampEvent{ .timestamp = 343721.803338 },
+        .{ .timestamp = 343721.803338 },
         .{ .session_id = session_id },
     );
 
@@ -246,7 +245,7 @@ fn navigate(cmd: anytype) !void {
     // TODO: partially hard coded
     try cmd.sendEvent(
         "Page.loadEventFired",
-        cdp.TimestampEvent{ .timestamp = 343721.824655 },
+        .{ .timestamp = 343721.824655 },
         .{ .session_id = session_id },
     );
 
@@ -264,7 +263,7 @@ fn navigate(cmd: anytype) !void {
     }, .{ .session_id = session_id });
 }
 
-const testing = @import("testing.zig");
+const testing = @import("../testing.zig");
 test "cdp.page: getFrameTree" {
     var ctx = testing.context();
     defer ctx.deinit();

src/cdp/domains/performance.zig

@@ -17,8 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
-const asUint = @import("../str/parser.zig").asUint;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/runtime.zig

@@ -16,9 +16,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-const builtin = @import("builtin");
 const std = @import("std");
-const cdp = @import("cdp.zig");
+const builtin = @import("builtin");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/security.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const cdp = @import("cdp.zig");
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {

src/cdp/domains/target.zig

@@ -413,7 +413,7 @@ const TargetInfo = struct {
     browserContextId: ?[]const u8 = null,
 };
 
-const testing = @import("testing.zig");
+const testing = @import("../testing.zig");
 test "cdp.target: getBrowserContexts" {
     var ctx = testing.context();
     defer ctx.deinit();
@@ -521,7 +521,7 @@ test "cdp.target: createTarget" {
     {
         try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .browserContextId = "BID-9" } });
         try testing.expectEqual(true, bc.target_id != null);
-        try testing.expectString(
+        try testing.expectEqual(
             \\{"isDefault":true,"type":"default","frameId":"TID-1"}
         , bc.session.page.?.aux_data);
 

src/cdp/testing.zig

@@ -17,7 +17,6 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-
 const json = std.json;
 const Allocator = std.mem.Allocator;
 
@@ -27,9 +26,13 @@ const main = @import("cdp.zig");
 const parser = @import("netsurf");
 const App = @import("../app.zig").App;
 
-pub const expectEqual = std.testing.expectEqual;
+pub const allocator = @import("../testing.zig").allocator;
-pub const expectError = std.testing.expectError;
+
-pub const expectString = std.testing.expectEqualStrings;
+pub const expectEqual = @import("../testing.zig").expectEqual;
+pub const expectError = @import("../testing.zig").expectError;
+pub const expectEqualSlices = @import("../testing.zig").expectEqualSlices;
+
+pub const Document = @import("../testing.zig").Document;
 
 const Browser = struct {
     session: ?*Session = null,
@@ -51,11 +54,11 @@ const Browser = struct {
             return error.MockBrowserSessionAlreadyExists;
         }
 
-        const allocator = self.arena.allocator();
+        const arena = self.arena.allocator();
-        self.session = try allocator.create(Session);
+        self.session = try arena.create(Session);
         self.session.?.* = .{
             .page = null,
-            .allocator = allocator,
+            .arena = arena,
         };
         return self.session.?;
     }
@@ -70,7 +73,7 @@ const Browser = struct {
 
 const Session = struct {
     page: ?Page = null,
-    allocator: Allocator,
+    arena: Allocator,
 
     pub fn currentPage(self: *Session) ?*Page {
         return &(self.page orelse return null);
@@ -82,7 +85,7 @@ const Session = struct {
         }
         self.page = .{
             .session = self,
-            .aux_data = try self.allocator.dupe(u8, aux_data orelse ""),
+            .aux_data = try self.arena.dupe(u8, aux_data orelse ""),
         };
         return &self.page.?;
     }
@@ -114,9 +117,9 @@ const Client = struct {
     sent: std.ArrayListUnmanaged(json.Value) = .{},
     serialized: std.ArrayListUnmanaged([]const u8) = .{},
 
-    fn init(allocator: Allocator) Client {
+    fn init(alloc: Allocator) Client {
         return .{
-            .allocator = allocator,
+            .allocator = alloc,
         };
     }
 
@@ -165,6 +168,7 @@ const TestContext = struct {
         id: ?[]const u8 = null,
         target_id: ?[]const u8 = null,
         session_id: ?[]const u8 = null,
+        html: ?[]const u8 = null,
     };
     pub fn loadBrowserContext(self: *TestContext, opts: BrowserContextOpts) !*main.BrowserContext(TestCDP) {
         var c = self.cdp();
@@ -189,6 +193,13 @@ const TestContext = struct {
         if (opts.session_id) |sid| {
             bc.session_id = sid;
         }
+
+        if (opts.html) |html| {
+            parser.deinit();
+            try parser.init();
+            const page = try bc.session.createPage(null);
+            page.doc = (try Document.init(html)).doc;
+        }
         return bc;
     }
 

src/main_unit_tests.zig

@@ -17,9 +17,9 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
-const builtin = @import("builtin");
-const parser = @import("netsurf");
 const tls = @import("tls");
+const parser = @import("netsurf");
+const builtin = @import("builtin");
 
 const Allocator = std.mem.Allocator;
 
@@ -32,7 +32,6 @@ test {
     std.testing.refAllDecls(@import("css/match_test.zig"));
     std.testing.refAllDecls(@import("css/parser.zig"));
     std.testing.refAllDecls(@import("generate.zig"));
-    std.testing.refAllDecls(@import("http/client.zig"));
     std.testing.refAllDecls(@import("storage/storage.zig"));
     std.testing.refAllDecls(@import("storage/cookie.zig"));
     std.testing.refAllDecls(@import("iterator/iterator.zig"));

src/testing.zig

@@ -17,10 +17,12 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
+const parser = @import("netsurf");
 
 pub const allocator = std.testing.allocator;
 pub const expectError = std.testing.expectError;
 pub const expectString = std.testing.expectEqualStrings;
+pub const expectEqualSlices = std.testing.expectEqualSlices;
 
 const App = @import("app.zig").App;
 
@@ -190,3 +192,37 @@ pub const Random = struct {
         return instance.?.random();
     }
 };
+
+pub const Document = struct {
+    doc: *parser.Document,
+    arena: std.heap.ArenaAllocator,
+
+    pub fn init(html: []const u8) !Document {
+        parser.deinit();
+        try parser.init();
+
+        var fbs = std.io.fixedBufferStream(html);
+        const html_doc = try parser.documentHTMLParse(fbs.reader(), "utf-8");
+
+        return .{
+            .arena = std.heap.ArenaAllocator.init(allocator),
+            .doc = parser.documentHTMLToDocument(html_doc),
+        };
+    }
+
+    pub fn deinit(self: *Document) void {
+        parser.deinit();
+        self.arena.deinit();
+    }
+
+    pub fn querySelectorAll(self: *Document, selector: []const u8) ![]const *parser.Node {
+        const css = @import("dom/css.zig");
+        const node_list = try css.querySelectorAll(self.arena.allocator(), parser.documentToNode(self.doc), selector);
+        return node_list.nodes.items;
+    }
+
+    pub fn querySelector(self: *Document, selector: []const u8) !?*parser.Node {
+        const css = @import("dom/css.zig");
+        return css.querySelector(self.arena.allocator(), parser.documentToNode(self.doc), selector);
+    }
+};