mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-03-28 07:33:16 +00:00
MCP/CDP: unify node registration
This fixes a bug in MCP where interactive elements were not assigned a backendNodeId, preventing agents from clicking or filling them. Also extracts link collection to a shared browser module.
This commit is contained in:
@@ -36,6 +36,7 @@ pub const InteractivityType = enum {
|
|||||||
};
|
};
|
||||||
|
|
||||||
pub const InteractiveElement = struct {
|
pub const InteractiveElement = struct {
|
||||||
|
backendNodeId: ?u32 = null,
|
||||||
node: *Node,
|
node: *Node,
|
||||||
tag_name: []const u8,
|
tag_name: []const u8,
|
||||||
role: ?[]const u8,
|
role: ?[]const u8,
|
||||||
@@ -55,6 +56,11 @@ pub const InteractiveElement = struct {
|
|||||||
pub fn jsonStringify(self: *const InteractiveElement, jw: anytype) !void {
|
pub fn jsonStringify(self: *const InteractiveElement, jw: anytype) !void {
|
||||||
try jw.beginObject();
|
try jw.beginObject();
|
||||||
|
|
||||||
|
if (self.backendNodeId) |id| {
|
||||||
|
try jw.objectField("backendNodeId");
|
||||||
|
try jw.write(id);
|
||||||
|
}
|
||||||
|
|
||||||
try jw.objectField("tagName");
|
try jw.objectField("tagName");
|
||||||
try jw.write(self.tag_name);
|
try jw.write(self.tag_name);
|
||||||
|
|
||||||
@@ -123,6 +129,15 @@ pub const InteractiveElement = struct {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// Populate backendNodeId on each interactive element by registering
|
||||||
|
/// their nodes in the given registry. Works with both CDP and MCP registries.
|
||||||
|
pub fn registerNodes(elements: []InteractiveElement, registry: anytype) !void {
|
||||||
|
for (elements) |*el| {
|
||||||
|
const registered = try registry.register(el.node);
|
||||||
|
el.backendNodeId = registered.id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Collect all interactive elements under `root`.
|
/// Collect all interactive elements under `root`.
|
||||||
pub fn collectInteractiveElements(
|
pub fn collectInteractiveElements(
|
||||||
root: *Node,
|
root: *Node,
|
||||||
|
|||||||
54
src/browser/links.zig
Normal file
54
src/browser/links.zig
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||||
|
//
|
||||||
|
// Francis Bouvier <francis@lightpanda.io>
|
||||||
|
// Pierre Tachoire <pierre@lightpanda.io>
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as
|
||||||
|
// published by the Free Software Foundation, either version 3 of the
|
||||||
|
// License, or (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
|
||||||
|
const Element = @import("webapi/Element.zig");
|
||||||
|
const Node = @import("webapi/Node.zig");
|
||||||
|
const Page = @import("Page.zig");
|
||||||
|
const Selector = @import("webapi/selector/Selector.zig");
|
||||||
|
|
||||||
|
const Allocator = std.mem.Allocator;
|
||||||
|
|
||||||
|
/// Collect all links (href attributes from anchor tags) under `root`.
|
||||||
|
/// Returns a slice of strings allocated with `arena`.
|
||||||
|
pub fn collectLinks(arena: Allocator, root: *Node, page: *Page) ![]const []const u8 {
|
||||||
|
var links: std.ArrayList([]const u8) = .empty;
|
||||||
|
|
||||||
|
if (Selector.querySelectorAll(root, "a[href]", page)) |list| {
|
||||||
|
defer list.deinit(page._session);
|
||||||
|
|
||||||
|
for (list._nodes) |node| {
|
||||||
|
if (node.is(Element.Html.Anchor)) |anchor| {
|
||||||
|
const href = anchor.getHref(page) catch |err| {
|
||||||
|
@import("../lightpanda.zig").log.err(.app, "resolve href failed", .{ .err = err });
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (href.len > 0) {
|
||||||
|
try links.append(arena, href);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else |err| {
|
||||||
|
@import("../lightpanda.zig").log.err(.app, "query links failed", .{ .err = err });
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
return links.items;
|
||||||
|
}
|
||||||
@@ -135,17 +135,10 @@ fn getInteractiveElements(cmd: anytype) !void {
|
|||||||
page.document.asNode();
|
page.document.asNode();
|
||||||
|
|
||||||
const elements = try interactive.collectInteractiveElements(root, cmd.arena, page);
|
const elements = try interactive.collectInteractiveElements(root, cmd.arena, page);
|
||||||
|
try interactive.registerNodes(elements, &bc.node_registry);
|
||||||
// Register nodes so nodeIds are valid for subsequent CDP calls.
|
|
||||||
var node_ids: std.ArrayList(Node.Id) = try .initCapacity(cmd.arena, elements.len);
|
|
||||||
for (elements) |el| {
|
|
||||||
const registered = try bc.node_registry.register(el.node);
|
|
||||||
node_ids.appendAssumeCapacity(registered.id);
|
|
||||||
}
|
|
||||||
|
|
||||||
return cmd.sendResult(.{
|
return cmd.sendResult(.{
|
||||||
.elements = elements,
|
.elements = elements,
|
||||||
.nodeIds = node_ids.items,
|
|
||||||
}, .{});
|
}, .{});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -308,7 +301,6 @@ test "cdp.lp: getInteractiveElements" {
|
|||||||
|
|
||||||
const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object;
|
const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object;
|
||||||
try testing.expect(result.get("elements") != null);
|
try testing.expect(result.get("elements") != null);
|
||||||
try testing.expect(result.get("nodeIds") != null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
test "cdp.lp: getStructuredData" {
|
test "cdp.lp: getStructuredData" {
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ pub const markdown = @import("browser/markdown.zig");
|
|||||||
pub const SemanticTree = @import("SemanticTree.zig");
|
pub const SemanticTree = @import("SemanticTree.zig");
|
||||||
pub const CDPNode = @import("cdp/Node.zig");
|
pub const CDPNode = @import("cdp/Node.zig");
|
||||||
pub const interactive = @import("browser/interactive.zig");
|
pub const interactive = @import("browser/interactive.zig");
|
||||||
|
pub const links = @import("browser/links.zig");
|
||||||
pub const forms = @import("browser/forms.zig");
|
pub const forms = @import("browser/forms.zig");
|
||||||
pub const actions = @import("browser/actions.zig");
|
pub const actions = @import("browser/actions.zig");
|
||||||
pub const structured_data = @import("browser/structured_data.zig");
|
pub const structured_data = @import("browser/structured_data.zig");
|
||||||
|
|||||||
@@ -203,23 +203,12 @@ const ToolStreamingText = struct {
|
|||||||
log.err(.mcp, "markdown dump failed", .{ .err = err });
|
log.err(.mcp, "markdown dump failed", .{ .err = err });
|
||||||
},
|
},
|
||||||
.links => {
|
.links => {
|
||||||
if (Selector.querySelectorAll(self.page.document.asNode(), "a[href]", self.page)) |list| {
|
if (lp.links.collectLinks(self.page.call_arena, self.page.document.asNode(), self.page)) |links| {
|
||||||
defer list.deinit(self.page._session);
|
|
||||||
|
|
||||||
var first = true;
|
var first = true;
|
||||||
for (list._nodes) |node| {
|
for (links) |href| {
|
||||||
if (node.is(Element.Html.Anchor)) |anchor| {
|
if (!first) try w.writeByte('\n');
|
||||||
const href = anchor.getHref(self.page) catch |err| {
|
try w.writeAll(href);
|
||||||
log.err(.mcp, "resolve href failed", .{ .err = err });
|
first = false;
|
||||||
continue;
|
|
||||||
};
|
|
||||||
|
|
||||||
if (href.len > 0) {
|
|
||||||
if (!first) try w.writeByte('\n');
|
|
||||||
try w.writeAll(href);
|
|
||||||
first = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else |err| {
|
} else |err| {
|
||||||
log.err(.mcp, "query links failed", .{ .err = err });
|
log.err(.mcp, "query links failed", .{ .err = err });
|
||||||
@@ -417,6 +406,12 @@ fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.
|
|||||||
log.err(.mcp, "elements collection failed", .{ .err = err });
|
log.err(.mcp, "elements collection failed", .{ .err = err });
|
||||||
return server.sendError(id, .InternalError, "Failed to collect interactive elements");
|
return server.sendError(id, .InternalError, "Failed to collect interactive elements");
|
||||||
};
|
};
|
||||||
|
|
||||||
|
lp.interactive.registerNodes(elements, &server.node_registry) catch |err| {
|
||||||
|
log.err(.mcp, "node registration failed", .{ .err = err });
|
||||||
|
return server.sendError(id, .InternalError, "Failed to register element nodes");
|
||||||
|
};
|
||||||
|
|
||||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||||
try std.json.Stringify.value(elements, .{}, &aw.writer);
|
try std.json.Stringify.value(elements, .{}, &aw.writer);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user