mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-03-30 17:18:57 +00:00
Compare commits
1 Commits
main
...
semantic-t
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9c8fe9b20f |
@@ -97,6 +97,7 @@ const NodeData = struct {
|
||||
options: ?[]OptionData = null,
|
||||
xpath: []const u8,
|
||||
is_interactive: bool,
|
||||
is_disabled: bool,
|
||||
node_name: []const u8,
|
||||
};
|
||||
|
||||
@@ -148,6 +149,7 @@ fn walk(
|
||||
const role = try axn.getRole();
|
||||
|
||||
var is_interactive = false;
|
||||
var is_disabled = false;
|
||||
var value: ?[]const u8 = null;
|
||||
var options: ?[]OptionData = null;
|
||||
var node_name: []const u8 = "text";
|
||||
@@ -172,6 +174,8 @@ fn walk(
|
||||
is_interactive = true;
|
||||
}
|
||||
}
|
||||
|
||||
is_disabled = el.isDisabled();
|
||||
} else if (node._type == .document or node._type == .document_fragment) {
|
||||
node_name = "root";
|
||||
}
|
||||
@@ -236,6 +240,7 @@ fn walk(
|
||||
.options = options,
|
||||
.xpath = xpath,
|
||||
.is_interactive = is_interactive,
|
||||
.is_disabled = is_disabled,
|
||||
.node_name = node_name,
|
||||
};
|
||||
|
||||
@@ -347,6 +352,11 @@ const JsonVisitor = struct {
|
||||
try self.jw.objectField("isInteractive");
|
||||
try self.jw.write(data.is_interactive);
|
||||
|
||||
if (data.is_disabled) {
|
||||
try self.jw.objectField("isDisabled");
|
||||
try self.jw.write(true);
|
||||
}
|
||||
|
||||
try self.jw.objectField("role");
|
||||
try self.jw.write(data.role);
|
||||
|
||||
@@ -459,6 +469,9 @@ const TextVisitor = struct {
|
||||
const is_text_only = std.mem.eql(u8, data.role, "StaticText") or std.mem.eql(u8, data.role, "none") or std.mem.eql(u8, data.role, "generic");
|
||||
|
||||
try self.writer.print("{d}", .{data.id});
|
||||
if (data.is_interactive) {
|
||||
try self.writer.writeAll(if (data.is_disabled) " [i:disabled]" else " [i]");
|
||||
}
|
||||
if (!is_text_only) {
|
||||
try self.writer.print(" {s}", .{data.role});
|
||||
}
|
||||
@@ -509,6 +522,177 @@ const TextVisitor = struct {
|
||||
}
|
||||
};
|
||||
|
||||
pub const NodeDetails = struct {
|
||||
backendNodeId: CDPNode.Id,
|
||||
tag_name: []const u8,
|
||||
role: []const u8,
|
||||
name: ?[]const u8,
|
||||
is_interactive: bool,
|
||||
is_disabled: bool,
|
||||
value: ?[]const u8 = null,
|
||||
input_type: ?[]const u8 = null,
|
||||
placeholder: ?[]const u8 = null,
|
||||
href: ?[]const u8 = null,
|
||||
id: ?[]const u8 = null,
|
||||
class: ?[]const u8 = null,
|
||||
checked: ?bool = null,
|
||||
options: ?[]OptionData = null,
|
||||
|
||||
pub fn jsonStringify(self: *const NodeDetails, jw: anytype) !void {
|
||||
try jw.beginObject();
|
||||
|
||||
try jw.objectField("backendNodeId");
|
||||
try jw.write(self.backendNodeId);
|
||||
|
||||
try jw.objectField("tagName");
|
||||
try jw.write(self.tag_name);
|
||||
|
||||
try jw.objectField("role");
|
||||
try jw.write(self.role);
|
||||
|
||||
if (self.name) |n| {
|
||||
try jw.objectField("name");
|
||||
try jw.write(n);
|
||||
}
|
||||
|
||||
try jw.objectField("isInteractive");
|
||||
try jw.write(self.is_interactive);
|
||||
|
||||
if (self.is_disabled) {
|
||||
try jw.objectField("isDisabled");
|
||||
try jw.write(true);
|
||||
}
|
||||
|
||||
if (self.value) |v| {
|
||||
try jw.objectField("value");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.input_type) |v| {
|
||||
try jw.objectField("inputType");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.placeholder) |v| {
|
||||
try jw.objectField("placeholder");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.href) |v| {
|
||||
try jw.objectField("href");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.id) |v| {
|
||||
try jw.objectField("id");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.class) |v| {
|
||||
try jw.objectField("class");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.checked) |c| {
|
||||
try jw.objectField("checked");
|
||||
try jw.write(c);
|
||||
}
|
||||
|
||||
if (self.options) |opts| {
|
||||
try jw.objectField("options");
|
||||
try jw.beginArray();
|
||||
for (opts) |opt| {
|
||||
try jw.beginObject();
|
||||
try jw.objectField("value");
|
||||
try jw.write(opt.value);
|
||||
try jw.objectField("text");
|
||||
try jw.write(opt.text);
|
||||
if (opt.selected) {
|
||||
try jw.objectField("selected");
|
||||
try jw.write(true);
|
||||
}
|
||||
try jw.endObject();
|
||||
}
|
||||
try jw.endArray();
|
||||
}
|
||||
|
||||
try jw.endObject();
|
||||
}
|
||||
};
|
||||
|
||||
pub fn getNodeDetails(node: *Node, registry: *CDPNode.Registry, page: *Page, arena: std.mem.Allocator) !NodeDetails {
|
||||
const cdp_node = try registry.register(node);
|
||||
const axn = AXNode.fromNode(node);
|
||||
const role = try axn.getRole();
|
||||
const name = try axn.getName(page, arena);
|
||||
|
||||
var is_interactive_val = false;
|
||||
var is_disabled = false;
|
||||
var tag_name: []const u8 = "text";
|
||||
var value: ?[]const u8 = null;
|
||||
var input_type: ?[]const u8 = null;
|
||||
var placeholder: ?[]const u8 = null;
|
||||
var href: ?[]const u8 = null;
|
||||
var id_attr: ?[]const u8 = null;
|
||||
var class_attr: ?[]const u8 = null;
|
||||
var checked: ?bool = null;
|
||||
var options: ?[]OptionData = null;
|
||||
|
||||
if (node.is(Element)) |el| {
|
||||
tag_name = el.getTagNameLower();
|
||||
is_disabled = el.isDisabled();
|
||||
id_attr = el.getAttributeSafe(comptime lp.String.wrap("id"));
|
||||
class_attr = el.getAttributeSafe(comptime lp.String.wrap("class"));
|
||||
placeholder = el.getAttributeSafe(comptime lp.String.wrap("placeholder"));
|
||||
|
||||
if (el.getAttributeSafe(comptime lp.String.wrap("href"))) |h| {
|
||||
const URL = lp.URL;
|
||||
href = URL.resolve(arena, page.base(), h, .{ .encode = true }) catch h;
|
||||
}
|
||||
|
||||
if (el.is(Element.Html.Input)) |input| {
|
||||
value = input.getValue();
|
||||
input_type = input._input_type.toString();
|
||||
if (input._input_type == .checkbox or input._input_type == .radio) {
|
||||
checked = input.getChecked();
|
||||
}
|
||||
if (el.getAttributeSafe(comptime lp.String.wrap("list"))) |list_id| {
|
||||
options = try extractDataListOptions(list_id, page, arena);
|
||||
}
|
||||
} else if (el.is(Element.Html.TextArea)) |textarea| {
|
||||
value = textarea.getValue();
|
||||
} else if (el.is(Element.Html.Select)) |select| {
|
||||
value = select.getValue(page);
|
||||
options = try extractSelectOptions(el.asNode(), page, arena);
|
||||
}
|
||||
|
||||
if (el.is(Element.Html)) |html_el| {
|
||||
const listener_targets = try interactive.buildListenerTargetMap(page, arena);
|
||||
var pointer_events_cache: Element.PointerEventsCache = .empty;
|
||||
if (interactive.classifyInteractivity(page, el, html_el, listener_targets, &pointer_events_cache) != null) {
|
||||
is_interactive_val = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return .{
|
||||
.backendNodeId = cdp_node.id,
|
||||
.tag_name = tag_name,
|
||||
.role = role,
|
||||
.name = name,
|
||||
.is_interactive = is_interactive_val,
|
||||
.is_disabled = is_disabled,
|
||||
.value = value,
|
||||
.input_type = input_type,
|
||||
.placeholder = placeholder,
|
||||
.href = href,
|
||||
.id = id_attr,
|
||||
.class = class_attr,
|
||||
.checked = checked,
|
||||
.options = options,
|
||||
};
|
||||
}
|
||||
|
||||
const testing = @import("testing.zig");
|
||||
|
||||
test "SemanticTree backendDOMNodeId" {
|
||||
|
||||
@@ -30,6 +30,7 @@ pub fn processMessage(cmd: anytype) !void {
|
||||
getMarkdown,
|
||||
getSemanticTree,
|
||||
getInteractiveElements,
|
||||
getNodeDetails,
|
||||
getStructuredData,
|
||||
detectForms,
|
||||
clickNode,
|
||||
@@ -42,6 +43,7 @@ pub fn processMessage(cmd: anytype) !void {
|
||||
.getMarkdown => return getMarkdown(cmd),
|
||||
.getSemanticTree => return getSemanticTree(cmd),
|
||||
.getInteractiveElements => return getInteractiveElements(cmd),
|
||||
.getNodeDetails => return getNodeDetails(cmd),
|
||||
.getStructuredData => return getStructuredData(cmd),
|
||||
.detectForms => return detectForms(cmd),
|
||||
.clickNode => return clickNode(cmd),
|
||||
@@ -141,6 +143,24 @@ fn getInteractiveElements(cmd: anytype) !void {
|
||||
}, .{});
|
||||
}
|
||||
|
||||
fn getNodeDetails(cmd: anytype) !void {
|
||||
const Params = struct {
|
||||
backendNodeId: Node.Id,
|
||||
};
|
||||
const params = (try cmd.params(Params)) orelse return error.InvalidParam;
|
||||
|
||||
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
||||
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
|
||||
|
||||
const node = (bc.node_registry.lookup_by_id.get(params.backendNodeId) orelse return error.InvalidNodeId).dom;
|
||||
|
||||
const details = SemanticTree.getNodeDetails(node, &bc.node_registry, page, cmd.arena) catch return error.InternalError;
|
||||
|
||||
return cmd.sendResult(.{
|
||||
.nodeDetails = details,
|
||||
}, .{});
|
||||
}
|
||||
|
||||
fn getStructuredData(cmd: anytype) !void {
|
||||
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
||||
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
|
||||
|
||||
@@ -75,6 +75,19 @@ pub const tool_list = [_]protocol.Tool{
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "nodeDetails",
|
||||
.description = "Get detailed information about a specific node by its backend node ID. Returns tag, role, name, interactivity, disabled state, value, input type, placeholder, href, checked state, and select options.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "backendNodeId": { "type": "integer", "description": "The backend node ID of the element to inspect." }
|
||||
\\ },
|
||||
\\ "required": ["backendNodeId"]
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "interactiveElements",
|
||||
.description = "Extract interactive elements from the opened page. If a url is provided, it navigates to that url first.",
|
||||
@@ -256,6 +269,7 @@ const ToolAction = enum {
|
||||
navigate,
|
||||
markdown,
|
||||
links,
|
||||
nodeDetails,
|
||||
interactiveElements,
|
||||
structuredData,
|
||||
detectForms,
|
||||
@@ -272,6 +286,7 @@ const tool_map = std.StaticStringMap(ToolAction).initComptime(.{
|
||||
.{ "navigate", .navigate },
|
||||
.{ "markdown", .markdown },
|
||||
.{ "links", .links },
|
||||
.{ "nodeDetails", .nodeDetails },
|
||||
.{ "interactiveElements", .interactiveElements },
|
||||
.{ "structuredData", .structuredData },
|
||||
.{ "detectForms", .detectForms },
|
||||
@@ -305,6 +320,7 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
|
||||
.goto, .navigate => try handleGoto(server, arena, req.id.?, call_params.arguments),
|
||||
.markdown => try handleMarkdown(server, arena, req.id.?, call_params.arguments),
|
||||
.links => try handleLinks(server, arena, req.id.?, call_params.arguments),
|
||||
.nodeDetails => try handleNodeDetails(server, arena, req.id.?, call_params.arguments),
|
||||
.interactiveElements => try handleInteractiveElements(server, arena, req.id.?, call_params.arguments),
|
||||
.structuredData => try handleStructuredData(server, arena, req.id.?, call_params.arguments),
|
||||
.detectForms => try handleDetectForms(server, arena, req.id.?, call_params.arguments),
|
||||
@@ -373,6 +389,32 @@ fn handleSemanticTree(server: *Server, arena: std.mem.Allocator, id: std.json.Va
|
||||
};
|
||||
}
|
||||
|
||||
fn handleNodeDetails(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const Params = struct {
|
||||
backendNodeId: CDPNode.Id,
|
||||
};
|
||||
const args = try parseArgs(Params, arena, arguments, server, id, "nodeDetails");
|
||||
|
||||
_ = server.session.currentPage() orelse {
|
||||
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
};
|
||||
|
||||
const node = server.node_registry.lookup_by_id.get(args.backendNodeId) orelse {
|
||||
return server.sendError(id, .InvalidParams, "Node not found");
|
||||
};
|
||||
|
||||
const page = server.session.currentPage().?;
|
||||
const details = lp.SemanticTree.getNodeDetails(node.dom, &server.node_registry, page, arena) catch {
|
||||
return server.sendError(id, .InternalError, "Failed to get node details");
|
||||
};
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
try std.json.Stringify.value(&details, .{}, &aw.writer);
|
||||
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = aw.written() }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
|
||||
Reference in New Issue
Block a user