mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-03-30 17:18:57 +00:00
Compare commits
1 Commits
main
...
semantic-t
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9c8fe9b20f |
@@ -97,6 +97,7 @@ const NodeData = struct {
|
|||||||
options: ?[]OptionData = null,
|
options: ?[]OptionData = null,
|
||||||
xpath: []const u8,
|
xpath: []const u8,
|
||||||
is_interactive: bool,
|
is_interactive: bool,
|
||||||
|
is_disabled: bool,
|
||||||
node_name: []const u8,
|
node_name: []const u8,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -148,6 +149,7 @@ fn walk(
|
|||||||
const role = try axn.getRole();
|
const role = try axn.getRole();
|
||||||
|
|
||||||
var is_interactive = false;
|
var is_interactive = false;
|
||||||
|
var is_disabled = false;
|
||||||
var value: ?[]const u8 = null;
|
var value: ?[]const u8 = null;
|
||||||
var options: ?[]OptionData = null;
|
var options: ?[]OptionData = null;
|
||||||
var node_name: []const u8 = "text";
|
var node_name: []const u8 = "text";
|
||||||
@@ -172,6 +174,8 @@ fn walk(
|
|||||||
is_interactive = true;
|
is_interactive = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
is_disabled = el.isDisabled();
|
||||||
} else if (node._type == .document or node._type == .document_fragment) {
|
} else if (node._type == .document or node._type == .document_fragment) {
|
||||||
node_name = "root";
|
node_name = "root";
|
||||||
}
|
}
|
||||||
@@ -236,6 +240,7 @@ fn walk(
|
|||||||
.options = options,
|
.options = options,
|
||||||
.xpath = xpath,
|
.xpath = xpath,
|
||||||
.is_interactive = is_interactive,
|
.is_interactive = is_interactive,
|
||||||
|
.is_disabled = is_disabled,
|
||||||
.node_name = node_name,
|
.node_name = node_name,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -347,6 +352,11 @@ const JsonVisitor = struct {
|
|||||||
try self.jw.objectField("isInteractive");
|
try self.jw.objectField("isInteractive");
|
||||||
try self.jw.write(data.is_interactive);
|
try self.jw.write(data.is_interactive);
|
||||||
|
|
||||||
|
if (data.is_disabled) {
|
||||||
|
try self.jw.objectField("isDisabled");
|
||||||
|
try self.jw.write(true);
|
||||||
|
}
|
||||||
|
|
||||||
try self.jw.objectField("role");
|
try self.jw.objectField("role");
|
||||||
try self.jw.write(data.role);
|
try self.jw.write(data.role);
|
||||||
|
|
||||||
@@ -459,6 +469,9 @@ const TextVisitor = struct {
|
|||||||
const is_text_only = std.mem.eql(u8, data.role, "StaticText") or std.mem.eql(u8, data.role, "none") or std.mem.eql(u8, data.role, "generic");
|
const is_text_only = std.mem.eql(u8, data.role, "StaticText") or std.mem.eql(u8, data.role, "none") or std.mem.eql(u8, data.role, "generic");
|
||||||
|
|
||||||
try self.writer.print("{d}", .{data.id});
|
try self.writer.print("{d}", .{data.id});
|
||||||
|
if (data.is_interactive) {
|
||||||
|
try self.writer.writeAll(if (data.is_disabled) " [i:disabled]" else " [i]");
|
||||||
|
}
|
||||||
if (!is_text_only) {
|
if (!is_text_only) {
|
||||||
try self.writer.print(" {s}", .{data.role});
|
try self.writer.print(" {s}", .{data.role});
|
||||||
}
|
}
|
||||||
@@ -509,6 +522,177 @@ const TextVisitor = struct {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub const NodeDetails = struct {
|
||||||
|
backendNodeId: CDPNode.Id,
|
||||||
|
tag_name: []const u8,
|
||||||
|
role: []const u8,
|
||||||
|
name: ?[]const u8,
|
||||||
|
is_interactive: bool,
|
||||||
|
is_disabled: bool,
|
||||||
|
value: ?[]const u8 = null,
|
||||||
|
input_type: ?[]const u8 = null,
|
||||||
|
placeholder: ?[]const u8 = null,
|
||||||
|
href: ?[]const u8 = null,
|
||||||
|
id: ?[]const u8 = null,
|
||||||
|
class: ?[]const u8 = null,
|
||||||
|
checked: ?bool = null,
|
||||||
|
options: ?[]OptionData = null,
|
||||||
|
|
||||||
|
pub fn jsonStringify(self: *const NodeDetails, jw: anytype) !void {
|
||||||
|
try jw.beginObject();
|
||||||
|
|
||||||
|
try jw.objectField("backendNodeId");
|
||||||
|
try jw.write(self.backendNodeId);
|
||||||
|
|
||||||
|
try jw.objectField("tagName");
|
||||||
|
try jw.write(self.tag_name);
|
||||||
|
|
||||||
|
try jw.objectField("role");
|
||||||
|
try jw.write(self.role);
|
||||||
|
|
||||||
|
if (self.name) |n| {
|
||||||
|
try jw.objectField("name");
|
||||||
|
try jw.write(n);
|
||||||
|
}
|
||||||
|
|
||||||
|
try jw.objectField("isInteractive");
|
||||||
|
try jw.write(self.is_interactive);
|
||||||
|
|
||||||
|
if (self.is_disabled) {
|
||||||
|
try jw.objectField("isDisabled");
|
||||||
|
try jw.write(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.value) |v| {
|
||||||
|
try jw.objectField("value");
|
||||||
|
try jw.write(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.input_type) |v| {
|
||||||
|
try jw.objectField("inputType");
|
||||||
|
try jw.write(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.placeholder) |v| {
|
||||||
|
try jw.objectField("placeholder");
|
||||||
|
try jw.write(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.href) |v| {
|
||||||
|
try jw.objectField("href");
|
||||||
|
try jw.write(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.id) |v| {
|
||||||
|
try jw.objectField("id");
|
||||||
|
try jw.write(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.class) |v| {
|
||||||
|
try jw.objectField("class");
|
||||||
|
try jw.write(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.checked) |c| {
|
||||||
|
try jw.objectField("checked");
|
||||||
|
try jw.write(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (self.options) |opts| {
|
||||||
|
try jw.objectField("options");
|
||||||
|
try jw.beginArray();
|
||||||
|
for (opts) |opt| {
|
||||||
|
try jw.beginObject();
|
||||||
|
try jw.objectField("value");
|
||||||
|
try jw.write(opt.value);
|
||||||
|
try jw.objectField("text");
|
||||||
|
try jw.write(opt.text);
|
||||||
|
if (opt.selected) {
|
||||||
|
try jw.objectField("selected");
|
||||||
|
try jw.write(true);
|
||||||
|
}
|
||||||
|
try jw.endObject();
|
||||||
|
}
|
||||||
|
try jw.endArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
try jw.endObject();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
pub fn getNodeDetails(node: *Node, registry: *CDPNode.Registry, page: *Page, arena: std.mem.Allocator) !NodeDetails {
|
||||||
|
const cdp_node = try registry.register(node);
|
||||||
|
const axn = AXNode.fromNode(node);
|
||||||
|
const role = try axn.getRole();
|
||||||
|
const name = try axn.getName(page, arena);
|
||||||
|
|
||||||
|
var is_interactive_val = false;
|
||||||
|
var is_disabled = false;
|
||||||
|
var tag_name: []const u8 = "text";
|
||||||
|
var value: ?[]const u8 = null;
|
||||||
|
var input_type: ?[]const u8 = null;
|
||||||
|
var placeholder: ?[]const u8 = null;
|
||||||
|
var href: ?[]const u8 = null;
|
||||||
|
var id_attr: ?[]const u8 = null;
|
||||||
|
var class_attr: ?[]const u8 = null;
|
||||||
|
var checked: ?bool = null;
|
||||||
|
var options: ?[]OptionData = null;
|
||||||
|
|
||||||
|
if (node.is(Element)) |el| {
|
||||||
|
tag_name = el.getTagNameLower();
|
||||||
|
is_disabled = el.isDisabled();
|
||||||
|
id_attr = el.getAttributeSafe(comptime lp.String.wrap("id"));
|
||||||
|
class_attr = el.getAttributeSafe(comptime lp.String.wrap("class"));
|
||||||
|
placeholder = el.getAttributeSafe(comptime lp.String.wrap("placeholder"));
|
||||||
|
|
||||||
|
if (el.getAttributeSafe(comptime lp.String.wrap("href"))) |h| {
|
||||||
|
const URL = lp.URL;
|
||||||
|
href = URL.resolve(arena, page.base(), h, .{ .encode = true }) catch h;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (el.is(Element.Html.Input)) |input| {
|
||||||
|
value = input.getValue();
|
||||||
|
input_type = input._input_type.toString();
|
||||||
|
if (input._input_type == .checkbox or input._input_type == .radio) {
|
||||||
|
checked = input.getChecked();
|
||||||
|
}
|
||||||
|
if (el.getAttributeSafe(comptime lp.String.wrap("list"))) |list_id| {
|
||||||
|
options = try extractDataListOptions(list_id, page, arena);
|
||||||
|
}
|
||||||
|
} else if (el.is(Element.Html.TextArea)) |textarea| {
|
||||||
|
value = textarea.getValue();
|
||||||
|
} else if (el.is(Element.Html.Select)) |select| {
|
||||||
|
value = select.getValue(page);
|
||||||
|
options = try extractSelectOptions(el.asNode(), page, arena);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (el.is(Element.Html)) |html_el| {
|
||||||
|
const listener_targets = try interactive.buildListenerTargetMap(page, arena);
|
||||||
|
var pointer_events_cache: Element.PointerEventsCache = .empty;
|
||||||
|
if (interactive.classifyInteractivity(page, el, html_el, listener_targets, &pointer_events_cache) != null) {
|
||||||
|
is_interactive_val = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return .{
|
||||||
|
.backendNodeId = cdp_node.id,
|
||||||
|
.tag_name = tag_name,
|
||||||
|
.role = role,
|
||||||
|
.name = name,
|
||||||
|
.is_interactive = is_interactive_val,
|
||||||
|
.is_disabled = is_disabled,
|
||||||
|
.value = value,
|
||||||
|
.input_type = input_type,
|
||||||
|
.placeholder = placeholder,
|
||||||
|
.href = href,
|
||||||
|
.id = id_attr,
|
||||||
|
.class = class_attr,
|
||||||
|
.checked = checked,
|
||||||
|
.options = options,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const testing = @import("testing.zig");
|
const testing = @import("testing.zig");
|
||||||
|
|
||||||
test "SemanticTree backendDOMNodeId" {
|
test "SemanticTree backendDOMNodeId" {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ pub fn processMessage(cmd: anytype) !void {
|
|||||||
getMarkdown,
|
getMarkdown,
|
||||||
getSemanticTree,
|
getSemanticTree,
|
||||||
getInteractiveElements,
|
getInteractiveElements,
|
||||||
|
getNodeDetails,
|
||||||
getStructuredData,
|
getStructuredData,
|
||||||
detectForms,
|
detectForms,
|
||||||
clickNode,
|
clickNode,
|
||||||
@@ -42,6 +43,7 @@ pub fn processMessage(cmd: anytype) !void {
|
|||||||
.getMarkdown => return getMarkdown(cmd),
|
.getMarkdown => return getMarkdown(cmd),
|
||||||
.getSemanticTree => return getSemanticTree(cmd),
|
.getSemanticTree => return getSemanticTree(cmd),
|
||||||
.getInteractiveElements => return getInteractiveElements(cmd),
|
.getInteractiveElements => return getInteractiveElements(cmd),
|
||||||
|
.getNodeDetails => return getNodeDetails(cmd),
|
||||||
.getStructuredData => return getStructuredData(cmd),
|
.getStructuredData => return getStructuredData(cmd),
|
||||||
.detectForms => return detectForms(cmd),
|
.detectForms => return detectForms(cmd),
|
||||||
.clickNode => return clickNode(cmd),
|
.clickNode => return clickNode(cmd),
|
||||||
@@ -141,6 +143,24 @@ fn getInteractiveElements(cmd: anytype) !void {
|
|||||||
}, .{});
|
}, .{});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn getNodeDetails(cmd: anytype) !void {
|
||||||
|
const Params = struct {
|
||||||
|
backendNodeId: Node.Id,
|
||||||
|
};
|
||||||
|
const params = (try cmd.params(Params)) orelse return error.InvalidParam;
|
||||||
|
|
||||||
|
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
||||||
|
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
|
||||||
|
|
||||||
|
const node = (bc.node_registry.lookup_by_id.get(params.backendNodeId) orelse return error.InvalidNodeId).dom;
|
||||||
|
|
||||||
|
const details = SemanticTree.getNodeDetails(node, &bc.node_registry, page, cmd.arena) catch return error.InternalError;
|
||||||
|
|
||||||
|
return cmd.sendResult(.{
|
||||||
|
.nodeDetails = details,
|
||||||
|
}, .{});
|
||||||
|
}
|
||||||
|
|
||||||
fn getStructuredData(cmd: anytype) !void {
|
fn getStructuredData(cmd: anytype) !void {
|
||||||
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
||||||
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
|
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
|
||||||
|
|||||||
@@ -75,6 +75,19 @@ pub const tool_list = [_]protocol.Tool{
|
|||||||
\\}
|
\\}
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
.{
|
||||||
|
.name = "nodeDetails",
|
||||||
|
.description = "Get detailed information about a specific node by its backend node ID. Returns tag, role, name, interactivity, disabled state, value, input type, placeholder, href, checked state, and select options.",
|
||||||
|
.inputSchema = protocol.minify(
|
||||||
|
\\{
|
||||||
|
\\ "type": "object",
|
||||||
|
\\ "properties": {
|
||||||
|
\\ "backendNodeId": { "type": "integer", "description": "The backend node ID of the element to inspect." }
|
||||||
|
\\ },
|
||||||
|
\\ "required": ["backendNodeId"]
|
||||||
|
\\}
|
||||||
|
),
|
||||||
|
},
|
||||||
.{
|
.{
|
||||||
.name = "interactiveElements",
|
.name = "interactiveElements",
|
||||||
.description = "Extract interactive elements from the opened page. If a url is provided, it navigates to that url first.",
|
.description = "Extract interactive elements from the opened page. If a url is provided, it navigates to that url first.",
|
||||||
@@ -256,6 +269,7 @@ const ToolAction = enum {
|
|||||||
navigate,
|
navigate,
|
||||||
markdown,
|
markdown,
|
||||||
links,
|
links,
|
||||||
|
nodeDetails,
|
||||||
interactiveElements,
|
interactiveElements,
|
||||||
structuredData,
|
structuredData,
|
||||||
detectForms,
|
detectForms,
|
||||||
@@ -272,6 +286,7 @@ const tool_map = std.StaticStringMap(ToolAction).initComptime(.{
|
|||||||
.{ "navigate", .navigate },
|
.{ "navigate", .navigate },
|
||||||
.{ "markdown", .markdown },
|
.{ "markdown", .markdown },
|
||||||
.{ "links", .links },
|
.{ "links", .links },
|
||||||
|
.{ "nodeDetails", .nodeDetails },
|
||||||
.{ "interactiveElements", .interactiveElements },
|
.{ "interactiveElements", .interactiveElements },
|
||||||
.{ "structuredData", .structuredData },
|
.{ "structuredData", .structuredData },
|
||||||
.{ "detectForms", .detectForms },
|
.{ "detectForms", .detectForms },
|
||||||
@@ -305,6 +320,7 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
|
|||||||
.goto, .navigate => try handleGoto(server, arena, req.id.?, call_params.arguments),
|
.goto, .navigate => try handleGoto(server, arena, req.id.?, call_params.arguments),
|
||||||
.markdown => try handleMarkdown(server, arena, req.id.?, call_params.arguments),
|
.markdown => try handleMarkdown(server, arena, req.id.?, call_params.arguments),
|
||||||
.links => try handleLinks(server, arena, req.id.?, call_params.arguments),
|
.links => try handleLinks(server, arena, req.id.?, call_params.arguments),
|
||||||
|
.nodeDetails => try handleNodeDetails(server, arena, req.id.?, call_params.arguments),
|
||||||
.interactiveElements => try handleInteractiveElements(server, arena, req.id.?, call_params.arguments),
|
.interactiveElements => try handleInteractiveElements(server, arena, req.id.?, call_params.arguments),
|
||||||
.structuredData => try handleStructuredData(server, arena, req.id.?, call_params.arguments),
|
.structuredData => try handleStructuredData(server, arena, req.id.?, call_params.arguments),
|
||||||
.detectForms => try handleDetectForms(server, arena, req.id.?, call_params.arguments),
|
.detectForms => try handleDetectForms(server, arena, req.id.?, call_params.arguments),
|
||||||
@@ -373,6 +389,32 @@ fn handleSemanticTree(server: *Server, arena: std.mem.Allocator, id: std.json.Va
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn handleNodeDetails(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||||
|
const Params = struct {
|
||||||
|
backendNodeId: CDPNode.Id,
|
||||||
|
};
|
||||||
|
const args = try parseArgs(Params, arena, arguments, server, id, "nodeDetails");
|
||||||
|
|
||||||
|
_ = server.session.currentPage() orelse {
|
||||||
|
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||||
|
};
|
||||||
|
|
||||||
|
const node = server.node_registry.lookup_by_id.get(args.backendNodeId) orelse {
|
||||||
|
return server.sendError(id, .InvalidParams, "Node not found");
|
||||||
|
};
|
||||||
|
|
||||||
|
const page = server.session.currentPage().?;
|
||||||
|
const details = lp.SemanticTree.getNodeDetails(node.dom, &server.node_registry, page, arena) catch {
|
||||||
|
return server.sendError(id, .InternalError, "Failed to get node details");
|
||||||
|
};
|
||||||
|
|
||||||
|
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||||
|
try std.json.Stringify.value(&details, .{}, &aw.writer);
|
||||||
|
|
||||||
|
const content = [_]protocol.TextContent([]const u8){.{ .text = aw.written() }};
|
||||||
|
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||||
|
}
|
||||||
|
|
||||||
fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||||
const page = try ensurePage(server, id, args.url);
|
const page = try ensurePage(server, id, args.url);
|
||||||
|
|||||||
Reference in New Issue
Block a user