mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-03-22 04:34:44 +00:00
add LP.getInteractiveElements CDP command
Returns a structured list of all interactive elements on a page: buttons, links, inputs, ARIA widgets, contenteditable regions, and elements with event listeners. Includes accessible names, roles, listener types, and key attributes. Event listener introspection (both addEventListener and inline handlers) is unique to LP — no other browser exposes this to automation code.
This commit is contained in:
egrs
481
src/browser/interactive.zig
Normal file
481
src/browser/interactive.zig
Normal file
@@ -0,0 +1,481 @@
|
||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||
//
|
||||
// Francis Bouvier <francis@lightpanda.io>
|
||||
// Pierre Tachoire <pierre@lightpanda.io>
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const Page = @import("Page.zig");
|
||||
const TreeWalker = @import("webapi/TreeWalker.zig");
|
||||
const Element = @import("webapi/Element.zig");
|
||||
const Node = @import("webapi/Node.zig");
|
||||
const EventTarget = @import("webapi/EventTarget.zig");
|
||||
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
pub const InteractivityType = enum {
|
||||
native,
|
||||
aria,
|
||||
contenteditable,
|
||||
listener,
|
||||
focusable,
|
||||
};
|
||||
|
||||
pub const InteractiveElement = struct {
|
||||
node: *Node,
|
||||
tag_name: []const u8,
|
||||
role: ?[]const u8,
|
||||
name: ?[]const u8,
|
||||
interactivity_type: InteractivityType,
|
||||
listener_types: []const []const u8,
|
||||
disabled: bool,
|
||||
tab_index: i32,
|
||||
id: ?[]const u8,
|
||||
class: ?[]const u8,
|
||||
href: ?[]const u8,
|
||||
input_type: ?[]const u8,
|
||||
value: ?[]const u8,
|
||||
element_name: ?[]const u8,
|
||||
placeholder: ?[]const u8,
|
||||
|
||||
pub fn jsonStringify(self: *const InteractiveElement, jw: anytype) !void {
|
||||
try jw.beginObject();
|
||||
|
||||
try jw.objectField("tagName");
|
||||
try jw.write(self.tag_name);
|
||||
|
||||
try jw.objectField("role");
|
||||
try jw.write(self.role);
|
||||
|
||||
try jw.objectField("name");
|
||||
try jw.write(self.name);
|
||||
|
||||
try jw.objectField("type");
|
||||
try jw.write(@tagName(self.interactivity_type));
|
||||
|
||||
if (self.listener_types.len > 0) {
|
||||
try jw.objectField("listeners");
|
||||
try jw.beginArray();
|
||||
for (self.listener_types) |lt| {
|
||||
try jw.write(lt);
|
||||
}
|
||||
try jw.endArray();
|
||||
}
|
||||
|
||||
if (self.disabled) {
|
||||
try jw.objectField("disabled");
|
||||
try jw.write(true);
|
||||
}
|
||||
|
||||
try jw.objectField("tabIndex");
|
||||
try jw.write(self.tab_index);
|
||||
|
||||
if (self.id) |v| {
|
||||
try jw.objectField("id");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.class) |v| {
|
||||
try jw.objectField("class");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.href) |v| {
|
||||
try jw.objectField("href");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.input_type) |v| {
|
||||
try jw.objectField("inputType");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.value) |v| {
|
||||
try jw.objectField("value");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.element_name) |v| {
|
||||
try jw.objectField("elementName");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
if (self.placeholder) |v| {
|
||||
try jw.objectField("placeholder");
|
||||
try jw.write(v);
|
||||
}
|
||||
|
||||
try jw.endObject();
|
||||
}
|
||||
};
|
||||
|
||||
/// Collect all interactive elements under `root`.
|
||||
pub fn collectInteractiveElements(
|
||||
root: *Node,
|
||||
page: *Page,
|
||||
arena: Allocator,
|
||||
) ![]InteractiveElement {
|
||||
// Pre-build a map of event_target pointer → event type names,
|
||||
// so classify and getListenerTypes are both O(1) per element.
|
||||
const listener_targets = try buildListenerTargetMap(page, arena);
|
||||
|
||||
var results: std.ArrayList(InteractiveElement) = .empty;
|
||||
|
||||
var tw = TreeWalker.FullExcludeSelf.init(root, .{});
|
||||
while (tw.next()) |node| {
|
||||
const el = node.is(Element) orelse continue;
|
||||
const html_el = el.is(Element.Html) orelse continue;
|
||||
|
||||
// Skip non-visual elements that are never user-interactive.
|
||||
switch (el.getTag()) {
|
||||
.script, .style, .link, .meta, .head, .noscript, .template => continue,
|
||||
else => {},
|
||||
}
|
||||
|
||||
const itype = classifyInteractivity(el, html_el, listener_targets) orelse continue;
|
||||
|
||||
const listener_types = getListenerTypes(
|
||||
el.asEventTarget(),
|
||||
listener_targets,
|
||||
);
|
||||
|
||||
try results.append(arena, .{
|
||||
.node = node,
|
||||
.tag_name = el.getTagNameLower(),
|
||||
.role = getRole(el),
|
||||
.name = getAccessibleName(el),
|
||||
.interactivity_type = itype,
|
||||
.listener_types = listener_types,
|
||||
.disabled = isDisabled(el),
|
||||
.tab_index = html_el.getTabIndex(),
|
||||
.id = el.getAttributeSafe(comptime .wrap("id")),
|
||||
.class = el.getAttributeSafe(comptime .wrap("class")),
|
||||
.href = el.getAttributeSafe(comptime .wrap("href")),
|
||||
.input_type = getInputType(el),
|
||||
.value = getInputValue(el),
|
||||
.element_name = el.getAttributeSafe(comptime .wrap("name")),
|
||||
.placeholder = el.getAttributeSafe(comptime .wrap("placeholder")),
|
||||
});
|
||||
}
|
||||
|
||||
return results.items;
|
||||
}
|
||||
|
||||
const ListenerTargetMap = std.AutoHashMapUnmanaged(usize, std.ArrayList([]const u8));
|
||||
|
||||
/// Pre-build a map from event_target pointer → list of event type names.
|
||||
/// This lets both classifyInteractivity (O(1) "has any?") and
|
||||
/// getListenerTypes (O(1) "which ones?") avoid re-iterating per element.
|
||||
fn buildListenerTargetMap(page: *Page, arena: Allocator) !ListenerTargetMap {
|
||||
var map = ListenerTargetMap{};
|
||||
|
||||
// addEventListener registrations
|
||||
var it = page._event_manager.lookup.iterator();
|
||||
while (it.next()) |entry| {
|
||||
const list = entry.value_ptr.*;
|
||||
if (list.first != null) {
|
||||
const gop = try map.getOrPut(arena, entry.key_ptr.event_target);
|
||||
if (!gop.found_existing) gop.value_ptr.* = .empty;
|
||||
try gop.value_ptr.append(arena, entry.key_ptr.type_string.str());
|
||||
}
|
||||
}
|
||||
|
||||
// Inline handlers (onclick, onmousedown, etc.)
|
||||
var attr_it = page._event_target_attr_listeners.iterator();
|
||||
while (attr_it.next()) |entry| {
|
||||
const gop = try map.getOrPut(arena, @intFromPtr(entry.key_ptr.target));
|
||||
if (!gop.found_existing) gop.value_ptr.* = .empty;
|
||||
// Strip "on" prefix to get the event type name.
|
||||
try gop.value_ptr.append(arena, @tagName(entry.key_ptr.handler)[2..]);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
fn classifyInteractivity(
|
||||
el: *Element,
|
||||
html_el: *Element.Html,
|
||||
listener_targets: ListenerTargetMap,
|
||||
) ?InteractivityType {
|
||||
// 1. Native interactive by tag
|
||||
switch (el.getTag()) {
|
||||
.button, .summary, .details, .select, .textarea => return .native,
|
||||
.anchor, .area => {
|
||||
if (el.getAttributeSafe(comptime .wrap("href")) != null) return .native;
|
||||
},
|
||||
.input => {
|
||||
if (el.is(Element.Html.Input)) |input| {
|
||||
if (input._input_type != .hidden) return .native;
|
||||
}
|
||||
},
|
||||
else => {},
|
||||
}
|
||||
|
||||
// 2. ARIA interactive role
|
||||
if (el.getAttributeSafe(comptime .wrap("role"))) |role| {
|
||||
if (isInteractiveRole(role)) return .aria;
|
||||
}
|
||||
|
||||
// 3. contenteditable (15 bytes, exceeds SSO limit for comptime)
|
||||
if (el.getAttributeSafe(.wrap("contenteditable"))) |ce| {
|
||||
if (ce.len == 0 or std.ascii.eqlIgnoreCase(ce, "true")) return .contenteditable;
|
||||
}
|
||||
|
||||
// 4. Event listeners (addEventListener or inline handlers)
|
||||
const et_ptr = @intFromPtr(html_el.asEventTarget());
|
||||
if (listener_targets.get(et_ptr) != null) return .listener;
|
||||
|
||||
// 5. Explicitly focusable via tabindex.
|
||||
// Only count elements with an EXPLICIT tabindex attribute,
|
||||
// since getTabIndex() returns 0 for all interactive tags by default
|
||||
// (including anchors without href and hidden inputs).
|
||||
if (el.getAttributeSafe(comptime .wrap("tabindex"))) |_| {
|
||||
if (html_el.getTabIndex() >= 0) return .focusable;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
fn isInteractiveRole(role: []const u8) bool {
|
||||
const interactive_roles = [_][]const u8{
|
||||
"button", "link", "tab", "menuitem",
|
||||
"menuitemcheckbox", "menuitemradio", "switch", "checkbox",
|
||||
"radio", "slider", "spinbutton", "searchbox",
|
||||
"combobox", "option", "treeitem",
|
||||
};
|
||||
for (interactive_roles) |r| {
|
||||
if (std.ascii.eqlIgnoreCase(role, r)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn getRole(el: *Element) ?[]const u8 {
|
||||
// Explicit role attribute takes precedence
|
||||
if (el.getAttributeSafe(comptime .wrap("role"))) |role| return role;
|
||||
|
||||
// Implicit role from tag
|
||||
return switch (el.getTag()) {
|
||||
.button, .summary => "button",
|
||||
.anchor, .area => if (el.getAttributeSafe(comptime .wrap("href")) != null) "link" else null,
|
||||
.input => blk: {
|
||||
if (el.is(Element.Html.Input)) |input| {
|
||||
break :blk switch (input._input_type) {
|
||||
.text, .tel, .url, .email => "textbox",
|
||||
.checkbox => "checkbox",
|
||||
.radio => "radio",
|
||||
.button, .submit, .reset, .image => "button",
|
||||
.range => "slider",
|
||||
.number => "spinbutton",
|
||||
.search => "searchbox",
|
||||
else => null,
|
||||
};
|
||||
}
|
||||
break :blk null;
|
||||
},
|
||||
.select => "combobox",
|
||||
.textarea => "textbox",
|
||||
.details => "group",
|
||||
else => null,
|
||||
};
|
||||
}
|
||||
|
||||
fn getAccessibleName(el: *Element) ?[]const u8 {
|
||||
// aria-label
|
||||
if (el.getAttributeSafe(comptime .wrap("aria-label"))) |v| {
|
||||
if (v.len > 0) return v;
|
||||
}
|
||||
|
||||
// alt (for img, input[type=image])
|
||||
if (el.getAttributeSafe(comptime .wrap("alt"))) |v| {
|
||||
if (v.len > 0) return v;
|
||||
}
|
||||
|
||||
// title
|
||||
if (el.getAttributeSafe(comptime .wrap("title"))) |v| {
|
||||
if (v.len > 0) return v;
|
||||
}
|
||||
|
||||
// placeholder
|
||||
if (el.getAttributeSafe(comptime .wrap("placeholder"))) |v| {
|
||||
if (v.len > 0) return v;
|
||||
}
|
||||
|
||||
// value (for buttons)
|
||||
if (el.getTag() == .input) {
|
||||
if (el.getAttributeSafe(comptime .wrap("value"))) |v| {
|
||||
if (v.len > 0) return v;
|
||||
}
|
||||
}
|
||||
|
||||
// Text content (first non-empty text node, trimmed)
|
||||
return getTextContent(el.asNode());
|
||||
}
|
||||
|
||||
fn getTextContent(node: *Node) ?[]const u8 {
|
||||
var tw = TreeWalker.FullExcludeSelf.init(node, .{});
|
||||
while (tw.next()) |child| {
|
||||
// Skip text inside script/style elements.
|
||||
if (child.is(Element)) |el| {
|
||||
switch (el.getTag()) {
|
||||
.script, .style => {
|
||||
tw.skipChildren();
|
||||
continue;
|
||||
},
|
||||
else => {},
|
||||
}
|
||||
}
|
||||
if (child.is(Node.CData)) |cdata| {
|
||||
if (cdata.is(Node.CData.Text)) |text| {
|
||||
const content = std.mem.trim(u8, text.getWholeText(), &std.ascii.whitespace);
|
||||
if (content.len > 0) return content;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn isDisabled(el: *Element) bool {
|
||||
return el.getAttributeSafe(comptime .wrap("disabled")) != null;
|
||||
}
|
||||
|
||||
fn getInputType(el: *Element) ?[]const u8 {
|
||||
if (el.is(Element.Html.Input)) |input| {
|
||||
return input._input_type.toString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn getInputValue(el: *Element) ?[]const u8 {
|
||||
if (el.is(Element.Html.Input)) |input| {
|
||||
return input.getValue();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Get all event listener types registered on this target.
|
||||
fn getListenerTypes(target: *EventTarget, listener_targets: ListenerTargetMap) []const []const u8 {
|
||||
if (listener_targets.get(@intFromPtr(target))) |types| return types.items;
|
||||
return &.{};
|
||||
}
|
||||
|
||||
const testing = @import("../testing.zig");
|
||||
|
||||
fn testInteractive(html: []const u8) ![]InteractiveElement {
|
||||
const page = try testing.test_session.createPage();
|
||||
defer testing.test_session.removePage();
|
||||
|
||||
const doc = page.window._document;
|
||||
const div = try doc.createElement("div", null, page);
|
||||
try page.parseHtmlAsChildren(div.asNode(), html);
|
||||
|
||||
return collectInteractiveElements(div.asNode(), page, page.call_arena);
|
||||
}
|
||||
|
||||
test "browser.interactive: button" {
|
||||
const elements = try testInteractive("<button>Click me</button>");
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expectEqual("button", elements[0].tag_name);
|
||||
try testing.expectEqual("button", elements[0].role.?);
|
||||
try testing.expectEqual("Click me", elements[0].name.?);
|
||||
try testing.expectEqual(InteractivityType.native, elements[0].interactivity_type);
|
||||
}
|
||||
|
||||
test "browser.interactive: anchor with href" {
|
||||
const elements = try testInteractive("<a href=\"/page\">Link</a>");
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expectEqual("a", elements[0].tag_name);
|
||||
try testing.expectEqual("link", elements[0].role.?);
|
||||
try testing.expectEqual("Link", elements[0].name.?);
|
||||
}
|
||||
|
||||
test "browser.interactive: anchor without href" {
|
||||
const elements = try testInteractive("<a>Not a link</a>");
|
||||
try testing.expectEqual(0, elements.len);
|
||||
}
|
||||
|
||||
test "browser.interactive: input types" {
|
||||
const elements = try testInteractive(
|
||||
\\<input type="text" placeholder="Search">
|
||||
\\<input type="hidden" name="csrf">
|
||||
);
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expectEqual("input", elements[0].tag_name);
|
||||
try testing.expectEqual("text", elements[0].input_type.?);
|
||||
try testing.expectEqual("Search", elements[0].placeholder.?);
|
||||
}
|
||||
|
||||
test "browser.interactive: select and textarea" {
|
||||
const elements = try testInteractive(
|
||||
\\<select name="color"><option>Red</option></select>
|
||||
\\<textarea name="msg"></textarea>
|
||||
);
|
||||
try testing.expectEqual(2, elements.len);
|
||||
try testing.expectEqual("select", elements[0].tag_name);
|
||||
try testing.expectEqual("textarea", elements[1].tag_name);
|
||||
}
|
||||
|
||||
test "browser.interactive: aria role" {
|
||||
const elements = try testInteractive("<div role=\"button\">Custom</div>");
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expectEqual("div", elements[0].tag_name);
|
||||
try testing.expectEqual("button", elements[0].role.?);
|
||||
try testing.expectEqual(InteractivityType.aria, elements[0].interactivity_type);
|
||||
}
|
||||
|
||||
test "browser.interactive: contenteditable" {
|
||||
const elements = try testInteractive("<div contenteditable=\"true\">Edit me</div>");
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expectEqual(InteractivityType.contenteditable, elements[0].interactivity_type);
|
||||
}
|
||||
|
||||
test "browser.interactive: tabindex" {
|
||||
const elements = try testInteractive("<div tabindex=\"0\">Focusable</div>");
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expectEqual(InteractivityType.focusable, elements[0].interactivity_type);
|
||||
try testing.expectEqual(@as(i32, 0), elements[0].tab_index);
|
||||
}
|
||||
|
||||
test "browser.interactive: disabled" {
|
||||
const elements = try testInteractive("<button disabled>Off</button>");
|
||||
try testing.expectEqual(1, elements.len);
|
||||
try testing.expect(elements[0].disabled);
|
||||
}
|
||||
|
||||
test "browser.interactive: non-interactive div" {
|
||||
const elements = try testInteractive("<div>Just text</div>");
|
||||
try testing.expectEqual(0, elements.len);
|
||||
}
|
||||
|
||||
test "browser.interactive: details and summary" {
|
||||
const elements = try testInteractive("<details><summary>More</summary><p>Content</p></details>");
|
||||
try testing.expectEqual(2, elements.len);
|
||||
try testing.expectEqual("details", elements[0].tag_name);
|
||||
try testing.expectEqual("summary", elements[1].tag_name);
|
||||
}
|
||||
|
||||
test "browser.interactive: mixed elements" {
|
||||
const elements = try testInteractive(
|
||||
\\<div>
|
||||
\\ <a href="/home">Home</a>
|
||||
\\ <p>Some text</p>
|
||||
\\ <button id="btn1">Submit</button>
|
||||
\\ <input type="email" placeholder="Email">
|
||||
\\ <div>Not interactive</div>
|
||||
\\ <div role="tab">Tab</div>
|
||||
\\</div>
|
||||
);
|
||||
try testing.expectEqual(4, elements.len);
|
||||
}
|
||||
@@ -19,15 +19,18 @@
|
||||
const std = @import("std");
|
||||
const lp = @import("lightpanda");
|
||||
const markdown = lp.markdown;
|
||||
const interactive = lp.interactive;
|
||||
const Node = @import("../Node.zig");
|
||||
|
||||
pub fn processMessage(cmd: anytype) !void {
|
||||
const action = std.meta.stringToEnum(enum {
|
||||
getMarkdown,
|
||||
getInteractiveElements,
|
||||
}, cmd.input.action) orelse return error.UnknownMethod;
|
||||
|
||||
switch (action) {
|
||||
.getMarkdown => return getMarkdown(cmd),
|
||||
.getInteractiveElements => return getInteractiveElements(cmd),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +57,35 @@ fn getMarkdown(cmd: anytype) !void {
|
||||
}, .{});
|
||||
}
|
||||
|
||||
fn getInteractiveElements(cmd: anytype) !void {
|
||||
const Params = struct {
|
||||
nodeId: ?Node.Id = null,
|
||||
};
|
||||
const params = (try cmd.params(Params)) orelse Params{};
|
||||
|
||||
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
||||
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
|
||||
|
||||
const root = if (params.nodeId) |nodeId|
|
||||
(bc.node_registry.lookup_by_id.get(nodeId) orelse return error.InvalidNodeId).dom
|
||||
else
|
||||
page.document.asNode();
|
||||
|
||||
const elements = try interactive.collectInteractiveElements(root, page, cmd.arena);
|
||||
|
||||
// Register nodes so nodeIds are valid for subsequent CDP calls.
|
||||
var node_ids: std.ArrayList(Node.Id) = try .initCapacity(cmd.arena, elements.len);
|
||||
for (elements) |el| {
|
||||
const registered = try bc.node_registry.register(el.node);
|
||||
node_ids.appendAssumeCapacity(registered.id);
|
||||
}
|
||||
|
||||
return cmd.sendResult(.{
|
||||
.elements = elements,
|
||||
.nodeIds = node_ids.items,
|
||||
}, .{});
|
||||
}
|
||||
|
||||
const testing = @import("../testing.zig");
|
||||
test "cdp.lp: getMarkdown" {
|
||||
var ctx = testing.context();
|
||||
@@ -70,3 +102,20 @@ test "cdp.lp: getMarkdown" {
|
||||
const result = ctx.client.?.sent.items[0].object.get("result").?.object;
|
||||
try testing.expect(result.get("markdown") != null);
|
||||
}
|
||||
|
||||
test "cdp.lp: getInteractiveElements" {
|
||||
var ctx = testing.context();
|
||||
defer ctx.deinit();
|
||||
|
||||
const bc = try ctx.loadBrowserContext(.{});
|
||||
_ = try bc.session.createPage();
|
||||
|
||||
try ctx.processMessage(.{
|
||||
.id = 1,
|
||||
.method = "LP.getInteractiveElements",
|
||||
});
|
||||
|
||||
const result = ctx.client.?.sent.items[0].object.get("result").?.object;
|
||||
try testing.expect(result.get("elements") != null);
|
||||
try testing.expect(result.get("nodeIds") != null);
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ pub const log = @import("log.zig");
|
||||
pub const js = @import("browser/js/js.zig");
|
||||
pub const dump = @import("browser/dump.zig");
|
||||
pub const markdown = @import("browser/markdown.zig");
|
||||
pub const interactive = @import("browser/interactive.zig");
|
||||
pub const mcp = @import("mcp.zig");
|
||||
pub const build_config = @import("build_config");
|
||||
pub const crash_handler = @import("crash_handler.zig");
|
||||
|
||||
Reference in New Issue
Block a user