mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-04-04 08:30:31 +00:00
Compare commits
68 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f75ce1778 | ||
|
|
7aabda9392 | ||
|
|
e29f33642c | ||
|
|
d94effb237 | ||
|
|
3b1ef66b51 | ||
|
|
15c0a7be83 | ||
|
|
a5d3d686b8 | ||
|
|
20c31a3f71 | ||
|
|
a81a24229b | ||
|
|
5826caf6dc | ||
|
|
b0c6c2d591 | ||
|
|
b33bb54442 | ||
|
|
06fe6c5e7d | ||
|
|
778b7eb8c2 | ||
|
|
ca5fa2b866 | ||
|
|
a71ff521aa | ||
|
|
5a551607c2 | ||
|
|
13ea4d1ee3 | ||
|
|
dc600c953f | ||
|
|
e00d569754 | ||
|
|
3d760e4577 | ||
|
|
1e8bdd7e28 | ||
|
|
31bab4cc05 | ||
|
|
a1a301666f | ||
|
|
619a2653d1 | ||
|
|
0b9cae5354 | ||
|
|
f098a991a8 | ||
|
|
7b5e4d6f52 | ||
|
|
9ffc99d6a2 | ||
|
|
855c3290ff | ||
|
|
d65a4b09f3 | ||
|
|
6a57d69359 | ||
|
|
9c5e67fbf5 | ||
|
|
7edb24e54d | ||
|
|
a60932bbe0 | ||
|
|
77e9f5caf7 | ||
|
|
cedc894445 | ||
|
|
9d62e58c9a | ||
|
|
609983da87 | ||
|
|
65f77af84d | ||
|
|
cd3e6b2364 | ||
|
|
557a4458a4 | ||
|
|
ce620e208d | ||
|
|
2de35a9db2 | ||
|
|
3eb05fdd1a | ||
|
|
186fdee59b | ||
|
|
3c8bb5bc00 | ||
|
|
66d190c047 | ||
|
|
5c2207ecc3 | ||
|
|
18d347e247 | ||
|
|
29dfbbfdea | ||
|
|
02f611bbc8 | ||
|
|
349d5a0a0b | ||
|
|
647d989191 | ||
|
|
41a24623fa | ||
|
|
d9a3d912c0 | ||
|
|
070baa8f46 | ||
|
|
4f78f299a3 | ||
|
|
9568c86326 | ||
|
|
6633b6effc | ||
|
|
2962864f3d | ||
|
|
749f21816c | ||
|
|
6bb8bc8391 | ||
|
|
72229f715a | ||
|
|
6c9a5ddab8 | ||
|
|
46a63e0b4b | ||
|
|
58143ee3d1 | ||
|
|
5e79af42f4 |
2
.github/workflows/e2e-integration-test.yml
vendored
2
.github/workflows/e2e-integration-test.yml
vendored
@@ -62,7 +62,7 @@ jobs:
|
||||
- name: run end to end integration tests
|
||||
continue-on-error: true
|
||||
run: |
|
||||
./lightpanda serve --log-level error & echo $! > LPD.pid
|
||||
./lightpanda serve --http-proxy ${{ secrets.MASSIVE_PROXY_RESIDENTIAL_US }} --log-level error & echo $! > LPD.pid
|
||||
go run integration/main.go |tee result.log
|
||||
kill `cat LPD.pid`
|
||||
|
||||
|
||||
37
build.zig
37
build.zig
@@ -85,32 +85,12 @@ pub fn build(b: *Build) !void {
|
||||
try linkV8(b, mod, enable_asan, enable_tsan, prebuilt_v8_path);
|
||||
try linkCurl(b, mod, enable_tsan);
|
||||
try linkHtml5Ever(b, mod);
|
||||
linkZenai(b, mod);
|
||||
linkLinenoise(b, mod);
|
||||
|
||||
break :blk mod;
|
||||
};
|
||||
|
||||
lightpanda_module.addCSourceFile(.{
|
||||
.file = b.path("lib/sqlite3/sqlite3.c"),
|
||||
.flags = &[_][]const u8{
|
||||
"-DSQLITE_DQS=0",
|
||||
"-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1",
|
||||
"-DSQLITE_USE_ALLOCA=1",
|
||||
"-DSQLITE_THREADSAFE=1",
|
||||
"-DSQLITE_TEMP_STORE=3",
|
||||
"-DSQLITE_ENABLE_API_ARMOR=1",
|
||||
"-DSQLITE_ENABLE_UNLOCK_NOTIFY",
|
||||
"-DSQLITE_DEFAULT_FILE_PERMISSIONS=0600",
|
||||
"-DSQLITE_OMIT_DECLTYPE=1",
|
||||
"-DSQLITE_OMIT_DEPRECATED=1",
|
||||
"-DSQLITE_OMIT_LOAD_EXTENSION=1",
|
||||
"-DSQLITE_OMIT_PROGRESS_CALLBACK=1",
|
||||
"-DSQLITE_OMIT_SHARED_CACHE",
|
||||
"-DSQLITE_OMIT_TRACE=1",
|
||||
"-DSQLITE_OMIT_UTF16=1",
|
||||
"-DHAVE_USLEEP=0",
|
||||
},
|
||||
});
|
||||
|
||||
// Check compilation
|
||||
const check = b.step("check", "Check if lightpanda compiles");
|
||||
|
||||
@@ -772,6 +752,19 @@ fn buildCurl(
|
||||
return lib;
|
||||
}
|
||||
|
||||
fn linkZenai(b: *Build, mod: *Build.Module) void {
|
||||
const dep = b.dependency("zenai", .{});
|
||||
mod.addImport("zenai", dep.module("zenai"));
|
||||
}
|
||||
|
||||
fn linkLinenoise(b: *Build, mod: *Build.Module) void {
|
||||
const dep = b.dependency("linenoise", .{});
|
||||
mod.addIncludePath(dep.path(""));
|
||||
mod.addCSourceFile(.{
|
||||
.file = dep.path("linenoise.c"),
|
||||
});
|
||||
}
|
||||
|
||||
/// Resolves the semantic version of the build.
|
||||
///
|
||||
/// The base version is read from `build.zig.zon`. This can be overridden
|
||||
|
||||
@@ -30,6 +30,13 @@
|
||||
.url = "https://github.com/curl/curl/releases/download/curl-8_18_0/curl-8.18.0.tar.gz",
|
||||
.hash = "N-V-__8AALp9QAGn6CCHZ6fK_FfMyGtG824LSHYHHasM3w-y",
|
||||
},
|
||||
.zenai = .{
|
||||
.path = "../zenai",
|
||||
},
|
||||
.linenoise = .{
|
||||
.url = "https://github.com/antirez/linenoise/archive/refs/tags/2.0.tar.gz",
|
||||
.hash = "N-V-__8AAJ4HAgCX79UDBfNwhqAqUVoGC44ib6UYa18q6oa_",
|
||||
},
|
||||
},
|
||||
.paths = .{""},
|
||||
}
|
||||
|
||||
265977
lib/sqlite3/sqlite3.c
265977
lib/sqlite3/sqlite3.c
File diff suppressed because it is too large
Load Diff
@@ -55,7 +55,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
|
||||
.arena_pool = undefined,
|
||||
};
|
||||
|
||||
app.network = try Network.init(allocator, config);
|
||||
app.network = try Network.init(allocator, app, config);
|
||||
errdefer app.network.deinit();
|
||||
|
||||
app.platform = try Platform.init();
|
||||
|
||||
181
src/Config.zig
181
src/Config.zig
@@ -32,6 +32,7 @@ pub const RunMode = enum {
|
||||
serve,
|
||||
version,
|
||||
mcp,
|
||||
agent,
|
||||
};
|
||||
|
||||
pub const CDP_MAX_HTTP_REQUEST_SIZE = 4096;
|
||||
@@ -63,56 +64,56 @@ pub fn deinit(self: *const Config, allocator: Allocator) void {
|
||||
|
||||
pub fn tlsVerifyHost(self: *const Config) bool {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.tls_verify_host,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.tls_verify_host,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn obeyRobots(self: *const Config) bool {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.obey_robots,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.obey_robots,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn httpProxy(self: *const Config) ?[:0]const u8 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_proxy,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.http_proxy,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn proxyBearerToken(self: *const Config) ?[:0]const u8 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.proxy_bearer_token,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.proxy_bearer_token,
|
||||
.help, .version => null,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn httpMaxConcurrent(self: *const Config) u8 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_max_concurrent orelse 10,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.http_max_concurrent orelse 10,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn httpMaxHostOpen(self: *const Config) u8 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_max_host_open orelse 4,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.http_max_host_open orelse 4,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn httpConnectTimeout(self: *const Config) u31 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_connect_timeout orelse 0,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.http_connect_timeout orelse 0,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn httpTimeout(self: *const Config) u31 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_timeout orelse 5000,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.http_timeout orelse 5000,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
@@ -123,39 +124,46 @@ pub fn httpMaxRedirects(_: *const Config) u8 {
|
||||
|
||||
pub fn httpMaxResponseSize(self: *const Config) ?usize {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_max_response_size,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.http_max_response_size,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn logLevel(self: *const Config) ?log.Level {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.log_level,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.log_level,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn logFormat(self: *const Config) ?log.Format {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.log_format,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.log_format,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn logFilterScopes(self: *const Config) ?[]const log.Scope {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.log_filter_scopes,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.log_filter_scopes,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.user_agent_suffix,
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| opts.common.user_agent_suffix,
|
||||
.help, .version => null,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn httpCacheDir(self: *const Config) ?[]const u8 {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| opts.common.http_cache_dir,
|
||||
else => null,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn cdpTimeout(self: *const Config) usize {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000,
|
||||
@@ -182,7 +190,7 @@ pub fn advertiseHost(self: *const Config) []const u8 {
|
||||
|
||||
pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
|
||||
inline .serve, .fetch, .mcp, .agent => |opts| WebBotAuthConfig{
|
||||
.key_file = opts.common.web_bot_auth_key_file orelse return null,
|
||||
.keyid = opts.common.web_bot_auth_keyid orelse return null,
|
||||
.domain = opts.common.web_bot_auth_domain orelse return null,
|
||||
@@ -213,6 +221,7 @@ pub const Mode = union(RunMode) {
|
||||
serve: Serve,
|
||||
version: void,
|
||||
mcp: Mcp,
|
||||
agent: Agent,
|
||||
};
|
||||
|
||||
pub const Serve = struct {
|
||||
@@ -231,6 +240,24 @@ pub const Mcp = struct {
|
||||
cdp_port: ?u16 = null,
|
||||
};
|
||||
|
||||
pub const AiProvider = enum {
|
||||
anthropic,
|
||||
openai,
|
||||
gemini,
|
||||
};
|
||||
|
||||
pub const Agent = struct {
|
||||
common: Common = .{},
|
||||
provider: AiProvider = .anthropic,
|
||||
model: ?[:0]const u8 = null,
|
||||
api_key: ?[:0]const u8 = null,
|
||||
system_prompt: ?[:0]const u8 = null,
|
||||
repl: bool = true,
|
||||
script_file: ?[]const u8 = null,
|
||||
record_file: ?[]const u8 = null,
|
||||
no_record: bool = false,
|
||||
};
|
||||
|
||||
pub const DumpFormat = enum {
|
||||
html,
|
||||
markdown,
|
||||
@@ -273,6 +300,7 @@ pub const Common = struct {
|
||||
log_format: ?log.Format = null,
|
||||
log_filter_scopes: ?[]log.Scope = null,
|
||||
user_agent_suffix: ?[]const u8 = null,
|
||||
http_cache_dir: ?[]const u8 = null,
|
||||
|
||||
web_bot_auth_key_file: ?[]const u8 = null,
|
||||
web_bot_auth_keyid: ?[]const u8 = null,
|
||||
@@ -392,13 +420,18 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
|
||||
\\
|
||||
\\--web-bot-auth-domain
|
||||
\\ Your domain e.g. yourdomain.com
|
||||
\\
|
||||
\\--http-cache-dir
|
||||
\\ Path to a directory to use as a Filesystem Cache for network resources.
|
||||
\\ Omitting this will result is no caching.
|
||||
\\ Defaults to no caching.
|
||||
;
|
||||
|
||||
// MAX_HELP_LEN|
|
||||
const usage =
|
||||
\\usage: {s} command [options] [URL]
|
||||
\\
|
||||
\\Command can be either 'fetch', 'serve', 'mcp' or 'help'
|
||||
\\Command can be either 'fetch', 'serve', 'mcp', 'agent' or 'help'
|
||||
\\
|
||||
\\fetch command
|
||||
\\Fetches the specified URL
|
||||
@@ -480,6 +513,24 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
|
||||
\\ Valid: 2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25.
|
||||
\\ Defaults to "2024-11-05".
|
||||
\\
|
||||
++ common_options ++
|
||||
\\
|
||||
\\agent command
|
||||
\\Starts an interactive AI agent that can browse the web
|
||||
\\Example: {s} agent --provider anthropic --model claude-sonnet-4-20250514
|
||||
\\
|
||||
\\Options:
|
||||
\\--provider The AI provider: anthropic, openai, or gemini.
|
||||
\\ Defaults to "anthropic".
|
||||
\\
|
||||
\\--model The model name to use.
|
||||
\\ Defaults to a sensible default per provider.
|
||||
\\
|
||||
\\--api-key The API key. Can also be set via environment variable:
|
||||
\\ ANTHROPIC_API_KEY, OPENAI_API_KEY, or GOOGLE_API_KEY.
|
||||
\\
|
||||
\\--system-prompt Override the default system prompt.
|
||||
\\
|
||||
++ common_options ++
|
||||
\\
|
||||
\\version command
|
||||
@@ -489,7 +540,7 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
|
||||
\\Displays this message
|
||||
\\
|
||||
;
|
||||
std.debug.print(usage, .{ self.exec_name, self.exec_name, self.exec_name, self.exec_name, self.exec_name });
|
||||
std.debug.print(usage, .{ self.exec_name, self.exec_name, self.exec_name, self.exec_name, self.exec_name, self.exec_name });
|
||||
if (success) {
|
||||
return std.process.cleanExit();
|
||||
}
|
||||
@@ -526,6 +577,8 @@ pub fn parseArgs(allocator: Allocator) !Config {
|
||||
return init(allocator, exec_name, .{ .help = false }) },
|
||||
.mcp => .{ .mcp = parseMcpArgs(allocator, &args) catch
|
||||
return init(allocator, exec_name, .{ .help = false }) },
|
||||
.agent => .{ .agent = parseAgentArgs(allocator, &args) catch
|
||||
return init(allocator, exec_name, .{ .help = false }) },
|
||||
.version => .{ .version = {} },
|
||||
};
|
||||
return init(allocator, exec_name, mode);
|
||||
@@ -871,6 +924,93 @@ fn parseFetchArgs(
|
||||
};
|
||||
}
|
||||
|
||||
fn parseAgentArgs(
|
||||
allocator: Allocator,
|
||||
args: *std.process.ArgIterator,
|
||||
) !Agent {
|
||||
var result: Agent = .{};
|
||||
|
||||
while (args.next()) |opt| {
|
||||
if (std.mem.eql(u8, "--provider", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = opt });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
result.provider = std.meta.stringToEnum(AiProvider, str) orelse {
|
||||
log.fatal(.app, "invalid provider", .{ .arg = opt, .val = str });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--model", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = opt });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
result.model = try allocator.dupeZ(u8, str);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--api-key", opt) or std.mem.eql(u8, "--api_key", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = opt });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
result.api_key = try allocator.dupeZ(u8, str);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--repl", opt)) {
|
||||
result.repl = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--run", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = opt });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
result.script_file = str;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--system-prompt", opt) or std.mem.eql(u8, "--system_prompt", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = opt });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
result.system_prompt = try allocator.dupeZ(u8, str);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--no-record", opt) or std.mem.eql(u8, "--no_record", opt)) {
|
||||
result.no_record = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (try parseCommonArg(allocator, opt, args, &result.common)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Positional argument: recording file for REPL mode (e.g. `agent --repl my_workflow.panda`)
|
||||
if (!std.mem.startsWith(u8, opt, "-")) {
|
||||
result.record_file = opt;
|
||||
continue;
|
||||
}
|
||||
|
||||
log.fatal(.app, "unknown argument", .{ .mode = "agent", .arg = opt });
|
||||
return error.UnkownOption;
|
||||
}
|
||||
|
||||
// If --no-record is set, clear the record file
|
||||
if (result.no_record) {
|
||||
result.record_file = null;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
fn parseCommonArg(
|
||||
allocator: Allocator,
|
||||
opt: []const u8,
|
||||
@@ -1066,5 +1206,14 @@ fn parseCommonArg(
|
||||
return true;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--http-cache-dir", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = "--http-cache-dir" });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
common.http_cache_dir = try allocator.dupe(u8, str);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
12
src/agent.zig
Normal file
12
src/agent.zig
Normal file
@@ -0,0 +1,12 @@
|
||||
pub const Agent = @import("agent/Agent.zig");
|
||||
pub const ToolExecutor = @import("agent/ToolExecutor.zig");
|
||||
pub const Terminal = @import("agent/Terminal.zig");
|
||||
pub const Command = @import("agent/Command.zig");
|
||||
pub const CommandExecutor = @import("agent/CommandExecutor.zig");
|
||||
pub const Recorder = @import("agent/Recorder.zig");
|
||||
|
||||
test {
|
||||
_ = Command;
|
||||
_ = CommandExecutor;
|
||||
_ = Recorder;
|
||||
}
|
||||
523
src/agent/Agent.zig
Normal file
523
src/agent/Agent.zig
Normal file
@@ -0,0 +1,523 @@
|
||||
const std = @import("std");
|
||||
const zenai = @import("zenai");
|
||||
const lp = @import("lightpanda");
|
||||
|
||||
const log = lp.log;
|
||||
const Config = lp.Config;
|
||||
const App = @import("../App.zig");
|
||||
const ToolExecutor = @import("ToolExecutor.zig");
|
||||
const Terminal = @import("Terminal.zig");
|
||||
const Command = @import("Command.zig");
|
||||
const CommandExecutor = @import("CommandExecutor.zig");
|
||||
const Recorder = @import("Recorder.zig");
|
||||
|
||||
const Self = @This();
|
||||
|
||||
const default_system_prompt =
|
||||
\\You are a web browsing assistant powered by the Lightpanda browser.
|
||||
\\You can navigate to websites, read their content, interact with forms,
|
||||
\\click links, and extract information.
|
||||
\\
|
||||
\\When helping the user, navigate to relevant pages and extract information.
|
||||
\\Use the semantic_tree or interactiveElements tools to understand page structure
|
||||
\\before clicking or filling forms. Be concise in your responses.
|
||||
;
|
||||
|
||||
const self_heal_prompt_prefix =
|
||||
\\A Pandascript command failed during replay. The original intent was:
|
||||
\\
|
||||
;
|
||||
|
||||
const self_heal_prompt_suffix =
|
||||
\\
|
||||
\\The command that failed was:
|
||||
\\
|
||||
;
|
||||
|
||||
const self_heal_prompt_page_state =
|
||||
\\
|
||||
\\Please analyze the current page state and execute the equivalent action.
|
||||
\\Use the available tools to accomplish the original intent.
|
||||
;
|
||||
|
||||
const login_prompt =
|
||||
\\Find the login form on the current page. Fill in the credentials using
|
||||
\\environment variables (look for $LP_EMAIL or $LP_USERNAME for the username
|
||||
\\field, and $LP_PASSWORD for the password field). Handle any cookie banners
|
||||
\\or popups first, then submit the login form.
|
||||
;
|
||||
|
||||
const accept_cookies_prompt =
|
||||
\\Find and dismiss the cookie consent banner on the current page.
|
||||
\\Look for "Accept", "Accept All", "I agree", or similar buttons and click them.
|
||||
;
|
||||
|
||||
allocator: std.mem.Allocator,
|
||||
ai_client: ?AiClient,
|
||||
tool_executor: *ToolExecutor,
|
||||
terminal: Terminal,
|
||||
cmd_executor: CommandExecutor,
|
||||
recorder: Recorder,
|
||||
messages: std.ArrayListUnmanaged(zenai.provider.Message),
|
||||
message_arena: std.heap.ArenaAllocator,
|
||||
tools: []const zenai.provider.Tool,
|
||||
model: []const u8,
|
||||
system_prompt: []const u8,
|
||||
script_file: ?[]const u8,
|
||||
record_file: ?[]const u8,
|
||||
|
||||
const AiClient = union(Config.AiProvider) {
|
||||
anthropic: *zenai.anthropic.Client,
|
||||
openai: *zenai.openai.Client,
|
||||
gemini: *zenai.gemini.Client,
|
||||
|
||||
fn toProvider(self: AiClient) zenai.provider.Client {
|
||||
return switch (self) {
|
||||
.anthropic => |c| .{ .anthropic = c },
|
||||
.openai => |c| .{ .openai = c },
|
||||
.gemini => |c| .{ .gemini = c },
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Self {
|
||||
const is_script_mode = opts.script_file != null;
|
||||
|
||||
// API key is only required for REPL mode and self-healing
|
||||
const api_key: ?[:0]const u8 = opts.api_key orelse getEnvApiKey(opts.provider) orelse if (!is_script_mode) {
|
||||
log.fatal(.app, "missing API key", .{
|
||||
.hint = "Set the API key via --api-key or environment variable",
|
||||
});
|
||||
return error.MissingApiKey;
|
||||
} else null;
|
||||
|
||||
const tool_executor = try ToolExecutor.init(allocator, app);
|
||||
errdefer tool_executor.deinit();
|
||||
|
||||
const self = try allocator.create(Self);
|
||||
errdefer allocator.destroy(self);
|
||||
|
||||
const ai_client: ?AiClient = if (api_key) |key| switch (opts.provider) {
|
||||
.anthropic => blk: {
|
||||
const client = try allocator.create(zenai.anthropic.Client);
|
||||
client.* = zenai.anthropic.Client.init(allocator, key, .{});
|
||||
break :blk .{ .anthropic = client };
|
||||
},
|
||||
.openai => blk: {
|
||||
const client = try allocator.create(zenai.openai.Client);
|
||||
client.* = zenai.openai.Client.init(allocator, key, .{});
|
||||
break :blk .{ .openai = client };
|
||||
},
|
||||
.gemini => blk: {
|
||||
const client = try allocator.create(zenai.gemini.Client);
|
||||
client.* = zenai.gemini.Client.init(allocator, key, .{});
|
||||
break :blk .{ .gemini = client };
|
||||
},
|
||||
} else null;
|
||||
|
||||
const tools = tool_executor.getTools() catch {
|
||||
log.fatal(.app, "failed to initialize tools", .{});
|
||||
return error.ToolInitFailed;
|
||||
};
|
||||
|
||||
self.* = .{
|
||||
.allocator = allocator,
|
||||
.ai_client = ai_client,
|
||||
.tool_executor = tool_executor,
|
||||
.terminal = Terminal.init(null),
|
||||
.cmd_executor = undefined,
|
||||
.recorder = Recorder.init(opts.record_file),
|
||||
.messages = .empty,
|
||||
.message_arena = std.heap.ArenaAllocator.init(allocator),
|
||||
.tools = tools,
|
||||
.model = opts.model orelse defaultModel(opts.provider),
|
||||
.system_prompt = opts.system_prompt orelse default_system_prompt,
|
||||
.script_file = opts.script_file,
|
||||
.record_file = opts.record_file,
|
||||
};
|
||||
|
||||
self.cmd_executor = CommandExecutor.init(allocator, tool_executor, &self.terminal);
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
self.recorder.deinit();
|
||||
self.message_arena.deinit();
|
||||
self.messages.deinit(self.allocator);
|
||||
self.tool_executor.deinit();
|
||||
if (self.ai_client) |ai_client| {
|
||||
switch (ai_client) {
|
||||
inline else => |c| {
|
||||
c.deinit();
|
||||
self.allocator.destroy(c);
|
||||
},
|
||||
}
|
||||
}
|
||||
self.allocator.destroy(self);
|
||||
}
|
||||
|
||||
pub fn run(self: *Self) void {
|
||||
if (self.script_file) |script_file| {
|
||||
self.runScript(script_file);
|
||||
} else {
|
||||
self.runRepl();
|
||||
}
|
||||
}
|
||||
|
||||
fn runRepl(self: *Self) void {
|
||||
self.terminal.printInfo("Lightpanda Agent (type 'quit' to exit)");
|
||||
log.debug(.app, "tools loaded", .{ .count = self.tools.len });
|
||||
const info = if (self.ai_client) |ai_client|
|
||||
std.fmt.allocPrint(self.allocator, "Provider: {s}, Model: {s}", .{
|
||||
@tagName(std.meta.activeTag(ai_client)),
|
||||
self.model,
|
||||
}) catch null
|
||||
else
|
||||
null;
|
||||
self.terminal.printInfo(info orelse "Ready.");
|
||||
if (info) |i| self.allocator.free(i);
|
||||
|
||||
while (true) {
|
||||
const line = self.terminal.readLine("> ") orelse break;
|
||||
defer self.terminal.freeLine(line);
|
||||
|
||||
if (line.len == 0) continue;
|
||||
|
||||
const cmd = Command.parse(line);
|
||||
switch (cmd) {
|
||||
.exit => break,
|
||||
.comment => continue,
|
||||
.login => {
|
||||
self.recorder.recordComment("# INTENT: LOGIN");
|
||||
self.processUserMessage(login_prompt) catch |err| {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "LOGIN failed: {s}", .{@errorName(err)}) catch "LOGIN failed";
|
||||
self.terminal.printError(msg);
|
||||
};
|
||||
},
|
||||
.accept_cookies => {
|
||||
self.recorder.recordComment("# INTENT: ACCEPT_COOKIES");
|
||||
self.processUserMessage(accept_cookies_prompt) catch |err| {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "ACCEPT_COOKIES failed: {s}", .{@errorName(err)}) catch "ACCEPT_COOKIES failed";
|
||||
self.terminal.printError(msg);
|
||||
};
|
||||
},
|
||||
.natural_language => {
|
||||
// "quit" as a convenience alias
|
||||
if (std.mem.eql(u8, line, "quit")) break;
|
||||
|
||||
self.processUserMessage(line) catch |err| {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "Request failed: {s}", .{@errorName(err)}) catch "Request failed";
|
||||
self.terminal.printError(msg);
|
||||
};
|
||||
},
|
||||
else => {
|
||||
self.cmd_executor.execute(cmd);
|
||||
self.recorder.record(line);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
self.terminal.printInfo("Goodbye!");
|
||||
}
|
||||
|
||||
fn runScript(self: *Self, path: []const u8) void {
|
||||
const file = std.fs.cwd().openFile(path, .{}) catch |err| {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "Failed to open script '{s}': {s}", .{ path, @errorName(err) }) catch "Failed to open script";
|
||||
self.terminal.printError(msg);
|
||||
return;
|
||||
};
|
||||
defer file.close();
|
||||
|
||||
const content = file.readToEndAlloc(self.allocator, 10 * 1024 * 1024) catch |err| {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "Failed to read script: {s}", .{@errorName(err)}) catch "Failed to read script";
|
||||
self.terminal.printError(msg);
|
||||
return;
|
||||
};
|
||||
defer self.allocator.free(content);
|
||||
|
||||
const script_info = std.fmt.allocPrint(self.allocator, "Running script: {s}", .{path}) catch null;
|
||||
self.terminal.printInfo(script_info orelse "Running script...");
|
||||
if (script_info) |i| self.allocator.free(i);
|
||||
|
||||
var script_arena = std.heap.ArenaAllocator.init(self.allocator);
|
||||
defer script_arena.deinit();
|
||||
|
||||
var iter = Command.ScriptIterator.init(content, script_arena.allocator());
|
||||
var last_intent: ?[]const u8 = null;
|
||||
|
||||
while (iter.next()) |entry| {
|
||||
switch (entry.command) {
|
||||
.exit => {
|
||||
self.terminal.printInfo("EXIT — stopping script.");
|
||||
return;
|
||||
},
|
||||
.comment => {
|
||||
// Track # INTENT: comments for self-healing
|
||||
if (std.mem.startsWith(u8, entry.raw_line, "# INTENT:")) {
|
||||
last_intent = std.mem.trim(u8, entry.raw_line["# INTENT:".len..], &std.ascii.whitespace);
|
||||
}
|
||||
continue;
|
||||
},
|
||||
.natural_language => {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "line {d}: unrecognized command: {s}", .{ entry.line_num, entry.raw_line }) catch "unrecognized command";
|
||||
self.terminal.printError(msg);
|
||||
return;
|
||||
},
|
||||
.login, .accept_cookies => {
|
||||
// High-level commands require LLM
|
||||
if (self.ai_client == null) {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "line {d}: {s} requires an API key for LLM resolution", .{
|
||||
entry.line_num,
|
||||
entry.raw_line,
|
||||
}) catch "LLM required";
|
||||
self.terminal.printError(msg);
|
||||
return;
|
||||
}
|
||||
const prompt = if (entry.command == .login) login_prompt else accept_cookies_prompt;
|
||||
self.processUserMessage(prompt) catch |err| {
|
||||
const msg = std.fmt.allocPrint(self.allocator, "line {d}: {s} failed: {s}", .{
|
||||
entry.line_num,
|
||||
entry.raw_line,
|
||||
@errorName(err),
|
||||
}) catch "command failed";
|
||||
self.terminal.printError(msg);
|
||||
return;
|
||||
};
|
||||
},
|
||||
else => {
|
||||
const line_info = std.fmt.allocPrint(self.allocator, "[{d}] {s}", .{ entry.line_num, entry.raw_line }) catch null;
|
||||
self.terminal.printInfo(line_info orelse entry.raw_line);
|
||||
if (line_info) |li| self.allocator.free(li);
|
||||
|
||||
// Execute with result checking for self-healing
|
||||
var cmd_arena = std.heap.ArenaAllocator.init(self.allocator);
|
||||
defer cmd_arena.deinit();
|
||||
|
||||
const result = self.cmd_executor.executeWithResult(cmd_arena.allocator(), entry.command);
|
||||
self.terminal.printAssistant(result.output);
|
||||
std.debug.print("\n", .{});
|
||||
|
||||
if (result.failed) {
|
||||
// Attempt self-healing via LLM
|
||||
if (self.ai_client != null) {
|
||||
self.terminal.printInfo("Command failed, attempting self-healing...");
|
||||
if (self.attemptSelfHeal(last_intent, entry.raw_line)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const msg = std.fmt.allocPrint(self.allocator, "line {d}: command failed: {s}", .{
|
||||
entry.line_num,
|
||||
entry.raw_line,
|
||||
}) catch "command failed";
|
||||
self.terminal.printError(msg);
|
||||
return;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
self.terminal.printInfo("Script completed.");
|
||||
}
|
||||
|
||||
/// Attempt to self-heal a failed command by asking the LLM to resolve it.
|
||||
fn attemptSelfHeal(self: *Self, intent: ?[]const u8, failed_command: []const u8) bool {
|
||||
var heal_arena = std.heap.ArenaAllocator.init(self.allocator);
|
||||
defer heal_arena.deinit();
|
||||
const ha = heal_arena.allocator();
|
||||
|
||||
// Build the self-healing prompt
|
||||
const prompt = std.fmt.allocPrint(ha, "{s}{s}{s}{s}{s}", .{
|
||||
self_heal_prompt_prefix,
|
||||
intent orelse "(no recorded intent)",
|
||||
self_heal_prompt_suffix,
|
||||
failed_command,
|
||||
self_heal_prompt_page_state,
|
||||
}) catch return false;
|
||||
|
||||
self.processUserMessage(prompt) catch return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
fn processUserMessage(self: *Self, user_input: []const u8) !void {
|
||||
const ma = self.message_arena.allocator();
|
||||
|
||||
// Add system prompt as first message if this is the first user message
|
||||
if (self.messages.items.len == 0) {
|
||||
try self.messages.append(self.allocator, .{
|
||||
.role = .system,
|
||||
.content = self.system_prompt,
|
||||
});
|
||||
}
|
||||
|
||||
// Add user message
|
||||
try self.messages.append(self.allocator, .{
|
||||
.role = .user,
|
||||
.content = try ma.dupe(u8, user_input),
|
||||
});
|
||||
|
||||
// Loop: send to LLM, execute tool calls, repeat until we get text
|
||||
var max_iterations: u32 = 20;
|
||||
while (max_iterations > 0) : (max_iterations -= 1) {
|
||||
const provider_client = (self.ai_client orelse return error.NoAiClient).toProvider();
|
||||
var result = provider_client.generateContent(self.model, self.messages.items, .{
|
||||
.tools = self.tools,
|
||||
.max_tokens = 4096,
|
||||
}) catch |err| {
|
||||
log.err(.app, "AI API error", .{ .err = err });
|
||||
return error.ApiError;
|
||||
};
|
||||
defer result.deinit();
|
||||
|
||||
log.debug(.app, "LLM response", .{
|
||||
.finish_reason = @tagName(result.finish_reason),
|
||||
.has_text = result.text != null,
|
||||
.has_tool_calls = result.tool_calls != null,
|
||||
});
|
||||
|
||||
// Handle tool calls (check for tool_calls presence, not just finish_reason,
|
||||
// because some providers like Gemini return finish_reason=STOP for tool calls)
|
||||
if (result.tool_calls) |tool_calls| {
|
||||
// Add the assistant message with tool calls
|
||||
try self.messages.append(self.allocator, .{
|
||||
.role = .assistant,
|
||||
.content = if (result.text) |t| try ma.dupe(u8, t) else null,
|
||||
.tool_calls = try self.dupeToolCalls(tool_calls),
|
||||
});
|
||||
|
||||
// Execute each tool call and collect results
|
||||
var tool_results: std.ArrayListUnmanaged(zenai.provider.ToolResult) = .empty;
|
||||
|
||||
for (tool_calls) |tc| {
|
||||
self.terminal.printToolCall(tc.name, tc.arguments);
|
||||
|
||||
var tool_arena = std.heap.ArenaAllocator.init(self.allocator);
|
||||
defer tool_arena.deinit();
|
||||
|
||||
const tool_result = self.tool_executor.call(tool_arena.allocator(), tc.name, tc.arguments) catch "Error: tool execution failed";
|
||||
self.terminal.printToolResult(tc.name, tool_result);
|
||||
|
||||
// Record resolved tool call as Pandascript command
|
||||
if (!std.mem.startsWith(u8, tool_result, "Error:")) {
|
||||
self.recordToolCall(tool_arena.allocator(), tc.name, tc.arguments);
|
||||
}
|
||||
|
||||
try tool_results.append(ma, .{
|
||||
.id = try ma.dupe(u8, tc.id),
|
||||
.name = try ma.dupe(u8, tc.name),
|
||||
.content = try ma.dupe(u8, tool_result),
|
||||
});
|
||||
}
|
||||
|
||||
// Add tool results as a message
|
||||
try self.messages.append(self.allocator, .{
|
||||
.role = .tool,
|
||||
.tool_results = try tool_results.toOwnedSlice(ma),
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Text response
|
||||
if (result.text) |text| {
|
||||
std.debug.print("\n", .{});
|
||||
self.terminal.printAssistant(text);
|
||||
std.debug.print("\n\n", .{});
|
||||
|
||||
try self.messages.append(self.allocator, .{
|
||||
.role = .assistant,
|
||||
.content = try ma.dupe(u8, text),
|
||||
});
|
||||
} else {
|
||||
self.terminal.printInfo("(no response from model)");
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a tool call (name + JSON arguments) into a Pandascript command and record it.
|
||||
fn recordToolCall(self: *Self, arena: std.mem.Allocator, tool_name: []const u8, arguments: []const u8) void {
|
||||
const parsed = std.json.parseFromSlice(std.json.Value, arena, arguments, .{}) catch return;
|
||||
const obj = switch (parsed.value) {
|
||||
.object => |o| o,
|
||||
else => return,
|
||||
};
|
||||
|
||||
const panda_cmd: ?[]const u8 = if (std.mem.eql(u8, tool_name, "goto") or std.mem.eql(u8, tool_name, "navigate")) blk: {
|
||||
const url = switch (obj.get("url") orelse break :blk null) {
|
||||
.string => |s| s,
|
||||
else => break :blk null,
|
||||
};
|
||||
break :blk std.fmt.allocPrint(arena, "GOTO {s}", .{url}) catch null;
|
||||
} else if (std.mem.eql(u8, tool_name, "click")) blk: {
|
||||
if (obj.get("selector")) |sel_val| {
|
||||
const sel = switch (sel_val) {
|
||||
.string => |s| s,
|
||||
else => break :blk null,
|
||||
};
|
||||
break :blk std.fmt.allocPrint(arena, "CLICK \"{s}\"", .{sel}) catch null;
|
||||
}
|
||||
if (obj.get("backendNodeId")) |_| {
|
||||
// Can't meaningfully record a node ID as Pandascript
|
||||
break :blk null;
|
||||
}
|
||||
break :blk null;
|
||||
} else if (std.mem.eql(u8, tool_name, "fill")) blk: {
|
||||
const sel = switch (obj.get("selector") orelse break :blk null) {
|
||||
.string => |s| s,
|
||||
else => break :blk null,
|
||||
};
|
||||
const val = switch (obj.get("value") orelse break :blk null) {
|
||||
.string => |s| s,
|
||||
else => break :blk null,
|
||||
};
|
||||
break :blk std.fmt.allocPrint(arena, "TYPE \"{s}\" \"{s}\"", .{ sel, val }) catch null;
|
||||
} else if (std.mem.eql(u8, tool_name, "waitForSelector")) blk: {
|
||||
// WAIT is read-only, not recorded — Recorder will skip it anyway
|
||||
break :blk null;
|
||||
} else if (std.mem.eql(u8, tool_name, "evaluate") or std.mem.eql(u8, tool_name, "eval")) blk: {
|
||||
const script = switch (obj.get("script") orelse break :blk null) {
|
||||
.string => |s| s,
|
||||
else => break :blk null,
|
||||
};
|
||||
// Use multi-line format if the script contains newlines
|
||||
if (std.mem.indexOfScalar(u8, script, '\n') != null) {
|
||||
break :blk std.fmt.allocPrint(arena, "EVAL \"\"\"\n{s}\n\"\"\"", .{script}) catch null;
|
||||
}
|
||||
break :blk std.fmt.allocPrint(arena, "EVAL \"{s}\"", .{script}) catch null;
|
||||
} else null;
|
||||
|
||||
if (panda_cmd) |cmd| {
|
||||
self.recorder.record(cmd);
|
||||
}
|
||||
}
|
||||
|
||||
fn dupeToolCalls(self: *Self, calls: []const zenai.provider.ToolCall) ![]const zenai.provider.ToolCall {
|
||||
const ma = self.message_arena.allocator();
|
||||
const duped = try ma.alloc(zenai.provider.ToolCall, calls.len);
|
||||
for (calls, 0..) |tc, i| {
|
||||
duped[i] = .{
|
||||
.id = try ma.dupe(u8, tc.id),
|
||||
.name = try ma.dupe(u8, tc.name),
|
||||
.arguments = try ma.dupe(u8, tc.arguments),
|
||||
};
|
||||
}
|
||||
return duped;
|
||||
}
|
||||
|
||||
fn getEnvApiKey(provider_type: Config.AiProvider) ?[:0]const u8 {
|
||||
return switch (provider_type) {
|
||||
.anthropic => std.posix.getenv("ANTHROPIC_API_KEY"),
|
||||
.openai => std.posix.getenv("OPENAI_API_KEY"),
|
||||
.gemini => std.posix.getenv("GOOGLE_API_KEY") orelse std.posix.getenv("GEMINI_API_KEY"),
|
||||
};
|
||||
}
|
||||
|
||||
fn defaultModel(provider_type: Config.AiProvider) []const u8 {
|
||||
return switch (provider_type) {
|
||||
.anthropic => "claude-sonnet-4-20250514",
|
||||
.openai => "gpt-4o",
|
||||
.gemini => "gemini-2.5-flash",
|
||||
};
|
||||
}
|
||||
409
src/agent/Command.zig
Normal file
409
src/agent/Command.zig
Normal file
@@ -0,0 +1,409 @@
|
||||
const std = @import("std");
|
||||
|
||||
pub const TypeArgs = struct {
|
||||
selector: []const u8,
|
||||
value: []const u8,
|
||||
};
|
||||
|
||||
pub const ExtractArgs = struct {
|
||||
selector: []const u8,
|
||||
file: ?[]const u8,
|
||||
};
|
||||
|
||||
pub const Command = union(enum) {
|
||||
goto: []const u8,
|
||||
click: []const u8,
|
||||
type_cmd: TypeArgs,
|
||||
wait: []const u8,
|
||||
tree: void,
|
||||
markdown: void,
|
||||
extract: ExtractArgs,
|
||||
eval_js: []const u8,
|
||||
login: void,
|
||||
accept_cookies: void,
|
||||
exit: void,
|
||||
comment: void,
|
||||
natural_language: []const u8,
|
||||
};
|
||||
|
||||
/// Parse a line of REPL input into a Pandascript command.
|
||||
/// Unrecognized input is returned as `.natural_language`.
|
||||
/// For multi-line EVAL blocks in scripts, use `ScriptParser`.
|
||||
pub fn parse(line: []const u8) Command {
|
||||
const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (trimmed.len == 0) return .{ .natural_language = trimmed };
|
||||
|
||||
// Skip comment lines
|
||||
if (trimmed[0] == '#') return .{ .comment = {} };
|
||||
|
||||
// Find the command word (first whitespace-delimited token)
|
||||
const cmd_end = std.mem.indexOfAny(u8, trimmed, &std.ascii.whitespace) orelse trimmed.len;
|
||||
const cmd_word = trimmed[0..cmd_end];
|
||||
const rest = std.mem.trim(u8, trimmed[cmd_end..], &std.ascii.whitespace);
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "GOTO")) {
|
||||
if (rest.len == 0) return .{ .natural_language = trimmed };
|
||||
return .{ .goto = rest };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "CLICK")) {
|
||||
const arg = extractQuoted(rest) orelse rest;
|
||||
if (arg.len == 0) return .{ .natural_language = trimmed };
|
||||
return .{ .click = arg };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "TYPE")) {
|
||||
const first = extractQuotedWithRemainder(rest) orelse return .{ .natural_language = trimmed };
|
||||
const second_arg = std.mem.trim(u8, first.remainder, &std.ascii.whitespace);
|
||||
const second = extractQuoted(second_arg) orelse return .{ .natural_language = trimmed };
|
||||
return .{ .type_cmd = .{ .selector = first.value, .value = second } };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "WAIT")) {
|
||||
const arg = extractQuoted(rest) orelse rest;
|
||||
if (arg.len == 0) return .{ .natural_language = trimmed };
|
||||
return .{ .wait = arg };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "TREE")) {
|
||||
return .{ .tree = {} };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "MARKDOWN") or eqlIgnoreCase(cmd_word, "MD")) {
|
||||
return .{ .markdown = {} };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "EXTRACT")) {
|
||||
const selector = extractQuoted(rest) orelse {
|
||||
if (rest.len == 0) return .{ .natural_language = trimmed };
|
||||
return .{ .extract = .{ .selector = rest, .file = null } };
|
||||
};
|
||||
// Look for > filename after the quoted selector
|
||||
const after_quote = extractQuotedWithRemainder(rest) orelse return .{ .extract = .{ .selector = selector, .file = null } };
|
||||
const after = std.mem.trim(u8, after_quote.remainder, &std.ascii.whitespace);
|
||||
if (after.len > 0 and after[0] == '>') {
|
||||
const file = std.mem.trim(u8, after[1..], &std.ascii.whitespace);
|
||||
return .{ .extract = .{ .selector = selector, .file = if (file.len > 0) file else null } };
|
||||
}
|
||||
return .{ .extract = .{ .selector = selector, .file = null } };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "EVAL")) {
|
||||
if (rest.len == 0) return .{ .natural_language = trimmed };
|
||||
const arg = extractQuoted(rest) orelse rest;
|
||||
return .{ .eval_js = arg };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "LOGIN")) {
|
||||
return .{ .login = {} };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "ACCEPT_COOKIES") or eqlIgnoreCase(cmd_word, "ACCEPT-COOKIES")) {
|
||||
return .{ .accept_cookies = {} };
|
||||
}
|
||||
|
||||
if (eqlIgnoreCase(cmd_word, "EXIT")) {
|
||||
return .{ .exit = {} };
|
||||
}
|
||||
|
||||
return .{ .natural_language = trimmed };
|
||||
}
|
||||
|
||||
/// Iterator for parsing a script file, handling multi-line EVAL """ ... """ blocks.
|
||||
pub const ScriptIterator = struct {
|
||||
lines: std.mem.SplitIterator(u8, .scalar),
|
||||
line_num: u32,
|
||||
allocator: std.mem.Allocator,
|
||||
|
||||
pub fn init(content: []const u8, allocator: std.mem.Allocator) ScriptIterator {
|
||||
return .{
|
||||
.lines = std.mem.splitScalar(u8, content, '\n'),
|
||||
.line_num = 0,
|
||||
.allocator = allocator,
|
||||
};
|
||||
}
|
||||
|
||||
pub const Entry = struct {
|
||||
line_num: u32,
|
||||
raw_line: []const u8,
|
||||
command: Command,
|
||||
};
|
||||
|
||||
/// Returns the next command from the script, or null at EOF.
|
||||
/// Multi-line EVAL blocks are assembled into a single eval_js command.
|
||||
pub fn next(self: *ScriptIterator) ?Entry {
|
||||
while (self.lines.next()) |line| {
|
||||
self.line_num += 1;
|
||||
const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (trimmed.len == 0) continue;
|
||||
|
||||
// Check for EVAL """ multi-line block
|
||||
if (isEvalTripleQuote(trimmed)) {
|
||||
const start_line = self.line_num;
|
||||
if (self.collectEvalBlock()) |js| {
|
||||
return .{
|
||||
.line_num = start_line,
|
||||
.raw_line = trimmed,
|
||||
.command = .{ .eval_js = js },
|
||||
};
|
||||
} else {
|
||||
return .{
|
||||
.line_num = start_line,
|
||||
.raw_line = trimmed,
|
||||
.command = .{ .natural_language = "unterminated EVAL block" },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return .{
|
||||
.line_num = self.line_num,
|
||||
.raw_line = trimmed,
|
||||
.command = parse(trimmed),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn isEvalTripleQuote(line: []const u8) bool {
|
||||
const cmd_end = std.mem.indexOfAny(u8, line, &std.ascii.whitespace) orelse line.len;
|
||||
const cmd_word = line[0..cmd_end];
|
||||
if (!eqlIgnoreCase(cmd_word, "EVAL")) return false;
|
||||
const rest = std.mem.trim(u8, line[cmd_end..], &std.ascii.whitespace);
|
||||
return std.mem.startsWith(u8, rest, "\"\"\"");
|
||||
}
|
||||
|
||||
/// Collect lines until closing """, return the JS content.
|
||||
fn collectEvalBlock(self: *ScriptIterator) ?[]const u8 {
|
||||
var parts: std.ArrayListUnmanaged(u8) = .empty;
|
||||
while (self.lines.next()) |line| {
|
||||
self.line_num += 1;
|
||||
const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (std.mem.eql(u8, trimmed, "\"\"\"")) {
|
||||
return parts.toOwnedSlice(self.allocator) catch null;
|
||||
}
|
||||
if (parts.items.len > 0) {
|
||||
parts.append(self.allocator, '\n') catch return null;
|
||||
}
|
||||
parts.appendSlice(self.allocator, line) catch return null;
|
||||
}
|
||||
// Unterminated
|
||||
parts.deinit(self.allocator);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const QuotedResult = struct {
|
||||
value: []const u8,
|
||||
remainder: []const u8,
|
||||
};
|
||||
|
||||
fn extractQuotedWithRemainder(s: []const u8) ?QuotedResult {
|
||||
if (s.len < 2 or s[0] != '"') return null;
|
||||
const end = std.mem.indexOfScalarPos(u8, s, 1, '"') orelse return null;
|
||||
return .{
|
||||
.value = s[1..end],
|
||||
.remainder = s[end + 1 ..],
|
||||
};
|
||||
}
|
||||
|
||||
fn extractQuoted(s: []const u8) ?[]const u8 {
|
||||
const result = extractQuotedWithRemainder(s) orelse return null;
|
||||
return result.value;
|
||||
}
|
||||
|
||||
fn eqlIgnoreCase(a: []const u8, comptime upper: []const u8) bool {
|
||||
if (a.len != upper.len) return false;
|
||||
for (a, upper) |ac, uc| {
|
||||
if (std.ascii.toUpper(ac) != uc) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
test "parse GOTO" {
|
||||
const cmd = parse("GOTO https://example.com");
|
||||
try std.testing.expectEqualStrings("https://example.com", cmd.goto);
|
||||
}
|
||||
|
||||
test "parse GOTO case insensitive" {
|
||||
const cmd = parse("goto https://example.com");
|
||||
try std.testing.expectEqualStrings("https://example.com", cmd.goto);
|
||||
}
|
||||
|
||||
test "parse GOTO missing url" {
|
||||
const cmd = parse("GOTO");
|
||||
try std.testing.expect(cmd == .natural_language);
|
||||
}
|
||||
|
||||
test "parse CLICK quoted" {
|
||||
const cmd = parse("CLICK \"Login\"");
|
||||
try std.testing.expectEqualStrings("Login", cmd.click);
|
||||
}
|
||||
|
||||
test "parse CLICK unquoted" {
|
||||
const cmd = parse("CLICK .submit-btn");
|
||||
try std.testing.expectEqualStrings(".submit-btn", cmd.click);
|
||||
}
|
||||
|
||||
test "parse TYPE two quoted args" {
|
||||
const cmd = parse("TYPE \"#email\" \"user@test.com\"");
|
||||
try std.testing.expectEqualStrings("#email", cmd.type_cmd.selector);
|
||||
try std.testing.expectEqualStrings("user@test.com", cmd.type_cmd.value);
|
||||
}
|
||||
|
||||
test "parse TYPE missing second arg" {
|
||||
const cmd = parse("TYPE \"#email\"");
|
||||
try std.testing.expect(cmd == .natural_language);
|
||||
}
|
||||
|
||||
test "parse WAIT" {
|
||||
const cmd = parse("WAIT \".dashboard\"");
|
||||
try std.testing.expectEqualStrings(".dashboard", cmd.wait);
|
||||
}
|
||||
|
||||
test "parse TREE" {
|
||||
const cmd = parse("TREE");
|
||||
try std.testing.expect(cmd == .tree);
|
||||
}
|
||||
|
||||
test "parse MARKDOWN alias MD" {
|
||||
try std.testing.expect(parse("MARKDOWN") == .markdown);
|
||||
try std.testing.expect(parse("md") == .markdown);
|
||||
}
|
||||
|
||||
test "parse EXTRACT with file" {
|
||||
const cmd = parse("EXTRACT \".title\" > titles.json");
|
||||
try std.testing.expectEqualStrings(".title", cmd.extract.selector);
|
||||
try std.testing.expectEqualStrings("titles.json", cmd.extract.file.?);
|
||||
}
|
||||
|
||||
test "parse EXTRACT without file" {
|
||||
const cmd = parse("EXTRACT \".title\"");
|
||||
try std.testing.expectEqualStrings(".title", cmd.extract.selector);
|
||||
try std.testing.expect(cmd.extract.file == null);
|
||||
}
|
||||
|
||||
test "parse EVAL single line" {
|
||||
const cmd = parse("EVAL \"document.title\"");
|
||||
try std.testing.expectEqualStrings("document.title", cmd.eval_js);
|
||||
}
|
||||
|
||||
test "parse LOGIN" {
|
||||
try std.testing.expect(parse("LOGIN") == .login);
|
||||
try std.testing.expect(parse("login") == .login);
|
||||
}
|
||||
|
||||
test "parse ACCEPT_COOKIES" {
|
||||
try std.testing.expect(parse("ACCEPT_COOKIES") == .accept_cookies);
|
||||
try std.testing.expect(parse("ACCEPT-COOKIES") == .accept_cookies);
|
||||
}
|
||||
|
||||
test "parse EXIT" {
|
||||
try std.testing.expect(parse("EXIT") == .exit);
|
||||
}
|
||||
|
||||
test "parse comment" {
|
||||
try std.testing.expect(parse("# this is a comment") == .comment);
|
||||
try std.testing.expect(parse("# INTENT: LOGIN") == .comment);
|
||||
}
|
||||
|
||||
test "parse natural language fallback" {
|
||||
const cmd = parse("what is on this page?");
|
||||
try std.testing.expectEqualStrings("what is on this page?", cmd.natural_language);
|
||||
}
|
||||
|
||||
test "parse whitespace trimming" {
|
||||
const cmd = parse(" GOTO https://example.com ");
|
||||
try std.testing.expectEqualStrings("https://example.com", cmd.goto);
|
||||
}
|
||||
|
||||
test "parse empty input" {
|
||||
const cmd = parse("");
|
||||
try std.testing.expect(cmd == .natural_language);
|
||||
}
|
||||
|
||||
test "ScriptIterator basic commands" {
|
||||
const script =
|
||||
\\GOTO https://example.com
|
||||
\\TREE
|
||||
\\CLICK "Login"
|
||||
;
|
||||
var iter = ScriptIterator.init(script, std.testing.allocator);
|
||||
|
||||
const e1 = iter.next().?;
|
||||
try std.testing.expectEqualStrings("https://example.com", e1.command.goto);
|
||||
try std.testing.expectEqual(@as(u32, 1), e1.line_num);
|
||||
|
||||
const e2 = iter.next().?;
|
||||
try std.testing.expect(e2.command == .tree);
|
||||
|
||||
const e3 = iter.next().?;
|
||||
try std.testing.expectEqualStrings("Login", e3.command.click);
|
||||
|
||||
try std.testing.expect(iter.next() == null);
|
||||
}
|
||||
|
||||
test "ScriptIterator skips blank lines and comments" {
|
||||
const script =
|
||||
\\# Navigate
|
||||
\\GOTO https://example.com
|
||||
\\
|
||||
\\# Extract
|
||||
\\TREE
|
||||
;
|
||||
var iter = ScriptIterator.init(script, std.testing.allocator);
|
||||
|
||||
const e1 = iter.next().?;
|
||||
try std.testing.expect(e1.command == .comment);
|
||||
|
||||
const e2 = iter.next().?;
|
||||
try std.testing.expect(e2.command == .goto);
|
||||
|
||||
const e3 = iter.next().?;
|
||||
try std.testing.expect(e3.command == .comment);
|
||||
|
||||
const e4 = iter.next().?;
|
||||
try std.testing.expect(e4.command == .tree);
|
||||
|
||||
try std.testing.expect(iter.next() == null);
|
||||
}
|
||||
|
||||
test "ScriptIterator multi-line EVAL" {
|
||||
const script =
|
||||
\\GOTO https://example.com
|
||||
\\EVAL """
|
||||
\\ const x = 1;
|
||||
\\ const y = 2;
|
||||
\\ return x + y;
|
||||
\\"""
|
||||
\\TREE
|
||||
;
|
||||
var iter = ScriptIterator.init(script, std.testing.allocator);
|
||||
|
||||
const e1 = iter.next().?;
|
||||
try std.testing.expect(e1.command == .goto);
|
||||
|
||||
const e2 = iter.next().?;
|
||||
try std.testing.expect(e2.command == .eval_js);
|
||||
try std.testing.expect(std.mem.indexOf(u8, e2.command.eval_js, "const x = 1;") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, e2.command.eval_js, "return x + y;") != null);
|
||||
defer std.testing.allocator.free(e2.command.eval_js);
|
||||
|
||||
const e3 = iter.next().?;
|
||||
try std.testing.expect(e3.command == .tree);
|
||||
|
||||
try std.testing.expect(iter.next() == null);
|
||||
}
|
||||
|
||||
test "ScriptIterator unterminated EVAL" {
|
||||
const script =
|
||||
\\EVAL """
|
||||
\\ const x = 1;
|
||||
;
|
||||
var iter = ScriptIterator.init(script, std.testing.allocator);
|
||||
|
||||
const e1 = iter.next().?;
|
||||
try std.testing.expect(e1.command == .natural_language);
|
||||
try std.testing.expectEqualStrings("unterminated EVAL block", e1.command.natural_language);
|
||||
}
|
||||
298
src/agent/CommandExecutor.zig
Normal file
298
src/agent/CommandExecutor.zig
Normal file
@@ -0,0 +1,298 @@
|
||||
const std = @import("std");
|
||||
const Command = @import("Command.zig");
|
||||
const ToolExecutor = @import("ToolExecutor.zig");
|
||||
const Terminal = @import("Terminal.zig");
|
||||
|
||||
const Self = @This();
|
||||
|
||||
tool_executor: *ToolExecutor,
|
||||
terminal: *Terminal,
|
||||
allocator: std.mem.Allocator,
|
||||
|
||||
pub fn init(allocator: std.mem.Allocator, tool_executor: *ToolExecutor, terminal: *Terminal) Self {
|
||||
return .{
|
||||
.allocator = allocator,
|
||||
.tool_executor = tool_executor,
|
||||
.terminal = terminal,
|
||||
};
|
||||
}
|
||||
|
||||
pub const ExecResult = struct {
|
||||
output: []const u8,
|
||||
failed: bool,
|
||||
};
|
||||
|
||||
/// Execute a command and return the result with success/failure status.
|
||||
pub fn executeWithResult(self: *Self, a: std.mem.Allocator, cmd: Command.Command) ExecResult {
|
||||
const result = switch (cmd) {
|
||||
.goto => |url| self.execGoto(a, url),
|
||||
.click => |target| self.execClick(a, target),
|
||||
.type_cmd => |args| self.execType(a, args),
|
||||
.wait => |selector| self.tool_executor.call(a, "waitForSelector", buildJson(a, .{ .selector = selector })) catch "Error: wait failed",
|
||||
.tree => self.tool_executor.call(a, "semantic_tree", "") catch "Error: tree failed",
|
||||
.markdown => self.tool_executor.call(a, "markdown", "") catch "Error: markdown failed",
|
||||
.extract => |args| self.execExtract(a, args),
|
||||
.eval_js => |script| self.tool_executor.call(a, "evaluate", buildJson(a, .{ .script = script })) catch "Error: eval failed",
|
||||
.exit, .natural_language, .comment, .login, .accept_cookies => unreachable,
|
||||
};
|
||||
|
||||
return .{
|
||||
.output = result,
|
||||
.failed = std.mem.startsWith(u8, result, "Error:"),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn execute(self: *Self, cmd: Command.Command) void {
|
||||
var arena = std.heap.ArenaAllocator.init(self.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const result = self.executeWithResult(arena.allocator(), cmd);
|
||||
|
||||
self.terminal.printAssistant(result.output);
|
||||
std.debug.print("\n", .{});
|
||||
}
|
||||
|
||||
fn execGoto(self: *Self, arena: std.mem.Allocator, raw_url: []const u8) []const u8 {
|
||||
const url = substituteEnvVars(arena, raw_url);
|
||||
return self.tool_executor.call(arena, "goto", buildJson(arena, .{ .url = url })) catch "Error: goto failed";
|
||||
}
|
||||
|
||||
fn execClick(self: *Self, arena: std.mem.Allocator, raw_target: []const u8) []const u8 {
|
||||
const target = substituteEnvVars(arena, raw_target);
|
||||
// Try as CSS selector via interactiveElements + click
|
||||
// First get interactive elements to find the target
|
||||
const elements_result = self.tool_executor.call(arena, "interactiveElements", "") catch
|
||||
return "Error: failed to get interactive elements";
|
||||
|
||||
// Try to find a backendNodeId by searching the elements result for the target text
|
||||
if (findNodeIdByText(arena, elements_result, target)) |node_id| {
|
||||
const args = std.fmt.allocPrint(arena, "{{\"backendNodeId\":{d}}}", .{node_id}) catch
|
||||
return "Error: failed to build click args";
|
||||
return self.tool_executor.call(arena, "click", args) catch "Error: click failed";
|
||||
}
|
||||
|
||||
return "Error: could not find element matching the target";
|
||||
}
|
||||
|
||||
fn execType(self: *Self, arena: std.mem.Allocator, args: Command.TypeArgs) []const u8 {
|
||||
const selector = escapeJs(arena, substituteEnvVars(arena, args.selector));
|
||||
const value = escapeJs(arena, substituteEnvVars(arena, args.value));
|
||||
|
||||
// Use JavaScript to set the value on the element matching the selector
|
||||
const script = std.fmt.allocPrint(arena,
|
||||
\\(function() {{
|
||||
\\ var el = document.querySelector("{s}");
|
||||
\\ if (!el) return "Error: element not found";
|
||||
\\ el.value = "{s}";
|
||||
\\ el.dispatchEvent(new Event("input", {{bubbles: true}}));
|
||||
\\ return "Typed into " + el.tagName;
|
||||
\\}})()
|
||||
, .{ selector, value }) catch return "Error: failed to build type script";
|
||||
|
||||
return self.tool_executor.call(arena, "evaluate", buildJson(arena, .{ .script = script })) catch "Error: type failed";
|
||||
}
|
||||
|
||||
fn execExtract(self: *Self, arena: std.mem.Allocator, args: Command.ExtractArgs) []const u8 {
|
||||
const selector = escapeJs(arena, substituteEnvVars(arena, args.selector));
|
||||
|
||||
const script = std.fmt.allocPrint(arena,
|
||||
\\JSON.stringify(Array.from(document.querySelectorAll("{s}")).map(el => el.textContent.trim()))
|
||||
, .{selector}) catch return "Error: failed to build extract script";
|
||||
|
||||
const result = self.tool_executor.call(arena, "evaluate", buildJson(arena, .{ .script = script })) catch
|
||||
return "Error: extract failed";
|
||||
|
||||
if (args.file) |raw_file| {
|
||||
const file = sanitizePath(raw_file) orelse {
|
||||
self.terminal.printError("Invalid output path: must be relative and not traverse above working directory");
|
||||
return result;
|
||||
};
|
||||
std.fs.cwd().writeFile(.{
|
||||
.sub_path = file,
|
||||
.data = result,
|
||||
}) catch {
|
||||
self.terminal.printError("Failed to write to file");
|
||||
return result;
|
||||
};
|
||||
const msg = std.fmt.allocPrint(arena, "Extracted to {s}", .{file}) catch return "Extracted.";
|
||||
return msg;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Substitute $VAR_NAME references with values from the environment.
|
||||
fn substituteEnvVars(arena: std.mem.Allocator, input: []const u8) []const u8 {
|
||||
// Quick scan: if no $ present, return as-is
|
||||
if (std.mem.indexOfScalar(u8, input, '$') == null) return input;
|
||||
|
||||
var result: std.ArrayListUnmanaged(u8) = .empty;
|
||||
var i: usize = 0;
|
||||
while (i < input.len) {
|
||||
if (input[i] == '$') {
|
||||
// Find the end of the variable name (alphanumeric + underscore)
|
||||
const var_start = i + 1;
|
||||
var var_end = var_start;
|
||||
while (var_end < input.len and (std.ascii.isAlphanumeric(input[var_end]) or input[var_end] == '_')) {
|
||||
var_end += 1;
|
||||
}
|
||||
if (var_end > var_start) {
|
||||
const var_name = input[var_start..var_end];
|
||||
// We need a null-terminated string for getenv
|
||||
const var_name_z = arena.dupeZ(u8, var_name) catch return input;
|
||||
if (std.posix.getenv(var_name_z)) |env_val| {
|
||||
result.appendSlice(arena, env_val) catch return input;
|
||||
} else {
|
||||
// Keep the original $VAR if not found
|
||||
result.appendSlice(arena, input[i..var_end]) catch return input;
|
||||
}
|
||||
i = var_end;
|
||||
} else {
|
||||
result.append(arena, '$') catch return input;
|
||||
i += 1;
|
||||
}
|
||||
} else {
|
||||
result.append(arena, input[i]) catch return input;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return result.toOwnedSlice(arena) catch input;
|
||||
}
|
||||
|
||||
/// Escape a string for safe interpolation inside a JS double-quoted string literal.
|
||||
fn escapeJs(arena: std.mem.Allocator, input: []const u8) []const u8 {
|
||||
// Quick scan: if nothing to escape, return as-is
|
||||
const needs_escape = for (input) |ch| {
|
||||
if (ch == '"' or ch == '\\' or ch == '\n' or ch == '\r' or ch == '\t') break true;
|
||||
} else false;
|
||||
if (!needs_escape) return input;
|
||||
|
||||
var out: std.ArrayListUnmanaged(u8) = .empty;
|
||||
for (input) |ch| {
|
||||
switch (ch) {
|
||||
'\\' => out.appendSlice(arena, "\\\\") catch return input,
|
||||
'"' => out.appendSlice(arena, "\\\"") catch return input,
|
||||
'\n' => out.appendSlice(arena, "\\n") catch return input,
|
||||
'\r' => out.appendSlice(arena, "\\r") catch return input,
|
||||
'\t' => out.appendSlice(arena, "\\t") catch return input,
|
||||
else => out.append(arena, ch) catch return input,
|
||||
}
|
||||
}
|
||||
return out.toOwnedSlice(arena) catch input;
|
||||
}
|
||||
|
||||
/// Validate that a file path is safe: relative, no traversal above cwd.
|
||||
fn sanitizePath(path: []const u8) ?[]const u8 {
|
||||
// Reject absolute paths
|
||||
if (path.len > 0 and path[0] == '/') return null;
|
||||
|
||||
// Reject paths containing ".." components
|
||||
var iter = std.mem.splitScalar(u8, path, '/');
|
||||
while (iter.next()) |component| {
|
||||
if (std.mem.eql(u8, component, "..")) return null;
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
fn findNodeIdByText(arena: std.mem.Allocator, elements_json: []const u8, target: []const u8) ?u32 {
|
||||
_ = arena;
|
||||
// Simple text search in the JSON result for the target text
|
||||
// Look for patterns like "backendNodeId":N near the target text
|
||||
// This is a heuristic — search for the target text, then scan backwards for backendNodeId
|
||||
var pos: usize = 0;
|
||||
while (std.mem.indexOfPos(u8, elements_json, pos, target)) |idx| {
|
||||
// Search backwards from idx for "backendNodeId":
|
||||
const search_start = if (idx > 200) idx - 200 else 0;
|
||||
const window = elements_json[search_start..idx];
|
||||
if (std.mem.lastIndexOf(u8, window, "\"backendNodeId\":")) |bid_offset| {
|
||||
const num_start = search_start + bid_offset + "\"backendNodeId\":".len;
|
||||
const num_end = std.mem.indexOfAnyPos(u8, elements_json, num_start, ",}] \n") orelse continue;
|
||||
const num_str = elements_json[num_start..num_end];
|
||||
return std.fmt.parseInt(u32, num_str, 10) catch {
|
||||
pos = idx + 1;
|
||||
continue;
|
||||
};
|
||||
}
|
||||
pos = idx + 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn buildJson(arena: std.mem.Allocator, value: anytype) []const u8 {
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
std.json.Stringify.value(value, .{}, &aw.writer) catch return "{}";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
test "escapeJs no escaping needed" {
|
||||
const result = escapeJs(std.testing.allocator, "hello world");
|
||||
try std.testing.expectEqualStrings("hello world", result);
|
||||
}
|
||||
|
||||
test "escapeJs quotes and backslashes" {
|
||||
const result = escapeJs(std.testing.allocator, "say \"hello\\world\"");
|
||||
defer std.testing.allocator.free(result);
|
||||
try std.testing.expectEqualStrings("say \\\"hello\\\\world\\\"", result);
|
||||
}
|
||||
|
||||
test "escapeJs newlines and tabs" {
|
||||
const result = escapeJs(std.testing.allocator, "line1\nline2\ttab");
|
||||
defer std.testing.allocator.free(result);
|
||||
try std.testing.expectEqualStrings("line1\\nline2\\ttab", result);
|
||||
}
|
||||
|
||||
test "escapeJs injection attempt" {
|
||||
const result = escapeJs(std.testing.allocator, "\"; alert(1); //");
|
||||
defer std.testing.allocator.free(result);
|
||||
try std.testing.expectEqualStrings("\\\"; alert(1); //", result);
|
||||
}
|
||||
|
||||
test "sanitizePath allows relative" {
|
||||
try std.testing.expectEqualStrings("output.json", sanitizePath("output.json").?);
|
||||
try std.testing.expectEqualStrings("dir/file.json", sanitizePath("dir/file.json").?);
|
||||
}
|
||||
|
||||
test "sanitizePath rejects absolute" {
|
||||
try std.testing.expect(sanitizePath("/etc/passwd") == null);
|
||||
}
|
||||
|
||||
test "sanitizePath rejects traversal" {
|
||||
try std.testing.expect(sanitizePath("../../../etc/passwd") == null);
|
||||
try std.testing.expect(sanitizePath("foo/../../bar") == null);
|
||||
}
|
||||
|
||||
test "substituteEnvVars no vars" {
|
||||
const result = substituteEnvVars(std.testing.allocator, "hello world");
|
||||
try std.testing.expectEqualStrings("hello world", result);
|
||||
}
|
||||
|
||||
test "substituteEnvVars with HOME" {
|
||||
// Use arena since substituteEnvVars makes intermediate allocations (dupeZ)
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const a = arena.allocator();
|
||||
|
||||
const result = substituteEnvVars(a, "dir=$HOME/test");
|
||||
// Result should not contain $HOME literally (it got substituted)
|
||||
try std.testing.expect(std.mem.indexOf(u8, result, "$HOME") == null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, result, "/test") != null);
|
||||
}
|
||||
|
||||
test "substituteEnvVars missing var kept literal" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const result = substituteEnvVars(arena.allocator(), "$UNLIKELY_VAR_12345");
|
||||
try std.testing.expectEqualStrings("$UNLIKELY_VAR_12345", result);
|
||||
}
|
||||
|
||||
test "substituteEnvVars bare dollar" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const result = substituteEnvVars(arena.allocator(), "price is $ 5");
|
||||
try std.testing.expectEqualStrings("price is $ 5", result);
|
||||
}
|
||||
146
src/agent/Recorder.zig
Normal file
146
src/agent/Recorder.zig
Normal file
@@ -0,0 +1,146 @@
|
||||
const std = @import("std");
|
||||
const Command = @import("Command.zig");
|
||||
|
||||
const Self = @This();
|
||||
|
||||
file: ?std.fs.File,
|
||||
|
||||
/// Commands that are read-only / ephemeral and should NOT be recorded.
|
||||
pub fn init(path: ?[]const u8) Self {
|
||||
const file: ?std.fs.File = if (path) |p|
|
||||
std.fs.cwd().createFile(p, .{ .truncate = false }) catch |err| blk: {
|
||||
std.debug.print("Warning: could not open recording file: {s}\n", .{@errorName(err)});
|
||||
break :blk null;
|
||||
}
|
||||
else
|
||||
null;
|
||||
|
||||
// Seek to end for appending
|
||||
if (file) |f| {
|
||||
f.seekFromEnd(0) catch {};
|
||||
}
|
||||
|
||||
return .{ .file = file };
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
if (self.file) |f| f.close();
|
||||
}
|
||||
|
||||
/// Record a successfully executed command line to the .panda file.
|
||||
/// Skips read-only commands (WAIT, TREE, MARKDOWN).
|
||||
pub fn record(self: *Self, line: []const u8) void {
|
||||
const f = self.file orelse return;
|
||||
|
||||
// Check if this command should be skipped
|
||||
const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (trimmed.len == 0) return;
|
||||
if (trimmed[0] == '#') return;
|
||||
|
||||
const cmd_end = std.mem.indexOfAny(u8, trimmed, &std.ascii.whitespace) orelse trimmed.len;
|
||||
const cmd_word = trimmed[0..cmd_end];
|
||||
|
||||
if (isNonRecordedCommand(cmd_word)) return;
|
||||
|
||||
f.writeAll(trimmed) catch return;
|
||||
f.writeAll("\n") catch return;
|
||||
}
|
||||
|
||||
/// Record a comment line (e.g. # INTENT: ...).
|
||||
pub fn recordComment(self: *Self, comment: []const u8) void {
|
||||
const f = self.file orelse return;
|
||||
f.writeAll(comment) catch return;
|
||||
f.writeAll("\n") catch return;
|
||||
}
|
||||
|
||||
fn isNonRecordedCommand(cmd_word: []const u8) bool {
|
||||
const non_recorded = [_][]const u8{ "WAIT", "TREE", "MARKDOWN", "MD" };
|
||||
inline for (non_recorded) |skip| {
|
||||
if (eqlIgnoreCase(cmd_word, skip)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn eqlIgnoreCase(a: []const u8, comptime upper: []const u8) bool {
|
||||
if (a.len != upper.len) return false;
|
||||
for (a, upper) |ac, uc| {
|
||||
if (std.ascii.toUpper(ac) != uc) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
test "isNonRecordedCommand" {
|
||||
try std.testing.expect(isNonRecordedCommand("WAIT"));
|
||||
try std.testing.expect(isNonRecordedCommand("wait"));
|
||||
try std.testing.expect(isNonRecordedCommand("TREE"));
|
||||
try std.testing.expect(isNonRecordedCommand("MARKDOWN"));
|
||||
try std.testing.expect(isNonRecordedCommand("MD"));
|
||||
try std.testing.expect(isNonRecordedCommand("md"));
|
||||
try std.testing.expect(!isNonRecordedCommand("GOTO"));
|
||||
try std.testing.expect(!isNonRecordedCommand("CLICK"));
|
||||
try std.testing.expect(!isNonRecordedCommand("EXTRACT"));
|
||||
}
|
||||
|
||||
test "record writes state-mutating commands" {
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
const file = tmp.dir.createFile("test.panda", .{ .read = true }) catch unreachable;
|
||||
|
||||
var recorder = Self{ .file = file };
|
||||
defer recorder.deinit();
|
||||
|
||||
recorder.record("GOTO https://example.com");
|
||||
recorder.record("CLICK \"Login\"");
|
||||
recorder.record("TREE"); // should be skipped
|
||||
recorder.record("WAIT \".dashboard\""); // should be skipped
|
||||
recorder.record("MARKDOWN"); // should be skipped
|
||||
recorder.record("EXTRACT \".title\"");
|
||||
recorder.recordComment("# INTENT: LOGIN");
|
||||
|
||||
// Read back and verify
|
||||
file.seekTo(0) catch unreachable;
|
||||
var buf: [512]u8 = undefined;
|
||||
const n = file.readAll(&buf) catch unreachable;
|
||||
const content = buf[0..n];
|
||||
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "GOTO https://example.com\n") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "CLICK \"Login\"\n") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "EXTRACT \".title\"\n") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "# INTENT: LOGIN\n") != null);
|
||||
// Verify skipped commands are NOT present
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "TREE") == null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "WAIT") == null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, content, "MARKDOWN") == null);
|
||||
}
|
||||
|
||||
test "record skips empty and comment lines" {
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
const file = tmp.dir.createFile("test2.panda", .{ .read = true }) catch unreachable;
|
||||
|
||||
var recorder = Self{ .file = file };
|
||||
defer recorder.deinit();
|
||||
|
||||
recorder.record("");
|
||||
recorder.record(" ");
|
||||
recorder.record("# this is a comment");
|
||||
recorder.record("GOTO https://example.com");
|
||||
|
||||
file.seekTo(0) catch unreachable;
|
||||
var buf: [256]u8 = undefined;
|
||||
const n = file.readAll(&buf) catch unreachable;
|
||||
const content = buf[0..n];
|
||||
|
||||
try std.testing.expectEqualStrings("GOTO https://example.com\n", content);
|
||||
}
|
||||
|
||||
test "recorder with null file is no-op" {
|
||||
var recorder = Self{ .file = null };
|
||||
recorder.record("GOTO https://example.com");
|
||||
recorder.recordComment("# test");
|
||||
recorder.deinit();
|
||||
}
|
||||
64
src/agent/Terminal.zig
Normal file
64
src/agent/Terminal.zig
Normal file
@@ -0,0 +1,64 @@
|
||||
const std = @import("std");
|
||||
const c = @cImport({
|
||||
@cInclude("linenoise.h");
|
||||
});
|
||||
|
||||
const Self = @This();
|
||||
|
||||
const ansi_reset = "\x1b[0m";
|
||||
const ansi_bold = "\x1b[1m";
|
||||
const ansi_dim = "\x1b[2m";
|
||||
const ansi_cyan = "\x1b[36m";
|
||||
const ansi_green = "\x1b[32m";
|
||||
const ansi_yellow = "\x1b[33m";
|
||||
const ansi_red = "\x1b[31m";
|
||||
|
||||
history_path: ?[:0]const u8,
|
||||
|
||||
pub fn init(history_path: ?[:0]const u8) Self {
|
||||
c.linenoiseSetMultiLine(1);
|
||||
const self = Self{ .history_path = history_path };
|
||||
if (history_path) |path| {
|
||||
_ = c.linenoiseHistoryLoad(path.ptr);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
pub fn readLine(self: *Self, prompt: [*:0]const u8) ?[]const u8 {
|
||||
const line = c.linenoise(prompt) orelse return null;
|
||||
const slice = std.mem.sliceTo(line, 0);
|
||||
if (slice.len > 0) {
|
||||
_ = c.linenoiseHistoryAdd(line);
|
||||
if (self.history_path) |path| {
|
||||
_ = c.linenoiseHistorySave(path.ptr);
|
||||
}
|
||||
}
|
||||
return slice;
|
||||
}
|
||||
|
||||
pub fn freeLine(_: *Self, line: []const u8) void {
|
||||
c.linenoiseFree(@ptrCast(@constCast(line.ptr)));
|
||||
}
|
||||
|
||||
pub fn printAssistant(_: *Self, text: []const u8) void {
|
||||
const fd = std.posix.STDOUT_FILENO;
|
||||
_ = std.posix.write(fd, text) catch {};
|
||||
}
|
||||
|
||||
pub fn printToolCall(_: *Self, name: []const u8, args: []const u8) void {
|
||||
std.debug.print("\n{s}{s}[tool: {s}]{s} {s}\n", .{ ansi_dim, ansi_cyan, name, ansi_reset, args });
|
||||
}
|
||||
|
||||
pub fn printToolResult(_: *Self, name: []const u8, result: []const u8) void {
|
||||
const truncated = if (result.len > 500) result[0..500] else result;
|
||||
const ellipsis: []const u8 = if (result.len > 500) "..." else "";
|
||||
std.debug.print("{s}{s}[result: {s}]{s} {s}{s}\n", .{ ansi_dim, ansi_green, name, ansi_reset, truncated, ellipsis });
|
||||
}
|
||||
|
||||
pub fn printError(_: *Self, msg: []const u8) void {
|
||||
std.debug.print("{s}{s}Error: {s}{s}\n", .{ ansi_bold, ansi_red, msg, ansi_reset });
|
||||
}
|
||||
|
||||
pub fn printInfo(_: *Self, msg: []const u8) void {
|
||||
std.debug.print("{s}{s}{s}\n", .{ ansi_dim, msg, ansi_reset });
|
||||
}
|
||||
446
src/agent/ToolExecutor.zig
Normal file
446
src/agent/ToolExecutor.zig
Normal file
@@ -0,0 +1,446 @@
|
||||
const std = @import("std");
|
||||
const lp = @import("lightpanda");
|
||||
const zenai = @import("zenai");
|
||||
|
||||
const App = @import("../App.zig");
|
||||
const HttpClient = @import("../browser/HttpClient.zig");
|
||||
const CDPNode = @import("../cdp/Node.zig");
|
||||
const mcp_tools = @import("../mcp/tools.zig");
|
||||
const protocol = @import("../mcp/protocol.zig");
|
||||
|
||||
const Self = @This();
|
||||
|
||||
allocator: std.mem.Allocator,
|
||||
app: *App,
|
||||
http_client: *HttpClient,
|
||||
notification: *lp.Notification,
|
||||
browser: lp.Browser,
|
||||
session: *lp.Session,
|
||||
node_registry: CDPNode.Registry,
|
||||
tool_schema_arena: std.heap.ArenaAllocator,
|
||||
|
||||
pub fn init(allocator: std.mem.Allocator, app: *App) !*Self {
|
||||
const http_client = try HttpClient.init(allocator, &app.network);
|
||||
errdefer http_client.deinit();
|
||||
|
||||
const notification = try lp.Notification.init(allocator);
|
||||
errdefer notification.deinit();
|
||||
|
||||
const self = try allocator.create(Self);
|
||||
errdefer allocator.destroy(self);
|
||||
|
||||
var browser = try lp.Browser.init(app, .{ .http_client = http_client });
|
||||
errdefer browser.deinit();
|
||||
|
||||
self.* = .{
|
||||
.allocator = allocator,
|
||||
.app = app,
|
||||
.http_client = http_client,
|
||||
.notification = notification,
|
||||
.browser = browser,
|
||||
.session = undefined,
|
||||
.node_registry = CDPNode.Registry.init(allocator),
|
||||
.tool_schema_arena = std.heap.ArenaAllocator.init(allocator),
|
||||
};
|
||||
|
||||
self.session = try self.browser.newSession(self.notification);
|
||||
return self;
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
self.tool_schema_arena.deinit();
|
||||
self.node_registry.deinit();
|
||||
self.browser.deinit();
|
||||
self.notification.deinit();
|
||||
self.http_client.deinit();
|
||||
self.allocator.destroy(self);
|
||||
}
|
||||
|
||||
/// Returns the list of tools in zenai provider.Tool format.
|
||||
pub fn getTools(self: *Self) ![]const zenai.provider.Tool {
|
||||
const arena = self.tool_schema_arena.allocator();
|
||||
const tools = try arena.alloc(zenai.provider.Tool, mcp_tools.tool_list.len);
|
||||
for (mcp_tools.tool_list, 0..) |t, i| {
|
||||
const parsed = try std.json.parseFromSliceLeaky(
|
||||
std.json.Value,
|
||||
arena,
|
||||
t.inputSchema,
|
||||
.{},
|
||||
);
|
||||
tools[i] = .{
|
||||
.name = t.name,
|
||||
.description = t.description orelse "",
|
||||
.parameters = parsed,
|
||||
};
|
||||
}
|
||||
return tools;
|
||||
}
|
||||
|
||||
/// Execute a tool by name with JSON arguments, returning the result as a string.
|
||||
pub fn call(self: *Self, arena: std.mem.Allocator, tool_name: []const u8, arguments_json: []const u8) ![]const u8 {
|
||||
const arguments = if (arguments_json.len > 0)
|
||||
(std.json.parseFromSlice(std.json.Value, arena, arguments_json, .{}) catch
|
||||
return "Error: invalid JSON arguments").value
|
||||
else
|
||||
null;
|
||||
|
||||
const Action = enum {
|
||||
goto,
|
||||
navigate,
|
||||
markdown,
|
||||
links,
|
||||
nodeDetails,
|
||||
interactiveElements,
|
||||
structuredData,
|
||||
detectForms,
|
||||
evaluate,
|
||||
eval,
|
||||
semantic_tree,
|
||||
click,
|
||||
fill,
|
||||
scroll,
|
||||
waitForSelector,
|
||||
};
|
||||
|
||||
const action_map = std.StaticStringMap(Action).initComptime(.{
|
||||
.{ "goto", .goto },
|
||||
.{ "navigate", .navigate },
|
||||
.{ "markdown", .markdown },
|
||||
.{ "links", .links },
|
||||
.{ "nodeDetails", .nodeDetails },
|
||||
.{ "interactiveElements", .interactiveElements },
|
||||
.{ "structuredData", .structuredData },
|
||||
.{ "detectForms", .detectForms },
|
||||
.{ "evaluate", .evaluate },
|
||||
.{ "eval", .eval },
|
||||
.{ "semantic_tree", .semantic_tree },
|
||||
.{ "click", .click },
|
||||
.{ "fill", .fill },
|
||||
.{ "scroll", .scroll },
|
||||
.{ "waitForSelector", .waitForSelector },
|
||||
});
|
||||
|
||||
const action = action_map.get(tool_name) orelse return "Error: unknown tool";
|
||||
|
||||
return switch (action) {
|
||||
.goto, .navigate => self.execGoto(arena, arguments),
|
||||
.markdown => self.execMarkdown(arena, arguments),
|
||||
.links => self.execLinks(arena, arguments),
|
||||
.nodeDetails => self.execNodeDetails(arena, arguments),
|
||||
.interactiveElements => self.execInteractiveElements(arena, arguments),
|
||||
.structuredData => self.execStructuredData(arena, arguments),
|
||||
.detectForms => self.execDetectForms(arena, arguments),
|
||||
.evaluate, .eval => self.execEvaluate(arena, arguments),
|
||||
.semantic_tree => self.execSemanticTree(arena, arguments),
|
||||
.click => self.execClick(arena, arguments),
|
||||
.fill => self.execFill(arena, arguments),
|
||||
.scroll => self.execScroll(arena, arguments),
|
||||
.waitForSelector => self.execWaitForSelector(arena, arguments),
|
||||
};
|
||||
}
|
||||
|
||||
fn execGoto(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const GotoParams = struct {
|
||||
url: [:0]const u8,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrErr(GotoParams, arena, arguments) orelse return "Error: missing or invalid 'url' argument";
|
||||
self.performGoto(args.url, args.timeout, args.waitUntil) catch return "Error: navigation failed";
|
||||
return "Navigated successfully.";
|
||||
}
|
||||
|
||||
fn execMarkdown(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const UrlParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(UrlParams, arena, arguments);
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
lp.markdown.dump(page.window._document.asNode(), .{}, &aw.writer, page) catch return "Error: failed to generate markdown";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execLinks(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const UrlParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(UrlParams, arena, arguments);
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
const links_list = lp.links.collectLinks(arena, page.window._document.asNode(), page) catch
|
||||
return "Error: failed to collect links";
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
for (links_list, 0..) |href, i| {
|
||||
if (i > 0) aw.writer.writeByte('\n') catch {};
|
||||
aw.writer.writeAll(href) catch {};
|
||||
}
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execNodeDetails(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const Params = struct { backendNodeId: CDPNode.Id };
|
||||
const args = parseArgsOrErr(Params, arena, arguments) orelse return "Error: missing backendNodeId";
|
||||
|
||||
_ = self.session.currentPage() orelse return "Error: page not loaded";
|
||||
|
||||
const node = self.node_registry.lookup_by_id.get(args.backendNodeId) orelse
|
||||
return "Error: node not found";
|
||||
|
||||
const page = self.session.currentPage().?;
|
||||
const details = lp.SemanticTree.getNodeDetails(arena, node.dom, &self.node_registry, page) catch
|
||||
return "Error: failed to get node details";
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
std.json.Stringify.value(&details, .{}, &aw.writer) catch return "Error: serialization failed";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execInteractiveElements(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const UrlParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(UrlParams, arena, arguments);
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
const elements = lp.interactive.collectInteractiveElements(page.window._document.asNode(), arena, page) catch
|
||||
return "Error: failed to collect interactive elements";
|
||||
lp.interactive.registerNodes(elements, &self.node_registry) catch
|
||||
return "Error: failed to register nodes";
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
std.json.Stringify.value(elements, .{}, &aw.writer) catch return "Error: serialization failed";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execStructuredData(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const UrlParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(UrlParams, arena, arguments);
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
const data = lp.structured_data.collectStructuredData(page.window._document.asNode(), arena, page) catch
|
||||
return "Error: failed to collect structured data";
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
std.json.Stringify.value(data, .{}, &aw.writer) catch return "Error: serialization failed";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execDetectForms(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const UrlParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(UrlParams, arena, arguments);
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
const forms_data = lp.forms.collectForms(arena, page.window._document.asNode(), page) catch
|
||||
return "Error: failed to collect forms";
|
||||
lp.forms.registerNodes(forms_data, &self.node_registry) catch
|
||||
return "Error: failed to register form nodes";
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
std.json.Stringify.value(forms_data, .{}, &aw.writer) catch return "Error: serialization failed";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execEvaluate(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const Params = struct {
|
||||
script: [:0]const u8,
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrErr(Params, arena, arguments) orelse return "Error: missing 'script' argument";
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
var ls: lp.js.Local.Scope = undefined;
|
||||
page.js.localScope(&ls);
|
||||
defer ls.deinit();
|
||||
|
||||
var try_catch: lp.js.TryCatch = undefined;
|
||||
try_catch.init(&ls.local);
|
||||
defer try_catch.deinit();
|
||||
|
||||
const js_result = ls.local.compileAndRun(args.script, null) catch |err| {
|
||||
const caught = try_catch.caughtOrError(arena, err);
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
caught.format(&aw.writer) catch {};
|
||||
return aw.written();
|
||||
};
|
||||
|
||||
return js_result.toStringSliceWithAlloc(arena) catch "undefined";
|
||||
}
|
||||
|
||||
fn execSemanticTree(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const TreeParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
backendNodeId: ?u32 = null,
|
||||
maxDepth: ?u32 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(TreeParams, arena, arguments);
|
||||
const page = self.ensurePage(args.url, args.timeout, args.waitUntil) catch return "Error: page not loaded";
|
||||
|
||||
var root_node = page.window._document.asNode();
|
||||
if (args.backendNodeId) |node_id| {
|
||||
if (self.node_registry.lookup_by_id.get(node_id)) |n| {
|
||||
root_node = n.dom;
|
||||
}
|
||||
}
|
||||
|
||||
const st = lp.SemanticTree{
|
||||
.dom_node = root_node,
|
||||
.registry = &self.node_registry,
|
||||
.page = page,
|
||||
.arena = arena,
|
||||
.prune = true,
|
||||
.max_depth = args.maxDepth orelse std.math.maxInt(u32) - 1,
|
||||
};
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
st.textStringify(&aw.writer) catch return "Error: failed to generate semantic tree";
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
fn execClick(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const Params = struct { backendNodeId: CDPNode.Id };
|
||||
const args = parseArgsOrErr(Params, arena, arguments) orelse return "Error: missing backendNodeId";
|
||||
|
||||
const page = self.session.currentPage() orelse return "Error: page not loaded";
|
||||
const node = self.node_registry.lookup_by_id.get(args.backendNodeId) orelse return "Error: node not found";
|
||||
|
||||
lp.actions.click(node.dom, page) catch |err| {
|
||||
if (err == error.InvalidNodeType) return "Error: node is not an HTML element";
|
||||
return "Error: failed to click element";
|
||||
};
|
||||
|
||||
const page_title = page.getTitle() catch null;
|
||||
return std.fmt.allocPrint(arena, "Clicked element (backendNodeId: {d}). Page url: {s}, title: {s}", .{
|
||||
args.backendNodeId,
|
||||
page.url,
|
||||
page_title orelse "(none)",
|
||||
}) catch "Clicked element.";
|
||||
}
|
||||
|
||||
fn execFill(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const Params = struct {
|
||||
backendNodeId: CDPNode.Id,
|
||||
text: []const u8,
|
||||
};
|
||||
const args = parseArgsOrErr(Params, arena, arguments) orelse return "Error: missing backendNodeId or text";
|
||||
|
||||
const page = self.session.currentPage() orelse return "Error: page not loaded";
|
||||
const node = self.node_registry.lookup_by_id.get(args.backendNodeId) orelse return "Error: node not found";
|
||||
|
||||
lp.actions.fill(node.dom, args.text, page) catch |err| {
|
||||
if (err == error.InvalidNodeType) return "Error: node is not an input, textarea or select";
|
||||
return "Error: failed to fill element";
|
||||
};
|
||||
|
||||
const page_title = page.getTitle() catch null;
|
||||
return std.fmt.allocPrint(arena, "Filled element (backendNodeId: {d}) with \"{s}\". Page url: {s}, title: {s}", .{
|
||||
args.backendNodeId,
|
||||
args.text,
|
||||
page.url,
|
||||
page_title orelse "(none)",
|
||||
}) catch "Filled element.";
|
||||
}
|
||||
|
||||
fn execScroll(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const Params = struct {
|
||||
backendNodeId: ?CDPNode.Id = null,
|
||||
x: ?i32 = null,
|
||||
y: ?i32 = null,
|
||||
};
|
||||
const args = parseArgsOrDefault(Params, arena, arguments);
|
||||
const page = self.session.currentPage() orelse return "Error: page not loaded";
|
||||
|
||||
var target_node: ?*@import("../browser/webapi/Node.zig") = null;
|
||||
if (args.backendNodeId) |node_id| {
|
||||
const node = self.node_registry.lookup_by_id.get(node_id) orelse return "Error: node not found";
|
||||
target_node = node.dom;
|
||||
}
|
||||
|
||||
lp.actions.scroll(target_node, args.x, args.y, page) catch |err| {
|
||||
if (err == error.InvalidNodeType) return "Error: node is not an element";
|
||||
return "Error: failed to scroll";
|
||||
};
|
||||
|
||||
const page_title = page.getTitle() catch null;
|
||||
return std.fmt.allocPrint(arena, "Scrolled to x: {d}, y: {d}. Page url: {s}, title: {s}", .{
|
||||
args.x orelse 0,
|
||||
args.y orelse 0,
|
||||
page.url,
|
||||
page_title orelse "(none)",
|
||||
}) catch "Scrolled.";
|
||||
}
|
||||
|
||||
fn execWaitForSelector(self: *Self, arena: std.mem.Allocator, arguments: ?std.json.Value) []const u8 {
|
||||
const Params = struct {
|
||||
selector: [:0]const u8,
|
||||
timeout: ?u32 = null,
|
||||
};
|
||||
const args = parseArgsOrErr(Params, arena, arguments) orelse return "Error: missing 'selector' argument";
|
||||
|
||||
_ = self.session.currentPage() orelse return "Error: page not loaded";
|
||||
|
||||
const timeout_ms = args.timeout orelse 5000;
|
||||
|
||||
const node = lp.actions.waitForSelector(args.selector, timeout_ms, self.session) catch |err| {
|
||||
if (err == error.InvalidSelector) return "Error: invalid selector";
|
||||
if (err == error.Timeout) return "Error: timeout waiting for selector";
|
||||
return "Error: failed waiting for selector";
|
||||
};
|
||||
|
||||
const registered = self.node_registry.register(node) catch return "Element found.";
|
||||
return std.fmt.allocPrint(arena, "Element found. backendNodeId: {d}", .{registered.id}) catch "Element found.";
|
||||
}
|
||||
|
||||
fn ensurePage(self: *Self, url: ?[:0]const u8, timeout: ?u32, waitUntil: ?lp.Config.WaitUntil) !*lp.Page {
|
||||
if (url) |u| {
|
||||
try self.performGoto(u, timeout, waitUntil);
|
||||
}
|
||||
return self.session.currentPage() orelse error.PageNotLoaded;
|
||||
}
|
||||
|
||||
fn performGoto(self: *Self, url: [:0]const u8, timeout: ?u32, waitUntil: ?lp.Config.WaitUntil) !void {
|
||||
const session = self.session;
|
||||
if (session.page != null) {
|
||||
session.removePage();
|
||||
}
|
||||
const page = try session.createPage();
|
||||
_ = try page.navigate(url, .{
|
||||
.reason = .address_bar,
|
||||
.kind = .{ .push = null },
|
||||
});
|
||||
|
||||
var runner = try session.runner(.{});
|
||||
try runner.wait(.{
|
||||
.ms = timeout orelse 10000,
|
||||
.until = waitUntil orelse .done,
|
||||
});
|
||||
}
|
||||
|
||||
fn parseArgsOrDefault(comptime T: type, arena: std.mem.Allocator, arguments: ?std.json.Value) T {
|
||||
const args_raw = arguments orelse return .{};
|
||||
return std.json.parseFromValueLeaky(T, arena, args_raw, .{ .ignore_unknown_fields = true }) catch .{};
|
||||
}
|
||||
|
||||
fn parseArgsOrErr(comptime T: type, arena: std.mem.Allocator, arguments: ?std.json.Value) ?T {
|
||||
const args_raw = arguments orelse return null;
|
||||
return std.json.parseFromValueLeaky(T, arena, args_raw, .{ .ignore_unknown_fields = true }) catch null;
|
||||
}
|
||||
@@ -32,6 +32,9 @@ const CookieJar = @import("webapi/storage/Cookie.zig").Jar;
|
||||
const http = @import("../network/http.zig");
|
||||
const Network = @import("../network/Network.zig");
|
||||
const Robots = @import("../network/Robots.zig");
|
||||
const Cache = @import("../network/cache/Cache.zig");
|
||||
const CacheMetadata = Cache.CachedMetadata;
|
||||
const CachedResponse = Cache.CachedResponse;
|
||||
|
||||
const IS_DEBUG = builtin.mode == .Debug;
|
||||
|
||||
@@ -311,7 +314,73 @@ pub fn request(self: *Client, req: Request) !void {
|
||||
return self.fetchRobotsThenProcessRequest(robots_url, req);
|
||||
}
|
||||
|
||||
fn serveFromCache(req: Request, cached: *const CachedResponse) !void {
|
||||
const response = Response.fromCached(req.ctx, cached);
|
||||
defer switch (cached.data) {
|
||||
.buffer => |_| {},
|
||||
.file => |f| f.file.close(),
|
||||
};
|
||||
|
||||
if (req.start_callback) |cb| {
|
||||
try cb(response);
|
||||
}
|
||||
|
||||
const proceed = try req.header_callback(response);
|
||||
if (!proceed) {
|
||||
req.error_callback(req.ctx, error.Abort);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (cached.data) {
|
||||
.buffer => |data| {
|
||||
if (data.len > 0) {
|
||||
try req.data_callback(response, data);
|
||||
}
|
||||
},
|
||||
.file => |f| {
|
||||
const file = f.file;
|
||||
|
||||
var buf: [1024]u8 = undefined;
|
||||
var file_reader = file.reader(&buf);
|
||||
try file_reader.seekTo(f.offset);
|
||||
const reader = &file_reader.interface;
|
||||
|
||||
var read_buf: [1024]u8 = undefined;
|
||||
var remaining = f.len;
|
||||
|
||||
while (remaining > 0) {
|
||||
const read_len = @min(read_buf.len, remaining);
|
||||
const n = try reader.readSliceShort(read_buf[0..read_len]);
|
||||
if (n == 0) break;
|
||||
remaining -= n;
|
||||
try req.data_callback(response, read_buf[0..n]);
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
try req.done_callback(req.ctx);
|
||||
}
|
||||
|
||||
fn processRequest(self: *Client, req: Request) !void {
|
||||
if (self.network.cache) |*cache| {
|
||||
if (req.method == .GET) {
|
||||
const arena = try self.network.app.arena_pool.acquire(.{ .debug = "HttpClient.processRequest.cache" });
|
||||
defer self.network.app.arena_pool.release(arena);
|
||||
|
||||
var iter = req.headers.iterator();
|
||||
const req_header_list = try iter.collect(arena);
|
||||
|
||||
if (cache.get(arena, .{
|
||||
.url = req.url,
|
||||
.timestamp = std.time.timestamp(),
|
||||
.request_headers = req_header_list.items,
|
||||
})) |cached| {
|
||||
defer req.headers.deinit();
|
||||
return serveFromCache(req, &cached);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const transfer = try self.makeTransfer(req);
|
||||
|
||||
transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
|
||||
@@ -399,8 +468,10 @@ fn fetchRobotsThenProcessRequest(self: *Client, robots_url: [:0]const u8, req: R
|
||||
try entry.value_ptr.append(self.allocator, req);
|
||||
}
|
||||
|
||||
fn robotsHeaderCallback(transfer: *Transfer) !bool {
|
||||
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn robotsHeaderCallback(response: Response) !bool {
|
||||
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx));
|
||||
// Robots callbacks only happen on real live requests.
|
||||
const transfer = response.inner.transfer;
|
||||
|
||||
if (transfer.response_header) |hdr| {
|
||||
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = ctx.robots_url });
|
||||
@@ -414,8 +485,8 @@ fn robotsHeaderCallback(transfer: *Transfer) !bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
fn robotsDataCallback(transfer: *Transfer, data: []const u8) !void {
|
||||
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn robotsDataCallback(response: Response, data: []const u8) !void {
|
||||
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx));
|
||||
try ctx.buffer.appendSlice(ctx.client.allocator, data);
|
||||
}
|
||||
|
||||
@@ -634,13 +705,43 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer {
|
||||
.id = id,
|
||||
.url = req.url,
|
||||
.req = req,
|
||||
.ctx = req.ctx,
|
||||
.client = self,
|
||||
.max_response_size = self.network.config.httpMaxResponseSize(),
|
||||
};
|
||||
return transfer;
|
||||
}
|
||||
|
||||
fn requestFailed(transfer: *Transfer, err: anyerror, comptime execute_callback: bool) void {
|
||||
if (transfer._notified_fail) {
|
||||
// we can force a failed request within a callback, which will eventually
|
||||
// result in this being called again in the more general loop. We do this
|
||||
// because we can raise a more specific error inside a callback in some cases
|
||||
return;
|
||||
}
|
||||
|
||||
transfer._notified_fail = true;
|
||||
|
||||
transfer.req.notification.dispatch(.http_request_fail, &.{
|
||||
.transfer = transfer,
|
||||
.err = err,
|
||||
});
|
||||
|
||||
if (execute_callback) {
|
||||
transfer.req.error_callback(transfer.req.ctx, err);
|
||||
} else if (transfer.req.shutdown_callback) |cb| {
|
||||
cb(transfer.req.ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// Same restriction as changeProxy. Should be ok since this is only called on
|
||||
// BrowserContext deinit.
|
||||
pub fn restoreOriginalProxy(self: *Client) !void {
|
||||
try self.ensureNoActiveConnection();
|
||||
|
||||
self.http_proxy = self.network.config.httpProxy();
|
||||
self.use_proxy = self.http_proxy != null;
|
||||
}
|
||||
|
||||
fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void {
|
||||
{
|
||||
// Reset per-response state for retries (auth challenge, queue).
|
||||
@@ -674,7 +775,7 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer
|
||||
self.active += 1;
|
||||
|
||||
if (transfer.req.start_callback) |cb| {
|
||||
cb(transfer) catch |err| {
|
||||
cb(Response.fromTransfer(transfer)) catch |err| {
|
||||
transfer.deinit();
|
||||
return err;
|
||||
};
|
||||
@@ -742,7 +843,10 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
|
||||
// TODO give a way to configure the number of auth retries.
|
||||
if (transfer._auth_challenge != null and transfer._tries < 10) {
|
||||
var wait_for_interception = false;
|
||||
transfer.req.notification.dispatch(.http_request_auth_required, &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception });
|
||||
transfer.req.notification.dispatch(
|
||||
.http_request_auth_required,
|
||||
&.{ .transfer = transfer, .wait_for_interception = &wait_for_interception },
|
||||
);
|
||||
if (wait_for_interception) {
|
||||
self.intercepted += 1;
|
||||
if (comptime IS_DEBUG) {
|
||||
@@ -841,10 +945,11 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
|
||||
}
|
||||
}
|
||||
|
||||
const body = transfer._stream_buffer.items;
|
||||
|
||||
// Replay buffered body through user's data_callback.
|
||||
if (transfer._stream_buffer.items.len > 0) {
|
||||
const body = transfer._stream_buffer.items;
|
||||
try transfer.req.data_callback(transfer, body);
|
||||
try transfer.req.data_callback(Response.fromTransfer(transfer), body);
|
||||
|
||||
transfer.req.notification.dispatch(.http_response_data, &.{
|
||||
.data = body,
|
||||
@@ -857,11 +962,19 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
|
||||
}
|
||||
}
|
||||
|
||||
if (transfer._pending_cache_metadata) |metadata| {
|
||||
const cache = &self.network.cache.?;
|
||||
cache.put(metadata.*, body) catch |err| {
|
||||
log.warn(.cache, "cache put failed", .{ .err = err });
|
||||
};
|
||||
}
|
||||
|
||||
// release conn ASAP so that it's available; some done_callbacks
|
||||
// will load more resources.
|
||||
transfer.releaseConn();
|
||||
|
||||
try transfer.req.done_callback(transfer.ctx);
|
||||
try transfer.req.done_callback(transfer.req.ctx);
|
||||
|
||||
transfer.req.notification.dispatch(.http_request_done, &.{
|
||||
.transfer = transfer,
|
||||
});
|
||||
@@ -939,9 +1052,9 @@ pub const Request = struct {
|
||||
// arbitrary data that can be associated with this request
|
||||
ctx: *anyopaque = undefined,
|
||||
|
||||
start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null,
|
||||
header_callback: *const fn (transfer: *Transfer) anyerror!bool,
|
||||
data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void,
|
||||
start_callback: ?*const fn (response: Response) anyerror!void = null,
|
||||
header_callback: *const fn (response: Response) anyerror!bool,
|
||||
data_callback: *const fn (response: Response, data: []const u8) anyerror!void,
|
||||
done_callback: *const fn (ctx: *anyopaque) anyerror!void,
|
||||
error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
|
||||
shutdown_callback: ?*const fn (ctx: *anyopaque) void = null,
|
||||
@@ -967,16 +1080,91 @@ pub const Request = struct {
|
||||
};
|
||||
};
|
||||
|
||||
pub const Response = struct {
|
||||
ctx: *anyopaque,
|
||||
inner: union(enum) {
|
||||
transfer: *Transfer,
|
||||
cached: *const CachedResponse,
|
||||
},
|
||||
|
||||
pub fn fromTransfer(transfer: *Transfer) Response {
|
||||
return .{ .ctx = transfer.req.ctx, .inner = .{ .transfer = transfer } };
|
||||
}
|
||||
|
||||
pub fn fromCached(ctx: *anyopaque, resp: *const CachedResponse) Response {
|
||||
return .{ .ctx = ctx, .inner = .{ .cached = resp } };
|
||||
}
|
||||
|
||||
pub fn status(self: Response) ?u16 {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| if (t.response_header) |rh| rh.status else null,
|
||||
.cached => |c| c.metadata.status,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn contentType(self: Response) ?[]const u8 {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| if (t.response_header) |*rh| rh.contentType() else null,
|
||||
.cached => |c| c.metadata.content_type,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn contentLength(self: Response) ?u32 {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| t.getContentLength(),
|
||||
.cached => |c| switch (c.data) {
|
||||
.buffer => |buf| @intCast(buf.len),
|
||||
.file => |f| @intCast(f.len),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
pub fn redirectCount(self: Response) ?u32 {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| if (t.response_header) |rh| rh.redirect_count else null,
|
||||
.cached => 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn url(self: Response) [:0]const u8 {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| t.url,
|
||||
.cached => |c| c.metadata.url,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn headerIterator(self: Response) HeaderIterator {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| t.responseHeaderIterator(),
|
||||
.cached => |c| HeaderIterator{ .list = .{ .list = c.metadata.headers } },
|
||||
};
|
||||
}
|
||||
|
||||
pub fn abort(self: Response, err: anyerror) void {
|
||||
switch (self.inner) {
|
||||
.transfer => |t| t.abort(err),
|
||||
.cached => {},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn format(self: Response, writer: *std.Io.Writer) !void {
|
||||
return switch (self.inner) {
|
||||
.transfer => |t| try t.format(writer),
|
||||
.cached => |c| try c.format(writer),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
pub const Transfer = struct {
|
||||
arena: ArenaAllocator,
|
||||
id: u32 = 0,
|
||||
req: Request,
|
||||
url: [:0]const u8,
|
||||
ctx: *anyopaque, // copied from req.ctx to make it easier for callback handlers
|
||||
client: *Client,
|
||||
// total bytes received in the response, including the response status line,
|
||||
// the headers, and the [encoded] body.
|
||||
bytes_received: usize = 0,
|
||||
_pending_cache_metadata: ?*CacheMetadata = null,
|
||||
|
||||
aborted: bool = false,
|
||||
|
||||
@@ -1065,7 +1253,7 @@ pub const Transfer = struct {
|
||||
// as abort (doesn't send a notification, doesn't invoke an error callback)
|
||||
fn kill(self: *Transfer) void {
|
||||
if (self.req.shutdown_callback) |cb| {
|
||||
cb(self.ctx);
|
||||
cb(self.req.ctx);
|
||||
}
|
||||
|
||||
if (self._performing or self.client.performing) {
|
||||
@@ -1101,9 +1289,9 @@ pub const Transfer = struct {
|
||||
});
|
||||
|
||||
if (execute_callback) {
|
||||
self.req.error_callback(self.ctx, err);
|
||||
self.req.error_callback(self.req.ctx, err);
|
||||
} else if (self.req.shutdown_callback) |cb| {
|
||||
cb(self.ctx);
|
||||
cb(self.req.ctx);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1352,11 +1540,61 @@ pub const Transfer = struct {
|
||||
.transfer = transfer,
|
||||
});
|
||||
|
||||
const proceed = transfer.req.header_callback(transfer) catch |err| {
|
||||
const proceed = transfer.req.header_callback(Response.fromTransfer(transfer)) catch |err| {
|
||||
log.err(.http, "header_callback", .{ .err = err, .req = transfer });
|
||||
return err;
|
||||
};
|
||||
|
||||
if (transfer.client.network.cache != null and transfer.req.method == .GET) {
|
||||
const rh = &transfer.response_header.?;
|
||||
const allocator = transfer.arena.allocator();
|
||||
|
||||
const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null;
|
||||
|
||||
const maybe_cm = try Cache.tryCache(
|
||||
allocator,
|
||||
std.time.timestamp(),
|
||||
transfer.url,
|
||||
rh.status,
|
||||
rh.contentType(),
|
||||
if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
|
||||
vary,
|
||||
if (conn.getResponseHeader("age", 0)) |h| h.value else null,
|
||||
conn.getResponseHeader("set-cookie", 0) != null,
|
||||
conn.getResponseHeader("authorization", 0) != null,
|
||||
);
|
||||
|
||||
if (maybe_cm) |cm| {
|
||||
var iter = transfer.responseHeaderIterator();
|
||||
var header_list = try iter.collect(allocator);
|
||||
const end_of_response = header_list.items.len;
|
||||
|
||||
if (vary) |vary_str| {
|
||||
var req_it = transfer.req.headers.iterator();
|
||||
|
||||
while (req_it.next()) |hdr| {
|
||||
var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
|
||||
|
||||
while (vary_iter.next()) |part| {
|
||||
const name = std.mem.trim(u8, part, &std.ascii.whitespace);
|
||||
if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
|
||||
try header_list.append(allocator, .{
|
||||
.name = try allocator.dupe(u8, hdr.name),
|
||||
.value = try allocator.dupe(u8, hdr.value),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = try transfer.arena.allocator().create(CacheMetadata);
|
||||
metadata.* = cm;
|
||||
metadata.headers = header_list.items[0..end_of_response];
|
||||
metadata.vary_headers = header_list.items[end_of_response..];
|
||||
transfer._pending_cache_metadata = metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return proceed and transfer.aborted == false;
|
||||
}
|
||||
|
||||
@@ -1455,7 +1693,7 @@ pub const Transfer = struct {
|
||||
fn _fulfill(transfer: *Transfer, status: u16, headers: []const http.Header, body: ?[]const u8) !void {
|
||||
const req = &transfer.req;
|
||||
if (req.start_callback) |cb| {
|
||||
try cb(transfer);
|
||||
try cb(Response.fromTransfer(transfer));
|
||||
}
|
||||
|
||||
transfer.response_header = .{
|
||||
@@ -1474,13 +1712,13 @@ pub const Transfer = struct {
|
||||
}
|
||||
|
||||
lp.assert(transfer._header_done_called == false, "Transfer.fulfill header_done_called", .{});
|
||||
if (try req.header_callback(transfer) == false) {
|
||||
if (try req.header_callback(Response.fromTransfer(transfer)) == false) {
|
||||
transfer.abort(error.Abort);
|
||||
return;
|
||||
}
|
||||
|
||||
if (body) |b| {
|
||||
try req.data_callback(transfer, b);
|
||||
try req.data_callback(Response.fromTransfer(transfer), b);
|
||||
}
|
||||
|
||||
try req.done_callback(req.ctx);
|
||||
@@ -1517,10 +1755,10 @@ pub const Transfer = struct {
|
||||
};
|
||||
|
||||
const Noop = struct {
|
||||
fn headerCallback(_: *Transfer) !bool {
|
||||
fn headerCallback(_: Response) !bool {
|
||||
return true;
|
||||
}
|
||||
fn dataCallback(_: *Transfer, _: []const u8) !void {}
|
||||
fn dataCallback(_: Response, _: []const u8) !void {}
|
||||
fn doneCallback(_: *anyopaque) !void {}
|
||||
fn errorCallback(_: *anyopaque, _: anyerror) void {}
|
||||
};
|
||||
|
||||
@@ -27,6 +27,9 @@ charset: [41]u8 = default_charset,
|
||||
charset_len: usize = default_charset_len,
|
||||
is_default_charset: bool = true,
|
||||
|
||||
type_buf: [127]u8 = @splat(0),
|
||||
sub_type_buf: [127]u8 = @splat(0),
|
||||
|
||||
/// String "UTF-8" continued by null characters.
|
||||
const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36;
|
||||
const default_charset_len = 5;
|
||||
@@ -61,7 +64,10 @@ pub const ContentType = union(ContentTypeEnum) {
|
||||
image_webp: void,
|
||||
application_json: void,
|
||||
unknown: void,
|
||||
other: struct { type: []const u8, sub_type: []const u8 },
|
||||
other: struct {
|
||||
type: []const u8,
|
||||
sub_type: []const u8,
|
||||
},
|
||||
};
|
||||
|
||||
pub fn contentTypeString(mime: *const Mime) []const u8 {
|
||||
@@ -112,17 +118,18 @@ fn parseCharset(value: []const u8) error{ CharsetTooBig, Invalid }![]const u8 {
|
||||
return value;
|
||||
}
|
||||
|
||||
pub fn parse(input: []u8) !Mime {
|
||||
pub fn parse(input: []const u8) !Mime {
|
||||
if (input.len > 255) {
|
||||
return error.TooBig;
|
||||
}
|
||||
|
||||
// Zig's trim API is broken. The return type is always `[]const u8`,
|
||||
// even if the input type is `[]u8`. @constCast is safe here.
|
||||
var normalized = @constCast(std.mem.trim(u8, input, &std.ascii.whitespace));
|
||||
var buf: [255]u8 = undefined;
|
||||
const normalized = std.ascii.lowerString(&buf, std.mem.trim(u8, input, &std.ascii.whitespace));
|
||||
_ = std.ascii.lowerString(normalized, normalized);
|
||||
|
||||
const content_type, const type_len = try parseContentType(normalized);
|
||||
var mime = Mime{ .content_type = undefined };
|
||||
|
||||
const content_type, const type_len = try parseContentType(normalized, &mime.type_buf, &mime.sub_type_buf);
|
||||
if (type_len >= normalized.len) {
|
||||
return .{ .content_type = content_type };
|
||||
}
|
||||
@@ -163,13 +170,12 @@ pub fn parse(input: []u8) !Mime {
|
||||
}
|
||||
}
|
||||
|
||||
return .{
|
||||
.params = params,
|
||||
.charset = charset,
|
||||
.charset_len = charset_len,
|
||||
.content_type = content_type,
|
||||
.is_default_charset = !has_explicit_charset,
|
||||
};
|
||||
mime.params = params;
|
||||
mime.charset = charset;
|
||||
mime.charset_len = charset_len;
|
||||
mime.content_type = content_type;
|
||||
mime.is_default_charset = !has_explicit_charset;
|
||||
return mime;
|
||||
}
|
||||
|
||||
/// Prescan the first 1024 bytes of an HTML document for a charset declaration.
|
||||
@@ -395,7 +401,7 @@ pub fn isText(mime: *const Mime) bool {
|
||||
}
|
||||
|
||||
// we expect value to be lowercase
|
||||
fn parseContentType(value: []const u8) !struct { ContentType, usize } {
|
||||
fn parseContentType(value: []const u8, type_buf: []u8, sub_type_buf: []u8) !struct { ContentType, usize } {
|
||||
const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len;
|
||||
const type_name = trimRight(value[0..end]);
|
||||
const attribute_start = end + 1;
|
||||
@@ -444,10 +450,18 @@ fn parseContentType(value: []const u8) !struct { ContentType, usize } {
|
||||
return error.Invalid;
|
||||
}
|
||||
|
||||
return .{ .{ .other = .{
|
||||
.type = main_type,
|
||||
.sub_type = sub_type,
|
||||
} }, attribute_start };
|
||||
@memcpy(type_buf[0..main_type.len], main_type);
|
||||
@memcpy(sub_type_buf[0..sub_type.len], sub_type);
|
||||
|
||||
return .{
|
||||
.{
|
||||
.other = .{
|
||||
.type = type_buf[0..main_type.len],
|
||||
.sub_type = sub_type_buf[0..sub_type.len],
|
||||
},
|
||||
},
|
||||
attribute_start,
|
||||
};
|
||||
}
|
||||
|
||||
const VALID_CODEPOINTS = blk: {
|
||||
@@ -461,6 +475,13 @@ const VALID_CODEPOINTS = blk: {
|
||||
break :blk v;
|
||||
};
|
||||
|
||||
pub fn typeString(self: *const Mime) []const u8 {
|
||||
return switch (self.content_type) {
|
||||
.other => |o| o.type[0..o.type_len],
|
||||
else => "",
|
||||
};
|
||||
}
|
||||
|
||||
fn validType(value: []const u8) bool {
|
||||
for (value) |b| {
|
||||
if (VALID_CODEPOINTS[b] == false) {
|
||||
|
||||
@@ -886,12 +886,10 @@ fn notifyParentLoadComplete(self: *Page) void {
|
||||
parent.iframeCompletedLoading(self.iframe.?);
|
||||
}
|
||||
|
||||
fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
var self: *Page = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn pageHeaderDoneCallback(response: HttpClient.Response) !bool {
|
||||
var self: *Page = @ptrCast(@alignCast(response.ctx));
|
||||
|
||||
const header = &transfer.response_header.?;
|
||||
|
||||
const response_url = std.mem.span(header.url);
|
||||
const response_url = response.url();
|
||||
if (std.mem.eql(u8, response_url, self.url) == false) {
|
||||
// would be different than self.url in the case of a redirect
|
||||
self.url = try self.arena.dupeZ(u8, response_url);
|
||||
@@ -905,8 +903,8 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
if (comptime IS_DEBUG) {
|
||||
log.debug(.page, "navigate header", .{
|
||||
.url = self.url,
|
||||
.status = header.status,
|
||||
.content_type = header.contentType(),
|
||||
.status = response.status(),
|
||||
.content_type = response.contentType(),
|
||||
.type = self._type,
|
||||
});
|
||||
}
|
||||
@@ -927,14 +925,14 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
||||
var self: *Page = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn pageDataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||
var self: *Page = @ptrCast(@alignCast(response.ctx));
|
||||
|
||||
if (self._parse_state == .pre) {
|
||||
// we lazily do this, because we might need the first chunk of data
|
||||
// to sniff the content type
|
||||
var mime: Mime = blk: {
|
||||
if (transfer.response_header.?.contentType()) |ct| {
|
||||
if (response.contentType()) |ct| {
|
||||
break :blk try Mime.parse(ct);
|
||||
}
|
||||
break :blk Mime.sniff(data);
|
||||
|
||||
@@ -273,25 +273,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
|
||||
// Let the outer errdefer handle releasing the arena if client.request fails
|
||||
}
|
||||
|
||||
try self.client.request(.{
|
||||
.url = url,
|
||||
.ctx = script,
|
||||
.method = .GET,
|
||||
.frame_id = page._frame_id,
|
||||
.headers = try self.getHeaders(),
|
||||
.blocking = is_blocking,
|
||||
.cookie_jar = &page._session.cookie_jar,
|
||||
.cookie_origin = page.url,
|
||||
.resource_type = .script,
|
||||
.notification = page._session.notification,
|
||||
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
|
||||
.header_callback = Script.headerCallback,
|
||||
.data_callback = Script.dataCallback,
|
||||
.done_callback = Script.doneCallback,
|
||||
.error_callback = Script.errorCallback,
|
||||
});
|
||||
handover = true;
|
||||
|
||||
if (comptime IS_DEBUG) {
|
||||
var ls: js.Local.Scope = undefined;
|
||||
page.js.localScope(&ls);
|
||||
@@ -304,6 +285,32 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
|
||||
.stack = ls.local.stackTrace() catch "???",
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
const was_evaluating = self.is_evaluating;
|
||||
self.is_evaluating = true;
|
||||
defer self.is_evaluating = was_evaluating;
|
||||
|
||||
try self.client.request(.{
|
||||
.url = url,
|
||||
.ctx = script,
|
||||
.method = .GET,
|
||||
.frame_id = page._frame_id,
|
||||
.headers = try self.getHeaders(),
|
||||
.blocking = is_blocking,
|
||||
.cookie_jar = &page._session.cookie_jar,
|
||||
.cookie_origin = page.url,
|
||||
.resource_type = .script,
|
||||
.notification = page._session.notification,
|
||||
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
|
||||
.header_callback = Script.headerCallback,
|
||||
.data_callback = Script.dataCallback,
|
||||
.done_callback = Script.doneCallback,
|
||||
.error_callback = Script.errorCallback,
|
||||
});
|
||||
}
|
||||
|
||||
handover = true;
|
||||
}
|
||||
|
||||
if (is_blocking == false) {
|
||||
@@ -694,82 +701,86 @@ pub const Script = struct {
|
||||
self.manager.page.releaseArena(self.arena);
|
||||
}
|
||||
|
||||
fn startCallback(transfer: *HttpClient.Transfer) !void {
|
||||
log.debug(.http, "script fetch start", .{ .req = transfer });
|
||||
fn startCallback(response: HttpClient.Response) !void {
|
||||
log.debug(.http, "script fetch start", .{ .req = response });
|
||||
}
|
||||
|
||||
fn headerCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
const self: *Script = @ptrCast(@alignCast(transfer.ctx));
|
||||
const header = &transfer.response_header.?;
|
||||
self.status = header.status;
|
||||
if (header.status != 200) {
|
||||
fn headerCallback(response: HttpClient.Response) !bool {
|
||||
const self: *Script = @ptrCast(@alignCast(response.ctx));
|
||||
|
||||
self.status = response.status().?;
|
||||
if (response.status() != 200) {
|
||||
log.info(.http, "script header", .{
|
||||
.req = transfer,
|
||||
.status = header.status,
|
||||
.content_type = header.contentType(),
|
||||
.req = response,
|
||||
.status = response.status(),
|
||||
.content_type = response.contentType(),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
if (comptime IS_DEBUG) {
|
||||
log.debug(.http, "script header", .{
|
||||
.req = transfer,
|
||||
.status = header.status,
|
||||
.content_type = header.contentType(),
|
||||
.req = response,
|
||||
.status = response.status(),
|
||||
.content_type = response.contentType(),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
// temp debug, trying to figure out why the next assert sometimes
|
||||
// fails. Is the buffer just corrupt or is headerCallback really
|
||||
// being called twice?
|
||||
lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{
|
||||
.m = @tagName(std.meta.activeTag(self.mode)),
|
||||
.a1 = self.debug_transfer_id,
|
||||
.a2 = self.debug_transfer_tries,
|
||||
.a3 = self.debug_transfer_aborted,
|
||||
.a4 = self.debug_transfer_bytes_received,
|
||||
.a5 = self.debug_transfer_notified_fail,
|
||||
.a7 = self.debug_transfer_intercept_state,
|
||||
.a8 = self.debug_transfer_auth_challenge,
|
||||
.a9 = self.debug_transfer_easy_id,
|
||||
.b1 = transfer.id,
|
||||
.b2 = transfer._tries,
|
||||
.b3 = transfer.aborted,
|
||||
.b4 = transfer.bytes_received,
|
||||
.b5 = transfer._notified_fail,
|
||||
.b7 = @intFromEnum(transfer._intercept_state),
|
||||
.b8 = transfer._auth_challenge != null,
|
||||
.b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
|
||||
});
|
||||
self.header_callback_called = true;
|
||||
self.debug_transfer_id = transfer.id;
|
||||
self.debug_transfer_tries = transfer._tries;
|
||||
self.debug_transfer_aborted = transfer.aborted;
|
||||
self.debug_transfer_bytes_received = transfer.bytes_received;
|
||||
self.debug_transfer_notified_fail = transfer._notified_fail;
|
||||
self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
|
||||
self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
|
||||
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
|
||||
switch (response.inner) {
|
||||
.transfer => |transfer| {
|
||||
// temp debug, trying to figure out why the next assert sometimes
|
||||
// fails. Is the buffer just corrupt or is headerCallback really
|
||||
// being called twice?
|
||||
lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{
|
||||
.m = @tagName(std.meta.activeTag(self.mode)),
|
||||
.a1 = self.debug_transfer_id,
|
||||
.a2 = self.debug_transfer_tries,
|
||||
.a3 = self.debug_transfer_aborted,
|
||||
.a4 = self.debug_transfer_bytes_received,
|
||||
.a5 = self.debug_transfer_notified_fail,
|
||||
.a7 = self.debug_transfer_intercept_state,
|
||||
.a8 = self.debug_transfer_auth_challenge,
|
||||
.a9 = self.debug_transfer_easy_id,
|
||||
.b1 = transfer.id,
|
||||
.b2 = transfer._tries,
|
||||
.b3 = transfer.aborted,
|
||||
.b4 = transfer.bytes_received,
|
||||
.b5 = transfer._notified_fail,
|
||||
.b7 = @intFromEnum(transfer._intercept_state),
|
||||
.b8 = transfer._auth_challenge != null,
|
||||
.b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
|
||||
});
|
||||
self.header_callback_called = true;
|
||||
self.debug_transfer_id = transfer.id;
|
||||
self.debug_transfer_tries = transfer._tries;
|
||||
self.debug_transfer_aborted = transfer.aborted;
|
||||
self.debug_transfer_bytes_received = transfer.bytes_received;
|
||||
self.debug_transfer_notified_fail = transfer._notified_fail;
|
||||
self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
|
||||
self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
|
||||
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
|
||||
},
|
||||
else => {},
|
||||
}
|
||||
|
||||
lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity });
|
||||
var buffer: std.ArrayList(u8) = .empty;
|
||||
if (transfer.getContentLength()) |cl| {
|
||||
if (response.contentLength()) |cl| {
|
||||
try buffer.ensureTotalCapacity(self.arena, cl);
|
||||
}
|
||||
self.source = .{ .remote = buffer };
|
||||
return true;
|
||||
}
|
||||
|
||||
fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
||||
const self: *Script = @ptrCast(@alignCast(transfer.ctx));
|
||||
self._dataCallback(transfer, data) catch |err| {
|
||||
log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });
|
||||
fn dataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||
const self: *Script = @ptrCast(@alignCast(response.ctx));
|
||||
self._dataCallback(response, data) catch |err| {
|
||||
log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = response, .len = data.len });
|
||||
return err;
|
||||
};
|
||||
}
|
||||
fn _dataCallback(self: *Script, _: *HttpClient.Transfer, data: []const u8) !void {
|
||||
|
||||
fn _dataCallback(self: *Script, _: HttpClient.Response, data: []const u8) !void {
|
||||
try self.source.remote.appendSlice(self.arena, data);
|
||||
}
|
||||
|
||||
|
||||
@@ -22,10 +22,23 @@ const DOMNode = @import("webapi/Node.zig");
|
||||
const Element = @import("webapi/Element.zig");
|
||||
const Event = @import("webapi/Event.zig");
|
||||
const MouseEvent = @import("webapi/event/MouseEvent.zig");
|
||||
const KeyboardEvent = @import("webapi/event/KeyboardEvent.zig");
|
||||
const Page = @import("Page.zig");
|
||||
const Session = @import("Session.zig");
|
||||
const Selector = @import("webapi/selector/Selector.zig");
|
||||
|
||||
fn dispatchInputAndChangeEvents(el: *Element, page: *Page) !void {
|
||||
const input_evt: *Event = try .initTrusted(comptime .wrap("input"), .{ .bubbles = true }, page);
|
||||
page._event_manager.dispatch(el.asEventTarget(), input_evt) catch |err| {
|
||||
lp.log.err(.app, "dispatch input event failed", .{ .err = err });
|
||||
};
|
||||
|
||||
const change_evt: *Event = try .initTrusted(comptime .wrap("change"), .{ .bubbles = true }, page);
|
||||
page._event_manager.dispatch(el.asEventTarget(), change_evt) catch |err| {
|
||||
lp.log.err(.app, "dispatch change event failed", .{ .err = err });
|
||||
};
|
||||
}
|
||||
|
||||
pub fn click(node: *DOMNode, page: *Page) !void {
|
||||
const el = node.is(Element) orelse return error.InvalidNodeType;
|
||||
|
||||
@@ -43,9 +56,107 @@ pub fn click(node: *DOMNode, page: *Page) !void {
|
||||
};
|
||||
}
|
||||
|
||||
pub fn hover(node: *DOMNode, page: *Page) !void {
|
||||
const el = node.is(Element) orelse return error.InvalidNodeType;
|
||||
|
||||
const mouseover_event: *MouseEvent = try .initTrusted(comptime .wrap("mouseover"), .{
|
||||
.bubbles = true,
|
||||
.cancelable = true,
|
||||
.composed = true,
|
||||
}, page);
|
||||
|
||||
page._event_manager.dispatch(el.asEventTarget(), mouseover_event.asEvent()) catch |err| {
|
||||
lp.log.err(.app, "hover mouseover failed", .{ .err = err });
|
||||
return error.ActionFailed;
|
||||
};
|
||||
|
||||
const mouseenter_event: *MouseEvent = try .initTrusted(comptime .wrap("mouseenter"), .{
|
||||
.composed = true,
|
||||
}, page);
|
||||
|
||||
page._event_manager.dispatch(el.asEventTarget(), mouseenter_event.asEvent()) catch |err| {
|
||||
lp.log.err(.app, "hover mouseenter failed", .{ .err = err });
|
||||
return error.ActionFailed;
|
||||
};
|
||||
}
|
||||
|
||||
pub fn press(node: ?*DOMNode, key: []const u8, page: *Page) !void {
|
||||
const target = if (node) |n|
|
||||
(n.is(Element) orelse return error.InvalidNodeType).asEventTarget()
|
||||
else
|
||||
page.document.asNode().asEventTarget();
|
||||
|
||||
const keydown_event: *KeyboardEvent = try .initTrusted(comptime .wrap("keydown"), .{
|
||||
.bubbles = true,
|
||||
.cancelable = true,
|
||||
.composed = true,
|
||||
.key = key,
|
||||
}, page);
|
||||
|
||||
page._event_manager.dispatch(target, keydown_event.asEvent()) catch |err| {
|
||||
lp.log.err(.app, "press keydown failed", .{ .err = err });
|
||||
return error.ActionFailed;
|
||||
};
|
||||
|
||||
const keyup_event: *KeyboardEvent = try .initTrusted(comptime .wrap("keyup"), .{
|
||||
.bubbles = true,
|
||||
.cancelable = true,
|
||||
.composed = true,
|
||||
.key = key,
|
||||
}, page);
|
||||
|
||||
page._event_manager.dispatch(target, keyup_event.asEvent()) catch |err| {
|
||||
lp.log.err(.app, "press keyup failed", .{ .err = err });
|
||||
return error.ActionFailed;
|
||||
};
|
||||
}
|
||||
|
||||
pub fn selectOption(node: *DOMNode, value: []const u8, page: *Page) !void {
|
||||
const el = node.is(Element) orelse return error.InvalidNodeType;
|
||||
const select = el.is(Element.Html.Select) orelse return error.InvalidNodeType;
|
||||
|
||||
select.setValue(value, page) catch |err| {
|
||||
lp.log.err(.app, "select setValue failed", .{ .err = err });
|
||||
return error.ActionFailed;
|
||||
};
|
||||
|
||||
try dispatchInputAndChangeEvents(el, page);
|
||||
}
|
||||
|
||||
pub fn setChecked(node: *DOMNode, checked: bool, page: *Page) !void {
|
||||
const el = node.is(Element) orelse return error.InvalidNodeType;
|
||||
const input = el.is(Element.Html.Input) orelse return error.InvalidNodeType;
|
||||
|
||||
if (input._input_type != .checkbox and input._input_type != .radio) {
|
||||
return error.InvalidNodeType;
|
||||
}
|
||||
|
||||
input.setChecked(checked, page) catch |err| {
|
||||
lp.log.err(.app, "setChecked failed", .{ .err = err });
|
||||
return error.ActionFailed;
|
||||
};
|
||||
|
||||
// Match browser event order: click fires first, then input and change.
|
||||
const click_event: *MouseEvent = try .initTrusted(comptime .wrap("click"), .{
|
||||
.bubbles = true,
|
||||
.cancelable = true,
|
||||
.composed = true,
|
||||
}, page);
|
||||
|
||||
page._event_manager.dispatch(el.asEventTarget(), click_event.asEvent()) catch |err| {
|
||||
lp.log.err(.app, "dispatch click event failed", .{ .err = err });
|
||||
};
|
||||
|
||||
try dispatchInputAndChangeEvents(el, page);
|
||||
}
|
||||
|
||||
pub fn fill(node: *DOMNode, text: []const u8, page: *Page) !void {
|
||||
const el = node.is(Element) orelse return error.InvalidNodeType;
|
||||
|
||||
el.focus(page) catch |err| {
|
||||
lp.log.err(.app, "fill focus failed", .{ .err = err });
|
||||
};
|
||||
|
||||
if (el.is(Element.Html.Input)) |input| {
|
||||
input.setValue(text, page) catch |err| {
|
||||
lp.log.err(.app, "fill input failed", .{ .err = err });
|
||||
@@ -65,15 +176,7 @@ pub fn fill(node: *DOMNode, text: []const u8, page: *Page) !void {
|
||||
return error.InvalidNodeType;
|
||||
}
|
||||
|
||||
const input_evt: *Event = try .initTrusted(comptime .wrap("input"), .{ .bubbles = true }, page);
|
||||
page._event_manager.dispatch(el.asEventTarget(), input_evt) catch |err| {
|
||||
lp.log.err(.app, "dispatch input event failed", .{ .err = err });
|
||||
};
|
||||
|
||||
const change_evt: *Event = try .initTrusted(comptime .wrap("change"), .{ .bubbles = true }, page);
|
||||
page._event_manager.dispatch(el.asEventTarget(), change_evt) catch |err| {
|
||||
lp.log.err(.app, "dispatch change event failed", .{ .err = err });
|
||||
};
|
||||
try dispatchInputAndChangeEvents(el, page);
|
||||
}
|
||||
|
||||
pub fn scroll(node: ?*DOMNode, x: ?i32, y: ?i32, page: *Page) !void {
|
||||
|
||||
@@ -10,5 +10,20 @@
|
||||
<div id="scrollbox" style="width: 100px; height: 100px; overflow: scroll;" onscroll="window.scrolled = true;">
|
||||
<div style="height: 500px;">Long content</div>
|
||||
</div>
|
||||
<div id="hoverTarget" onmouseover="window.hovered = true;">Hover Me</div>
|
||||
<input id="keyTarget" onkeydown="window.keyPressed = event.key;" onkeyup="window.keyReleased = event.key;">
|
||||
<select id="sel2" onchange="window.sel2Changed = this.value">
|
||||
<option value="a">Alpha</option>
|
||||
<option value="b">Beta</option>
|
||||
<option value="c">Gamma</option>
|
||||
</select>
|
||||
<input id="chk" type="checkbox">
|
||||
<input id="rad" type="radio" name="group1">
|
||||
<script>
|
||||
document.getElementById('chk').addEventListener('click', function() { window.chkClicked = true; });
|
||||
document.getElementById('chk').addEventListener('change', function() { window.chkChanged = true; });
|
||||
document.getElementById('rad').addEventListener('click', function() { window.radClicked = true; });
|
||||
document.getElementById('rad').addEventListener('change', function() { window.radChanged = true; });
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -127,16 +127,16 @@ fn handleBlobUrl(url: []const u8, resolver: js.PromiseResolver, page: *Page) !js
|
||||
return resolver.promise();
|
||||
}
|
||||
|
||||
fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
|
||||
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn httpStartCallback(response: HttpClient.Response) !void {
|
||||
const self: *Fetch = @ptrCast(@alignCast(response.ctx));
|
||||
if (comptime IS_DEBUG) {
|
||||
log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" });
|
||||
}
|
||||
self._response._transfer = transfer;
|
||||
self._response._http_response = response;
|
||||
}
|
||||
|
||||
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn httpHeaderDoneCallback(response: HttpClient.Response) !bool {
|
||||
const self: *Fetch = @ptrCast(@alignCast(response.ctx));
|
||||
|
||||
if (self._signal) |signal| {
|
||||
if (signal._aborted) {
|
||||
@@ -145,25 +145,24 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
}
|
||||
|
||||
const arena = self._response._arena;
|
||||
if (transfer.getContentLength()) |cl| {
|
||||
if (response.contentLength()) |cl| {
|
||||
try self._buf.ensureTotalCapacity(arena, cl);
|
||||
}
|
||||
|
||||
const res = self._response;
|
||||
const header = transfer.response_header.?;
|
||||
|
||||
if (comptime IS_DEBUG) {
|
||||
log.debug(.http, "request header", .{
|
||||
.source = "fetch",
|
||||
.url = self._url,
|
||||
.status = header.status,
|
||||
.status = response.status(),
|
||||
});
|
||||
}
|
||||
|
||||
res._status = header.status;
|
||||
res._status_text = std.http.Status.phrase(@enumFromInt(header.status)) orelse "";
|
||||
res._url = try arena.dupeZ(u8, std.mem.span(header.url));
|
||||
res._is_redirected = header.redirect_count > 0;
|
||||
res._status = response.status().?;
|
||||
res._status_text = std.http.Status.phrase(@enumFromInt(response.status().?)) orelse "";
|
||||
res._url = try arena.dupeZ(u8, response.url());
|
||||
res._is_redirected = response.redirectCount().? > 0;
|
||||
|
||||
// Determine response type based on origin comparison
|
||||
const page_origin = URL.getOrigin(arena, self._page.url) catch null;
|
||||
@@ -183,7 +182,7 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
res._type = .basic;
|
||||
}
|
||||
|
||||
var it = transfer.responseHeaderIterator();
|
||||
var it = response.headerIterator();
|
||||
while (it.next()) |hdr| {
|
||||
try res._headers.append(hdr.name, hdr.value, self._page);
|
||||
}
|
||||
@@ -191,8 +190,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
||||
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||
const self: *Fetch = @ptrCast(@alignCast(response.ctx));
|
||||
|
||||
// Check if aborted
|
||||
if (self._signal) |signal| {
|
||||
@@ -207,7 +206,7 @@ fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
||||
fn httpDoneCallback(ctx: *anyopaque) !void {
|
||||
const self: *Fetch = @ptrCast(@alignCast(ctx));
|
||||
var response = self._response;
|
||||
response._transfer = null;
|
||||
response._http_response = null;
|
||||
response._body = self._buf.items;
|
||||
|
||||
log.info(.http, "request complete", .{
|
||||
@@ -230,7 +229,7 @@ fn httpErrorCallback(ctx: *anyopaque, _: anyerror) void {
|
||||
const self: *Fetch = @ptrCast(@alignCast(ctx));
|
||||
|
||||
var response = self._response;
|
||||
response._transfer = null;
|
||||
response._http_response = null;
|
||||
// the response is only passed on v8 on success, if we're here, it's safe to
|
||||
// clear this. (defer since `self is in the response's arena).
|
||||
|
||||
@@ -256,7 +255,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {
|
||||
|
||||
if (self._owns_response) {
|
||||
var response = self._response;
|
||||
response._transfer = null;
|
||||
response._http_response = null;
|
||||
response.deinit(self._page._session);
|
||||
// Do not access `self` after this point: the Fetch struct was
|
||||
// allocated from response._arena which has been released.
|
||||
|
||||
@@ -48,7 +48,7 @@ _type: Type,
|
||||
_status_text: []const u8,
|
||||
_url: [:0]const u8,
|
||||
_is_redirected: bool,
|
||||
_transfer: ?*HttpClient.Transfer = null,
|
||||
_http_response: ?HttpClient.Response = null,
|
||||
|
||||
const InitOpts = struct {
|
||||
status: u16 = 200,
|
||||
@@ -81,9 +81,9 @@ pub fn init(body_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*Response {
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Response, session: *Session) void {
|
||||
if (self._transfer) |transfer| {
|
||||
transfer.abort(error.Abort);
|
||||
self._transfer = null;
|
||||
if (self._http_response) |resp| {
|
||||
resp.abort(error.Abort);
|
||||
self._http_response = null;
|
||||
}
|
||||
session.releaseArena(self._arena);
|
||||
}
|
||||
@@ -191,7 +191,7 @@ pub fn clone(self: *const Response, page: *Page) !*Response {
|
||||
._type = self._type,
|
||||
._is_redirected = self._is_redirected,
|
||||
._headers = try Headers.init(.{ .obj = self._headers }, page),
|
||||
._transfer = null,
|
||||
._http_response = null,
|
||||
};
|
||||
return cloned;
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ _rc: lp.RC(u8) = .{},
|
||||
_page: *Page,
|
||||
_proto: *XMLHttpRequestEventTarget,
|
||||
_arena: Allocator,
|
||||
_transfer: ?*HttpClient.Transfer = null,
|
||||
_http_response: ?HttpClient.Response = null,
|
||||
_active_request: bool = false,
|
||||
|
||||
_url: [:0]const u8 = "",
|
||||
@@ -100,9 +100,9 @@ pub fn init(page: *Page) !*XMLHttpRequest {
|
||||
}
|
||||
|
||||
pub fn deinit(self: *XMLHttpRequest, session: *Session) void {
|
||||
if (self._transfer) |transfer| {
|
||||
transfer.abort(error.Abort);
|
||||
self._transfer = null;
|
||||
if (self._http_response) |resp| {
|
||||
resp.abort(error.Abort);
|
||||
self._http_response = null;
|
||||
}
|
||||
|
||||
if (self._on_ready_state_change) |func| {
|
||||
@@ -184,9 +184,9 @@ pub fn setWithCredentials(self: *XMLHttpRequest, value: bool) !void {
|
||||
// TODO: url should be a union, as it can be multiple things
|
||||
pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void {
|
||||
// Abort any in-progress request
|
||||
if (self._transfer) |transfer| {
|
||||
if (self._http_response) |transfer| {
|
||||
transfer.abort(error.Abort);
|
||||
self._transfer = null;
|
||||
self._http_response = null;
|
||||
}
|
||||
|
||||
// Reset internal state
|
||||
@@ -402,34 +402,32 @@ pub fn getResponseXML(self: *XMLHttpRequest, page: *Page) !?*Node.Document {
|
||||
};
|
||||
}
|
||||
|
||||
fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn httpStartCallback(response: HttpClient.Response) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||
if (comptime IS_DEBUG) {
|
||||
log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });
|
||||
}
|
||||
self._transfer = transfer;
|
||||
self._http_response = response;
|
||||
}
|
||||
|
||||
fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: http.Header) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn httpHeaderCallback(response: HttpClient.Response, header: http.Header) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });
|
||||
try self._response_headers.append(self._arena, joined);
|
||||
}
|
||||
|
||||
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
||||
|
||||
const header = &transfer.response_header.?;
|
||||
fn httpHeaderDoneCallback(response: HttpClient.Response) !bool {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||
|
||||
if (comptime IS_DEBUG) {
|
||||
log.debug(.http, "request header", .{
|
||||
.source = "xhr",
|
||||
.url = self._url,
|
||||
.status = header.status,
|
||||
.status = response.status(),
|
||||
});
|
||||
}
|
||||
|
||||
if (header.contentType()) |ct| {
|
||||
if (response.contentType()) |ct| {
|
||||
self._response_mime = Mime.parse(ct) catch |e| {
|
||||
log.info(.http, "invalid content type", .{
|
||||
.content_Type = ct,
|
||||
@@ -440,18 +438,18 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
};
|
||||
}
|
||||
|
||||
var it = transfer.responseHeaderIterator();
|
||||
var it = response.headerIterator();
|
||||
while (it.next()) |hdr| {
|
||||
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value });
|
||||
try self._response_headers.append(self._arena, joined);
|
||||
}
|
||||
|
||||
self._response_status = header.status;
|
||||
if (transfer.getContentLength()) |cl| {
|
||||
self._response_status = response.status().?;
|
||||
if (response.contentLength()) |cl| {
|
||||
self._response_len = cl;
|
||||
try self._response_data.ensureTotalCapacity(self._arena, cl);
|
||||
}
|
||||
self._response_url = try self._arena.dupeZ(u8, std.mem.span(header.url));
|
||||
self._response_url = try self._arena.dupeZ(u8, response.url());
|
||||
|
||||
const page = self._page;
|
||||
|
||||
@@ -466,8 +464,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
||||
fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||
try self._response_data.appendSlice(self._arena, data);
|
||||
|
||||
const page = self._page;
|
||||
@@ -490,7 +488,7 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
|
||||
|
||||
// Not that the request is done, the http/client will free the transfer
|
||||
// object. It isn't safe to keep it around.
|
||||
self._transfer = null;
|
||||
self._http_response = null;
|
||||
|
||||
const page = self._page;
|
||||
|
||||
@@ -513,23 +511,23 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
|
||||
// http client will close it after an error, it isn't safe to keep around
|
||||
self.handleError(err);
|
||||
if (self._transfer != null) {
|
||||
self._transfer = null;
|
||||
if (self._http_response != null) {
|
||||
self._http_response = null;
|
||||
}
|
||||
self.releaseSelfRef();
|
||||
}
|
||||
|
||||
fn httpShutdownCallback(ctx: *anyopaque) void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
|
||||
self._transfer = null;
|
||||
self._http_response = null;
|
||||
self.releaseSelfRef();
|
||||
}
|
||||
|
||||
pub fn abort(self: *XMLHttpRequest) void {
|
||||
self.handleError(error.Abort);
|
||||
if (self._transfer) |transfer| {
|
||||
self._transfer = null;
|
||||
transfer.abort(error.Abort);
|
||||
if (self._http_response) |resp| {
|
||||
self._http_response = null;
|
||||
resp.abort(error.Abort);
|
||||
}
|
||||
self.releaseSelfRef();
|
||||
}
|
||||
|
||||
@@ -52,17 +52,19 @@ fn dispatchKeyEvent(cmd: *CDP.Command) !void {
|
||||
|
||||
try cmd.sendResult(null, .{});
|
||||
|
||||
// quickly ignore types we know we don't handle
|
||||
switch (params.type) {
|
||||
.keyUp, .rawKeyDown, .char => return,
|
||||
.keyDown => {},
|
||||
}
|
||||
// rawKeyDown is a Chrome-internal event type not used for JS dispatch
|
||||
if (params.type == .rawKeyDown) return;
|
||||
|
||||
const bc = cmd.browser_context orelse return;
|
||||
const page = bc.session.currentPage() orelse return;
|
||||
|
||||
const KeyboardEvent = @import("../../browser/webapi/event/KeyboardEvent.zig");
|
||||
const keyboard_event = try KeyboardEvent.initTrusted(comptime .wrap("keydown"), .{
|
||||
const keyboard_event = try KeyboardEvent.initTrusted(switch (params.type) {
|
||||
.keyDown => comptime .wrap("keydown"),
|
||||
.keyUp => comptime .wrap("keyup"),
|
||||
.char => comptime .wrap("keypress"),
|
||||
.rawKeyDown => unreachable,
|
||||
}, .{
|
||||
.key = params.key,
|
||||
.code = params.code,
|
||||
.altKey = params.modifiers & 1 == 1,
|
||||
|
||||
@@ -40,6 +40,7 @@ pub const forms = @import("browser/forms.zig");
|
||||
pub const actions = @import("browser/actions.zig");
|
||||
pub const structured_data = @import("browser/structured_data.zig");
|
||||
pub const mcp = @import("mcp.zig");
|
||||
pub const agent = @import("agent.zig");
|
||||
pub const build_config = @import("build_config");
|
||||
pub const crash_handler = @import("crash_handler.zig");
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ pub const Scope = enum {
|
||||
telemetry,
|
||||
unknown_prop,
|
||||
mcp,
|
||||
cache,
|
||||
};
|
||||
|
||||
const Opts = struct {
|
||||
|
||||
177
src/main.zig
177
src/main.zig
@@ -43,8 +43,6 @@ pub fn main() !void {
|
||||
const main_arena = main_arena_instance.allocator();
|
||||
defer main_arena_instance.deinit();
|
||||
|
||||
try useSqlite3();
|
||||
|
||||
run(gpa, main_arena) catch |err| {
|
||||
log.fatal(.app, "exit", .{ .err = err });
|
||||
std.posix.exit(1);
|
||||
@@ -167,10 +165,30 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {
|
||||
|
||||
app.network.run();
|
||||
},
|
||||
.agent => |opts| {
|
||||
log.info(.app, "starting agent", .{});
|
||||
|
||||
var worker_thread = try std.Thread.spawn(.{}, agentThread, .{ allocator, app, opts });
|
||||
defer worker_thread.join();
|
||||
|
||||
app.network.run();
|
||||
},
|
||||
else => unreachable,
|
||||
}
|
||||
}
|
||||
|
||||
fn agentThread(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) void {
|
||||
defer app.network.stop();
|
||||
|
||||
var agent_instance = lp.agent.Agent.init(allocator, app, opts) catch |err| {
|
||||
log.fatal(.app, "agent init error", .{ .err = err });
|
||||
return;
|
||||
};
|
||||
defer agent_instance.deinit();
|
||||
|
||||
agent_instance.run();
|
||||
}
|
||||
|
||||
fn fetchThread(app: *App, url: [:0]const u8, fetch_opts: lp.FetchOpts) void {
|
||||
defer app.network.stop();
|
||||
lp.fetch(app, url, fetch_opts) catch |err| {
|
||||
@@ -194,158 +212,3 @@ fn mcpThread(allocator: std.mem.Allocator, app: *App) void {
|
||||
log.fatal(.mcp, "mcp error", .{ .err = err });
|
||||
};
|
||||
}
|
||||
|
||||
fn useSqlite3() !void {
|
||||
const c = @cImport(@cInclude("sqlite3.h"));
|
||||
|
||||
const flags = c.SQLITE_OPEN_READWRITE;
|
||||
|
||||
var conn: ?*c.sqlite3 = null;
|
||||
{
|
||||
const rc = c.sqlite3_open_v2(":memory:", &conn, flags, null);
|
||||
if (rc != c.SQLITE_OK) {
|
||||
return sqlite3Error(rc);
|
||||
}
|
||||
}
|
||||
defer _ = c.sqlite3_close_v2(conn);
|
||||
|
||||
var stmt: ?*c.sqlite3_stmt = null;
|
||||
{
|
||||
const sql = "select sqlite_version()";
|
||||
var tail: [*:0]const u8 = undefined;
|
||||
const rc = c.sqlite3_prepare_v2(conn, sql, @intCast(sql.len), &stmt, @ptrCast(&tail));
|
||||
if (rc != c.SQLITE_OK) {
|
||||
return sqlite3Error(rc);
|
||||
}
|
||||
}
|
||||
defer _ = c.sqlite3_finalize(stmt);
|
||||
|
||||
{
|
||||
const rc = c.sqlite3_step(stmt);
|
||||
if (rc == c.SQLITE_DONE) {
|
||||
return error.NoRow;
|
||||
}
|
||||
if (rc != c.SQLITE_ROW) {
|
||||
return sqlite3Error(rc);
|
||||
}
|
||||
|
||||
const data = c.sqlite3_column_text(stmt, 0);
|
||||
const len = c.sqlite3_column_bytes(stmt, 0);
|
||||
if (len == 0) {
|
||||
return error.EmptyValue;
|
||||
}
|
||||
std.debug.print("sqlite version: {s}\n", .{@as([*c]const u8, @ptrCast(data))[0..@intCast(len)]});
|
||||
}
|
||||
}
|
||||
|
||||
fn sqlite3Error(result: c_int) !void {
|
||||
const c = @cImport(@cInclude("sqlite3.h"));
|
||||
return switch (result) {
|
||||
c.SQLITE_ABORT => error.Abort,
|
||||
c.SQLITE_AUTH => error.Auth,
|
||||
c.SQLITE_BUSY => error.Busy,
|
||||
c.SQLITE_CANTOPEN => error.CantOpen,
|
||||
c.SQLITE_CONSTRAINT => error.Constraint,
|
||||
c.SQLITE_CORRUPT => error.Corrupt,
|
||||
c.SQLITE_EMPTY => error.Empty,
|
||||
c.SQLITE_ERROR => error.Error,
|
||||
c.SQLITE_FORMAT => error.Format,
|
||||
c.SQLITE_FULL => error.Full,
|
||||
c.SQLITE_INTERNAL => error.Internal,
|
||||
c.SQLITE_INTERRUPT => error.Interrupt,
|
||||
c.SQLITE_IOERR => error.IoErr,
|
||||
c.SQLITE_LOCKED => error.Locked,
|
||||
c.SQLITE_MISMATCH => error.Mismatch,
|
||||
c.SQLITE_MISUSE => error.Misuse,
|
||||
c.SQLITE_NOLFS => error.NoLFS,
|
||||
c.SQLITE_NOMEM => error.NoMem,
|
||||
c.SQLITE_NOTADB => error.NotADB,
|
||||
c.SQLITE_NOTFOUND => error.Notfound,
|
||||
c.SQLITE_NOTICE => error.Notice,
|
||||
c.SQLITE_PERM => error.Perm,
|
||||
c.SQLITE_PROTOCOL => error.Protocol,
|
||||
c.SQLITE_RANGE => error.Range,
|
||||
c.SQLITE_READONLY => error.ReadOnly,
|
||||
c.SQLITE_SCHEMA => error.Schema,
|
||||
c.SQLITE_TOOBIG => error.TooBig,
|
||||
c.SQLITE_WARNING => error.Warning,
|
||||
|
||||
// extended codes:
|
||||
c.SQLITE_ERROR_MISSING_COLLSEQ => error.ErrorMissingCollseq,
|
||||
c.SQLITE_ERROR_RETRY => error.ErrorRetry,
|
||||
c.SQLITE_ERROR_SNAPSHOT => error.ErrorSnapshot,
|
||||
c.SQLITE_IOERR_READ => error.IoerrRead,
|
||||
c.SQLITE_IOERR_SHORT_READ => error.IoerrShortRead,
|
||||
c.SQLITE_IOERR_WRITE => error.IoerrWrite,
|
||||
c.SQLITE_IOERR_FSYNC => error.IoerrFsync,
|
||||
c.SQLITE_IOERR_DIR_FSYNC => error.IoerrDir_fsync,
|
||||
c.SQLITE_IOERR_TRUNCATE => error.IoerrTruncate,
|
||||
c.SQLITE_IOERR_FSTAT => error.IoerrFstat,
|
||||
c.SQLITE_IOERR_UNLOCK => error.IoerrUnlock,
|
||||
c.SQLITE_IOERR_RDLOCK => error.IoerrRdlock,
|
||||
c.SQLITE_IOERR_DELETE => error.IoerrDelete,
|
||||
c.SQLITE_IOERR_BLOCKED => error.IoerrBlocked,
|
||||
c.SQLITE_IOERR_NOMEM => error.IoerrNomem,
|
||||
c.SQLITE_IOERR_ACCESS => error.IoerrAccess,
|
||||
c.SQLITE_IOERR_CHECKRESERVEDLOCK => error.IoerrCheckreservedlock,
|
||||
c.SQLITE_IOERR_LOCK => error.IoerrLock,
|
||||
c.SQLITE_IOERR_CLOSE => error.IoerrClose,
|
||||
c.SQLITE_IOERR_DIR_CLOSE => error.IoerrDirClose,
|
||||
c.SQLITE_IOERR_SHMOPEN => error.IoerrShmopen,
|
||||
c.SQLITE_IOERR_SHMSIZE => error.IoerrShmsize,
|
||||
c.SQLITE_IOERR_SHMLOCK => error.IoerrShmlock,
|
||||
c.SQLITE_IOERR_SHMMAP => error.ioerrshmmap,
|
||||
c.SQLITE_IOERR_SEEK => error.IoerrSeek,
|
||||
c.SQLITE_IOERR_DELETE_NOENT => error.IoerrDeleteNoent,
|
||||
c.SQLITE_IOERR_MMAP => error.IoerrMmap,
|
||||
c.SQLITE_IOERR_GETTEMPPATH => error.IoerrGetTempPath,
|
||||
c.SQLITE_IOERR_CONVPATH => error.IoerrConvPath,
|
||||
c.SQLITE_IOERR_VNODE => error.IoerrVnode,
|
||||
c.SQLITE_IOERR_AUTH => error.IoerrAuth,
|
||||
c.SQLITE_IOERR_BEGIN_ATOMIC => error.IoerrBeginAtomic,
|
||||
c.SQLITE_IOERR_COMMIT_ATOMIC => error.IoerrCommitAtomic,
|
||||
c.SQLITE_IOERR_ROLLBACK_ATOMIC => error.IoerrRollbackAtomic,
|
||||
c.SQLITE_IOERR_DATA => error.IoerrData,
|
||||
c.SQLITE_IOERR_CORRUPTFS => error.IoerrCorruptFS,
|
||||
c.SQLITE_LOCKED_SHAREDCACHE => error.LockedSharedCache,
|
||||
c.SQLITE_LOCKED_VTAB => error.LockedVTab,
|
||||
c.SQLITE_BUSY_RECOVERY => error.BusyRecovery,
|
||||
c.SQLITE_BUSY_SNAPSHOT => error.BusySnapshot,
|
||||
c.SQLITE_BUSY_TIMEOUT => error.BusyTimeout,
|
||||
c.SQLITE_CANTOPEN_NOTEMPDIR => error.CantOpenNoTempDir,
|
||||
c.SQLITE_CANTOPEN_ISDIR => error.CantOpenIsDir,
|
||||
c.SQLITE_CANTOPEN_FULLPATH => error.CantOpenFullPath,
|
||||
c.SQLITE_CANTOPEN_CONVPATH => error.CantOpenConvPath,
|
||||
c.SQLITE_CANTOPEN_DIRTYWAL => error.CantOpenDirtyWal,
|
||||
c.SQLITE_CANTOPEN_SYMLINK => error.CantOpenSymlink,
|
||||
c.SQLITE_CORRUPT_VTAB => error.CorruptVTab,
|
||||
c.SQLITE_CORRUPT_SEQUENCE => error.CorruptSequence,
|
||||
c.SQLITE_CORRUPT_INDEX => error.CorruptIndex,
|
||||
c.SQLITE_READONLY_RECOVERY => error.ReadonlyRecovery,
|
||||
c.SQLITE_READONLY_CANTLOCK => error.ReadonlyCantlock,
|
||||
c.SQLITE_READONLY_ROLLBACK => error.ReadonlyRollback,
|
||||
c.SQLITE_READONLY_DBMOVED => error.ReadonlyDbMoved,
|
||||
c.SQLITE_READONLY_CANTINIT => error.ReadonlyCantInit,
|
||||
c.SQLITE_READONLY_DIRECTORY => error.ReadonlyDirectory,
|
||||
c.SQLITE_ABORT_ROLLBACK => error.AbortRollback,
|
||||
c.SQLITE_CONSTRAINT_CHECK => error.ConstraintCheck,
|
||||
c.SQLITE_CONSTRAINT_COMMITHOOK => error.ConstraintCommithook,
|
||||
c.SQLITE_CONSTRAINT_FOREIGNKEY => error.ConstraintForeignKey,
|
||||
c.SQLITE_CONSTRAINT_FUNCTION => error.ConstraintFunction,
|
||||
c.SQLITE_CONSTRAINT_NOTNULL => error.ConstraintNotNull,
|
||||
c.SQLITE_CONSTRAINT_PRIMARYKEY => error.ConstraintPrimaryKey,
|
||||
c.SQLITE_CONSTRAINT_TRIGGER => error.ConstraintTrigger,
|
||||
c.SQLITE_CONSTRAINT_UNIQUE => error.ConstraintUnique,
|
||||
c.SQLITE_CONSTRAINT_VTAB => error.ConstraintVTab,
|
||||
c.SQLITE_CONSTRAINT_ROWID => error.ConstraintRowId,
|
||||
c.SQLITE_CONSTRAINT_PINNED => error.ConstraintPinned,
|
||||
c.SQLITE_CONSTRAINT_DATATYPE => error.ConstraintDatatype,
|
||||
c.SQLITE_NOTICE_RECOVER_WAL => error.NoticeRecoverWal,
|
||||
c.SQLITE_NOTICE_RECOVER_ROLLBACK => error.NoticeRecoverRollback,
|
||||
c.SQLITE_WARNING_AUTOINDEX => error.WarningAutoIndex,
|
||||
c.SQLITE_AUTH_USER => error.AuthUser,
|
||||
c.SQLITE_OK_LOAD_PERMANENTLY => error.OkLoadPermanently,
|
||||
|
||||
else => std.debug.panic("{s} {d}", .{ c.sqlite3_errstr(result), result }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -175,6 +175,74 @@ pub const tool_list = [_]protocol.Tool{
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "hover",
|
||||
.description = "Hover over an element, triggering mouseover and mouseenter events. Useful for menus, tooltips, and hover states.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "backendNodeId": { "type": "integer", "description": "The backend node ID of the element to hover over." }
|
||||
\\ },
|
||||
\\ "required": ["backendNodeId"]
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "press",
|
||||
.description = "Press a keyboard key, dispatching keydown and keyup events. Use key names like 'Enter', 'Tab', 'Escape', 'ArrowDown', 'Backspace', or single characters like 'a', '1'.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "key": { "type": "string", "description": "The key to press (e.g. 'Enter', 'Tab', 'a')." },
|
||||
\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID of the element to target. Defaults to the document." }
|
||||
\\ },
|
||||
\\ "required": ["key"]
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "selectOption",
|
||||
.description = "Select an option in a <select> dropdown element by its value. Dispatches input and change events.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "backendNodeId": { "type": "integer", "description": "The backend node ID of the <select> element." },
|
||||
\\ "value": { "type": "string", "description": "The value of the option to select." }
|
||||
\\ },
|
||||
\\ "required": ["backendNodeId", "value"]
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "setChecked",
|
||||
.description = "Check or uncheck a checkbox or radio button. Dispatches input, change, and click events.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "backendNodeId": { "type": "integer", "description": "The backend node ID of the checkbox or radio input element." },
|
||||
\\ "checked": { "type": "boolean", "description": "Whether to check (true) or uncheck (false) the element." }
|
||||
\\ },
|
||||
\\ "required": ["backendNodeId", "checked"]
|
||||
\\}
|
||||
),
|
||||
},
|
||||
.{
|
||||
.name = "findElement",
|
||||
.description = "Find interactive elements by role and/or accessible name. Returns matching elements with their backend node IDs. Useful for locating specific elements without parsing the full semantic tree.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "role": { "type": "string", "description": "Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox')." },
|
||||
\\ "name": { "type": "string", "description": "Optional accessible name substring to match (case-insensitive)." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
pub fn handleList(server: *Server, arena: std.mem.Allocator, req: protocol.Request) !void {
|
||||
@@ -282,6 +350,11 @@ const ToolAction = enum {
|
||||
fill,
|
||||
scroll,
|
||||
waitForSelector,
|
||||
hover,
|
||||
press,
|
||||
selectOption,
|
||||
setChecked,
|
||||
findElement,
|
||||
};
|
||||
|
||||
const tool_map = std.StaticStringMap(ToolAction).initComptime(.{
|
||||
@@ -300,6 +373,11 @@ const tool_map = std.StaticStringMap(ToolAction).initComptime(.{
|
||||
.{ "fill", .fill },
|
||||
.{ "scroll", .scroll },
|
||||
.{ "waitForSelector", .waitForSelector },
|
||||
.{ "hover", .hover },
|
||||
.{ "press", .press },
|
||||
.{ "selectOption", .selectOption },
|
||||
.{ "setChecked", .setChecked },
|
||||
.{ "findElement", .findElement },
|
||||
});
|
||||
|
||||
pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Request) !void {
|
||||
@@ -334,6 +412,11 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
|
||||
.fill => try handleFill(server, arena, req.id.?, call_params.arguments),
|
||||
.scroll => try handleScroll(server, arena, req.id.?, call_params.arguments),
|
||||
.waitForSelector => try handleWaitForSelector(server, arena, req.id.?, call_params.arguments),
|
||||
.hover => try handleHover(server, arena, req.id.?, call_params.arguments),
|
||||
.press => try handlePress(server, arena, req.id.?, call_params.arguments),
|
||||
.selectOption => try handleSelectOption(server, arena, req.id.?, call_params.arguments),
|
||||
.setChecked => try handleSetChecked(server, arena, req.id.?, call_params.arguments),
|
||||
.findElement => try handleFindElement(server, arena, req.id.?, call_params.arguments),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,17 +483,9 @@ fn handleNodeDetails(server: *Server, arena: std.mem.Allocator, id: std.json.Val
|
||||
backendNodeId: CDPNode.Id,
|
||||
};
|
||||
const args = try parseArgs(Params, arena, arguments, server, id, "nodeDetails");
|
||||
const resolved = try resolveNodeAndPage(server, id, args.backendNodeId);
|
||||
|
||||
_ = server.session.currentPage() orelse {
|
||||
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
};
|
||||
|
||||
const node = server.node_registry.lookup_by_id.get(args.backendNodeId) orelse {
|
||||
return server.sendError(id, .InvalidParams, "Node not found");
|
||||
};
|
||||
|
||||
const page = server.session.currentPage().?;
|
||||
const details = lp.SemanticTree.getNodeDetails(arena, node.dom, &server.node_registry, page) catch {
|
||||
const details = lp.SemanticTree.getNodeDetails(arena, resolved.node, &server.node_registry, resolved.page) catch {
|
||||
return server.sendError(id, .InternalError, "Failed to get node details");
|
||||
};
|
||||
|
||||
@@ -510,26 +585,19 @@ fn handleClick(server: *Server, arena: std.mem.Allocator, id: std.json.Value, ar
|
||||
backendNodeId: CDPNode.Id,
|
||||
};
|
||||
const args = try parseArgs(ClickParams, arena, arguments, server, id, "click");
|
||||
const resolved = try resolveNodeAndPage(server, id, args.backendNodeId);
|
||||
|
||||
const page = server.session.currentPage() orelse {
|
||||
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
};
|
||||
|
||||
const node = server.node_registry.lookup_by_id.get(args.backendNodeId) orelse {
|
||||
return server.sendError(id, .InvalidParams, "Node not found");
|
||||
};
|
||||
|
||||
lp.actions.click(node.dom, page) catch |err| {
|
||||
lp.actions.click(resolved.node, resolved.page) catch |err| {
|
||||
if (err == error.InvalidNodeType) {
|
||||
return server.sendError(id, .InvalidParams, "Node is not an HTML element");
|
||||
}
|
||||
return server.sendError(id, .InternalError, "Failed to click element");
|
||||
};
|
||||
|
||||
const page_title = page.getTitle() catch null;
|
||||
const page_title = resolved.page.getTitle() catch null;
|
||||
const result_text = try std.fmt.allocPrint(arena, "Clicked element (backendNodeId: {d}). Page url: {s}, title: {s}", .{
|
||||
args.backendNodeId,
|
||||
page.url,
|
||||
resolved.page.url,
|
||||
page_title orelse "(none)",
|
||||
});
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = result_text }};
|
||||
@@ -542,27 +610,20 @@ fn handleFill(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arg
|
||||
text: []const u8,
|
||||
};
|
||||
const args = try parseArgs(FillParams, arena, arguments, server, id, "fill");
|
||||
const resolved = try resolveNodeAndPage(server, id, args.backendNodeId);
|
||||
|
||||
const page = server.session.currentPage() orelse {
|
||||
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
};
|
||||
|
||||
const node = server.node_registry.lookup_by_id.get(args.backendNodeId) orelse {
|
||||
return server.sendError(id, .InvalidParams, "Node not found");
|
||||
};
|
||||
|
||||
lp.actions.fill(node.dom, args.text, page) catch |err| {
|
||||
lp.actions.fill(resolved.node, args.text, resolved.page) catch |err| {
|
||||
if (err == error.InvalidNodeType) {
|
||||
return server.sendError(id, .InvalidParams, "Node is not an input, textarea or select");
|
||||
}
|
||||
return server.sendError(id, .InternalError, "Failed to fill element");
|
||||
};
|
||||
|
||||
const page_title = page.getTitle() catch null;
|
||||
const page_title = resolved.page.getTitle() catch null;
|
||||
const result_text = try std.fmt.allocPrint(arena, "Filled element (backendNodeId: {d}) with \"{s}\". Page url: {s}, title: {s}", .{
|
||||
args.backendNodeId,
|
||||
args.text,
|
||||
page.url,
|
||||
resolved.page.url,
|
||||
page_title orelse "(none)",
|
||||
});
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = result_text }};
|
||||
@@ -636,6 +697,189 @@ fn handleWaitForSelector(server: *Server, arena: std.mem.Allocator, id: std.json
|
||||
return server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn handleHover(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const Params = struct {
|
||||
backendNodeId: CDPNode.Id,
|
||||
};
|
||||
const args = try parseArgs(Params, arena, arguments, server, id, "hover");
|
||||
const resolved = try resolveNodeAndPage(server, id, args.backendNodeId);
|
||||
|
||||
lp.actions.hover(resolved.node, resolved.page) catch |err| {
|
||||
if (err == error.InvalidNodeType) {
|
||||
return server.sendError(id, .InvalidParams, "Node is not an HTML element");
|
||||
}
|
||||
return server.sendError(id, .InternalError, "Failed to hover element");
|
||||
};
|
||||
|
||||
const page_title = resolved.page.getTitle() catch null;
|
||||
const result_text = try std.fmt.allocPrint(arena, "Hovered element (backendNodeId: {d}). Page url: {s}, title: {s}", .{
|
||||
args.backendNodeId,
|
||||
resolved.page.url,
|
||||
page_title orelse "(none)",
|
||||
});
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = result_text }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn handlePress(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const Params = struct {
|
||||
key: []const u8,
|
||||
backendNodeId: ?CDPNode.Id = null,
|
||||
};
|
||||
const args = try parseArgs(Params, arena, arguments, server, id, "press");
|
||||
|
||||
const page = server.session.currentPage() orelse {
|
||||
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
};
|
||||
|
||||
var target_node: ?*DOMNode = null;
|
||||
if (args.backendNodeId) |node_id| {
|
||||
const node = server.node_registry.lookup_by_id.get(node_id) orelse {
|
||||
return server.sendError(id, .InvalidParams, "Node not found");
|
||||
};
|
||||
target_node = node.dom;
|
||||
}
|
||||
|
||||
lp.actions.press(target_node, args.key, page) catch |err| {
|
||||
if (err == error.InvalidNodeType) {
|
||||
return server.sendError(id, .InvalidParams, "Node is not an HTML element");
|
||||
}
|
||||
return server.sendError(id, .InternalError, "Failed to press key");
|
||||
};
|
||||
|
||||
const page_title = page.getTitle() catch null;
|
||||
const result_text = try std.fmt.allocPrint(arena, "Pressed key '{s}'. Page url: {s}, title: {s}", .{
|
||||
args.key,
|
||||
page.url,
|
||||
page_title orelse "(none)",
|
||||
});
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = result_text }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn handleSelectOption(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const Params = struct {
|
||||
backendNodeId: CDPNode.Id,
|
||||
value: []const u8,
|
||||
};
|
||||
const args = try parseArgs(Params, arena, arguments, server, id, "selectOption");
|
||||
const resolved = try resolveNodeAndPage(server, id, args.backendNodeId);
|
||||
|
||||
lp.actions.selectOption(resolved.node, args.value, resolved.page) catch |err| {
|
||||
if (err == error.InvalidNodeType) {
|
||||
return server.sendError(id, .InvalidParams, "Node is not a <select> element");
|
||||
}
|
||||
return server.sendError(id, .InternalError, "Failed to select option");
|
||||
};
|
||||
|
||||
const page_title = resolved.page.getTitle() catch null;
|
||||
const result_text = try std.fmt.allocPrint(arena, "Selected option '{s}' (backendNodeId: {d}). Page url: {s}, title: {s}", .{
|
||||
args.value,
|
||||
args.backendNodeId,
|
||||
resolved.page.url,
|
||||
page_title orelse "(none)",
|
||||
});
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = result_text }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn handleSetChecked(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const Params = struct {
|
||||
backendNodeId: CDPNode.Id,
|
||||
checked: bool,
|
||||
};
|
||||
const args = try parseArgs(Params, arena, arguments, server, id, "setChecked");
|
||||
const resolved = try resolveNodeAndPage(server, id, args.backendNodeId);
|
||||
|
||||
lp.actions.setChecked(resolved.node, args.checked, resolved.page) catch |err| {
|
||||
if (err == error.InvalidNodeType) {
|
||||
return server.sendError(id, .InvalidParams, "Node is not a checkbox or radio input");
|
||||
}
|
||||
return server.sendError(id, .InternalError, "Failed to set checked state");
|
||||
};
|
||||
|
||||
const state_str = if (args.checked) "checked" else "unchecked";
|
||||
const page_title = resolved.page.getTitle() catch null;
|
||||
const result_text = try std.fmt.allocPrint(arena, "Set element (backendNodeId: {d}) to {s}. Page url: {s}, title: {s}", .{
|
||||
args.backendNodeId,
|
||||
state_str,
|
||||
resolved.page.url,
|
||||
page_title orelse "(none)",
|
||||
});
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = result_text }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn handleFindElement(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const Params = struct {
|
||||
role: ?[]const u8 = null,
|
||||
name: ?[]const u8 = null,
|
||||
};
|
||||
const args = try parseArgsOrDefault(Params, arena, arguments, server, id);
|
||||
|
||||
if (args.role == null and args.name == null) {
|
||||
return server.sendError(id, .InvalidParams, "At least one of 'role' or 'name' must be provided");
|
||||
}
|
||||
|
||||
const page = server.session.currentPage() orelse {
|
||||
return server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
};
|
||||
|
||||
const elements = lp.interactive.collectInteractiveElements(page.document.asNode(), arena, page) catch |err| {
|
||||
log.err(.mcp, "elements collection failed", .{ .err = err });
|
||||
return server.sendError(id, .InternalError, "Failed to collect interactive elements");
|
||||
};
|
||||
|
||||
var matches: std.ArrayList(lp.interactive.InteractiveElement) = .empty;
|
||||
for (elements) |el| {
|
||||
if (args.role) |role| {
|
||||
const el_role = el.role orelse continue;
|
||||
if (!std.ascii.eqlIgnoreCase(el_role, role)) continue;
|
||||
}
|
||||
if (args.name) |name| {
|
||||
const el_name = el.name orelse continue;
|
||||
if (!containsIgnoreCase(el_name, name)) continue;
|
||||
}
|
||||
try matches.append(arena, el);
|
||||
}
|
||||
|
||||
const matched = try matches.toOwnedSlice(arena);
|
||||
lp.interactive.registerNodes(matched, &server.node_registry) catch |err| {
|
||||
log.err(.mcp, "node registration failed", .{ .err = err });
|
||||
return server.sendError(id, .InternalError, "Failed to register element nodes");
|
||||
};
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
try std.json.Stringify.value(matched, .{}, &aw.writer);
|
||||
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = aw.written() }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn containsIgnoreCase(haystack: []const u8, needle: []const u8) bool {
|
||||
if (needle.len > haystack.len) return false;
|
||||
if (needle.len == 0) return true;
|
||||
const end = haystack.len - needle.len + 1;
|
||||
for (0..end) |i| {
|
||||
if (std.ascii.eqlIgnoreCase(haystack[i..][0..needle.len], needle)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const NodeAndPage = struct { node: *DOMNode, page: *lp.Page };
|
||||
|
||||
fn resolveNodeAndPage(server: *Server, id: std.json.Value, node_id: CDPNode.Id) !NodeAndPage {
|
||||
const page = server.session.currentPage() orelse {
|
||||
try server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
return error.PageNotLoaded;
|
||||
};
|
||||
const node = server.node_registry.lookup_by_id.get(node_id) orelse {
|
||||
try server.sendError(id, .InvalidParams, "Node not found");
|
||||
return error.InvalidParams;
|
||||
};
|
||||
return .{ .node = node.dom, .page = page };
|
||||
}
|
||||
|
||||
fn ensurePage(server: *Server, id: std.json.Value, url: ?[:0]const u8, timeout: ?u32, waitUntil: ?lp.Config.WaitUntil) !*lp.Page {
|
||||
if (url) |u| {
|
||||
try performGoto(server, u, id, timeout, waitUntil);
|
||||
@@ -736,7 +980,7 @@ test "MCP - evaluate error reporting" {
|
||||
} }, out.written());
|
||||
}
|
||||
|
||||
test "MCP - Actions: click, fill, scroll" {
|
||||
test "MCP - Actions: click, fill, scroll, hover, press, selectOption, setChecked" {
|
||||
defer testing.reset();
|
||||
const aa = testing.arena_allocator;
|
||||
|
||||
@@ -797,7 +1041,67 @@ test "MCP - Actions: click, fill, scroll" {
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
// Evaluate assertions
|
||||
{
|
||||
// Test Hover
|
||||
const el = page.document.getElementById("hoverTarget", page).?.asNode();
|
||||
const el_id = (try server.node_registry.register(el)).id;
|
||||
var id_buf: [12]u8 = undefined;
|
||||
const id_str = std.fmt.bufPrint(&id_buf, "{d}", .{el_id}) catch unreachable;
|
||||
const msg = try std.mem.concat(aa, u8, &.{ "{\"jsonrpc\":\"2.0\",\"id\":5,\"method\":\"tools/call\",\"params\":{\"name\":\"hover\",\"arguments\":{\"backendNodeId\":", id_str, "}}}" });
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Hovered element") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Test Press
|
||||
const el = page.document.getElementById("keyTarget", page).?.asNode();
|
||||
const el_id = (try server.node_registry.register(el)).id;
|
||||
var id_buf: [12]u8 = undefined;
|
||||
const id_str = std.fmt.bufPrint(&id_buf, "{d}", .{el_id}) catch unreachable;
|
||||
const msg = try std.mem.concat(aa, u8, &.{ "{\"jsonrpc\":\"2.0\",\"id\":6,\"method\":\"tools/call\",\"params\":{\"name\":\"press\",\"arguments\":{\"key\":\"Enter\",\"backendNodeId\":", id_str, "}}}" });
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Pressed key") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Test SelectOption
|
||||
const el = page.document.getElementById("sel2", page).?.asNode();
|
||||
const el_id = (try server.node_registry.register(el)).id;
|
||||
var id_buf: [12]u8 = undefined;
|
||||
const id_str = std.fmt.bufPrint(&id_buf, "{d}", .{el_id}) catch unreachable;
|
||||
const msg = try std.mem.concat(aa, u8, &.{ "{\"jsonrpc\":\"2.0\",\"id\":7,\"method\":\"tools/call\",\"params\":{\"name\":\"selectOption\",\"arguments\":{\"backendNodeId\":", id_str, ",\"value\":\"b\"}}}" });
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Selected option") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Test SetChecked (checkbox)
|
||||
const el = page.document.getElementById("chk", page).?.asNode();
|
||||
const el_id = (try server.node_registry.register(el)).id;
|
||||
var id_buf: [12]u8 = undefined;
|
||||
const id_str = std.fmt.bufPrint(&id_buf, "{d}", .{el_id}) catch unreachable;
|
||||
const msg = try std.mem.concat(aa, u8, &.{ "{\"jsonrpc\":\"2.0\",\"id\":8,\"method\":\"tools/call\",\"params\":{\"name\":\"setChecked\",\"arguments\":{\"backendNodeId\":", id_str, ",\"checked\":true}}}" });
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "checked") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Test SetChecked (radio)
|
||||
const el = page.document.getElementById("rad", page).?.asNode();
|
||||
const el_id = (try server.node_registry.register(el)).id;
|
||||
var id_buf: [12]u8 = undefined;
|
||||
const id_str = std.fmt.bufPrint(&id_buf, "{d}", .{el_id}) catch unreachable;
|
||||
const msg = try std.mem.concat(aa, u8, &.{ "{\"jsonrpc\":\"2.0\",\"id\":9,\"method\":\"tools/call\",\"params\":{\"name\":\"setChecked\",\"arguments\":{\"backendNodeId\":", id_str, ",\"checked\":true}}}" });
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "checked") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
// Evaluate JS assertions for all actions
|
||||
var ls: js.Local.Scope = undefined;
|
||||
page.js.localScope(&ls);
|
||||
defer ls.deinit();
|
||||
@@ -809,12 +1113,66 @@ test "MCP - Actions: click, fill, scroll" {
|
||||
const result = try ls.local.exec(
|
||||
\\ window.clicked === true && window.inputVal === 'hello' &&
|
||||
\\ window.changed === true && window.selChanged === 'opt2' &&
|
||||
\\ window.scrolled === true
|
||||
\\ window.scrolled === true &&
|
||||
\\ window.hovered === true &&
|
||||
\\ window.keyPressed === 'Enter' && window.keyReleased === 'Enter' &&
|
||||
\\ window.sel2Changed === 'b' &&
|
||||
\\ window.chkClicked === true && window.chkChanged === true &&
|
||||
\\ window.radClicked === true && window.radChanged === true
|
||||
, null);
|
||||
|
||||
try testing.expect(result.isTrue());
|
||||
}
|
||||
|
||||
test "MCP - findElement" {
|
||||
defer testing.reset();
|
||||
const aa = testing.arena_allocator;
|
||||
|
||||
var out: std.io.Writer.Allocating = .init(aa);
|
||||
const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_actions.html", &out.writer);
|
||||
defer server.deinit();
|
||||
|
||||
{
|
||||
// Find by role
|
||||
const msg =
|
||||
\\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"findElement","arguments":{"role":"button"}}}
|
||||
;
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Click Me") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Find by name (case-insensitive substring)
|
||||
const msg =
|
||||
\\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"findElement","arguments":{"name":"click"}}}
|
||||
;
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Click Me") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Find with no matches
|
||||
const msg =
|
||||
\\{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"findElement","arguments":{"role":"slider"}}}
|
||||
;
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "[]") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
// Error: no params provided
|
||||
const msg =
|
||||
\\{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"findElement","arguments":{}}}
|
||||
;
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "error") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
test "MCP - waitForSelector: existing element" {
|
||||
defer testing.reset();
|
||||
var out: std.io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
const std = @import("std");
|
||||
const log = @import("../log.zig");
|
||||
const builtin = @import("builtin");
|
||||
const net = std.net;
|
||||
const posix = std.posix;
|
||||
@@ -30,6 +31,10 @@ const http = @import("http.zig");
|
||||
const RobotStore = @import("Robots.zig").RobotStore;
|
||||
const WebBotAuth = @import("WebBotAuth.zig");
|
||||
|
||||
const Cache = @import("cache/Cache.zig");
|
||||
const FsCache = @import("cache/FsCache.zig");
|
||||
|
||||
const App = @import("../App.zig");
|
||||
const Network = @This();
|
||||
|
||||
const Listener = struct {
|
||||
@@ -45,10 +50,12 @@ const MAX_TICK_CALLBACKS = 16;
|
||||
|
||||
allocator: Allocator,
|
||||
|
||||
app: *App,
|
||||
config: *const Config,
|
||||
ca_blob: ?http.Blob,
|
||||
robot_store: RobotStore,
|
||||
web_bot_auth: ?WebBotAuth,
|
||||
cache: ?Cache,
|
||||
|
||||
connections: []http.Connection,
|
||||
available: std.DoublyLinkedList = .{},
|
||||
@@ -200,7 +207,7 @@ fn globalDeinit() void {
|
||||
libcurl.curl_global_cleanup();
|
||||
}
|
||||
|
||||
pub fn init(allocator: Allocator, config: *const Config) !Network {
|
||||
pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
|
||||
globalInit(allocator);
|
||||
errdefer globalDeinit();
|
||||
|
||||
@@ -233,6 +240,22 @@ pub fn init(allocator: Allocator, config: *const Config) !Network {
|
||||
else
|
||||
null;
|
||||
|
||||
const cache = if (config.httpCacheDir()) |cache_dir_path|
|
||||
Cache{
|
||||
.kind = .{
|
||||
.fs = FsCache.init(cache_dir_path) catch |e| {
|
||||
log.err(.cache, "failed to init", .{
|
||||
.kind = "FsCache",
|
||||
.path = cache_dir_path,
|
||||
.err = e,
|
||||
});
|
||||
return e;
|
||||
},
|
||||
},
|
||||
}
|
||||
else
|
||||
null;
|
||||
|
||||
return .{
|
||||
.allocator = allocator,
|
||||
.config = config,
|
||||
@@ -244,8 +267,10 @@ pub fn init(allocator: Allocator, config: *const Config) !Network {
|
||||
.available = available,
|
||||
.connections = connections,
|
||||
|
||||
.app = app,
|
||||
.robot_store = RobotStore.init(allocator),
|
||||
.web_bot_auth = web_bot_auth,
|
||||
.cache = cache,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -278,6 +303,8 @@ pub fn deinit(self: *Network) void {
|
||||
wba.deinit(self.allocator);
|
||||
}
|
||||
|
||||
if (self.cache) |*cache| cache.deinit();
|
||||
|
||||
globalDeinit();
|
||||
}
|
||||
|
||||
|
||||
213
src/network/cache/Cache.zig
vendored
Normal file
213
src/network/cache/Cache.zig
vendored
Normal file
@@ -0,0 +1,213 @@
|
||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||
//
|
||||
// Francis Bouvier <francis@lightpanda.io>
|
||||
// Pierre Tachoire <pierre@lightpanda.io>
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
const std = @import("std");
|
||||
const log = @import("../../log.zig");
|
||||
const Http = @import("../http.zig");
|
||||
const FsCache = @import("FsCache.zig");
|
||||
|
||||
/// A browser-wide cache for resources across the network.
|
||||
/// This mostly conforms to RFC9111 with regards to caching behavior.
|
||||
pub const Cache = @This();
|
||||
|
||||
kind: union(enum) {
|
||||
fs: FsCache,
|
||||
},
|
||||
|
||||
pub fn deinit(self: *Cache) void {
|
||||
return switch (self.kind) {
|
||||
inline else => |*c| c.deinit(),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse {
|
||||
return switch (self.kind) {
|
||||
inline else => |*c| c.get(arena, req),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void {
|
||||
return switch (self.kind) {
|
||||
inline else => |*c| c.put(metadata, body),
|
||||
};
|
||||
}
|
||||
|
||||
pub const CacheControl = struct {
|
||||
max_age: u64,
|
||||
|
||||
pub fn parse(value: []const u8) ?CacheControl {
|
||||
var cc: CacheControl = .{ .max_age = undefined };
|
||||
|
||||
var max_age_set = false;
|
||||
var max_s_age_set = false;
|
||||
var is_public = false;
|
||||
|
||||
var iter = std.mem.splitScalar(u8, value, ',');
|
||||
while (iter.next()) |part| {
|
||||
const directive = std.mem.trim(u8, part, &std.ascii.whitespace);
|
||||
if (std.ascii.eqlIgnoreCase(directive, "no-store")) {
|
||||
return null;
|
||||
} else if (std.ascii.eqlIgnoreCase(directive, "no-cache")) {
|
||||
return null;
|
||||
} else if (std.ascii.eqlIgnoreCase(directive, "public")) {
|
||||
is_public = true;
|
||||
} else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) {
|
||||
if (!max_s_age_set) {
|
||||
if (std.fmt.parseInt(u64, directive[8..], 10) catch null) |max_age| {
|
||||
cc.max_age = max_age;
|
||||
max_age_set = true;
|
||||
}
|
||||
}
|
||||
} else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) {
|
||||
if (std.fmt.parseInt(u64, directive[9..], 10) catch null) |max_age| {
|
||||
cc.max_age = max_age;
|
||||
max_age_set = true;
|
||||
max_s_age_set = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!max_age_set) return null;
|
||||
if (!is_public) return null;
|
||||
if (cc.max_age == 0) return null;
|
||||
|
||||
return cc;
|
||||
}
|
||||
};
|
||||
|
||||
pub const CachedMetadata = struct {
|
||||
url: [:0]const u8,
|
||||
content_type: []const u8,
|
||||
|
||||
status: u16,
|
||||
stored_at: i64,
|
||||
age_at_store: u64,
|
||||
|
||||
cache_control: CacheControl,
|
||||
/// Response Headers
|
||||
headers: []const Http.Header,
|
||||
|
||||
/// These are Request Headers used by Vary.
|
||||
vary_headers: []const Http.Header,
|
||||
|
||||
pub fn format(self: CachedMetadata, writer: *std.Io.Writer) !void {
|
||||
try writer.print("url={s} | status={d} | content_type={s} | max_age={d} | vary=[", .{
|
||||
self.url,
|
||||
self.status,
|
||||
self.content_type,
|
||||
self.cache_control.max_age,
|
||||
});
|
||||
|
||||
// Logging all headers gets pretty verbose...
|
||||
// so we just log the Vary ones that matter for caching.
|
||||
|
||||
if (self.vary_headers.len > 0) {
|
||||
for (self.vary_headers, 0..) |hdr, i| {
|
||||
if (i > 0) try writer.print(", ", .{});
|
||||
try writer.print("{s}: {s}", .{ hdr.name, hdr.value });
|
||||
}
|
||||
}
|
||||
try writer.print("]", .{});
|
||||
}
|
||||
};
|
||||
|
||||
pub const CacheRequest = struct {
|
||||
url: []const u8,
|
||||
timestamp: i64,
|
||||
request_headers: []const Http.Header,
|
||||
};
|
||||
|
||||
pub const CachedData = union(enum) {
|
||||
buffer: []const u8,
|
||||
file: struct {
|
||||
file: std.fs.File,
|
||||
offset: usize,
|
||||
len: usize,
|
||||
},
|
||||
|
||||
pub fn format(self: CachedData, writer: *std.Io.Writer) !void {
|
||||
switch (self) {
|
||||
.buffer => |buf| try writer.print("buffer({d} bytes)", .{buf.len}),
|
||||
.file => |f| try writer.print("file(offset={d}, len={d} bytes)", .{ f.offset, f.len }),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
pub const CachedResponse = struct {
|
||||
metadata: CachedMetadata,
|
||||
data: CachedData,
|
||||
|
||||
pub fn format(self: *const CachedResponse, writer: *std.Io.Writer) !void {
|
||||
try writer.print("metadata=(", .{});
|
||||
try self.metadata.format(writer);
|
||||
try writer.print("), data=", .{});
|
||||
try self.data.format(writer);
|
||||
}
|
||||
};
|
||||
|
||||
pub fn tryCache(
|
||||
arena: std.mem.Allocator,
|
||||
timestamp: i64,
|
||||
url: [:0]const u8,
|
||||
status: u16,
|
||||
content_type: ?[]const u8,
|
||||
cache_control: ?[]const u8,
|
||||
vary: ?[]const u8,
|
||||
age: ?[]const u8,
|
||||
has_set_cookie: bool,
|
||||
has_authorization: bool,
|
||||
) !?CachedMetadata {
|
||||
if (status != 200) {
|
||||
log.debug(.cache, "no store", .{ .url = url, .code = status, .reason = "status" });
|
||||
return null;
|
||||
}
|
||||
if (has_set_cookie) {
|
||||
log.debug(.cache, "no store", .{ .url = url, .reason = "has_cookies" });
|
||||
return null;
|
||||
}
|
||||
if (has_authorization) {
|
||||
log.debug(.cache, "no store", .{ .url = url, .reason = "has_authorization" });
|
||||
return null;
|
||||
}
|
||||
if (vary) |v| if (std.mem.eql(u8, v, "*")) {
|
||||
log.debug(.cache, "no store", .{ .url = url, .vary = v, .reason = "vary" });
|
||||
return null;
|
||||
};
|
||||
const cc = blk: {
|
||||
if (cache_control == null) {
|
||||
log.debug(.cache, "no store", .{ .url = url, .reason = "no cache control" });
|
||||
return null;
|
||||
}
|
||||
if (CacheControl.parse(cache_control.?)) |cc| {
|
||||
break :blk cc;
|
||||
}
|
||||
log.debug(.cache, "no store", .{ .url = url, .cache_control = cache_control.?, .reason = "cache control" });
|
||||
return null;
|
||||
};
|
||||
|
||||
return .{
|
||||
.url = try arena.dupeZ(u8, url),
|
||||
.content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",
|
||||
.status = status,
|
||||
.stored_at = timestamp,
|
||||
.age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0,
|
||||
.cache_control = cc,
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{},
|
||||
};
|
||||
}
|
||||
612
src/network/cache/FsCache.zig
vendored
Normal file
612
src/network/cache/FsCache.zig
vendored
Normal file
@@ -0,0 +1,612 @@
|
||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||
//
|
||||
// Francis Bouvier <francis@lightpanda.io>
|
||||
// Pierre Tachoire <pierre@lightpanda.io>
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
const std = @import("std");
|
||||
const log = @import("../../log.zig");
|
||||
const Cache = @import("Cache.zig");
|
||||
const Http = @import("../http.zig");
|
||||
const CacheRequest = Cache.CacheRequest;
|
||||
const CachedMetadata = Cache.CachedMetadata;
|
||||
const CachedResponse = Cache.CachedResponse;
|
||||
|
||||
const CACHE_VERSION: usize = 1;
|
||||
const LOCK_STRIPES = 16;
|
||||
comptime {
|
||||
std.debug.assert(std.math.isPowerOfTwo(LOCK_STRIPES));
|
||||
}
|
||||
|
||||
pub const FsCache = @This();
|
||||
|
||||
dir: std.fs.Dir,
|
||||
locks: [LOCK_STRIPES]std.Thread.Mutex = .{std.Thread.Mutex{}} ** LOCK_STRIPES,
|
||||
|
||||
const CacheMetadataJson = struct {
|
||||
version: usize,
|
||||
metadata: CachedMetadata,
|
||||
};
|
||||
|
||||
fn getLockPtr(self: *FsCache, key: *const [HASHED_KEY_LEN]u8) *std.Thread.Mutex {
|
||||
const lock_idx = std.hash.Wyhash.hash(0, key[0..]) & (LOCK_STRIPES - 1);
|
||||
return &self.locks[lock_idx];
|
||||
}
|
||||
|
||||
const BODY_LEN_HEADER_LEN = 8;
|
||||
const HASHED_KEY_LEN = 64;
|
||||
const HASHED_PATH_LEN = HASHED_KEY_LEN + 6;
|
||||
const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4;
|
||||
|
||||
fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 {
|
||||
var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
|
||||
std.crypto.hash.sha2.Sha256.hash(key, &digest, .{});
|
||||
var hex: [HASHED_KEY_LEN]u8 = undefined;
|
||||
_ = std.fmt.bufPrint(&hex, "{s}", .{std.fmt.bytesToHex(&digest, .lower)}) catch unreachable;
|
||||
return hex;
|
||||
}
|
||||
|
||||
fn cachePath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 {
|
||||
var path: [HASHED_PATH_LEN]u8 = undefined;
|
||||
_ = std.fmt.bufPrint(&path, "{s}.cache", .{hashed_key}) catch unreachable;
|
||||
return path;
|
||||
}
|
||||
|
||||
fn cacheTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {
|
||||
var path: [HASHED_TMP_PATH_LEN]u8 = undefined;
|
||||
_ = std.fmt.bufPrint(&path, "{s}.cache.tmp", .{hashed_key}) catch unreachable;
|
||||
return path;
|
||||
}
|
||||
|
||||
pub fn init(path: []const u8) !FsCache {
|
||||
const cwd = std.fs.cwd();
|
||||
|
||||
cwd.makeDir(path) catch |err| switch (err) {
|
||||
error.PathAlreadyExists => {},
|
||||
else => return err,
|
||||
};
|
||||
|
||||
const dir = try cwd.openDir(path, .{ .iterate = true });
|
||||
return .{ .dir = dir };
|
||||
}
|
||||
|
||||
pub fn deinit(self: *FsCache) void {
|
||||
self.dir.close();
|
||||
}
|
||||
|
||||
pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.CachedResponse {
|
||||
const hashed_key = hashKey(req.url);
|
||||
const cache_p = cachePath(&hashed_key);
|
||||
|
||||
const lock = self.getLockPtr(&hashed_key);
|
||||
lock.lock();
|
||||
defer lock.unlock();
|
||||
|
||||
const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch |e| {
|
||||
switch (e) {
|
||||
std.fs.File.OpenError.FileNotFound => {
|
||||
log.debug(.cache, "miss", .{ .url = req.url, .hash = &hashed_key, .reason = "missing" });
|
||||
},
|
||||
else => |err| {
|
||||
log.warn(.cache, "open file err", .{ .url = req.url, .err = err });
|
||||
},
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
var cleanup = false;
|
||||
defer if (cleanup) {
|
||||
file.close();
|
||||
self.dir.deleteFile(&cache_p) catch |e| {
|
||||
log.err(.cache, "clean fail", .{ .url = req.url, .file = &cache_p, .err = e });
|
||||
};
|
||||
};
|
||||
|
||||
var file_buf: [1024]u8 = undefined;
|
||||
var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined;
|
||||
|
||||
var file_reader = file.reader(&file_buf);
|
||||
const file_reader_iface = &file_reader.interface;
|
||||
|
||||
file_reader_iface.readSliceAll(&len_buf) catch |e| {
|
||||
log.warn(.cache, "read header", .{ .url = req.url, .err = e });
|
||||
cleanup = true;
|
||||
return null;
|
||||
};
|
||||
const body_len = std.mem.readInt(u64, &len_buf, .little);
|
||||
|
||||
// Now we read metadata.
|
||||
file_reader.seekTo(body_len + BODY_LEN_HEADER_LEN) catch |e| {
|
||||
log.warn(.cache, "seek metadata", .{ .url = req.url, .err = e });
|
||||
cleanup = true;
|
||||
return null;
|
||||
};
|
||||
|
||||
var json_reader = std.json.Reader.init(arena, file_reader_iface);
|
||||
const cache_file: CacheMetadataJson = std.json.parseFromTokenSourceLeaky(
|
||||
CacheMetadataJson,
|
||||
arena,
|
||||
&json_reader,
|
||||
.{ .allocate = .alloc_always },
|
||||
) catch |e| {
|
||||
// Warn because malformed metadata can be a deeper symptom.
|
||||
log.warn(.cache, "miss", .{ .url = req.url, .err = e, .reason = "malformed metadata" });
|
||||
cleanup = true;
|
||||
return null;
|
||||
};
|
||||
|
||||
if (cache_file.version != CACHE_VERSION) {
|
||||
log.debug(.cache, "miss", .{
|
||||
.url = req.url,
|
||||
.reason = "version mismatch",
|
||||
.expected = CACHE_VERSION,
|
||||
.got = cache_file.version,
|
||||
});
|
||||
cleanup = true;
|
||||
return null;
|
||||
}
|
||||
|
||||
const metadata = cache_file.metadata;
|
||||
|
||||
// Check entry expiration.
|
||||
const now = req.timestamp;
|
||||
const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store));
|
||||
if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) {
|
||||
log.debug(.cache, "miss", .{ .url = req.url, .reason = "expired" });
|
||||
cleanup = true;
|
||||
return null;
|
||||
}
|
||||
|
||||
// If we have Vary headers, ensure they are present & matching.
|
||||
for (metadata.vary_headers) |vary_hdr| {
|
||||
const name = vary_hdr.name;
|
||||
const value = vary_hdr.value;
|
||||
|
||||
const incoming = for (req.request_headers) |h| {
|
||||
if (std.ascii.eqlIgnoreCase(h.name, name)) break h.value;
|
||||
} else "";
|
||||
|
||||
if (!std.ascii.eqlIgnoreCase(value, incoming)) {
|
||||
log.debug(.cache, "miss", .{
|
||||
.url = req.url,
|
||||
.reason = "vary mismatch",
|
||||
.header = name,
|
||||
.expected = value,
|
||||
.got = incoming,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// On the case of a hash collision.
|
||||
if (!std.ascii.eqlIgnoreCase(metadata.url, req.url)) {
|
||||
log.warn(.cache, "collision", .{ .url = req.url, .expected = metadata.url, .got = req.url });
|
||||
cleanup = true;
|
||||
return null;
|
||||
}
|
||||
|
||||
log.debug(.cache, "hit", .{ .url = req.url, .hash = &hashed_key });
|
||||
|
||||
return .{
|
||||
.metadata = metadata,
|
||||
.data = .{
|
||||
.file = .{
|
||||
.file = file,
|
||||
.offset = BODY_LEN_HEADER_LEN,
|
||||
.len = body_len,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void {
|
||||
const hashed_key = hashKey(meta.url);
|
||||
const cache_p = cachePath(&hashed_key);
|
||||
const cache_tmp_p = cacheTmpPath(&hashed_key);
|
||||
|
||||
const lock = self.getLockPtr(&hashed_key);
|
||||
lock.lock();
|
||||
defer lock.unlock();
|
||||
|
||||
const file = self.dir.createFile(&cache_tmp_p, .{ .truncate = true }) catch |e| {
|
||||
log.err(.cache, "create file", .{ .url = meta.url, .file = &cache_tmp_p, .err = e });
|
||||
return e;
|
||||
};
|
||||
defer file.close();
|
||||
|
||||
var writer_buf: [1024]u8 = undefined;
|
||||
var file_writer = file.writer(&writer_buf);
|
||||
var file_writer_iface = &file_writer.interface;
|
||||
|
||||
var len_buf: [8]u8 = undefined;
|
||||
std.mem.writeInt(u64, &len_buf, body.len, .little);
|
||||
|
||||
file_writer_iface.writeAll(&len_buf) catch |e| {
|
||||
log.err(.cache, "write body len", .{ .url = meta.url, .err = e });
|
||||
return e;
|
||||
};
|
||||
file_writer_iface.writeAll(body) catch |e| {
|
||||
log.err(.cache, "write body", .{ .url = meta.url, .err = e });
|
||||
return e;
|
||||
};
|
||||
std.json.Stringify.value(
|
||||
CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta },
|
||||
.{ .whitespace = .minified },
|
||||
file_writer_iface,
|
||||
) catch |e| {
|
||||
log.err(.cache, "write metadata", .{ .url = meta.url, .err = e });
|
||||
return e;
|
||||
};
|
||||
file_writer_iface.flush() catch |e| {
|
||||
log.err(.cache, "flush", .{ .url = meta.url, .err = e });
|
||||
return e;
|
||||
};
|
||||
self.dir.rename(&cache_tmp_p, &cache_p) catch |e| {
|
||||
log.err(.cache, "rename", .{ .url = meta.url, .from = &cache_tmp_p, .to = &cache_p, .err = e });
|
||||
return e;
|
||||
};
|
||||
|
||||
log.debug(.cache, "put", .{ .url = meta.url, .hash = &hashed_key, .body_len = body.len });
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
fn setupCache() !struct { tmp: testing.TmpDir, cache: Cache } {
|
||||
var tmp = testing.tmpDir(.{});
|
||||
errdefer tmp.cleanup();
|
||||
|
||||
const path = try tmp.dir.realpathAlloc(testing.allocator, ".");
|
||||
defer testing.allocator.free(path);
|
||||
|
||||
return .{
|
||||
.tmp = tmp,
|
||||
.cache = Cache{ .kind = .{ .fs = try FsCache.init(path) } },
|
||||
};
|
||||
}
|
||||
|
||||
test "FsCache: basic put and get" {
|
||||
var setup = try setupCache();
|
||||
defer {
|
||||
setup.cache.deinit();
|
||||
setup.tmp.cleanup();
|
||||
}
|
||||
|
||||
const cache = &setup.cache;
|
||||
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const now = std.time.timestamp();
|
||||
const meta = CachedMetadata{
|
||||
.url = "https://example.com",
|
||||
.content_type = "text/html",
|
||||
.status = 200,
|
||||
.stored_at = now,
|
||||
.age_at_store = 0,
|
||||
.cache_control = .{ .max_age = 600 },
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{},
|
||||
};
|
||||
|
||||
const body = "hello world";
|
||||
try cache.put(meta, body);
|
||||
|
||||
const result = cache.get(
|
||||
arena.allocator(),
|
||||
.{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{},
|
||||
},
|
||||
) orelse return error.CacheMiss;
|
||||
const f = result.data.file;
|
||||
const file = f.file;
|
||||
defer file.close();
|
||||
|
||||
var buf: [64]u8 = undefined;
|
||||
var file_reader = file.reader(&buf);
|
||||
try file_reader.seekTo(f.offset);
|
||||
|
||||
const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len);
|
||||
defer testing.allocator.free(read_buf);
|
||||
try testing.expectEqualStrings(body, read_buf);
|
||||
}
|
||||
|
||||
test "FsCache: get expiration" {
|
||||
var setup = try setupCache();
|
||||
defer {
|
||||
setup.cache.deinit();
|
||||
setup.tmp.cleanup();
|
||||
}
|
||||
|
||||
const cache = &setup.cache;
|
||||
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const now = 5000;
|
||||
const max_age = 1000;
|
||||
|
||||
const meta = CachedMetadata{
|
||||
.url = "https://example.com",
|
||||
.content_type = "text/html",
|
||||
.status = 200,
|
||||
.stored_at = now,
|
||||
.age_at_store = 900,
|
||||
.cache_control = .{ .max_age = max_age },
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{},
|
||||
};
|
||||
|
||||
const body = "hello world";
|
||||
try cache.put(meta, body);
|
||||
|
||||
const result = cache.get(
|
||||
arena.allocator(),
|
||||
.{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now + 50,
|
||||
.request_headers = &.{},
|
||||
},
|
||||
) orelse return error.CacheMiss;
|
||||
result.data.file.file.close();
|
||||
|
||||
try testing.expectEqual(null, cache.get(
|
||||
arena.allocator(),
|
||||
.{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now + 200,
|
||||
.request_headers = &.{},
|
||||
},
|
||||
));
|
||||
|
||||
try testing.expectEqual(null, cache.get(
|
||||
arena.allocator(),
|
||||
.{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{},
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
test "FsCache: put override" {
|
||||
var setup = try setupCache();
|
||||
defer {
|
||||
setup.cache.deinit();
|
||||
setup.tmp.cleanup();
|
||||
}
|
||||
|
||||
const cache = &setup.cache;
|
||||
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
{
|
||||
const now = 5000;
|
||||
const max_age = 1000;
|
||||
|
||||
const meta = CachedMetadata{
|
||||
.url = "https://example.com",
|
||||
.content_type = "text/html",
|
||||
.status = 200,
|
||||
.stored_at = now,
|
||||
.age_at_store = 900,
|
||||
.cache_control = .{ .max_age = max_age },
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{},
|
||||
};
|
||||
|
||||
const body = "hello world";
|
||||
try cache.put(meta, body);
|
||||
|
||||
const result = cache.get(
|
||||
arena.allocator(),
|
||||
.{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{},
|
||||
},
|
||||
) orelse return error.CacheMiss;
|
||||
const f = result.data.file;
|
||||
const file = f.file;
|
||||
defer file.close();
|
||||
|
||||
var buf: [64]u8 = undefined;
|
||||
var file_reader = file.reader(&buf);
|
||||
try file_reader.seekTo(f.offset);
|
||||
|
||||
const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len);
|
||||
defer testing.allocator.free(read_buf);
|
||||
|
||||
try testing.expectEqualStrings(body, read_buf);
|
||||
}
|
||||
|
||||
{
|
||||
const now = 10000;
|
||||
const max_age = 2000;
|
||||
|
||||
const meta = CachedMetadata{
|
||||
.url = "https://example.com",
|
||||
.content_type = "text/html",
|
||||
.status = 200,
|
||||
.stored_at = now,
|
||||
.age_at_store = 0,
|
||||
.cache_control = .{ .max_age = max_age },
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{},
|
||||
};
|
||||
|
||||
const body = "goodbye world";
|
||||
try cache.put(meta, body);
|
||||
|
||||
const result = cache.get(
|
||||
arena.allocator(),
|
||||
.{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{},
|
||||
},
|
||||
) orelse return error.CacheMiss;
|
||||
const f = result.data.file;
|
||||
const file = f.file;
|
||||
defer file.close();
|
||||
|
||||
var buf: [64]u8 = undefined;
|
||||
var file_reader = file.reader(&buf);
|
||||
try file_reader.seekTo(f.offset);
|
||||
|
||||
const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len);
|
||||
defer testing.allocator.free(read_buf);
|
||||
|
||||
try testing.expectEqualStrings(body, read_buf);
|
||||
}
|
||||
}
|
||||
|
||||
test "FsCache: garbage file" {
|
||||
var setup = try setupCache();
|
||||
defer {
|
||||
setup.cache.deinit();
|
||||
setup.tmp.cleanup();
|
||||
}
|
||||
|
||||
const hashed_key = hashKey("https://example.com");
|
||||
const cache_p = cachePath(&hashed_key);
|
||||
const file = try setup.cache.kind.fs.dir.createFile(&cache_p, .{});
|
||||
try file.writeAll("this is not a valid cache file !@#$%");
|
||||
file.close();
|
||||
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
try testing.expectEqual(
|
||||
null,
|
||||
setup.cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = 5000,
|
||||
.request_headers = &.{},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
test "FsCache: vary hit and miss" {
|
||||
var setup = try setupCache();
|
||||
defer {
|
||||
setup.cache.deinit();
|
||||
setup.tmp.cleanup();
|
||||
}
|
||||
|
||||
const cache = &setup.cache;
|
||||
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const now = std.time.timestamp();
|
||||
const meta = CachedMetadata{
|
||||
.url = "https://example.com",
|
||||
.content_type = "text/html",
|
||||
.status = 200,
|
||||
.stored_at = now,
|
||||
.age_at_store = 0,
|
||||
.cache_control = .{ .max_age = 600 },
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "gzip" },
|
||||
},
|
||||
};
|
||||
|
||||
try cache.put(meta, "hello world");
|
||||
|
||||
const result = cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "gzip" },
|
||||
},
|
||||
}) orelse return error.CacheMiss;
|
||||
result.data.file.file.close();
|
||||
|
||||
try testing.expectEqual(null, cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "br" },
|
||||
},
|
||||
}));
|
||||
|
||||
try testing.expectEqual(null, cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{},
|
||||
}));
|
||||
|
||||
const result2 = cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "gzip" },
|
||||
},
|
||||
}) orelse return error.CacheMiss;
|
||||
result2.data.file.file.close();
|
||||
}
|
||||
|
||||
test "FsCache: vary multiple headers" {
|
||||
var setup = try setupCache();
|
||||
defer {
|
||||
setup.cache.deinit();
|
||||
setup.tmp.cleanup();
|
||||
}
|
||||
|
||||
const cache = &setup.cache;
|
||||
|
||||
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const now = std.time.timestamp();
|
||||
const meta = CachedMetadata{
|
||||
.url = "https://example.com",
|
||||
.content_type = "text/html",
|
||||
.status = 200,
|
||||
.stored_at = now,
|
||||
.age_at_store = 0,
|
||||
.cache_control = .{ .max_age = 600 },
|
||||
.headers = &.{},
|
||||
.vary_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "gzip" },
|
||||
.{ .name = "Accept-Language", .value = "en" },
|
||||
},
|
||||
};
|
||||
|
||||
try cache.put(meta, "hello world");
|
||||
|
||||
const result = cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "gzip" },
|
||||
.{ .name = "Accept-Language", .value = "en" },
|
||||
},
|
||||
}) orelse return error.CacheMiss;
|
||||
result.data.file.file.close();
|
||||
|
||||
try testing.expectEqual(null, cache.get(arena.allocator(), .{
|
||||
.url = "https://example.com",
|
||||
.timestamp = now,
|
||||
.request_headers = &.{
|
||||
.{ .name = "Accept-Encoding", .value = "gzip" },
|
||||
.{ .name = "Accept-Language", .value = "fr" },
|
||||
},
|
||||
}));
|
||||
}
|
||||
@@ -79,7 +79,7 @@ pub const Headers = struct {
|
||||
self.headers = updated_headers;
|
||||
}
|
||||
|
||||
fn parseHeader(header_str: []const u8) ?Header {
|
||||
pub fn parseHeader(header_str: []const u8) ?Header {
|
||||
const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
|
||||
|
||||
const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
|
||||
@@ -88,22 +88,9 @@ pub const Headers = struct {
|
||||
return .{ .name = name, .value = value };
|
||||
}
|
||||
|
||||
pub fn iterator(self: *Headers) Iterator {
|
||||
return .{
|
||||
.header = self.headers,
|
||||
};
|
||||
pub fn iterator(self: Headers) HeaderIterator {
|
||||
return .{ .curl_slist = .{ .header = self.headers } };
|
||||
}
|
||||
|
||||
const Iterator = struct {
|
||||
header: [*c]libcurl.CurlSList,
|
||||
|
||||
pub fn next(self: *Iterator) ?Header {
|
||||
const h = self.header orelse return null;
|
||||
|
||||
self.header = h.*.next;
|
||||
return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
// In normal cases, the header iterator comes from the curl linked list.
|
||||
@@ -112,6 +99,7 @@ pub const Headers = struct {
|
||||
// This union, is an iterator that exposes the same API for either case.
|
||||
pub const HeaderIterator = union(enum) {
|
||||
curl: CurlHeaderIterator,
|
||||
curl_slist: CurlSListIterator,
|
||||
list: ListHeaderIterator,
|
||||
|
||||
pub fn next(self: *HeaderIterator) ?Header {
|
||||
@@ -120,6 +108,19 @@ pub const HeaderIterator = union(enum) {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn collect(self: *HeaderIterator, allocator: std.mem.Allocator) !std.ArrayList(Header) {
|
||||
var list: std.ArrayList(Header) = .empty;
|
||||
|
||||
while (self.next()) |hdr| {
|
||||
try list.append(allocator, .{
|
||||
.name = try allocator.dupe(u8, hdr.name),
|
||||
.value = try allocator.dupe(u8, hdr.value),
|
||||
});
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
const CurlHeaderIterator = struct {
|
||||
conn: *const Connection,
|
||||
prev: ?*libcurl.CurlHeader = null,
|
||||
@@ -136,6 +137,16 @@ pub const HeaderIterator = union(enum) {
|
||||
}
|
||||
};
|
||||
|
||||
const CurlSListIterator = struct {
|
||||
header: [*c]libcurl.CurlSList,
|
||||
|
||||
pub fn next(self: *CurlSListIterator) ?Header {
|
||||
const h = self.header orelse return null;
|
||||
self.header = h.*.next;
|
||||
return Headers.parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));
|
||||
}
|
||||
};
|
||||
|
||||
const ListHeaderIterator = struct {
|
||||
index: usize = 0,
|
||||
list: []const Header,
|
||||
|
||||
13968
src/sqlite3.h
13968
src/sqlite3.h
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user