From 6b832815394e18c6d35a047408ab16ad046a0375 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 3 Mar 2025 19:49:24 +0800 Subject: [PATCH] Add navigate telemetry --- .github/workflows/e2e-test.yml | 1 + README.md | 3 +++ src/app.zig | 14 +++++++++++++- src/browser/browser.zig | 19 +++++++++++++------ src/cdp/cdp.zig | 11 ++++------- src/cdp/testing.zig | 12 +++++++----- src/main.zig | 15 ++++----------- src/server.zig | 10 +++++----- src/telemetry/lightpanda.zig | 2 +- src/telemetry/telemetry.zig | 2 +- src/unit_tests.zig | 14 +++++++------- 11 files changed, 59 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index fe7104a4..53427e09 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -64,6 +64,7 @@ jobs: env: MAX_MEMORY: 28000 MAX_AVG_DURATION: 24 + LIGHTPANDA_DISABLE_TELEMETRY: true runs-on: ubuntu-latest diff --git a/README.md b/README.md index 5fa92e53..b0b7d168 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ await context.close(); await browser.disconnect(); ``` +### Telemetry +By default, Lightpanda collects and sends usage telemetry. This can be disabled by setting an environment variable `LIGHTPANDA_DISABLE_TELEMETRY=true`. You can read Lightpanda's privacy policy at: [https://lightpanda.io/privacy-policy](https://lightpanda.io/privacy-policy). + ## Status Lightpanda is still a work in progress and is currently at a Beta stage. diff --git a/src/app.zig b/src/app.zig index 158ea5d5..ea08043d 100644 --- a/src/app.zig +++ b/src/app.zig @@ -7,18 +7,30 @@ const Telemetry = @import("telemetry/telemetry.zig").Telemetry; // Container for global state / objects that various parts of the system // might need. pub const App = struct { + loop: *Loop, + allocator: Allocator, telemetry: Telemetry, - pub fn init(allocator: Allocator, loop: *Loop) !App { + pub fn init(allocator: Allocator) !App { + const loop = try allocator.create(Loop); + errdefer allocator.destroy(loop); + + loop.* = try Loop.init(allocator); + errdefer loop.deinit(); + const telemetry = Telemetry.init(allocator, loop); errdefer telemetry.deinit(); return .{ + .loop = loop, + .allocator = allocator, .telemetry = telemetry, }; } pub fn deinit(self: *App) void { self.telemetry.deinit(); + self.loop.deinit(); + self.allocator.destroy(self.loop); } }; diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 4c1d6f52..c00ae52f 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -33,6 +33,7 @@ const Loop = jsruntime.Loop; const Env = jsruntime.Env; const Module = jsruntime.Module; +const App = @import("../app.zig").App; const apiweb = @import("../apiweb.zig"); const Window = @import("../html/window.zig").Window; @@ -59,7 +60,7 @@ pub const user_agent = "Lightpanda/1.0"; // A browser contains only one session. // TODO allow multiple sessions per browser. pub const Browser = struct { - loop: *Loop, + app: *App, session: ?*Session, allocator: Allocator, http_client: HttpClient, @@ -68,9 +69,10 @@ pub const Browser = struct { const SessionPool = std.heap.MemoryPool(Session); - pub fn init(allocator: Allocator, loop: *Loop) Browser { + pub fn init(app: *App) Browser { + const allocator = app.allocator; return .{ - .loop = loop, + .app = app, .session = null, .allocator = allocator, .http_client = .{ .allocator = allocator }, @@ -109,6 +111,8 @@ pub const Browser = struct { // You can create successively multiple pages for a session, but you must // deinit a page before running another one. pub const Session = struct { + app: *App, + browser: *Browser, // The arena is used only to bound the js env init b/c it leaks memory. @@ -133,8 +137,10 @@ pub const Session = struct { jstypes: [Types.len]usize = undefined, fn init(self: *Session, browser: *Browser, ctx: anytype) !void { - const allocator = browser.allocator; + const app = browser.app; + const allocator = app.allocator; self.* = .{ + .app = app, .env = undefined, .browser = browser, .inspector = undefined, @@ -145,7 +151,7 @@ pub const Session = struct { }; const arena = self.arena.allocator(); - Env.init(&self.env, arena, browser.loop, null); + Env.init(&self.env, arena, app.loop, null); errdefer self.env.deinit(); try self.env.load(&self.jstypes); @@ -238,7 +244,7 @@ pub const Session = struct { std.debug.assert(self.page != null); // Reset all existing callbacks. - self.browser.loop.reset(); + self.app.loop.reset(); self.env.stop(); // TODO unload document: https://html.spec.whatwg.org/#unloading-documents @@ -333,6 +339,7 @@ pub const Page = struct { // see Inspector.contextCreated pub fn navigate(self: *Page, uri: []const u8, aux_data: ?[]const u8) !void { const arena = self.arena; + self.session.app.telemetry.record(.{ .navigate = {} }); log.debug("starting GET {s}", .{uri}); diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 1543ba76..37da6e85 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -20,7 +20,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const json = std.json; -const Loop = @import("jsruntime").Loop; +const App = @import("../app.zig").App; const asUint = @import("../str/parser.zig").asUint; const Incrementing = @import("../id.zig").Incrementing; @@ -34,7 +34,6 @@ pub const TimestampEvent = struct { }; pub const CDP = CDPT(struct { - const Loop = *@import("jsruntime").Loop; const Client = *@import("../server.zig").Client; const Browser = @import("../browser/browser.zig").Browser; const Session = @import("../browser/browser.zig").Session; @@ -47,8 +46,6 @@ const BrowserContextIdGen = Incrementing(u32, "BID"); // Generic so that we can inject mocks into it. pub fn CDPT(comptime TypeProvider: type) type { return struct { - loop: TypeProvider.Loop, - // Used for sending message to the client and closing on error client: TypeProvider.Client, @@ -73,13 +70,13 @@ pub fn CDPT(comptime TypeProvider: type) type { pub const Browser = TypeProvider.Browser; pub const Session = TypeProvider.Session; - pub fn init(allocator: Allocator, client: TypeProvider.Client, loop: TypeProvider.Loop) Self { + pub fn init(app: *App, client: TypeProvider.Client) Self { + const allocator = app.allocator; return .{ - .loop = loop, .client = client, .allocator = allocator, .browser_context = null, - .browser = Browser.init(allocator, loop), + .browser = Browser.init(app), .message_arena = std.heap.ArenaAllocator.init(allocator), .browser_context_pool = std.heap.MemoryPool(BrowserContext(Self)).init(allocator), }; diff --git a/src/cdp/testing.zig b/src/cdp/testing.zig index d08d2e09..0e5b1508 100644 --- a/src/cdp/testing.zig +++ b/src/cdp/testing.zig @@ -7,6 +7,7 @@ const Testing = @This(); const main = @import("cdp.zig"); const parser = @import("netsurf"); +const App = @import("../app.zig").App; pub const expectEqual = std.testing.expectEqual; pub const expectError = std.testing.expectError; @@ -16,10 +17,9 @@ const Browser = struct { session: ?*Session = null, arena: std.heap.ArenaAllocator, - pub fn init(allocator: Allocator, loop: anytype) Browser { - _ = loop; + pub fn init(app: *App) Browser { return .{ - .arena = std.heap.ArenaAllocator.init(allocator), + .arena = std.heap.ArenaAllocator.init(app.allocator), }; } @@ -112,13 +112,13 @@ const Client = struct { }; const TestCDP = main.CDPT(struct { - pub const Loop = void; pub const Browser = Testing.Browser; pub const Session = Testing.Session; pub const Client = *Testing.Client; }); const TestContext = struct { + app: App, client: ?Client = null, cdp_: ?TestCDP = null, arena: std.heap.ArenaAllocator, @@ -127,6 +127,7 @@ const TestContext = struct { if (self.cdp_) |*c| { c.deinit(); } + self.app.deinit(); self.arena.deinit(); } @@ -135,7 +136,7 @@ const TestContext = struct { self.client = Client.init(self.arena.allocator()); // Don't use the arena here. We want to detect leaks in CDP. // The arena is only for test-specific stuff - self.cdp_ = TestCDP.init(std.testing.allocator, &self.client.?, {}); + self.cdp_ = TestCDP.init(&self.app, &self.client.?); } return &self.cdp_.?; } @@ -262,6 +263,7 @@ const TestContext = struct { pub fn context() TestContext { return .{ + .app = App.init(std.testing.allocator) catch unreachable, .arena = std.heap.ArenaAllocator.init(std.testing.allocator), }; } diff --git a/src/main.zig b/src/main.zig index 363ed585..d6dd955e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -70,15 +70,12 @@ pub fn main() !void { return args.printUsageAndExit(false); }; - var loop = try jsruntime.Loop.init(alloc); - defer loop.deinit(); - - var app = try @import("app.zig").App.init(alloc, &loop); + var app = try @import("app.zig").App.init(alloc); defer app.deinit(); app.telemetry.record(.{ .run = .{ .mode = .serve, .version = version } }); const timeout = std.time.ns_per_s * @as(u64, opts.timeout); - server.run(alloc, address, timeout, &loop, &app) catch |err| { + server.run(&app, address, timeout) catch |err| { log.err("Server error", .{}); return err; }; @@ -86,11 +83,7 @@ pub fn main() !void { .fetch => |opts| { log.debug("Fetch mode: url {s}, dump {any}", .{ opts.url, opts.dump }); - // loop - var loop = try jsruntime.Loop.init(alloc); - defer loop.deinit(); - - var app = try @import("app.zig").App.init(alloc, &loop); + var app = try @import("app.zig").App.init(alloc); defer app.deinit(); app.telemetry.record(.{ .run = .{ .mode = .fetch, .version = version } }); @@ -99,7 +92,7 @@ pub fn main() !void { defer vm.deinit(); // browser - var browser = Browser.init(alloc, &loop); + var browser = Browser.init(&app); defer browser.deinit(); var session = try browser.newSession({}); diff --git a/src/server.zig b/src/server.zig index 18bbd112..fb4869fd 100644 --- a/src/server.zig +++ b/src/server.zig @@ -72,7 +72,6 @@ const Server = struct { fn deinit(self: *Server) void { self.client_pool.deinit(); - self.allocator.free(self.json_version_response); } fn queueAccept(self: *Server) void { @@ -467,7 +466,7 @@ pub const Client = struct { }; self.mode = .websocket; - self.cdp = CDP.init(self.server.allocator, self, self.server.loop); + self.cdp = CDP.init(self.server.app, self); return self.send(arena, response); } @@ -1016,11 +1015,9 @@ fn websocketHeader(buf: []u8, op_code: OpCode, payload_len: usize) []const u8 { } pub fn run( - allocator: Allocator, + app: *App, address: net.Address, timeout: u64, - loop: *jsruntime.Loop, - app: *App, ) !void { // create socket const flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | posix.SOCK.NONBLOCK; @@ -1043,7 +1040,10 @@ pub fn run( const vm = jsruntime.VM.init(); defer vm.deinit(); + var loop = app.loop; + const allocator = app.allocator; const json_version_response = try buildJSONVersionResponse(allocator, address); + defer allocator.free(json_version_response); var server = Server{ .app = app, diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 32172cc8..0c16aeca 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -7,7 +7,7 @@ const Client = @import("asyncio").Client; const log = std.log.scoped(.telemetry); -const URL = "https://stats.lightpanda.io"; +const URL = "https://telemetry.lightpanda.io/"; pub const LightPanda = struct { uri: std.Uri, diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 4ca8a442..be9c911c 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.telemetry); const ID_FILE = "lightpanda.id"; pub const Telemetry = TelemetryT(blk: { - // if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; + if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; break :blk @import("lightpanda.zig").LightPanda; }); diff --git a/src/unit_tests.zig b/src/unit_tests.zig index 96335131..3d625251 100644 --- a/src/unit_tests.zig +++ b/src/unit_tests.zig @@ -45,10 +45,7 @@ pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; const allocator = gpa.allocator(); - var loop = try jsruntime.Loop.init(allocator); - defer loop.deinit(); - - var app = try App.init(allocator, &loop); + var app = try App.init(allocator); defer app.deinit(); const env = Env.init(allocator); @@ -71,7 +68,10 @@ pub fn main() !void { const cdp_thread = blk: { const address = try std.net.Address.parseIp("127.0.0.1", 9583); - const thread = try std.Thread.spawn(.{}, serveCDP, .{ allocator, address, &loop, &app }); + const thread = try std.Thread.spawn(.{}, serveCDP, .{ + &app, + address, + }); break :blk thread; }; defer cdp_thread.join(); @@ -353,9 +353,9 @@ fn serveHTTP(address: std.net.Address) !void { } } -fn serveCDP(allocator: Allocator, address: std.net.Address, loop: *jsruntime.Loop, app: *App) !void { +fn serveCDP(app: *App, address: std.net.Address) !void { const server = @import("server.zig"); - server.run(allocator, address, std.time.ns_per_s * 2, loop, app) catch |err| { + server.run(app, address, std.time.ns_per_s * 2) catch |err| { std.debug.print("CDP server error: {}", .{err}); return err; };