From 56ddcc8e2944108181a69b3a1d766378d1365ade Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 25 Feb 2025 12:40:01 +0800 Subject: [PATCH 01/17] Initial usage telemetry --- src/app.zig | 23 ++++++ src/main.zig | 8 +- src/telemetry/lightpanda.zig | 87 ++++++++++++++++++++++ src/telemetry/telemetry.zig | 140 +++++++++++++++++++++++++++++++++++ src/unit_tests.zig | 1 + 5 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 src/app.zig create mode 100644 src/telemetry/lightpanda.zig create mode 100644 src/telemetry/telemetry.zig diff --git a/src/app.zig b/src/app.zig new file mode 100644 index 00000000..a92f3b9f --- /dev/null +++ b/src/app.zig @@ -0,0 +1,23 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const Telemetry = @import("telemetry/telemetry.zig").Telemetry; + +// Container for global state / objects that various parts of the system +// might need. +pub const App = struct { + telemetry: Telemetry, + + pub fn init(allocator: Allocator) !App { + const telemetry = Telemetry.init(allocator); + errdefer telemetry.deinit(); + + return .{ + .telemetry = telemetry, + }; + } + + pub fn deinit(self: *App) void { + self.telemetry.deinit(); + } +}; diff --git a/src/main.zig b/src/main.zig index 624d0ce1..2aaf879c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -31,6 +31,7 @@ const apiweb = @import("apiweb.zig"); pub const Types = jsruntime.reflect(apiweb.Interfaces); pub const UserContext = apiweb.UserContext; pub const IO = @import("asyncio").Wrapper(jsruntime.Loop); +const version = @import("build_info").git_commit; const log = std.log.scoped(.cli); @@ -53,6 +54,9 @@ pub fn main() !void { _ = gpa.detectLeaks(); }; + var app = try @import("app.zig").App.init(alloc); + defer app.deinit(); + var args_arena = std.heap.ArenaAllocator.init(alloc); defer args_arena.deinit(); const args = try parseArgs(args_arena.allocator()); @@ -60,10 +64,11 @@ pub fn main() !void { switch (args.mode) { .help => args.printUsageAndExit(args.mode.help), .version => { - std.debug.print("{s}\n", .{@import("build_info").git_commit}); + std.debug.print("{s}\n", .{version}); return std.process.cleanExit(); }, .serve => |opts| { + app.telemetry.record(.{ .run = .{ .mode = .serve, .version = version } }); const address = std.net.Address.parseIp4(opts.host, opts.port) catch |err| { log.err("address (host:port) {any}\n", .{err}); return args.printUsageAndExit(false); @@ -79,6 +84,7 @@ pub fn main() !void { }; }, .fetch => |opts| { + app.telemetry.record(.{ .run = .{ .mode = .fetch, .version = version } }); log.debug("Fetch mode: url {s}, dump {any}", .{ opts.url, opts.dump }); // vm diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig new file mode 100644 index 00000000..b3f481a7 --- /dev/null +++ b/src/telemetry/lightpanda.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const ArenAallocator = std.heap.ArenaAllocator; + +const Event = @import("telemetry.zig").Event; +const log = std.log.scoped(.telemetry); + +const URL = "https://lightpanda.io/browser-stats"; + +pub const Lightpanda = struct { + uri: std.Uri, + arena: ArenAallocator, + client: std.http.Client, + headers: [1]std.http.Header, + + pub fn init(allocator: Allocator) !Lightpanda { + return .{ + .client = .{ .allocator = allocator }, + .arena = std.heap.ArenaAllocator.init(allocator), + .uri = std.Uri.parse(URL) catch unreachable, + .headers = [1]std.http.Header{ + .{ .name = "Content-Type", .value = "application/json" }, + }, + }; + } + + pub fn deinit(self: *Lightpanda) void { + self.arena.deinit(); + self.client.deinit(); + } + + pub fn send(self: *Lightpanda, iid: ?[]const u8, eid: []const u8, events: []Event) !void { + std.debug.print("SENDING: {s} {s} {d}", .{iid, eid, events.len}) + // defer _ = self.arena.reset(.{ .retain_capacity = {} }); + // const body = try std.json.stringifyAlloc(self.arena.allocator(), PlausibleEvent{ .event = event }, .{}); + + // var server_headers: [2048]u8 = undefined; + // var req = try self.client.open(.POST, self.uri, .{ + // .redirect_behavior = .not_allowed, + // .extra_headers = &self.headers, + // .server_header_buffer = &server_headers, + // }); + // req.transfer_encoding = .{ .content_length = body.len }; + // try req.send(); + + // try req.writeAll(body); + // try req.finish(); + // try req.wait(); + + // const status = req.response.status; + // if (status != .accepted) { + // log.warn("telemetry '{s}' event error: {d}", .{ @tagName(event), @intFromEnum(status) }); + // } else { + // log.warn("telemetry '{s}' sent", .{@tagName(event)}); + // } + } +}; + +// wraps a telemetry event so that we can serialize it to plausible's event endpoint +const PlausibleEvent = struct { + event: Event, + + pub fn jsonStringify(self: PlausibleEvent, jws: anytype) !void { + try jws.beginObject(); + try jws.objectField("name"); + try jws.write(@tagName(self.event)); + try jws.objectField("url"); + try jws.write(EVENT_URL); + try jws.objectField("domain"); + try jws.write(DOMAIN_KEY); + try jws.objectField("props"); + switch (self.event) { + inline else => |props| try jws.write(props), + } + try jws.endObject(); + } +}; + +const testing = std.testing; +test "plausible: json event" { + const json = try std.json.stringifyAlloc(testing.allocator, PlausibleEvent{ .event = .{ .run = .{ .mode = .serve, .version = "over 9000!" } } }, .{}); + defer testing.allocator.free(json); + + try testing.expectEqualStrings( + \\{"name":"run","url":"https://lightpanda.io/browser-stats","domain":"localhost","props":{"version":"over 9000!","mode":"serve"}} + , json); +} diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig new file mode 100644 index 00000000..90d55c15 --- /dev/null +++ b/src/telemetry/telemetry.zig @@ -0,0 +1,140 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const Allocator = std.mem.Allocator; +const uuidv4 = @import("../id.zig").uuidv4; + +const log = std.log.scoped(.telemetry); + +const BATCH_SIZE = 5; +const BATCH_END = BATCH_SIZE - 1; +const ID_FILE = "lightpanda.id"; + +pub const Telemetry = TelemetryT(blk: { + if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; + break :blk @import("ligtpanda.zig").Lightpanda; +}); + +fn TelemetryT(comptime P: type) type { + return struct { + // an "install" id that we [try to] persist and re-use between runs + // null on IO error + iid: ?[36]u8, + + // a "execution" id is an id that represents this specific run + eid: [36]u8, + provider: P, + + // batch of events, pending[0..count] are pending + pending: [BATCH_SIZE]Event, + count: usize, + disabled: bool, + + const Self = @This(); + + pub fn init(allocator: Allocator) Self { + const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); + + var eid: [36]u8 = undefined; + uuidv4(&eid) + + return .{ + .eid = eid, + .iid = if (disabled) null else getOrCreateId(), + .disabled = disabled, + .provider = try P.init(allocator), + }; + } + + pub fn deinit(self: *Self) void { + self.provider.deinit(); + } + + pub fn record(self: *Self, event: Event) void { + if (self.disabled) { + return; + } + + const count = self.count; + self.pending[count] = event; + if (count < BATCH_END) { + self.count = count + 1; + return; + } + + const iid = if (self.iid) |*iid| *iid else null; + self.provider.send(iid, &self.eid, &self.pending) catch |err| { + log.warn("failed to record event: {}", .{err}); + }; + self.count = 0; + } + }; +} + +fn getOrCreateId() ?[36]u8 { + var buf: [37]u8 = undefined; + const data = std.fs.cwd().readFile(ID_FILE, &buf) catch |err| switch (err) blk: { + error.FileNotFound => break :bkl &.{}, + else => { + log.warn("failed to open id file: {}", .{err}); + return null, + }, + } + + var id: [36]u8 = undefined; + if (data.len == 36) { + @memcpy(id[0..36], data) + return id; + } + + uuidv4(&id); + std.fs.cwd().writeFile(.{.sub_path = ID_FILE, .data = buf[0..36]}) catch |err| { + log.warn("failed to write to id file: {}", .{err}); + return null; + }; + return id; +} + +pub const Event = union(enum) { + run: Run, + + const Run = struct { + version: []const u8, + mode: RunMode, + + const RunMode = enum { + fetch, + serve, + }; + }; +}; + +const NoopProvider = struct { + fn init(_: Allocator) !NoopProvider { + return .{}; + } + fn deinit(_: NoopProvider) void {} + pub fn record(_: NoopProvider, _: Event) !void {} +}; + +extern fn setenv(name: [*:0]u8, value: [*:0]u8, override: c_int) c_int; +extern fn unsetenv(name: [*:0]u8) c_int; +const testing = std.testing; +test "telemetry: disabled by environment" { + _ = setenv(@constCast("LIGHTPANDA_DISABLE_TELEMETRY"), @constCast(""), 0); + defer _ = unsetenv(@constCast("LIGHTPANDA_DISABLE_TELEMETRY")); + + const FailingProvider = struct { + fn init(_: Allocator) !@This() { + return .{}; + } + fn deinit(_: @This()) void {} + pub fn record(_: @This(), _: Event) !void { + unreachable; + } + }; + + var telemetry = TelemetryT(FailingProvider).init(testing.allocator); + defer telemetry.deinit(); + telemetry.record(.{ .run = .{ .mode = .serve, .version = "123" } }); +} diff --git a/src/unit_tests.zig b/src/unit_tests.zig index 440a1f41..d677e5d7 100644 --- a/src/unit_tests.zig +++ b/src/unit_tests.zig @@ -384,4 +384,5 @@ test { std.testing.refAllDecls(@import("cdp/cdp.zig")); std.testing.refAllDecls(@import("log.zig")); std.testing.refAllDecls(@import("datetime.zig")); + std.testing.refAllDecls(@import("telemetry/telemetry.zig")); } From 53f6e66c230672ac913bbb62e24195578dfbf8dd Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 3 Mar 2025 15:27:39 +0800 Subject: [PATCH 02/17] Remove plausible, leave a dummy provider for now Add batching, add install optional id (persisted) and execution id (per run) --- .gitignore | 1 + src/id.zig | 2 +- src/telemetry/lightpanda.zig | 55 ++++++++++--------- src/telemetry/telemetry.zig | 100 ++++++++++++++++++++++++++++++----- 4 files changed, 119 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 49ae9a0b..4c5a0f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ zig-cache zig-out /vendor/netsurf/out /vendor/libiconv/ +lightpanda.id diff --git a/src/id.zig b/src/id.zig index f21af778..f3dc248d 100644 --- a/src/id.zig +++ b/src/id.zig @@ -66,7 +66,7 @@ pub fn Incrementing(comptime T: type, comptime prefix: []const u8) type { }; } -fn uuidv4(hex: []u8) void { +pub fn uuidv4(hex: []u8) void { std.debug.assert(hex.len == 36); var bin: [16]u8 = undefined; diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index b3f481a7..92ac0521 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -30,7 +30,10 @@ pub const Lightpanda = struct { } pub fn send(self: *Lightpanda, iid: ?[]const u8, eid: []const u8, events: []Event) !void { - std.debug.print("SENDING: {s} {s} {d}", .{iid, eid, events.len}) + _ = self; + _ = iid; + _ = eid; + _ = events; // defer _ = self.arena.reset(.{ .retain_capacity = {} }); // const body = try std.json.stringifyAlloc(self.arena.allocator(), PlausibleEvent{ .event = event }, .{}); @@ -57,31 +60,31 @@ pub const Lightpanda = struct { }; // wraps a telemetry event so that we can serialize it to plausible's event endpoint -const PlausibleEvent = struct { - event: Event, +// const PlausibleEvent = struct { +// event: Event, - pub fn jsonStringify(self: PlausibleEvent, jws: anytype) !void { - try jws.beginObject(); - try jws.objectField("name"); - try jws.write(@tagName(self.event)); - try jws.objectField("url"); - try jws.write(EVENT_URL); - try jws.objectField("domain"); - try jws.write(DOMAIN_KEY); - try jws.objectField("props"); - switch (self.event) { - inline else => |props| try jws.write(props), - } - try jws.endObject(); - } -}; +// pub fn jsonStringify(self: PlausibleEvent, jws: anytype) !void { +// try jws.beginObject(); +// try jws.objectField("name"); +// try jws.write(@tagName(self.event)); +// try jws.objectField("url"); +// try jws.write(EVENT_URL); +// try jws.objectField("domain"); +// try jws.write(DOMAIN_KEY); +// try jws.objectField("props"); +// switch (self.event) { +// inline else => |props| try jws.write(props), +// } +// try jws.endObject(); +// } +// }; -const testing = std.testing; -test "plausible: json event" { - const json = try std.json.stringifyAlloc(testing.allocator, PlausibleEvent{ .event = .{ .run = .{ .mode = .serve, .version = "over 9000!" } } }, .{}); - defer testing.allocator.free(json); +// const testing = std.testing; +// test "plausible: json event" { +// const json = try std.json.stringifyAlloc(testing.allocator, PlausibleEvent{ .event = .{ .run = .{ .mode = .serve, .version = "over 9000!" } } }, .{}); +// defer testing.allocator.free(json); - try testing.expectEqualStrings( - \\{"name":"run","url":"https://lightpanda.io/browser-stats","domain":"localhost","props":{"version":"over 9000!","mode":"serve"}} - , json); -} +// try testing.expectEqualStrings( +// \\{"name":"run","url":"https://lightpanda.io/browser-stats","domain":"localhost","props":{"version":"over 9000!","mode":"serve"}} +// , json); +// } diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 90d55c15..383a36aa 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -12,7 +12,7 @@ const ID_FILE = "lightpanda.id"; pub const Telemetry = TelemetryT(blk: { if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; - break :blk @import("ligtpanda.zig").Lightpanda; + break :blk @import("lightpanda.zig").Lightpanda; }); fn TelemetryT(comptime P: type) type { @@ -36,11 +36,13 @@ fn TelemetryT(comptime P: type) type { const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); var eid: [36]u8 = undefined; - uuidv4(&eid) + uuidv4(&eid); return .{ - .eid = eid, .iid = if (disabled) null else getOrCreateId(), + .eid = eid, + .count = 0, + .pending = undefined, .disabled = disabled, .provider = try P.init(allocator), }; @@ -62,7 +64,7 @@ fn TelemetryT(comptime P: type) type { return; } - const iid = if (self.iid) |*iid| *iid else null; + const iid: ?[]const u8 = if (self.iid) |*iid| iid else null; self.provider.send(iid, &self.eid, &self.pending) catch |err| { log.warn("failed to record event: {}", .{err}); }; @@ -73,22 +75,22 @@ fn TelemetryT(comptime P: type) type { fn getOrCreateId() ?[36]u8 { var buf: [37]u8 = undefined; - const data = std.fs.cwd().readFile(ID_FILE, &buf) catch |err| switch (err) blk: { - error.FileNotFound => break :bkl &.{}, + const data = std.fs.cwd().readFile(ID_FILE, &buf) catch |err| switch (err) { + error.FileNotFound => &.{}, else => { log.warn("failed to open id file: {}", .{err}); - return null, + return null; }, - } + }; var id: [36]u8 = undefined; if (data.len == 36) { - @memcpy(id[0..36], data) + @memcpy(id[0..36], data); return id; } uuidv4(&id); - std.fs.cwd().writeFile(.{.sub_path = ID_FILE, .data = buf[0..36]}) catch |err| { + std.fs.cwd().writeFile(.{ .sub_path = ID_FILE, .data = &id }) catch |err| { log.warn("failed to write to id file: {}", .{err}); return null; }; @@ -97,6 +99,7 @@ fn getOrCreateId() ?[36]u8 { pub const Event = union(enum) { run: Run, + flag: []const u8, // used for testing const Run = struct { version: []const u8, @@ -114,7 +117,7 @@ const NoopProvider = struct { return .{}; } fn deinit(_: NoopProvider) void {} - pub fn record(_: NoopProvider, _: Event) !void {} + pub fn send(_: NoopProvider, _: ?[]const u8, _: []const u8, _: []Event) !void {} }; extern fn setenv(name: [*:0]u8, value: [*:0]u8, override: c_int) c_int; @@ -129,7 +132,7 @@ test "telemetry: disabled by environment" { return .{}; } fn deinit(_: @This()) void {} - pub fn record(_: @This(), _: Event) !void { + pub fn send(_: @This(), _: ?[]const u8, _: []const u8, _: []Event) !void { unreachable; } }; @@ -138,3 +141,76 @@ test "telemetry: disabled by environment" { defer telemetry.deinit(); telemetry.record(.{ .run = .{ .mode = .serve, .version = "123" } }); } + +test "telemetry: getOrCreateId" { + defer std.fs.cwd().deleteFile(ID_FILE) catch {}; + + std.fs.cwd().deleteFile(ID_FILE) catch {}; + + const id1 = getOrCreateId().?; + const id2 = getOrCreateId().?; + try testing.expectEqualStrings(&id1, &id2); + + std.fs.cwd().deleteFile(ID_FILE) catch {}; + const id3 = getOrCreateId().?; + try testing.expectEqual(false, std.mem.eql(u8, &id1, &id3)); +} + +test "telemetry: sends batch" { + defer std.fs.cwd().deleteFile(ID_FILE) catch {}; + std.fs.cwd().deleteFile(ID_FILE) catch {}; + + var telemetry = TelemetryT(MockProvider).init(testing.allocator); + defer telemetry.deinit(); + const mock = &telemetry.provider; + + telemetry.record(.{ .flag = "1" }); + telemetry.record(.{ .flag = "2" }); + telemetry.record(.{ .flag = "3" }); + telemetry.record(.{ .flag = "4" }); + try testing.expectEqual(0, mock.events.items.len); + telemetry.record(.{ .flag = "5" }); + try testing.expectEqual(5, mock.events.items.len); + + telemetry.record(.{ .flag = "6" }); + telemetry.record(.{ .flag = "7" }); + telemetry.record(.{ .flag = "8" }); + telemetry.record(.{ .flag = "9" }); + try testing.expectEqual(5, mock.events.items.len); + telemetry.record(.{ .flag = "a" }); + try testing.expectEqual(10, mock.events.items.len); + + for (mock.events.items, 0..) |event, i| { + try testing.expectEqual(i + 1, std.fmt.parseInt(usize, event.flag, 16)); + } +} + +const MockProvider = struct { + iid: ?[]const u8, + eid: ?[]const u8, + allocator: Allocator, + events: std.ArrayListUnmanaged(Event), + + fn init(allocator: Allocator) !@This() { + return .{ + .iid = null, + .eid = null, + .events = .{}, + .allocator = allocator, + }; + } + fn deinit(self: *MockProvider) void { + self.events.deinit(self.allocator); + } + pub fn send(self: *MockProvider, iid: ?[]const u8, eid: []const u8, events: []Event) !void { + if (self.iid == null) { + try testing.expectEqual(null, self.eid); + self.iid = iid.?; + self.eid = eid; + } else { + try testing.expectEqualStrings(self.iid.?, iid.?); + try testing.expectEqualStrings(self.eid.?, eid); + } + try self.events.appendSlice(self.allocator, events); + } +}; From accf2c0e5eec99396829f317a7790ad57eac4619 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 3 Mar 2025 19:32:18 +0800 Subject: [PATCH 03/17] use async-client for telemetry --- src/app.zig | 5 +- src/main.zig | 23 ++-- src/server.zig | 4 + src/telemetry/lightpanda.zig | 205 ++++++++++++++++++++++++++--------- src/telemetry/telemetry.zig | 63 ++++------- src/unit_tests.zig | 10 +- 6 files changed, 201 insertions(+), 109 deletions(-) diff --git a/src/app.zig b/src/app.zig index a92f3b9f..158ea5d5 100644 --- a/src/app.zig +++ b/src/app.zig @@ -1,5 +1,6 @@ const std = @import("std"); +const Loop = @import("jsruntime").Loop; const Allocator = std.mem.Allocator; const Telemetry = @import("telemetry/telemetry.zig").Telemetry; @@ -8,8 +9,8 @@ const Telemetry = @import("telemetry/telemetry.zig").Telemetry; pub const App = struct { telemetry: Telemetry, - pub fn init(allocator: Allocator) !App { - const telemetry = Telemetry.init(allocator); + pub fn init(allocator: Allocator, loop: *Loop) !App { + const telemetry = Telemetry.init(allocator, loop); errdefer telemetry.deinit(); return .{ diff --git a/src/main.zig b/src/main.zig index 2aaf879c..363ed585 100644 --- a/src/main.zig +++ b/src/main.zig @@ -54,9 +54,6 @@ pub fn main() !void { _ = gpa.detectLeaks(); }; - var app = try @import("app.zig").App.init(alloc); - defer app.deinit(); - var args_arena = std.heap.ArenaAllocator.init(alloc); defer args_arena.deinit(); const args = try parseArgs(args_arena.allocator()); @@ -68,7 +65,6 @@ pub fn main() !void { return std.process.cleanExit(); }, .serve => |opts| { - app.telemetry.record(.{ .run = .{ .mode = .serve, .version = version } }); const address = std.net.Address.parseIp4(opts.host, opts.port) catch |err| { log.err("address (host:port) {any}\n", .{err}); return args.printUsageAndExit(false); @@ -77,24 +73,31 @@ pub fn main() !void { var loop = try jsruntime.Loop.init(alloc); defer loop.deinit(); + var app = try @import("app.zig").App.init(alloc, &loop); + defer app.deinit(); + app.telemetry.record(.{ .run = .{ .mode = .serve, .version = version } }); + const timeout = std.time.ns_per_s * @as(u64, opts.timeout); - server.run(alloc, address, timeout, &loop) catch |err| { + server.run(alloc, address, timeout, &loop, &app) catch |err| { log.err("Server error", .{}); return err; }; }, .fetch => |opts| { - app.telemetry.record(.{ .run = .{ .mode = .fetch, .version = version } }); log.debug("Fetch mode: url {s}, dump {any}", .{ opts.url, opts.dump }); - // vm - const vm = jsruntime.VM.init(); - defer vm.deinit(); - // loop var loop = try jsruntime.Loop.init(alloc); defer loop.deinit(); + var app = try @import("app.zig").App.init(alloc, &loop); + defer app.deinit(); + app.telemetry.record(.{ .run = .{ .mode = .fetch, .version = version } }); + + // vm + const vm = jsruntime.VM.init(); + defer vm.deinit(); + // browser var browser = Browser.init(alloc, &loop); defer browser.deinit(); diff --git a/src/server.zig b/src/server.zig index cbb0b500..18bbd112 100644 --- a/src/server.zig +++ b/src/server.zig @@ -34,6 +34,7 @@ const CloseError = jsruntime.IO.CloseError; const CancelError = jsruntime.IO.CancelOneError; const TimeoutError = jsruntime.IO.TimeoutError; +const App = @import("app.zig").App; const CDP = @import("cdp/cdp.zig").CDP; const TimeoutCheck = std.time.ns_per_ms * 100; @@ -48,6 +49,7 @@ const MAX_HTTP_REQUEST_SIZE = 2048; const MAX_MESSAGE_SIZE = 256 * 1024 + 14; const Server = struct { + app: *App, allocator: Allocator, loop: *jsruntime.Loop, @@ -1018,6 +1020,7 @@ pub fn run( address: net.Address, timeout: u64, loop: *jsruntime.Loop, + app: *App, ) !void { // create socket const flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | posix.SOCK.NONBLOCK; @@ -1043,6 +1046,7 @@ pub fn run( const json_version_response = try buildJSONVersionResponse(allocator, address); var server = Server{ + .app = app, .loop = loop, .timeout = timeout, .listener = listener, diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 92ac0521..7bb1b220 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -2,75 +2,176 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const ArenAallocator = std.heap.ArenaAllocator; -const Event = @import("telemetry.zig").Event; +const Loop = @import("jsruntime").Loop; +const Client = @import("asyncio").Client; + const log = std.log.scoped(.telemetry); -const URL = "https://lightpanda.io/browser-stats"; +const URL = "https://stats.lightpanda.io"; -pub const Lightpanda = struct { +pub const LightPanda = struct { uri: std.Uri, - arena: ArenAallocator, - client: std.http.Client, - headers: [1]std.http.Header, + io: Client.IO, + client: Client, + allocator: Allocator, + sending_pool: std.heap.MemoryPool(Sending), + client_context_pool: std.heap.MemoryPool(Client.Ctx), - pub fn init(allocator: Allocator) !Lightpanda { + pub fn init(allocator: Allocator, loop: *Loop) !LightPanda { return .{ + .allocator = allocator, + .io = Client.IO.init(loop), .client = .{ .allocator = allocator }, - .arena = std.heap.ArenaAllocator.init(allocator), .uri = std.Uri.parse(URL) catch unreachable, - .headers = [1]std.http.Header{ - .{ .name = "Content-Type", .value = "application/json" }, - }, + .sending_pool = std.heap.MemoryPool(Sending).init(allocator), + .client_context_pool = std.heap.MemoryPool(Client.Ctx).init(allocator), }; } - pub fn deinit(self: *Lightpanda) void { - self.arena.deinit(); + pub fn deinit(self: *LightPanda) void { self.client.deinit(); + self.sending_pool.deinit(); + self.client_context_pool.deinit(); } - pub fn send(self: *Lightpanda, iid: ?[]const u8, eid: []const u8, events: []Event) !void { - _ = self; - _ = iid; - _ = eid; - _ = events; - // defer _ = self.arena.reset(.{ .retain_capacity = {} }); - // const body = try std.json.stringifyAlloc(self.arena.allocator(), PlausibleEvent{ .event = event }, .{}); + pub fn send(self: *LightPanda, iid: ?[]const u8, eid: []const u8, event: anytype) !void { + var arena = std.heap.ArenaAllocator.init(self.allocator); + errdefer arena.deinit(); - // var server_headers: [2048]u8 = undefined; - // var req = try self.client.open(.POST, self.uri, .{ - // .redirect_behavior = .not_allowed, - // .extra_headers = &self.headers, - // .server_header_buffer = &server_headers, - // }); - // req.transfer_encoding = .{ .content_length = body.len }; - // try req.send(); + const resp_header_buffer = try arena.allocator().alloc(u8, 4096); + const body = try std.json.stringifyAlloc(arena.allocator(), .{ + .iid = iid, + .eid = eid, + .event = event, + }, .{}); - // try req.writeAll(body); - // try req.finish(); - // try req.wait(); + const sending = try self.sending_pool.create(); + errdefer self.sending_pool.destroy(sending); - // const status = req.response.status; - // if (status != .accepted) { - // log.warn("telemetry '{s}' event error: {d}", .{ @tagName(event), @intFromEnum(status) }); - // } else { - // log.warn("telemetry '{s}' sent", .{@tagName(event)}); - // } + sending.* = .{ + .body = body, + .arena = arena, + .lightpanda = self, + .request = try self.client.create(.POST, self.uri, .{ + .server_header_buffer = resp_header_buffer, + }), + }; + errdefer sending.request.deinit(); + + const ctx = try self.client_context_pool.create(); + errdefer self.client_context_pool.destroy(ctx); + + ctx.* = try Client.Ctx.init(&self.io, &sending.request); + ctx.userData = sending; + + try self.client.async_open( + .POST, + self.uri, + .{ .server_header_buffer = resp_header_buffer }, + ctx, + onRequestConnect, + ); + } + + fn handleError(self: *LightPanda, ctx: *Client.Ctx, err: anyerror) anyerror!void { + ctx.deinit(); + self.client_context_pool.destroy(ctx); + + var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + sending.deinit(); + self.sending_pool.destroy(sending); + log.info("request failure: {}", .{err}); + } + + fn onRequestConnect(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { + var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return sending.lightpanda.handleError(ctx, err); + + ctx.req.transfer_encoding = .{ .content_length = sending.body.len }; + return ctx.req.async_send(ctx, onRequestSend) catch |err| { + return sending.lightpanda.handleError(ctx, err); + }; + } + + fn onRequestSend(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { + var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return sending.lightpanda.handleError(ctx, err); + + return ctx.req.async_writeAll(sending.body, ctx, onRequestWrite) catch |err| { + return sending.lightpanda.handleError(ctx, err); + }; + } + + fn onRequestWrite(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { + var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return sending.lightpanda.handleError(ctx, err); + return ctx.req.async_finish(ctx, onRequestFinish) catch |err| { + return sending.lightpanda.handleError(ctx, err); + }; + } + + fn onRequestFinish(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { + var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return sending.lightpanda.handleError(ctx, err); + return ctx.req.async_wait(ctx, onRequestWait) catch |err| { + return sending.lightpanda.handleError(ctx, err); + }; + } + + fn onRequestWait(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { + var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return sending.lightpanda.handleError(ctx, err); + + const lightpanda = sending.lightpanda; + + defer { + ctx.deinit(); + lightpanda.client_context_pool.destroy(ctx); + + sending.deinit(); + lightpanda.sending_pool.destroy(sending); + } + + var buffer: [2048]u8 = undefined; + const reader = ctx.req.reader(); + while (true) { + const n = reader.read(&buffer) catch 0; + if (n == 0) { + break; + } + } + if (ctx.req.response.status != .ok) { + log.info("invalid response: {d}", .{@intFromEnum(ctx.req.response.status)}); + } } }; -// wraps a telemetry event so that we can serialize it to plausible's event endpoint -// const PlausibleEvent = struct { -// event: Event, +const Sending = struct { + body: []const u8, + request: Client.Request, + lightpanda: *LightPanda, + arena: std.heap.ArenaAllocator, -// pub fn jsonStringify(self: PlausibleEvent, jws: anytype) !void { + pub fn deinit(self: *Sending) void { + self.arena.deinit(); + self.request.deinit(); + } +}; + +// // wraps a telemetry event so that we can serialize it to plausible's event endpoint +// const EventWrap = struct { +// iid: ?[]const u8, +// eid: []const u8, +// event: *const Event, + +// pub fn jsonStringify(self: *const EventWrap, jws: anytype) !void { // try jws.beginObject(); -// try jws.objectField("name"); -// try jws.write(@tagName(self.event)); -// try jws.objectField("url"); -// try jws.write(EVENT_URL); -// try jws.objectField("domain"); -// try jws.write(DOMAIN_KEY); +// try jws.objectField("iid"); +// try jws.write(self.iid); +// try jws.objectField("eid"); +// try jws.write(self.eid); +// try jws.objectField("event"); +// try jws.write(@tagName(self.event.*)); // try jws.objectField("props"); // switch (self.event) { // inline else => |props| try jws.write(props), @@ -80,11 +181,15 @@ pub const Lightpanda = struct { // }; // const testing = std.testing; -// test "plausible: json event" { -// const json = try std.json.stringifyAlloc(testing.allocator, PlausibleEvent{ .event = .{ .run = .{ .mode = .serve, .version = "over 9000!" } } }, .{}); +// test "telemetry: lightpanda json event" { +// const json = try std.json.stringifyAlloc(testing.allocator, EventWrap{ +// .iid = "1234", +// .eid = "abc!", +// .event = .{ .run = .{ .mode = .serve, .version = "over 9000!" } } +// }, .{}); // defer testing.allocator.free(json); // try testing.expectEqualStrings( -// \\{"name":"run","url":"https://lightpanda.io/browser-stats","domain":"localhost","props":{"version":"over 9000!","mode":"serve"}} +// \\{"event":"run","iid""1234","eid":"abc!","props":{"version":"over 9000!","mode":"serve"}} // , json); // } diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 383a36aa..be9c911c 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -2,17 +2,16 @@ const std = @import("std"); const builtin = @import("builtin"); const Allocator = std.mem.Allocator; + +const Loop = @import("jsruntime").Loop; const uuidv4 = @import("../id.zig").uuidv4; const log = std.log.scoped(.telemetry); - -const BATCH_SIZE = 5; -const BATCH_END = BATCH_SIZE - 1; const ID_FILE = "lightpanda.id"; pub const Telemetry = TelemetryT(blk: { if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; - break :blk @import("lightpanda.zig").Lightpanda; + break :blk @import("lightpanda.zig").LightPanda; }); fn TelemetryT(comptime P: type) type { @@ -25,14 +24,11 @@ fn TelemetryT(comptime P: type) type { eid: [36]u8, provider: P, - // batch of events, pending[0..count] are pending - pending: [BATCH_SIZE]Event, - count: usize, disabled: bool, const Self = @This(); - pub fn init(allocator: Allocator) Self { + pub fn init(allocator: Allocator, loop: *Loop) Self { const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); var eid: [36]u8 = undefined; @@ -41,10 +37,8 @@ fn TelemetryT(comptime P: type) type { return .{ .iid = if (disabled) null else getOrCreateId(), .eid = eid, - .count = 0, - .pending = undefined, .disabled = disabled, - .provider = try P.init(allocator), + .provider = try P.init(allocator, loop), }; } @@ -56,19 +50,10 @@ fn TelemetryT(comptime P: type) type { if (self.disabled) { return; } - - const count = self.count; - self.pending[count] = event; - if (count < BATCH_END) { - self.count = count + 1; - return; - } - const iid: ?[]const u8 = if (self.iid) |*iid| iid else null; - self.provider.send(iid, &self.eid, &self.pending) catch |err| { + self.provider.send(iid, &self.eid, &event) catch |err| { log.warn("failed to record event: {}", .{err}); }; - self.count = 0; } }; } @@ -99,6 +84,7 @@ fn getOrCreateId() ?[36]u8 { pub const Event = union(enum) { run: Run, + navigate: void, flag: []const u8, // used for testing const Run = struct { @@ -113,11 +99,11 @@ pub const Event = union(enum) { }; const NoopProvider = struct { - fn init(_: Allocator) !NoopProvider { + fn init(_: Allocator, _: *Loop) !NoopProvider { return .{}; } fn deinit(_: NoopProvider) void {} - pub fn send(_: NoopProvider, _: ?[]const u8, _: []const u8, _: []Event) !void {} + pub fn send(_: NoopProvider, _: ?[]const u8, _: []const u8, _: anytype) !void {} }; extern fn setenv(name: [*:0]u8, value: [*:0]u8, override: c_int) c_int; @@ -128,16 +114,16 @@ test "telemetry: disabled by environment" { defer _ = unsetenv(@constCast("LIGHTPANDA_DISABLE_TELEMETRY")); const FailingProvider = struct { - fn init(_: Allocator) !@This() { + fn init(_: Allocator, _: *Loop) !@This() { return .{}; } fn deinit(_: @This()) void {} - pub fn send(_: @This(), _: ?[]const u8, _: []const u8, _: []Event) !void { + pub fn send(_: @This(), _: ?[]const u8, _: []const u8, _: anytype) !void { unreachable; } }; - var telemetry = TelemetryT(FailingProvider).init(testing.allocator); + var telemetry = TelemetryT(FailingProvider).init(testing.allocator, undefined); defer telemetry.deinit(); telemetry.record(.{ .run = .{ .mode = .serve, .version = "123" } }); } @@ -156,32 +142,21 @@ test "telemetry: getOrCreateId" { try testing.expectEqual(false, std.mem.eql(u8, &id1, &id3)); } -test "telemetry: sends batch" { +test "telemetry: sends event to provider" { defer std.fs.cwd().deleteFile(ID_FILE) catch {}; std.fs.cwd().deleteFile(ID_FILE) catch {}; - var telemetry = TelemetryT(MockProvider).init(testing.allocator); + var telemetry = TelemetryT(MockProvider).init(testing.allocator, undefined); defer telemetry.deinit(); const mock = &telemetry.provider; telemetry.record(.{ .flag = "1" }); telemetry.record(.{ .flag = "2" }); telemetry.record(.{ .flag = "3" }); - telemetry.record(.{ .flag = "4" }); - try testing.expectEqual(0, mock.events.items.len); - telemetry.record(.{ .flag = "5" }); - try testing.expectEqual(5, mock.events.items.len); - - telemetry.record(.{ .flag = "6" }); - telemetry.record(.{ .flag = "7" }); - telemetry.record(.{ .flag = "8" }); - telemetry.record(.{ .flag = "9" }); - try testing.expectEqual(5, mock.events.items.len); - telemetry.record(.{ .flag = "a" }); - try testing.expectEqual(10, mock.events.items.len); + try testing.expectEqual(3, mock.events.items.len); for (mock.events.items, 0..) |event, i| { - try testing.expectEqual(i + 1, std.fmt.parseInt(usize, event.flag, 16)); + try testing.expectEqual(i + 1, std.fmt.parseInt(usize, event.flag, 10)); } } @@ -191,7 +166,7 @@ const MockProvider = struct { allocator: Allocator, events: std.ArrayListUnmanaged(Event), - fn init(allocator: Allocator) !@This() { + fn init(allocator: Allocator, _: *Loop) !@This() { return .{ .iid = null, .eid = null, @@ -202,7 +177,7 @@ const MockProvider = struct { fn deinit(self: *MockProvider) void { self.events.deinit(self.allocator); } - pub fn send(self: *MockProvider, iid: ?[]const u8, eid: []const u8, events: []Event) !void { + pub fn send(self: *MockProvider, iid: ?[]const u8, eid: []const u8, events: *const Event) !void { if (self.iid == null) { try testing.expectEqual(null, self.eid); self.iid = iid.?; @@ -211,6 +186,6 @@ const MockProvider = struct { try testing.expectEqualStrings(self.iid.?, iid.?); try testing.expectEqualStrings(self.eid.?, eid); } - try self.events.appendSlice(self.allocator, events); + try self.events.append(self.allocator, events.*); } }; diff --git a/src/unit_tests.zig b/src/unit_tests.zig index d677e5d7..96335131 100644 --- a/src/unit_tests.zig +++ b/src/unit_tests.zig @@ -22,6 +22,7 @@ const parser = @import("netsurf"); const Allocator = std.mem.Allocator; +const App = @import("app.zig").App; const jsruntime = @import("jsruntime"); pub const Types = jsruntime.reflect(@import("generate.zig").Tuple(.{}){}); pub const UserContext = @import("user_context.zig").UserContext; @@ -47,6 +48,9 @@ pub fn main() !void { var loop = try jsruntime.Loop.init(allocator); defer loop.deinit(); + var app = try App.init(allocator, &loop); + defer app.deinit(); + const env = Env.init(allocator); defer env.deinit(allocator); @@ -67,7 +71,7 @@ pub fn main() !void { const cdp_thread = blk: { const address = try std.net.Address.parseIp("127.0.0.1", 9583); - const thread = try std.Thread.spawn(.{}, serveCDP, .{ allocator, address, &loop }); + const thread = try std.Thread.spawn(.{}, serveCDP, .{ allocator, address, &loop, &app }); break :blk thread; }; defer cdp_thread.join(); @@ -349,9 +353,9 @@ fn serveHTTP(address: std.net.Address) !void { } } -fn serveCDP(allocator: Allocator, address: std.net.Address, loop: *jsruntime.Loop) !void { +fn serveCDP(allocator: Allocator, address: std.net.Address, loop: *jsruntime.Loop, app: *App) !void { const server = @import("server.zig"); - server.run(allocator, address, std.time.ns_per_s * 2, loop) catch |err| { + server.run(allocator, address, std.time.ns_per_s * 2, loop, app) catch |err| { std.debug.print("CDP server error: {}", .{err}); return err; }; From 2609671982b7630a0dd99254a948d5a1c6e74be4 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 3 Mar 2025 19:44:38 +0800 Subject: [PATCH 04/17] don't try (and fail) to get userData after clearing context --- src/telemetry/lightpanda.zig | 39 ++++++++++++++++++------------------ src/telemetry/telemetry.zig | 2 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 7bb1b220..32172cc8 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -73,54 +73,55 @@ pub const LightPanda = struct { ); } - fn handleError(self: *LightPanda, ctx: *Client.Ctx, err: anyerror) anyerror!void { - ctx.deinit(); - self.client_context_pool.destroy(ctx); + fn handleError(sending: *Sending, ctx: *Client.Ctx, err: anyerror) anyerror!void { + const lightpanda = sending.lightpanda; + + ctx.deinit(); + lightpanda.client_context_pool.destroy(ctx); - var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); sending.deinit(); - self.sending_pool.destroy(sending); + lightpanda.sending_pool.destroy(sending); log.info("request failure: {}", .{err}); } fn onRequestConnect(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return sending.lightpanda.handleError(ctx, err); + const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return handleError(sending, ctx, err); ctx.req.transfer_encoding = .{ .content_length = sending.body.len }; return ctx.req.async_send(ctx, onRequestSend) catch |err| { - return sending.lightpanda.handleError(ctx, err); + return handleError(sending, ctx, err); }; } fn onRequestSend(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return sending.lightpanda.handleError(ctx, err); + const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return handleError(sending, ctx, err); return ctx.req.async_writeAll(sending.body, ctx, onRequestWrite) catch |err| { - return sending.lightpanda.handleError(ctx, err); + return handleError(sending, ctx, err); }; } fn onRequestWrite(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return sending.lightpanda.handleError(ctx, err); + const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return handleError(sending, ctx, err); return ctx.req.async_finish(ctx, onRequestFinish) catch |err| { - return sending.lightpanda.handleError(ctx, err); + return handleError(sending, ctx, err); }; } fn onRequestFinish(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return sending.lightpanda.handleError(ctx, err); + const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return handleError(sending, ctx, err); return ctx.req.async_wait(ctx, onRequestWait) catch |err| { - return sending.lightpanda.handleError(ctx, err); + return handleError(sending, ctx, err); }; } fn onRequestWait(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - var sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return sending.lightpanda.handleError(ctx, err); + const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); + res catch |err| return handleError(sending, ctx, err); const lightpanda = sending.lightpanda; diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index be9c911c..4ca8a442 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.telemetry); const ID_FILE = "lightpanda.id"; pub const Telemetry = TelemetryT(blk: { - if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; + // if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; break :blk @import("lightpanda.zig").LightPanda; }); From 6b832815394e18c6d35a047408ab16ad046a0375 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 3 Mar 2025 19:49:24 +0800 Subject: [PATCH 05/17] Add navigate telemetry --- .github/workflows/e2e-test.yml | 1 + README.md | 3 +++ src/app.zig | 14 +++++++++++++- src/browser/browser.zig | 19 +++++++++++++------ src/cdp/cdp.zig | 11 ++++------- src/cdp/testing.zig | 12 +++++++----- src/main.zig | 15 ++++----------- src/server.zig | 10 +++++----- src/telemetry/lightpanda.zig | 2 +- src/telemetry/telemetry.zig | 2 +- src/unit_tests.zig | 14 +++++++------- 11 files changed, 59 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index fe7104a4..53427e09 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -64,6 +64,7 @@ jobs: env: MAX_MEMORY: 28000 MAX_AVG_DURATION: 24 + LIGHTPANDA_DISABLE_TELEMETRY: true runs-on: ubuntu-latest diff --git a/README.md b/README.md index 5fa92e53..b0b7d168 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ await context.close(); await browser.disconnect(); ``` +### Telemetry +By default, Lightpanda collects and sends usage telemetry. This can be disabled by setting an environment variable `LIGHTPANDA_DISABLE_TELEMETRY=true`. You can read Lightpanda's privacy policy at: [https://lightpanda.io/privacy-policy](https://lightpanda.io/privacy-policy). + ## Status Lightpanda is still a work in progress and is currently at a Beta stage. diff --git a/src/app.zig b/src/app.zig index 158ea5d5..ea08043d 100644 --- a/src/app.zig +++ b/src/app.zig @@ -7,18 +7,30 @@ const Telemetry = @import("telemetry/telemetry.zig").Telemetry; // Container for global state / objects that various parts of the system // might need. pub const App = struct { + loop: *Loop, + allocator: Allocator, telemetry: Telemetry, - pub fn init(allocator: Allocator, loop: *Loop) !App { + pub fn init(allocator: Allocator) !App { + const loop = try allocator.create(Loop); + errdefer allocator.destroy(loop); + + loop.* = try Loop.init(allocator); + errdefer loop.deinit(); + const telemetry = Telemetry.init(allocator, loop); errdefer telemetry.deinit(); return .{ + .loop = loop, + .allocator = allocator, .telemetry = telemetry, }; } pub fn deinit(self: *App) void { self.telemetry.deinit(); + self.loop.deinit(); + self.allocator.destroy(self.loop); } }; diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 4c1d6f52..c00ae52f 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -33,6 +33,7 @@ const Loop = jsruntime.Loop; const Env = jsruntime.Env; const Module = jsruntime.Module; +const App = @import("../app.zig").App; const apiweb = @import("../apiweb.zig"); const Window = @import("../html/window.zig").Window; @@ -59,7 +60,7 @@ pub const user_agent = "Lightpanda/1.0"; // A browser contains only one session. // TODO allow multiple sessions per browser. pub const Browser = struct { - loop: *Loop, + app: *App, session: ?*Session, allocator: Allocator, http_client: HttpClient, @@ -68,9 +69,10 @@ pub const Browser = struct { const SessionPool = std.heap.MemoryPool(Session); - pub fn init(allocator: Allocator, loop: *Loop) Browser { + pub fn init(app: *App) Browser { + const allocator = app.allocator; return .{ - .loop = loop, + .app = app, .session = null, .allocator = allocator, .http_client = .{ .allocator = allocator }, @@ -109,6 +111,8 @@ pub const Browser = struct { // You can create successively multiple pages for a session, but you must // deinit a page before running another one. pub const Session = struct { + app: *App, + browser: *Browser, // The arena is used only to bound the js env init b/c it leaks memory. @@ -133,8 +137,10 @@ pub const Session = struct { jstypes: [Types.len]usize = undefined, fn init(self: *Session, browser: *Browser, ctx: anytype) !void { - const allocator = browser.allocator; + const app = browser.app; + const allocator = app.allocator; self.* = .{ + .app = app, .env = undefined, .browser = browser, .inspector = undefined, @@ -145,7 +151,7 @@ pub const Session = struct { }; const arena = self.arena.allocator(); - Env.init(&self.env, arena, browser.loop, null); + Env.init(&self.env, arena, app.loop, null); errdefer self.env.deinit(); try self.env.load(&self.jstypes); @@ -238,7 +244,7 @@ pub const Session = struct { std.debug.assert(self.page != null); // Reset all existing callbacks. - self.browser.loop.reset(); + self.app.loop.reset(); self.env.stop(); // TODO unload document: https://html.spec.whatwg.org/#unloading-documents @@ -333,6 +339,7 @@ pub const Page = struct { // see Inspector.contextCreated pub fn navigate(self: *Page, uri: []const u8, aux_data: ?[]const u8) !void { const arena = self.arena; + self.session.app.telemetry.record(.{ .navigate = {} }); log.debug("starting GET {s}", .{uri}); diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 1543ba76..37da6e85 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -20,7 +20,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const json = std.json; -const Loop = @import("jsruntime").Loop; +const App = @import("../app.zig").App; const asUint = @import("../str/parser.zig").asUint; const Incrementing = @import("../id.zig").Incrementing; @@ -34,7 +34,6 @@ pub const TimestampEvent = struct { }; pub const CDP = CDPT(struct { - const Loop = *@import("jsruntime").Loop; const Client = *@import("../server.zig").Client; const Browser = @import("../browser/browser.zig").Browser; const Session = @import("../browser/browser.zig").Session; @@ -47,8 +46,6 @@ const BrowserContextIdGen = Incrementing(u32, "BID"); // Generic so that we can inject mocks into it. pub fn CDPT(comptime TypeProvider: type) type { return struct { - loop: TypeProvider.Loop, - // Used for sending message to the client and closing on error client: TypeProvider.Client, @@ -73,13 +70,13 @@ pub fn CDPT(comptime TypeProvider: type) type { pub const Browser = TypeProvider.Browser; pub const Session = TypeProvider.Session; - pub fn init(allocator: Allocator, client: TypeProvider.Client, loop: TypeProvider.Loop) Self { + pub fn init(app: *App, client: TypeProvider.Client) Self { + const allocator = app.allocator; return .{ - .loop = loop, .client = client, .allocator = allocator, .browser_context = null, - .browser = Browser.init(allocator, loop), + .browser = Browser.init(app), .message_arena = std.heap.ArenaAllocator.init(allocator), .browser_context_pool = std.heap.MemoryPool(BrowserContext(Self)).init(allocator), }; diff --git a/src/cdp/testing.zig b/src/cdp/testing.zig index d08d2e09..0e5b1508 100644 --- a/src/cdp/testing.zig +++ b/src/cdp/testing.zig @@ -7,6 +7,7 @@ const Testing = @This(); const main = @import("cdp.zig"); const parser = @import("netsurf"); +const App = @import("../app.zig").App; pub const expectEqual = std.testing.expectEqual; pub const expectError = std.testing.expectError; @@ -16,10 +17,9 @@ const Browser = struct { session: ?*Session = null, arena: std.heap.ArenaAllocator, - pub fn init(allocator: Allocator, loop: anytype) Browser { - _ = loop; + pub fn init(app: *App) Browser { return .{ - .arena = std.heap.ArenaAllocator.init(allocator), + .arena = std.heap.ArenaAllocator.init(app.allocator), }; } @@ -112,13 +112,13 @@ const Client = struct { }; const TestCDP = main.CDPT(struct { - pub const Loop = void; pub const Browser = Testing.Browser; pub const Session = Testing.Session; pub const Client = *Testing.Client; }); const TestContext = struct { + app: App, client: ?Client = null, cdp_: ?TestCDP = null, arena: std.heap.ArenaAllocator, @@ -127,6 +127,7 @@ const TestContext = struct { if (self.cdp_) |*c| { c.deinit(); } + self.app.deinit(); self.arena.deinit(); } @@ -135,7 +136,7 @@ const TestContext = struct { self.client = Client.init(self.arena.allocator()); // Don't use the arena here. We want to detect leaks in CDP. // The arena is only for test-specific stuff - self.cdp_ = TestCDP.init(std.testing.allocator, &self.client.?, {}); + self.cdp_ = TestCDP.init(&self.app, &self.client.?); } return &self.cdp_.?; } @@ -262,6 +263,7 @@ const TestContext = struct { pub fn context() TestContext { return .{ + .app = App.init(std.testing.allocator) catch unreachable, .arena = std.heap.ArenaAllocator.init(std.testing.allocator), }; } diff --git a/src/main.zig b/src/main.zig index 363ed585..d6dd955e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -70,15 +70,12 @@ pub fn main() !void { return args.printUsageAndExit(false); }; - var loop = try jsruntime.Loop.init(alloc); - defer loop.deinit(); - - var app = try @import("app.zig").App.init(alloc, &loop); + var app = try @import("app.zig").App.init(alloc); defer app.deinit(); app.telemetry.record(.{ .run = .{ .mode = .serve, .version = version } }); const timeout = std.time.ns_per_s * @as(u64, opts.timeout); - server.run(alloc, address, timeout, &loop, &app) catch |err| { + server.run(&app, address, timeout) catch |err| { log.err("Server error", .{}); return err; }; @@ -86,11 +83,7 @@ pub fn main() !void { .fetch => |opts| { log.debug("Fetch mode: url {s}, dump {any}", .{ opts.url, opts.dump }); - // loop - var loop = try jsruntime.Loop.init(alloc); - defer loop.deinit(); - - var app = try @import("app.zig").App.init(alloc, &loop); + var app = try @import("app.zig").App.init(alloc); defer app.deinit(); app.telemetry.record(.{ .run = .{ .mode = .fetch, .version = version } }); @@ -99,7 +92,7 @@ pub fn main() !void { defer vm.deinit(); // browser - var browser = Browser.init(alloc, &loop); + var browser = Browser.init(&app); defer browser.deinit(); var session = try browser.newSession({}); diff --git a/src/server.zig b/src/server.zig index 18bbd112..fb4869fd 100644 --- a/src/server.zig +++ b/src/server.zig @@ -72,7 +72,6 @@ const Server = struct { fn deinit(self: *Server) void { self.client_pool.deinit(); - self.allocator.free(self.json_version_response); } fn queueAccept(self: *Server) void { @@ -467,7 +466,7 @@ pub const Client = struct { }; self.mode = .websocket; - self.cdp = CDP.init(self.server.allocator, self, self.server.loop); + self.cdp = CDP.init(self.server.app, self); return self.send(arena, response); } @@ -1016,11 +1015,9 @@ fn websocketHeader(buf: []u8, op_code: OpCode, payload_len: usize) []const u8 { } pub fn run( - allocator: Allocator, + app: *App, address: net.Address, timeout: u64, - loop: *jsruntime.Loop, - app: *App, ) !void { // create socket const flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | posix.SOCK.NONBLOCK; @@ -1043,7 +1040,10 @@ pub fn run( const vm = jsruntime.VM.init(); defer vm.deinit(); + var loop = app.loop; + const allocator = app.allocator; const json_version_response = try buildJSONVersionResponse(allocator, address); + defer allocator.free(json_version_response); var server = Server{ .app = app, diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 32172cc8..0c16aeca 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -7,7 +7,7 @@ const Client = @import("asyncio").Client; const log = std.log.scoped(.telemetry); -const URL = "https://stats.lightpanda.io"; +const URL = "https://telemetry.lightpanda.io/"; pub const LightPanda = struct { uri: std.Uri, diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 4ca8a442..be9c911c 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.telemetry); const ID_FILE = "lightpanda.id"; pub const Telemetry = TelemetryT(blk: { - // if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; + if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; break :blk @import("lightpanda.zig").LightPanda; }); diff --git a/src/unit_tests.zig b/src/unit_tests.zig index 96335131..3d625251 100644 --- a/src/unit_tests.zig +++ b/src/unit_tests.zig @@ -45,10 +45,7 @@ pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; const allocator = gpa.allocator(); - var loop = try jsruntime.Loop.init(allocator); - defer loop.deinit(); - - var app = try App.init(allocator, &loop); + var app = try App.init(allocator); defer app.deinit(); const env = Env.init(allocator); @@ -71,7 +68,10 @@ pub fn main() !void { const cdp_thread = blk: { const address = try std.net.Address.parseIp("127.0.0.1", 9583); - const thread = try std.Thread.spawn(.{}, serveCDP, .{ allocator, address, &loop, &app }); + const thread = try std.Thread.spawn(.{}, serveCDP, .{ + &app, + address, + }); break :blk thread; }; defer cdp_thread.join(); @@ -353,9 +353,9 @@ fn serveHTTP(address: std.net.Address) !void { } } -fn serveCDP(allocator: Allocator, address: std.net.Address, loop: *jsruntime.Loop, app: *App) !void { +fn serveCDP(app: *App, address: std.net.Address) !void { const server = @import("server.zig"); - server.run(allocator, address, std.time.ns_per_s * 2, loop, app) catch |err| { + server.run(app, address, std.time.ns_per_s * 2) catch |err| { std.debug.print("CDP server error: {}", .{err}); return err; }; From cd33a089d15109b6bc32c7799d05eb9936210869 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 5 Mar 2025 22:57:41 +0800 Subject: [PATCH 06/17] flatten events, include aarch + os, remove eid --- src/app.zig | 9 +- src/cdp/testing.zig | 2 +- src/main.zig | 8 +- src/telemetry/lightpanda.zig | 251 +++++++++++++---------------------- src/telemetry/telemetry.zig | 51 +++---- src/unit_tests.zig | 2 +- 6 files changed, 124 insertions(+), 199 deletions(-) diff --git a/src/app.zig b/src/app.zig index ea08043d..1249ccdf 100644 --- a/src/app.zig +++ b/src/app.zig @@ -4,6 +4,11 @@ const Loop = @import("jsruntime").Loop; const Allocator = std.mem.Allocator; const Telemetry = @import("telemetry/telemetry.zig").Telemetry; +pub const RunMode = enum { + serve, + fetch, +}; + // Container for global state / objects that various parts of the system // might need. pub const App = struct { @@ -11,14 +16,14 @@ pub const App = struct { allocator: Allocator, telemetry: Telemetry, - pub fn init(allocator: Allocator) !App { + pub fn init(allocator: Allocator, run_mode: RunMode) !App { const loop = try allocator.create(Loop); errdefer allocator.destroy(loop); loop.* = try Loop.init(allocator); errdefer loop.deinit(); - const telemetry = Telemetry.init(allocator, loop); + const telemetry = Telemetry.init(allocator, loop, run_mode); errdefer telemetry.deinit(); return .{ diff --git a/src/cdp/testing.zig b/src/cdp/testing.zig index 0e5b1508..5bc5f029 100644 --- a/src/cdp/testing.zig +++ b/src/cdp/testing.zig @@ -263,7 +263,7 @@ const TestContext = struct { pub fn context() TestContext { return .{ - .app = App.init(std.testing.allocator) catch unreachable, + .app = App.init(std.testing.allocator, .serve) catch unreachable, .arena = std.heap.ArenaAllocator.init(std.testing.allocator), }; } diff --git a/src/main.zig b/src/main.zig index d6dd955e..a7d27f6f 100644 --- a/src/main.zig +++ b/src/main.zig @@ -70,9 +70,9 @@ pub fn main() !void { return args.printUsageAndExit(false); }; - var app = try @import("app.zig").App.init(alloc); + var app = try @import("app.zig").App.init(alloc, .serve); defer app.deinit(); - app.telemetry.record(.{ .run = .{ .mode = .serve, .version = version } }); + app.telemetry.record(.{ .run = {} }); const timeout = std.time.ns_per_s * @as(u64, opts.timeout); server.run(&app, address, timeout) catch |err| { @@ -83,9 +83,9 @@ pub fn main() !void { .fetch => |opts| { log.debug("Fetch mode: url {s}, dump {any}", .{ opts.url, opts.dump }); - var app = try @import("app.zig").App.init(alloc); + var app = try @import("app.zig").App.init(alloc, .fetch); defer app.deinit(); - app.telemetry.record(.{ .run = .{ .mode = .fetch, .version = version } }); + app.telemetry.record(.{ .run = {} }); // vm const vm = jsruntime.VM.init(); diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 0c16aeca..d77818ac 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -1,196 +1,127 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; -const ArenAallocator = std.heap.ArenaAllocator; +const builtin = @import("builtin"); +const build_info = @import("build_info"); -const Loop = @import("jsruntime").Loop; -const Client = @import("asyncio").Client; +const Thread = std.Thread; +const Allocator = std.mem.Allocator; + +const telemetry = @import("telemetry.zig"); +const RunMode = @import("../app.zig").RunMode; const log = std.log.scoped(.telemetry); - -const URL = "https://telemetry.lightpanda.io/"; +const URL = "https://telemetry.lightpanda.io"; pub const LightPanda = struct { uri: std.Uri, - io: Client.IO, - client: Client, + pending: List, + running: bool, + thread: ?std.Thread, allocator: Allocator, - sending_pool: std.heap.MemoryPool(Sending), - client_context_pool: std.heap.MemoryPool(Client.Ctx), + mutex: std.Thread.Mutex, + cond: Thread.Condition, + node_pool: std.heap.MemoryPool(List.Node), - pub fn init(allocator: Allocator, loop: *Loop) !LightPanda { + const List = std.DoublyLinkedList(LightPandaEvent); + + pub fn init(allocator: Allocator) !LightPanda { return .{ + .cond = .{}, + .mutex = .{}, + .pending = .{}, + .thread = null, + .running = true, .allocator = allocator, - .io = Client.IO.init(loop), - .client = .{ .allocator = allocator }, .uri = std.Uri.parse(URL) catch unreachable, - .sending_pool = std.heap.MemoryPool(Sending).init(allocator), - .client_context_pool = std.heap.MemoryPool(Client.Ctx).init(allocator), + .node_pool = std.heap.MemoryPool(List.Node).init(allocator), }; } pub fn deinit(self: *LightPanda) void { - self.client.deinit(); - self.sending_pool.deinit(); - self.client_context_pool.deinit(); + if (self.thread) |*thread| { + self.mutex.lock(); + self.running = false; + self.mutex.unlock(); + self.cond.signal(); + thread.join(); + } + self.node_pool.deinit(); } - pub fn send(self: *LightPanda, iid: ?[]const u8, eid: []const u8, event: anytype) !void { - var arena = std.heap.ArenaAllocator.init(self.allocator); - errdefer arena.deinit(); - - const resp_header_buffer = try arena.allocator().alloc(u8, 4096); - const body = try std.json.stringifyAlloc(arena.allocator(), .{ + pub fn send(self: *LightPanda, iid: ?[]const u8, run_mode: RunMode, raw_event: telemetry.Event) !void { + const event = LightPandaEvent{ .iid = iid, - .eid = eid, - .event = event, - }, .{}); - - const sending = try self.sending_pool.create(); - errdefer self.sending_pool.destroy(sending); - - sending.* = .{ - .body = body, - .arena = arena, - .lightpanda = self, - .request = try self.client.create(.POST, self.uri, .{ - .server_header_buffer = resp_header_buffer, - }), + .driver = if (std.meta.activeTag(raw_event) == .navigate) "cdp" else null, + .mode = run_mode, + .os = builtin.os.tag, + .arch = builtin.cpu.arch, + .version = build_info.git_commit, + .event = @tagName(std.meta.activeTag(raw_event)), }; - errdefer sending.request.deinit(); - const ctx = try self.client_context_pool.create(); - errdefer self.client_context_pool.destroy(ctx); + self.mutex.lock(); + defer self.mutex.unlock(); + if (self.thread == null) { + self.thread = try std.Thread.spawn(.{}, run, .{self}); + } - ctx.* = try Client.Ctx.init(&self.io, &sending.request); - ctx.userData = sending; - - try self.client.async_open( - .POST, - self.uri, - .{ .server_header_buffer = resp_header_buffer }, - ctx, - onRequestConnect, - ); + const node = try self.node_pool.create(); + errdefer self.node_pool.destroy(node); + node.data = event; + self.pending.append(node); + self.cond.signal(); } - fn handleError(sending: *Sending, ctx: *Client.Ctx, err: anyerror) anyerror!void { - const lightpanda = sending.lightpanda; - - ctx.deinit(); - lightpanda.client_context_pool.destroy(ctx); - - sending.deinit(); - lightpanda.sending_pool.destroy(sending); - log.info("request failure: {}", .{err}); - } - - fn onRequestConnect(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return handleError(sending, ctx, err); - - ctx.req.transfer_encoding = .{ .content_length = sending.body.len }; - return ctx.req.async_send(ctx, onRequestSend) catch |err| { - return handleError(sending, ctx, err); - }; - } - - fn onRequestSend(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return handleError(sending, ctx, err); - - return ctx.req.async_writeAll(sending.body, ctx, onRequestWrite) catch |err| { - return handleError(sending, ctx, err); - }; - } - - fn onRequestWrite(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return handleError(sending, ctx, err); - return ctx.req.async_finish(ctx, onRequestFinish) catch |err| { - return handleError(sending, ctx, err); - }; - } - - fn onRequestFinish(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return handleError(sending, ctx, err); - return ctx.req.async_wait(ctx, onRequestWait) catch |err| { - return handleError(sending, ctx, err); - }; - } - - fn onRequestWait(ctx: *Client.Ctx, res: anyerror!void) anyerror!void { - const sending: *Sending = @ptrCast(@alignCast(ctx.userData)); - res catch |err| return handleError(sending, ctx, err); - - const lightpanda = sending.lightpanda; + fn run(self: *LightPanda) void { + var arr: std.ArrayListUnmanaged(u8) = .{}; + var client = std.http.Client{ .allocator = self.allocator }; defer { - ctx.deinit(); - lightpanda.client_context_pool.destroy(ctx); - - sending.deinit(); - lightpanda.sending_pool.destroy(sending); + arr.deinit(self.allocator); + client.deinit(); } - var buffer: [2048]u8 = undefined; - const reader = ctx.req.reader(); + self.mutex.lock(); while (true) { - const n = reader.read(&buffer) catch 0; - if (n == 0) { - break; + while (self.pending.popFirst()) |node| { + self.mutex.unlock(); + self.postEvent(&node.data, &client, &arr) catch |err| { + log.warn("Telementry reporting error: {}", .{err}); + }; + self.mutex.lock(); + self.node_pool.destroy(node); } + if (self.running == false) { + return; + } + self.cond.wait(&self.mutex); } - if (ctx.req.response.status != .ok) { - log.info("invalid response: {d}", .{@intFromEnum(ctx.req.response.status)}); + } + + fn postEvent(self: *const LightPanda, event: *const LightPandaEvent, client: *std.http.Client, arr: *std.ArrayListUnmanaged(u8)) !void { + defer arr.clearRetainingCapacity(); + try std.json.stringify(event, .{ .emit_null_optional_fields = false }, arr.writer(self.allocator)); + + var response_header_buffer: [2048]u8 = undefined; + + const result = try client.fetch(.{ + .method = .POST, + .payload = arr.items, + .response_storage = .ignore, + .location = .{ .uri = self.uri }, + .server_header_buffer = &response_header_buffer, + }); + if (result.status != .ok) { + log.warn("server error status: {}", .{result.status}); } } }; -const Sending = struct { - body: []const u8, - request: Client.Request, - lightpanda: *LightPanda, - arena: std.heap.ArenaAllocator, - - pub fn deinit(self: *Sending) void { - self.arena.deinit(); - self.request.deinit(); - } +const LightPandaEvent = struct { + iid: ?[]const u8, + mode: RunMode, + driver: ?[]const u8, + os: std.Target.Os.Tag, + arch: std.Target.Cpu.Arch, + version: []const u8, + event: []const u8, }; - -// // wraps a telemetry event so that we can serialize it to plausible's event endpoint -// const EventWrap = struct { -// iid: ?[]const u8, -// eid: []const u8, -// event: *const Event, - -// pub fn jsonStringify(self: *const EventWrap, jws: anytype) !void { -// try jws.beginObject(); -// try jws.objectField("iid"); -// try jws.write(self.iid); -// try jws.objectField("eid"); -// try jws.write(self.eid); -// try jws.objectField("event"); -// try jws.write(@tagName(self.event.*)); -// try jws.objectField("props"); -// switch (self.event) { -// inline else => |props| try jws.write(props), -// } -// try jws.endObject(); -// } -// }; - -// const testing = std.testing; -// test "telemetry: lightpanda json event" { -// const json = try std.json.stringifyAlloc(testing.allocator, EventWrap{ -// .iid = "1234", -// .eid = "abc!", -// .event = .{ .run = .{ .mode = .serve, .version = "over 9000!" } } -// }, .{}); -// defer testing.allocator.free(json); - -// try testing.expectEqualStrings( -// \\{"event":"run","iid""1234","eid":"abc!","props":{"version":"over 9000!","mode":"serve"}} -// , json); -// } diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index be9c911c..b76d96ef 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -5,6 +5,7 @@ const Allocator = std.mem.Allocator; const Loop = @import("jsruntime").Loop; const uuidv4 = @import("../id.zig").uuidv4; +const RunMode = @import("../app.zig").RunMode; const log = std.log.scoped(.telemetry); const ID_FILE = "lightpanda.id"; @@ -20,24 +21,21 @@ fn TelemetryT(comptime P: type) type { // null on IO error iid: ?[36]u8, - // a "execution" id is an id that represents this specific run - eid: [36]u8, provider: P, disabled: bool, + run_mode: RunMode, + const Self = @This(); - pub fn init(allocator: Allocator, loop: *Loop) Self { + pub fn init(allocator: Allocator, loop: *Loop, run_mode: RunMode) Self { const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); - var eid: [36]u8 = undefined; - uuidv4(&eid); - return .{ - .iid = if (disabled) null else getOrCreateId(), - .eid = eid, .disabled = disabled, + .run_mode = run_mode, + .iid = if (disabled) null else getOrCreateId(), .provider = try P.init(allocator, loop), }; } @@ -51,7 +49,7 @@ fn TelemetryT(comptime P: type) type { return; } const iid: ?[]const u8 = if (self.iid) |*iid| iid else null; - self.provider.send(iid, &self.eid, &event) catch |err| { + self.provider.send(iid, self.run_mode, &event) catch |err| { log.warn("failed to record event: {}", .{err}); }; } @@ -83,19 +81,9 @@ fn getOrCreateId() ?[36]u8 { } pub const Event = union(enum) { - run: Run, + run: void, navigate: void, flag: []const u8, // used for testing - - const Run = struct { - version: []const u8, - mode: RunMode, - - const RunMode = enum { - fetch, - serve, - }; - }; }; const NoopProvider = struct { @@ -103,11 +91,12 @@ const NoopProvider = struct { return .{}; } fn deinit(_: NoopProvider) void {} - pub fn send(_: NoopProvider, _: ?[]const u8, _: []const u8, _: anytype) !void {} + pub fn send(_: NoopProvider, _: ?[]const u8, _: RunMode, _: *const Event) !void {} }; extern fn setenv(name: [*:0]u8, value: [*:0]u8, override: c_int) c_int; extern fn unsetenv(name: [*:0]u8) c_int; + const testing = std.testing; test "telemetry: disabled by environment" { _ = setenv(@constCast("LIGHTPANDA_DISABLE_TELEMETRY"), @constCast(""), 0); @@ -118,14 +107,14 @@ test "telemetry: disabled by environment" { return .{}; } fn deinit(_: @This()) void {} - pub fn send(_: @This(), _: ?[]const u8, _: []const u8, _: anytype) !void { + pub fn send(_: @This(), _: ?[]const u8, _: RunMode, _: *const Event) !void { unreachable; } }; - var telemetry = TelemetryT(FailingProvider).init(testing.allocator, undefined); + var telemetry = TelemetryT(FailingProvider).init(testing.allocator, undefined, .serve); defer telemetry.deinit(); - telemetry.record(.{ .run = .{ .mode = .serve, .version = "123" } }); + telemetry.record(.{ .run = {} }); } test "telemetry: getOrCreateId" { @@ -146,7 +135,7 @@ test "telemetry: sends event to provider" { defer std.fs.cwd().deleteFile(ID_FILE) catch {}; std.fs.cwd().deleteFile(ID_FILE) catch {}; - var telemetry = TelemetryT(MockProvider).init(testing.allocator, undefined); + var telemetry = TelemetryT(MockProvider).init(testing.allocator, undefined, .serve); defer telemetry.deinit(); const mock = &telemetry.provider; @@ -162,14 +151,14 @@ test "telemetry: sends event to provider" { const MockProvider = struct { iid: ?[]const u8, - eid: ?[]const u8, + run_mode: ?RunMode, allocator: Allocator, events: std.ArrayListUnmanaged(Event), fn init(allocator: Allocator, _: *Loop) !@This() { return .{ .iid = null, - .eid = null, + .run_mode = null, .events = .{}, .allocator = allocator, }; @@ -177,14 +166,14 @@ const MockProvider = struct { fn deinit(self: *MockProvider) void { self.events.deinit(self.allocator); } - pub fn send(self: *MockProvider, iid: ?[]const u8, eid: []const u8, events: *const Event) !void { + pub fn send(self: *MockProvider, iid: ?[]const u8, run_mode: RunMode, events: *const Event) !void { if (self.iid == null) { - try testing.expectEqual(null, self.eid); + try testing.expectEqual(null, self.run_mode); self.iid = iid.?; - self.eid = eid; + self.run_mode = run_mode; } else { try testing.expectEqualStrings(self.iid.?, iid.?); - try testing.expectEqualStrings(self.eid.?, eid); + try testing.expectEqual(self.run_mode.?, run_mode); } try self.events.append(self.allocator, events.*); } diff --git a/src/unit_tests.zig b/src/unit_tests.zig index 3d625251..1cc9975f 100644 --- a/src/unit_tests.zig +++ b/src/unit_tests.zig @@ -45,7 +45,7 @@ pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; const allocator = gpa.allocator(); - var app = try App.init(allocator); + var app = try App.init(allocator, .serve); defer app.deinit(); const env = Env.init(allocator); From 75512602c3bfa3332417e1d05aef32eb1dce7526 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 5 Mar 2025 23:01:11 +0800 Subject: [PATCH 07/17] Add log to display telemetry state --- src/telemetry/telemetry.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index b76d96ef..3e9cecda 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -31,6 +31,9 @@ fn TelemetryT(comptime P: type) type { pub fn init(allocator: Allocator, loop: *Loop, run_mode: RunMode) Self { const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); + if (builtin.mode != .Debug and builtin.is_test == false) { + log.info("telemetry {s}", .{if (disabled) "disabled" else "enabled"}); + } return .{ .disabled = disabled, From 3b4de6a4052d3c19f76f3bafbead801f0b29ea1c Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 5 Mar 2025 23:31:46 +0800 Subject: [PATCH 08/17] remove [incorrect] data version --- src/telemetry/telemetry.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 3e9cecda..b2ed5ced 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -52,7 +52,7 @@ fn TelemetryT(comptime P: type) type { return; } const iid: ?[]const u8 = if (self.iid) |*iid| iid else null; - self.provider.send(iid, self.run_mode, &event) catch |err| { + self.provider.send(iid, self.run_mode, event) catch |err| { log.warn("failed to record event: {}", .{err}); }; } @@ -94,7 +94,7 @@ const NoopProvider = struct { return .{}; } fn deinit(_: NoopProvider) void {} - pub fn send(_: NoopProvider, _: ?[]const u8, _: RunMode, _: *const Event) !void {} + pub fn send(_: NoopProvider, _: ?[]const u8, _: RunMode, _: Event) !void {} }; extern fn setenv(name: [*:0]u8, value: [*:0]u8, override: c_int) c_int; @@ -110,7 +110,7 @@ test "telemetry: disabled by environment" { return .{}; } fn deinit(_: @This()) void {} - pub fn send(_: @This(), _: ?[]const u8, _: RunMode, _: *const Event) !void { + pub fn send(_: @This(), _: ?[]const u8, _: RunMode, _: Event) !void { unreachable; } }; @@ -169,7 +169,7 @@ const MockProvider = struct { fn deinit(self: *MockProvider) void { self.events.deinit(self.allocator); } - pub fn send(self: *MockProvider, iid: ?[]const u8, run_mode: RunMode, events: *const Event) !void { + pub fn send(self: *MockProvider, iid: ?[]const u8, run_mode: RunMode, events: Event) !void { if (self.iid == null) { try testing.expectEqual(null, self.run_mode); self.iid = iid.?; @@ -178,6 +178,6 @@ const MockProvider = struct { try testing.expectEqualStrings(self.iid.?, iid.?); try testing.expectEqual(self.run_mode.?, run_mode); } - try self.events.append(self.allocator, events.*); + try self.events.append(self.allocator, events); } }; From 25bf4fa7382c83f21149a51c53c102fbae6a58c4 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 6 Mar 2025 23:06:09 +0800 Subject: [PATCH 09/17] update telemetry URL --- vendor/tls.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tls.zig b/vendor/tls.zig index 0ea9e6d7..7eb35dab 160000 --- a/vendor/tls.zig +++ b/vendor/tls.zig @@ -1 +1 @@ -Subproject commit 0ea9e6d769a74946d6554edef4f05850734a48d2 +Subproject commit 7eb35dabf8798a88e15ddb7cd409e4f0b15912c4 From 288761632fe1de2c812012427b239fc83e77559a Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 6 Mar 2025 23:24:16 +0800 Subject: [PATCH 10/17] Revert "update telemetry URL" This reverts commit 88850bcdd38026720f03087be8ef7e9869072ac6. --- src/telemetry/lightpanda.zig | 2 +- vendor/tls.zig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index d77818ac..03dbf408 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -9,7 +9,7 @@ const telemetry = @import("telemetry.zig"); const RunMode = @import("../app.zig").RunMode; const log = std.log.scoped(.telemetry); -const URL = "https://telemetry.lightpanda.io"; +const URL = "https://lightpanda.io/browser-stats"; pub const LightPanda = struct { uri: std.Uri, diff --git a/vendor/tls.zig b/vendor/tls.zig index 7eb35dab..0ea9e6d7 160000 --- a/vendor/tls.zig +++ b/vendor/tls.zig @@ -1 +1 @@ -Subproject commit 7eb35dabf8798a88e15ddb7cd409e4f0b15912c4 +Subproject commit 0ea9e6d769a74946d6554edef4f05850734a48d2 From a6a81302344e51c526141bca5d9ceda285e5e258 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 6 Mar 2025 23:27:54 +0800 Subject: [PATCH 11/17] update telemetry URL (but not vendored dependency this time) --- src/telemetry/lightpanda.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 03dbf408..d77818ac 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -9,7 +9,7 @@ const telemetry = @import("telemetry.zig"); const RunMode = @import("../app.zig").RunMode; const log = std.log.scoped(.telemetry); -const URL = "https://lightpanda.io/browser-stats"; +const URL = "https://telemetry.lightpanda.io"; pub const LightPanda = struct { uri: std.Uri, From a5ee34a2db7ceaf5acf970d3fc0bebf37f540f78 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 10 Mar 2025 14:17:23 +0800 Subject: [PATCH 12/17] send telemetry synchronously in a background thread --- src/app.zig | 2 +- src/telemetry/telemetry.zig | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app.zig b/src/app.zig index 1249ccdf..c3e77bcd 100644 --- a/src/app.zig +++ b/src/app.zig @@ -23,7 +23,7 @@ pub const App = struct { loop.* = try Loop.init(allocator); errdefer loop.deinit(); - const telemetry = Telemetry.init(allocator, loop, run_mode); + const telemetry = Telemetry.init(allocator, run_mode); errdefer telemetry.deinit(); return .{ diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index b2ed5ced..74117f6d 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -29,7 +29,7 @@ fn TelemetryT(comptime P: type) type { const Self = @This(); - pub fn init(allocator: Allocator, loop: *Loop, run_mode: RunMode) Self { + pub fn init(allocator: Allocator, run_mode: RunMode) Self { const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); if (builtin.mode != .Debug and builtin.is_test == false) { log.info("telemetry {s}", .{if (disabled) "disabled" else "enabled"}); @@ -38,8 +38,8 @@ fn TelemetryT(comptime P: type) type { return .{ .disabled = disabled, .run_mode = run_mode, + .provider = try P.init(allocator), .iid = if (disabled) null else getOrCreateId(), - .provider = try P.init(allocator, loop), }; } From b0a2087015dcd0430b13db197ff644001c7b04a9 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 10 Mar 2025 14:23:32 +0800 Subject: [PATCH 13/17] fix unit test --- src/telemetry/telemetry.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 74117f6d..8c6d5ac2 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -90,7 +90,7 @@ pub const Event = union(enum) { }; const NoopProvider = struct { - fn init(_: Allocator, _: *Loop) !NoopProvider { + fn init(_: Allocator) !NoopProvider { return .{}; } fn deinit(_: NoopProvider) void {} @@ -106,7 +106,7 @@ test "telemetry: disabled by environment" { defer _ = unsetenv(@constCast("LIGHTPANDA_DISABLE_TELEMETRY")); const FailingProvider = struct { - fn init(_: Allocator, _: *Loop) !@This() { + fn init(_: Allocator) !@This() { return .{}; } fn deinit(_: @This()) void {} @@ -115,7 +115,7 @@ test "telemetry: disabled by environment" { } }; - var telemetry = TelemetryT(FailingProvider).init(testing.allocator, undefined, .serve); + var telemetry = TelemetryT(FailingProvider).init(testing.allocator, .serve); defer telemetry.deinit(); telemetry.record(.{ .run = {} }); } @@ -138,7 +138,7 @@ test "telemetry: sends event to provider" { defer std.fs.cwd().deleteFile(ID_FILE) catch {}; std.fs.cwd().deleteFile(ID_FILE) catch {}; - var telemetry = TelemetryT(MockProvider).init(testing.allocator, undefined, .serve); + var telemetry = TelemetryT(MockProvider).init(testing.allocator, .serve); defer telemetry.deinit(); const mock = &telemetry.provider; @@ -158,7 +158,7 @@ const MockProvider = struct { allocator: Allocator, events: std.ArrayListUnmanaged(Event), - fn init(allocator: Allocator, _: *Loop) !@This() { + fn init(allocator: Allocator) !@This() { return .{ .iid = null, .run_mode = null, From 1e6a1bd3af9279ea9bfb87179c417a62263e9fc1 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 10 Mar 2025 14:46:15 +0800 Subject: [PATCH 14/17] store iid in application data directory --- src/app.zig | 28 +++++++++++++++++++++++++- src/telemetry/telemetry.zig | 39 +++++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/app.zig b/src/app.zig index c3e77bcd..fac2f2ff 100644 --- a/src/app.zig +++ b/src/app.zig @@ -4,6 +4,8 @@ const Loop = @import("jsruntime").Loop; const Allocator = std.mem.Allocator; const Telemetry = @import("telemetry/telemetry.zig").Telemetry; +const log = std.log.scoped(.app); + pub const RunMode = enum { serve, fetch, @@ -13,6 +15,7 @@ pub const RunMode = enum { // might need. pub const App = struct { loop: *Loop, + app_dir_path: ?[]const u8, allocator: Allocator, telemetry: Telemetry, @@ -23,19 +26,42 @@ pub const App = struct { loop.* = try Loop.init(allocator); errdefer loop.deinit(); - const telemetry = Telemetry.init(allocator, run_mode); + const app_dir_path = getAndMakeAppDir(allocator); + const telemetry = Telemetry.init(allocator, run_mode, app_dir_path); errdefer telemetry.deinit(); return .{ .loop = loop, .allocator = allocator, .telemetry = telemetry, + .app_dir_path = app_dir_path, }; } pub fn deinit(self: *App) void { + if (self.app_dir_path) |app_dir_path| { + self.allocator.free(app_dir_path); + } + self.telemetry.deinit(); self.loop.deinit(); self.allocator.destroy(self.loop); } }; + +fn getAndMakeAppDir(allocator: Allocator) ?[]const u8 { + const app_dir_path = std.fs.getAppDataDir(allocator, "lightpanda") catch |err| { + log.warn("failed to get lightpanda data dir: {}", .{err}); + return null; + }; + + std.fs.makeDirAbsolute(app_dir_path) catch |err| switch (err) { + error.PathAlreadyExists => return app_dir_path, + else => { + allocator.free(app_dir_path); + log.warn("failed to create lightpanda data dir: {}", .{err}); + return null; + } + }; + return app_dir_path; +} diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 8c6d5ac2..c55b4a19 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -8,7 +8,7 @@ const uuidv4 = @import("../id.zig").uuidv4; const RunMode = @import("../app.zig").RunMode; const log = std.log.scoped(.telemetry); -const ID_FILE = "lightpanda.id"; +const IID_FILE = "iid"; pub const Telemetry = TelemetryT(blk: { if (builtin.mode == .Debug or builtin.is_test) break :blk NoopProvider; @@ -29,7 +29,7 @@ fn TelemetryT(comptime P: type) type { const Self = @This(); - pub fn init(allocator: Allocator, run_mode: RunMode) Self { + pub fn init(allocator: Allocator, run_mode: RunMode, app_dir_path: ?[]const u8) Self { const disabled = std.process.hasEnvVarConstant("LIGHTPANDA_DISABLE_TELEMETRY"); if (builtin.mode != .Debug and builtin.is_test == false) { log.info("telemetry {s}", .{if (disabled) "disabled" else "enabled"}); @@ -39,7 +39,7 @@ fn TelemetryT(comptime P: type) type { .disabled = disabled, .run_mode = run_mode, .provider = try P.init(allocator), - .iid = if (disabled) null else getOrCreateId(), + .iid = if (disabled) null else getOrCreateId(app_dir_path), }; } @@ -59,9 +59,17 @@ fn TelemetryT(comptime P: type) type { }; } -fn getOrCreateId() ?[36]u8 { +fn getOrCreateId(app_dir_path_: ?[]const u8) ?[36]u8 { + const app_dir_path = app_dir_path_ orelse return null; + var buf: [37]u8 = undefined; - const data = std.fs.cwd().readFile(ID_FILE, &buf) catch |err| switch (err) { + var dir = std.fs.openDirAbsolute(app_dir_path, .{}) catch |err| { + log.warn("failed to open data directory '{s}': {}", .{ app_dir_path, err }); + return null; + }; + defer dir.close(); + + const data = dir.readFile(IID_FILE, &buf) catch |err| switch (err) { error.FileNotFound => &.{}, else => { log.warn("failed to open id file: {}", .{err}); @@ -76,7 +84,7 @@ fn getOrCreateId() ?[36]u8 { } uuidv4(&id); - std.fs.cwd().writeFile(.{ .sub_path = ID_FILE, .data = &id }) catch |err| { + dir.writeFile(.{ .sub_path = IID_FILE, .data = &id }) catch |err| { log.warn("failed to write to id file: {}", .{err}); return null; }; @@ -115,30 +123,27 @@ test "telemetry: disabled by environment" { } }; - var telemetry = TelemetryT(FailingProvider).init(testing.allocator, .serve); + var telemetry = TelemetryT(FailingProvider).init(testing.allocator, .serve, null); defer telemetry.deinit(); telemetry.record(.{ .run = {} }); } test "telemetry: getOrCreateId" { - defer std.fs.cwd().deleteFile(ID_FILE) catch {}; + defer std.fs.cwd().deleteFile("/tmp/" ++ IID_FILE) catch {}; - std.fs.cwd().deleteFile(ID_FILE) catch {}; + std.fs.cwd().deleteFile("/tmp/" ++ IID_FILE) catch {}; - const id1 = getOrCreateId().?; - const id2 = getOrCreateId().?; + const id1 = getOrCreateId("/tmp/").?; + const id2 = getOrCreateId("/tmp/").?; try testing.expectEqualStrings(&id1, &id2); - std.fs.cwd().deleteFile(ID_FILE) catch {}; - const id3 = getOrCreateId().?; + std.fs.cwd().deleteFile("/tmp/" ++ IID_FILE) catch {}; + const id3 = getOrCreateId("/tmp/").?; try testing.expectEqual(false, std.mem.eql(u8, &id1, &id3)); } test "telemetry: sends event to provider" { - defer std.fs.cwd().deleteFile(ID_FILE) catch {}; - std.fs.cwd().deleteFile(ID_FILE) catch {}; - - var telemetry = TelemetryT(MockProvider).init(testing.allocator, .serve); + var telemetry = TelemetryT(MockProvider).init(testing.allocator, .serve, "/tmp/"); defer telemetry.deinit(); const mock = &telemetry.provider; From b75b36dc61dd84af585bfb63707828819503b1e3 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 10 Mar 2025 15:44:09 +0800 Subject: [PATCH 15/17] zig fmt --- src/app.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.zig b/src/app.zig index fac2f2ff..f64c88e4 100644 --- a/src/app.zig +++ b/src/app.zig @@ -61,7 +61,7 @@ fn getAndMakeAppDir(allocator: Allocator) ?[]const u8 { allocator.free(app_dir_path); log.warn("failed to create lightpanda data dir: {}", .{err}); return null; - } + }, }; return app_dir_path; } From c9bc5be42b9e2d5490f8a31be3f087dea23ef2db Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 18 Mar 2025 09:56:57 +0800 Subject: [PATCH 16/17] add additition navigate fields --- src/browser/browser.zig | 6 ++++- src/telemetry/lightpanda.zig | 49 ++++++++++++++++++++++++++++-------- src/telemetry/telemetry.zig | 7 +++++- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/browser/browser.zig b/src/browser/browser.zig index c00ae52f..2469c94f 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -339,7 +339,6 @@ pub const Page = struct { // see Inspector.contextCreated pub fn navigate(self: *Page, uri: []const u8, aux_data: ?[]const u8) !void { const arena = self.arena; - self.session.app.telemetry.record(.{ .navigate = {} }); log.debug("starting GET {s}", .{uri}); @@ -366,6 +365,11 @@ pub const Page = struct { // TODO handle fragment in url. + self.session.app.telemetry.record(.{ .navigate = .{ + .proxy = false, + .tls = std.ascii.eqlIgnoreCase(self.uri.scheme, "https"), + } }); + // load the data var resp = try self.session.loader.get(arena, self.uri); defer resp.deinit(); diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index d77818ac..7a53cd38 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -50,12 +50,8 @@ pub const LightPanda = struct { pub fn send(self: *LightPanda, iid: ?[]const u8, run_mode: RunMode, raw_event: telemetry.Event) !void { const event = LightPandaEvent{ .iid = iid, - .driver = if (std.meta.activeTag(raw_event) == .navigate) "cdp" else null, .mode = run_mode, - .os = builtin.os.tag, - .arch = builtin.cpu.arch, - .version = build_info.git_commit, - .event = @tagName(std.meta.activeTag(raw_event)), + .event = raw_event, }; self.mutex.lock(); @@ -119,9 +115,42 @@ pub const LightPanda = struct { const LightPandaEvent = struct { iid: ?[]const u8, mode: RunMode, - driver: ?[]const u8, - os: std.Target.Os.Tag, - arch: std.Target.Cpu.Arch, - version: []const u8, - event: []const u8, + event: telemetry.Event, + + pub fn jsonStringify(self: *const LightPandaEvent, writer: anytype) !void { + try writer.beginObject(); + + try writer.objectField("iid"); + try writer.write(self.iid); + + try writer.objectField("mode"); + try writer.write(self.mode); + + try writer.objectField("os"); + try writer.write(builtin.os.tag); + + try writer.objectField("arch"); + try writer.write(builtin.cpu.arch); + + try writer.objectField("version"); + try writer.write(build_info.git_commit); + + try writer.objectField("event"); + try writer.write(@tagName(std.meta.activeTag(self.event))); + + inline for (@typeInfo(telemetry.Event).Union.fields) |union_field| { + if (self.event == @field(telemetry.Event, union_field.name)) { + const inner = @field(self.event, union_field.name); + const TI = @typeInfo(@TypeOf(inner)); + if (TI == .Struct) { + inline for (TI.Struct.fields) |field| { + try writer.objectField(field.name); + try writer.write(@field(inner, field.name)); + } + } + } + } + + try writer.endObject(); + } }; diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index c55b4a19..4d170a19 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -93,8 +93,13 @@ fn getOrCreateId(app_dir_path_: ?[]const u8) ?[36]u8 { pub const Event = union(enum) { run: void, - navigate: void, + navigate: Navigate, flag: []const u8, // used for testing + + const Navigate = struct { + tls: bool, + proxy: bool, + }; }; const NoopProvider = struct { From 2e7342a59c1cf58cf7343495a2c0cc628c3b5322 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 18 Mar 2025 10:40:04 +0800 Subject: [PATCH 17/17] add driver field to navigate telemetry --- src/telemetry/lightpanda.zig | 1 - src/telemetry/telemetry.zig | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 7a53cd38..6dba26d0 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -98,7 +98,6 @@ pub const LightPanda = struct { try std.json.stringify(event, .{ .emit_null_optional_fields = false }, arr.writer(self.allocator)); var response_header_buffer: [2048]u8 = undefined; - const result = try client.fetch(.{ .method = .POST, .payload = arr.items, diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index 4d170a19..fd6d0447 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -99,6 +99,7 @@ pub const Event = union(enum) { const Navigate = struct { tls: bool, proxy: bool, + driver: []const u8 = "cdp", }; };