diff --git a/package-lock.json b/package-lock.json index 942bdd2..8f90078 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "live-testing-tool", + "name": "bugtrail", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "live-testing-tool", + "name": "bugtrail", "version": "0.1.0", "workspaces": [ "packages/shared", @@ -1082,6 +1082,12 @@ "win32" ] }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.19", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.19.tgz", + "integrity": "sha512-+yFUIDUaASFz0vQpWBErPxun/Lb+F0TdrlTsj50gm7K797YzEx2TmV6fEiwciHyFNghlZYPgQXH5qRcwMHcJFQ==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -1106,6 +1112,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/wicg-file-system-access": { + "version": "2020.9.8", + "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.8.tgz", + "integrity": "sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==", + "license": "MIT" + }, "node_modules/@vitejs/plugin-vue": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.2.4.tgz", @@ -2742,6 +2754,17 @@ "integrity": "sha512-97TBmpoWJEE+3nFBQ4VocyCdLKfw54rFaJ6EVQYLBCXqCIpLSZkwGgASpv4oPt9gdKCJ80RJlcmNzNn008Ag6Q==", "license": "MPL-2.0" }, + "node_modules/webm-muxer": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/webm-muxer/-/webm-muxer-5.1.4.tgz", + "integrity": "sha512-ditzgFVFbfqPaugkIr4mGhAdob5K9HY6Rzlh7TRsA368yA1sp/m5O7nQCcMLdgFDeNGtFPg8B+MeXLtpzKWX6Q==", + "deprecated": "This library is superseded by Mediabunny. Please migrate to it.", + "license": "MIT", + "dependencies": { + "@types/dom-webcodecs": "^0.1.4", + "@types/wicg-file-system-access": "^2020.9.5" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -2815,7 +2838,8 @@ "gifenc": "^1.0.3", "gnexus-ui-kit": "git+https://git.gnexus.space/git/root/gnexus-ui-kit.git#5227ba022e5da5ef7df5a4b4ed463c25abd1f85b", "vue": "^3.5.0", - "webextension-polyfill": "^0.12.0" + "webextension-polyfill": "^0.12.0", + "webm-muxer": "^5.1.4" }, "devDependencies": { "@types/webextension-polyfill": "^0.12.0", diff --git a/packages/extension/e2e.mjs b/packages/extension/e2e.mjs index 8c510fd..a8c6499 100644 --- a/packages/extension/e2e.mjs +++ b/packages/extension/e2e.mjs @@ -192,26 +192,64 @@ const all = itemsAfter.items ?? itemsAfter; const recordingSummary = await api(`/api/reports/${all[0].share_token}`); const steps = recordingSummary.steps ?? []; - // the track is delivered as an animated GIF attachment (512×512 cursor view) + // the screen recording is delivered as a silent WebM video (GIF fallback) + const videoAttachment = (recordingSummary.attachments ?? []).find((a) => a.mime === "video/webm"); const gifAttachment = (recordingSummary.attachments ?? []).find((a) => a.mime === "image/gif"); let videoOk = false; - if (gifAttachment) { + const mediaAttachment = videoAttachment ?? gifAttachment; + if (mediaAttachment) { const fileResponse = await fetch( - `${SERVER}/api/reports/by-token/${all[0].share_token}/files/${gifAttachment.file_id}` + `${SERVER}/api/reports/by-token/${all[0].share_token}/files/${mediaAttachment.file_id}` ); const { writeFile } = await import("node:fs/promises"); const buf = Buffer.from(await fileResponse.arrayBuffer()); - await writeFile("/tmp/ltt-shots/25-recording.gif", buf); - // GIF logical screen size lives at offset 6 (little-endian); frames each - // carry a Graphic Control Extension (0x21 0xF9) - const isGif = buf.subarray(0, 3).toString() === "GIF"; - const gifWidth = buf.readUInt16LE(6); - let frameCount = 0; - for (let i = 0; i < buf.length - 1; i++) { - if (buf[i] === 0x21 && buf[i + 1] === 0xf9) frameCount++; + await writeFile(videoAttachment ? "/tmp/ltt-shots/25-recording.webm" : "/tmp/ltt-shots/25-recording.gif", buf); + if (videoAttachment) { + // EBML magic (0x1A45DFA3) marks a Matroska/WebM container + const isWebm = buf[0] === 0x1a && buf[1] === 0x45 && buf[2] === 0xdf && buf[3] === 0xa3; + videoOk = isWebm && buf.length > 4000; + console.log("screen video:", JSON.stringify({ size: buf.length, isWebm })); + // full container validation when ffprobe is available: the file must be + // a decodable VP8 stream with a positive duration (catches muxer bugs + // that produce structurally valid but unplayable output) + if (videoOk) { + try { + const { execFile } = await import("node:child_process"); + const { promisify } = await import("node:util"); + const out = JSON.parse( + ( + await promisify(execFile)("ffprobe", [ + "-v", "error", + "-select_streams", "v:0", + "-show_entries", "stream=codec_name,width,height:format=duration", + "-of", "json", + "/tmp/ltt-shots/25-recording.webm", + ]) + ).stdout + ); + const stream = out.streams?.[0]; + console.log("ffprobe:", JSON.stringify({ codec: stream?.codec_name, size: `${stream?.width}x${stream?.height}`, duration: out.format?.duration })); + videoOk = stream?.codec_name === "vp8" && Number(out.format?.duration) > 0; + } catch (error) { + if (String(error).includes("ENOENT")) console.log("ffprobe: not installed, skipping"); + else { + console.log("ffprobe: FAILED", String(error).slice(0, 200)); + videoOk = false; + } + } + } + } else { + // GIF fallback: logical screen size at offset 6 (little-endian); frames + // each carry a Graphic Control Extension (0x21 0xF9) + const isGif = buf.subarray(0, 3).toString() === "GIF"; + const gifWidth = buf.readUInt16LE(6); + let frameCount = 0; + for (let i = 0; i < buf.length - 1; i++) { + if (buf[i] === 0x21 && buf[i + 1] === 0xf9) frameCount++; + } + videoOk = isGif && gifWidth === 512 && frameCount >= 3; + console.log("track gif:", JSON.stringify({ size: buf.length, isGif, gifWidth, frameCount })); } - videoOk = isGif && gifWidth === 512 && frameCount >= 3; - console.log("track gif:", JSON.stringify({ size: buf.length, isGif, gifWidth, frameCount })); } console.log( "recording:", @@ -220,6 +258,7 @@ stepCount: steps.length, types: steps.map((s) => s.type), inputValue: steps.find((s) => s.type === "input")?.data?.value, + hasVideo: Boolean(videoAttachment), hasGif: Boolean(gifAttachment), }) ); @@ -228,7 +267,7 @@ recordingSummary.type === "recording" && steps.some((s) => s.type === "click") && steps.some((s) => s.type === "input" && s.data?.value === "Hello recorder") && - Boolean(gifAttachment); + Boolean(mediaAttachment); // recording survives a full navigation: url_change recorded, and a click // captured *after* it (capture listeners reinstalled on the new page); // the report's page_url must be the URL where recording started diff --git a/packages/extension/manifest.template.json b/packages/extension/manifest.template.json index 231f052..6cf9f7a 100644 --- a/packages/extension/manifest.template.json +++ b/packages/extension/manifest.template.json @@ -3,7 +3,7 @@ "name": "BugTrail", "description": "Capture bug reports: element notes with screenshots and recorded reproduction steps.", "version": "0.1.0", - "permissions": ["activeTab", "scripting", "storage", "tabs", "debugger"], + "permissions": ["activeTab", "scripting", "storage", "tabs", "debugger", "offscreen"], "host_permissions": [""], "action": { "default_title": "BugTrail", diff --git a/packages/extension/package.json b/packages/extension/package.json index 73f84b5..3fcfedb 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -18,7 +18,8 @@ "gifenc": "^1.0.3", "gnexus-ui-kit": "git+https://git.gnexus.space/git/root/gnexus-ui-kit.git#5227ba022e5da5ef7df5a4b4ed463c25abd1f85b", "vue": "^3.5.0", - "webextension-polyfill": "^0.12.0" + "webextension-polyfill": "^0.12.0", + "webm-muxer": "^5.1.4" }, "devDependencies": { "@types/webextension-polyfill": "^0.12.0", diff --git a/packages/extension/scripts/build.mjs b/packages/extension/scripts/build.mjs index dcfce19..d95090d 100644 --- a/packages/extension/scripts/build.mjs +++ b/packages/extension/scripts/build.mjs @@ -22,7 +22,7 @@ // order matters: every pass writes assets/style.css (cssCodeSplit:false), so // the content pass — whose stylesheet is the union of all styles — runs last -const entries = ["background", "options", "popup", "relay", "console-tap", "content"]; +const entries = ["background", "options", "popup", "relay", "console-tap", "offscreen", "content"]; function run(entry) { return new Promise((resolve, reject) => { diff --git a/packages/extension/src/background/index.ts b/packages/extension/src/background/index.ts index 91580dc..e2ace8e 100644 --- a/packages/extension/src/background/index.ts +++ b/packages/extension/src/background/index.ts @@ -203,14 +203,21 @@ } /** - * Uploads the recorded screen as an animated GIF. Prefers the continuous - * screencast frames (cropped to a square following the cursor); when the - * video isn't available (Firefox, debugger busy, setting off) falls back to - * the per-event screenshots. Returns the uploaded file id. + * Uploads the recorded screen as a silent WebM video; when video encoding + * isn't possible (Firefox, WebCodecs missing) falls back to the cursor GIF + * and then to per-event screenshots. Returns the uploaded file id. */ async function uploadTrackGif(buffer: RecorderBuffer): Promise { - try { - if (buffer.video.length) { + if (buffer.video.length) { + try { + const video = await encodeVideoOffscreen(buffer.video, buffer.startedAt ?? Date.now()); + if (video) return await uploadRecordingMedia(video, "recording.webm"); + debugLog("screen video failed: offscreen API unavailable"); + } catch (error) { + debugLog(`screen video failed: ${error instanceof Error ? error.message : String(error)}`); + } + try { + // video encoding failed but the screencast frames are here — cursor GIF const viewport = (buffer.environment?.viewport as { w?: number; h?: number } | undefined) ?? null; const track = buildMouseTrack(buffer)?.points ?? []; const blob = await buildCursorGif(buffer.video, { @@ -219,11 +226,11 @@ startedAt: buffer.startedAt ?? Date.now(), track, }); - if (blob) return await uploadGifBlob(blob); + if (blob) return await uploadRecordingMedia(blob, "recording.gif"); debugLog("cursor gif failed: no frames encoded"); + } catch (error) { + debugLog(`cursor gif failed: ${error instanceof Error ? error.message : String(error)}`); } - } catch (error) { - debugLog(`cursor gif failed: ${error instanceof Error ? error.message : String(error)}`); } const indices = [...buffer.screenshots.keys()].sort((a, b) => a - b); @@ -240,21 +247,111 @@ try { const blob = await buildGif(frames); if (!blob) return null; - return await uploadGifBlob(blob); + return await uploadRecordingMedia(blob, "recording.gif"); } catch (error) { debugLog(`track gif failed: ${error instanceof Error ? error.message : String(error)}`); return null; } } -async function uploadGifBlob(blob: Blob): Promise { +async function uploadRecordingMedia(blob: Blob, filename: string): Promise { const client = await getHttpClient(); - const file = new File([blob], "recording.gif", { type: "image/gif" }); + const file = new File([blob], filename, { type: blob.type }); const uploaded = await client.upload<{ file_id: string }>("/api/uploads", file); - debugLog(`track gif uploaded: ${uploaded.file_id}`); + debugLog(`recording media uploaded: ${filename} (${Math.round(blob.size / 1024)} KB)`); return uploaded.file_id; } +// ---------- offscreen video encoding (WebCodecs lives only there) ---------- + +/** Minimal shape of the chrome.offscreen API (absent in Firefox). */ +interface OffscreenApiLike { + createDocument(params: { url: string; reasons: string[]; justification: string }): Promise; + closeDocument(): Promise; +} + +/** base64 → bytes (wire format to the offscreen document) */ +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + binary += String.fromCharCode(...bytes.subarray(i, i + chunk)); + } + return btoa(binary); +} + +async function ensureOffscreenDocument(): Promise { + const api = (browser as unknown as { offscreen?: OffscreenApiLike }).offscreen; + if (!api) return null; // Firefox / old Chrome — GIF fallback + try { + await api.createDocument({ + url: "src/offscreen/offscreen.html", + reasons: ["BLOBS"], + justification: "Encode recorded screencast frames into a WebM video", + }); + } catch (error) { + // already open (previous encode crashed) — reuse it + if (!String(error).includes("already exists")) throw error; + } + return api; +} + +/** per-message payload cap for the frame transfer (~8 MB of base64) */ +const VIDEO_BATCH_BYTES = 6 * 1024 * 1024; +const VIDEO_ENCODE_TIMEOUT_MS = 120_000; + +/** + * Sends the recorded frames to the offscreen document and gets the encoded + * WebM back. Runs over a dedicated runtime port: the service worker's own + * async onMessage listener would win the response race for plain messages. + */ +async function encodeVideoOffscreen(frames: VideoFrame[], startedAt: number): Promise { + const api = await ensureOffscreenDocument(); + if (!api) return null; + try { + return await new Promise((resolve, reject) => { + let settled = false; + const port = browser.runtime.connect({ name: "video-encode" }); + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + port.disconnect(); + fn(); + }; + const timer = setTimeout(() => finish(() => reject(new Error("video encoding timed out"))), VIDEO_ENCODE_TIMEOUT_MS); + port.onMessage.addListener((raw: unknown) => { + const msg = raw as { type: string; ok?: boolean; b64?: string; error?: string }; + if (msg.type !== "encode_video_result") return; + if (msg.ok && msg.b64) { + const bytes = base64ToBytes(msg.b64); + finish(() => resolve(new Blob([bytes.buffer as ArrayBuffer], { type: "video/webm" }))); + } else { + finish(() => reject(new Error(msg.error ?? "video encoding failed"))); + } + }); + port.onDisconnect.addListener(() => finish(() => reject(new Error("offscreen encoder disconnected")))); + + port.postMessage({ type: "encode_video_start" }); + let batch: { at: number; b64: string; w: number; h: number }[] = []; + let batchBytes = 0; + for (const frame of frames) { + batch.push({ at: frame.at, b64: bytesToBase64(frame.jpeg), w: frame.w, h: frame.h }); + batchBytes += frame.jpeg.length; + if (batchBytes >= VIDEO_BATCH_BYTES) { + port.postMessage({ type: "encode_video_frames", frames: batch }); + batch = []; + batchBytes = 0; + } + } + if (batch.length) port.postMessage({ type: "encode_video_frames", frames: batch }); + port.postMessage({ type: "encode_video_run" }); + }); + } finally { + await api.closeDocument().catch(() => {}); + } +} + // ---------- content-script injection ---------- async function ensureContentScript(tabId: number): Promise { @@ -786,6 +883,7 @@ case "recorder_console": { // console dump relayed from the MAIN-world tap — becomes timeline steps const buffer = recorder(tabId!); + debugLog(`console event received (recording: ${buffer.recording}, tab: ${tabId})`); if (!buffer.recording) return { ok: true, data: { recording: false } }; const settings = await getSettings(); if (!settings.consoleCapture) return { ok: true, data: { recording: false } }; diff --git a/packages/extension/src/background/video.ts b/packages/extension/src/background/video.ts new file mode 100644 index 0000000..555319b --- /dev/null +++ b/packages/extension/src/background/video.ts @@ -0,0 +1,75 @@ +/** + * Encodes the recorded screencast frames into a silent WebM video. WebCodecs + * VideoEncoder is not exposed to service workers, so this runs in the + * offscreen document (hosted by src/offscreen/main.ts). Returns null when + * WebCodecs is unavailable (Firefox) so the caller can fall back to the GIF + * path. + */ +import { Muxer, ArrayBufferTarget } from "webm-muxer"; +import type { VideoFrame } from "./gif"; + +/** VP8 needs even dimensions. */ +function even(value: number): number { + return Math.max(2, Math.floor(value / 2) * 2); +} + +export async function buildVideoWebm(frames: import("./gif").VideoFrame[]): Promise { + if (!frames.length || typeof VideoEncoder === "undefined") return null; + + // fixed output size: first frame's dimensions capped at 800px wide + const first = frames[0]; + const scale = Math.min(1, 800 / first.w); + const width = even(first.w * scale); + const height = even(first.h * scale); + + const muxer = new Muxer({ + target: new ArrayBufferTarget(), + video: { codec: "V_VP8", width, height, frameRate: 15 }, + firstTimestampBehavior: "offset", + }); + + // the encoder reports async failures through its error callback + const failure: { error: Error | null } = { error: null }; + const encoder = new VideoEncoder({ + output: (chunk, meta) => muxer.addVideoChunk(chunk, meta), + error: (error) => { + failure.error = error instanceof Error ? error : new Error(String(error)); + }, + }); + encoder.configure({ codec: "vp8", width, height, bitrate: 1_200_000 }); + + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + if (!context) throw new Error("no 2d context"); + + try { + // media timestamps start at the first frame: the muxer requires a + // zero-based timeline (see firstTimestampBehavior above) + const baseAt = frames[0].at; + for (let position = 0; position < frames.length; position++) { + const frame = frames[position]; + const nextAt = position + 1 < frames.length ? frames[position + 1].at : null; + const timestampUs = Math.max(frame.at - baseAt, 0) * 1000; + const durationUs = nextAt != null ? Math.max(nextAt - frame.at, 1) * 1000 : 33_000; + + const image = await createImageBitmap(new Blob([frame.jpeg.buffer as ArrayBuffer], { type: "image/jpeg" })); + try { + context.drawImage(image, 0, 0, width, height); + } finally { + image.close(); + } + const videoFrame = new VideoFrame(canvas, { timestamp: timestampUs, duration: durationUs }); + encoder.encode(videoFrame, { keyFrame: position % 30 === 0 }); + videoFrame.close(); + if (failure.error) throw failure.error; + } + if (encoder.state === "configured") await encoder.flush(); + } finally { + encoder.close(); + } + + if (failure.error) throw failure.error; + muxer.finalize(); + const { buffer } = muxer.target as ArrayBufferTarget; + return new Blob([buffer], { type: "video/webm" }); +} \ No newline at end of file diff --git a/packages/extension/src/offscreen/main.ts b/packages/extension/src/offscreen/main.ts new file mode 100644 index 0000000..2d666c1 --- /dev/null +++ b/packages/extension/src/offscreen/main.ts @@ -0,0 +1,87 @@ +/** + * Offscreen document: the only extension context where WebCodecs lives + * (VideoEncoder is not exposed to service workers). The background ships the + * screencast frames over a dedicated runtime port in base64 batches, this + * page encodes them into a silent WebM, verifies it with a local