diff --git a/package-lock.json b/package-lock.json index 7f81c91..942bdd2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1777,6 +1777,12 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/gifenc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/gifenc/-/gifenc-1.0.3.tgz", + "integrity": "sha512-xdr6AdrfGBcfzncONUOlXMBuc5wJDtOueE3c5rdG0oNgtINLD+f2iFZltrBRZYzACRbKr+mSVU/x98zv2u3jmw==", + "license": "MIT" + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -2806,6 +2812,7 @@ "dependencies": { "@ltt/shared": "*", "@ltt/ui": "*", + "gifenc": "^1.0.3", "gnexus-ui-kit": "git+https://git.gnexus.space/git/root/gnexus-ui-kit.git#5227ba022e5da5ef7df5a4b4ed463c25abd1f85b", "vue": "^3.5.0", "webextension-polyfill": "^0.12.0" diff --git a/packages/extension/e2e.mjs b/packages/extension/e2e.mjs index 6f7d373..cf465af 100644 --- a/packages/extension/e2e.mjs +++ b/packages/extension/e2e.mjs @@ -135,7 +135,10 @@ let clipboardOk = false; try { const clipboard = await page.evaluate(() => navigator.clipboard.readText()); - clipboardOk = clipboard === `${SERVER}/r/${items[0].share_token}`; + // the share link points at the web panel (panelUrl), not the API server + const expected = `http://localhost:5173/r/${items[0].share_token}`; + clipboardOk = clipboard === expected; + if (!clipboardOk) console.log("[clipboard]", JSON.stringify({ got: clipboard, expected })); } catch (error) { console.log("[clipboard read failed]", String(error).slice(0, 120)); } @@ -158,27 +161,16 @@ const all = itemsAfter.items ?? itemsAfter; const recordingSummary = await api(`/api/reports/${all[0].share_token}`); const steps = recordingSummary.steps ?? []; - // keep a step screenshot around for manual inspection: the recorder bar - // must not appear in it (plugin UI is hidden during captures) - const shotStep = steps.find((s) => s.screenshot_attachment_id != null); - if (shotStep) { - // steps reference the attachment row id; the files route wants the file id - const attachment = (recordingSummary.attachments ?? []).find((a) => a.id === shotStep.screenshot_attachment_id); - const fileId = attachment?.file_id ?? shotStep.screenshot_attachment_id; + // the track is delivered as an animated GIF attachment instead of step screenshots + const gifAttachment = (recordingSummary.attachments ?? []).find((a) => a.mime === "image/gif"); + if (gifAttachment) { const fileResponse = await fetch( - `${SERVER}/api/reports/by-token/${all[0].share_token}/files/${fileId}` + `${SERVER}/api/reports/by-token/${all[0].share_token}/files/${gifAttachment.file_id}` ); - if (!fileResponse.ok) { - console.log( - "step screenshot fetch failed:", - fileResponse.status, - JSON.stringify({ attachments: recordingSummary.attachments?.map((a) => a.file_id), steps: steps.map((s) => s.screenshot_attachment_id) }) - ); - } else { - const { writeFile } = await import("node:fs/promises"); - await writeFile("/tmp/ltt-shots/23-step-screenshot.png", Buffer.from(await fileResponse.arrayBuffer())); - console.log("step screenshot saved for inspection"); - } + const { writeFile } = await import("node:fs/promises"); + const buf = Buffer.from(await fileResponse.arrayBuffer()); + await writeFile("/tmp/ltt-shots/25-recording.gif", buf); + console.log("track gif:", JSON.stringify({ size: buf.length, isGif: buf.subarray(0, 3).toString() === "GIF" })); } console.log( "recording:", @@ -187,39 +179,72 @@ stepCount: steps.length, types: steps.map((s) => s.type), inputValue: steps.find((s) => s.type === "input")?.data?.value, + hasGif: Boolean(gifAttachment), }) ); const recorderOk = recordingSummary.type === "recording" && steps.some((s) => s.type === "click") && steps.some((s) => s.type === "input" && s.data?.value === "Hello recorder") && - steps.some((s) => s.screenshot_attachment_id != null); + Boolean(gifAttachment); - // --- popup: mode buttons, latest report, delete from the popup --- + // --- popup: mode buttons, latest report, replay + delete from the popup --- const popup = await context.newPage(); await popup.goto(`chrome-extension://${extensionId}/src/popup/popup.html`); await popup.waitForSelector("text=Element note", { timeout: 10000 }); const hasRecord = await popup.locator("text=Record steps").count(); const latestTitle = await popup.locator(".popup-report-title").textContent(); const latestOk = Boolean(latestTitle && latestTitle.length > 0); + const pagesBefore = context.pages().length; + + // replay the recorded track in a new tab and verify it re-executed the steps + let replayOk = false; + if (await popup.locator(".popup-mini:has-text('Replay')").count()) { + await popup.click(".popup-mini:has-text('Replay')"); + // the popup closes itself once the replay resolves; poll for the new tab + let replayPage = null; + for (let i = 0; i < 24 && !replayPage; i++) { + await new Promise((resolve) => setTimeout(resolve, 500)); + replayPage = context.pages().find( + (candidate) => candidate !== popup && candidate !== page && candidate.url().includes(`localhost:${PAGE_PORT}`) + ); + } + if (replayPage) { + // poll until the replayed input step has actually typed the value + let inputValue = null; + for (let i = 0; i < 16; i++) { + await replayPage.waitForTimeout(500); + inputValue = await replayPage.evaluate(() => document.querySelector("#name-input")?.value ?? null); + if (inputValue === "Hello recorder") break; + } + replayOk = inputValue === "Hello recorder"; + console.log("replay:", JSON.stringify({ url: replayPage.url(), inputValue })); + } + } + + // the popup closed itself during the replay — reopen it for the delete check + const popup2 = await context.newPage(); + await popup2.goto(`chrome-extension://${extensionId}/src/popup/popup.html`); + await popup2.waitForSelector("text=Element note", { timeout: 10000 }); // two-step delete: arm then confirm - await popup.click(".popup-mini-danger"); - await popup.click(".popup-mini-danger"); - await popup.waitForTimeout(1500); - await popup.close(); + await popup2.click(".popup-mini-danger"); + await popup2.click(".popup-mini-danger"); + await popup2.waitForTimeout(1500); + await popup2.close(); const itemsAfterDelete = (await api(`/api/projects/${project.id}/reports`, { headers: { Authorization: `Bearer ${login.token}` }, })); const deleteOk = (itemsAfterDelete.items ?? itemsAfterDelete).length === all.length - 1; const popupOk = hasRecord > 0 && latestOk && deleteOk; - console.log("popup:", JSON.stringify({ hasRecord: hasRecord > 0, latestOk, deleteOk })); + console.log("popup:", JSON.stringify({ hasRecord: hasRecord > 0, latestOk, deleteOk, replayOk })); - const ok = noteOk && recorderOk && clipboardOk && popupOk; + const ok = noteOk && recorderOk && clipboardOk && popupOk && replayOk; console.log("background log:", JSON.stringify(await worker.evaluate(() => self.__lttDebug()), null, 1)); console.log(noteOk ? "note flow OK" : "note flow MISMATCH"); console.log(recorderOk ? "recorder flow OK" : "recorder flow MISMATCH"); console.log(clipboardOk ? "clipboard link OK" : "clipboard link MISMATCH"); console.log(popupOk ? "popup OK" : "popup MISMATCH"); + console.log(replayOk ? "replay OK" : "replay MISMATCH"); console.log(ok ? "E2E OK" : "E2E FAILED"); process.exitCode = ok ? 0 : 1; } catch (error) { diff --git a/packages/extension/package.json b/packages/extension/package.json index a870a82..73f84b5 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -15,6 +15,7 @@ "dependencies": { "@ltt/shared": "*", "@ltt/ui": "*", + "gifenc": "^1.0.3", "gnexus-ui-kit": "git+https://git.gnexus.space/git/root/gnexus-ui-kit.git#5227ba022e5da5ef7df5a4b4ed463c25abd1f85b", "vue": "^3.5.0", "webextension-polyfill": "^0.12.0" diff --git a/packages/extension/src/background/gif.ts b/packages/extension/src/background/gif.ts new file mode 100644 index 0000000..c6dd685 --- /dev/null +++ b/packages/extension/src/background/gif.ts @@ -0,0 +1,41 @@ +import { GIFEncoder, quantize, applyPalette } from "gifenc"; + +/** + * Builds an animated GIF from event frames captured during a recording. + * Frames are scaled to max 800px wide; every frame carries the delay of the + * gap that followed its event (clamped), so the GIF paces like the track. + */ +export interface GifFrame { + dataUrl: string; + delayMs: number; +} + +export async function buildGif(frames: GifFrame[], maxWidth = 800): Promise { + if (!frames.length) return null; + const encoder = GIFEncoder(); + let size: { w: number; h: number } | null = null; + + for (const frame of frames) { + const blob = await (await fetch(frame.dataUrl)).blob(); + const image = await createImageBitmap(blob); + try { + if (!size) { + const scale = Math.min(1, maxWidth / image.width); + size = { w: Math.max(1, Math.round(image.width * scale)), h: Math.max(1, Math.round(image.height * scale)) }; + } + const canvas = new OffscreenCanvas(size.w, size.h); + const context = canvas.getContext("2d"); + if (!context) throw new Error("no 2d context"); + context.drawImage(image, 0, 0, size.w, size.h); + const { data } = context.getImageData(0, 0, size.w, size.h); + const palette = quantize(data, 256, { format: "rgb444" }); + const index = applyPalette(data, palette, "rgb444"); + encoder.writeFrame(index, size.w, size.h, { palette, delay: Math.round(frame.delayMs) }); + } finally { + image.close(); + } + } + + encoder.finish(); + return new Blob([encoder.bytes().buffer as ArrayBuffer], { type: "image/gif" }); +} \ No newline at end of file diff --git a/packages/extension/src/background/index.ts b/packages/extension/src/background/index.ts index d9dd059..7b1bb92 100644 --- a/packages/extension/src/background/index.ts +++ b/packages/extension/src/background/index.ts @@ -1,7 +1,9 @@ import browser from "webextension-polyfill"; import type { Runtime } from "webextension-polyfill"; +import type { ReportDetail } from "@ltt/shared"; import { submitNoteReport, submitRecordingReport } from "./api"; import { getSettings, saveSettings, login, logout, getHttpClient, panelBase } from "./settings"; +import { buildGif } from "./gif"; import type { BackgroundResponse, RecordEvent, @@ -118,12 +120,12 @@ if (!buffer.recording) return recorderStateOf(buffer); const settings = await getSettings(); - const attachmentIdByStep = await uploadStepScreenshots(buffer); - const steps = buffer.events.map((event, index) => ({ + const gifFileId = await uploadTrackGif(buffer); + const steps = buffer.events.map((event) => ({ type: event.type, offset_ms: buffer.startedAt ? Math.max(event.at - buffer.startedAt, 0) : 0, data: event.data, - attachment_id: attachmentIdByStep.get(index) ?? null, + attachment_id: null, })); if (buffer.events.length > 0) { @@ -136,7 +138,7 @@ page_title: buffer.pageTitle, environment: buffer.environment ?? {}, steps, - attachment_ids: [...attachmentIdByStep.values()], + attachment_ids: gifFileId ? [gifFileId] : [], }, settings, }).catch(async (error) => { @@ -153,26 +155,31 @@ return recorderStateOf(buffer); } -/** Uploads captured step screenshots; returns attachment id by event index. */ -async function uploadStepScreenshots(buffer: RecorderBuffer): Promise> { - const client = await getHttpClient(); - // buffer.screenshots is keyed by the event index, which matches the step index - const attachmentIdByStep = new Map(); - - for (const [eventIndex, dataUrl] of buffer.screenshots) { - try { - const response = await fetch(dataUrl); - const blob = await response.blob(); - const file = new File([blob], `step-${eventIndex + 1}.png`, { type: "image/png" }); - const uploaded = await client.upload<{ file_id: string }>("/api/uploads", file); - attachmentIdByStep.set(eventIndex, uploaded.file_id); - debugLog(`step screenshot ${eventIndex} uploaded: ${uploaded.file_id}`); - } catch (error) { - debugLog(`step screenshot ${eventIndex} failed: ${error instanceof Error ? error.message : String(error)}`); - } +/** Uploads the recorded track as an animated GIF; returns its file id. */ +async function uploadTrackGif(buffer: RecorderBuffer): Promise { + const indices = [...buffer.screenshots.keys()].sort((a, b) => a - b); + const frames = indices + .filter((index) => buffer.screenshots.has(index)) + .map((index, position, list) => { + const at = buffer.events[index]?.at ?? Date.now(); + const nextAt = position + 1 < list.length ? buffer.events[list[position + 1]]?.at : undefined; + // pace the GIF like the track; degenerate gaps get sane bounds + const delayMs = nextAt != null ? Math.min(Math.max(nextAt - at, 400), 2000) : 1500; + return { dataUrl: buffer.screenshots.get(index)!, delayMs }; + }); + if (!frames.length) return null; + try { + const blob = await buildGif(frames); + if (!blob) return null; + const client = await getHttpClient(); + const file = new File([blob], "recording.gif", { type: "image/gif" }); + const uploaded = await client.upload<{ file_id: string }>("/api/uploads", file); + debugLog(`track gif uploaded: ${uploaded.file_id} (${frames.length} frames)`); + return uploaded.file_id; + } catch (error) { + debugLog(`track gif failed: ${error instanceof Error ? error.message : String(error)}`); + return null; } - - return attachmentIdByStep; } // ---------- content-script injection ---------- @@ -219,6 +226,69 @@ return startRecorder(tabId); } +// ---------- replay (extension users can reproduce a recorded track) ---------- + +function waitTabComplete(tabId: number, timeoutMs = 20000): Promise { + return new Promise((resolve) => { + const finish = () => { + clearTimeout(timer); + browser.tabs.onUpdated.removeListener(listener); + resolve(); + }; + const timer = setTimeout(finish, timeoutMs); + const listener = (id: number, changeInfo: { status?: string }) => { + if (id === tabId && changeInfo.status === "complete") finish(); + }; + browser.tabs.onUpdated.addListener(listener); + }); +} + +/** + * Replays a recorded track in a fresh tab: opens the recorded page, then + * performs the steps with their recorded pacing (gaps capped so stale + * timings can't stall the replay). Page interactions are executed by the + * content script of the replayed tab. + */ +async function replayReport(token: string): Promise<{ played: number; total: number }> { + const client = await getHttpClient(); + const report = await client.get(`/api/reports/${token}`); + if (report.type !== "recording") throw new Error("Only recordings can be replayed"); + if (!report.page_url) throw new Error("Recording has no start page URL"); + + const tab = await browser.tabs.create({ url: report.page_url, active: true }); + const tabId = tab.id; + if (tabId == null) throw new Error("Could not open replay tab"); + await waitTabComplete(tabId); + // the replay tab is fresh — the content script may not be there yet + await ensureContentScript(tabId); + + const steps = report.steps ?? []; + const startedAt = Date.now(); + let played = 0; + for (const step of steps) { + // reproduce the recorded pacing, but cap each gap so stale timings don't stall + const wait = Math.min(Math.max(step.offset_ms - (Date.now() - startedAt), 0), 2000); + if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)); + if (step.type === "url_change" && typeof step.data?.to_url === "string") { + await browser.tabs.update(tabId, { url: step.data.to_url }); + await waitTabComplete(tabId); + played++; + } else if (step.type === "click" || step.type === "input") { + const response = (await browser.tabs.sendMessage(tabId, { type: "replay_step", data: step.data }).catch(() => null)) as + | { ok: boolean } + | null; + if (response?.ok) played++; + } + // note/screenshot steps have no page effect — skip + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + await browser.tabs + .sendMessage(tabId, { type: "show_flash", text: `Replay finished: ${played}/${steps.length} steps` }) + .catch(() => {}); + return { played, total: steps.length }; +} + /** Latest report of the default project + its project share token (for the panel link). */ async function latestReport(): Promise<{ report: { title: string; type: string; status: string; created_at: string; share_token: string } | null; @@ -302,12 +372,13 @@ if (changeInfo.url) buffer.pageUrl = changeInfo.url; if (changeInfo.title) buffer.pageTitle = changeInfo.title; if (changeInfo.url && buffer.lastUrl !== changeInfo.url && tab.url) { - buffer.events.push({ + const eventIndex = buffer.events.push({ type: "url_change", at: Date.now(), data: { from_url: buffer.lastUrl, to_url: tab.url, trigger: "tab_updated" }, - }); + }) - 1; buffer.lastUrl = tab.url; + void captureForEvent(tabId, eventIndex); } }); @@ -362,6 +433,14 @@ return { ok: true }; } + case "replay_report": { + try { + return { ok: true, data: await replayReport(msg.token as string) }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } + } + case "capture": { debugLog("capture requested"); try { @@ -393,8 +472,8 @@ ensureRecordingFreshness(buffer); const { event } = msg as unknown as { event: RecordEvent }; buffer.events.push(event); - // auto-screenshot on clicks (bounded to avoid runaway captures) - if (event.type === "click" && buffer.events.length <= 30) { + // a frame per event feeds the track GIF (bounded to avoid runaway memory) + if (buffer.events.length <= 60) { await captureForEvent(tabId!, buffer.events.length - 1); } return { ok: true, data: { stepCount: buffer.events.length } }; diff --git a/packages/extension/src/background/settings.ts b/packages/extension/src/background/settings.ts index 96facd9..247283b 100644 --- a/packages/extension/src/background/settings.ts +++ b/packages/extension/src/background/settings.ts @@ -14,7 +14,8 @@ const DEFAULTS: Settings = { serverUrl: "http://localhost:8001", - panelUrl: null, + // dev stack serves the panel on Vite's port; in prod panel and API share an origin + panelUrl: "http://localhost:5173", token: null, tokenExpiresAt: null, defaultProjectId: null, diff --git a/packages/extension/src/content/index.ts b/packages/extension/src/content/index.ts index 2652bec..305b56b 100644 --- a/packages/extension/src/content/index.ts +++ b/packages/extension/src/content/index.ts @@ -96,6 +96,42 @@ captureListeners = null; } +// ---------- replay (executes recorded steps on the page) ---------- + +function resolveRecordedElement(element: unknown): Element | null { + const context = element as { selector?: string | null; unique_selector?: string | null } | null; + if (!context) return null; + for (const selector of [context.unique_selector, context.selector]) { + if (!selector) continue; + try { + const found = document.querySelector(selector); + if (found) return found; + } catch { + // invalid selector — try the next one + } + } + return null; +} + +function replayStep(data: Record): { ok: boolean } { + const target = resolveRecordedElement(data.element); + if (!target) return { ok: false }; + const value = typeof data.value === "string" ? data.value : undefined; + const isField = + target instanceof HTMLInputElement || + target instanceof HTMLTextAreaElement || + target instanceof HTMLSelectElement; + if (value !== undefined && isField) { + const field = target as HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement; + field.value = value; + field.dispatchEvent(new Event("input", { bubbles: true })); + field.dispatchEvent(new Event("change", { bubbles: true })); + } else { + (target as HTMLElement).click(); + } + return { ok: true }; +} + // ---------- message wiring ---------- browser.runtime.onMessage.addListener((message, _sender, sendResponse) => { @@ -125,6 +161,17 @@ return true; } + case "replay_step": { + const result = replayStep((msg.data ?? {}) as Record); + sendResponse({ ok: result.ok }); + return true; + } + + case "show_flash": + getOverlay().setFlash(typeof msg.text === "string" ? msg.text : null); + sendResponse({ ok: true }); + return true; + case "set_ui_hidden": // the background hides all plugin UI while it grabs a screenshot if (Boolean(msg.hidden)) pushUiHidden(); diff --git a/packages/extension/src/content/overlay/App.vue b/packages/extension/src/content/overlay/App.vue index 2be4f68..0d21059 100644 --- a/packages/extension/src/content/overlay/App.vue +++ b/packages/extension/src/content/overlay/App.vue @@ -126,6 +126,10 @@
Link copied to clipboard
+ +
+ {{ state.flash }} +
@@ -145,4 +149,19 @@ font-size: 13px; z-index: 2; } +.ltt-flash-toast { + position: fixed; + left: 50%; + bottom: 76px; + transform: translateX(-50%); + display: flex; + align-items: center; + gap: 8px; + padding: 10px 16px; + background: var(--color-surface, #1f2335); + border: 2px solid var(--color-accent, #7aa2f7); + color: var(--color-text, #c0caf5); + font-size: 13px; + z-index: 2; +} \ No newline at end of file diff --git a/packages/extension/src/content/overlay/mount.ts b/packages/extension/src/content/overlay/mount.ts index 02e170f..9099a9b 100644 --- a/packages/extension/src/content/overlay/mount.ts +++ b/packages/extension/src/content/overlay/mount.ts @@ -18,11 +18,14 @@ export interface OverlayState { mode: "idle" | "picker"; recording: boolean; + /** transient message shown in the flash toast (replay results etc.) */ + flash: string | null; } export interface Overlay { startPicker: () => void; setRecorderState: (recording: boolean) => void; + setFlash: (text: string | null) => void; destroy: () => void; } @@ -62,10 +65,12 @@ mountPoint.style.cssText = `all: initial; pointer-events: none; width: 100%; height: 100%; ${vars}`; shadowRoot.appendChild(mountPoint); - const state = reactive({ mode: "idle", recording: false }); + const state = reactive({ mode: "idle", recording: false, flash: null }); const app = createApp(App, { state }); app.mount(mountPoint); + let flashTimer: number | null = null; + // styles load asynchronously; the overlay works unstyled until then void (async () => { try { @@ -79,6 +84,22 @@ } })(); + // constructable stylesheets ignore @font-face (Chromium limitation), so the + // Phosphor font must be registered through the FontFace API instead + void (async () => { + try { + const font = new FontFace( + "Phosphor", + `url(${browser.runtime.getURL("assets/fonts/phosphor-icons/src/fonts/Phosphor.woff2")})`, + { weight: "400", style: "normal", display: "block" } + ); + await font.load(); + document.fonts.add(font); + } catch { + // icons degrade to empty boxes; the overlay keeps working + } + })(); + return { startPicker: () => { state.mode = "picker"; @@ -89,6 +110,11 @@ state.recording = recording; if (!recording && state.mode === "picker") state.mode = "idle"; }, + setFlash: (text: string | null) => { + state.flash = text; + if (flashTimer) window.clearTimeout(flashTimer); + if (text) flashTimer = window.setTimeout(() => (state.flash = null), 5000); + }, destroy: () => { app.unmount(); host.remove(); diff --git a/packages/extension/src/popup/Popup.vue b/packages/extension/src/popup/Popup.vue index 17807be..a35f428 100644 --- a/packages/extension/src/popup/Popup.vue +++ b/packages/extension/src/popup/Popup.vue @@ -94,6 +94,19 @@ window.close(); } +async function replayLatest() { + if (!latest.value) return; + busy.value = true; + try { + await sendMessage({ type: "replay_report", token: latest.value.share_token }); + window.close(); + } catch (err) { + error.value = err instanceof Error ? err.message : String(err); + } finally { + busy.value = false; + } +} + async function deleteLatest() { if (!latest.value) return; if (!confirmingDelete.value) { @@ -171,6 +184,15 @@ Open +