diff --git a/packages/extension/e2e.mjs b/packages/extension/e2e.mjs index cf465af..1a5e18f 100644 --- a/packages/extension/e2e.mjs +++ b/packages/extension/e2e.mjs @@ -143,12 +143,22 @@ console.log("[clipboard read failed]", String(error).slice(0, 120)); } - // --- recorder: start, click the button, type, stop → recording report --- + // --- recorder: start, move the mouse around, click the button, type, stop --- + // human-like movement: several move events with pauses so the cursor + // sampler (~25 Hz) actually accumulates a track + const wiggle = async (box) => { + for (let i = 0; i < 6; i++) { + await page.mouse.move(box.x + box.width / 2 + i * 4, box.y + box.height / 2 - i * 2); + await page.waitForTimeout(70); + } + }; await worker.evaluate(() => self.__lttToggleRecorder()); await page.waitForTimeout(800); const buttonBox = await page.locator("#target").boundingBox(); - await page.mouse.click(buttonBox.x + buttonBox.width / 2, buttonBox.y + buttonBox.height / 2); + await wiggle(buttonBox); + await page.mouse.click(buttonBox.x + box.width / 2, buttonBox.y + buttonBox.height / 2); const inputBox = await page.locator("#name-input").boundingBox(); + await wiggle(inputBox); await page.mouse.click(inputBox.x + 10, inputBox.y + inputBox.height / 2); await page.keyboard.type("Hello recorder"); await page.waitForTimeout(600); // let the input debounce flush @@ -187,63 +197,114 @@ steps.some((s) => s.type === "click") && steps.some((s) => s.type === "input" && s.data?.value === "Hello recorder") && Boolean(gifAttachment); + // the recorded cursor path must be stored alongside the steps + const trackPoints = recordingSummary.mouse_track?.points?.length ?? 0; + const mouseTrackOk = trackPoints >= 2; + console.log("mouse track:", JSON.stringify({ points: trackPoints })); - // --- popup: mode buttons, latest report, replay + delete from the popup --- + // --- popup: mode buttons + latest report card --- const popup = await context.newPage(); await popup.goto(`chrome-extension://${extensionId}/src/popup/popup.html`); await popup.waitForSelector("text=Element note", { timeout: 10000 }); const hasRecord = await popup.locator("text=Record steps").count(); const latestTitle = await popup.locator(".popup-report-title").textContent(); - const latestOk = Boolean(latestTitle && latestTitle.length > 0); - const pagesBefore = context.pages().length; + const popupOk = hasRecord > 0 && Boolean(latestTitle && latestTitle.length > 0); - // replay the recorded track in a new tab and verify it re-executed the steps + // --- panel relay: the web panel talks to the extension through window --- + // messages (the relay content script), then replays the recording and the + // virtual cursor should appear in the replay tab + let relayOk = false; + let cursorOk = false; let replayOk = false; - if (await popup.locator(".popup-mini:has-text('Replay')").count()) { - await popup.click(".popup-mini:has-text('Replay')"); - // the popup closes itself once the replay resolves; poll for the new tab + let replayResultOk = false; + const panelPage = await context.newPage(); + await panelPage.goto(`http://localhost:5173/r/${all[0].share_token}`); + await panelPage.waitForTimeout(2000); // let the relay content script install + relayOk = await panelPage.evaluate( + () => + new Promise((resolve) => { + const handler = (event) => { + if (event.source !== window) return; + const data = event.data; + if (data?.source === "bugtrail-ext" && data.type === "pong") { + window.removeEventListener("message", handler); + resolve(true); + } + }; + window.addEventListener("message", handler); + window.postMessage({ source: "bugtrail-panel", type: "ping" }, window.location.origin); + setTimeout(() => resolve(false), 2500); + }) + ); + console.log("panel relay ping:", relayOk); + if (relayOk) { + // replay through the exact path the panel's "Start replay" button uses + const resultPromise = panelPage.evaluate( + (token) => + new Promise((resolve) => { + const handler = (event) => { + const data = event.data; + if (data?.source === "bugtrail-ext" && data.type === "replay_result") { + window.removeEventListener("message", handler); + resolve(data); + } + }; + window.addEventListener("message", handler); + window.postMessage({ source: "bugtrail-panel", type: "replay_report", token }, window.location.origin); + setTimeout(() => window.removeEventListener("message", handler), 90000); + }), + all[0].share_token + ); + // the virtual cursor lives only while the replay animates — poll for it let replayPage = null; - for (let i = 0; i < 24 && !replayPage; i++) { - await new Promise((resolve) => setTimeout(resolve, 500)); + for (let i = 0; i < 60 && !replayPage; i++) { + await new Promise((resolve) => setTimeout(resolve, 250)); replayPage = context.pages().find( - (candidate) => candidate !== popup && candidate !== page && candidate.url().includes(`localhost:${PAGE_PORT}`) + (candidate) => + candidate !== popup && candidate !== page && candidate !== panelPage && candidate.url().includes(`localhost:${PAGE_PORT}`) ); } if (replayPage) { + for (let i = 0; i < 30 && !cursorOk; i++) { + if (replayPage.isClosed()) break; + cursorOk = await replayPage.evaluate(() => Boolean(document.getElementById("ltt-cursor-host"))).catch(() => false); + if (cursorOk) break; + await replayPage.waitForTimeout(300); + } // poll until the replayed input step has actually typed the value let inputValue = null; for (let i = 0; i < 16; i++) { await replayPage.waitForTimeout(500); + if (replayPage.isClosed()) break; inputValue = await replayPage.evaluate(() => document.querySelector("#name-input")?.value ?? null); if (inputValue === "Hello recorder") break; } replayOk = inputValue === "Hello recorder"; - console.log("replay:", JSON.stringify({ url: replayPage.url(), inputValue })); } + const result = await resultPromise; + replayResultOk = result?.ok === true; + console.log("panel replay:", JSON.stringify({ result, replayOk, cursorOk })); } - // the popup closed itself during the replay — reopen it for the delete check - const popup2 = await context.newPage(); - await popup2.goto(`chrome-extension://${extensionId}/src/popup/popup.html`); - await popup2.waitForSelector("text=Element note", { timeout: 10000 }); - // two-step delete: arm then confirm - await popup2.click(".popup-mini-danger"); - await popup2.click(".popup-mini-danger"); - await popup2.waitForTimeout(1500); - await popup2.close(); + // --- delete the latest report from the popup (two-step: arm, confirm) --- + await popup.click(".popup-mini-danger"); + await popup.click(".popup-mini-danger"); + await popup.waitForTimeout(1500); + await popup.close(); const itemsAfterDelete = (await api(`/api/projects/${project.id}/reports`, { headers: { Authorization: `Bearer ${login.token}` }, })); const deleteOk = (itemsAfterDelete.items ?? itemsAfterDelete).length === all.length - 1; - const popupOk = hasRecord > 0 && latestOk && deleteOk; - console.log("popup:", JSON.stringify({ hasRecord: hasRecord > 0, latestOk, deleteOk, replayOk })); + console.log("popup:", JSON.stringify({ hasRecord: hasRecord > 0, latestOk: popupOk, deleteOk })); - const ok = noteOk && recorderOk && clipboardOk && popupOk && replayOk; + const ok = noteOk && recorderOk && clipboardOk && popupOk && relayOk && replayOk && replayResultOk && cursorOk && mouseTrackOk; console.log("background log:", JSON.stringify(await worker.evaluate(() => self.__lttDebug()), null, 1)); console.log(noteOk ? "note flow OK" : "note flow MISMATCH"); console.log(recorderOk ? "recorder flow OK" : "recorder flow MISMATCH"); + console.log(mouseTrackOk ? "mouse track OK" : "mouse track MISMATCH"); console.log(clipboardOk ? "clipboard link OK" : "clipboard link MISMATCH"); console.log(popupOk ? "popup OK" : "popup MISMATCH"); + console.log(relayOk && replayResultOk ? "panel relay OK" : "panel relay MISMATCH"); console.log(replayOk ? "replay OK" : "replay MISMATCH"); console.log(ok ? "E2E OK" : "E2E FAILED"); process.exitCode = ok ? 0 : 1; diff --git a/packages/extension/manifest.template.json b/packages/extension/manifest.template.json index 31f0b84..9ffe58e 100644 --- a/packages/extension/manifest.template.json +++ b/packages/extension/manifest.template.json @@ -21,6 +21,13 @@ "48": "icons/icon-48.png", "128": "icons/icon-128.png" }, + "content_scripts": [ + { + "matches": ["http://*/*", "https://*/*"], + "js": ["assets/relay.js"], + "run_at": "document_idle" + } + ], "web_accessible_resources": [ { "resources": ["assets/*"], diff --git a/packages/extension/scripts/build.mjs b/packages/extension/scripts/build.mjs index 1973b70..8ee82e9 100644 --- a/packages/extension/scripts/build.mjs +++ b/packages/extension/scripts/build.mjs @@ -22,7 +22,7 @@ // order matters: every pass writes assets/style.css (cssCodeSplit:false), so // the content pass — whose stylesheet is the union of all styles — runs last -const entries = ["background", "options", "popup", "content"]; +const entries = ["background", "options", "popup", "relay", "content"]; function run(entry) { return new Promise((resolve, reject) => { diff --git a/packages/extension/src/background/index.ts b/packages/extension/src/background/index.ts index 7b1bb92..f1da6bc 100644 --- a/packages/extension/src/background/index.ts +++ b/packages/extension/src/background/index.ts @@ -31,6 +31,8 @@ /** screenshots captured for click events, keyed by event index */ screenshots: Map; lastUrl: string | null; + /** sampled cursor positions: {at, x, y} with client timestamps */ + moveTrack: { at: number; x: number; y: number }[]; } const recorders = new Map(); @@ -47,6 +49,7 @@ environment: null, screenshots: new Map(), lastUrl: null, + moveTrack: [], }; recorders.set(tabId, buffer); } @@ -99,6 +102,7 @@ buffer.startedAt = Date.now(); buffer.events = []; buffer.screenshots = new Map(); + buffer.moveTrack = []; buffer.pageUrl = tab.url ?? null; buffer.pageTitle = tab.title ?? null; buffer.lastUrl = tab.url ?? null; @@ -114,11 +118,32 @@ }; } +/** Builds the persisted mouse track: offsets from recording start, capped. */ +function buildMouseTrack(buffer: RecorderBuffer): { viewport: { w: number; h: number }; points: { t: number; x: number; y: number }[] } | null { + if (!buffer.moveTrack.length || !buffer.startedAt) return null; + const viewport = (buffer.environment?.viewport as { w?: number; h?: number } | undefined) ?? null; + if (!viewport?.w || !viewport?.h) return null; + let points = buffer.moveTrack.map((p) => ({ + t: Math.max(p.at - buffer.startedAt!, 0), + x: p.x, + y: p.y, + })); + // decimate very long tracks instead of dropping them + while (points.length > 5000) points = points.filter((_, i) => i % 2 === 0); + return { viewport: { w: viewport.w, h: viewport.h }, points }; +} + async function recorderStop(tabId: number): Promise { const buffer = recorder(tabId); ensureRecordingFreshness(buffer); if (!buffer.recording) return recorderStateOf(buffer); + // collect the tail of the cursor track before anything else can race it + const flushed = (await browser.tabs.sendMessage(tabId, { type: "recorder_flush_moves" }).catch(() => null)) as + | { points?: { at: number; x: number; y: number }[] } + | null; + if (flushed?.points?.length) buffer.moveTrack.push(...flushed.points); + const settings = await getSettings(); const gifFileId = await uploadTrackGif(buffer); const steps = buffer.events.map((event) => ({ @@ -139,6 +164,7 @@ environment: buffer.environment ?? {}, steps, attachment_ids: gifFileId ? [gifFileId] : [], + mouse_track: buildMouseTrack(buffer), }, settings, }).catch(async (error) => { @@ -152,6 +178,7 @@ buffer.startedAt = null; buffer.events = []; buffer.screenshots = new Map(); + buffer.moveTrack = []; return recorderStateOf(buffer); } @@ -228,28 +255,47 @@ // ---------- replay (extension users can reproduce a recorded track) ---------- -function waitTabComplete(tabId: number, timeoutMs = 20000): Promise { +function waitTabComplete(tabId: number, timeoutMs = 20000): Promise { return new Promise((resolve) => { - const finish = () => { + const finish = (completed: boolean) => { clearTimeout(timer); browser.tabs.onUpdated.removeListener(listener); - resolve(); + browser.tabs.onRemoved.removeListener(removed); + resolve(completed); }; - const timer = setTimeout(finish, timeoutMs); + const timer = setTimeout(() => finish(false), timeoutMs); const listener = (id: number, changeInfo: { status?: string }) => { - if (id === tabId && changeInfo.status === "complete") finish(); + if (id === tabId && changeInfo.status === "complete") finish(true); + }; + const removed = (id: number) => { + if (id === tabId) finish(false); }; browser.tabs.onUpdated.addListener(listener); + browser.tabs.onRemoved.addListener(removed); }); } +interface ReplayFailure { + index: number; + type: string; + reason: string; +} + +export interface ReplayResult { + played: number; + total: number; + failures: ReplayFailure[]; +} + /** * Replays a recorded track in a fresh tab: opens the recorded page, then * performs the steps with their recorded pacing (gaps capped so stale - * timings can't stall the replay). Page interactions are executed by the - * content script of the replayed tab. + * timings can't stall the replay) while a virtual cursor walks the recorded + * mouse path. Page interactions are executed by the content script of the + * replayed tab. Fails loudly when the page can't be loaded or the tab dies, + * and collects per-step failures (e.g. elements that no longer exist). */ -async function replayReport(token: string): Promise<{ played: number; total: number }> { +async function replayReport(token: string): Promise { const client = await getHttpClient(); const report = await client.get(`/api/reports/${token}`); if (report.type !== "recording") throw new Error("Only recordings can be replayed"); @@ -258,35 +304,58 @@ const tab = await browser.tabs.create({ url: report.page_url, active: true }); const tabId = tab.id; if (tabId == null) throw new Error("Could not open replay tab"); - await waitTabComplete(tabId); + if (!(await waitTabComplete(tabId))) { + await browser.tabs.remove(tabId).catch(() => {}); + throw new Error(`Page did not load within 20 s: ${report.page_url}`); + } // the replay tab is fresh — the content script may not be there yet await ensureContentScript(tabId); + // a virtual cursor walks the recorded mouse path while the steps play + if (report.mouse_track?.points?.length) { + await browser.tabs.sendMessage(tabId, { type: "replay_start", track: report.mouse_track }).catch(() => {}); + } const steps = report.steps ?? []; const startedAt = Date.now(); + const failures: ReplayFailure[] = []; let played = 0; for (const step of steps) { + const stepIndex = step.step_index ?? played; // reproduce the recorded pacing, but cap each gap so stale timings don't stall const wait = Math.min(Math.max(step.offset_ms - (Date.now() - startedAt), 0), 2000); if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)); + // the tab may have been closed mid-replay (by the user or the page itself) + const alive = await browser.tabs.get(tabId).then( + () => true, + () => false + ); + if (!alive) throw new Error("Replay tab was closed before the track finished"); if (step.type === "url_change" && typeof step.data?.to_url === "string") { await browser.tabs.update(tabId, { url: step.data.to_url }); - await waitTabComplete(tabId); + if (!(await waitTabComplete(tabId))) { + throw new Error(`Page did not load within 20 s: ${step.data.to_url}`); + } played++; } else if (step.type === "click" || step.type === "input") { const response = (await browser.tabs.sendMessage(tabId, { type: "replay_step", data: step.data }).catch(() => null)) as | { ok: boolean } | null; - if (response?.ok) played++; + if (response?.ok) { + played++; + } else { + failures.push({ index: stepIndex, type: step.type, reason: "Element not found on the current page" }); + } } // note/screenshot steps have no page effect — skip } await new Promise((resolve) => setTimeout(resolve, 500)); - await browser.tabs - .sendMessage(tabId, { type: "show_flash", text: `Replay finished: ${played}/${steps.length} steps` }) - .catch(() => {}); - return { played, total: steps.length }; + await browser.tabs.sendMessage(tabId, { type: "replay_stop" }).catch(() => {}); + const summary = failures.length + ? `Replay finished: ${played}/${steps.length} steps (${failures.length} failed)` + : `Replay finished: ${played}/${steps.length} steps`; + await browser.tabs.sendMessage(tabId, { type: "show_flash", text: summary }).catch(() => {}); + return { played, total: steps.length, failures }; } /** Latest report of the default project + its project share token (for the panel link). */ @@ -485,6 +554,14 @@ return { ok: true }; } + case "recorder_move_batch": { + const buffer = recorder(tabId!); + if (!buffer.recording) return { ok: true, data: { recording: false } }; + const points = (msg.points ?? []) as { at: number; x: number; y: number }[]; + if (buffer.moveTrack.length + points.length <= 20000) buffer.moveTrack.push(...points); + return { ok: true }; + } + case "recorder_screenshot": { // a manual screenshot becomes its own step attached to the recording const buffer = recorder(tabId!); diff --git a/packages/extension/src/content/index.ts b/packages/extension/src/content/index.ts index 305b56b..1f0ec9a 100644 --- a/packages/extension/src/content/index.ts +++ b/packages/extension/src/content/index.ts @@ -52,6 +52,42 @@ return target instanceof Element && target.id === "ltt-overlay-host"; } +// ---------- cursor sampling (while recording) ---------- + +interface MoveSample { + /** client timestamp (Date.now()) — background converts to track offset */ + at: number; + x: number; + y: number; +} + +let moveSamples: MoveSample[] = []; +let moveFlushTimer: number | null = null; +let lastMove: { x: number; y: number; at: number } | null = null; + +const MOVE_MIN_INTERVAL_MS = 40; // ~25 samples/sec is plenty to redraw the path +const MOVE_MIN_DISTANCE_PX = 2; + +function onMouseMove(event: MouseEvent) { + if (isOurOverlay(event.target)) return; + const now = Date.now(); + if (lastMove) { + const moved = + Math.abs(event.clientX - lastMove.x) >= MOVE_MIN_DISTANCE_PX || + Math.abs(event.clientY - lastMove.y) >= MOVE_MIN_DISTANCE_PX; + if (!moved || now - lastMove.at < MOVE_MIN_INTERVAL_MS) return; + } + lastMove = { x: event.clientX, y: event.clientY, at: now }; + moveSamples.push({ at: now, x: event.clientX, y: event.clientY }); +} + +/** Hands the accumulated samples to the background and clears the local buffer. */ +function takeMoveSamples(): MoveSample[] { + const out = moveSamples; + moveSamples = []; + return out; +} + function attachCaptureListeners() { if (captureListeners) return; @@ -82,9 +118,19 @@ document.addEventListener("click", onClick, true); document.addEventListener("input", onInput, true); + document.addEventListener("mousemove", onMouseMove, true); + // the content script dies on full page navigations — flush regularly so + // each page contributes its own slice of the track + moveFlushTimer = window.setInterval(() => { + if (!moveSamples.length) return; + void browser.runtime.sendMessage({ type: "recorder_move_batch", points: takeMoveSamples() }).catch(() => {}); + }, 1000); captureListeners = () => { document.removeEventListener("click", onClick, true); document.removeEventListener("input", onInput, true); + document.removeEventListener("mousemove", onMouseMove, true); + if (moveFlushTimer != null) window.clearInterval(moveFlushTimer); + moveFlushTimer = null; for (const [element, pending] of pendingInputs) window.clearTimeout(pending.timer); pendingInputs.clear(); }; @@ -113,9 +159,97 @@ return null; } +// ---------- replay cursor (a virtual mouse that walks the recorded path) ---------- + +interface CursorState { + el: HTMLElement; + raf: number; + track: { viewport: { w: number; h: number }; points: { t: number; x: number; y: number }[] }; + startedAt: number; + lastDispatch: number; +} + +let cursor: CursorState | null = null; + +function ensureCursorEl(): HTMLElement { + let host = document.getElementById("ltt-cursor-host"); + if (host) return host; + host = document.createElement("div"); + host.id = "ltt-cursor-host"; + host.style.cssText = "position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;will-change:transform;"; + host.innerHTML = + '' + + ''; + document.documentElement.appendChild(host); + return host; +} + +function setCursorPosition(state: CursorState, x: number, y: number, dispatch: boolean) { + state.el.style.transform = `translate(${x}px, ${y}px)`; + if (!dispatch) return; + // synthetic mousemove keeps JS hover/move handlers live during replay; + // CSS :hover can't be triggered synthetically — that's a browser limitation + const now = performance.now(); + if (now - state.lastDispatch < 30) return; + state.lastDispatch = now; + const target = document.elementFromPoint(x, y); + if (target && target.id !== "ltt-cursor-host") { + target.dispatchEvent(new MouseEvent("mousemove", { clientX: x, clientY: y, bubbles: true })); + } +} + +function startReplayCursor(track: CursorState["track"]) { + stopReplayCursor(); + if (!track?.points?.length || !track.viewport?.w || !track.viewport?.h) return; + const el = ensureCursorEl(); + const state: CursorState = { el, raf: 0, track, startedAt: performance.now(), lastDispatch: 0 }; + cursor = state; + const points = track.points; + let index = 0; + const loop = () => { + if (!cursor || cursor !== state) return; + const t = performance.now() - state.startedAt; + // scale recorded viewport coordinates to the current window size + const sx = window.innerWidth / track.viewport.w; + const sy = window.innerHeight / track.viewport.h; + while (index + 1 < points.length && points[index + 1].t <= t) index++; + const a = points[index]; + const b = points[index + 1]; + let x: number; + let y: number; + if (!b) { + x = a.x * sx; + y = a.y * sy; + } else { + const k = Math.min(Math.max((t - a.t) / (b.t - a.t), 0), 1); + x = (a.x + (b.x - a.x) * k) * sx; + y = (a.y + (b.y - a.y) * k) * sy; + } + setCursorPosition(state, x, y, true); + state.raf = requestAnimationFrame(loop); + }; + setCursorPosition(state, points[0].x * (window.innerWidth / track.viewport.w), points[0].y * (window.innerHeight / track.viewport.h), false); + state.raf = requestAnimationFrame(loop); +} + +function moveCursorToElement(target: Element) { + if (!cursor) return; + const rect = target.getBoundingClientRect(); + setCursorPosition(cursor, rect.left + rect.width / 2, rect.top + rect.height / 2, true); +} + +function stopReplayCursor() { + if (!cursor) return; + cancelAnimationFrame(cursor.raf); + cursor.el.remove(); + cursor = null; +} + function replayStep(data: Record): { ok: boolean } { const target = resolveRecordedElement(data.element); if (!target) return { ok: false }; + moveCursorToElement(target); const value = typeof data.value === "string" ? data.value : undefined; const isField = target instanceof HTMLInputElement || @@ -150,23 +284,44 @@ const recording = Boolean((message as { recording?: unknown }).recording); getOverlay().setRecorderState(recording); if (recording) { + moveSamples = []; + lastMove = null; attachCaptureListeners(); void browser.runtime .sendMessage({ type: "recorder_environment", environment: collectEnvironment() }) .catch(() => {}); } else { detachCaptureListeners(); + // hand the tail of the track to the background before it submits + const points = takeMoveSamples(); + if (points.length) { + void browser.runtime.sendMessage({ type: "recorder_move_batch", points }).catch(() => {}); + } } sendResponse({ ok: true }); return true; } + case "recorder_flush_moves": + sendResponse({ ok: true, points: takeMoveSamples() }); + return true; + case "replay_step": { const result = replayStep((msg.data ?? {}) as Record); sendResponse({ ok: result.ok }); return true; } + case "replay_start": + startReplayCursor(msg.track as CursorState["track"]); + sendResponse({ ok: true }); + return true; + + case "replay_stop": + stopReplayCursor(); + sendResponse({ ok: true }); + return true; + case "show_flash": getOverlay().setFlash(typeof msg.text === "string" ? msg.text : null); sendResponse({ ok: true }); diff --git a/packages/extension/src/relay.ts b/packages/extension/src/relay.ts new file mode 100644 index 0000000..bcfd8c6 --- /dev/null +++ b/packages/extension/src/relay.ts @@ -0,0 +1,42 @@ +/** + * Tiny relay content script present on every http(s) page. Its only job is to + * let the BugTrail web panel talk to the extension through window messages — + * the panel can't use runtime.sendMessage directly (it would need the + * extension id in externally_connectable, which differs per installation). + * The overlay is still injected on demand; this relay holds no UI. + */ +import browser from "webextension-polyfill"; + +const PANEL_SOURCE = "bugtrail-panel"; +const EXT_SOURCE = "bugtrail-ext"; + +window.addEventListener("message", (event) => { + // only accept messages from this same window (the page itself) + if (event.source !== window) return; + const data = event.data as { source?: string; type?: string; token?: string } | null; + if (!data || data.source !== PANEL_SOURCE) return; + + if (data.type === "ping") { + window.postMessage({ source: EXT_SOURCE, type: "pong", ok: true }, window.location.origin); + return; + } + + if (data.type === "replay_report" && typeof data.token === "string" && /^https?:$/.test(window.location.protocol)) { + void browser.runtime + .sendMessage({ type: "replay_report", token: data.token }) + .then((response) => { + const r = response as { ok: boolean; data?: unknown; error?: string } | undefined; + window.postMessage( + { source: EXT_SOURCE, type: "replay_result", ok: r?.ok ?? false, data: r?.data ?? null, error: r?.error }, + window.location.origin + ); + }) + .catch((error) => { + // e.g. the extension context is gone after an update + window.postMessage( + { source: EXT_SOURCE, type: "replay_result", ok: false, data: null, error: String(error) }, + window.location.origin + ); + }); + } +}); \ No newline at end of file diff --git a/packages/extension/verify-cursor.mjs b/packages/extension/verify-cursor.mjs new file mode 100644 index 0000000..4d5743a --- /dev/null +++ b/packages/extension/verify-cursor.mjs @@ -0,0 +1,104 @@ +/** Visual check: the virtual cursor walking the recorded path during replay. */ +import { chromium } from "playwright-core"; +import { createServer } from "node:http"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { homedir } from "node:os"; + +const root = dirname(fileURLToPath(import.meta.url)); +const SERVER = "http://localhost:8001"; +const PAGE_PORT = 8898; +const stamp = Date.now(); + +const pageHtml = `Cursor target + +

Hover me

+ +`; +const httpServer = createServer((req, res) => { res.writeHead(200, { "Content-Type": "text/html" }); res.end(pageHtml); }); +await new Promise((resolve) => httpServer.listen(PAGE_PORT, resolve)); + +async function api(path, options = {}) { + const response = await fetch(SERVER + path, options); + if (!response.ok) throw new Error(`${path} -> ${response.status}: ${await response.text()}`); + return response.json(); +} + +const email = `cursor-${stamp}@example.com`; +const login = await api("/api/auth/login", { + method: "POST", headers: { "Content-Type": "application/json", "X-Client": "extension" }, + body: JSON.stringify({ email, password: "cursor-password-1" }), +}).catch(async () => { + await api("/api/auth/register", { method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ nickname: `cursor-${stamp}`, email, password: "cursor-password-1" }) }); + return api("/api/auth/login", { method: "POST", headers: { "Content-Type": "application/json", "X-Client": "extension" }, + body: JSON.stringify({ email, password: "cursor-password-1" }) }); +}); +const project = await api("/api/projects", { + method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${login.token}` }, + body: JSON.stringify({ name: "Cursor check" }), +}); +// a synthetic track: a zigzag mouse path over 2.5 s ending on the button +const points = []; +for (let i = 0; i < 40; i++) { + points.push({ t: i * 50, x: 60 + i * 12, y: 120 + (i % 2) * 90 }); +} +points.push({ t: 2050, x: 160, y: 165 }); +const report = await api("/api/reports", { + method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${login.token}` }, + body: JSON.stringify({ + project_id: project.id, type: "recording", title: "Cursor replay check", + page_url: `http://localhost:${PAGE_PORT}/`, + environment: { viewport: { w: 1280, h: 720 } }, + steps: [{ type: "click", offset_ms: 2100, data: { element: { selector: "#target", unique_selector: "#target" }, button: 0 } }], + mouse_track: { viewport: { w: 1280, h: 720 }, points }, + }), +}); + +const executablePath = join(homedir(), ".cache/ms-playwright/chromium-1234/chrome-linux64/chrome"); +const context = await chromium.launchPersistentContext("", { + executablePath, headless: true, + args: [`--disable-extensions-except=${root}/dist/chrome`, `--load-extension=${root}/dist/chrome`], +}); +let [worker] = context.serviceWorkers(); +if (!worker) worker = await context.waitForEvent("serviceworker", { timeout: 10000 }); +const extensionId = new URL(worker.url()).host; + +const options = await context.newPage(); +await options.goto(`chrome-extension://${extensionId}/src/options/options.html`); +await options.fill('input[type="email"]', email); +await options.fill('input[type="password"]', "cursor-password-1"); +await options.click("text=Sign in"); +await options.waitForSelector("text=Default project", { timeout: 10000 }); +await options.selectOption("select", project.id); + +// trigger the replay from the panel page exactly like the button does +const panel = await context.newPage(); +await panel.goto(`${SERVER.replace("8001", "5173")}/r/${report.share_token}`); +await panel.waitForTimeout(2000); +void panel.evaluate( + (token) => window.postMessage({ source: "bugtrail-panel", type: "replay_report", token }, window.location.origin), + report.share_token +); +// catch the replay tab mid-animation and screenshot the moving cursor +let replayPage = null; +for (let i = 0; i < 60 && !replayPage; i++) { + await new Promise((r) => setTimeout(r, 250)); + replayPage = context.pages().find((c) => c !== panel && c.url().includes(`localhost:${PAGE_PORT}`)); +} +if (!replayPage) throw new Error("replay tab never opened"); +for (let i = 0; i < 40; i++) { + const has = await replayPage.evaluate(() => Boolean(document.getElementById("ltt-cursor-host"))).catch(() => false); + if (has) break; + await replayPage.waitForTimeout(200); +} +await replayPage.waitForTimeout(600); // let it move a bit along the path +await replayPage.screenshot({ path: "/tmp/ltt-shots/30-replay-cursor.png" }); +const pos = await replayPage.evaluate(() => document.getElementById("ltt-cursor-host")?.style.transform ?? null); +console.log("cursor transform mid-replay:", pos); +await new Promise((r) => setTimeout(r, 4000)); +const posEnd = await replayPage.evaluate(() => document.getElementById("ltt-cursor-host")?.style.transform ?? null); +console.log("cursor transform after steps:", posEnd); +await context.close(); +httpServer.close(); +console.log("done"); diff --git a/packages/extension/vite.config.ts b/packages/extension/vite.config.ts index b621e9f..049ffa9 100644 --- a/packages/extension/vite.config.ts +++ b/packages/extension/vite.config.ts @@ -9,7 +9,7 @@ // One build pass per entry (LTT_ENTRY), driven by scripts/build.mjs: // Vite/Rollup cannot emit IIFE for a code-splitting (multi-entry) build, and // content scripts / MV3 service workers must be single self-contained files. -const entryNames = ["background", "content", "options", "popup"] as const; +const entryNames = ["background", "content", "options", "popup", "relay"] as const; type EntryName = (typeof entryNames)[number]; const entryPaths: Record = { @@ -17,6 +17,7 @@ content: resolve(__dirname, "src/content/index.ts"), options: resolve(__dirname, "src/options/options.html"), popup: resolve(__dirname, "src/popup/popup.html"), + relay: resolve(__dirname, "src/relay.ts"), }; // Build targets: --mode chrome | --mode firefox diff --git a/packages/shared/src/api.ts b/packages/shared/src/api.ts index 231fde2..85785b9 100644 --- a/packages/shared/src/api.ts +++ b/packages/shared/src/api.ts @@ -63,6 +63,13 @@ screenshot_attachment_id: string | null; } +export interface MouseTrack { + /** viewport the points were captured in, CSS px */ + viewport: { w: number; h: number }; + /** cursor samples; t = ms since recording start, x/y viewport-relative */ + points: { t: number; x: number; y: number }[]; +} + export interface ElementContext { selector?: string | null; unique_selector?: string | null; @@ -87,6 +94,7 @@ page_title: string | null; environment: Record; element: ElementContext | null; + mouse_track: MouseTrack | null; share_token: string; created_at: string; updated_at: string; @@ -133,6 +141,7 @@ page_title?: string | null; environment?: Record | null; element?: ElementContext | null; + mouse_track?: MouseTrack | null; attachment_ids?: string[] | null; annotation_shapes?: Record | null; steps?: StepIn[] | null; diff --git a/packages/web/src/i18n/en.json b/packages/web/src/i18n/en.json index 1466f20..fcc4baf 100644 --- a/packages/web/src/i18n/en.json +++ b/packages/web/src/i18n/en.json @@ -104,7 +104,14 @@ "navigation": "Navigation", "note": "Note", "screenshot": "Screenshot" - } + }, + "replay": "Start replay", + "replayNoExtension": "BugTrail extension not found in this browser — install it and reload the panel", + "replayNoResponse": "The extension did not respond — make sure you are signed in", + "replayDone": "Replay finished: {played}/{total} steps", + "replayPartialTitle": "Replay finished: {played}/{total} steps", + "replayPartialText": "{failed} steps failed — the page or its elements may have changed", + "replayFailed": "Could not start the replay" }, "settings": { "title": "Settings", diff --git a/packages/web/src/i18n/ru.json b/packages/web/src/i18n/ru.json index 003a58c..cd9139c 100644 --- a/packages/web/src/i18n/ru.json +++ b/packages/web/src/i18n/ru.json @@ -104,7 +104,14 @@ "navigation": "Переход", "note": "Заметка", "screenshot": "Скриншот" - } + }, + "replay": "Начать реплей", + "replayNoExtension": "Расширение BugTrail не найдено в этом браузере — установите его и обновите страницу", + "replayNoResponse": "Расширение не ответило — проверьте, что в нём выполнен вход", + "replayDone": "Реплей завершён: {played}/{total} шагов", + "replayPartialTitle": "Реплей завершён: {played}/{total} шагов", + "replayPartialText": "{failed} шагов не выполнилось — страница или элементы могли измениться", + "replayFailed": "Не удалось запустить реплей" }, "settings": { "title": "Настройки", diff --git a/packages/web/src/pages/ReportPage.vue b/packages/web/src/pages/ReportPage.vue index 1364770..d877be7 100644 --- a/packages/web/src/pages/ReportPage.vue +++ b/packages/web/src/pages/ReportPage.vue @@ -36,6 +36,86 @@ const savingStatus = ref(false); const tab = ref("details"); +const replayBusy = ref(false); + +// ---------- replay via the browser extension ---------- +// The panel can't reach the extension directly (that would need the extension +// id in externally_connectable, which differs per installation), so the +// extension ships a tiny relay content script that listens for window +// messages and forwards them to its background worker. + +const EXT_SOURCE = "bugtrail-ext"; +const PANEL_SOURCE = "bugtrail-panel"; + +interface ReplayResult { + ok: boolean; + data?: { played: number; total: number; failures: { index: number; type: string; reason: string }[] } | null; + error?: string; +} + +/** Sends a message to the relay and waits for the matching reply, or null on timeout. */ +function sendToExtension( + message: Record, + replyType: string, + timeoutMs: number +): Promise { + return new Promise((resolve) => { + const cleanup = () => { + window.removeEventListener("message", handler); + window.clearTimeout(timer); + }; + const handler = (event: MessageEvent) => { + if (event.source !== window) return; + const data = event.data as { source?: string; type?: string } | null; + if (!data || data.source !== EXT_SOURCE || data.type !== replyType) return; + cleanup(); + resolve(data as unknown as T); + }; + const timer = window.setTimeout(() => { + cleanup(); + resolve(null); + }, timeoutMs); + window.addEventListener("message", handler); + window.postMessage({ source: PANEL_SOURCE, ...message }, window.location.origin); + }); +} + +async function startReplay() { + if (!report.value || report.value.type !== "recording" || replayBusy.value) return; + replayBusy.value = true; + try { + // presence check first, so "no extension" fails fast instead of hanging + const pong = await sendToExtension<{ ok: boolean }>({ type: "ping" }, "pong", 1500); + if (!pong) { + toast.warning({ title: t("report.replayNoExtension") }); + return; + } + const result = await sendToExtension( + { type: "replay_report", token: report.value.share_token }, + "replay_result", + 5 * 60 * 1000 + ); + if (!result) { + toast.error({ title: t("report.replayNoResponse") }); + return; + } + if (!result.ok) { + toast.error({ title: t("report.replayFailed"), text: result.error ?? undefined }); + return; + } + const { played, total, failures } = result.data ?? { played: 0, total: 0, failures: [] }; + if (failures?.length) { + toast.warning({ + title: t("report.replayPartialTitle", { played, total }), + text: t("report.replayPartialText", { failed: failures.length }), + }); + } else { + toast.success({ title: t("report.replayDone", { played, total }) }); + } + } finally { + replayBusy.value = false; + } +} const tabs = computed(() => { const items = [{ id: "details", label: t("report.details") }]; if (report.value?.steps?.length) items.push({ id: "steps", label: t("report.steps") }); @@ -178,6 +258,16 @@