Harvest virtualized ChatGPT conversations

2026-07-15 19:09:44 -05:00
parent 3f26b2f01a
commit 8962d0faf8
4 changed files with 875 additions and 137 deletions
@@ -22,11 +22,6 @@ function selectMessageCandidate(candidates, turnRole) {
    || null;
 }

-function parseSequencePosition(section) {
-  const match = (section.getAttribute("data-testid") || "").match(/^conversation-turn-(\d+)$/i);
-  return match ? Number(match[1]) : NaN;
-}
-
 export function discoverChatGPTTurns(root = document) {
  const conversationRoot = findConversationRoot(root);
  if (!conversationRoot) {
@@ -78,7 +73,6 @@ export function discoverChatGPTTurns(root = document) {
      turnIndex: index,
      turnId,
      hasStableTurnId: Boolean(stableTurnId),
-      sequencePosition: parseSequencePosition(section),
      role,
      sourceElement: section,
      messageElement,
@@ -1,89 +1,259 @@
 import { discoverChatGPTTurns } from "./chatgptDiscovery.js";

 const DEFAULT_OPTIONS = {
-  renderDelayMs: 250,
+  renderDelayMs: 300,
  stablePassesRequired: 3,
-  maximumPasses: 80,
-  maximumElapsedMs: 30000,
-  scrollStepRatio: 0.8
+  maximumPasses: 160,
+  maximumElapsedMs: 60000,
+  scrollStepRatio: 0.55,
+  minimumScrollStep: 40,
+  maximumOverlapRetries: 4
 };

 function wait(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }

-function identifyScrollContainer(root) {
-  const conversationRoot = root?.matches?.("#thread") ? root : root?.querySelector?.("#thread");
-  return conversationRoot?.closest?.("[data-scroll-root]") || null;
+function findConversationRoot(root) {
+  return root?.matches?.("#thread") ? root : root?.querySelector?.("#thread") || null;
 }

-function snapshotTurn(turn, firstObservedIndex) {
+function isScrollable(element) {
+  if (!element) {
+    return false;
+  }
+
+  if (element.matches?.("[data-scroll-root]")) {
+    return true;
+  }
+
+  if (element.scrollHeight <= element.clientHeight) {
+    return false;
+  }
+
+  if (typeof getComputedStyle !== "function") {
+    return false;
+  }
+
+  const overflowY = getComputedStyle(element).overflowY;
+  return overflowY === "auto" || overflowY === "scroll";
+}
+
+function identifyScrollContainer(root) {
+  const conversationRoot = findConversationRoot(root);
+  let candidate = conversationRoot?.parentElement || null;
+
+  while (candidate) {
+    if (isScrollable(candidate)) {
+      return candidate;
+    }
+    candidate = candidate.parentElement;
+  }
+
+  return null;
+}
+
+function textFromMessage(messageElement) {
+  return (messageElement?.innerText || messageElement?.textContent || "").replace(/\s+/g, " ").trim();
+}
+
+function snapshotTurn(turn, passNumber) {
  return {
    ...turn,
    sourceElement: turn.sourceElement?.cloneNode?.(true) || null,
    messageElement: turn.messageElement?.cloneNode?.(true) || null,
-    firstObservedIndex
+    firstSeenPass: passNumber,
+    lastSeenPass: passNumber
  };
 }

-function collectRenderedTurns(root, collectedTurns, nextObservedIndex) {
+function reconcileTurn(turnStore, observedTurn, passNumber) {
+  const existing = turnStore.get(observedTurn.turnId);
+  if (!existing) {
+    turnStore.set(observedTurn.turnId, snapshotTurn(observedTurn, passNumber));
+    return true;
+  }
+
+  existing.lastSeenPass = passNumber;
+
+  if (existing.role !== observedTurn.role) {
+    console.warn(`[Thoth] contradictory roles for turn ${observedTurn.turnId}: ${existing.role} vs ${observedTurn.role}`);
+  }
+
+  const existingText = textFromMessage(existing.messageElement);
+  const observedText = textFromMessage(observedTurn.messageElement);
+  if (!existingText && observedText) {
+    existing.messageElement = observedTurn.messageElement.cloneNode(true);
+    existing.sourceElement = observedTurn.sourceElement?.cloneNode?.(true) || existing.sourceElement;
+  } else if (existingText && observedText && existingText !== observedText) {
+    console.warn(`[Thoth] contradictory authored content for turn: ${observedTurn.turnId}`);
+  }
+
+  return false;
+}
+
+function observeWindow(root, turnStore, passNumber, scrollContainer) {
  const renderedTurns = discoverChatGPTTurns(root);
+  const orderedTurnIds = [];
+  const idsInWindow = new Set();
+  let duplicateIds = 0;
  let newUniqueTurns = 0;
-  let observedIndex = nextObservedIndex;

  renderedTurns.forEach((turn) => {
    if (!turn.hasStableTurnId) {
-      console.warn(`[Thoth] turn ID is missing; using fallback identity: ${turn.turnId}`);
+      console.warn(`[Thoth] missing data-turn-id; turn omitted from overlap ordering: ${turn.turnId}`);
+      return;
    }

-    if (!collectedTurns.has(turn.turnId)) {
-      collectedTurns.set(turn.turnId, snapshotTurn(turn, observedIndex));
-      observedIndex += 1;
+    if (idsInWindow.has(turn.turnId)) {
+      duplicateIds += 1;
+      console.error(`[Thoth] repeated ID inside rendered window: ${turn.turnId}`);
+      return;
+    }
+
+    idsInWindow.add(turn.turnId);
+    orderedTurnIds.push(turn.turnId);
+    if (reconcileTurn(turnStore, turn, passNumber)) {
      newUniqueTurns += 1;
    }
  });

  return {
-    renderedTurns,
-    newUniqueTurns,
-    nextObservedIndex: observedIndex
+    passNumber,
+    scrollTop: scrollContainer.scrollTop,
+    scrollHeight: scrollContainer.scrollHeight,
+    clientHeight: scrollContainer.clientHeight,
+    orderedTurnIds,
+    duplicateIds,
+    newUniqueTurns
+  };
+}
+
+function findLongestContiguousOverlap(leftIds, rightIds) {
+  let best = null;
+  let ambiguous = false;
+
+  for (let leftStart = 0; leftStart < leftIds.length; leftStart += 1) {
+    for (let rightStart = 0; rightStart < rightIds.length; rightStart += 1) {
+      let length = 0;
+      while (
+        leftStart + length < leftIds.length
+        && rightStart + length < rightIds.length
+        && leftIds[leftStart + length] === rightIds[rightStart + length]
+      ) {
+        length += 1;
+      }
+
+      if (length === 0) {
+        continue;
+      }
+
+      if (!best || length > best.length) {
+        best = { leftStart, rightStart, length };
+        ambiguous = false;
+      } else if (length === best.length && (leftStart !== best.leftStart || rightStart !== best.rightStart)) {
+        ambiguous = true;
+      }
+    }
+  }
+
+  return best ? { ...best, ambiguous } : { length: 0, ambiguous: false };
+}
+
+function wouldCreateCycle(successors, fromId, toId) {
+  let current = toId;
+  const visited = new Set();
+  while (current && !visited.has(current)) {
+    if (current === fromId) {
+      return true;
+    }
+    visited.add(current);
+    current = successors.get(current);
+  }
+  return false;
+}
+
+function addWindowEdges(windowIds, successors, predecessors) {
+  for (let index = 0; index < windowIds.length - 1; index += 1) {
+    const fromId = windowIds[index];
+    const toId = windowIds[index + 1];
+    const existingSuccessor = successors.get(fromId);
+    const existingPredecessor = predecessors.get(toId);
+
+    if ((existingSuccessor && existingSuccessor !== toId) || (existingPredecessor && existingPredecessor !== fromId)) {
+      console.error(`[Thoth] contradictory ordering: ${fromId} -> ${toId}`);
+      return false;
+    }
+
+    if (!existingSuccessor && wouldCreateCycle(successors, fromId, toId)) {
+      console.error(`[Thoth] contradictory ordering cycle: ${fromId} -> ${toId}`);
+      return false;
+    }
+
+    successors.set(fromId, toId);
+    predecessors.set(toId, fromId);
+  }
+
+  return true;
+}
+
+function countDisconnectedSequences(turnIds, successors, predecessors) {
+  const unvisited = new Set(turnIds);
+  let components = 0;
+
+  while (unvisited.size > 0) {
+    components += 1;
+    const stack = [unvisited.values().next().value];
+    while (stack.length > 0) {
+      const turnId = stack.pop();
+      if (!unvisited.delete(turnId)) {
+        continue;
+      }
+      const successor = successors.get(turnId);
+      const predecessor = predecessors.get(turnId);
+      if (successor) stack.push(successor);
+      if (predecessor) stack.push(predecessor);
+    }
+  }
+
+  return Math.max(0, components - 1);
+}
+
+function reconstructGlobalOrder(turnStore, successors, predecessors) {
+  const turnIds = Array.from(turnStore.keys());
+  const disconnectedSequences = countDisconnectedSequences(turnIds, successors, predecessors);
+  const heads = turnIds.filter((turnId) => !predecessors.has(turnId));
+
+  if (turnIds.length > 0 && heads.length !== 1) {
+    console.error(`[Thoth] disconnected global sequences: ${Math.max(disconnectedSequences, heads.length - 1)}`);
+    return { orderedTurns: [], disconnectedSequences: Math.max(disconnectedSequences, heads.length - 1), complete: false };
+  }
+
+  const orderedTurns = [];
+  const visited = new Set();
+  let current = heads[0];
+  while (current && !visited.has(current)) {
+    visited.add(current);
+    orderedTurns.push(turnStore.get(current));
+    current = successors.get(current);
+  }
+
+  const complete = visited.size === turnIds.length && disconnectedSequences === 0;
+  return {
+    orderedTurns: orderedTurns.map((turn, index) => ({ ...turn, turnIndex: index })),
+    disconnectedSequences,
+    complete
  };
 }

 function hasActiveLoadingIndicator(root) {
-  const conversationRoot = root?.matches?.("#thread") ? root : root?.querySelector?.("#thread");
-  return Boolean(conversationRoot?.querySelector?.('[aria-busy="true"], [role="progressbar"]'));
+  return Boolean(findConversationRoot(root)?.querySelector?.('[aria-busy="true"], [role="progressbar"]'));
 }

-function orderCollectedTurns(collectedTurns) {
-  const turns = Array.from(collectedTurns.values());
-  const sequenceOwners = new Map();
-
-  turns.forEach((turn) => {
-    if (!Number.isFinite(turn.sequencePosition)) {
-      console.warn(`[Thoth] ordering cannot be established from sequence metadata for turn: ${turn.turnId}`);
-      return;
-    }
-
-    if (sequenceOwners.has(turn.sequencePosition)) {
-      console.warn(`[Thoth] inconsistent visible sequence metadata: ${turn.sequencePosition}`);
-    } else {
-      sequenceOwners.set(turn.sequencePosition, turn.turnId);
-    }
-  });
-
-  return turns.sort((left, right) => {
-    const leftHasSequence = Number.isFinite(left.sequencePosition);
-    const rightHasSequence = Number.isFinite(right.sequencePosition);
-
-    if (leftHasSequence && rightHasSequence && left.sequencePosition !== right.sequencePosition) {
-      return left.sequencePosition - right.sequencePosition;
-    }
-    if (leftHasSequence !== rightHasSequence) {
-      return leftHasSequence ? -1 : 1;
-    }
-    return left.firstObservedIndex - right.firstObservedIndex;
-  }).map((turn, index) => ({ ...turn, turnIndex: index }));
+function atEndpoint(observation, direction) {
+  return direction === "up"
+    ? observation.scrollTop <= 1
+    : observation.scrollTop + observation.clientHeight >= observation.scrollHeight - 1;
 }

 function restoreScrollPosition(scrollContainer, originalBottomOffset) {
@@ -92,115 +262,185 @@ function restoreScrollPosition(scrollContainer, originalBottomOffset) {
  return Math.abs(scrollContainer.scrollTop - targetPosition) <= Math.max(1, scrollContainer.clientHeight);
 }

+async function moveAndObserve({
+  root,
+  scrollContainer,
+  turnStore,
+  previousWindow,
+  direction,
+  passState,
+  settings
+}) {
+  const startingPosition = scrollContainer.scrollTop;
+  let step = Math.max(settings.minimumScrollStep, scrollContainer.clientHeight * settings.scrollStepRatio);
+
+  for (let retry = 0; retry <= settings.maximumOverlapRetries; retry += 1) {
+    const target = direction === "up"
+      ? Math.max(0, startingPosition - step)
+      : Math.min(scrollContainer.scrollHeight - scrollContainer.clientHeight, startingPosition + step);
+    scrollContainer.scrollTop = Math.max(0, target);
+    await wait(settings.renderDelayMs);
+
+    passState.value += 1;
+    const observation = observeWindow(root, turnStore, passState.value, scrollContainer);
+    const overlap = findLongestContiguousOverlap(previousWindow.orderedTurnIds, observation.orderedTurnIds);
+
+    console.log(`[Thoth] pass: ${observation.passNumber}`);
+    console.log(`[Thoth] direction: ${direction}`);
+    console.log(`[Thoth] scrollTop: ${observation.scrollTop}`);
+    console.log(`[Thoth] scrollHeight: ${observation.scrollHeight}`);
+    console.log(`[Thoth] clientHeight: ${observation.clientHeight}`);
+    console.log(`[Thoth] rendered window size: ${observation.orderedTurnIds.length}`);
+    console.log(`[Thoth] current window IDs: ${observation.orderedTurnIds.join(",")}`);
+    console.log(`[Thoth] overlap size: ${overlap.length}`);
+    console.log(`[Thoth] new unique turns: ${observation.newUniqueTurns}`);
+    console.log(`[Thoth] total unique turns: ${turnStore.size}`);
+
+    if (observation.duplicateIds > 0) {
+      return { accepted: false, fatal: true, observation, overlap };
+    }
+
+    if (overlap.ambiguous) {
+      console.error("[Thoth] ambiguous overlap between consecutive windows");
+      return { accepted: false, fatal: true, observation, overlap };
+    }
+
+    if (overlap.length > 0) {
+      return { accepted: true, fatal: false, observation, overlap };
+    }
+
+    console.warn(`[Thoth] no overlap; retrying with smaller step (retry ${retry + 1})`);
+    scrollContainer.scrollTop = startingPosition;
+    await wait(settings.renderDelayMs);
+    step /= 2;
+    if (step < settings.minimumScrollStep) {
+      break;
+    }
+  }
+
+  console.error("[Thoth] no overlap could be established after retries");
+  return { accepted: false, fatal: true, observation: null, overlap: { length: 0 } };
+}
+
 export async function prepareChatGPTForCapture(options = {}) {
  const root = options.root || document;
  const settings = { ...DEFAULT_OPTIONS, ...(options.materialization || {}) };
  const scrollContainer = identifyScrollContainer(root);
-  const collectedTurns = new Map();
-  let nextObservedIndex = 0;
-  let pass = 0;
-  let stabilizationCount = 0;
-  let previousPosition = null;
-  let previousHeight = null;
-  let limitReached = false;
+  const turnStore = new Map();
+  const successors = new Map();
+  const predecessors = new Map();
+  const windows = [];
+  const passState = { value: 1 };
+  let duplicateTurnIds = 0;
+  let incompleteReason;
  const startedAt = Date.now();

  if (!scrollContainer) {
-    console.warn("[Thoth] no conversation scroll container identified");
-    const collection = collectRenderedTurns(root, collectedTurns, nextObservedIndex);
-    console.log(`[Thoth] initial rendered turns: ${collection.renderedTurns.length}`);
-    console.log("[Thoth] materialization complete");
-
-    const turns = orderCollectedTurns(collectedTurns);
-    console.log(`[Thoth] user turns collected: ${turns.filter((turn) => turn.role === "user").length}`);
-    console.log(`[Thoth] assistant turns collected: ${turns.filter((turn) => turn.role === "assistant").length}`);
+    console.error("[Thoth] no scroll container");
+    const visibleWindow = observeWindow(
+      root,
+      turnStore,
+      passState.value,
+      { scrollTop: 0, scrollHeight: 0, clientHeight: 0 }
+    );
+    duplicateTurnIds += visibleWindow.duplicateIds;
+    windows.push(visibleWindow);
+    addWindowEdges(visibleWindow.orderedTurnIds, successors, predecessors);
+    const visibleReconstruction = reconstructGlobalOrder(turnStore, successors, predecessors);

    return {
      prepared: false,
-      turns,
-      warning: "The conversation scroll container could not be identified; only currently rendered turns were captured.",
-      passes: 0,
-      limitReached: false,
+      turns: visibleReconstruction.orderedTurns,
+      warning: "The conversation scroll container could not be identified; only the currently rendered window was captured.",
+      windows,
+      duplicateTurnIds,
+      disconnectedSequences: visibleReconstruction.disconnectedSequences,
      positionRestored: false
    };
  }

  console.log("[Thoth] scroll container identified");
+  console.log(`[Thoth] selected scroll container: ${scrollContainer.tagName || "unknown"}, scrollTop=${scrollContainer.scrollTop}, scrollHeight=${scrollContainer.scrollHeight}, clientHeight=${scrollContainer.clientHeight}`);
+  console.log(`[Thoth] original scrollTop: ${scrollContainer.scrollTop}`);

  const originalBottomOffset = scrollContainer.scrollHeight - scrollContainer.scrollTop;
-
-  while (pass < settings.maximumPasses && Date.now() - startedAt < settings.maximumElapsedMs) {
-    pass += 1;
-    console.log(`[Thoth] materialization pass: ${pass}`);
-
-    const collection = collectRenderedTurns(root, collectedTurns, nextObservedIndex);
-    nextObservedIndex = collection.nextObservedIndex;
-    const position = scrollContainer.scrollTop;
-    const height = scrollContainer.scrollHeight;
-    const atOldestPosition = position <= 1;
-    const metricsStable = previousPosition !== null
-      && Math.abs(position - previousPosition) <= 1
-      && Math.abs(height - previousHeight) <= 1;
-    const loading = hasActiveLoadingIndicator(root);
-
-    if (pass === 1) {
-      console.log(`[Thoth] initial rendered turns: ${collection.renderedTurns.length}`);
-    }
-    console.log(`[Thoth] rendered turns this pass: ${collection.renderedTurns.length}`);
-    console.log(`[Thoth] new unique turns this pass: ${collection.newUniqueTurns}`);
-    console.log(`[Thoth] total unique turns collected: ${collectedTurns.size}`);
-    console.log(`[Thoth] scroll position: ${position}`);
-    console.log(`[Thoth] scroll height: ${height}`);
-
-    if (collection.newUniqueTurns === 0 && previousPosition !== null && Math.abs(position - previousPosition) > 1) {
-      console.warn("[Thoth] scroll position changed without revealing new turns");
-    }
-
-    stabilizationCount = atOldestPosition && metricsStable && collection.newUniqueTurns === 0 && !loading
-      ? stabilizationCount + 1
-      : 0;
-    console.log(`[Thoth] stabilization count: ${stabilizationCount}`);
-
-    if (stabilizationCount >= settings.stablePassesRequired) {
-      break;
-    }
-
-    previousPosition = position;
-    previousHeight = height;
-
-    if (atOldestPosition) {
-      scrollContainer.scrollTop = 0;
-    } else {
-      const step = Math.max(1, scrollContainer.clientHeight * settings.scrollStepRatio);
-      scrollContainer.scrollTop = Math.max(0, position - step);
-    }
-
-    await wait(settings.renderDelayMs);
+  let currentWindow = observeWindow(root, turnStore, passState.value, scrollContainer);
+  duplicateTurnIds += currentWindow.duplicateIds;
+  windows.push(currentWindow);
+  if (!addWindowEdges(currentWindow.orderedTurnIds, successors, predecessors)) {
+    incompleteReason = "The initial rendered window contained contradictory ordering.";
  }

-  if (stabilizationCount < settings.stablePassesRequired) {
-    limitReached = true;
-    console.warn("[Thoth] materialization stabilization limit reached");
+  for (const direction of ["up", "down"]) {
+    let stablePasses = 0;
+
+    while (!incompleteReason && stablePasses < settings.stablePassesRequired) {
+      if (passState.value >= settings.maximumPasses || Date.now() - startedAt >= settings.maximumElapsedMs) {
+        incompleteReason = "Conversation harvesting reached a safety limit.";
+        console.error("[Thoth] safety-limit termination");
+        break;
+      }
+
+      const result = await moveAndObserve({
+        root,
+        scrollContainer,
+        turnStore,
+        previousWindow: currentWindow,
+        direction,
+        passState,
+        settings
+      });
+
+      if (!result.accepted) {
+        duplicateTurnIds += result.observation?.duplicateIds || 0;
+        incompleteReason = "A continuous overlap could not be established between rendered windows.";
+        break;
+      }
+
+      if (!addWindowEdges(result.observation.orderedTurnIds, successors, predecessors)) {
+        incompleteReason = "Contradictory ordering was observed between overlapping windows.";
+        break;
+      }
+
+      currentWindow = result.observation;
+      windows.push(currentWindow);
+      const endpointReached = atEndpoint(currentWindow, direction);
+      stablePasses = endpointReached && currentWindow.newUniqueTurns === 0 && !hasActiveLoadingIndicator(root)
+        ? stablePasses + 1
+        : 0;
+    }
+
+    if (!incompleteReason) {
+      console.log(`[Thoth] ${direction === "up" ? "top" : "bottom"} endpoint reached`);
+    }
+  }
+
+  const reconstruction = reconstructGlobalOrder(turnStore, successors, predecessors);
+  if (!reconstruction.complete && !incompleteReason) {
+    incompleteReason = "The harvested windows did not form one continuous global sequence.";
  }

-  const turns = orderCollectedTurns(collectedTurns);
  const positionRestored = restoreScrollPosition(scrollContainer, originalBottomOffset);
  if (!positionRestored) {
-    console.warn("[Thoth] original conversation scroll position could not be restored exactly");
+    console.warn("[Thoth] original scroll position could not be restored approximately");
  }

-  console.log("[Thoth] materialization complete");
-  console.log(`[Thoth] user turns collected: ${turns.filter((turn) => turn.role === "user").length}`);
-  console.log(`[Thoth] assistant turns collected: ${turns.filter((turn) => turn.role === "assistant").length}`);
+  const turns = reconstruction.orderedTurns;
+  const userTurns = turns.filter((turn) => turn.role === "user").length;
+  const assistantTurns = turns.filter((turn) => turn.role === "assistant").length;
+  console.log(`[Thoth] user turns collected: ${userTurns}`);
+  console.log(`[Thoth] assistant turns collected: ${assistantTurns}`);
+  if (!incompleteReason) {
+    console.log("[Thoth] global ordering complete");
+  }

  return {
-    prepared: !limitReached,
+    prepared: !incompleteReason,
    turns,
-    warning: limitReached
-      ? "Conversation materialization reached its safety limit; the export contains every turn observed before capture stopped."
-      : undefined,
-    passes: pass,
-    limitReached,
+    warning: incompleteReason,
+    windows,
+    duplicateTurnIds,
+    disconnectedSequences: reconstruction.disconnectedSequences,
    positionRestored
  };
 }
@@ -0,0 +1,484 @@
+# Work Order: Incrementally Harvest a Virtualized ChatGPT Conversation
+
+## Objective
+
+Refactor the ChatGPT capture connector so it captures a complete conversation from ChatGPT’s virtualized DOM.
+
+The extension must no longer assume that the complete conversation can exist in the DOM simultaneously.
+
+Instead, it must:
+
+1. Traverse the conversation’s scrollable surface.
+2. Observe overlapping windows of rendered turns.
+3. Extract each unique turn when it appears.
+4. Reconstruct the full conversation order using overlap between consecutive windows.
+5. Write the reconstructed conversation to the existing Markdown download pipeline.
+
+Formatting fidelity is not part of this work order.
+
+## Governing context
+
+Before making changes:
+
+1. Locate the .thoth directory at the root of the repository.
+2. Read every document in that directory.
+3. Review the ChatGPT research artifacts in the repository, including:
+    - the DOM specification,
+    - the saved full-page HTML,
+    - previous capture samples,
+    - and relevant earlier work orders.
+4. Treat the DOM specification as the human-authored structural interpretation.
+5. Treat the saved HTML and runtime observations as primary-source evidence.
+
+## Confirmed runtime behavior
+
+Manual DevTools testing established the following:
+
+- ChatGPT uses a virtualized conversation DOM.
+- The complete conversation is not rendered simultaneously.
+- Different scroll positions exposed different subsets of turns.
+- Manual observations found 12, 11, and 9 user turns at different positions.
+- Previously visible turns leave the DOM as other turns enter it.
+- The Jellyfin test conversation contains 28 user turns and corresponding assistant turns.
+- The current extension captures only 4 user turns and 7 assistant turns.
+- The rendered `data-testid="conversation-turn-N"` sequence is reused for each virtualized window.
+- `data-testid="conversation-turn-N"` is therefore not a stable global conversation index.
+- `data-turn-id` is the stable observed identity for a turn.
+- Consecutive virtualized windows must overlap so their local sequences can be joined.
+
+The identified scrollable ancestor was a `div` with runtime values approximately equivalent to:
+
+```
+scrollTop: 50027
+scrollHeight: 67237
+clientHeight: 961
+```
+
+Do not depend on the observed generated CSS class name as a permanent selector.
+
+## Current working behavior
+
+The extension currently:
+
+- loads successfully,
+- identifies rendered section[data-turn] elements,
+- extracts ordinary user and assistant text,
+- avoids systematic duplication within the current DOM window,
+- creates a Markdown file,
+- and saves that file to Downloads.
+
+Preserve these working behaviors.
+
+## Problem statement
+
+The existing implementation captures either:
+
+- the currently rendered window, or
+- an incomplete collection that does not traverse and merge the virtualized conversation correctly.
+
+Scrolling to an endpoint and extracting the final DOM cannot solve this problem because previously rendered turns may have already been removed.
+
+The acquisition process must collect turns during traversal.
+
+## Required conceptual model
+
+Each observation of the DOM produces a locally ordered window:
+
+```
+Window 1:
+A B C D E F
+
+Window 2:
+D E F G H I
+
+Window 3:
+G H I J K L
+```
+
+The overlap provides the ordering relationship:
+
+```
+A B C D E F G H I J K L
+```
+
+Stable identity:
+
+```
+data-turn-id
+```
+
+Reliable local order:
+
+```
+DOM order within the current observation
+```
+
+Unreliable for global order:
+
+```
+data-testid="conversation-turn-N"
+```
+
+## Required data structures
+
+Turn store
+
+Maintain a turn store keyed by stable turn ID:
+
+```
+Map<data-turn-id, TurnRecord>
+```
+
+Each `TurnRecord` should contain at minimum:
+
+```
+turnId
+role
+content
+firstSeenPass
+lastSeenPass
+```
+
+It may also include diagnostic or ordering metadata where useful.
+
+## Ordered window observations
+
+For every collection pass, record the current sequence of stable turn IDs in DOM order:
+
+```
+WindowObservation
+    passNumber
+    scrollTop
+    scrollHeight
+    clientHeight
+    orderedTurnIds[]
+```
+
+## Global ordering representation
+
+Reconstruct the global order using either:
+
+1. Ordered-window overlap merging, or
+2. An adjacency graph derived from locally adjacent turn IDs.
+
+The implementation choice must be documented.
+
+Required implementation behavior
+
+1. Identify the scroll container
+
+Identify the actual conversation scroll container dynamically.
+
+Do not assume `window` is the scroll surface.
+
+Do not rely solely on a generated CSS class.
+
+A defensible approach may inspect ancestors of `#thread` or an element inside it and find the nearest ancestor where:
+
+```
+overflow-y is auto or scroll
+scrollHeight > clientHeight
+```
+
+Log the selected element and its dimensions.
+
+2. Preserve the initial state
+
+Before traversal:
+
+- record the initial scrollTop,
+- observe the current window,
+- collect all currently rendered turns,
+- and initialize the ordered collection.
+
+3. Scroll in overlap-preserving increments
+
+Use a scroll increment small enough that consecutive DOM windows retain shared turn IDs.
+
+Start with an increment based on approximately 50–65% of `clientHeight`.
+
+For an observed `clientHeight` near 961 pixels, an initial step might be around 480–625 pixels.
+
+This is a starting policy, not a hard-coded pixel requirement.
+
+4. Observe after every scroll
+
+After each scroll step:
+
+1. Wait for the virtualized DOM to settle.
+2. Enumerate all:
+```css
+section[data-turn]
+```
+3. Record their data-turn-id values in DOM order.
+4. Extract and store any previously unseen turn.
+5. Update any already-known turn if the new observation supplies content that was previously absent.
+6. Compare the current window with the previous window.
+
+5. Require overlap
+
+Every consecutive window must share at least one stable turn ID unless the traversal is at a known endpoint or an explicitly handled discontinuity.
+
+Prefer more than one overlapping turn where practical.
+
+If no overlap is detected:
+
+1. Do not guess the order.
+2. Retry from the preceding scroll position with a smaller step.
+3. Continue reducing the step within reasonable limits.
+4. Abort with an explicit diagnostic if overlap cannot be established.
+
+6. Merge windows
+
+The implementation must merge each locally ordered window into the accumulated global sequence.
+
+For downward traversal, a common case will be:
+
+```
+master:
+A B C D E F
+
+current:
+D E F G H I
+```
+
+Merge result:
+
+```
+A B C D E F G H I
+```
+
+For upward traversal, the inverse may occur:
+
+```
+current:
+X Y Z A B C
+
+master:
+A B C D E F
+```
+
+Merge result:
+
+```
+X Y Z A B C D E F
+```
+
+Do not append or prepend a window merely because it contains new IDs. The overlap must establish its position.
+
+7. Use the longest valid overlap
+
+When merging windows, prefer the longest contiguous overlap that is consistent with both local sequences.
+
+Detect and report:
+
+- contradictory ordering,
+- repeated IDs inside a window,
+- multiple ambiguous overlaps,
+- and disconnected windows.
+
+Do not silently choose an arbitrary ordering.
+
+8. Traverse the full conversation
+
+Traverse to one endpoint, then through the complete conversation to the opposite endpoint.
+
+A defensible approach is:
+
+1. Begin from the current location.
+2. Harvest while moving to the top.
+3. Confirm the top endpoint.
+4. Harvest while moving from top to bottom.
+5. Confirm the bottom endpoint.
+
+Alternatively, Codex may choose another method if it preserves overlap and can demonstrate full coverage.
+
+9. Extract turn content immediately
+
+When a turn appears in a virtualized window, extract and store its content immediately.
+
+Do not defer content extraction until the end because the node may no longer exist.
+
+Use the documented relationship:
+
+```
+section[data-turn]
+    [data-message-author-role]
+```
+
+Preserve unsupported or empty-content turns as placeholders under the existing behavior.
+
+10. Reconcile repeated observations
+
+When the same turn ID is observed again:
+
+- do not create a duplicate record,
+- confirm that role remains consistent,
+- compare content state,
+- and replace an empty placeholder with authored content if a later observation contains it.
+
+Warn if the same turn ID appears with contradictory roles or materially contradictory authored content.
+
+11. Determine completion
+
+Traversal is complete only when:
+
+- the physical endpoint has been reached,
+- several consecutive observation passes reveal no new stable turn IDs,
+- the final passes maintain valid overlap,
+- and no loading state appears active.
+
+Do not interpret one pass with no new IDs as completion.
+
+12. Restore user position
+
+After capture, restore the user’s approximate original scroll position where practical.
+
+Because virtualization changes `scrollHeight`, exact restoration may not be possible. Report whether restoration succeeded approximately.
+
+13. Generate Markdown from the reconstructed sequence
+
+Once collection is complete:
+
+- iterate the reconstructed global turn sequence,
+- resolve each ID to its stored TurnRecord,
+- and pass that ordered collection into the existing Markdown generation and download pipeline.
+
+Do not generate Markdown from the final live DOM.
+
+## Diagnostics
+
+Log at minimum:
+
+```
+[Thoth] capture requested
+[Thoth] scroll container identified
+[Thoth] original scrollTop: <value>
+[Thoth] pass: <number>
+[Thoth] direction: <up|down>
+[Thoth] scrollTop: <value>
+[Thoth] scrollHeight: <value>
+[Thoth] clientHeight: <value>
+[Thoth] rendered window size: <count>
+[Thoth] current window IDs: <ordered IDs or concise representation>
+[Thoth] overlap size: <count>
+[Thoth] new unique turns: <count>
+[Thoth] total unique turns: <count>
+[Thoth] user turns collected: <count>
+[Thoth] assistant turns collected: <count>
+[Thoth] top endpoint reached
+[Thoth] bottom endpoint reached
+[Thoth] global ordering complete
+[Thoth] turns written: <count>
+[Thoth] download completed
+```
+
+Log explicit errors or warnings for:
+
+- no scroll container,
+- missing data-turn-id,
+- no overlap,
+- ambiguous overlap,
+- contradictory ordering,
+- repeated IDs within one window,
+- contradictory roles,
+- safety-limit termination,
+- incomplete endpoint traversal,
+- or disconnected global sequences.
+
+## Safety limits
+
+Prevent infinite or excessively long traversal.
+
+Use:
+
+- maximum elapsed time,
+- maximum number of passes,
+- minimum scroll-step size,
+- maximum overlap retries per position,
+- and required consecutive stable passes.
+
+If a safety limit is reached:
+
+- do not falsely report complete capture,
+- export only if existing product behavior permits partial export,
+- and include an explicit incompleteness warning with diagnostic counts.
+
+##Constraints
+
+- Do not work on Markdown formatting.
+- Do not add image extraction.
+- Do not add attachment handling.
+- Do not change metadata-generation behavior.
+- Do not redesign the download pipeline.
+- Do not use data-testid="conversation-turn-N" for global ordering.
+- Do not sort by UUID.
+- Do not assume every turn can coexist in the DOM.
+- Do not guess ordering where no overlap exists.
+- Do not perform unrelated refactoring.
+- Preserve Manifest V3 compatibility.
+
+## Acceptance criteria
+
+Using the Jellyfin test conversation:
+
+1. The connector collects all 28 user turns.
+2. The connector collects every corresponding assistant turn represented by a stable data-turn-id.
+3. Every unique data-turn-id appears no more than once in the final ordered collection.
+4. Every consecutive observation window overlaps the previous accepted window by at least one stable turn ID, except for explicitly documented endpoint handling.
+5. No global ordering decision depends on data-testid="conversation-turn-N".
+6. The final sequence forms one continuous ordered conversation.
+7. The Markdown contains all collected turns in reconstructed conversation order.
+8. The connector preserves unsupported non-text turns as placeholders rather than omitting them silently.
+9. The logs show:
+    - number of traversal passes,
+    - overlap size for each accepted pass,
+    - total unique turns,
+    - user-turn count,
+    - assistant-turn count,
+    - endpoint detection,
+    - and final turns written.
+10. A second capture of the same conversation produces the same ordered set of stable turn IDs.
+11. Validate against:
+    - the 28-user-turn Jellyfin conversation,
+    - one short conversation that does not require virtualization traversal,
+    - and one medium-length conversation.
+12. The Jellyfin test must report:
+
+```
+Expected user turns: 28
+Collected user turns: 28
+Duplicate turn IDs: 0
+Disconnected sequences: 0
+```
+
+Any failure of those criteria means the work order is incomplete.
+
+## Required completion report
+
+After implementation, report:
+
+- the dynamically identified scroll container,
+- the chosen scroll-step policy,
+- the DOM-settling policy,
+- the observed window sizes,
+- the overlap-merging algorithm,
+- how missing overlap is retried,
+- the global ordering representation,
+- the endpoint and stabilization conditions,
+- the files changed,
+- expected and collected counts for every test conversation,
+- any unsupported turn types,
+- any safety limit reached,
+- and any remaining known limitation.
+
+## Out of scope
+
+- High-fidelity Markdown
+- Lists, tables, links, citations, and code formatting
+- Image and generated-artifact extraction
+- Attachments
+- Manifest creation
+- Metadata enrichment
+- General architectural cleanup
+- Connector support for other AI platforms
@@ -192,4 +192,24 @@ The capture implementation must therefore retain each observed turn by `data-tur

 ### Ordering evidence

-Observed `data-testid="conversation-turn-N"` values provide numeric sequence evidence and should be used to reconstruct order after incremental collection. UUID turn IDs identify turns but do not encode order and must never be sorted lexically.
+Runtime observation confirms that `data-testid="conversation-turn-N"` values are reused within different virtualized windows. They are local presentation indices and must not be used as global conversation order.
+
+Global ordering must instead be reconstructed from:
+
+- stable identity from `data-turn-id`,
+- DOM order inside each rendered window,
+- and contiguous stable-ID overlap between consecutive windows.
+
+UUID turn IDs identify turns but do not encode order and must never be sorted lexically.
+
+### Confirmed virtualization behavior
+
+Manual runtime observation established that:
+
+- different scroll positions expose different subsets of the conversation,
+- observed windows contained 12, 11, and 9 user turns,
+- turns leave the DOM as other turns enter it,
+- the complete conversation does not coexist in the DOM,
+- and the Jellyfin conversation contains 28 user turns with corresponding assistant turns.
+
+Capture must therefore extract each stable turn immediately and retain ordered window observations. The final live DOM is not a complete source artifact.