From 5b56f354c7fa49d814928cbd0b4f1d48b9ec21e2 Mon Sep 17 00:00:00 2001 From: pionxzh Date: Mon, 6 Jul 2026 03:08:03 +0800 Subject: [PATCH] fix: export answer source lists Fixes #361 --- src/api.ts | 27 +++--- src/constants.ts | 1 + src/exporter/html.ts | 61 ++------------ src/exporter/markdown.ts | 68 ++------------- src/exporter/text.ts | 40 ++------- src/locales/en.json | 3 + src/locales/es.json | 3 + src/locales/fr.json | 3 + src/locales/id.json | 3 + src/locales/jp.json | 3 + src/locales/ru.json | 3 + src/locales/tr.json | 3 + src/locales/zh-Hans.json | 3 + src/locales/zh-Hant.json | 3 + src/ui/SettingContext.tsx | 8 ++ src/ui/SettingDialog.tsx | 14 ++++ src/utils/citations.ts | 170 ++++++++++++++++++++++++++++++++++++++ 17 files changed, 256 insertions(+), 160 deletions(-) create mode 100644 src/utils/citations.ts diff --git a/src/api.ts b/src/api.ts index 6a79bb2..5f0b696 100644 --- a/src/api.ts +++ b/src/api.ts @@ -53,8 +53,15 @@ export interface Citation { } } +export interface ContentReferenceSource { + title?: string + url?: string + attribution?: string + supporting_websites?: ContentReferenceSource[] +} + export interface ContentReference { - type: 'grouped_webpages' | 'sources_footnote' | 'nav_list' | 'alt_text' & (string & {}) + type: 'grouped_webpages' | 'sources_footnote' | 'nav_list' | 'alt_text' | 'webpage' | (string & {}) /** The text that was matched in the content, e.g., "citeturn0search3" */ matched_text?: string start_idx: number @@ -62,20 +69,16 @@ export interface ContentReference { /** Pre-formatted markdown link, e.g., "([Title](url))" */ alt?: string /** Array of actual reference items with URL and title */ - items?: Array<{ - title: string - url: string - attribution?: string - /** Additional sources for multi-citations */ - supporting_websites?: Array<{ - title: string - url: string - attribution?: string - }> - }> + items?: ContentReferenceSource[] + /** Sources shown in ChatGPT's end-of-answer source list */ + sources?: ContentReferenceSource[] + fallback_items?: ContentReferenceSource[] + safe_urls?: string[] + refs?: string[] // Legacy fields (may still be present in some responses) url?: string title?: string + attribution?: string } interface CiteMetadata { diff --git a/src/constants.ts b/src/constants.ts index 3b2b3eb..0d7ec6e 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,7 @@ export const KEY_TIMESTAMP_HTML = 'exporter:timestamp_html' export const KEY_META_ENABLED = 'exporter:enable_meta' export const KEY_META_LIST = 'exporter:meta_list' export const KEY_THINKING_ENABLED = 'exporter:enable_thinking' +export const KEY_SOURCES_ENABLED = 'exporter:enable_sources' export const KEY_EXPORT_ALL_LIMIT = 'exporter:export_all_limit' export const KEY_OAI_LOCALE = 'oai/apps/locale' diff --git a/src/exporter/html.ts b/src/exporter/html.ts index 316cb96..5f8c023 100644 --- a/src/exporter/html.ts +++ b/src/exporter/html.ts @@ -1,9 +1,10 @@ import JSZip from 'jszip' import { fetchConversation, getCurrentChatId, processConversation, shouldSkipMessageInExport } from '../api' -import { KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_HTML, baseUrl } from '../constants' +import { KEY_SOURCES_ENABLED, KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_HTML, baseUrl } from '../constants' import i18n from '../i18n' import { checkIfConversationStarted, getUserAvatar } from '../page' import templateHtml from '../template.html?raw' +import { transformContentReferences } from '../utils/citations' import { buildZipFileName, downloadFile, getFileNameWithFormat } from '../utils/download' import { fromMarkdown, toHtml } from '../utils/markdown' import { ScriptStorage } from '../utils/storage' @@ -80,6 +81,7 @@ function conversationToHtml(conversation: ConversationResult, avatar: string, me const enableTimestamp = ScriptStorage.get(KEY_TIMESTAMP_ENABLED) ?? false const timeStampHtml = ScriptStorage.get(KEY_TIMESTAMP_HTML) ?? false const timeStamp24H = ScriptStorage.get(KEY_TIMESTAMP_24H) ?? false + const enableSources = ScriptStorage.get(KEY_SOURCES_ENABLED) ?? true const LatexRegex = /(\s\$\$.+?\$\$\s|\s\$.+?\$\s|\\\[.+?\\\]|\\\(.+?\\\))|(^\$$[\S\s]+?^\$$)|(^\$\$[\S\s]+?^\$\$\$)/gm @@ -100,7 +102,10 @@ function conversationToHtml(conversation: ConversationResult, avatar: string, me // Handle old-style footnotes (【11†(PrintWiki)】 format) postSteps.push(input => transformFootNotes(input, message.metadata)) // Handle new-style content references (web search citations with Unicode markers) - postSteps.push(input => transformContentReferences(input, message.metadata)) + postSteps.push(input => transformContentReferences(input, message.metadata, { + includeSourceList: enableSources, + sourceListLabel: i18n.t('Sources'), + })) postSteps.push((input) => { const matches = input.match(LatexRegex) @@ -243,58 +248,6 @@ function transformFootNotes( }) } -function transformContentReferences( - input: string, - metadata: ConversationNodeMessage['metadata'], -) { - const contentRefs = metadata?.content_references - if (!contentRefs || contentRefs.length === 0) return input - - const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0)) - - // Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII - const normalize = (s: string) => s - .replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ') - .replaceAll(/[\u2010-\u2015\u2212]/gu, '-') - - let output = normalize(input) - - for (const ref of sortedRefs) { - if (!ref.matched_text) continue - - const matchedText = normalize(ref.matched_text) - - switch (ref.type) { - case 'sources_footnote': - break - case 'grouped_webpages': { - // For citations, build links from items including supporting_websites - const item = ref.items?.[0] - if (item) { - const links: string[] = [] - // Primary source - links.push(`[${item.attribution || item.title}](${item.url})`) - // Supporting sources - for (const sw of item.supporting_websites || []) { - links.push(`[${sw.attribution || sw.title}](${sw.url})`) - } - // Markdown links will be converted to HTML by the subsequent toHtml step - output = output.replaceAll(matchedText, `(${links.join(', ')})`) - } - else { - output = output.replaceAll(matchedText, ref.alt || '') - } - break - } - default: - // Use ref.alt which contains display text or pre-formatted markdown link - // Markdown links will be converted to HTML by the subsequent toHtml step - output = output.replaceAll(matchedText, ref.alt || '') - } - } - return output -} - /** * Convert the content based on the type of message */ diff --git a/src/exporter/markdown.ts b/src/exporter/markdown.ts index 097c540..f3e6fdc 100644 --- a/src/exporter/markdown.ts +++ b/src/exporter/markdown.ts @@ -1,8 +1,9 @@ import JSZip from 'jszip' import { fetchConversation, getCurrentChatId, processConversation, shouldSkipMessageInExport } from '../api' -import { KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_MARKDOWN, baseUrl } from '../constants' +import { KEY_SOURCES_ENABLED, KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_MARKDOWN, baseUrl } from '../constants' import i18n from '../i18n' import { checkIfConversationStarted } from '../page' +import { transformContentReferences } from '../utils/citations' import { buildZipFileName, downloadFile, getFileNameWithFormat } from '../utils/download' import { fromMarkdown, toMarkdown } from '../utils/markdown' import { ScriptStorage } from '../utils/storage' @@ -98,6 +99,7 @@ function conversationToMarkdown(conversation: ConversationResult, metaList?: Exp const enableTimestamp = ScriptStorage.get(KEY_TIMESTAMP_ENABLED) ?? false const timeStampMarkdown = ScriptStorage.get(KEY_TIMESTAMP_MARKDOWN) ?? false const timeStamp24H = ScriptStorage.get(KEY_TIMESTAMP_24H) ?? false + const enableSources = ScriptStorage.get(KEY_SOURCES_ENABLED) ?? true const content = conversationNodes.map(({ message, thinking }) => { if (!message || !message.content) return null @@ -120,7 +122,10 @@ function conversationToMarkdown(conversation: ConversationResult, metaList?: Exp const postSteps: Array<(input: string) => string> = [] if (message.author.role === 'assistant') { // Handle new-style content references (web search citations with Unicode markers) - postSteps.push(input => transformContentReferences(input, message.metadata)) + postSteps.push(input => transformContentReferences(input, message.metadata, { + includeSourceList: enableSources, + sourceListLabel: i18n.t('Sources'), + })) // Handle old-style footnotes (【11†(PrintWiki)】 format) postSteps.push(input => transformFootNotes(input, message.metadata)) } @@ -202,6 +207,7 @@ function transformFootNotes( return match }) + const citationText = citationList.map((citation) => { const citeIndex = citation.metadata?.extra?.cited_message_idx ?? 1 const citeTitle = citation.metadata?.title ?? 'No title' @@ -212,64 +218,6 @@ function transformFootNotes( return `${output}\n\n${citationText}` } -/** - * Transform new-style content references (web search citations) in assistant's message. - * Citations appear as plain text like "citeturn1search1" or "citeturn0search2turn1search8". - * The metadata.content_references array contains the URL and title for each citation. - */ -function transformContentReferences( - input: string, - metadata: ConversationNodeMessage['metadata'], -): string { - const contentRefs = metadata?.content_references - if (!contentRefs || contentRefs.length === 0) return input - - // Sort by matched_text length descending to match longer patterns first - // (e.g., "citeturn0search2turn1search8" before "citeturn0search2") - const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0)) - - // Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII - const normalize = (s: string) => s - .replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ') - .replaceAll(/[\u2010-\u2015\u2212]/gu, '-') - - let output = normalize(input) - - for (const ref of sortedRefs) { - if (!ref.matched_text) continue - - const matchedText = normalize(ref.matched_text) - - switch (ref.type) { - case 'sources_footnote': - break - case 'grouped_webpages': { - // For citations, build links from items including supporting_websites - const item = ref.items?.[0] - if (item) { - const links: string[] = [] - // Primary source - links.push(`[${item.attribution || item.title}](${item.url})`) - // Supporting sources - for (const sw of item.supporting_websites || []) { - links.push(`[${sw.attribution || sw.title}](${sw.url})`) - } - output = output.replaceAll(matchedText, `(${links.join(', ')})`) - } - else { - output = output.replaceAll(matchedText, ref.alt || '') - } - break - } - default: - // Use ref.alt which contains display text or pre-formatted markdown link - output = output.replaceAll(matchedText, ref.alt || '') - } - } - - return output -} - /** * Convert the content based on the type of message */ diff --git a/src/exporter/text.ts b/src/exporter/text.ts index 409a0d7..8bd3efe 100644 --- a/src/exporter/text.ts +++ b/src/exporter/text.ts @@ -1,6 +1,7 @@ import { fetchConversation, getCurrentChatId, processConversation, shouldSkipMessageInExport } from '../api' import i18n from '../i18n' import { checkIfConversationStarted } from '../page' +import { transformContentReferences } from '../utils/citations' import { copyToClipboard } from '../utils/clipboard' import { flatMap, fromMarkdown, toMarkdown } from '../utils/markdown' import { standardizeLineBreaks } from '../utils/text' @@ -49,7 +50,11 @@ function transformMessage(message?: ConversationNodeMessage) { } if (message.author.role === 'assistant') { - content = transformContentReferences(content, message.metadata) + content = transformContentReferences(content, message.metadata, { + output: 'text', + inlineReferenceMode: 'alt', + includeSourceList: false, + }) content = transformFootNotes(content, message.metadata) } @@ -150,38 +155,6 @@ function transformAuthor(author: ConversationNodeMessage['author']): string { } } -function transformContentReferences( - input: string, - metadata: ConversationNodeMessage['metadata'], -) { - const contentRefs = metadata?.content_references - if (!contentRefs || contentRefs.length === 0) return input - - const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0)) - - // Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII - const normalize = (s: string) => s - .replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ') - .replaceAll(/[\u2010-\u2015\u2212]/gu, '-') - - let output = normalize(input) - - for (const ref of sortedRefs) { - if (!ref.matched_text) continue - - const matchedText = normalize(ref.matched_text) - - switch (ref.type) { - case 'sources_footnote': - break - default: - // Use ref.alt which contains display text (links won't render in plain text) - output = output.replaceAll(matchedText, ref.alt || '') - } - } - return output -} - /** * Transform foot notes in assistant's message */ @@ -193,7 +166,6 @@ function transformFootNotes( const footNoteMarkRegex = /【(\d+)†\((.+?)\)】/g return input.replace(footNoteMarkRegex, (match, citeIndex, _evidenceText) => { const citation = metadata?.citations?.find(cite => cite.metadata?.extra?.cited_message_idx === +citeIndex) - // We simply remove the foot note mark in text output if (citation) return '' return match diff --git a/src/locales/en.json b/src/locales/en.json index a135f73..821318e 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -35,6 +35,9 @@ "Export Metadata Description": "Add metadata to exported Markdown and HTML files.", "Export Thinking Process": "Export Thinking Process", "Export Thinking Process Description": "Include the model's thinking/reasoning process in exported Markdown and HTML files.", + "Export Sources": "Export Sources", + "Export Sources Description": "Include the source list shown at the end of each answer in exported Markdown and HTML files.", + "Sources": "Sources", "OpenAI Official Format": "OpenAI Official Format", "Conversation Archive Alert": "Are you sure you want to archive all selected conversations?", "Conversation Archived Message": "All selected conversations have been archived. Please refresh the page to see the changes.", diff --git a/src/locales/es.json b/src/locales/es.json index a9bbb9c..21ac2fb 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -35,6 +35,9 @@ "Export Metadata Description": "Añadir Metadatos a los archivos Markdown y HTML exportados.", "Export Thinking Process": "Exportar Proceso de Pensamiento", "Export Thinking Process Description": "Incluir el proceso de pensamiento/razonamiento del modelo en los archivos Markdown y HTML exportados.", + "Export Sources": "Exportar fuentes", + "Export Sources Description": "Incluir la lista de fuentes que se muestra al final de cada respuesta en los archivos Markdown y HTML exportados.", + "Sources": "Fuentes", "OpenAI Official Format": "Formato Oficial de OpenAI", "Conversation Archive Alert": "¿Estás seguro que quieres archivar todas las conversaciones seleccionadas?", "Conversation Archived Message": "Todos las conversaciones seleccionadas se han archivado. Por favor refresca la página para ver los cambios.", diff --git a/src/locales/fr.json b/src/locales/fr.json index 884ce4c..83d92c0 100644 --- a/src/locales/fr.json +++ b/src/locales/fr.json @@ -35,6 +35,9 @@ "Export Metadata Description": "Ajouter des métadonnées aux fichiers Markdown et HTML exportés.", "Export Thinking Process": "Exporter le processus de réflexion", "Export Thinking Process Description": "Inclure le processus de réflexion/raisonnement du modèle dans les fichiers Markdown et HTML exportés.", + "Export Sources": "Exporter les sources", + "Export Sources Description": "Inclure la liste des sources affichée à la fin de chaque réponse dans les fichiers Markdown et HTML exportés.", + "Sources": "Sources", "OpenAI Official Format": "Format officiel OpenAI", "Conversation Archive Alert": "Êtes-vous sûr de vouloir archiver toutes les conversations sélectionnées ?", "Conversation Archived Message": "Toutes les conversations sélectionnées ont été archivées. Veuillez actualiser la page pour voir les changements.", diff --git a/src/locales/id.json b/src/locales/id.json index a684d47..8ecd8ca 100644 --- a/src/locales/id.json +++ b/src/locales/id.json @@ -35,6 +35,9 @@ "Export Metadata Description": "Tambahkan metadata ke file Markdown dan HTML yang diekspor.", "Export Thinking Process": "Ekspor Proses Berpikir", "Export Thinking Process Description": "Sertakan proses berpikir/penalaran model dalam file Markdown dan HTML yang diekspor.", + "Export Sources": "Ekspor Sumber", + "Export Sources Description": "Sertakan daftar sumber yang ditampilkan di akhir setiap jawaban dalam file Markdown dan HTML yang diekspor.", + "Sources": "Sumber", "OpenAI Official Format": "Format Resmi OpenAI", "Conversation Archive Alert": "Apakah Anda yakin ingin mengarsipkan semua percakapan yang dipilih?", "Conversation Archived Message": "Semua percakapan yang dipilih telah diarsipkan. Harap segarkan halaman untuk melihat perubahan.", diff --git a/src/locales/jp.json b/src/locales/jp.json index 704f03f..5a4a508 100644 --- a/src/locales/jp.json +++ b/src/locales/jp.json @@ -35,6 +35,9 @@ "Export Metadata Description": "エクスポートされたMarkdownおよびHTMLファイルにメタデータを追加します。", "Export Thinking Process": "思考プロセスをエクスポート", "Export Thinking Process Description": "エクスポートされたMarkdownおよびHTMLファイルにモデルの思考・推論プロセスを含めます。", + "Export Sources": "ソースをエクスポート", + "Export Sources Description": "各回答の末尾に表示されるソース一覧を、エクスポートされた Markdown および HTML ファイルに含めます。", + "Sources": "出典", "OpenAI Official Format": "OpenAI公式フォーマット", "Conversation Archive Alert": "選択したすべての会話をアーカイブしてもよろしいですか？", "Conversation Archived Message": "選択したすべての会話がアーカイブされました。変更を表示するには、ページを更新してください。", diff --git a/src/locales/ru.json b/src/locales/ru.json index e94515d..5d59768 100644 --- a/src/locales/ru.json +++ b/src/locales/ru.json @@ -35,6 +35,9 @@ "Export Metadata Description": "Добавляйте метаданные в экспортированные файлы Markdown и HTML.", "Export Thinking Process": "Экспорт процесса мышления", "Export Thinking Process Description": "Включить процесс мышления/рассуждения модели в экспортированные файлы Markdown и HTML.", + "Export Sources": "Экспорт источников", + "Export Sources Description": "Включить список источников, показанный в конце каждого ответа, в экспортированные файлы Markdown и HTML.", + "Sources": "Источники", "OpenAI Official Format": "Официальный формат OpenAI", "Conversation Archive Alert": "Вы уверены, что хотите архивировать все выбранные разговоры?", "Conversation Archived Message": "Все выбранные разговоры были заархивированы. Пожалуйста, обновите страницу, чтобы увидеть изменения.", diff --git a/src/locales/tr.json b/src/locales/tr.json index 77a78d0..bd04c47 100644 --- a/src/locales/tr.json +++ b/src/locales/tr.json @@ -35,6 +35,9 @@ "Export Metadata Description": "Dışa aktarılan Markdown ve HTML dosyalarına üst veri ekle", "Export Thinking Process": "Düşünme Sürecini Dışa Aktar", "Export Thinking Process Description": "Dışa aktarılan Markdown ve HTML dosyalarına modelin düşünme/akıl yürütme sürecini dahil et.", + "Export Sources": "Kaynakları Dışa Aktar", + "Export Sources Description": "Her yanıtın sonunda gösterilen kaynak listesini dışa aktarılan Markdown ve HTML dosyalarına dahil et.", + "Sources": "Kaynaklar", "OpenAI Official Format": "OpenAI Resmi Format", "Conversation Archive Alert": "Seçilen tüm konuşmaları arşivlemek istediğinizden emin misiniz?", "Conversation Archived Message": "Seçilen tüm konuşmalar arşivlendi. Değişiklikleri görmek için sayfayı yenileyin.", diff --git a/src/locales/zh-Hans.json b/src/locales/zh-Hans.json index c48185c..66b7043 100644 --- a/src/locales/zh-Hans.json +++ b/src/locales/zh-Hans.json @@ -35,6 +35,9 @@ "Export Metadata Description": "会添加至 Markdown 以及 HTML 导出。", "Export Thinking Process": "导出思考过程", "Export Thinking Process Description": "在导出的 Markdown 和 HTML 文件中包含模型的思考/推理过程。", + "Export Sources": "导出来源", + "Export Sources Description": "在导出的 Markdown 和 HTML 文件中包含每个回答末尾显示的来源列表。", + "Sources": "来源", "OpenAI Official Format": "OpenAI 官方格式", "Conversation Archive Alert": "确定要归档所有选取的对话？", "Conversation Archived Message": "所有所选的对话已归档。请刷新页面。", diff --git a/src/locales/zh-Hant.json b/src/locales/zh-Hant.json index f929d68..3c3969a 100644 --- a/src/locales/zh-Hant.json +++ b/src/locales/zh-Hant.json @@ -35,6 +35,9 @@ "Export Metadata Description": "會添加至 Markdown 以及 HTML 匯出。", "Export Thinking Process": "匯出思考過程", "Export Thinking Process Description": "在匯出的 Markdown 和 HTML 檔案中包含模型的思考/推理過程。", + "Export Sources": "匯出來源", + "Export Sources Description": "在匯出的 Markdown 和 HTML 檔案中包含每個回答末尾顯示的來源列表。", + "Sources": "來源", "OpenAI Official Format": "OpenAI 官方格式", "Conversation Archive Alert": "確定要封存所有選取的對話？", "Conversation Archived Message": "所有選取的對話已封存。請重新整理頁面。", diff --git a/src/ui/SettingContext.tsx b/src/ui/SettingContext.tsx index 7bb2690..6d27800 100644 --- a/src/ui/SettingContext.tsx +++ b/src/ui/SettingContext.tsx @@ -5,6 +5,7 @@ import { KEY_FILENAME_FORMAT, KEY_META_ENABLED, KEY_META_LIST, + KEY_SOURCES_ENABLED, KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, @@ -46,6 +47,8 @@ const SettingContext = createContext({ setExportMetaList: (_: ExportMeta[]) => {}, enableThinking: false, setEnableThinking: (_: boolean) => {}, + enableSources: true, + setEnableSources: (_: boolean) => {}, exportAllLimit: defaultExportAllLimit, setExportAllLimit: (_: number) => {}, resetDefault: () => {}, @@ -63,6 +66,7 @@ export const SettingProvider: FC = ({ children }) => { const [exportMetaList, setExportMetaList] = useGMStorage(KEY_META_LIST, defaultExportMetaList) const [enableThinking, setEnableThinking] = useGMStorage(KEY_THINKING_ENABLED, false) + const [enableSources, setEnableSources] = useGMStorage(KEY_SOURCES_ENABLED, true) const [exportAllLimit, setExportAllLimit] = useGMStorage(KEY_EXPORT_ALL_LIMIT, defaultExportAllLimit) const resetDefault = useCallback(() => { @@ -71,6 +75,7 @@ export const SettingProvider: FC = ({ children }) => { setEnableMeta(false) setExportMetaList(defaultExportMetaList) setEnableThinking(false) + setEnableSources(true) setExportAllLimit(defaultExportAllLimit) }, [ setFormat, @@ -78,6 +83,7 @@ export const SettingProvider: FC = ({ children }) => { setEnableMeta, setExportMetaList, setEnableThinking, + setEnableSources, setExportAllLimit, ]) @@ -103,6 +109,8 @@ export const SettingProvider: FC = ({ children }) => { enableThinking, setEnableThinking, + enableSources, + setEnableSources, exportAllLimit, setExportAllLimit, diff --git a/src/ui/SettingDialog.tsx b/src/ui/SettingDialog.tsx index a28b281..75cef41 100644 --- a/src/ui/SettingDialog.tsx +++ b/src/ui/SettingDialog.tsx @@ -36,6 +36,7 @@ export const SettingDialog: FC = ({ enableMeta, setEnableMeta, exportMetaList, setExportMetaList, enableThinking, setEnableThinking, + enableSources, setEnableSources, exportAllLimit, setExportAllLimit, /* eslint-enable pionxzh/consistent-list-newline */ } = useSettingContext() @@ -125,6 +126,19 @@ export const SettingDialog: FC = ({ +
+
+
+ {t('Export Sources')} +
+
+ {t('Export Sources Description')} +
+
+
+ +
+
diff --git a/src/utils/citations.ts b/src/utils/citations.ts new file mode 100644 index 0000000..7590d64 --- /dev/null +++ b/src/utils/citations.ts @@ -0,0 +1,170 @@ +import type { ContentReference, ContentReferenceSource, ConversationNodeMessage } from '../api' + +type CitationOutput = 'markdown' | 'text' + +interface TransformContentReferenceOptions { + output?: CitationOutput + inlineReferenceMode?: 'expanded' | 'alt' + includeSourceList?: boolean + sourceListLabel?: string +} + +const CitationMarkerRegex = /\uE200cite(?:\uE202[^\uE200\uE201]*)+\uE201/gu + +export function normalizeCitationText(input: string): string { + return input + .replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ') + .replaceAll(/[\u2010-\u2015\u2212]/gu, '-') + .replaceAll(/[\uE203\uE204]/gu, '') +} + +export function transformContentReferences( + input: string, + metadata: ConversationNodeMessage['metadata'], + options: TransformContentReferenceOptions = {}, +): string { + const outputType = options.output ?? 'markdown' + const inlineReferenceMode = options.inlineReferenceMode ?? 'expanded' + const contentRefs = metadata?.content_references ?? [] + let output = normalizeCitationText(input) + + const sortedRefs = [...contentRefs] + .filter(ref => ref.type !== 'sources_footnote') + .sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0)) + + for (const ref of sortedRefs) { + if (!ref.matched_text) continue + + const matchedText = normalizeCitationText(ref.matched_text) + if (!matchedText) continue + + const replacement = formatInlineReference(ref, outputType, inlineReferenceMode) + output = output.replaceAll(matchedText, replacement) + } + + output = output.replace(CitationMarkerRegex, '') + + if (options.includeSourceList !== false) { + const sources = getSourcesFootnoteSources(contentRefs) + if (sources.length > 0) { + output = appendSourcesSection(output, sources, outputType, options.sourceListLabel ?? 'Sources') + } + } + + return output +} + +export function formatCitationSource(source: ContentReferenceSource, output: CitationOutput): string { + const label = getSourceLabel(source) + const url = source.url?.trim() + + if (!url) return output === 'markdown' ? escapeMarkdownText(label) : label + + if (output === 'text') { + return `${label}: ${url}` + } + + return `[${escapeMarkdownText(label)}](<${escapeMarkdownUrl(url)}>)` +} + +function formatInlineReference(ref: ContentReference, output: CitationOutput, mode: 'expanded' | 'alt'): string { + if (mode === 'alt') return ref.alt || '' + + const sources = getInlineSources(ref) + + if (sources.length > 0) { + const separator = output === 'markdown' ? ', ' : '; ' + return `(${sources.map(source => formatCitationSource(source, output)).join(separator)})` + } + + if (ref.alt) return ref.alt + + return '' +} + +function getInlineSources(ref: ContentReference): ContentReferenceSource[] { + const sources: ContentReferenceSource[] = [] + + for (const item of ref.items ?? []) { + sources.push(item) + sources.push(...(item.supporting_websites ?? [])) + } + + sources.push(...(ref.fallback_items ?? [])) + + if (sources.length === 0 && (ref.url || ref.title || ref.attribution)) { + sources.push(ref) + } + + if (sources.length === 0 && ref.safe_urls?.length) { + sources.push(...ref.safe_urls.map(url => ({ title: url, url }))) + } + + return dedupeSources(sources) +} + +function getSourcesFootnoteSources(contentRefs: ContentReference[]): ContentReferenceSource[] { + const sources = contentRefs + .filter(ref => ref.type === 'sources_footnote') + .flatMap((ref) => { + if (ref.sources?.length) return ref.sources + if (ref.items?.length) return ref.items + if (ref.fallback_items?.length) return ref.fallback_items + if (ref.safe_urls?.length) return ref.safe_urls.map(url => ({ title: url, url })) + return [] + }) + + return dedupeSources(sources) +} + +function appendSourcesSection(input: string, sources: ContentReferenceSource[], output: CitationOutput, label: string): string { + const trimmed = input.trimEnd() + const sourceList = output === 'markdown' + ? [ + `**${escapeMarkdownText(label)}:**`, + '', + ...sources.map(source => `- ${formatCitationSource(source, output)}`), + ].join('\n') + : [ + `${label}:`, + ...sources.map((source, index) => `${index + 1}. ${formatCitationSource(source, output)}`), + ].join('\n') + + return trimmed ? `${trimmed}\n\n${sourceList}` : sourceList +} + +function dedupeSources(sources: ContentReferenceSource[]): ContentReferenceSource[] { + const seen = new Set() + const result: ContentReferenceSource[] = [] + + for (const source of sources) { + const key = source.url?.trim() || getSourceLabel(source) + if (!key || seen.has(key)) continue + seen.add(key) + result.push(source) + } + + return result +} + +function getSourceLabel(source: ContentReferenceSource): string { + return source.attribution?.trim() + || source.title?.trim() + || source.url?.trim() + || 'Source' +} + +function escapeMarkdownText(input: string): string { + return input + .replaceAll('\\', '\\\\') + .replaceAll('[', '\\[') + .replaceAll(']', '\\]') + .replaceAll('\n', ' ') +} + +function escapeMarkdownUrl(input: string): string { + return input + .replaceAll('<', '%3C') + .replaceAll('>', '%3E') + .replaceAll('\n', '') +}