upstream-archive/chatgpt-exporter
1
0
Fork 0
mirror of https://github.com/pionxzh/chatgpt-exporter.git synced 2026-07-23 09:00:51 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: export answer source lists

Browse Source 
Fixes #361
This commit is contained in:
pionxzh
2026-07-06 03:08:03 +08:00
parent f5ef7fe301
commit 5b56f354c7
17 changed files with 256 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/api.ts
View File

@@ -53,8 +53,15 @@ export interface Citation {
}
}
export interface ContentReferenceSource {
title?: string
url?: string
attribution?: string
supporting_websites?: ContentReferenceSource[]
}
export interface ContentReference {
type: 'grouped_webpages' | 'sources_footnote' | 'nav_list' | 'alt_text' & (string & {})
type: 'grouped_webpages' | 'sources_footnote' | 'nav_list' | 'alt_text' | 'webpage' | (string & {})
/** The text that was matched in the content, e.g., "citeturn0search3" */
matched_text?: string
start_idx: number
@@ -62,20 +69,16 @@ export interface ContentReference {
/** Pre-formatted markdown link, e.g., "([Title](url))" */
alt?: string
/** Array of actual reference items with URL and title */
items?: Array<{
title: string
url: string
attribution?: string
/** Additional sources for multi-citations */
supporting_websites?: Array<{
title: string
url: string
attribution?: string
}>
}>
items?: ContentReferenceSource[]
/** Sources shown in ChatGPT's end-of-answer source list */
sources?: ContentReferenceSource[]
fallback_items?: ContentReferenceSource[]
safe_urls?: string[]
refs?: string[]
// Legacy fields (may still be present in some responses)
url?: string
title?: string
attribution?: string
}
interface CiteMetadata {

1
src/constants.ts
View File

@@ -15,6 +15,7 @@ export const KEY_TIMESTAMP_HTML = 'exporter:timestamp_html'
export const KEY_META_ENABLED = 'exporter:enable_meta'
export const KEY_META_LIST = 'exporter:meta_list'
export const KEY_THINKING_ENABLED = 'exporter:enable_thinking'
export const KEY_SOURCES_ENABLED = 'exporter:enable_sources'
export const KEY_EXPORT_ALL_LIMIT = 'exporter:export_all_limit'
export const KEY_OAI_LOCALE = 'oai/apps/locale'

61
src/exporter/html.ts
View File

@@ -1,9 +1,10 @@
import JSZip from 'jszip'
import { fetchConversation, getCurrentChatId, processConversation, shouldSkipMessageInExport } from '../api'
import { KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_HTML, baseUrl } from '../constants'
import { KEY_SOURCES_ENABLED, KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_HTML, baseUrl } from '../constants'
import i18n from '../i18n'
import { checkIfConversationStarted, getUserAvatar } from '../page'
import templateHtml from '../template.html?raw'
import { transformContentReferences } from '../utils/citations'
import { buildZipFileName, downloadFile, getFileNameWithFormat } from '../utils/download'
import { fromMarkdown, toHtml } from '../utils/markdown'
import { ScriptStorage } from '../utils/storage'
@@ -80,6 +81,7 @@ function conversationToHtml(conversation: ConversationResult, avatar: string, me
const enableTimestamp = ScriptStorage.get<boolean>(KEY_TIMESTAMP_ENABLED) ?? false
const timeStampHtml = ScriptStorage.get<boolean>(KEY_TIMESTAMP_HTML) ?? false
const timeStamp24H = ScriptStorage.get<boolean>(KEY_TIMESTAMP_24H) ?? false
const enableSources = ScriptStorage.get<boolean>(KEY_SOURCES_ENABLED) ?? true
const LatexRegex = /(\s\$\$.+?\$\$\s|\s\$.+?\$\s|\\\[.+?\\\]|\\\(.+?\\\))|(^\$$[\S\s]+?^\$$)|(^\$\$[\S\s]+?^\$\$\$)/gm
@@ -100,7 +102,10 @@ function conversationToHtml(conversation: ConversationResult, avatar: string, me
// Handle old-style footnotes (【11†(PrintWiki)】 format)
postSteps.push(input => transformFootNotes(input, message.metadata))
// Handle new-style content references (web search citations with Unicode markers)
postSteps.push(input => transformContentReferences(input, message.metadata))
postSteps.push(input => transformContentReferences(input, message.metadata, {
includeSourceList: enableSources,
sourceListLabel: i18n.t('Sources'),
}))
postSteps.push((input) => {
const matches = input.match(LatexRegex)
@@ -243,58 +248,6 @@ function transformFootNotes(
})
}
function transformContentReferences(
input: string,
metadata: ConversationNodeMessage['metadata'],
) {
const contentRefs = metadata?.content_references
if (!contentRefs || contentRefs.length === 0) return input
const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
// Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII
const normalize = (s: string) => s
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
let output = normalize(input)
for (const ref of sortedRefs) {
if (!ref.matched_text) continue
const matchedText = normalize(ref.matched_text)
switch (ref.type) {
case 'sources_footnote':
break
case 'grouped_webpages': {
// For citations, build links from items including supporting_websites
const item = ref.items?.[0]
if (item) {
const links: string[] = []
// Primary source
links.push(`[${item.attribution || item.title}](${item.url})`)
// Supporting sources
for (const sw of item.supporting_websites || []) {
links.push(`[${sw.attribution || sw.title}](${sw.url})`)
}
// Markdown links will be converted to HTML by the subsequent toHtml step
output = output.replaceAll(matchedText, `(${links.join(', ')})`)
}
else {
output = output.replaceAll(matchedText, ref.alt || '')
}
break
}
default:
// Use ref.alt which contains display text or pre-formatted markdown link
// Markdown links will be converted to HTML by the subsequent toHtml step
output = output.replaceAll(matchedText, ref.alt || '')
}
}
return output
}
/**
* Convert the content based on the type of message
*/

68
src/exporter/markdown.ts
View File

@@ -1,8 +1,9 @@
import JSZip from 'jszip'
import { fetchConversation, getCurrentChatId, processConversation, shouldSkipMessageInExport } from '../api'
import { KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_MARKDOWN, baseUrl } from '../constants'
import { KEY_SOURCES_ENABLED, KEY_THINKING_ENABLED, KEY_TIMESTAMP_24H, KEY_TIMESTAMP_ENABLED, KEY_TIMESTAMP_MARKDOWN, baseUrl } from '../constants'
import i18n from '../i18n'
import { checkIfConversationStarted } from '../page'
import { transformContentReferences } from '../utils/citations'
import { buildZipFileName, downloadFile, getFileNameWithFormat } from '../utils/download'
import { fromMarkdown, toMarkdown } from '../utils/markdown'
import { ScriptStorage } from '../utils/storage'
@@ -98,6 +99,7 @@ function conversationToMarkdown(conversation: ConversationResult, metaList?: Exp
const enableTimestamp = ScriptStorage.get<boolean>(KEY_TIMESTAMP_ENABLED) ?? false
const timeStampMarkdown = ScriptStorage.get<boolean>(KEY_TIMESTAMP_MARKDOWN) ?? false
const timeStamp24H = ScriptStorage.get<boolean>(KEY_TIMESTAMP_24H) ?? false
const enableSources = ScriptStorage.get<boolean>(KEY_SOURCES_ENABLED) ?? true
const content = conversationNodes.map(({ message, thinking }) => {
if (!message || !message.content) return null
@@ -120,7 +122,10 @@ function conversationToMarkdown(conversation: ConversationResult, metaList?: Exp
const postSteps: Array<(input: string) => string> = []
if (message.author.role === 'assistant') {
// Handle new-style content references (web search citations with Unicode markers)
postSteps.push(input => transformContentReferences(input, message.metadata))
postSteps.push(input => transformContentReferences(input, message.metadata, {
includeSourceList: enableSources,
sourceListLabel: i18n.t('Sources'),
}))
// Handle old-style footnotes (【11†(PrintWiki)】 format)
postSteps.push(input => transformFootNotes(input, message.metadata))
}
@@ -202,6 +207,7 @@ function transformFootNotes(
return match
})
const citationText = citationList.map((citation) => {
const citeIndex = citation.metadata?.extra?.cited_message_idx ?? 1
const citeTitle = citation.metadata?.title ?? 'No title'
@@ -212,64 +218,6 @@ function transformFootNotes(
return `${output}\n\n${citationText}`
}
/**
* Transform new-style content references (web search citations) in assistant's message.
* Citations appear as plain text like "citeturn1search1" or "citeturn0search2turn1search8".
* The metadata.content_references array contains the URL and title for each citation.
*/
function transformContentReferences(
input: string,
metadata: ConversationNodeMessage['metadata'],
): string {
const contentRefs = metadata?.content_references
if (!contentRefs || contentRefs.length === 0) return input
// Sort by matched_text length descending to match longer patterns first
// (e.g., "citeturn0search2turn1search8" before "citeturn0search2")
const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
// Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII
const normalize = (s: string) => s
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
let output = normalize(input)
for (const ref of sortedRefs) {
if (!ref.matched_text) continue
const matchedText = normalize(ref.matched_text)
switch (ref.type) {
case 'sources_footnote':
break
case 'grouped_webpages': {
// For citations, build links from items including supporting_websites
const item = ref.items?.[0]
if (item) {
const links: string[] = []
// Primary source
links.push(`[${item.attribution || item.title}](${item.url})`)
// Supporting sources
for (const sw of item.supporting_websites || []) {
links.push(`[${sw.attribution || sw.title}](${sw.url})`)
}
output = output.replaceAll(matchedText, `(${links.join(', ')})`)
}
else {
output = output.replaceAll(matchedText, ref.alt || '')
}
break
}
default:
// Use ref.alt which contains display text or pre-formatted markdown link
output = output.replaceAll(matchedText, ref.alt || '')
}
}
return output
}
/**
* Convert the content based on the type of message
*/

40
src/exporter/text.ts
View File

@@ -1,6 +1,7 @@
import { fetchConversation, getCurrentChatId, processConversation, shouldSkipMessageInExport } from '../api'
import i18n from '../i18n'
import { checkIfConversationStarted } from '../page'
import { transformContentReferences } from '../utils/citations'
import { copyToClipboard } from '../utils/clipboard'
import { flatMap, fromMarkdown, toMarkdown } from '../utils/markdown'
import { standardizeLineBreaks } from '../utils/text'
@@ -49,7 +50,11 @@ function transformMessage(message?: ConversationNodeMessage) {
}
if (message.author.role === 'assistant') {
content = transformContentReferences(content, message.metadata)
content = transformContentReferences(content, message.metadata, {
output: 'text',
inlineReferenceMode: 'alt',
includeSourceList: false,
})
content = transformFootNotes(content, message.metadata)
}
@@ -150,38 +155,6 @@ function transformAuthor(author: ConversationNodeMessage['author']): string {
}
}
function transformContentReferences(
input: string,
metadata: ConversationNodeMessage['metadata'],
) {
const contentRefs = metadata?.content_references
if (!contentRefs || contentRefs.length === 0) return input
const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
// Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII
const normalize = (s: string) => s
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
let output = normalize(input)
for (const ref of sortedRefs) {
if (!ref.matched_text) continue
const matchedText = normalize(ref.matched_text)
switch (ref.type) {
case 'sources_footnote':
break
default:
// Use ref.alt which contains display text (links won't render in plain text)
output = output.replaceAll(matchedText, ref.alt || '')
}
}
return output
}
/**
* Transform foot notes in assistant's message
*/
@@ -193,7 +166,6 @@ function transformFootNotes(
const footNoteMarkRegex = /【(\d+)†\((.+?)\)】/g
return input.replace(footNoteMarkRegex, (match, citeIndex, _evidenceText) => {
const citation = metadata?.citations?.find(cite => cite.metadata?.extra?.cited_message_idx === +citeIndex)
// We simply remove the foot note mark in text output
if (citation) return ''
return match

3
src/locales/en.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "Add metadata to exported Markdown and HTML files.",
"Export Thinking Process": "Export Thinking Process",
"Export Thinking Process Description": "Include the model's thinking/reasoning process in exported Markdown and HTML files.",
"Export Sources": "Export Sources",
"Export Sources Description": "Include the source list shown at the end of each answer in exported Markdown and HTML files.",
"Sources": "Sources",
"OpenAI Official Format": "OpenAI Official Format",
"Conversation Archive Alert": "Are you sure you want to archive all selected conversations?",
"Conversation Archived Message": "All selected conversations have been archived. Please refresh the page to see the changes.",

3
src/locales/es.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "Añadir Metadatos a los archivos Markdown y HTML exportados.",
"Export Thinking Process": "Exportar Proceso de Pensamiento",
"Export Thinking Process Description": "Incluir el proceso de pensamiento/razonamiento del modelo en los archivos Markdown y HTML exportados.",
"Export Sources": "Exportar fuentes",
"Export Sources Description": "Incluir la lista de fuentes que se muestra al final de cada respuesta en los archivos Markdown y HTML exportados.",
"Sources": "Fuentes",
"OpenAI Official Format": "Formato Oficial de OpenAI",
"Conversation Archive Alert": "¿Estás seguro que quieres archivar todas las conversaciones seleccionadas?",
"Conversation Archived Message": "Todos las conversaciones seleccionadas se han archivado. Por favor refresca la página para ver los cambios.",

3
src/locales/fr.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "Ajouter des métadonnées aux fichiers Markdown et HTML exportés.",
"Export Thinking Process": "Exporter le processus de réflexion",
"Export Thinking Process Description": "Inclure le processus de réflexion/raisonnement du modèle dans les fichiers Markdown et HTML exportés.",
"Export Sources": "Exporter les sources",
"Export Sources Description": "Inclure la liste des sources affichée à la fin de chaque réponse dans les fichiers Markdown et HTML exportés.",
"Sources": "Sources",
"OpenAI Official Format": "Format officiel OpenAI",
"Conversation Archive Alert": "Êtes-vous sûr de vouloir archiver toutes les conversations sélectionnées ?",
"Conversation Archived Message": "Toutes les conversations sélectionnées ont été archivées. Veuillez actualiser la page pour voir les changements.",

3
src/locales/id.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "Tambahkan metadata ke file Markdown dan HTML yang diekspor.",
"Export Thinking Process": "Ekspor Proses Berpikir",
"Export Thinking Process Description": "Sertakan proses berpikir/penalaran model dalam file Markdown dan HTML yang diekspor.",
"Export Sources": "Ekspor Sumber",
"Export Sources Description": "Sertakan daftar sumber yang ditampilkan di akhir setiap jawaban dalam file Markdown dan HTML yang diekspor.",
"Sources": "Sumber",
"OpenAI Official Format": "Format Resmi OpenAI",
"Conversation Archive Alert": "Apakah Anda yakin ingin mengarsipkan semua percakapan yang dipilih?",
"Conversation Archived Message": "Semua percakapan yang dipilih telah diarsipkan. Harap segarkan halaman untuk melihat perubahan.",

3
src/locales/jp.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "エクスポートされたMarkdownおよびHTMLファイルにメタデータを追加します。",
"Export Thinking Process": "思考プロセスをエクスポート",
"Export Thinking Process Description": "エクスポートされたMarkdownおよびHTMLファイルにモデルの思考・推論プロセスを含めます。",
"Export Sources": "ソースをエクスポート",
"Export Sources Description": "各回答の末尾に表示されるソース一覧を、エクスポートされた Markdown および HTML ファイルに含めます。",
"Sources": "出典",
"OpenAI Official Format": "OpenAI公式フォーマット",
"Conversation Archive Alert": "選択したすべての会話をアーカイブしてもよろしいですか？",
"Conversation Archived Message": "選択したすべての会話がアーカイブされました。変更を表示するには、ページを更新してください。",

3
src/locales/ru.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "Добавляйте метаданные в экспортированные файлы Markdown и HTML.",
"Export Thinking Process": "Экспорт процесса мышления",
"Export Thinking Process Description": "Включить процесс мышления/рассуждения модели в экспортированные файлы Markdown и HTML.",
"Export Sources": "Экспорт источников",
"Export Sources Description": "Включить список источников, показанный в конце каждого ответа, в экспортированные файлы Markdown и HTML.",
"Sources": "Источники",
"OpenAI Official Format": "Официальный формат OpenAI",
"Conversation Archive Alert": "Вы уверены, что хотите архивировать все выбранные разговоры?",
"Conversation Archived Message": "Все выбранные разговоры были заархивированы. Пожалуйста, обновите страницу, чтобы увидеть изменения.",

3
src/locales/tr.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "Dışa aktarılan Markdown ve HTML dosyalarına üst veri ekle",
"Export Thinking Process": "Düşünme Sürecini Dışa Aktar",
"Export Thinking Process Description": "Dışa aktarılan Markdown ve HTML dosyalarına modelin düşünme/akıl yürütme sürecini dahil et.",
"Export Sources": "Kaynakları Dışa Aktar",
"Export Sources Description": "Her yanıtın sonunda gösterilen kaynak listesini dışa aktarılan Markdown ve HTML dosyalarına dahil et.",
"Sources": "Kaynaklar",
"OpenAI Official Format": "OpenAI Resmi Format",
"Conversation Archive Alert": "Seçilen tüm konuşmaları arşivlemek istediğinizden emin misiniz?",
"Conversation Archived Message": "Seçilen tüm konuşmalar arşivlendi. Değişiklikleri görmek için sayfayı yenileyin.",

3
src/locales/zh-Hans.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "会添加至 Markdown 以及 HTML 导出。",
"Export Thinking Process": "导出思考过程",
"Export Thinking Process Description": "在导出的 Markdown 和 HTML 文件中包含模型的思考/推理过程。",
"Export Sources": "导出来源",
"Export Sources Description": "在导出的 Markdown 和 HTML 文件中包含每个回答末尾显示的来源列表。",
"Sources": "来源",
"OpenAI Official Format": "OpenAI 官方格式",
"Conversation Archive Alert": "确定要归档所有选取的对话？",
"Conversation Archived Message": "所有所选的对话已归档。请刷新页面。",

3
src/locales/zh-Hant.json
View File

@@ -35,6 +35,9 @@
"Export Metadata Description": "會添加至 Markdown 以及 HTML 匯出。",
"Export Thinking Process": "匯出思考過程",
"Export Thinking Process Description": "在匯出的 Markdown 和 HTML 檔案中包含模型的思考/推理過程。",
"Export Sources": "匯出來源",
"Export Sources Description": "在匯出的 Markdown 和 HTML 檔案中包含每個回答末尾顯示的來源列表。",
"Sources": "來源",
"OpenAI Official Format": "OpenAI 官方格式",
"Conversation Archive Alert": "確定要封存所有選取的對話？",
"Conversation Archived Message": "所有選取的對話已封存。請重新整理頁面。",

8
src/ui/SettingContext.tsx
View File

@@ -5,6 +5,7 @@ import {
KEY_FILENAME_FORMAT,
KEY_META_ENABLED,
KEY_META_LIST,
KEY_SOURCES_ENABLED,
KEY_THINKING_ENABLED,
KEY_TIMESTAMP_24H,
KEY_TIMESTAMP_ENABLED,
@@ -46,6 +47,8 @@ const SettingContext = createContext({
setExportMetaList: (_: ExportMeta[]) => {},
enableThinking: false,
setEnableThinking: (_: boolean) => {},
enableSources: true,
setEnableSources: (_: boolean) => {},
exportAllLimit: defaultExportAllLimit,
setExportAllLimit: (_: number) => {},
resetDefault: () => {},
@@ -63,6 +66,7 @@ export const SettingProvider: FC = ({ children }) => {
const [exportMetaList, setExportMetaList] = useGMStorage(KEY_META_LIST, defaultExportMetaList)
const [enableThinking, setEnableThinking] = useGMStorage(KEY_THINKING_ENABLED, false)
const [enableSources, setEnableSources] = useGMStorage(KEY_SOURCES_ENABLED, true)
const [exportAllLimit, setExportAllLimit] = useGMStorage(KEY_EXPORT_ALL_LIMIT, defaultExportAllLimit)
const resetDefault = useCallback(() => {
@@ -71,6 +75,7 @@ export const SettingProvider: FC = ({ children }) => {
setEnableMeta(false)
setExportMetaList(defaultExportMetaList)
setEnableThinking(false)
setEnableSources(true)
setExportAllLimit(defaultExportAllLimit)
}, [
setFormat,
@@ -78,6 +83,7 @@ export const SettingProvider: FC = ({ children }) => {
setEnableMeta,
setExportMetaList,
setEnableThinking,
setEnableSources,
setExportAllLimit,
])
@@ -103,6 +109,8 @@ export const SettingProvider: FC = ({ children }) => {
enableThinking,
setEnableThinking,
enableSources,
setEnableSources,
exportAllLimit,
setExportAllLimit,

14
src/ui/SettingDialog.tsx
View File

@@ -36,6 +36,7 @@ export const SettingDialog: FC<SettingDialogProps> = ({
enableMeta, setEnableMeta,
exportMetaList, setExportMetaList,
enableThinking, setEnableThinking,
enableSources, setEnableSources,
exportAllLimit, setExportAllLimit,
/* eslint-enable pionxzh/consistent-list-newline */
} = useSettingContext()
@@ -125,6 +126,19 @@ export const SettingDialog: FC<SettingDialogProps> = ({
<Toggle label="" checked={enableThinking} onCheckedUpdate={setEnableThinking} />
</div>
</div>
<div className="relative flex bg-white dark:bg-white/5 rounded p-4">
<div>
<dt className="text-md font-medium text-gray-800 dark:text-white">
{t('Export Sources')}
</dt>
<dd className="text-sm text-gray-700 dark:text-gray-300">
{t('Export Sources Description')}
</dd>
</div>
<div className="absolute right-4">
<Toggle label="" checked={enableSources} onCheckedUpdate={setEnableSources} />
</div>
</div>
<div className="relative flex bg-white dark:bg-white/5 rounded p-4">
<div>
<dt className="text-md font-medium text-gray-800 dark:text-white">

170
src/utils/citations.ts Normal file
View File

@@ -0,0 +1,170 @@
import type { ContentReference, ContentReferenceSource, ConversationNodeMessage } from '../api'
type CitationOutput = 'markdown' | 'text'
interface TransformContentReferenceOptions {
output?: CitationOutput
inlineReferenceMode?: 'expanded' | 'alt'
includeSourceList?: boolean
sourceListLabel?: string
}
const CitationMarkerRegex = /\uE200cite(?:\uE202[^\uE200\uE201]*)+\uE201/gu
export function normalizeCitationText(input: string): string {
return input
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
.replaceAll(/[\uE203\uE204]/gu, '')
}
export function transformContentReferences(
input: string,
metadata: ConversationNodeMessage['metadata'],
options: TransformContentReferenceOptions = {},
): string {
const outputType = options.output ?? 'markdown'
const inlineReferenceMode = options.inlineReferenceMode ?? 'expanded'
const contentRefs = metadata?.content_references ?? []
let output = normalizeCitationText(input)
const sortedRefs = [...contentRefs]
.filter(ref => ref.type !== 'sources_footnote')
.sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
for (const ref of sortedRefs) {
if (!ref.matched_text) continue
const matchedText = normalizeCitationText(ref.matched_text)
if (!matchedText) continue
const replacement = formatInlineReference(ref, outputType, inlineReferenceMode)
output = output.replaceAll(matchedText, replacement)
}
output = output.replace(CitationMarkerRegex, '')
if (options.includeSourceList !== false) {
const sources = getSourcesFootnoteSources(contentRefs)
if (sources.length > 0) {
output = appendSourcesSection(output, sources, outputType, options.sourceListLabel ?? 'Sources')
}
}
return output
}
export function formatCitationSource(source: ContentReferenceSource, output: CitationOutput): string {
const label = getSourceLabel(source)
const url = source.url?.trim()
if (!url) return output === 'markdown' ? escapeMarkdownText(label) : label
if (output === 'text') {
return `${label}: ${url}`
}
return `[${escapeMarkdownText(label)}](<${escapeMarkdownUrl(url)}>)`
}
function formatInlineReference(ref: ContentReference, output: CitationOutput, mode: 'expanded' | 'alt'): string {
if (mode === 'alt') return ref.alt || ''
const sources = getInlineSources(ref)
if (sources.length > 0) {
const separator = output === 'markdown' ? ', ' : '; '
return `(${sources.map(source => formatCitationSource(source, output)).join(separator)})`
}
if (ref.alt) return ref.alt
return ''
}
function getInlineSources(ref: ContentReference): ContentReferenceSource[] {
const sources: ContentReferenceSource[] = []
for (const item of ref.items ?? []) {
sources.push(item)
sources.push(...(item.supporting_websites ?? []))
}
sources.push(...(ref.fallback_items ?? []))
if (sources.length === 0 && (ref.url || ref.title || ref.attribution)) {
sources.push(ref)
}
if (sources.length === 0 && ref.safe_urls?.length) {
sources.push(...ref.safe_urls.map(url => ({ title: url, url })))
}
return dedupeSources(sources)
}
function getSourcesFootnoteSources(contentRefs: ContentReference[]): ContentReferenceSource[] {
const sources = contentRefs
.filter(ref => ref.type === 'sources_footnote')
.flatMap((ref) => {
if (ref.sources?.length) return ref.sources
if (ref.items?.length) return ref.items
if (ref.fallback_items?.length) return ref.fallback_items
if (ref.safe_urls?.length) return ref.safe_urls.map(url => ({ title: url, url }))
return []
})
return dedupeSources(sources)
}
function appendSourcesSection(input: string, sources: ContentReferenceSource[], output: CitationOutput, label: string): string {
const trimmed = input.trimEnd()
const sourceList = output === 'markdown'
? [
`**${escapeMarkdownText(label)}:**`,
'',
...sources.map(source => `- ${formatCitationSource(source, output)}`),
].join('\n')
: [
`${label}:`,
...sources.map((source, index) => `${index + 1}. ${formatCitationSource(source, output)}`),
].join('\n')
return trimmed ? `${trimmed}\n\n${sourceList}` : sourceList
}
function dedupeSources(sources: ContentReferenceSource[]): ContentReferenceSource[] {
const seen = new Set<string>()
const result: ContentReferenceSource[] = []
for (const source of sources) {
const key = source.url?.trim() || getSourceLabel(source)
if (!key || seen.has(key)) continue
seen.add(key)
result.push(source)
}
return result
}
function getSourceLabel(source: ContentReferenceSource): string {
return source.attribution?.trim()
|| source.title?.trim()
|| source.url?.trim()
|| 'Source'
}
function escapeMarkdownText(input: string): string {
return input
.replaceAll('\\', '\\\\')
.replaceAll('[', '\\[')
.replaceAll(']', '\\]')
.replaceAll('\n', ' ')
}
function escapeMarkdownUrl(input: string): string {
return input
.replaceAll('<', '%3C')
.replaceAll('>', '%3E')
.replaceAll('\n', '')
}