mirror of
https://github.com/pionxzh/chatgpt-exporter.git
synced 2026-07-23 09:00:51 -05:00
fix: improve content reference handling for citations and entities
- Add support for alt_text content reference type (entity mentions) - Fix Unicode normalization to include non-breaking hyphens and preserve newlines - Render citations as inline links instead of footnotes - Include all sources for multi-citations - Fix HTML exporter to retain links instead of stripping them
This commit is contained in:
Eddy G
@@ -50,7 +50,7 @@ export interface Citation {
|
||||
}
|
||||
|
||||
export interface ContentReference {
|
||||
type: 'grouped_webpages' | 'sources_footnote' | 'nav_list' & (string & {})
|
||||
type: 'grouped_webpages' | 'sources_footnote' | 'nav_list' | 'alt_text' & (string & {})
|
||||
/** The text that was matched in the content, e.g., "citeturn0search3" */
|
||||
matched_text?: string
|
||||
start_idx: number
|
||||
@@ -62,6 +62,12 @@ export interface ContentReference {
|
||||
title: string
|
||||
url: string
|
||||
attribution?: string
|
||||
/** Additional sources for multi-citations */
|
||||
supporting_websites?: Array<{
|
||||
title: string
|
||||
url: string
|
||||
attribution?: string
|
||||
}>
|
||||
}>
|
||||
// Legacy fields (may still be present in some responses)
|
||||
url?: string
|
||||
|
||||
@@ -108,9 +108,9 @@ function conversationToHtml(conversation: ConversationResult, avatar: string, me
|
||||
|
||||
let postSteps: Array<(input: string) => string> = []
|
||||
if (message.author.role === 'assistant') {
|
||||
// Handle new-style content references (web search citations with Unicode markers)
|
||||
postSteps.push(input => transformFootNotes(input, message.metadata))
|
||||
// Handle old-style footnotes (【11†(PrintWiki)】 format)
|
||||
postSteps.push(input => transformFootNotes(input, message.metadata))
|
||||
// Handle new-style content references (web search citations with Unicode markers)
|
||||
postSteps.push(input => transformContentReferences(input, message.metadata))
|
||||
|
||||
postSteps.push((input) => {
|
||||
@@ -260,20 +260,47 @@ function transformContentReferences(
|
||||
|
||||
const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
|
||||
|
||||
// Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII
|
||||
const normalize = (s: string) => s
|
||||
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
|
||||
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
|
||||
|
||||
let output = normalize(input)
|
||||
|
||||
for (const ref of sortedRefs) {
|
||||
if (!ref.matched_text) continue
|
||||
|
||||
// For some reason, the matched_text contains non-breaking spaces but the content doesn't!
|
||||
const matchedText = ref.matched_text.replaceAll(/\s/gu, ' ')
|
||||
const matchedText = normalize(ref.matched_text)
|
||||
|
||||
switch (ref.type) {
|
||||
case 'sources_footnote':
|
||||
break
|
||||
case 'grouped_webpages': {
|
||||
// For citations, build links from items including supporting_websites
|
||||
const item = ref.items?.[0]
|
||||
if (item) {
|
||||
const links: string[] = []
|
||||
// Primary source
|
||||
links.push(`[${item.attribution || item.title}](${item.url})`)
|
||||
// Supporting sources
|
||||
for (const sw of item.supporting_websites || []) {
|
||||
links.push(`[${sw.attribution || sw.title}](${sw.url})`)
|
||||
}
|
||||
// Markdown links will be converted to HTML by the subsequent toHtml step
|
||||
output = output.replaceAll(matchedText, `(${links.join(', ')})`)
|
||||
}
|
||||
else {
|
||||
output = output.replaceAll(matchedText, ref.alt || '')
|
||||
}
|
||||
break
|
||||
}
|
||||
default:
|
||||
input = input.replaceAll(matchedText, '')
|
||||
// Use ref.alt which contains display text or pre-formatted markdown link
|
||||
// Markdown links will be converted to HTML by the subsequent toHtml step
|
||||
output = output.replaceAll(matchedText, ref.alt || '')
|
||||
}
|
||||
}
|
||||
return input
|
||||
return output
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -245,47 +245,45 @@ function transformContentReferences(
|
||||
// (e.g., "citeturn0search2turn1search8" before "citeturn0search2")
|
||||
const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
|
||||
|
||||
const usedRefs: Array<{ url: string; title: string }> = []
|
||||
let output = input
|
||||
// Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII
|
||||
const normalize = (s: string) => s
|
||||
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
|
||||
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
|
||||
|
||||
let output = normalize(input)
|
||||
|
||||
for (const ref of sortedRefs) {
|
||||
if (!ref.matched_text) continue
|
||||
|
||||
// For some reason, the matched_text contains non-breaking spaces but the content doesn't!
|
||||
const matchedText = ref.matched_text.replaceAll(/\s/gu, ' ')
|
||||
const matchedText = normalize(ref.matched_text)
|
||||
|
||||
switch (ref.type) {
|
||||
case 'sources_footnote':
|
||||
break
|
||||
case 'nav_list':
|
||||
output = output.replaceAll(matchedText, ref.alt || '')
|
||||
break
|
||||
default: {
|
||||
// Get URL and title from items array (new format) or direct fields
|
||||
case 'grouped_webpages': {
|
||||
// For citations, build links from items including supporting_websites
|
||||
const item = ref.items?.[0]
|
||||
const url = item?.url || ref.url
|
||||
const title = item?.title || ref.title
|
||||
|
||||
if (!url) continue
|
||||
|
||||
// Replace all occurrences of this matched_text
|
||||
if (output.includes(ref.matched_text)) {
|
||||
usedRefs.push({ url, title: title || url })
|
||||
const refIndex = usedRefs.length
|
||||
output = output.replaceAll(matchedText, `[^${refIndex}]`)
|
||||
if (item) {
|
||||
const links: string[] = []
|
||||
// Primary source
|
||||
links.push(`[${item.attribution || item.title}](${item.url})`)
|
||||
// Supporting sources
|
||||
for (const sw of item.supporting_websites || []) {
|
||||
links.push(`[${sw.attribution || sw.title}](${sw.url})`)
|
||||
}
|
||||
output = output.replaceAll(matchedText, `(${links.join(', ')})`)
|
||||
}
|
||||
else {
|
||||
output = output.replaceAll(matchedText, ref.alt || '')
|
||||
}
|
||||
break
|
||||
}
|
||||
default:
|
||||
// Use ref.alt which contains display text or pre-formatted markdown link
|
||||
output = output.replaceAll(matchedText, ref.alt || '')
|
||||
}
|
||||
}
|
||||
|
||||
// Add footnote definitions at the end
|
||||
if (usedRefs.length > 0) {
|
||||
const footnotes = usedRefs.map((ref, index) => {
|
||||
return `[^${index + 1}]: [${ref.title}](${ref.url})`
|
||||
}).join('\n')
|
||||
output = `${output}\n\n${footnotes}`
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
|
||||
@@ -175,20 +175,27 @@ function transformContentReferences(
|
||||
|
||||
const sortedRefs = [...contentRefs].sort((a, b) => (b.matched_text?.length || 0) - (a.matched_text?.length || 0))
|
||||
|
||||
// Normalize unicode variants (non-breaking spaces, non-breaking hyphens) to regular ASCII
|
||||
const normalize = (s: string) => s
|
||||
.replaceAll(/[\u00A0\u202F\u2007\u2060]/gu, ' ')
|
||||
.replaceAll(/[\u2010-\u2015\u2212]/gu, '-')
|
||||
|
||||
let output = normalize(input)
|
||||
|
||||
for (const ref of sortedRefs) {
|
||||
if (!ref.matched_text) continue
|
||||
|
||||
// For some reason, the matched_text contains non-breaking spaces but the content doesn't!
|
||||
const matchedText = ref.matched_text.replaceAll(/\s/gu, ' ')
|
||||
const matchedText = normalize(ref.matched_text)
|
||||
|
||||
switch (ref.type) {
|
||||
case 'sources_footnote':
|
||||
break
|
||||
default:
|
||||
input = input.replaceAll(matchedText, '')
|
||||
// Use ref.alt which contains display text (links won't render in plain text)
|
||||
output = output.replaceAll(matchedText, ref.alt || '')
|
||||
}
|
||||
}
|
||||
return input
|
||||
return output
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user