feat: support footnote

packages/userscript/src/api.ts

@@ -17,6 +17,33 @@ interface ApiSession {
 
 type ModelSlug = 'text-davinci-002-render-sha' | 'text-davinci-002-render-paid' | 'text-davinci-002-browse' | 'gpt-4' | 'gpt-4-browsing'
 
+export interface Citation {
+    start_ix: number
+    end_ix: number
+    citation_format_type: 'tether_og' & (string & {})
+    metadata?: {
+        extra?: {
+            cited_message_idx: number
+            evidence_text: string
+        }
+        text: string
+        title: string
+        type: 'webpage' & (string & {})
+        url: string
+    }
+}
+
+interface CiteMetadata {
+    citation_format: {
+        name: 'tether_og' & (string & {})
+    }
+    metadata_list: Array<{
+        title: string
+        url: string
+        text: string
+    }>
+}
+
 interface MessageMeta {
     command: 'click' | 'search' | 'quote' | 'quote_lines' | 'scroll' & (string & {})
     args: unknown
@@ -26,16 +53,8 @@ interface MessageMeta {
     }
     model_slug?: ModelSlug & (string & {})
     timestamp_: 'absolute' & (string & {})
-    _cite_metadata?: {
-        citation_format: {
-            name: 'tether_og' & (string & {})
-        }
-        metadata_list: Array<{
-            title: string
-            url: string
-            text: string
-        }>
-    }
+    citations?: Citation[]
+    _cite_metadata?: CiteMetadata
 }
 
 export type AuthorRole = 'system' | 'assistant' | 'user' | 'tool'
@@ -252,7 +271,7 @@ const modelMapping: { [key in ModelSlug]: string } & { [key: string]: string } =
     'text-davinci-002-render-paid': 'GTP-3.5',
     'text-davinci-002-browse': 'GTP-3.5',
     'gpt-4': 'GPT-4',
-    'gpt-4-browsing': 'GPT-4 (Browsing)',
+    'gpt-4-browsing': 'GPT-4 (Browser)',
 
     // fuzzy matching
     'text-davinci-002': 'GTP-3.5',

packages/userscript/src/exporter/html.ts

@@ -121,6 +121,24 @@ const transformContent = (
     }
 }
 
+/**
+ * Transform foot notes in assistant's message
+ */
+const transformFootNotes = (
+    input: string,
+    metadata: ConversationNodeMessage['metadata'],
+) => {
+    // 【11†(PrintWiki)】
+    const footNoteMarkRegex = /【(\d+)†\((.+?)\)】/g
+    return input.replace(footNoteMarkRegex, (match, citeIndex, _evidenceText) => {
+        const citation = metadata?.citations?.find(cite => cite.metadata?.extra?.cited_message_idx === +citeIndex)
+        // We simply remove the foot note mark in html output
+        if (citation) return ''
+
+        return match
+    })
+}
+
 function conversationToHtml(conversation: ConversationResult, avatar: string, metaList?: ExportMeta[]) {
     const { id, title, model, modelSlug, createTime, updateTime, conversationNodes } = conversation
 
@@ -135,13 +153,18 @@ function conversationToHtml(conversation: ConversationResult, avatar: string, me
         if (message.author.role === 'tool') return null // Skip tool's intermediate message
 
         const isUser = message.author.role === 'user'
+        const isAssistant = message.author.role === 'assistant'
         const author = transformAuthor(message.author)
         const model = message?.metadata?.model_slug === 'gpt-4' ? 'GPT-4' : 'GPT-3'
         const authorType = isUser ? 'user' : model
         const avatarEl = isUser
             ? `<img alt="${author}" />`
             : '<svg width="41" height="41"><use xlink:href="#chatgpt" /></svg>'
-        const content = transformContent(message.content, message.metadata)
+        let content = transformContent(message.content, message.metadata)
+        if (isAssistant) {
+            content = transformFootNotes(content, message.metadata)
+        }
+
         let conversationContent = content
 
         if (isUser) {

packages/userscript/src/exporter/markdown.ts

@@ -8,7 +8,7 @@ import { fromMarkdown, toMarkdown } from '../utils/markdown'
 import { ScriptStorage } from '../utils/storage'
 import { standardizeLineBreaks } from '../utils/text'
 import { dateStr, timestamp, unixTimestampToISOString } from '../utils/utils'
-import type { ApiConversationWithId, ConversationNodeMessage, ConversationResult } from '../api'
+import type { ApiConversationWithId, Citation, ConversationNodeMessage, ConversationResult } from '../api'
 import type { ExportMeta } from '../ui/SettingContext'
 
 export async function exportToMarkdown(fileNameFormat: string, metaList: ExportMeta[]) {
@@ -116,6 +116,37 @@ const transformContent = (
     }
 }
 
+/**
+ * Transform foot notes in assistant's message
+ */
+const transformFootNotes = (
+    input: string,
+    metadata: ConversationNodeMessage['metadata'],
+) => {
+    // 【11†(PrintWiki)】
+    const footNoteMarkRegex = /【(\d+)†\((.+?)\)】/g
+
+    const citationList: Citation[] = []
+    const output = input.replace(footNoteMarkRegex, (match, citeIndex, _evidenceText) => {
+        const citation = metadata?.citations?.find(cite => cite.metadata?.extra?.cited_message_idx === +citeIndex)
+        if (citation) {
+            citationList.push(citation)
+            // Use markdown caret to represent foot note ([^1])
+            return `[^${citeIndex}]`
+        }
+
+        return match
+    })
+    const citationText = citationList.map((citation) => {
+        const citeIndex = citation.metadata?.extra?.cited_message_idx ?? 1
+        const citeTitle = citation.metadata?.title ?? 'No title'
+        return `[^${citeIndex}]: ${citeTitle}`
+    }).join('\n')
+
+    // Foot notes are placed at the end of the conversation node, not the end of the whole document
+    return `${output}\n\n${citationText}`
+}
+
 function conversationToMarkdown(conversation: ConversationResult, metaList?: ExportMeta[]) {
     const { id, title, model, modelSlug, createTime, updateTime, conversationNodes } = conversation
     const source = `${baseUrl}/c/${id}`
@@ -163,6 +194,9 @@ function conversationToMarkdown(conversation: ConversationResult, metaList?: Exp
         const isUser = message.author.role === 'user'
         const author = transformAuthor(message.author)
         let content = transformContent(message.content, message.metadata)
+        if (message.author.role === 'assistant') {
+            content = transformFootNotes(content, message.metadata)
+        }
 
         // User's message will not be reformatted
         if (!isUser && content) {

packages/userscript/src/exporter/text.ts

@@ -59,6 +59,24 @@ const transformContent = (
     }
 }
 
+/**
+ * Transform foot notes in assistant's message
+ */
+const transformFootNotes = (
+    input: string,
+    metadata: ConversationNodeMessage['metadata'],
+) => {
+    // 【11†(PrintWiki)】
+    const footNoteMarkRegex = /【(\d+)†\((.+?)\)】/g
+    return input.replace(footNoteMarkRegex, (match, citeIndex, _evidenceText) => {
+        const citation = metadata?.citations?.find(cite => cite.metadata?.extra?.cited_message_idx === +citeIndex)
+        // We simply remove the foot note mark in text output
+        if (citation) return ''
+
+        return match
+    })
+}
+
 /**
  * Remove some markdown syntaxes from the content
  */
@@ -93,6 +111,9 @@ export async function exportToText() {
 
         const author = transformAuthor(message.author)
         let content = transformContent(message.content, message.metadata)
+        if (message.author.role === 'assistant') {
+            content = transformFootNotes(content, message.metadata)
+        }
 
         // User's message will not be reformatted
         if (message.author.role !== 'user' && content) {