feat: support export to tavern and ooba format

#210

.vscode/settings.json

@@ -35,6 +35,7 @@
     "headlessui",
     "mdast",
     "micromark",
+    "ooba",
     "preact",
     "tabler",
     "unist"

src/constants.ts

@@ -8,7 +8,7 @@ export const apiUrl = API_MAPPING[baseUrl]
 
 export const KEY_LANGUAGE = 'exporter:language'
 export const KEY_FILENAME_FORMAT = 'exporter:filename_format'
-export const KEY_OFFICIAL_JSON_FORMAT = 'exporter:official_json_format'
+// export const KEY_OFFICIAL_JSON_FORMAT = 'exporter:official_json_format'
 export const KEY_TIMESTAMP_ENABLED = 'exporter:enable_timestamp'
 export const KEY_TIMESTAMP_24H = 'exporter:timestamp_24h'
 export const KEY_TIMESTAMP_MARKDOWN = 'exporter:timestamp_markdown'

src/exporter/json.ts

@@ -2,10 +2,11 @@ import JSZip from 'jszip'
 import { fetchConversation, getCurrentChatId, processConversation } from '../api'
 import i18n from '../i18n'
 import { checkIfConversationStarted } from '../page'
+import { convertToOoba, convertToTavern } from '../utils/conversion'
 import { downloadFile, getFileNameWithFormat } from '../utils/download'
 import type { ApiConversationWithId } from '../api'
 
-export async function exportToJson(fileNameFormat: string, options: { officialFormat: boolean }) {
+export async function exportToJson(fileNameFormat: string) {
     if (!checkIfConversationStarted()) {
         alert(i18n.t('Please start a conversation first'))
         return false
@@ -19,7 +20,41 @@ export async function exportToJson(fileNameFormat: string, options: { officialFo
     /**
      * The official format is just an array of the API response.
      */
-    const content = conversationToJson(options.officialFormat ? [rawConversation] : rawConversation)
+    const content = conversationToJson([rawConversation])
+    downloadFile(fileName, 'application/json', content)
+
+    return true
+}
+
+export async function exportToTavern(fileNameFormat: string) {
+    if (!checkIfConversationStarted()) {
+        alert(i18n.t('Please start a conversation first'))
+        return false
+    }
+
+    const chatId = await getCurrentChatId()
+    const rawConversation = await fetchConversation(chatId, false)
+    const conversation = processConversation(rawConversation)
+
+    const fileName = getFileNameWithFormat(`${fileNameFormat}.tavern`, 'jsonl', { title: conversation.title, chatId })
+    const content = convertToTavern(conversation)
+    downloadFile(fileName, 'application/json-lines', content)
+
+    return true
+}
+
+export async function exportToOoba(fileNameFormat: string) {
+    if (!checkIfConversationStarted()) {
+        alert(i18n.t('Please start a conversation first'))
+        return false
+    }
+
+    const chatId = await getCurrentChatId()
+    const rawConversation = await fetchConversation(chatId, false)
+    const conversation = processConversation(rawConversation)
+
+    const fileName = getFileNameWithFormat(`${fileNameFormat}.ooba`, 'json', { title: conversation.title, chatId })
+    const content = convertToOoba(conversation)
     downloadFile(fileName, 'application/json', content)
 
     return true

src/locales/en.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "Enable on Markdown files",
   "Use 24-hour format": "Use 24-hour format (eg. 23:59)",
   "Export Format": "Export Format",
-  "Export JSON Format Description": "Export JSON in OpenAI Official Format",
   "Export Metadata": "Export Metadata",
   "Export Metadata Description": "Add metadata to exported Markdown and HTML files.",
+  "OpenAI Official Format": "OpenAI Official Format",
   "Conversation Archive Alert": "Are you sure you want to archive all selected conversations?",
   "Conversation Archived Message": "All selected conversations have been archived. Please refresh the page to see the changes.",
   "Conversation Delete Alert": "Are you sure you want to delete all selected conversations?",

src/locales/es.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "Habilitar en archivos Markdown",
   "Use 24-hour format": "Usar formato de 24 horas (ej. 23:59)",
   "Export Format": "Formato de Exportación",
-  "Export JSON Format Description": "Exportar JSON en el Formato Oficial de OpenAI",
   "Export Metadata": "Exportar Metadatos",
   "Export Metadata Description": "Añadir Metadatos a los archivos Markdown y HTML exportados.",
+  "OpenAI Official Format": "Formato Oficial de OpenAI",
   "Conversation Archive Alert": "¿Estás seguro que quieres archivar todas las conversaciones seleccionadas?",
   "Conversation Archived Message": "Todos las conversaciones seleccionadas se han archivado. Por favor refresca la página para ver los cambios.",
   "Conversation Delete Alert": "¿Estás seguro que quieres borrar todas las conversaciones seleccionadas?",

src/locales/id.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "Aktifkan pada file Markdown",
   "Use 24-hour format": "Gunakan format 24 jam (contohnya: 23:59)",
   "Export Format": "Format Ekspor",
-  "Export JSON Format Description": "Ekspor JSON dalam Format Resmi OpenAI",
   "Export Metadata": "Ekspor Metada",
   "Export Metadata Description": "Tambahkan metadata ke file Markdown dan HTML yang diekspor.",
+  "OpenAI Official Format": "Format Resmi OpenAI",
   "Conversation Archive Alert": "Apakah Anda yakin ingin mengarsipkan semua percakapan yang dipilih?",
   "Conversation Archived Message": "Semua percakapan yang dipilih telah diarsipkan. Harap segarkan halaman untuk melihat perubahan.",
   "Conversation Delete Alert": "Apakah Anda yakin ingin menghapus semua percakapan yang dipilih?",

src/locales/jp.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "Markdown ファイルで有効にする",
   "Use 24-hour format": "24時間形式を使用する (例: 23:59)",
   "Export Format": "エクスポートフォーマット",
-  "Export JSON Format Description": "OpenAI公式フォーマットでのJSONのエクスポート",
   "Export Metadata": "メタデータをエクスポート",
   "Export Metadata Description": "エクスポートされたMarkdownおよびHTMLファイルにメタデータを追加します。",
+  "OpenAI Official Format": "OpenAI公式フォーマット",
   "Conversation Archive Alert": "選択したすべての会話をアーカイブしてもよろしいですか？",
   "Conversation Archived Message": "選択したすべての会話がアーカイブされました。変更を表示するには、ページを更新してください。",
   "Conversation Delete Alert": "選択したすべての会話を削除してもよろしいですか？",

src/locales/tr.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "Markdown dosyalarında etkinleştir",
   "Use 24-hour format": "24 saat biçimini kullan (örn. 23:59)",
   "Export Format": "Dışa Aktarma Formatı",
-  "Export JSON Format Description": "OpenAI Resmi Formatında JSON Dışa Aktarma",
   "Export Metadata": "Üst veriyi dışa aktar",
   "Export Metadata Description": "Dışa aktarılan Markdown ve HTML dosyalarına üst veri ekle",
+  "OpenAI Official Format": "OpenAI Resmi Format",
   "Conversation Archive Alert": "Seçilen tüm konuşmaları arşivlemek istediğinizden emin misiniz?",
   "Conversation Archived Message": "Seçilen tüm konuşmalar arşivlendi. Değişiklikleri görmek için sayfayı yenileyin.",
   "Conversation Delete Alert": "Seçilen tüm konuşmaları silmek istediğinizden emin misiniz?",

src/locales/zh-Hans.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "在 Markdown 文件上启用",
   "Use 24-hour format": "使用24小时制 (例如 23:59)",
   "Export Format": "导出格式",
-  "Export JSON Format Description": "以 OpenAI 官方格式导出JSON",
   "Export Metadata": "导出元数据",
   "Export Metadata Description": "会添加至 Markdown 以及 HTML 导出。",
+  "OpenAI Official Format": "OpenAI 官方格式",
   "Conversation Archive Alert": "确定要归档所有选取的对话？",
   "Conversation Archived Message": "所有所选的对话已归档。请刷新页面。",
   "Conversation Delete Alert": "确定要删除所有选取的对话？",

src/locales/zh-Hant.json

@@ -31,9 +31,9 @@
   "Enable on Markdown": "在 Markdown 檔案上啟用",
   "Use 24-hour format": "使用24小時制 (例如 23:59)",
   "Export Format": "匯出格式",
-  "Export JSON Format Description": "以 OpenAI 官方格式匯出 JSON",
   "Export Metadata": "匯出元資料",
   "Export Metadata Description": "會添加至 Markdown 以及 HTML 匯出。",
+  "OpenAI Official Format": "OpenAI 官方格式",
   "Conversation Archive Alert": "確定要封存所有選取的對話？",
   "Conversation Archived Message": "所有選取的對話已封存。請重新整理頁面。",
   "Conversation Delete Alert": "確定要刪除所有選取的對話？",

src/ui/Dialog.css

@@ -21,6 +21,7 @@
     overflow-y: auto;
     padding: 16px 24px;
     z-index: 1001;
+    outline: none;
     animation: contentShow 150ms cubic-bezier(0.16, 1, 0.3, 1);
 }
 

src/ui/ExportDialog.tsx

@@ -75,14 +75,15 @@ interface DialogContentProps {
 
 const DialogContent: FC<DialogContentProps> = ({ format }) => {
     const { t } = useTranslation()
-    const { enableMeta, exportMetaList, exportOfficialJsonFormat } = useSettingContext()
+    const { enableMeta, exportMetaList } = useSettingContext()
     const metaList = useMemo(() => enableMeta ? exportMetaList : [], [enableMeta, exportMetaList])
 
     const exportAllOptions = useMemo(() => [
         { label: 'Markdown', callback: exportAllToMarkdown },
-        { label: 'JSON', callback: exportOfficialJsonFormat ? exportAllToOfficialJson : exportAllToJson },
         { label: 'HTML', callback: exportAllToHtml },
-    ], [exportOfficialJsonFormat])
+        { label: 'JSON', callback: exportAllToOfficialJson },
+        { label: 'JSON (ZIP)', callback: exportAllToJson },
+    ], [])
 
     const fileInputRef = useRef<HTMLInputElement>(null)
     const [exportSource, setExportSource] = useState<ExportSource>('API')

src/ui/Menu.tsx

@@ -1,9 +1,10 @@
+import * as Dialog from '@radix-ui/react-dialog'
 import * as HoverCard from '@radix-ui/react-hover-card'
 import { useCallback, useEffect, useMemo, useState } from 'preact/hooks'
 import { useTranslation } from 'react-i18next'
 import { exportToHtml } from '../exporter/html'
 import { exportToPng } from '../exporter/image'
-import { exportToJson } from '../exporter/json'
+import { exportToJson, exportToOoba, exportToTavern } from '../exporter/json'
 import { exportToMarkdown } from '../exporter/markdown'
 import { exportToText } from '../exporter/text'
 import { useWindowResize } from '../hooks/useWindowResize'
@@ -23,12 +24,12 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
     const disabled = getHistoryDisabled()
 
     const [open, setOpen] = useState(false)
+    const [jsonOpen, setJsonOpen] = useState(false)
     const [exportOpen, setExportOpen] = useState(false)
     const [settingOpen, setSettingOpen] = useState(false)
 
     const {
         format,
-        exportOfficialJsonFormat,
         enableTimestamp,
         timeStamp24H,
         enableMeta,
@@ -50,7 +51,13 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
     const onClickPng = useCallback(() => exportToPng(format), [format])
     const onClickMarkdown = useCallback(() => exportToMarkdown(format, metaList), [format, metaList])
     const onClickHtml = useCallback(() => exportToHtml(format, metaList), [format, metaList])
-    const onClickJSON = useCallback(() => exportToJson(format, { officialFormat: exportOfficialJsonFormat }), [format, exportOfficialJsonFormat])
+    const onClickJSON = useCallback(() => {
+        setJsonOpen(true)
+        return true
+    }, [])
+    const onClickOfficialJSON = useCallback(() => exportToJson(format), [format])
+    const onClickTavern = useCallback(() => exportToTavern(format), [format])
+    const onClickOoba = useCallback(() => exportToOoba(format), [format])
 
     const width = useWindowResize(() => window.innerWidth)
     const isMobile = width < 768
@@ -95,7 +102,7 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
                 </HoverCard.Trigger>
                 <Portal
                     container={isMobile ? container : document.body}
-                    forceMount={open || settingOpen || exportOpen}
+                    forceMount={open || jsonOpen || settingOpen || exportOpen}
                 >
                     <HoverCard.Content
                         className={`
@@ -151,12 +158,43 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
                             className="row-half"
                             onClick={onClickHtml}
                         />
+                        <Dialog.Root
+                            open={jsonOpen}
+                            onOpenChange={setJsonOpen}
+                        >
+                            <Dialog.Trigger asChild>
                                 <MenuItem
                                     text={t('JSON')}
                                     icon={IconJSON}
                                     className="row-half"
                                     onClick={onClickJSON}
                                 />
+                            </Dialog.Trigger>
+                            <Dialog.Portal>
+                                <Dialog.Overlay className="DialogOverlay" />
+                                <Dialog.Content className="DialogContent" style={{ width: '320px' }}>
+                                    <Dialog.Title className="DialogTitle">{t('JSON')}</Dialog.Title>
+                                    <MenuItem
+                                        text={t('OpenAI Official Format')}
+                                        icon={IconCopy}
+                                        className="row-full"
+                                        onClick={onClickOfficialJSON}
+                                    />
+                                    <MenuItem
+                                        text="JSONL (TavernAI, SillyTavern)"
+                                        icon={IconCopy}
+                                        className="row-full"
+                                        onClick={onClickTavern}
+                                    />
+                                    <MenuItem
+                                        text="Ooba (text-generation-webui)"
+                                        icon={IconCopy}
+                                        className="row-full"
+                                        onClick={onClickOoba}
+                                    />
+                                </Dialog.Content>
+                            </Dialog.Portal>
+                        </Dialog.Root>
                         <ExportDialog
                             format={format}
                             open={exportOpen}

src/ui/SettingContext.tsx

@@ -4,7 +4,6 @@ import {
     KEY_FILENAME_FORMAT,
     KEY_META_ENABLED,
     KEY_META_LIST,
-    KEY_OFFICIAL_JSON_FORMAT,
     KEY_TIMESTAMP_24H,
     KEY_TIMESTAMP_ENABLED,
     KEY_TIMESTAMP_HTML,
@@ -29,9 +28,6 @@ const SettingContext = createContext({
     format: defaultFormat,
     setFormat: (_: string) => {},
 
-    exportOfficialJsonFormat: false,
-    setExportOfficialJsonFormat: (_: boolean) => {},
-
     enableTimestamp: false,
     setEnableTimestamp: (_: boolean) => {},
     timeStamp24H: false,
@@ -52,8 +48,6 @@ const SettingContext = createContext({
 export const SettingProvider: FC = ({ children }) => {
     const [format, setFormat] = useGMStorage(KEY_FILENAME_FORMAT, defaultFormat)
 
-    const [exportOfficialJsonFormat, setExportOfficialJsonFormat] = useGMStorage(KEY_OFFICIAL_JSON_FORMAT, false)
-
     const [enableTimestamp, setEnableTimestamp] = useGMStorage(KEY_TIMESTAMP_ENABLED, false)
     const [timeStamp24H, setTimeStamp24H] = useGMStorage(KEY_TIMESTAMP_24H, false)
     const [enableTimestampHTML, setEnableTimestampHTML] = useGMStorage(KEY_TIMESTAMP_HTML, false)
@@ -75,9 +69,6 @@ export const SettingProvider: FC = ({ children }) => {
                 format,
                 setFormat,
 
-                exportOfficialJsonFormat,
-                setExportOfficialJsonFormat,
-
                 enableTimestamp,
                 setEnableTimestamp,
                 timeStamp24H,

src/ui/SettingDialog.tsx

@@ -33,7 +33,6 @@ export const SettingDialog: FC<SettingDialogProps> = ({
         timeStamp24H, setTimeStamp24H,
         enableTimestampHTML, setEnableTimestampHTML,
         enableTimestampMarkdown, setEnableTimestampMarkdown,
-        exportOfficialJsonFormat, setExportOfficialJsonFormat,
         enableMeta, setEnableMeta,
         exportMetaList, setExportMetaList,
         /* eslint-enable pionxzh/consistent-list-newline */
@@ -111,22 +110,6 @@ export const SettingDialog: FC<SettingDialogProps> = ({
                                 </dd>
                             </div>
                         </div>
-                        <div className="relative flex bg-white dark:bg-white/5 rounded p-4">
-                            <div>
-                                <dt className="text-md font-medium text-gray-800 dark:text-white">
-                                    {t('Export Format')}
-                                </dt>
-                                <dd className="text-sm text-gray-700 dark:text-gray-300">
-                                    <div className="mt-2">
-                                        <Toggle
-                                            label={t('Export JSON Format Description')}
-                                            checked={exportOfficialJsonFormat}
-                                            onCheckedUpdate={setExportOfficialJsonFormat}
-                                        />
-                                    </div>
-                                </dd>
-                            </div>
-                        </div>
                         <div className="relative flex bg-white dark:bg-white/5 rounded p-4">
                             <div>
                                 <dt className="text-md font-medium text-gray-800 dark:text-white">

src/utils/conversion.ts

@@ -1,23 +1,9 @@
-interface Message {
+import { jsonlStringify, nonNullable } from './utils'
-    parent?: string
+import type { ConversationNode, ConversationResult } from '../api'
-    message?: {
-        author: {
-            role: string
-        }
-        create_time: number
-        content: {
-            parts: string[]
-        }
-    }
-}
 
-interface MessageMapping {
+interface NameMessage {
-    [key: string]: Message
+    user_name: string
-}
+    character_name: string
-
-interface ConversationData {
-    current_node: string
-    mapping: MessageMapping
 }
 
 interface TavernMessage {
@@ -30,50 +16,25 @@ interface TavernMessage {
     swipe_id: number
 }
 
-interface NameMessage {
-    user_name: string
-    character_name: string
-}
-
 interface OobaData {
     internal: [string, string][]
     visible: [string, string][]
 }
 
-function extractConversation(data: ConversationData): Message[] {
+function convertMessageToTavern(node: ConversationNode): TavernMessage | null {
-    const currentNode = data.current_node
+    if (!node.message || node.message.content.content_type !== 'text') {
-    const mapping = data.mapping
-
-    const messagesReversed: Message[] = []
-    let nodeId: string | undefined = currentNode
-
-    while (nodeId !== null && nodeId !== undefined) {
-        const currentMessage: Message = mapping[nodeId]
-        messagesReversed.push(currentMessage)
-        nodeId = currentMessage.parent
-    }
-
-    return messagesReversed.reverse()
-}
-
-function convertMessageToTavern(messageData: Message): TavernMessage | null {
-    if (!messageData.message) {
         return null
     }
 
-    const senderRole: string = messageData.message.author.role
+    const authorRole = node.message.author.role
-    if (senderRole === 'system') {
+    const createTime = node.message.create_time || (new Date()).getTime() / 1000
-        return null
+    const text = node.message.content.parts.join('\n')
-    }
-
-    const isAssistant = senderRole === 'assistant'
-    const createTime: number = messageData.message.create_time
-    const text: string = messageData.message.content.parts[0]
 
     return {
-        name: isAssistant ? 'Assistant' : 'You',
+        name: authorRole === 'assistant' ? 'Assistant' : 'You',
-        is_user: !isAssistant,
+        is_user: authorRole === 'user',
-        is_name: isAssistant,
+        // This seems to be always true
+        is_name: true,
         send_date: createTime,
         mes: text,
         swipes: [text],
@@ -81,55 +42,41 @@ function convertMessageToTavern(messageData: Message): TavernMessage | null {
     }
 }
 
-function jsonlStringify(messageArray: any[]): string {
+export function convertToTavern(conversation: ConversationResult): string {
-    return messageArray.map((msg: any) => JSON.stringify(msg)).join('\n')
+    const messages: (NameMessage | TavernMessage)[] = [
-}
+        {
-
-export function getTavernString(jsonData: ConversationData): string {
-    // Takes the OAI JSON data as input, outputs the JSONL string
-    const conversation = extractConversation(jsonData)
-
-    const convertedConvo: (TavernMessage | NameMessage)[] = [{
             user_name: 'You',
             character_name: 'Assistant',
-    }]
+        },
+        ...conversation.conversationNodes.map(convertMessageToTavern).filter(nonNullable),
+    ]
 
-    conversation.forEach((message) => {
+    return jsonlStringify(messages)
-        const convertedMsg = convertMessageToTavern(message)
-        if (convertedMsg !== null) {
-            convertedConvo.push(convertedMsg)
-        }
-    })
-    // This _has_ to be stringified without adding any indentation, due to the JSONL format.
-    return jsonlStringify(convertedConvo)
 }
 
-export function getOobaString(jsonData: ConversationData): string {
+export function convertToOoba(conversation: ConversationResult): string {
-    // Takes the OAI JSON data as input, outputs the serialized JSON
+    const pairs: [string, string][] = []
-    const messages = extractConversation(jsonData)
+    const messages = conversation.conversationNodes.filter(node => node.message?.author.role !== 'tool' && node.message?.content.content_type === 'text')
-    const pairs: any[] = []
+
     let idx = 0
-
     while (idx < messages.length - 1) {
         const message = messages[idx]
         const nextMessage = messages[idx + 1]
-        let role: string, text: string, nextRole: string, nextText: string
 
-        if (!message.message || !nextMessage.message) {
+        if (
+            !message.message
+            || !nextMessage.message
+            || message.message.content.content_type !== 'text'
+            || nextMessage.message.content.content_type !== 'text'
+        ) {
             idx += 1
             continue
         }
 
-        try {
+        const role = message.message.author.role
-            role = message.message.author.role
+        const text = message.message.content.parts[0]
-            text = message.message.content.parts[0]
+        const nextRole = nextMessage.message.author.role
-            nextRole = nextMessage.message.author.role
+        const nextText = nextMessage.message.content.parts[0]
-            nextText = nextMessage.message.content.parts[0]
-        }
-        catch (error) {
-            idx += 1
-            continue
-        }
 
         if (role === 'system') {
             if (text !== '') {
@@ -157,6 +104,7 @@ export function getOobaString(jsonData: ConversationData): string {
             idx += 1
         }
     }
+
     const oobaData: OobaData = {
         internal: pairs,
         visible: JSON.parse(JSON.stringify(pairs)),

src/utils/utils.ts

@@ -1,5 +1,9 @@
 export function noop() {}
 
+export function nonNullable<T>(x: T): x is NonNullable<T> {
+    return x != null
+}
+
 export function onloadSafe(fn: () => void) {
     if (document.readyState === 'complete') {
         fn()
@@ -32,3 +36,8 @@ export function unixTimestampToISOString(timestamp: number) {
     if (!timestamp) return ''
     return (new Date(timestamp * 1000)).toISOString()
 }
+
+export function jsonlStringify(list: any[]): string {
+    // This _has_ to be stringified without adding any indentation
+    return list.map((msg: any) => JSON.stringify(msg)).join('\n')
+}