From a28f9fdc353464d32d92a5d977272de324628e3a Mon Sep 17 00:00:00 2001 From: Pionxzh Date: Sat, 24 Feb 2024 23:05:44 +0800 Subject: [PATCH] feat: support export to tavern and ooba format #210 --- .vscode/settings.json | 1 + src/constants.ts | 2 +- src/exporter/json.ts | 39 +++++++++++- src/locales/en.json | 2 +- src/locales/es.json | 2 +- src/locales/id.json | 2 +- src/locales/jp.json | 2 +- src/locales/tr.json | 2 +- src/locales/zh-Hans.json | 2 +- src/locales/zh-Hant.json | 2 +- src/ui/Dialog.css | 1 + src/ui/ExportDialog.tsx | 7 ++- src/ui/Menu.tsx | 58 ++++++++++++++--- src/ui/SettingContext.tsx | 9 --- src/ui/SettingDialog.tsx | 17 ----- src/utils/conversion.ts | 128 +++++++++++--------------------------- src/utils/utils.ts | 9 +++ 17 files changed, 146 insertions(+), 139 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index bdc920f..4a08ef5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -35,6 +35,7 @@ "headlessui", "mdast", "micromark", + "ooba", "preact", "tabler", "unist" diff --git a/src/constants.ts b/src/constants.ts index 14b169e..691c207 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -8,7 +8,7 @@ export const apiUrl = API_MAPPING[baseUrl] export const KEY_LANGUAGE = 'exporter:language' export const KEY_FILENAME_FORMAT = 'exporter:filename_format' -export const KEY_OFFICIAL_JSON_FORMAT = 'exporter:official_json_format' +// export const KEY_OFFICIAL_JSON_FORMAT = 'exporter:official_json_format' export const KEY_TIMESTAMP_ENABLED = 'exporter:enable_timestamp' export const KEY_TIMESTAMP_24H = 'exporter:timestamp_24h' export const KEY_TIMESTAMP_MARKDOWN = 'exporter:timestamp_markdown' diff --git a/src/exporter/json.ts b/src/exporter/json.ts index 98b189d..7256af8 100644 --- a/src/exporter/json.ts +++ b/src/exporter/json.ts @@ -2,10 +2,11 @@ import JSZip from 'jszip' import { fetchConversation, getCurrentChatId, processConversation } from '../api' import i18n from '../i18n' import { checkIfConversationStarted } from '../page' +import { convertToOoba, convertToTavern } from '../utils/conversion' import { downloadFile, getFileNameWithFormat } from '../utils/download' import type { ApiConversationWithId } from '../api' -export async function exportToJson(fileNameFormat: string, options: { officialFormat: boolean }) { +export async function exportToJson(fileNameFormat: string) { if (!checkIfConversationStarted()) { alert(i18n.t('Please start a conversation first')) return false @@ -19,7 +20,41 @@ export async function exportToJson(fileNameFormat: string, options: { officialFo /** * The official format is just an array of the API response. */ - const content = conversationToJson(options.officialFormat ? [rawConversation] : rawConversation) + const content = conversationToJson([rawConversation]) + downloadFile(fileName, 'application/json', content) + + return true +} + +export async function exportToTavern(fileNameFormat: string) { + if (!checkIfConversationStarted()) { + alert(i18n.t('Please start a conversation first')) + return false + } + + const chatId = await getCurrentChatId() + const rawConversation = await fetchConversation(chatId, false) + const conversation = processConversation(rawConversation) + + const fileName = getFileNameWithFormat(`${fileNameFormat}.tavern`, 'jsonl', { title: conversation.title, chatId }) + const content = convertToTavern(conversation) + downloadFile(fileName, 'application/json-lines', content) + + return true +} + +export async function exportToOoba(fileNameFormat: string) { + if (!checkIfConversationStarted()) { + alert(i18n.t('Please start a conversation first')) + return false + } + + const chatId = await getCurrentChatId() + const rawConversation = await fetchConversation(chatId, false) + const conversation = processConversation(rawConversation) + + const fileName = getFileNameWithFormat(`${fileNameFormat}.ooba`, 'json', { title: conversation.title, chatId }) + const content = convertToOoba(conversation) downloadFile(fileName, 'application/json', content) return true diff --git a/src/locales/en.json b/src/locales/en.json index a99e239..a970696 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -31,9 +31,9 @@ "Enable on Markdown": "Enable on Markdown files", "Use 24-hour format": "Use 24-hour format (eg. 23:59)", "Export Format": "Export Format", - "Export JSON Format Description": "Export JSON in OpenAI Official Format", "Export Metadata": "Export Metadata", "Export Metadata Description": "Add metadata to exported Markdown and HTML files.", + "OpenAI Official Format": "OpenAI Official Format", "Conversation Archive Alert": "Are you sure you want to archive all selected conversations?", "Conversation Archived Message": "All selected conversations have been archived. Please refresh the page to see the changes.", "Conversation Delete Alert": "Are you sure you want to delete all selected conversations?", diff --git a/src/locales/es.json b/src/locales/es.json index fd63d22..001c212 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -31,9 +31,9 @@ "Enable on Markdown": "Habilitar en archivos Markdown", "Use 24-hour format": "Usar formato de 24 horas (ej. 23:59)", "Export Format": "Formato de Exportación", - "Export JSON Format Description": "Exportar JSON en el Formato Oficial de OpenAI", "Export Metadata": "Exportar Metadatos", "Export Metadata Description": "Añadir Metadatos a los archivos Markdown y HTML exportados.", + "OpenAI Official Format": "Formato Oficial de OpenAI", "Conversation Archive Alert": "¿Estás seguro que quieres archivar todas las conversaciones seleccionadas?", "Conversation Archived Message": "Todos las conversaciones seleccionadas se han archivado. Por favor refresca la página para ver los cambios.", "Conversation Delete Alert": "¿Estás seguro que quieres borrar todas las conversaciones seleccionadas?", diff --git a/src/locales/id.json b/src/locales/id.json index e83e1ef..0f4fa87 100644 --- a/src/locales/id.json +++ b/src/locales/id.json @@ -31,9 +31,9 @@ "Enable on Markdown": "Aktifkan pada file Markdown", "Use 24-hour format": "Gunakan format 24 jam (contohnya: 23:59)", "Export Format": "Format Ekspor", - "Export JSON Format Description": "Ekspor JSON dalam Format Resmi OpenAI", "Export Metadata": "Ekspor Metada", "Export Metadata Description": "Tambahkan metadata ke file Markdown dan HTML yang diekspor.", + "OpenAI Official Format": "Format Resmi OpenAI", "Conversation Archive Alert": "Apakah Anda yakin ingin mengarsipkan semua percakapan yang dipilih?", "Conversation Archived Message": "Semua percakapan yang dipilih telah diarsipkan. Harap segarkan halaman untuk melihat perubahan.", "Conversation Delete Alert": "Apakah Anda yakin ingin menghapus semua percakapan yang dipilih?", diff --git a/src/locales/jp.json b/src/locales/jp.json index ee6771f..5e94f27 100644 --- a/src/locales/jp.json +++ b/src/locales/jp.json @@ -31,9 +31,9 @@ "Enable on Markdown": "Markdown ファイルで有効にする", "Use 24-hour format": "24時間形式を使用する (例: 23:59)", "Export Format": "エクスポートフォーマット", - "Export JSON Format Description": "OpenAI公式フォーマットでのJSONのエクスポート", "Export Metadata": "メタデータをエクスポート", "Export Metadata Description": "エクスポートされたMarkdownおよびHTMLファイルにメタデータを追加します。", + "OpenAI Official Format": "OpenAI公式フォーマット", "Conversation Archive Alert": "選択したすべての会話をアーカイブしてもよろしいですか？", "Conversation Archived Message": "選択したすべての会話がアーカイブされました。変更を表示するには、ページを更新してください。", "Conversation Delete Alert": "選択したすべての会話を削除してもよろしいですか？", diff --git a/src/locales/tr.json b/src/locales/tr.json index 52b5bf5..75f47fe 100644 --- a/src/locales/tr.json +++ b/src/locales/tr.json @@ -31,9 +31,9 @@ "Enable on Markdown": "Markdown dosyalarında etkinleştir", "Use 24-hour format": "24 saat biçimini kullan (örn. 23:59)", "Export Format": "Dışa Aktarma Formatı", - "Export JSON Format Description": "OpenAI Resmi Formatında JSON Dışa Aktarma", "Export Metadata": "Üst veriyi dışa aktar", "Export Metadata Description": "Dışa aktarılan Markdown ve HTML dosyalarına üst veri ekle", + "OpenAI Official Format": "OpenAI Resmi Format", "Conversation Archive Alert": "Seçilen tüm konuşmaları arşivlemek istediğinizden emin misiniz?", "Conversation Archived Message": "Seçilen tüm konuşmalar arşivlendi. Değişiklikleri görmek için sayfayı yenileyin.", "Conversation Delete Alert": "Seçilen tüm konuşmaları silmek istediğinizden emin misiniz?", diff --git a/src/locales/zh-Hans.json b/src/locales/zh-Hans.json index 24e1a38..e8172ac 100644 --- a/src/locales/zh-Hans.json +++ b/src/locales/zh-Hans.json @@ -31,9 +31,9 @@ "Enable on Markdown": "在 Markdown 文件上启用", "Use 24-hour format": "使用24小时制 (例如 23:59)", "Export Format": "导出格式", - "Export JSON Format Description": "以 OpenAI 官方格式导出JSON", "Export Metadata": "导出元数据", "Export Metadata Description": "会添加至 Markdown 以及 HTML 导出。", + "OpenAI Official Format": "OpenAI 官方格式", "Conversation Archive Alert": "确定要归档所有选取的对话？", "Conversation Archived Message": "所有所选的对话已归档。请刷新页面。", "Conversation Delete Alert": "确定要删除所有选取的对话？", diff --git a/src/locales/zh-Hant.json b/src/locales/zh-Hant.json index b36cc47..deb83df 100644 --- a/src/locales/zh-Hant.json +++ b/src/locales/zh-Hant.json @@ -31,9 +31,9 @@ "Enable on Markdown": "在 Markdown 檔案上啟用", "Use 24-hour format": "使用24小時制 (例如 23:59)", "Export Format": "匯出格式", - "Export JSON Format Description": "以 OpenAI 官方格式匯出 JSON", "Export Metadata": "匯出元資料", "Export Metadata Description": "會添加至 Markdown 以及 HTML 匯出。", + "OpenAI Official Format": "OpenAI 官方格式", "Conversation Archive Alert": "確定要封存所有選取的對話？", "Conversation Archived Message": "所有選取的對話已封存。請重新整理頁面。", "Conversation Delete Alert": "確定要刪除所有選取的對話？", diff --git a/src/ui/Dialog.css b/src/ui/Dialog.css index 7ca65a9..9e5e8b3 100644 --- a/src/ui/Dialog.css +++ b/src/ui/Dialog.css @@ -21,6 +21,7 @@ overflow-y: auto; padding: 16px 24px; z-index: 1001; + outline: none; animation: contentShow 150ms cubic-bezier(0.16, 1, 0.3, 1); } diff --git a/src/ui/ExportDialog.tsx b/src/ui/ExportDialog.tsx index bb422f5..7ed5bdf 100644 --- a/src/ui/ExportDialog.tsx +++ b/src/ui/ExportDialog.tsx @@ -75,14 +75,15 @@ interface DialogContentProps { const DialogContent: FC = ({ format }) => { const { t } = useTranslation() - const { enableMeta, exportMetaList, exportOfficialJsonFormat } = useSettingContext() + const { enableMeta, exportMetaList } = useSettingContext() const metaList = useMemo(() => enableMeta ? exportMetaList : [], [enableMeta, exportMetaList]) const exportAllOptions = useMemo(() => [ { label: 'Markdown', callback: exportAllToMarkdown }, - { label: 'JSON', callback: exportOfficialJsonFormat ? exportAllToOfficialJson : exportAllToJson }, { label: 'HTML', callback: exportAllToHtml }, - ], [exportOfficialJsonFormat]) + { label: 'JSON', callback: exportAllToOfficialJson }, + { label: 'JSON (ZIP)', callback: exportAllToJson }, + ], []) const fileInputRef = useRef(null) const [exportSource, setExportSource] = useState('API') diff --git a/src/ui/Menu.tsx b/src/ui/Menu.tsx index d55f856..6cc872e 100644 --- a/src/ui/Menu.tsx +++ b/src/ui/Menu.tsx @@ -1,9 +1,10 @@ +import * as Dialog from '@radix-ui/react-dialog' import * as HoverCard from '@radix-ui/react-hover-card' import { useCallback, useEffect, useMemo, useState } from 'preact/hooks' import { useTranslation } from 'react-i18next' import { exportToHtml } from '../exporter/html' import { exportToPng } from '../exporter/image' -import { exportToJson } from '../exporter/json' +import { exportToJson, exportToOoba, exportToTavern } from '../exporter/json' import { exportToMarkdown } from '../exporter/markdown' import { exportToText } from '../exporter/text' import { useWindowResize } from '../hooks/useWindowResize' @@ -23,12 +24,12 @@ function MenuInner({ container }: { container: HTMLDivElement }) { const disabled = getHistoryDisabled() const [open, setOpen] = useState(false) + const [jsonOpen, setJsonOpen] = useState(false) const [exportOpen, setExportOpen] = useState(false) const [settingOpen, setSettingOpen] = useState(false) const { format, - exportOfficialJsonFormat, enableTimestamp, timeStamp24H, enableMeta, @@ -50,7 +51,13 @@ function MenuInner({ container }: { container: HTMLDivElement }) { const onClickPng = useCallback(() => exportToPng(format), [format]) const onClickMarkdown = useCallback(() => exportToMarkdown(format, metaList), [format, metaList]) const onClickHtml = useCallback(() => exportToHtml(format, metaList), [format, metaList]) - const onClickJSON = useCallback(() => exportToJson(format, { officialFormat: exportOfficialJsonFormat }), [format, exportOfficialJsonFormat]) + const onClickJSON = useCallback(() => { + setJsonOpen(true) + return true + }, []) + const onClickOfficialJSON = useCallback(() => exportToJson(format), [format]) + const onClickTavern = useCallback(() => exportToTavern(format), [format]) + const onClickOoba = useCallback(() => exportToOoba(format), [format]) const width = useWindowResize(() => window.innerWidth) const isMobile = width < 768 @@ -95,7 +102,7 @@ function MenuInner({ container }: { container: HTMLDivElement }) { - + + + + + + + + {t('JSON')} + + + + + + {}, - exportOfficialJsonFormat: false, - setExportOfficialJsonFormat: (_: boolean) => {}, - enableTimestamp: false, setEnableTimestamp: (_: boolean) => {}, timeStamp24H: false, @@ -52,8 +48,6 @@ const SettingContext = createContext({ export const SettingProvider: FC = ({ children }) => { const [format, setFormat] = useGMStorage(KEY_FILENAME_FORMAT, defaultFormat) - const [exportOfficialJsonFormat, setExportOfficialJsonFormat] = useGMStorage(KEY_OFFICIAL_JSON_FORMAT, false) - const [enableTimestamp, setEnableTimestamp] = useGMStorage(KEY_TIMESTAMP_ENABLED, false) const [timeStamp24H, setTimeStamp24H] = useGMStorage(KEY_TIMESTAMP_24H, false) const [enableTimestampHTML, setEnableTimestampHTML] = useGMStorage(KEY_TIMESTAMP_HTML, false) @@ -75,9 +69,6 @@ export const SettingProvider: FC = ({ children }) => { format, setFormat, - exportOfficialJsonFormat, - setExportOfficialJsonFormat, - enableTimestamp, setEnableTimestamp, timeStamp24H, diff --git a/src/ui/SettingDialog.tsx b/src/ui/SettingDialog.tsx index fd7d2c8..d9f66b8 100644 --- a/src/ui/SettingDialog.tsx +++ b/src/ui/SettingDialog.tsx @@ -33,7 +33,6 @@ export const SettingDialog: FC = ({ timeStamp24H, setTimeStamp24H, enableTimestampHTML, setEnableTimestampHTML, enableTimestampMarkdown, setEnableTimestampMarkdown, - exportOfficialJsonFormat, setExportOfficialJsonFormat, enableMeta, setEnableMeta, exportMetaList, setExportMetaList, /* eslint-enable pionxzh/consistent-list-newline */ @@ -111,22 +110,6 @@ export const SettingDialog: FC = ({ -
-
-
- {t('Export Format')} -
-
-
- -
-
-
-
diff --git a/src/utils/conversion.ts b/src/utils/conversion.ts index 53dc835..282c3ea 100644 --- a/src/utils/conversion.ts +++ b/src/utils/conversion.ts @@ -1,23 +1,9 @@ -interface Message { - parent?: string - message?: { - author: { - role: string - } - create_time: number - content: { - parts: string[] - } - } -} +import { jsonlStringify, nonNullable } from './utils' +import type { ConversationNode, ConversationResult } from '../api' -interface MessageMapping { - [key: string]: Message -} - -interface ConversationData { - current_node: string - mapping: MessageMapping +interface NameMessage { + user_name: string + character_name: string } interface TavernMessage { @@ -30,50 +16,25 @@ interface TavernMessage { swipe_id: number } -interface NameMessage { - user_name: string - character_name: string -} - interface OobaData { internal: [string, string][] visible: [string, string][] } -function extractConversation(data: ConversationData): Message[] { - const currentNode = data.current_node - const mapping = data.mapping - - const messagesReversed: Message[] = [] - let nodeId: string | undefined = currentNode - - while (nodeId !== null && nodeId !== undefined) { - const currentMessage: Message = mapping[nodeId] - messagesReversed.push(currentMessage) - nodeId = currentMessage.parent - } - - return messagesReversed.reverse() -} - -function convertMessageToTavern(messageData: Message): TavernMessage | null { - if (!messageData.message) { +function convertMessageToTavern(node: ConversationNode): TavernMessage | null { + if (!node.message || node.message.content.content_type !== 'text') { return null } - const senderRole: string = messageData.message.author.role - if (senderRole === 'system') { - return null - } - - const isAssistant = senderRole === 'assistant' - const createTime: number = messageData.message.create_time - const text: string = messageData.message.content.parts[0] + const authorRole = node.message.author.role + const createTime = node.message.create_time || (new Date()).getTime() / 1000 + const text = node.message.content.parts.join('\n') return { - name: isAssistant ? 'Assistant' : 'You', - is_user: !isAssistant, - is_name: isAssistant, + name: authorRole === 'assistant' ? 'Assistant' : 'You', + is_user: authorRole === 'user', + // This seems to be always true + is_name: true, send_date: createTime, mes: text, swipes: [text], @@ -81,55 +42,41 @@ function convertMessageToTavern(messageData: Message): TavernMessage | null { } } -function jsonlStringify(messageArray: any[]): string { - return messageArray.map((msg: any) => JSON.stringify(msg)).join('\n') +export function convertToTavern(conversation: ConversationResult): string { + const messages: (NameMessage | TavernMessage)[] = [ + { + user_name: 'You', + character_name: 'Assistant', + }, + ...conversation.conversationNodes.map(convertMessageToTavern).filter(nonNullable), + ] + + return jsonlStringify(messages) } -export function getTavernString(jsonData: ConversationData): string { - // Takes the OAI JSON data as input, outputs the JSONL string - const conversation = extractConversation(jsonData) +export function convertToOoba(conversation: ConversationResult): string { + const pairs: [string, string][] = [] + const messages = conversation.conversationNodes.filter(node => node.message?.author.role !== 'tool' && node.message?.content.content_type === 'text') - const convertedConvo: (TavernMessage | NameMessage)[] = [{ - user_name: 'You', - character_name: 'Assistant', - }] - - conversation.forEach((message) => { - const convertedMsg = convertMessageToTavern(message) - if (convertedMsg !== null) { - convertedConvo.push(convertedMsg) - } - }) - // This _has_ to be stringified without adding any indentation, due to the JSONL format. - return jsonlStringify(convertedConvo) -} - -export function getOobaString(jsonData: ConversationData): string { - // Takes the OAI JSON data as input, outputs the serialized JSON - const messages = extractConversation(jsonData) - const pairs: any[] = [] let idx = 0 - while (idx < messages.length - 1) { const message = messages[idx] const nextMessage = messages[idx + 1] - let role: string, text: string, nextRole: string, nextText: string - if (!message.message || !nextMessage.message) { + if ( + !message.message + || !nextMessage.message + || message.message.content.content_type !== 'text' + || nextMessage.message.content.content_type !== 'text' + ) { idx += 1 continue } - try { - role = message.message.author.role - text = message.message.content.parts[0] - nextRole = nextMessage.message.author.role - nextText = nextMessage.message.content.parts[0] - } - catch (error) { - idx += 1 - continue - } + const role = message.message.author.role + const text = message.message.content.parts[0] + const nextRole = nextMessage.message.author.role + const nextText = nextMessage.message.content.parts[0] if (role === 'system') { if (text !== '') { @@ -157,6 +104,7 @@ export function getOobaString(jsonData: ConversationData): string { idx += 1 } } + const oobaData: OobaData = { internal: pairs, visible: JSON.parse(JSON.stringify(pairs)), diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 063d65e..6046a5a 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,5 +1,9 @@ export function noop() {} +export function nonNullable(x: T): x is NonNullable { + return x != null +} + export function onloadSafe(fn: () => void) { if (document.readyState === 'complete') { fn() @@ -32,3 +36,8 @@ export function unixTimestampToISOString(timestamp: number) { if (!timestamp) return '' return (new Date(timestamp * 1000)).toISOString() } + +export function jsonlStringify(list: any[]): string { + // This _has_ to be stringified without adding any indentation + return list.map((msg: any) => JSON.stringify(msg)).join('\n') +}