upstream-archive/chatgpt-exporter
1
0
Fork 0
mirror of https://github.com/pionxzh/chatgpt-exporter.git synced 2026-07-23 09:00:51 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: support export to tavern and ooba format

Browse Source 
#210
This commit is contained in:
Pionxzh
2024-02-24 23:05:44 +08:00
parent 4ab9b19cf9
commit a28f9fdc35
17 changed files with 146 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.vscode/settings.json vendored
View File

@@ -35,6 +35,7 @@
"headlessui",
"mdast",
"micromark",
"ooba",
"preact",
"tabler",
"unist"

2
src/constants.ts
View File

@@ -8,7 +8,7 @@ export const apiUrl = API_MAPPING[baseUrl]
export const KEY_LANGUAGE = 'exporter:language'
export const KEY_FILENAME_FORMAT = 'exporter:filename_format'
export const KEY_OFFICIAL_JSON_FORMAT = 'exporter:official_json_format'
// export const KEY_OFFICIAL_JSON_FORMAT = 'exporter:official_json_format'
export const KEY_TIMESTAMP_ENABLED = 'exporter:enable_timestamp'
export const KEY_TIMESTAMP_24H = 'exporter:timestamp_24h'
export const KEY_TIMESTAMP_MARKDOWN = 'exporter:timestamp_markdown'

39
src/exporter/json.ts
View File

@@ -2,10 +2,11 @@ import JSZip from 'jszip'
import { fetchConversation, getCurrentChatId, processConversation } from '../api'
import i18n from '../i18n'
import { checkIfConversationStarted } from '../page'
import { convertToOoba, convertToTavern } from '../utils/conversion'
import { downloadFile, getFileNameWithFormat } from '../utils/download'
import type { ApiConversationWithId } from '../api'
export async function exportToJson(fileNameFormat: string, options: { officialFormat: boolean }) {
export async function exportToJson(fileNameFormat: string) {
if (!checkIfConversationStarted()) {
alert(i18n.t('Please start a conversation first'))
return false
@@ -19,7 +20,41 @@ export async function exportToJson(fileNameFormat: string, options: { officialFo
/**
* The official format is just an array of the API response.
*/
const content = conversationToJson(options.officialFormat ? [rawConversation] : rawConversation)
const content = conversationToJson([rawConversation])
downloadFile(fileName, 'application/json', content)
return true
}
export async function exportToTavern(fileNameFormat: string) {
if (!checkIfConversationStarted()) {
alert(i18n.t('Please start a conversation first'))
return false
}
const chatId = await getCurrentChatId()
const rawConversation = await fetchConversation(chatId, false)
const conversation = processConversation(rawConversation)
const fileName = getFileNameWithFormat(`${fileNameFormat}.tavern`, 'jsonl', { title: conversation.title, chatId })
const content = convertToTavern(conversation)
downloadFile(fileName, 'application/json-lines', content)
return true
}
export async function exportToOoba(fileNameFormat: string) {
if (!checkIfConversationStarted()) {
alert(i18n.t('Please start a conversation first'))
return false
}
const chatId = await getCurrentChatId()
const rawConversation = await fetchConversation(chatId, false)
const conversation = processConversation(rawConversation)
const fileName = getFileNameWithFormat(`${fileNameFormat}.ooba`, 'json', { title: conversation.title, chatId })
const content = convertToOoba(conversation)
downloadFile(fileName, 'application/json', content)
return true

2
src/locales/en.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "Enable on Markdown files",
"Use 24-hour format": "Use 24-hour format (eg. 23:59)",
"Export Format": "Export Format",
"Export JSON Format Description": "Export JSON in OpenAI Official Format",
"Export Metadata": "Export Metadata",
"Export Metadata Description": "Add metadata to exported Markdown and HTML files.",
"OpenAI Official Format": "OpenAI Official Format",
"Conversation Archive Alert": "Are you sure you want to archive all selected conversations?",
"Conversation Archived Message": "All selected conversations have been archived. Please refresh the page to see the changes.",
"Conversation Delete Alert": "Are you sure you want to delete all selected conversations?",

2
src/locales/es.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "Habilitar en archivos Markdown",
"Use 24-hour format": "Usar formato de 24 horas (ej. 23:59)",
"Export Format": "Formato de Exportación",
"Export JSON Format Description": "Exportar JSON en el Formato Oficial de OpenAI",
"Export Metadata": "Exportar Metadatos",
"Export Metadata Description": "Añadir Metadatos a los archivos Markdown y HTML exportados.",
"OpenAI Official Format": "Formato Oficial de OpenAI",
"Conversation Archive Alert": "¿Estás seguro que quieres archivar todas las conversaciones seleccionadas?",
"Conversation Archived Message": "Todos las conversaciones seleccionadas se han archivado. Por favor refresca la página para ver los cambios.",
"Conversation Delete Alert": "¿Estás seguro que quieres borrar todas las conversaciones seleccionadas?",

2
src/locales/id.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "Aktifkan pada file Markdown",
"Use 24-hour format": "Gunakan format 24 jam (contohnya: 23:59)",
"Export Format": "Format Ekspor",
"Export JSON Format Description": "Ekspor JSON dalam Format Resmi OpenAI",
"Export Metadata": "Ekspor Metada",
"Export Metadata Description": "Tambahkan metadata ke file Markdown dan HTML yang diekspor.",
"OpenAI Official Format": "Format Resmi OpenAI",
"Conversation Archive Alert": "Apakah Anda yakin ingin mengarsipkan semua percakapan yang dipilih?",
"Conversation Archived Message": "Semua percakapan yang dipilih telah diarsipkan. Harap segarkan halaman untuk melihat perubahan.",
"Conversation Delete Alert": "Apakah Anda yakin ingin menghapus semua percakapan yang dipilih?",

2
src/locales/jp.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "Markdown ファイルで有効にする",
"Use 24-hour format": "24時間形式を使用する (例: 23:59)",
"Export Format": "エクスポートフォーマット",
"Export JSON Format Description": "OpenAI公式フォーマットでのJSONのエクスポート",
"Export Metadata": "メタデータをエクスポート",
"Export Metadata Description": "エクスポートされたMarkdownおよびHTMLファイルにメタデータを追加します。",
"OpenAI Official Format": "OpenAI公式フォーマット",
"Conversation Archive Alert": "選択したすべての会話をアーカイブしてもよろしいですか？",
"Conversation Archived Message": "選択したすべての会話がアーカイブされました。変更を表示するには、ページを更新してください。",
"Conversation Delete Alert": "選択したすべての会話を削除してもよろしいですか？",

2
src/locales/tr.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "Markdown dosyalarında etkinleştir",
"Use 24-hour format": "24 saat biçimini kullan (örn. 23:59)",
"Export Format": "Dışa Aktarma Formatı",
"Export JSON Format Description": "OpenAI Resmi Formatında JSON Dışa Aktarma",
"Export Metadata": "Üst veriyi dışa aktar",
"Export Metadata Description": "Dışa aktarılan Markdown ve HTML dosyalarına üst veri ekle",
"OpenAI Official Format": "OpenAI Resmi Format",
"Conversation Archive Alert": "Seçilen tüm konuşmaları arşivlemek istediğinizden emin misiniz?",
"Conversation Archived Message": "Seçilen tüm konuşmalar arşivlendi. Değişiklikleri görmek için sayfayı yenileyin.",
"Conversation Delete Alert": "Seçilen tüm konuşmaları silmek istediğinizden emin misiniz?",

2
src/locales/zh-Hans.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "在 Markdown 文件上启用",
"Use 24-hour format": "使用24小时制 (例如 23:59)",
"Export Format": "导出格式",
"Export JSON Format Description": "以 OpenAI 官方格式导出JSON",
"Export Metadata": "导出元数据",
"Export Metadata Description": "会添加至 Markdown 以及 HTML 导出。",
"OpenAI Official Format": "OpenAI 官方格式",
"Conversation Archive Alert": "确定要归档所有选取的对话？",
"Conversation Archived Message": "所有所选的对话已归档。请刷新页面。",
"Conversation Delete Alert": "确定要删除所有选取的对话？",

2
src/locales/zh-Hant.json
View File

@@ -31,9 +31,9 @@
"Enable on Markdown": "在 Markdown 檔案上啟用",
"Use 24-hour format": "使用24小時制 (例如 23:59)",
"Export Format": "匯出格式",
"Export JSON Format Description": "以 OpenAI 官方格式匯出 JSON",
"Export Metadata": "匯出元資料",
"Export Metadata Description": "會添加至 Markdown 以及 HTML 匯出。",
"OpenAI Official Format": "OpenAI 官方格式",
"Conversation Archive Alert": "確定要封存所有選取的對話？",
"Conversation Archived Message": "所有選取的對話已封存。請重新整理頁面。",
"Conversation Delete Alert": "確定要刪除所有選取的對話？",

1
src/ui/Dialog.css
View File

@@ -21,6 +21,7 @@
overflow-y: auto;
padding: 16px 24px;
z-index: 1001;
outline: none;
animation: contentShow 150ms cubic-bezier(0.16, 1, 0.3, 1);
}

7
src/ui/ExportDialog.tsx
View File

@@ -75,14 +75,15 @@ interface DialogContentProps {
const DialogContent: FC<DialogContentProps> = ({ format }) => {
const { t } = useTranslation()
const { enableMeta, exportMetaList, exportOfficialJsonFormat } = useSettingContext()
const { enableMeta, exportMetaList } = useSettingContext()
const metaList = useMemo(() => enableMeta ? exportMetaList : [], [enableMeta, exportMetaList])
const exportAllOptions = useMemo(() => [
{ label: 'Markdown', callback: exportAllToMarkdown },
{ label: 'JSON', callback: exportOfficialJsonFormat ? exportAllToOfficialJson : exportAllToJson },
{ label: 'HTML', callback: exportAllToHtml },
], [exportOfficialJsonFormat])
{ label: 'JSON', callback: exportAllToOfficialJson },
{ label: 'JSON (ZIP)', callback: exportAllToJson },
], [])
const fileInputRef = useRef<HTMLInputElement>(null)
const [exportSource, setExportSource] = useState<ExportSource>('API')

46
src/ui/Menu.tsx
View File

@@ -1,9 +1,10 @@
import * as Dialog from '@radix-ui/react-dialog'
import * as HoverCard from '@radix-ui/react-hover-card'
import { useCallback, useEffect, useMemo, useState } from 'preact/hooks'
import { useTranslation } from 'react-i18next'
import { exportToHtml } from '../exporter/html'
import { exportToPng } from '../exporter/image'
import { exportToJson } from '../exporter/json'
import { exportToJson, exportToOoba, exportToTavern } from '../exporter/json'
import { exportToMarkdown } from '../exporter/markdown'
import { exportToText } from '../exporter/text'
import { useWindowResize } from '../hooks/useWindowResize'
@@ -23,12 +24,12 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
const disabled = getHistoryDisabled()
const [open, setOpen] = useState(false)
const [jsonOpen, setJsonOpen] = useState(false)
const [exportOpen, setExportOpen] = useState(false)
const [settingOpen, setSettingOpen] = useState(false)
const {
format,
exportOfficialJsonFormat,
enableTimestamp,
timeStamp24H,
enableMeta,
@@ -50,7 +51,13 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
const onClickPng = useCallback(() => exportToPng(format), [format])
const onClickMarkdown = useCallback(() => exportToMarkdown(format, metaList), [format, metaList])
const onClickHtml = useCallback(() => exportToHtml(format, metaList), [format, metaList])
const onClickJSON = useCallback(() => exportToJson(format, { officialFormat: exportOfficialJsonFormat }), [format, exportOfficialJsonFormat])
const onClickJSON = useCallback(() => {
setJsonOpen(true)
return true
}, [])
const onClickOfficialJSON = useCallback(() => exportToJson(format), [format])
const onClickTavern = useCallback(() => exportToTavern(format), [format])
const onClickOoba = useCallback(() => exportToOoba(format), [format])
const width = useWindowResize(() => window.innerWidth)
const isMobile = width < 768
@@ -95,7 +102,7 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
</HoverCard.Trigger>
<Portal
container={isMobile ? container : document.body}
forceMount={open || settingOpen || exportOpen}
forceMount={open || jsonOpen || settingOpen || exportOpen}
>
<HoverCard.Content
className={`
@@ -151,12 +158,43 @@ function MenuInner({ container }: { container: HTMLDivElement }) {
className="row-half"
onClick={onClickHtml}
/>
<Dialog.Root
open={jsonOpen}
onOpenChange={setJsonOpen}
>
<Dialog.Trigger asChild>
<MenuItem
text={t('JSON')}
icon={IconJSON}
className="row-half"
onClick={onClickJSON}
/>
</Dialog.Trigger>
<Dialog.Portal>
<Dialog.Overlay className="DialogOverlay" />
<Dialog.Content className="DialogContent" style={{ width: '320px' }}>
<Dialog.Title className="DialogTitle">{t('JSON')}</Dialog.Title>
<MenuItem
text={t('OpenAI Official Format')}
icon={IconCopy}
className="row-full"
onClick={onClickOfficialJSON}
/>
<MenuItem
text="JSONL (TavernAI, SillyTavern)"
icon={IconCopy}
className="row-full"
onClick={onClickTavern}
/>
<MenuItem
text="Ooba (text-generation-webui)"
icon={IconCopy}
className="row-full"
onClick={onClickOoba}
/>
</Dialog.Content>
</Dialog.Portal>
</Dialog.Root>
<ExportDialog
format={format}
open={exportOpen}

9
src/ui/SettingContext.tsx
View File

@@ -4,7 +4,6 @@ import {
KEY_FILENAME_FORMAT,
KEY_META_ENABLED,
KEY_META_LIST,
KEY_OFFICIAL_JSON_FORMAT,
KEY_TIMESTAMP_24H,
KEY_TIMESTAMP_ENABLED,
KEY_TIMESTAMP_HTML,
@@ -29,9 +28,6 @@ const SettingContext = createContext({
format: defaultFormat,
setFormat: (_: string) => {},
exportOfficialJsonFormat: false,
setExportOfficialJsonFormat: (_: boolean) => {},
enableTimestamp: false,
setEnableTimestamp: (_: boolean) => {},
timeStamp24H: false,
@@ -52,8 +48,6 @@ const SettingContext = createContext({
export const SettingProvider: FC = ({ children }) => {
const [format, setFormat] = useGMStorage(KEY_FILENAME_FORMAT, defaultFormat)
const [exportOfficialJsonFormat, setExportOfficialJsonFormat] = useGMStorage(KEY_OFFICIAL_JSON_FORMAT, false)
const [enableTimestamp, setEnableTimestamp] = useGMStorage(KEY_TIMESTAMP_ENABLED, false)
const [timeStamp24H, setTimeStamp24H] = useGMStorage(KEY_TIMESTAMP_24H, false)
const [enableTimestampHTML, setEnableTimestampHTML] = useGMStorage(KEY_TIMESTAMP_HTML, false)
@@ -75,9 +69,6 @@ export const SettingProvider: FC = ({ children }) => {
format,
setFormat,
exportOfficialJsonFormat,
setExportOfficialJsonFormat,
enableTimestamp,
setEnableTimestamp,
timeStamp24H,

17
src/ui/SettingDialog.tsx
View File

@@ -33,7 +33,6 @@ export const SettingDialog: FC<SettingDialogProps> = ({
timeStamp24H, setTimeStamp24H,
enableTimestampHTML, setEnableTimestampHTML,
enableTimestampMarkdown, setEnableTimestampMarkdown,
exportOfficialJsonFormat, setExportOfficialJsonFormat,
enableMeta, setEnableMeta,
exportMetaList, setExportMetaList,
/* eslint-enable pionxzh/consistent-list-newline */
@@ -111,22 +110,6 @@ export const SettingDialog: FC<SettingDialogProps> = ({
</dd>
</div>
</div>
<div className="relative flex bg-white dark:bg-white/5 rounded p-4">
<div>
<dt className="text-md font-medium text-gray-800 dark:text-white">
{t('Export Format')}
</dt>
<dd className="text-sm text-gray-700 dark:text-gray-300">
<div className="mt-2">
<Toggle
label={t('Export JSON Format Description')}
checked={exportOfficialJsonFormat}
onCheckedUpdate={setExportOfficialJsonFormat}
/>
</div>
</dd>
</div>
</div>
<div className="relative flex bg-white dark:bg-white/5 rounded p-4">
<div>
<dt className="text-md font-medium text-gray-800 dark:text-white">

124
src/utils/conversion.ts
View File

@@ -1,23 +1,9 @@
interface Message {
parent?: string
message?: {
author: {
role: string
}
create_time: number
content: {
parts: string[]
}
}
}
import { jsonlStringify, nonNullable } from './utils'
import type { ConversationNode, ConversationResult } from '../api'
interface MessageMapping {
[key: string]: Message
}
interface ConversationData {
current_node: string
mapping: MessageMapping
interface NameMessage {
user_name: string
character_name: string
}
interface TavernMessage {
@@ -30,50 +16,25 @@ interface TavernMessage {
swipe_id: number
}
interface NameMessage {
user_name: string
character_name: string
}
interface OobaData {
internal: [string, string][]
visible: [string, string][]
}
function extractConversation(data: ConversationData): Message[] {
const currentNode = data.current_node
const mapping = data.mapping
const messagesReversed: Message[] = []
let nodeId: string | undefined = currentNode
while (nodeId !== null && nodeId !== undefined) {
const currentMessage: Message = mapping[nodeId]
messagesReversed.push(currentMessage)
nodeId = currentMessage.parent
}
return messagesReversed.reverse()
}
function convertMessageToTavern(messageData: Message): TavernMessage | null {
if (!messageData.message) {
function convertMessageToTavern(node: ConversationNode): TavernMessage | null {
if (!node.message || node.message.content.content_type !== 'text') {
return null
}
const senderRole: string = messageData.message.author.role
if (senderRole === 'system') {
return null
}
const isAssistant = senderRole === 'assistant'
const createTime: number = messageData.message.create_time
const text: string = messageData.message.content.parts[0]
const authorRole = node.message.author.role
const createTime = node.message.create_time || (new Date()).getTime() / 1000
const text = node.message.content.parts.join('\n')
return {
name: isAssistant ? 'Assistant' : 'You',
is_user: !isAssistant,
is_name: isAssistant,
name: authorRole === 'assistant' ? 'Assistant' : 'You',
is_user: authorRole === 'user',
// This seems to be always true
is_name: true,
send_date: createTime,
mes: text,
swipes: [text],
@@ -81,55 +42,41 @@ function convertMessageToTavern(messageData: Message): TavernMessage | null {
}
}
function jsonlStringify(messageArray: any[]): string {
return messageArray.map((msg: any) => JSON.stringify(msg)).join('\n')
}
export function getTavernString(jsonData: ConversationData): string {
// Takes the OAI JSON data as input, outputs the JSONL string
const conversation = extractConversation(jsonData)
const convertedConvo: (TavernMessage | NameMessage)[] = [{
export function convertToTavern(conversation: ConversationResult): string {
const messages: (NameMessage | TavernMessage)[] = [
{
user_name: 'You',
character_name: 'Assistant',
}]
},
...conversation.conversationNodes.map(convertMessageToTavern).filter(nonNullable),
]
conversation.forEach((message) => {
const convertedMsg = convertMessageToTavern(message)
if (convertedMsg !== null) {
convertedConvo.push(convertedMsg)
}
})
// This _has_ to be stringified without adding any indentation, due to the JSONL format.
return jsonlStringify(convertedConvo)
return jsonlStringify(messages)
}
export function getOobaString(jsonData: ConversationData): string {
// Takes the OAI JSON data as input, outputs the serialized JSON
const messages = extractConversation(jsonData)
const pairs: any[] = []
let idx = 0
export function convertToOoba(conversation: ConversationResult): string {
const pairs: [string, string][] = []
const messages = conversation.conversationNodes.filter(node => node.message?.author.role !== 'tool' && node.message?.content.content_type === 'text')
let idx = 0
while (idx < messages.length - 1) {
const message = messages[idx]
const nextMessage = messages[idx + 1]
let role: string, text: string, nextRole: string, nextText: string
if (!message.message || !nextMessage.message) {
if (
!message.message
|| !nextMessage.message
|| message.message.content.content_type !== 'text'
|| nextMessage.message.content.content_type !== 'text'
) {
idx += 1
continue
}
try {
role = message.message.author.role
text = message.message.content.parts[0]
nextRole = nextMessage.message.author.role
nextText = nextMessage.message.content.parts[0]
}
catch (error) {
idx += 1
continue
}
const role = message.message.author.role
const text = message.message.content.parts[0]
const nextRole = nextMessage.message.author.role
const nextText = nextMessage.message.content.parts[0]
if (role === 'system') {
if (text !== '') {
@@ -157,6 +104,7 @@ export function getOobaString(jsonData: ConversationData): string {
idx += 1
}
}
const oobaData: OobaData = {
internal: pairs,
visible: JSON.parse(JSON.stringify(pairs)),

9
src/utils/utils.ts
View File

@@ -1,5 +1,9 @@
export function noop() {}
export function nonNullable<T>(x: T): x is NonNullable<T> {
return x != null
}
export function onloadSafe(fn: () => void) {
if (document.readyState === 'complete') {
fn()
@@ -32,3 +36,8 @@ export function unixTimestampToISOString(timestamp: number) {
if (!timestamp) return ''
return (new Date(timestamp * 1000)).toISOString()
}
export function jsonlStringify(list: any[]): string {
// This _has_ to be stringified without adding any indentation
return list.map((msg: any) => JSON.stringify(msg)).join('\n')
}