mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-20 09:47:42 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
184862801a | ||
|
|
79e81b0bd4 | ||
|
|
059cac2848 | ||
|
|
cf4153a9ba | ||
|
|
8e7862411d | ||
|
|
e626db7caa | ||
|
|
c6a25fc622 | ||
|
|
5f837145d8 | ||
|
|
36cd3aff40 | ||
|
|
28de970cc1 | ||
|
|
4b69c79cca |
@@ -18,6 +18,10 @@ name: UI Lib Publish
|
||||
# ui-lib-publish.yml, and Environment left blank - this job names no environment, since creating
|
||||
# one needs repository admin. Merging this workflow before that exists leaves the publish step
|
||||
# red on master: the release is late rather than wrong, and re-running the job publishes it.
|
||||
#
|
||||
# The unscoped llama-ui is an alias: once the scoped package is out, the same build is renamed
|
||||
# and published again at the same version, so the two never drift. It is a package of its own
|
||||
# on npm, and needs the same trusted publisher set on it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -128,6 +132,30 @@ jobs:
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Rename the build to the unscoped alias
|
||||
# after the scoped publish, so nothing above sees the changed name. Not on a PR: the
|
||||
# advisory check there is about the version, which the alias shares.
|
||||
if: github.event_name != 'pull_request'
|
||||
run: node scripts/alias-package.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the alias with npm
|
||||
id: alias
|
||||
if: github.event_name != 'pull_request'
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Publish the alias
|
||||
# its own verdict rather than the scoped one's, so an alias that fell behind catches up
|
||||
# on the next run instead of waiting for a bump
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# every value reaches the shell through env, not through the script, so a package name
|
||||
@@ -140,6 +168,11 @@ jobs:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
SHOULD: ${{ steps.version.outputs.should_publish }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
ALIAS_LATEST: ${{ steps.alias.outputs.latest }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
ALIAS_SHOULD: ${{ steps.alias.outputs.should_publish }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$SHOULD" = "true" ]; then published=yes
|
||||
@@ -156,6 +189,23 @@ jobs:
|
||||
echo
|
||||
echo "> $NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
[ "$EVENT" = "pull_request" ] && exit 0
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$ALIAS_SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo
|
||||
echo "### llama-ui (alias) $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${ALIAS_LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$ALIAS_VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $ALIAS_NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
if: always() && github.event_name != 'pull_request' && steps.version.outcome == 'success'
|
||||
|
||||
@@ -19,6 +19,7 @@ on:
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'ui/lib/voice/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -28,6 +29,7 @@ on:
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'ui/lib/voice/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
|
||||
@@ -150,6 +150,37 @@ jobs:
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
voice-checks:
|
||||
name: Voice checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/voice
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
needs: ui-build
|
||||
|
||||
+9
-1
@@ -3420,12 +3420,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused unless --ui-mcp-proxy-allow-private-urls is set - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY"));
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy-allow-private-urls"},
|
||||
"allow the MCP CORS proxy to reach local or private addresses (e.g. MCP servers on localhost or LAN) - anyone who can reach the proxy can then reach your local network (default: disabled, only public addresses)\n"
|
||||
"note: only has effect with --ui-mcp-proxy",
|
||||
[](common_params & params) {
|
||||
params.ui_mcp_proxy_allow_private_urls = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY_ALLOW_PRIVATE_URLS"));
|
||||
add_opt(common_arg(
|
||||
{"--tools"}, "TOOL1,TOOL2,...",
|
||||
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
|
||||
|
||||
@@ -702,6 +702,7 @@ struct common_params {
|
||||
// UI configs
|
||||
bool ui = true;
|
||||
bool ui_mcp_proxy = false;
|
||||
bool ui_mcp_proxy_allow_private_urls = false; // allow the MCP CORS proxy to reach local or private addresses
|
||||
std::string ui_config_json;
|
||||
|
||||
// "advanced" endpoints are disabled by default for better security
|
||||
|
||||
@@ -6798,7 +6798,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_RWKV_WKV7:
|
||||
return true;
|
||||
case GGML_OP_GATED_DELTA_NET:
|
||||
return true;
|
||||
return ggml_are_same_stride(op->src[0], op->src[1]);
|
||||
case GGML_OP_DSV4_HC_COMB:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->src[2]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
|
||||
|
||||
@@ -264,7 +264,7 @@ def benchmark(
|
||||
|
||||
bin_max = np.ceil(token_t_last) + 1
|
||||
plt.figure()
|
||||
plt.hist(token_t, np.linspace(0, bin_max, bin_max, endpoint=False).tolist())
|
||||
plt.hist(token_t, np.linspace(0, bin_max, int(bin_max), endpoint=False).tolist())
|
||||
plt.xlim(0, bin_max + 1)
|
||||
plt.title(name or "")
|
||||
plt.xlabel("Time [s]")
|
||||
|
||||
+151
-17
@@ -157,6 +157,29 @@ static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x,
|
||||
return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], x->nb[1], idx * x->nb[2]);
|
||||
}
|
||||
|
||||
class llm_graph_input_gemma4_prune : public llm_graph_input_i {
|
||||
int64_t n_slices;
|
||||
int64_t n_slice_tokens;
|
||||
int64_t n_slice_outputs;
|
||||
public:
|
||||
ggml_tensor * full_mask = nullptr;
|
||||
ggml_tensor * full_mask_swa = nullptr;
|
||||
ggml_tensor * out_mask = nullptr;
|
||||
ggml_tensor * out_mask_swa = nullptr;
|
||||
ggml_tensor * inp_prune_ids = nullptr;
|
||||
ggml_tensor * inp_prune_keep = nullptr;
|
||||
llm_graph_input_gemma4_prune(int64_t n_slices, int64_t n_slice_tokens, int64_t n_slice_outputs)
|
||||
: n_slices(n_slices), n_slice_tokens(n_slice_tokens), n_slice_outputs(n_slice_outputs) {}
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override;
|
||||
|
||||
bool can_reuse(const llm_graph_params & params) override;
|
||||
|
||||
static int64_t max_outputs_per_slice(
|
||||
const llama_ubatch & ubatch, int64_t n_slices, int64_t n_slice_tokens
|
||||
);
|
||||
};
|
||||
|
||||
llama_model_gemma4::graph::graph(const llm_graph_params & params) :
|
||||
llm_graph_context(params),
|
||||
n_embd_per_layer(model.hparams.n_embd_per_layer) {
|
||||
@@ -181,11 +204,8 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
|
||||
// TODO: is causal == true correct? might need some changes
|
||||
auto * inp_attn = build_attn_inp_kv_iswa();
|
||||
|
||||
auto inp_attn_full = *inp_attn;
|
||||
auto inp_attn_prune = *inp_attn;
|
||||
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
if (model.per_layer_tok_embd) {
|
||||
// fold per_layer_input_scale (1/sqrt(2)) from project_per_layer_inputs() into tok_embd_scale
|
||||
const float tok_embd_scale = sqrtf((float) n_embd_per_layer) / sqrtf(2.0f);
|
||||
@@ -197,34 +217,57 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
|
||||
inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer);
|
||||
}
|
||||
|
||||
const bool needs_unmasked_nextn =
|
||||
cparams.embeddings_nextn && !cparams.embeddings_nextn_masked;
|
||||
|
||||
// KV-shared layers do not write to the KV cache, so prune non-output token rows before the first KV-shared layer
|
||||
bool prune_shared = crop_before_nextn(inp_out_ids) &&
|
||||
bool prune_shared = !needs_unmasked_nextn &&
|
||||
hparams.n_layer_kv_from_start > 0 && hparams.n_layer_kv_from_start < n_layer &&
|
||||
!cparams.embeddings && cparams.n_seq_max == 1;
|
||||
!cparams.embeddings;
|
||||
for (int il = (int) hparams.n_layer_kv_from_start; prune_shared && il <= n_layer; ++il) {
|
||||
prune_shared = !cparams.embeddings_layer_inp[il];
|
||||
}
|
||||
|
||||
int64_t n_rows = n_tokens;
|
||||
|
||||
ggml_tensor * inp_prune_ids = nullptr;
|
||||
ggml_tensor * inp_prune_keep = nullptr;
|
||||
ggml_tensor * inp_out_ids = prune_shared ? nullptr : build_inp_out_ids();
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
if (prune_shared && il == (int) hparams.n_layer_kv_from_start) {
|
||||
inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
|
||||
inp_pos = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inp_pos, 1, n_tokens), inp_out_ids), n_outputs);
|
||||
ggml_tensor * full = inp_attn->self_attn_mask.raw;
|
||||
ggml_tensor * full_swa = inp_attn->self_attn_mask_swa.raw;
|
||||
|
||||
// TODO: instead of doing get_rows here, extend llm_attn_mask to have a `out` mask that
|
||||
// is construted during set_input to contain just the pruned output cells
|
||||
// this can be used to enable pruning with more than 1 sequence, both with unified and split KV cache
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/31356#discussion_r4455286628
|
||||
inp_attn_prune.self_attn_mask.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask.raw, inp_out_ids);
|
||||
inp_attn_prune.self_attn_mask_swa.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask_swa.raw, inp_out_ids);
|
||||
const int64_t n_slices = full->ne[3], n_slice_tokens = full->ne[1];
|
||||
const int64_t n_slice_outputs = llm_graph_input_gemma4_prune::max_outputs_per_slice(ubatch, n_slices, n_slice_tokens);
|
||||
|
||||
inp_attn_prune.self_attn_mask.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask.raw, GGML_TYPE_F16);
|
||||
inp_attn_prune.self_attn_mask_swa.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask_swa.raw, GGML_TYPE_F16);
|
||||
n_rows = n_slices * n_slice_outputs;
|
||||
|
||||
n_rows = n_outputs;
|
||||
auto prune_input = std::make_unique<llm_graph_input_gemma4_prune>(n_slices, n_slice_tokens, n_slice_outputs);
|
||||
prune_input->full_mask = full;
|
||||
prune_input->full_mask_swa = full_swa;
|
||||
prune_input->out_mask = ggml_new_tensor_4d(ctx0, full->type, full->ne[0], n_slice_outputs, 1, n_slices);
|
||||
prune_input->out_mask_swa = ggml_new_tensor_4d(ctx0, full_swa->type, full_swa->ne[0], n_slice_outputs, 1, n_slices);
|
||||
prune_input->inp_prune_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_rows);
|
||||
prune_input->inp_prune_keep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs);
|
||||
|
||||
ggml_set_input(prune_input->out_mask);
|
||||
ggml_set_input(prune_input->out_mask_swa);
|
||||
ggml_set_input(prune_input->inp_prune_ids);
|
||||
ggml_set_input(prune_input->inp_prune_keep);
|
||||
|
||||
inp_prune_ids = prune_input->inp_prune_ids;
|
||||
inp_prune_keep = prune_input->inp_prune_keep;
|
||||
|
||||
inpL = ggml_get_rows(ctx0, inpL, inp_prune_ids);
|
||||
inp_pos = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inp_pos, 1, n_tokens), inp_prune_ids), n_rows);
|
||||
|
||||
inp_attn_prune.self_attn_mask.raw = prune_input->out_mask;
|
||||
inp_attn_prune.self_attn_mask_swa.raw = prune_input->out_mask_swa;
|
||||
inp_attn = &inp_attn_prune;
|
||||
|
||||
res->add_input(std::move(prune_input));
|
||||
}
|
||||
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k(il);
|
||||
@@ -406,7 +449,9 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
|
||||
|
||||
ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens]
|
||||
|
||||
if ((prune_shared && !hparams.has_kv(il)) || (il == n_layer - 1 && crop_before_nextn(inp_out_ids))) {
|
||||
if (prune_shared && !hparams.has_kv(il)) {
|
||||
inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_prune_ids);
|
||||
} else if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
|
||||
inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids);
|
||||
}
|
||||
|
||||
@@ -432,6 +477,9 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
|
||||
inpL = cur;
|
||||
}
|
||||
cur = inpL;
|
||||
if (prune_shared) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_prune_keep);
|
||||
}
|
||||
|
||||
cur = build_norm(cur,
|
||||
model.output_norm, nullptr,
|
||||
@@ -558,3 +606,89 @@ ggml_tensor * llama_model_gemma4::graph::project_per_layer_inputs(ggml_tensor *
|
||||
// permute to shape: [n_embd_per_layer, n_tokens, n_layer]
|
||||
return ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3);
|
||||
}
|
||||
|
||||
void llm_graph_input_gemma4_prune::set_input(const llama_ubatch * ubatch) {
|
||||
std::vector<std::vector<int32_t>> selected(n_slices);
|
||||
|
||||
for (int32_t s = 0; s < n_slices; ++s) {
|
||||
for (int32_t t = 0; t < n_slice_tokens; ++t) {
|
||||
const int32_t i = s * n_slice_tokens + t;
|
||||
if (ubatch->output[i]) {
|
||||
selected[s].push_back(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<int32_t> take, keep;
|
||||
for (int32_t s = 0; s < n_slices; ++s) {
|
||||
const auto & ids = selected[s];
|
||||
|
||||
for (int32_t j = 0; j < int32_t(ids.size()); ++j) {
|
||||
keep.push_back(s * n_slice_outputs + j);
|
||||
}
|
||||
|
||||
take.insert(take.end(), ids.begin(), ids.end());
|
||||
const int32_t dummy = ids.empty() ? s * n_slice_tokens : ids.back();
|
||||
take.insert(take.end(), n_slice_outputs - ids.size(), dummy);
|
||||
}
|
||||
|
||||
GGML_ASSERT(take.size() == size_t(inp_prune_ids->ne[0]));
|
||||
if (!take.empty()) {
|
||||
ggml_backend_tensor_set(inp_prune_ids, take.data(), 0,
|
||||
take.size() * sizeof(int32_t));
|
||||
}
|
||||
|
||||
auto copy_mask = [&](const ggml_tensor * full, ggml_tensor * out) {
|
||||
if (take.empty() || !out->buffer) {
|
||||
return;
|
||||
}
|
||||
|
||||
GGML_ASSERT(full->buffer && ggml_backend_buffer_is_host(full->buffer));
|
||||
const size_t row_bytes = full->ne[0] * ggml_element_size(full);
|
||||
const auto * src = static_cast<const uint8_t *>(full->data);
|
||||
std::vector<uint8_t> packed(take.size() * row_bytes);
|
||||
|
||||
for (size_t j = 0; j < take.size(); ++j) {
|
||||
memcpy(packed.data() + j * row_bytes,
|
||||
src + take[j] * row_bytes,
|
||||
row_bytes);
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set(out, packed.data(), 0, packed.size());
|
||||
};
|
||||
|
||||
copy_mask(full_mask, out_mask);
|
||||
copy_mask(full_mask_swa, out_mask_swa);
|
||||
|
||||
if (!keep.empty()) {
|
||||
ggml_backend_tensor_set(
|
||||
inp_prune_keep, keep.data(), 0,
|
||||
keep.size() * sizeof(int32_t));
|
||||
}
|
||||
}
|
||||
|
||||
bool llm_graph_input_gemma4_prune::can_reuse(const llm_graph_params & params) {
|
||||
const auto & ubatch = params.ubatch;
|
||||
|
||||
if (ubatch.n_tokens != n_slices * n_slice_tokens || !ubatch.output) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return max_outputs_per_slice(ubatch, n_slices, n_slice_tokens) == n_slice_outputs;
|
||||
}
|
||||
|
||||
int64_t llm_graph_input_gemma4_prune::max_outputs_per_slice(const llama_ubatch & ubatch, int64_t n_slices, int64_t n_slice_tokens) {
|
||||
int64_t res = 0;
|
||||
|
||||
for (int32_t s = 0; s < n_slices; ++s) {
|
||||
int64_t m = 0;
|
||||
for (int32_t t = 0; t < n_slice_tokens; ++t) {
|
||||
const int32_t i = s * n_slice_tokens + t;
|
||||
if (ubatch.output[i]) {
|
||||
m++;
|
||||
}
|
||||
}
|
||||
res = std::max(res, m);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -202,7 +202,8 @@ For the full list of features, please refer to [server's changelog](https://gith
|
||||
| `--api-prefix PREFIX` | prefix path the server serves from, without the trailing slash (default: )<br/>(env: LLAMA_ARG_API_PREFIX) |
|
||||
| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
|
||||
| `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
|
||||
| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
|
||||
| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused unless --ui-mcp-proxy-allow-private-urls is set - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
|
||||
| `--ui-mcp-proxy-allow-private-urls` | allow the MCP CORS proxy to reach local or private addresses (e.g. MCP servers on localhost or LAN) - anyone who can reach the proxy can then reach your local network (default: disabled, only public addresses)<br/>note: only has effect with --ui-mcp-proxy<br/>(env: LLAMA_ARG_UI_MCP_PROXY_ALLOW_PRIVATE_URLS) |
|
||||
| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
|
||||
| `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/> 'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/> 'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/> 'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
|
||||
| `--allow-unauthenticated-tools` | allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)<br/>(env: LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS) |
|
||||
@@ -393,7 +394,7 @@ Every server is spawned once at startup to list its tools, then stopped, and res
|
||||
|
||||
The child process runs with the same privileges as the server, so only declare commands you trust. As with `--tools`, `--cors-origins` then defaults to `localhost`.
|
||||
|
||||
Note: `--ui-mcp-proxy` is unrelated, it only lets Llama Chat reach public MCP servers that do not send CORS headers. It refuses local and private addresses (loopback, LAN, link-local); connect to those from Llama Chat directly.
|
||||
Note: `--ui-mcp-proxy` is unrelated, it only lets Llama Chat reach public MCP servers that do not send CORS headers. By default it refuses local and private addresses (loopback, LAN, link-local); either connect to those from Llama Chat directly (the MCP server must send CORS headers), or add `--ui-mcp-proxy-allow-private-urls`.
|
||||
|
||||
Any server written against the [MCP specification](https://modelcontextprotocol.io) works as is, whether it uses an official SDK or not: the transport is one JSON-RPC message per line on stdio, so a script wrapping an existing program is a valid server too.
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ static std::string proxy_header_to_lower(std::string header) {
|
||||
return header;
|
||||
}
|
||||
|
||||
static server_http_res_ptr proxy_request(const server_http_req & req, std::string method) {
|
||||
static server_http_res_ptr proxy_request(const server_http_req & req, std::string method, bool allow_private_urls) {
|
||||
std::string target_url = req.get_param("url");
|
||||
common_http_url parsed_url = common_http_parse_url(target_url);
|
||||
|
||||
@@ -70,8 +70,7 @@ static server_http_res_ptr proxy_request(const server_http_req & req, std::strin
|
||||
req.should_stop,
|
||||
600, // timeout_read (default to 10 minutes)
|
||||
600, // timeout_write (default to 10 minutes)
|
||||
// test-only escape hatch
|
||||
server_resolve_public_addr(parsed_url.host, !common_get_env("LLAMA_SERVER_DEBUG_PROXY_ALLOW_PRIVATE").empty())
|
||||
server_resolve_public_addr(parsed_url.host, allow_private_urls)
|
||||
);
|
||||
|
||||
// route redirects back through the proxy, so each hop is checked
|
||||
@@ -86,10 +85,8 @@ static server_http_res_ptr proxy_request(const server_http_req & req, std::strin
|
||||
return proxy;
|
||||
}
|
||||
|
||||
static server_http_context::handler_t proxy_handler_post = [](const server_http_req & req) -> server_http_res_ptr {
|
||||
return proxy_request(req, "POST");
|
||||
};
|
||||
|
||||
static server_http_context::handler_t proxy_handler_get = [](const server_http_req & req) -> server_http_res_ptr {
|
||||
return proxy_request(req, "GET");
|
||||
};
|
||||
static server_http_context::handler_t proxy_make_handler(const std::string & method, bool allow_private_urls) {
|
||||
return [method, allow_private_urls](const server_http_req & req) -> server_http_res_ptr {
|
||||
return proxy_request(req, method, allow_private_urls);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -281,9 +281,11 @@ static bool server_setup_features(
|
||||
|
||||
// CORS proxy (EXPERIMENTAL, only used by Llama Chat for MCP)
|
||||
if (params.ui_mcp_proxy) {
|
||||
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
|
||||
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
|
||||
warn_names.push_back("MCP proxy (experimental)");
|
||||
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_make_handler("GET", params.ui_mcp_proxy_allow_private_urls)));
|
||||
ctx_http.post("/cors-proxy", ex_wrapper(proxy_make_handler("POST", params.ui_mcp_proxy_allow_private_urls)));
|
||||
warn_names.push_back(params.ui_mcp_proxy_allow_private_urls
|
||||
? "MCP proxy (experimental, local and private addresses allowed)"
|
||||
: "MCP proxy (experimental)");
|
||||
} else {
|
||||
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
|
||||
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
|
||||
|
||||
@@ -6,4 +6,5 @@ numpy~=2.5.4; python_version >= "3.14"
|
||||
openai~=2.14.0
|
||||
prometheus-client~=0.20.0
|
||||
requests~=2.32.3
|
||||
websockets~=15.0
|
||||
wget~=3.2
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import base64
|
||||
import pytest
|
||||
import requests
|
||||
import struct
|
||||
@@ -135,3 +136,78 @@ def test_speech_router():
|
||||
server = ServerPreset.router()
|
||||
server.start()
|
||||
assert_wav(post_audio("/v1/audio/speech", model=model))
|
||||
|
||||
|
||||
#
|
||||
# tts only model, with voice cloning
|
||||
#
|
||||
|
||||
def pocket_tts(path: str = "/v1/audio/speech", **extra) -> requests.Response:
|
||||
body = {"input": "Hello there.", "seed": 42, "n_predict": 8, **extra}
|
||||
return requests.post(server.make_url(path), json=body, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
|
||||
|
||||
def test_tts_only():
|
||||
global server
|
||||
server = ServerPreset.tinypockettts()
|
||||
server.start()
|
||||
res = server.make_request("GET", "/props")
|
||||
assert not res.body.get("voices") # this slice has no predefined voices
|
||||
res = server.make_request("GET", "/v1/models")
|
||||
assert res.body["data"][0]["architecture"]["output_modalities"] == ["audio"]
|
||||
res = pocket_tts()
|
||||
assert_wav(res)
|
||||
assert res.content == pocket_tts().content
|
||||
|
||||
|
||||
def test_tts_only_speaker_ref():
|
||||
global server
|
||||
server = ServerPreset.tinypockettts()
|
||||
server.start()
|
||||
ref_a = make_wav(seconds=2.0, rate=24000, freq=220)
|
||||
ref_b = make_wav(seconds=2.0, rate=24000, freq=440)
|
||||
res = pocket_tts("/tts", speaker_ref_b64=base64.b64encode(ref_a).decode())
|
||||
assert_wav(res)
|
||||
assert res.content != pocket_tts("/tts").content
|
||||
assert res.content != pocket_tts("/tts", speaker_ref_b64=base64.b64encode(ref_b).decode()).content
|
||||
# the same reference as an uploaded file
|
||||
fields = {"input": (None, "Hello there."), "seed": (None, "42"), "n_predict": (None, "8"), "speaker_ref": ("ref.wav", ref_a)}
|
||||
res_file = requests.post(server.make_url("/tts"), files=fields, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
assert res_file.content == res.content
|
||||
# /v1/audio/speech passes it through
|
||||
assert pocket_tts(speaker_ref_b64=base64.b64encode(ref_a).decode()).content == res.content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("speaker_ref_b64,message", [
|
||||
("!!!", "not valid base64"),
|
||||
(base64.b64encode(b"not an audio file").decode(), "failed to decode"),
|
||||
])
|
||||
def test_tts_only_invalid_speaker_ref(speaker_ref_b64: str, message: str):
|
||||
global server
|
||||
server = ServerPreset.tinypockettts()
|
||||
server.start()
|
||||
res = pocket_tts("/tts", speaker_ref_b64=speaker_ref_b64)
|
||||
assert res.status_code == 400
|
||||
assert message in res.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_tts_only_end_of_speech():
|
||||
global server
|
||||
server = ServerPreset.tinypockettts()
|
||||
server.start()
|
||||
# the end-of-speech head stops the generation before n_predict
|
||||
res = pocket_tts(n_predict=16)
|
||||
assert_wav(res)
|
||||
assert res.content == pocket_tts(n_predict=1000).content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/chat/completions", "/v1/audio/transcriptions"])
|
||||
def test_tts_only_no_text_output(path: str):
|
||||
global server
|
||||
server = ServerPreset.tinypockettts()
|
||||
server.start()
|
||||
if path == "/v1/audio/transcriptions":
|
||||
res = requests.post(server.make_url(path), files={"file": ("audio.wav", make_wav())}, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
else:
|
||||
res = requests.post(server.make_url(path), json={"messages": [{"role": "user", "content": "hello"}]}, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
assert res.status_code == 501
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import base64
|
||||
import json
|
||||
import pytest
|
||||
import requests
|
||||
from utils import *
|
||||
|
||||
server: ServerProcess
|
||||
|
||||
MP3_PATH = os.path.join(os.path.dirname(__file__), "../../../mtmd/test-2.mp3")
|
||||
|
||||
# full random sampling, so that the output depends on the audio
|
||||
SAMPLING_RANDOM = {"temperature": 1.0, "top_k": 0, "top_p": 1.0, "min_p": 0.0, "seed": 42}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def create_server():
|
||||
global server
|
||||
server = ServerPreset.tinylfm3omni()
|
||||
server.jinja = True
|
||||
server.reasoning = "off"
|
||||
|
||||
|
||||
def post_transcription(path: str = "/v1/audio/transcriptions", audio: bytes | None = None, filename: str = "audio.wav", **fields) -> requests.Response:
|
||||
data = {"max_tokens": "8", **fields}
|
||||
files = {"file": (filename, make_wav() if audio is None else audio)}
|
||||
return requests.post(server.make_url(path), data=data, files=files, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
|
||||
|
||||
def chat_with_audio(audio: bytes, fmt: str = "wav", extra: dict | None = None) -> str:
|
||||
res = server.make_request("POST", "/v1/chat/completions", data={
|
||||
"max_tokens": 8,
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "input_audio", "input_audio": {"data": base64.b64encode(audio).decode(), "format": fmt}},
|
||||
]}],
|
||||
**(extra or {}),
|
||||
})
|
||||
assert res.status_code == 200, res.body
|
||||
return res.body["choices"][0]["message"]["content"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
|
||||
def test_transcription(path: str):
|
||||
server.start()
|
||||
res = post_transcription(path)
|
||||
assert res.status_code == 200
|
||||
body = res.json()
|
||||
assert body["type"] == "transcript.text.done"
|
||||
assert len(body["text"]) > 0
|
||||
assert 0 < body["usage"]["output_tokens"] <= 8
|
||||
assert body["usage"]["input_tokens"] > 0
|
||||
|
||||
|
||||
def test_transcription_mp3():
|
||||
server.start()
|
||||
with open(MP3_PATH, "rb") as f:
|
||||
res = post_transcription(audio=f.read(), filename="audio.mp3")
|
||||
assert res.status_code == 200
|
||||
assert len(res.json()["text"]) > 0
|
||||
|
||||
|
||||
def test_transcription_stream():
|
||||
server.start()
|
||||
res = post_transcription(stream="true")
|
||||
assert res.status_code == 200
|
||||
lines = [line[len("data: "):] for line in res.text.splitlines() if line.startswith("data: ")]
|
||||
assert lines[-1] == "[DONE]"
|
||||
events = [json.loads(line) for line in lines[:-1]]
|
||||
assert len(events) > 1
|
||||
assert all(ev["type"] == "transcript.text.delta" for ev in events[:-1])
|
||||
assert events[-1]["type"] == "transcript.text.done"
|
||||
text = "".join(ev["delta"] for ev in events[:-1])
|
||||
assert text == events[-1]["text"]
|
||||
assert text == post_transcription().json()["text"]
|
||||
|
||||
|
||||
def test_transcription_language():
|
||||
server.start()
|
||||
# the LFM3 template puts the language in the system prompt
|
||||
n_default = post_transcription().json()["usage"]["input_tokens"]
|
||||
n_fr = post_transcription(language="fr").json()["usage"]["input_tokens"]
|
||||
assert n_fr > n_default
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fields,files,message", [
|
||||
({"response_format": "text"}, None, "response_format"),
|
||||
({}, {"file": ("audio.wav", b"not an audio file")}, None),
|
||||
({}, {"other": ("audio.wav", b"")}, None),
|
||||
])
|
||||
def test_transcription_invalid_request(fields: dict, files: dict | None, message: str | None):
|
||||
server.start()
|
||||
res = requests.post(server.make_url("/v1/audio/transcriptions"), data={"max_tokens": "8", **fields},
|
||||
files=files or {"file": ("audio.wav", make_wav())}, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
assert res.status_code == 400
|
||||
if message:
|
||||
assert message in res.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("preset", [
|
||||
"tinyllama2", # no audio input
|
||||
"tinyvoxtralrt", # live transcription only
|
||||
])
|
||||
def test_transcription_not_supported(preset: str):
|
||||
global server
|
||||
server = getattr(ServerPreset, preset)()
|
||||
server.start()
|
||||
assert post_transcription().status_code == 501 # ERROR_TYPE_NOT_SUPPORTED
|
||||
|
||||
|
||||
def test_transcription_router():
|
||||
global server
|
||||
model = server.model_hf_repo
|
||||
server = ServerPreset.router()
|
||||
server.start()
|
||||
res = post_transcription(model=model)
|
||||
assert res.status_code == 200
|
||||
assert res.json()["type"] == "transcript.text.done"
|
||||
|
||||
|
||||
def test_chat_input_audio():
|
||||
server.start()
|
||||
a = make_wav(freq=220)
|
||||
b = make_wav(freq=880)
|
||||
assert chat_with_audio(a, extra=SAMPLING_RANDOM) == chat_with_audio(a, extra=SAMPLING_RANDOM)
|
||||
assert chat_with_audio(a, extra=SAMPLING_RANDOM) != chat_with_audio(b, extra=SAMPLING_RANDOM)
|
||||
|
||||
|
||||
def test_chat_input_audio_mp3():
|
||||
server.start()
|
||||
with open(MP3_PATH, "rb") as f:
|
||||
assert len(chat_with_audio(f.read(), fmt="mp3")) > 0
|
||||
|
||||
|
||||
def test_chat_input_audio_no_audio_model():
|
||||
global server
|
||||
server = ServerPreset.tinyllama2()
|
||||
server.start()
|
||||
res = server.make_request("POST", "/v1/chat/completions", data={
|
||||
"max_tokens": 8,
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "input_audio", "input_audio": {"data": base64.b64encode(make_wav()).decode(), "format": "wav"}},
|
||||
]}],
|
||||
})
|
||||
assert res.status_code != 200
|
||||
assert "audio input is not supported" in res.body["error"]["message"]
|
||||
@@ -35,7 +35,7 @@ def test_mcp_proxy():
|
||||
def test_mcp_proxy_custom_port():
|
||||
global server
|
||||
server.ui_mcp_proxy = True
|
||||
server.debug_proxy_allow_private = True
|
||||
server.ui_mcp_proxy_allow_private_urls = True
|
||||
server.start()
|
||||
|
||||
# try getting the server's models API via the proxy
|
||||
@@ -61,7 +61,7 @@ def test_mcp_proxy_no_content():
|
||||
try:
|
||||
global server
|
||||
server.ui_mcp_proxy = True
|
||||
server.debug_proxy_allow_private = True
|
||||
server.ui_mcp_proxy_allow_private_urls = True
|
||||
server.start()
|
||||
|
||||
res = server.make_request("POST", f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/", data={})
|
||||
@@ -116,7 +116,7 @@ def test_mcp_proxy_redirect_goes_through_proxy():
|
||||
try:
|
||||
global server
|
||||
server.ui_mcp_proxy = True
|
||||
server.debug_proxy_allow_private = True
|
||||
server.ui_mcp_proxy_allow_private_urls = True
|
||||
server.start()
|
||||
|
||||
url = server.make_url(f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/start")
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import pytest
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
from websockets.sync.client import connect, ClientConnection
|
||||
from websockets.sync.connection import Connection
|
||||
from utils import *
|
||||
|
||||
server: ServerProcess
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def create_server():
|
||||
global server
|
||||
server = ServerPreset.tinylfm3omni()
|
||||
server.jinja = True
|
||||
server.reasoning = "off"
|
||||
|
||||
|
||||
def ws_connect(path: str = "/v1/realtime") -> ClientConnection:
|
||||
return connect(server.make_url(path).replace("http://", "ws://", 1), max_size=None, open_timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
|
||||
|
||||
def send(ws: Connection, type: str, **fields):
|
||||
ws.send(json.dumps({"type": type, **fields}))
|
||||
|
||||
|
||||
def send_audio(ws: Connection, pcm: bytes):
|
||||
send(ws, "input_audio_buffer.append", audio=base64.b64encode(pcm).decode())
|
||||
|
||||
|
||||
def send_text(ws: Connection, text: str):
|
||||
send(ws, "conversation.item.create", item={"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]})
|
||||
|
||||
|
||||
def recv(ws: Connection) -> dict:
|
||||
return json.loads(ws.recv(timeout=DEFAULT_REQUEST_TIMEOUT))
|
||||
|
||||
|
||||
def recv_until(ws: Connection, type: str) -> list[dict]:
|
||||
events = []
|
||||
while True:
|
||||
ev = recv(ws)
|
||||
events.append(ev)
|
||||
if ev["type"] == type:
|
||||
return events
|
||||
assert ev["type"] != "error", ev
|
||||
|
||||
|
||||
def recv_all(ws: Connection) -> list[dict]:
|
||||
""" read events until the server closes the socket """
|
||||
events = []
|
||||
try:
|
||||
while True:
|
||||
events.append(recv(ws))
|
||||
except ConnectionClosed:
|
||||
return events
|
||||
|
||||
|
||||
def get_transcripts(events: list[dict]) -> list[tuple[str, str]]:
|
||||
""" (item_id, transcript) of each completed item, checked against its deltas """
|
||||
deltas: dict[str, str] = {}
|
||||
out = []
|
||||
for ev in events:
|
||||
assert ev["type"] != "error", ev
|
||||
if ev["type"] == "conversation.item.input_audio_transcription.delta":
|
||||
deltas[ev["item_id"]] = deltas.get(ev["item_id"], "") + ev["delta"]
|
||||
elif ev["type"] == "conversation.item.input_audio_transcription.completed":
|
||||
assert ev["transcript"] == deltas.get(ev["item_id"], "")
|
||||
out.append((ev["item_id"], ev["transcript"]))
|
||||
return out
|
||||
|
||||
|
||||
def session_update(ws: Connection, **session) -> dict:
|
||||
send(ws, "session.update", session=session)
|
||||
return recv_until(ws, "session.updated")[-1]["session"]
|
||||
|
||||
|
||||
#
|
||||
# live transcription
|
||||
#
|
||||
|
||||
LIVE_ASR = [
|
||||
# a chat model with an "asr" chat template, live only in sessions opened with ?intent=transcription
|
||||
("tinylfm3omni", "/v1/realtime?intent=transcription"),
|
||||
# no chat, the text follows the audio by a few tokens
|
||||
("tinyvoxtralrt", "/v1/realtime"),
|
||||
]
|
||||
|
||||
|
||||
def start_live_asr(preset: str):
|
||||
global server
|
||||
if preset != "tinylfm3omni":
|
||||
server = getattr(ServerPreset, preset)()
|
||||
server.start()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("preset,path", LIVE_ASR)
|
||||
def test_live_asr(preset: str, path: str):
|
||||
start_live_asr(preset)
|
||||
with ws_connect(path) as ws:
|
||||
session = recv(ws)
|
||||
assert session["type"] == "session.created"
|
||||
assert session["session"]["object"] == "realtime.transcription_session"
|
||||
assert session["session"]["audio"]["input"]["format"] == {"type": "audio/pcm", "rate": 16000}
|
||||
send_audio(ws, make_pcm16())
|
||||
send(ws, "input_audio_buffer.commit")
|
||||
send_audio(ws, make_pcm16(freq=880))
|
||||
send(ws, "input_audio_buffer.commit", final=True)
|
||||
events = recv_all(ws)
|
||||
committed = [ev for ev in events if ev["type"] == "input_audio_buffer.committed"]
|
||||
assert len(committed) == 1
|
||||
assert committed[0]["previous_item_id"] is None
|
||||
transcripts = get_transcripts(events)
|
||||
assert len(transcripts) == 2
|
||||
assert transcripts[0][0] == committed[0]["item_id"]
|
||||
assert transcripts[1][0] != transcripts[0][0]
|
||||
assert all(len(text) > 0 for _, text in transcripts)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("preset,path", LIVE_ASR)
|
||||
def test_live_asr_chunked_audio(preset: str, path: str):
|
||||
start_live_asr(preset)
|
||||
pcm = make_pcm16()
|
||||
results = []
|
||||
for n_chunks in [1, 7]:
|
||||
size = len(pcm) // n_chunks // 2 * 2
|
||||
with ws_connect(path) as ws:
|
||||
recv(ws)
|
||||
for i in range(0, len(pcm), size):
|
||||
send_audio(ws, pcm[i:i + size])
|
||||
send(ws, "input_audio_buffer.commit", final=True)
|
||||
results.append([text for _, text in get_transcripts(recv_all(ws))])
|
||||
assert results[0] == results[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("preset,path", LIVE_ASR)
|
||||
def test_live_asr_input_rate(preset: str, path: str):
|
||||
start_live_asr(preset)
|
||||
with ws_connect(path) as ws:
|
||||
recv(ws)
|
||||
send(ws, "session.update", session={"audio": {"input": {"format": {"type": "audio/pcm", "rate": 4000}}}})
|
||||
assert "between 8000 and 192000" in recv_until(ws, "error")[-1]["error"]["message"]
|
||||
send(ws, "session.update", session={"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}})
|
||||
assert recv_until(ws, "session.updated")[-1]["session"]["audio"]["input"]["format"]["rate"] == 24000
|
||||
send_audio(ws, make_pcm16(rate=24000))
|
||||
send(ws, "input_audio_buffer.commit", final=True)
|
||||
transcripts = get_transcripts(recv_all(ws))
|
||||
assert len(transcripts) == 1
|
||||
assert len(transcripts[0][1]) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("preset,path", LIVE_ASR)
|
||||
def test_live_asr_invalid_events(preset: str, path: str):
|
||||
start_live_asr(preset)
|
||||
with ws_connect(path) as ws:
|
||||
recv(ws)
|
||||
ws.send("not json")
|
||||
assert recv_until(ws, "error")[-1]["error"]["message"] == "invalid JSON"
|
||||
send(ws, "response.create")
|
||||
assert "unknown event type" in recv_until(ws, "error")[-1]["error"]["message"]
|
||||
send(ws, "input_audio_buffer.clear")
|
||||
recv_until(ws, "input_audio_buffer.cleared")
|
||||
# the session still works
|
||||
send_audio(ws, make_pcm16())
|
||||
send(ws, "input_audio_buffer.commit", final=True)
|
||||
assert len(get_transcripts(recv_all(ws))) == 1
|
||||
|
||||
|
||||
def test_live_asr_depends_on_audio():
|
||||
# the random weights of this model give a different text for each input
|
||||
start_live_asr("tinyvoxtralrt")
|
||||
results = []
|
||||
for freq in [220, 880, 220]:
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
send_audio(ws, make_pcm16(freq=freq))
|
||||
send(ws, "input_audio_buffer.commit", final=True)
|
||||
results.append(get_transcripts(recv_all(ws))[0][1])
|
||||
assert results[0] != results[1]
|
||||
assert results[0] == results[2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("preset", ["tinypockettts", "tinyllama2"])
|
||||
def test_realtime_not_supported(preset: str):
|
||||
global server
|
||||
server = getattr(ServerPreset, preset)()
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
events = recv_all(ws)
|
||||
assert len(events) == 1
|
||||
assert events[0]["type"] == "error"
|
||||
assert "does not support realtime" in events[0]["error"]["message"]
|
||||
|
||||
|
||||
#
|
||||
# realtime conversation
|
||||
#
|
||||
|
||||
def test_conversation_text():
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
session = recv(ws)
|
||||
assert session["type"] == "session.created"
|
||||
assert session["session"]["object"] == "realtime.session"
|
||||
assert session["session"]["audio"]["output"]["format"]["rate"] > 0
|
||||
session_update(ws, output_modalities=["text"], max_output_tokens=8)
|
||||
send_text(ws, "hello")
|
||||
user_item = recv_until(ws, "conversation.item.done")[-1]["item"]
|
||||
assert user_item["role"] == "user"
|
||||
assert user_item["content"] == [{"type": "input_text", "text": "hello"}]
|
||||
send(ws, "response.create")
|
||||
events = recv_until(ws, "response.done")
|
||||
|
||||
types = [ev["type"] for ev in events]
|
||||
n_deltas = types.count("response.output_text.delta")
|
||||
assert n_deltas == 8
|
||||
assert types == [
|
||||
"response.created",
|
||||
"response.output_item.added",
|
||||
"response.content_part.added",
|
||||
*["response.output_text.delta"] * n_deltas,
|
||||
"response.output_text.done",
|
||||
"response.content_part.done",
|
||||
"response.output_item.done",
|
||||
"response.done",
|
||||
]
|
||||
response = events[-1]["response"]
|
||||
assert response["status"] == "completed"
|
||||
assert response["id"] == events[0]["response"]["id"]
|
||||
assert response["usage"]["output_tokens"] == 8
|
||||
text = "".join(ev["delta"] for ev in events if ev["type"] == "response.output_text.delta")
|
||||
assert response["output"][0]["role"] == "assistant"
|
||||
assert response["output"][0]["content"] == [{"type": "output_text", "text": text}]
|
||||
|
||||
|
||||
def test_conversation_audio():
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
session_update(ws, output_modalities=["audio"], max_output_tokens=8)
|
||||
send_audio(ws, make_pcm16())
|
||||
# response.create commits the pending audio
|
||||
send(ws, "response.create")
|
||||
events = recv_until(ws, "response.done")
|
||||
types = [ev["type"] for ev in events]
|
||||
assert types[:3] == ["input_audio_buffer.committed", "conversation.item.added", "conversation.item.done"]
|
||||
assert events[2]["item"]["content"] == [{"type": "input_audio"}]
|
||||
assert "response.output_text.delta" not in types
|
||||
audio = b"".join(base64.b64decode(ev["delta"]) for ev in events if ev["type"] == "response.output_audio.delta")
|
||||
assert len(audio) > 0 and len(audio) % 2 == 0
|
||||
response = events[-1]["response"]
|
||||
assert response["status"] == "completed"
|
||||
assert response["output"][0]["content"][0]["type"] == "output_audio"
|
||||
|
||||
|
||||
def test_conversation_multi_turn():
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
session_update(ws, output_modalities=["text"], max_output_tokens=4)
|
||||
send_text(ws, "hello")
|
||||
recv_until(ws, "conversation.item.done")
|
||||
send(ws, "response.create")
|
||||
first = recv_until(ws, "response.done")[-1]["response"]
|
||||
send_audio(ws, make_pcm16())
|
||||
send(ws, "input_audio_buffer.commit")
|
||||
committed = recv_until(ws, "input_audio_buffer.committed")[-1]
|
||||
assert committed["previous_item_id"] == first["output"][0]["id"]
|
||||
send(ws, "response.create")
|
||||
second = recv_until(ws, "response.done")[-1]["response"]
|
||||
assert second["status"] == "completed"
|
||||
assert second["id"] != first["id"]
|
||||
|
||||
|
||||
def test_conversation_cancel():
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
session_update(ws, output_modalities=["text"])
|
||||
send_text(ws, "hello")
|
||||
send(ws, "response.create")
|
||||
recv_until(ws, "response.output_text.delta")
|
||||
send(ws, "response.cancel")
|
||||
assert recv_until(ws, "response.done")[-1]["response"]["status"] == "cancelled"
|
||||
# the session still works
|
||||
send_text(ws, "hello again")
|
||||
send(ws, "response.create", response={"max_output_tokens": 4})
|
||||
response = recv_until(ws, "response.done")[-1]["response"]
|
||||
assert response["status"] == "completed"
|
||||
assert response["usage"]["output_tokens"] == 4
|
||||
|
||||
|
||||
def test_conversation_session_update():
|
||||
server.start()
|
||||
tools = [{"type": "function", "name": "get_weather", "description": "Get the weather",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}]
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
session = session_update(ws, instructions="Be brief.", tools=tools, tool_choice="none", max_output_tokens=16,
|
||||
audio={"output": {"voice": {"id": "male"}}})
|
||||
assert session["instructions"] == "Be brief."
|
||||
assert session["tools"] == tools
|
||||
assert session["tool_choice"] == "none"
|
||||
assert session["max_output_tokens"] == 16
|
||||
assert session["audio"]["output"]["voice"] == "male"
|
||||
session = session_update(ws, max_output_tokens="inf", output_modalities=["text", "audio"])
|
||||
assert session["max_output_tokens"] == "inf"
|
||||
assert session["output_modalities"] == ["text", "audio"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("events,message", [
|
||||
([{"type": "response.create"}], "commit audio or create a message first"),
|
||||
([{"type": "input_audio_buffer.commit"}], "the buffer is empty"),
|
||||
([{"type": "conversation.item.create", "item": {"type": "message", "role": "assistant", "content": []}}], "only user messages"),
|
||||
([{"type": "conversation.item.create", "item": {"type": "message", "role": "user", "content": [{"type": "input_image"}]}}], "unsupported content type"),
|
||||
([{"type": "conversation.item.create", "item": {"type": "function_call_output", "call_id": "x", "output": "{}"}}], "must follow the response"),
|
||||
([{"type": "session.update", "session": {"type": "foo"}}], "unknown session type"),
|
||||
([{"type": "session.update", "session": {"output_modalities": []}}], "non-empty array"),
|
||||
([{"type": "session.update", "session": {"output_modalities": ["video"]}}], "unknown modality"),
|
||||
([{"type": "session.update", "session": {"max_output_tokens": 0}}], "max_output_tokens"),
|
||||
([{"type": "session.update", "session": {"tool_choice": "maybe"}}], "tool_choice"),
|
||||
([{"type": "session.update", "session": {"audio": {"output": {"voice": "alloy"}}}}], "unknown voice"),
|
||||
([{"type": "conversation.item.create", "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}},
|
||||
{"type": "session.update", "session": {"instructions": "Be brief."}}], "cannot change after"),
|
||||
([{"type": "conversation.item.create", "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}},
|
||||
{"type": "session.update", "session": {"type": "transcription"}}], "cannot change after"),
|
||||
])
|
||||
def test_conversation_invalid_events(events: list[dict], message: str):
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
for ev in events:
|
||||
ws.send(json.dumps(ev))
|
||||
err = recv_until(ws, "error")[-1]
|
||||
assert message in err["error"]["message"]
|
||||
|
||||
|
||||
def test_transcription_session():
|
||||
server.start()
|
||||
with ws_connect() as ws:
|
||||
recv(ws)
|
||||
session = session_update(ws, type="transcription", max_output_tokens=8,
|
||||
audio={"input": {"transcription": {"language": "fr"}}})
|
||||
assert session["object"] == "realtime.transcription_session"
|
||||
assert session["audio"]["input"]["transcription"]["language"] == "fr"
|
||||
send_text(ws, "hello")
|
||||
assert "not supported in a transcription session" in recv_until(ws, "error")[-1]["error"]["message"]
|
||||
transcripts = []
|
||||
for freq in [440, 880]:
|
||||
send_audio(ws, make_pcm16(freq=freq))
|
||||
send(ws, "input_audio_buffer.commit")
|
||||
committed = recv_until(ws, "input_audio_buffer.committed")[-1]
|
||||
events = recv_until(ws, "conversation.item.input_audio_transcription.completed")
|
||||
transcripts += get_transcripts(events)
|
||||
assert transcripts[-1][0] == committed["item_id"]
|
||||
assert len(transcripts) == 2
|
||||
assert all(len(text) > 0 for _, text in transcripts)
|
||||
|
||||
|
||||
#
|
||||
# slot and server state
|
||||
#
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/realtime", "/v1/realtime?intent=transcription"])
|
||||
def test_realtime_disconnect_frees_slot(path: str):
|
||||
server.n_slots = 1
|
||||
server.start()
|
||||
with ws_connect(path) as ws:
|
||||
recv(ws)
|
||||
send_audio(ws, make_pcm16())
|
||||
if path == "/v1/realtime":
|
||||
send(ws, "response.create")
|
||||
recv_until(ws, "response.output_item.added")
|
||||
# the socket is closed in the middle of the session
|
||||
res = server.make_request("POST", "/v1/chat/completions", data={
|
||||
"max_tokens": 4,
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
})
|
||||
assert res.status_code == 200
|
||||
|
||||
|
||||
def test_realtime_router():
|
||||
global server
|
||||
model = server.model_hf_repo
|
||||
server = ServerPreset.router()
|
||||
server.start()
|
||||
with ws_connect(f"/v1/realtime?model={model}") as ws:
|
||||
assert recv(ws)["type"] == "session.created"
|
||||
session_update(ws, output_modalities=["text"], max_output_tokens=4)
|
||||
send_text(ws, "hello")
|
||||
send(ws, "response.create")
|
||||
assert recv_until(ws, "response.done")[-1]["response"]["status"] == "completed"
|
||||
|
||||
|
||||
def test_realtime_sleep():
|
||||
server.sleep_idle_seconds = 1
|
||||
server.start()
|
||||
start = time.time()
|
||||
while not server.make_request("GET", "/props").body["is_sleeping"]:
|
||||
assert time.time() - start < 10, "server did not go to sleep"
|
||||
time.sleep(0.1)
|
||||
# the session wakes the server up
|
||||
with ws_connect() as ws:
|
||||
assert recv(ws)["type"] == "session.created"
|
||||
session_update(ws, output_modalities=["text"], max_output_tokens=4)
|
||||
send_text(ws, "hello")
|
||||
send(ws, "response.create")
|
||||
assert recv_until(ws, "response.done")[-1]["response"]["status"] == "completed"
|
||||
@@ -314,7 +314,7 @@ def test_cors_proxy_only_forwards_explicit_proxy_headers():
|
||||
server = ServerPreset.tinyllama2()
|
||||
server.api_key = TEST_API_KEY
|
||||
server.ui_mcp_proxy = True
|
||||
server.debug_proxy_allow_private = True
|
||||
server.ui_mcp_proxy_allow_private_urls = True
|
||||
server.start()
|
||||
|
||||
res = server.make_request("GET", f"/cors-proxy?url=http://127.0.0.1:{target.server_port}/capture", headers={
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import io
|
||||
import math
|
||||
import struct
|
||||
import wave
|
||||
|
||||
TMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tmp")
|
||||
import re
|
||||
@@ -126,7 +130,7 @@ class ServerProcess:
|
||||
no_cache_idle_slots: bool = False
|
||||
log_path: str | None = None
|
||||
ui_mcp_proxy: bool = False
|
||||
debug_proxy_allow_private: bool = False # test-only: lets the CORS proxy reach local servers
|
||||
ui_mcp_proxy_allow_private_urls: bool = False
|
||||
media_allow_private_urls: bool = False
|
||||
allow_unauthenticated_tools: bool = False
|
||||
backend_sampling: bool = False
|
||||
@@ -155,8 +159,6 @@ class ServerProcess:
|
||||
**os.environ,
|
||||
"LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
|
||||
}
|
||||
if self.debug_proxy_allow_private:
|
||||
env["LLAMA_SERVER_DEBUG_PROXY_ALLOW_PRIVATE"] = "1"
|
||||
if self.llama_cache is not None:
|
||||
env["LLAMA_CACHE"] = self.llama_cache
|
||||
elif "LLAMA_CACHE" not in os.environ:
|
||||
@@ -313,6 +315,8 @@ class ServerProcess:
|
||||
server_args.append("--no-cache-idle-slots")
|
||||
if self.ui_mcp_proxy:
|
||||
server_args.append("--ui-mcp-proxy")
|
||||
if self.ui_mcp_proxy_allow_private_urls:
|
||||
server_args.append("--ui-mcp-proxy-allow-private-urls")
|
||||
if self.media_allow_private_urls:
|
||||
server_args.append("--media-allow-private-urls")
|
||||
if self.allow_unauthenticated_tools:
|
||||
@@ -726,6 +730,36 @@ class ServerPreset:
|
||||
server.seed = 42
|
||||
return server
|
||||
|
||||
@staticmethod
|
||||
def tinypockettts() -> ServerProcess:
|
||||
# tts only, with voice cloning
|
||||
server = ServerProcess()
|
||||
server.offline = True # will be downloaded by load_all()
|
||||
# mmproj is already provided by HF registry API
|
||||
server.model_hf_file = None
|
||||
server.model_hf_repo = "ggml-org-dev/tinyPocketTTS-for-testing-gguf:Q8_0"
|
||||
server.model_alias = "tinypockettts"
|
||||
server.n_ctx = 1024
|
||||
server.n_batch = 512
|
||||
server.n_slots = 1
|
||||
server.seed = 42
|
||||
return server
|
||||
|
||||
@staticmethod
|
||||
def tinyvoxtralrt() -> ServerProcess:
|
||||
# live transcription only, no chat
|
||||
server = ServerProcess()
|
||||
server.offline = True # will be downloaded by load_all()
|
||||
# mmproj is already provided by HF registry API
|
||||
server.model_hf_file = None
|
||||
server.model_hf_repo = "ggml-org-dev/tinyVoxtral-Realtime-for-testing-gguf:Q8_0"
|
||||
server.model_alias = "tinyvoxtralrt"
|
||||
server.n_ctx = 1024
|
||||
server.n_batch = 512
|
||||
server.n_slots = 1
|
||||
server.seed = 42
|
||||
return server
|
||||
|
||||
@staticmethod
|
||||
def router() -> ServerProcess:
|
||||
server = ServerProcess()
|
||||
@@ -832,3 +866,19 @@ def get_lan_ip(server: ServerProcess) -> str | None:
|
||||
|
||||
def is_slow_test_allowed():
|
||||
return os.environ.get("SLOW_TESTS") == "1" or os.environ.get("SLOW_TESTS") == "ON"
|
||||
|
||||
|
||||
def make_pcm16(seconds: float = 1.0, rate: int = 16000, freq: float = 440.0) -> bytes:
|
||||
""" sine tone as raw PCM16 mono samples """
|
||||
n = int(seconds * rate)
|
||||
return struct.pack(f"<{n}h", *(int(0.3 * 32767 * math.sin(2 * math.pi * freq * i / rate)) for i in range(n)))
|
||||
|
||||
|
||||
def make_wav(seconds: float = 1.0, rate: int = 16000, freq: float = 440.0) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(rate)
|
||||
w.writeframes(make_pcm16(seconds, rate, freq))
|
||||
return buf.getvalue()
|
||||
|
||||
+3
-2
@@ -86,8 +86,9 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
mtmd_context_params mtmd_params = mtmd_context_params_default();
|
||||
mtmd_params.use_gpu = params.mmproj_use_gpu;
|
||||
mtmd_params.device = params.mmproj_device;
|
||||
mtmd_params.use_gpu = params.mmproj_use_gpu;
|
||||
mtmd_params.device = params.mmproj_device;
|
||||
mtmd_params.n_threads = params.cpuparams.n_threads;
|
||||
mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params));
|
||||
if (!mctx) {
|
||||
LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str());
|
||||
|
||||
+6
-2
@@ -5,7 +5,8 @@ The UI of llama.cpp: the packages in `lib` and the apps built on them.
|
||||
```text
|
||||
ui/
|
||||
├── lib/
|
||||
│ └── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components
|
||||
│ ├── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components
|
||||
│ └── voice/ @ggml-org/llama-voice: microphone capture, transcription and dictation
|
||||
└── apps/
|
||||
└── chat/ Llama Chat, the chat app that llama-server serves
|
||||
```
|
||||
@@ -13,12 +14,15 @@ ui/
|
||||
- [`lib/ui`](lib/ui/README.md) draws the parts and handles interaction, with no business logic. It is built with
|
||||
`@sveltejs/package` and published to npm as `@ggml-org/llama-ui`. Apps in this repo skip the package: they compile
|
||||
its sources as part of their own build, so they hold no copy of a component, a token or a font.
|
||||
- [`lib/voice`](lib/voice/README.md) records the microphone and turns speech into text through llama-server, with no
|
||||
UI: Llama UI draws the voice parts. It is plain TypeScript built with `tsc`, plus a Svelte entry
|
||||
(`@ggml-org/llama-voice/svelte`). Apps in this repo compile its sources too. It is not published to npm yet.
|
||||
- [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop. CMake
|
||||
builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as `@ggml-org/llama-chat`
|
||||
(with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server.
|
||||
|
||||
Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building,
|
||||
checking and testing `apps/chat` needs no install in `lib/ui`.
|
||||
checking and testing `apps/chat` needs no install in `lib/ui` or `lib/voice`.
|
||||
|
||||
## Releases
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ const config: StorybookConfig = {
|
||||
'../tests/stories/**/*.stories.@(js|ts|svelte)',
|
||||
{
|
||||
directory: '../../../lib/ui/tests/stories',
|
||||
files: '{foundations,components,app,models,mockups}/**/*.stories.svelte'
|
||||
files: '{foundations,components,app,models,mockups,voice}/**/*.stories.svelte'
|
||||
}
|
||||
],
|
||||
addons: [
|
||||
|
||||
@@ -1154,6 +1154,12 @@ screens. Status lamps, avatars and the round icon buttons in the chat form are f
|
||||
(`--sidebar-reveal-x`), taking no taps while the page is aside. The page keeps room under its
|
||||
last row (`max-md:pb-safe-24`) so the key never covers it. One per page, only for its main
|
||||
action; from `md` up the action stays a button in the header.
|
||||
- **Voice clips:** a recorded clip is a round primary play key, its waveform and the time in
|
||||
`caption`: bars in the muted foreground at 40%, the played part in the primary colour, so
|
||||
the clip stays neutral in both modes. The live waveform while recording is as tall as the
|
||||
input's line (24px), the newest level on the right, and holds still with reduced motion. A
|
||||
voice message's transcript opens under the clip on hover, keyboard focus or a tap, and
|
||||
stays in the DOM for screen readers while closed.
|
||||
- **Copy feedback:** a copy action turns its icon into a `fg-success` check for 1.5s
|
||||
(`COPY_FEEDBACK_RESET_MS`), only when the copy succeeded, then back. The icon pops in with a
|
||||
150ms fade and scale, skipped on first render and with reduced motion.
|
||||
|
||||
@@ -89,6 +89,7 @@ The app imports the library as `@ggml-org/llama-ui/ui/<name>` and compiles its s
|
||||
| Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` |
|
||||
| App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area |
|
||||
| UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI |
|
||||
| Voice | `ui/lib/voice` | Microphone capture, transcription and dictation, from Llama Voice (`@ggml-org/llama-voice`) |
|
||||
| Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard |
|
||||
| Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings |
|
||||
| Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox |
|
||||
|
||||
@@ -115,7 +115,7 @@ Records every `scripts/docs/*.demo.ts` from the storage target and uploads `vide
|
||||
|
||||
## Writing a demo
|
||||
|
||||
Scripts get `test` and `expect` from `helpers.ts` without importing them. `demo.click`, `demo.hover` and `demo.type` glide a visible cursor to the target before acting, with timing varied like a person's (seeded per demo, so re-takes match); `demo.scroll` turns the mouse wheel over a target; `demo.zoom` pinches in on a control or a point to show a detail too small to read in the frame, holds, and pinches back out (with the fingertips drawn on a touch device); `demo.caption` shows a caption at the top of the frame. Use `test.use({ settings, startPath })` to seed other settings or start on another route.
|
||||
Scripts get `test` and `expect` from `helpers.ts` without importing them. `demo.click`, `demo.hover` and `demo.type` glide a visible cursor to the target before acting, with timing varied like a person's (seeded per demo, so re-takes match); `demo.scroll` turns the mouse wheel over a target; `demo.zoom` pinches in on a control or a point to show a detail too small to read in the frame, holds, and pinches back out (with the fingertips drawn on a touch device); `demo.caption` shows a caption at the top of the frame. For voice input, `demo.microphone()` gives the page a microphone that hears nothing but `demo.speak(text)`, which reads the text in a synthetic voice (macOS `say`) and shows it as a caption, since the video has no sound. Use `test.use({ settings, startPath })` to seed other settings or start on another route.
|
||||
|
||||
## Conversations
|
||||
|
||||
|
||||
+117
-13
@@ -3,7 +3,15 @@ import { generateTrace } from './fixtures/traces.ts';
|
||||
import { ScreencastRecorder } from './recorder.ts';
|
||||
import { type CDPSession, type Locator, type Page, test as base } from '@playwright/test';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import {
|
||||
appendFileSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { basename, join, resolve } from 'node:path';
|
||||
|
||||
const VIDEOS_DIR = resolve(import.meta.dirname, 'videos');
|
||||
@@ -112,6 +120,68 @@ function seedSettings(config: Record<string, unknown>): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Swaps the page's microphone for one that hears only what `window.demoSpeak` plays into it,
|
||||
* silence otherwise. Chromium's fake capture device crashes the page, so voice input is
|
||||
* recorded with this instead. Runs in the page.
|
||||
*/
|
||||
function installFakeMicrophone(): void {
|
||||
const context = new AudioContext();
|
||||
const output = context.createMediaStreamDestination();
|
||||
|
||||
navigator.mediaDevices.getUserMedia = async (constraints) => {
|
||||
if (!constraints?.audio || constraints.video) {
|
||||
throw new DOMException('demo: only the microphone is faked', 'NotSupportedError');
|
||||
}
|
||||
|
||||
await context.resume();
|
||||
|
||||
// each opening gets its own tracks, so a recording that stops them leaves the source on
|
||||
return new MediaStream(output.stream.getAudioTracks().map((track) => track.clone()));
|
||||
};
|
||||
|
||||
Object.assign(window, {
|
||||
demoSpeak: async (wavBase64: string) => {
|
||||
const bytes = Uint8Array.from(atob(wavBase64), (c) => c.charCodeAt(0));
|
||||
const source = context.createBufferSource();
|
||||
|
||||
source.buffer = await context.decodeAudioData(bytes.buffer);
|
||||
source.connect(output);
|
||||
await context.resume();
|
||||
await new Promise((done) => {
|
||||
source.onended = done;
|
||||
source.start();
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** `text` read by a synthetic voice, as a base64 WAV. Uses macOS `say`. */
|
||||
function synthesizeSpeech(text: string, voice: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'demo-speech-'));
|
||||
const file = join(dir, 'speech.wav');
|
||||
|
||||
try {
|
||||
execFileSync('say', [
|
||||
'-v',
|
||||
voice,
|
||||
'-o',
|
||||
file,
|
||||
'--file-format=WAVE',
|
||||
'--data-format=LEI16@24000',
|
||||
text
|
||||
]);
|
||||
|
||||
return readFileSync(file).toString('base64');
|
||||
} catch (error) {
|
||||
throw new Error(`demo: speak() needs macOS \`say\` with the voice ${voice}`, {
|
||||
cause: error
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Seeded randomness, so a demo moves the same way on every recording. */
|
||||
function createRandom(seed: string): () => number {
|
||||
let state = [...seed].reduce((h, c) => Math.imul(h ^ c.charCodeAt(0), 2654435761), 1);
|
||||
@@ -143,19 +213,9 @@ export class Demo {
|
||||
|
||||
/** Shows a caption at the top of the frame for `ms`, clear of the chat form. */
|
||||
async caption(text: string, ms = 2200): Promise<void> {
|
||||
await this.page.evaluate((t) => {
|
||||
const el = document.getElementById('demo-caption');
|
||||
|
||||
if (!el) return;
|
||||
|
||||
el.textContent = t;
|
||||
el.classList.add('visible');
|
||||
}, text);
|
||||
await this.showCaption(text);
|
||||
await this.page.waitForTimeout(ms);
|
||||
await this.page.evaluate(() =>
|
||||
document.getElementById('demo-caption')?.classList.remove('visible')
|
||||
);
|
||||
await this.page.waitForTimeout(250);
|
||||
await this.hideCaption();
|
||||
}
|
||||
|
||||
async click(target: Locator, ms = 400): Promise<void> {
|
||||
@@ -182,6 +242,15 @@ export class Demo {
|
||||
await this.pause(ms);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gives the page a microphone that hears only what `speak` says, and allows the page to
|
||||
* use it. Call it before the microphone is first used.
|
||||
*/
|
||||
async microphone(): Promise<void> {
|
||||
await this.page.context().grantPermissions(['microphone']);
|
||||
await this.page.evaluate(installFakeMicrophone);
|
||||
}
|
||||
|
||||
/** Glides the cursor to a point near the center of `target`. */
|
||||
async moveTo(target: Locator): Promise<void> {
|
||||
await target.scrollIntoViewIfNeeded();
|
||||
@@ -240,6 +309,23 @@ export class Demo {
|
||||
await this.pause(ms);
|
||||
}
|
||||
|
||||
/**
|
||||
* Says `text` into the microphone from `microphone` in a synthetic voice (macOS `say`), and
|
||||
* returns once it is said. The words show as a caption, since the video has no sound.
|
||||
*/
|
||||
async speak(text: string, voice = 'Samantha', ms = 400): Promise<void> {
|
||||
const wav = synthesizeSpeech(text, voice);
|
||||
|
||||
await this.showCaption(`“${text}”`);
|
||||
await this.page.evaluate(
|
||||
(base64) =>
|
||||
(window as unknown as { demoSpeak(wav: string): Promise<void> }).demoSpeak(base64),
|
||||
wav
|
||||
);
|
||||
await this.hideCaption();
|
||||
await this.pause(ms);
|
||||
}
|
||||
|
||||
/** Types quickly with uneven key timing, a little slower after spaces and punctuation. */
|
||||
async type(target: Locator, text: string, ms = 500): Promise<void> {
|
||||
await this.click(target, 200);
|
||||
@@ -305,6 +391,13 @@ export class Demo {
|
||||
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
||||
}
|
||||
|
||||
private async hideCaption(): Promise<void> {
|
||||
await this.page.evaluate(() =>
|
||||
document.getElementById('demo-caption')?.classList.remove('visible')
|
||||
);
|
||||
await this.page.waitForTimeout(250);
|
||||
}
|
||||
|
||||
/** A pinch that reaches the screen's edge can end without its last touchend reaching the page. */
|
||||
private async liftFingers(): Promise<void> {
|
||||
await this.page.evaluate(() =>
|
||||
@@ -340,6 +433,17 @@ export class Demo {
|
||||
|
||||
return this.cdp;
|
||||
}
|
||||
|
||||
private async showCaption(text: string): Promise<void> {
|
||||
await this.page.evaluate((t) => {
|
||||
const el = document.getElementById('demo-caption');
|
||||
|
||||
if (!el) return;
|
||||
|
||||
el.textContent = t;
|
||||
el.classList.add('visible');
|
||||
}, text);
|
||||
}
|
||||
}
|
||||
|
||||
interface DemoOptions {
|
||||
|
||||
@@ -26,6 +26,7 @@ const LLAMA_UI_EXPORTS = [
|
||||
['@ggml-org/llama-ui/enums', `${source}/enums/index.ts`],
|
||||
['@ggml-org/llama-ui/code', `${source}/utils/index.ts`],
|
||||
['@ggml-org/llama-ui/utils', `${source}/components/ui/utils.ts`],
|
||||
['@ggml-org/llama-ui/voice', `${source}/components/voice/index.ts`],
|
||||
['@ggml-org/llama-ui/styles/*', `${source}/styles/*`],
|
||||
['@ggml-org/llama-ui/theme.css', `${source}/styles/theme.css`],
|
||||
['@ggml-org/llama-ui/tokens.css', `${source}/styles/tokens.css`],
|
||||
|
||||
+13
-2
@@ -1,9 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { VoiceWaveformSize } from '@ggml-org/llama-ui/voice';
|
||||
import {
|
||||
ChatAttachmentsListItemMcpPrompt,
|
||||
ChatAttachmentsListItemMcpResource,
|
||||
ChatAttachmentsListItemThumbnailFile,
|
||||
ChatAttachmentsListItemThumbnailImage
|
||||
ChatAttachmentsListItemThumbnailImage,
|
||||
ChatAttachmentsListItemVoice
|
||||
} from '#lib/components/app/index.ts';
|
||||
import { AttachmentType } from '#lib/enums/index.ts';
|
||||
import type {
|
||||
@@ -12,7 +14,7 @@
|
||||
DatabaseMessageExtraMcpResource,
|
||||
MCPResourceAttachment
|
||||
} from '#lib/types/index.ts';
|
||||
import { isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts';
|
||||
import { isAudioFile, isMcpPrompt, isMcpResource, isPdfFile } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
@@ -93,6 +95,15 @@
|
||||
class="flex-shrink-0 {className} {scrollClasses}"
|
||||
onclick={() => onMcpResourcePreview?.(mcpResource)}
|
||||
/>
|
||||
{:else if isAudioFile(item.attachment, item.uploadedFile)}
|
||||
<!-- a clip plays where it is: the waveform takes the presses a chip would -->
|
||||
<ChatAttachmentsListItemVoice
|
||||
class="{layoutClasses} {className} {scrollClasses}"
|
||||
{item}
|
||||
onRemove={onFileRemove}
|
||||
{readonly}
|
||||
size={readonly ? VoiceWaveformSize.MD : VoiceWaveformSize.SM}
|
||||
/>
|
||||
{:else if item.isImage && item.preview}
|
||||
<ChatAttachmentsListItemThumbnailImage
|
||||
class="{layoutClasses} cursor-pointer {className} {scrollClasses}"
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
<!--
|
||||
An audio attachment drawn as a clip: a playable waveform, and for a voice message its transcript under it, open while
|
||||
it comes in and after, unless the Collapse transcripts setting folds it (then it opens on hover, keyboard focus or a
|
||||
tap). `sm` is the chip in the chat form, `md` the clip in a sent message. A clip
|
||||
stored without peaks (an older message, an uploaded file) gets them computed from its audio.
|
||||
-->
|
||||
<script lang="ts">
|
||||
import {
|
||||
spokenLength,
|
||||
VoiceTranscriptReveal,
|
||||
VoiceWaveform,
|
||||
VoiceWaveformSize
|
||||
} from '@ggml-org/llama-ui/voice';
|
||||
import { X } from '@lucide/svelte';
|
||||
import { ActionIcon } from '#lib/components/app/index.ts';
|
||||
import { SETTINGS_KEYS, VOICE_MESSAGE_LABEL } from '#lib/constants/index.ts';
|
||||
import { settingsStore, voiceTranscriptsStore } from '#lib/stores/index.ts';
|
||||
import type { ChatAttachmentDisplayItem } from '#lib/types/index.ts';
|
||||
import { audioUrl, lazyPeaks, voiceAttachmentSource } from '#lib/utils/voice-attachment.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
item: ChatAttachmentDisplayItem;
|
||||
onRemove?: (id: string) => void;
|
||||
readonly?: boolean;
|
||||
size?: VoiceWaveformSize;
|
||||
}
|
||||
|
||||
let {
|
||||
class: className = '',
|
||||
item,
|
||||
onRemove,
|
||||
readonly = false,
|
||||
size = VoiceWaveformSize.MD
|
||||
}: Props = $props();
|
||||
|
||||
let source = $derived(voiceAttachmentSource(item));
|
||||
let computedPeaks = $state<number[] | null>(null);
|
||||
let src = $state<string | undefined>();
|
||||
|
||||
let peaks = $derived(source.peaks ?? computedPeaks ?? []);
|
||||
let shown = $derived(voiceTranscriptsStore.shown(source.name, source.transcript));
|
||||
let transcript = $derived(shown.transcript);
|
||||
let status = $derived(shown.status);
|
||||
let errorReason = $derived(shown.errorReason);
|
||||
let label = $derived(source.voice ? VOICE_MESSAGE_LABEL : source.name);
|
||||
// the transcript lights up word by word as the clip plays, and stays lit
|
||||
let playbackMs = $state<number | null>(null);
|
||||
let spoken = $derived(
|
||||
transcript && playbackMs !== null && source.durationMs
|
||||
? spokenLength(transcript, peaks, source.durationMs, playbackMs)
|
||||
: null
|
||||
);
|
||||
let collapsed = $derived(settingsStore.config[SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS] === true);
|
||||
|
||||
// an object URL lives as long as the clip is on screen
|
||||
$effect(() => {
|
||||
const audio = source.audio;
|
||||
|
||||
if (!audio) {
|
||||
src = undefined;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const playable = audioUrl(audio, source.mimeType);
|
||||
|
||||
src = playable?.url;
|
||||
|
||||
return () => {
|
||||
if (playable?.revoke) URL.revokeObjectURL(playable.url);
|
||||
};
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
const { audio, name, peaks: stored } = source;
|
||||
|
||||
computedPeaks = null;
|
||||
|
||||
if (stored || !audio) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
lazyPeaks(name, audio).then((value) => {
|
||||
if (!cancelled) computedPeaks = value;
|
||||
});
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<div
|
||||
class={[
|
||||
'group relative min-w-0',
|
||||
size === VoiceWaveformSize.SM
|
||||
? 'w-60 max-w-[calc(100vw-4rem)] rounded-lg border border-border bg-muted py-2 pr-9 pl-2'
|
||||
: 'w-80 max-w-full rounded-2xl matte-sheen bg-(--user-bubble-background) px-3 py-2.5 backdrop-blur-md dark:bg-primary/15',
|
||||
className
|
||||
]}
|
||||
data-slot="chat-attachment-voice"
|
||||
data-voice={source.voice || undefined}
|
||||
>
|
||||
{#key collapsed}
|
||||
<VoiceTranscriptReveal defaultOpen={!collapsed} {errorReason} {spoken} {status} {transcript}>
|
||||
<VoiceWaveform bind:playbackMs durationMs={source.durationMs} {label} {peaks} {size} {src} />
|
||||
</VoiceTranscriptReveal>
|
||||
{/key}
|
||||
|
||||
{#if !source.voice}
|
||||
<p class="mt-1 truncate px-1 text-caption text-muted-foreground">{source.name}</p>
|
||||
{/if}
|
||||
|
||||
{#if !readonly && onRemove}
|
||||
<div class="absolute top-1/2 right-1 -translate-y-1/2">
|
||||
<ActionIcon
|
||||
ariaLabel={source.voice ? 'Remove voice message' : `Remove ${source.name}`}
|
||||
icon={X}
|
||||
onclick={() => onRemove?.(item.id)}
|
||||
stopPropagationOnClick
|
||||
tooltip="Remove"
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
+26
-2
@@ -1,12 +1,23 @@
|
||||
<script lang="ts">
|
||||
import { VoiceTranscriptStatus } from '@ggml-org/llama-ui/voice';
|
||||
import { Music } from '@lucide/svelte';
|
||||
import { VOICE_MESSAGE_LABEL } from '#lib/constants/index.ts';
|
||||
import { voiceTranscriptsStore } from '#lib/stores/index.ts';
|
||||
import type { ChatAttachmentDisplayItem } from '#lib/types/index.ts';
|
||||
import { voiceAttachmentSource } from '#lib/utils/voice-attachment.ts';
|
||||
|
||||
interface Props {
|
||||
currentItem: { name?: string } | null;
|
||||
currentItem: ChatAttachmentDisplayItem | null;
|
||||
audioSrc: string | null;
|
||||
}
|
||||
|
||||
let { audioSrc, currentItem }: Props = $props();
|
||||
|
||||
let source = $derived(currentItem ? voiceAttachmentSource(currentItem) : null);
|
||||
// a voice message's transcript, from the message or still on its way
|
||||
let shown = $derived(source ? voiceTranscriptsStore.shown(source.name, source.transcript) : null);
|
||||
let transcript = $derived(shown?.transcript);
|
||||
let transcribing = $derived(shown?.status === VoiceTranscriptStatus.TRANSCRIBING);
|
||||
</script>
|
||||
|
||||
<div class="flex flex-1 items-center justify-center p-8">
|
||||
@@ -21,6 +32,19 @@
|
||||
<p class="mb-4 text-white/70">Audio preview not available</p>
|
||||
{/if}
|
||||
|
||||
<p class="text-body-sm text-white/50">{currentItem?.name || 'Audio'}</p>
|
||||
<p class="text-body-sm text-white/50">
|
||||
{source?.voice ? VOICE_MESSAGE_LABEL : currentItem?.name || 'Audio'}
|
||||
</p>
|
||||
|
||||
{#if transcript}
|
||||
<p
|
||||
class="mt-6 max-h-[40vh] overflow-y-auto text-left text-body-sm whitespace-pre-wrap text-white/80"
|
||||
data-slot="voice-preview-transcript"
|
||||
>
|
||||
<span class="sr-only">Transcript: </span>{transcript}
|
||||
</p>
|
||||
{:else if transcribing}
|
||||
<p class="mt-6 text-caption text-white/50">Transcribing...</p>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
<script lang="ts">
|
||||
import ContextGaugePopup from './ChatFormContextGauge/ContextGaugePopup.svelte';
|
||||
import { microphoneUnavailableReason, warmUpVoiceCapture } from '@ggml-org/llama-voice';
|
||||
import { VoiceRecorder } from '@ggml-org/llama-voice/svelte';
|
||||
import { Maximize, Minimize } from '@lucide/svelte';
|
||||
import {
|
||||
ActionIcon,
|
||||
ChatAttachmentsList,
|
||||
ChatFormActions,
|
||||
ChatFormActionsRecording,
|
||||
ChatFormCurrentWorkingDirectory,
|
||||
ChatFormInput,
|
||||
ChatFormInputFileInputInvisible,
|
||||
@@ -18,7 +21,8 @@
|
||||
ICON_CLASS_DEFAULT,
|
||||
INITIAL_FILE_SIZE,
|
||||
PROMPT_CONTENT_SEPARATOR,
|
||||
SETTING_CONFIG_DEFAULT
|
||||
SETTING_CONFIG_DEFAULT,
|
||||
VOICE_EDITOR_SELECTOR
|
||||
} from '#lib/constants/index.ts';
|
||||
import {
|
||||
ContentPartType,
|
||||
@@ -30,6 +34,10 @@
|
||||
} from '#lib/enums/index.ts';
|
||||
import { useChatFormExpansion } from '#lib/hooks/use-chat-form-expansion.svelte.ts';
|
||||
import { useChatFormPickers } from '#lib/hooks/use-chat-form-pickers.svelte.ts';
|
||||
import {
|
||||
useChatFormVoice,
|
||||
type VoiceRecorderLike
|
||||
} from '#lib/hooks/use-chat-form-voice.svelte.ts';
|
||||
import {
|
||||
chatStore,
|
||||
conversationsStore,
|
||||
@@ -39,7 +47,8 @@
|
||||
serverStore,
|
||||
settingsStore,
|
||||
toolsStore,
|
||||
uiStore
|
||||
uiStore,
|
||||
voiceTranscriptsStore
|
||||
} from '#lib/stores/index.ts';
|
||||
import type {
|
||||
FileMentionEntry,
|
||||
@@ -48,14 +57,6 @@
|
||||
MCPResourceInfo,
|
||||
PromptMessage
|
||||
} from '#lib/types/index.ts';
|
||||
import {
|
||||
AudioRecorder,
|
||||
convertToWav,
|
||||
createAudioFile,
|
||||
describeMicrophoneError,
|
||||
isAudioRecordingSupported,
|
||||
microphoneUnavailableReason
|
||||
} from '#lib/utils/browser-only.ts';
|
||||
import {
|
||||
buildMentionInsertion,
|
||||
findCommandToken,
|
||||
@@ -65,7 +66,7 @@
|
||||
parseClipboardContent,
|
||||
uuid
|
||||
} from '#lib/utils/index.ts';
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { onMount, tick, untrack } from 'svelte';
|
||||
import type { ClassValue } from 'svelte/elements';
|
||||
import { toast } from 'svelte-sonner';
|
||||
|
||||
@@ -96,6 +97,9 @@
|
||||
onUploadedFileRemove?: (fileId: string) => void;
|
||||
onUploadedFilesChange?: (files: ChatUploadedFile[]) => void;
|
||||
onValueChange?: (value: string) => void;
|
||||
|
||||
/** Records voice input; tests and stories pass a fake, the app a VoiceRecorder. */
|
||||
voiceRecorder?: VoiceRecorderLike;
|
||||
}
|
||||
|
||||
let {
|
||||
@@ -117,21 +121,20 @@
|
||||
showModelSelector = true,
|
||||
singleRow = false,
|
||||
uploadedFiles = $bindable([]),
|
||||
value = $bindable('')
|
||||
value = $bindable(''),
|
||||
voiceRecorder
|
||||
}: Props = $props();
|
||||
|
||||
// Component References
|
||||
|
||||
let audioRecorder: AudioRecorder | undefined;
|
||||
let chatFormActionsRef: ChatFormActions | undefined = $state(undefined);
|
||||
let fileInputRef: ChatFormInputFileInputInvisible | undefined = $state(undefined);
|
||||
let pickersRef: { handleKeydown: (event: KeyboardEvent) => boolean } | undefined =
|
||||
$state(undefined);
|
||||
let inputRef: ChatFormInput | undefined = $state(undefined);
|
||||
|
||||
// Audio Recording State
|
||||
let isRecording = $state(false);
|
||||
let recordingSupported = $state(false);
|
||||
// Voice input
|
||||
const recorder: VoiceRecorderLike = untrack(() => voiceRecorder) ?? new VoiceRecorder();
|
||||
/** Why this page can't record at all (no HTTPS, no recorder), which no tap fixes. */
|
||||
let micUnavailableReason = $state<string | null>(null);
|
||||
/** Why the last start failed (denied, no microphone, busy); the next tap tries again. */
|
||||
@@ -189,6 +192,25 @@
|
||||
}
|
||||
});
|
||||
|
||||
const voice = useChatFormVoice({
|
||||
addVoiceFile: (file) => {
|
||||
uploadedFiles = [...uploadedFiles, file];
|
||||
onUploadedFilesChange?.(uploadedFiles);
|
||||
},
|
||||
getCaretOffset: () => inputRef?.getCaretOffset() ?? value.length,
|
||||
getValue: () => value,
|
||||
onMicError: (message) => (micError = message),
|
||||
recorder,
|
||||
// on a phone placing the caret would open the keyboard
|
||||
setCaretOffset: (offset) => {
|
||||
if (!deviceStore.isMobile) inputRef?.setCaretOffset(offset);
|
||||
},
|
||||
setValue: (next) => {
|
||||
value = next;
|
||||
onValueChange?.(next);
|
||||
}
|
||||
});
|
||||
|
||||
async function handleWorkingDirectoryChange(newDir: string | null) {
|
||||
// Committing a directory consumes the `/cwd` token; the chip's
|
||||
// clear-X path has no token to consume.
|
||||
@@ -234,6 +256,14 @@
|
||||
(attachments && attachments.length > 0) || (uploadedFiles && uploadedFiles.length > 0)
|
||||
);
|
||||
let canSubmit = $derived(value.trim().length > 0 || hasAttachments);
|
||||
// while a voice message records, what is being said shows in the empty message box; once
|
||||
// it stops, the text goes with the clip, as its transcript
|
||||
let voicePlaceholder = $derived(
|
||||
voice.isActive && voice.liveCaption && !value.trim() ? voice.liveCaption : null
|
||||
);
|
||||
|
||||
// while a reply streams, sending what is recorded waits for its first words in the form
|
||||
let hasVoiceText = $derived(!!value.trim() || !!voice.liveCaption);
|
||||
|
||||
const EXPANDED_SHEET_FILL = 'max-md:bg-(--chat-form-sheet-fill)';
|
||||
|
||||
@@ -263,6 +293,7 @@
|
||||
singleRow &&
|
||||
!deviceStore.isMobile &&
|
||||
!isExpanded &&
|
||||
!voice.isActive &&
|
||||
!hasAttachments &&
|
||||
!mcpStore.resources.hasAttachments &&
|
||||
!value.includes('\n')
|
||||
@@ -327,9 +358,7 @@
|
||||
);
|
||||
|
||||
onMount(() => {
|
||||
recordingSupported = isAudioRecordingSupported();
|
||||
micUnavailableReason = microphoneUnavailableReason();
|
||||
audioRecorder = new AudioRecorder();
|
||||
});
|
||||
|
||||
/** Expands for the message as it stands, e.g. one put back after a send failed. */
|
||||
@@ -375,6 +404,10 @@
|
||||
onAttachmentRemove?.(index);
|
||||
}
|
||||
} else {
|
||||
const removed = uploadedFiles.find((file) => file.id === fileId);
|
||||
|
||||
if (removed?.voice) voiceTranscriptsStore.forget(removed.name);
|
||||
|
||||
onUploadedFileRemove?.(fileId);
|
||||
}
|
||||
}
|
||||
@@ -399,9 +432,7 @@
|
||||
|
||||
event.preventDefault();
|
||||
|
||||
if (!canSubmit || disabled || hasLoadingAttachments) return;
|
||||
|
||||
onSubmit?.();
|
||||
void submit();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -586,7 +617,7 @@
|
||||
inputRef?.replaceText(built.newValue, built.caretOffset);
|
||||
}
|
||||
|
||||
async function handleMicClick() {
|
||||
function handleMicClick() {
|
||||
// a phone shows no tooltip on a tap, so the reason comes as a toast
|
||||
if (micUnavailableReason) {
|
||||
toast.error(micUnavailableReason);
|
||||
@@ -594,36 +625,61 @@
|
||||
return;
|
||||
}
|
||||
|
||||
if (!audioRecorder || !recordingSupported) {
|
||||
console.warn('Audio recording not supported');
|
||||
void voice.start();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
// the audio set-up takes a moment, so it is made while the message is being written;
|
||||
// it asks for no microphone, so the browser shows no recording indicator
|
||||
function handleFocusIn(event: FocusEvent) {
|
||||
if (voice.showButton && !voice.isActive) warmUpVoiceCapture();
|
||||
|
||||
if (isRecording) {
|
||||
isRecording = false;
|
||||
try {
|
||||
const audioBlob = await audioRecorder.stopRecording();
|
||||
const wavBlob = await convertToWav(audioBlob);
|
||||
const audioFile = createAudioFile(wavBlob);
|
||||
|
||||
onFilesAdd?.([audioFile]);
|
||||
} catch (error) {
|
||||
console.error('Failed to stop recording:', error);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await audioRecorder.startRecording();
|
||||
isRecording = true;
|
||||
micError = null;
|
||||
} catch (error) {
|
||||
micError = describeMicrophoneError(error);
|
||||
toast.error(micError);
|
||||
}
|
||||
// the voice message's live text shown as the placeholder turns into text to edit
|
||||
if (voicePlaceholder && (event.target as Element | null)?.closest(VOICE_EDITOR_SELECTOR)) {
|
||||
voice.adoptLiveText();
|
||||
}
|
||||
}
|
||||
|
||||
// a send asked for while the dictated text is still being refined waits for it
|
||||
let isSendWaiting = $state(false);
|
||||
|
||||
/** Sends the message; dictated text that is still being refined is sent refined. */
|
||||
async function submit() {
|
||||
if (!canSubmit || disabled || hasLoadingAttachments || isSendWaiting) return;
|
||||
|
||||
if (voice.isRefining) {
|
||||
isSendWaiting = true;
|
||||
|
||||
try {
|
||||
await voice.settled();
|
||||
await tick();
|
||||
} finally {
|
||||
isSendWaiting = false;
|
||||
}
|
||||
|
||||
if (!canSubmit || disabled || hasLoadingAttachments) return;
|
||||
}
|
||||
|
||||
onSubmit?.();
|
||||
}
|
||||
|
||||
/** Stops the recording and sends the message once its text or clip is in the form. */
|
||||
async function handleStopAndSend() {
|
||||
await voice.stop();
|
||||
await tick();
|
||||
await submit();
|
||||
}
|
||||
|
||||
// Escape drops a recording, wherever the focus is
|
||||
function handleWindowKeydown(event: KeyboardEvent) {
|
||||
if (event.key !== KeyboardKey.ESCAPE || !voice.isActive || event.defaultPrevented) return;
|
||||
|
||||
event.preventDefault();
|
||||
voice.cancel();
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:window onkeydown={handleWindowKeydown} />
|
||||
|
||||
<ChatFormInputFileInputInvisible bind:this={fileInputRef} onFileSelect={handleFileSelect} />
|
||||
|
||||
<form
|
||||
@@ -636,12 +692,10 @@
|
||||
'max-h-[calc(100dvh-var(--chat-tabs-offset,0px)-var(--chat-form-padding-top)-2rem)] max-md:max-h-dvh',
|
||||
className
|
||||
]}
|
||||
onfocusin={handleFocusIn}
|
||||
onsubmit={(event) => {
|
||||
event.preventDefault();
|
||||
|
||||
if (!canSubmit || disabled || hasLoadingAttachments) return;
|
||||
|
||||
onSubmit?.();
|
||||
void submit();
|
||||
}}
|
||||
>
|
||||
<!-- the pickers' own triggers fill this box, so all of them share the anchor -->
|
||||
@@ -753,13 +807,14 @@
|
||||
{disabled}
|
||||
expanded={isExpanded}
|
||||
onInput={() => {
|
||||
voice.handleUserInput();
|
||||
expansion.handleInput();
|
||||
pickers.handleInput();
|
||||
onValueChange?.(value);
|
||||
}}
|
||||
onKeydown={handleKeydown}
|
||||
onPaste={handlePaste}
|
||||
{placeholder}
|
||||
placeholder={voicePlaceholder ?? placeholder}
|
||||
/>
|
||||
|
||||
{#if mcpStore.resources.hasAttachments}
|
||||
@@ -774,29 +829,55 @@
|
||||
|
||||
<!-- its own box, so the phone sheet can hold it by the bottom edge while it moves -->
|
||||
<div bind:this={actionsEl} class={isSingleRow ? 'contents' : 'relative mt-auto'}>
|
||||
<ChatFormActions
|
||||
bind:this={chatFormActionsRef}
|
||||
canSend={canSubmit}
|
||||
class={isSingleRow ? undefined : 'px-3'}
|
||||
{disabled}
|
||||
inline={isSingleRow}
|
||||
{isLoading}
|
||||
isReasoning={chatStore.isReasoning}
|
||||
{isRecording}
|
||||
micWarning={micUnavailableReason ?? micError}
|
||||
onFileUpload={handleFileUpload}
|
||||
onMicClick={handleMicClick}
|
||||
{onStop}
|
||||
onSystemPromptClick={() =>
|
||||
onSystemPromptClick?.({ files: uploadedFiles, message: value })}
|
||||
onWorkingDirectoryClick={isWorkingDirectoryInAddMenu
|
||||
? pickers.handleWorkingDirectoryOpen
|
||||
: undefined}
|
||||
{showAddButton}
|
||||
{showModelSelector}
|
||||
{uploadedFiles}
|
||||
workingDirectory={cwd}
|
||||
/>
|
||||
{#if voice.isActive}
|
||||
<ChatFormActionsRecording
|
||||
class="px-3"
|
||||
elapsedMs={voice.elapsedMs}
|
||||
levels={voice.levels}
|
||||
liveCaption={voicePlaceholder ? '' : voice.liveCaption}
|
||||
mode={voice.mode}
|
||||
onCancel={voice.cancel}
|
||||
onSend={isLoading && !hasVoiceText ? undefined : () => void handleStopAndSend()}
|
||||
onStop={() => void voice.stop()}
|
||||
quiet={voice.quiet}
|
||||
state={voice.state}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- kept while recording, so its selector and menus keep their state -->
|
||||
<div class={voice.isActive ? 'hidden' : 'contents'}>
|
||||
<ChatFormActions
|
||||
bind:this={chatFormActionsRef}
|
||||
canSend={canSubmit}
|
||||
class={isSingleRow ? undefined : 'px-3'}
|
||||
{disabled}
|
||||
inline={isSingleRow}
|
||||
{isLoading}
|
||||
isReasoning={chatStore.isReasoning}
|
||||
{isSendWaiting}
|
||||
isVoiceRefining={voice.isRefining}
|
||||
micWarning={micUnavailableReason ?? micError}
|
||||
onFileUpload={handleFileUpload}
|
||||
onMicClick={handleMicClick}
|
||||
{onStop}
|
||||
onSystemPromptClick={() =>
|
||||
onSystemPromptClick?.({ files: uploadedFiles, message: value })}
|
||||
onWorkingDirectoryClick={isWorkingDirectoryInAddMenu
|
||||
? pickers.handleWorkingDirectoryOpen
|
||||
: undefined}
|
||||
{showAddButton}
|
||||
{showModelSelector}
|
||||
voice={{
|
||||
action: voice.label,
|
||||
hint: voice.hint,
|
||||
prepare: () => void voice.prepare(),
|
||||
show: voice.showButton,
|
||||
unavailableReason: voice.unavailableReason,
|
||||
unprepare: voice.unprepare
|
||||
}}
|
||||
workingDirectory={cwd}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+88
-14
@@ -2,34 +2,95 @@
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { Mic, MicOff, Square } from '@lucide/svelte';
|
||||
import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts';
|
||||
import {
|
||||
ICON_CLASS_DEFAULT,
|
||||
VOICE_MODE_MESSAGE,
|
||||
VOICE_PRESS_CLICK_MS
|
||||
} from '#lib/constants/index.ts';
|
||||
import { MouseButton, PointerType } from '#lib/enums/index.ts';
|
||||
|
||||
interface Props {
|
||||
/** What a press does: send a voice message or dictate. */
|
||||
action?: string;
|
||||
class?: string;
|
||||
disabled?: boolean;
|
||||
hasAudioModality?: boolean;
|
||||
/** Said after the action in the tooltip, e.g. that another model may be loaded. */
|
||||
hint?: string | null;
|
||||
isLoading?: boolean;
|
||||
isRecording?: boolean;
|
||||
/** Why the microphone can't record (no HTTPS, access denied...), shown on the button. */
|
||||
micWarning?: string | null;
|
||||
/** Why no model can take the recording; disables the button. */
|
||||
unavailableReason?: string | null;
|
||||
/** A quieter (tertiary) button, beside the main one: send or stop. */
|
||||
secondary?: boolean;
|
||||
/** With `secondary`: filled like the stop button it stands beside, not bare. */
|
||||
filled?: boolean;
|
||||
onMicClick?: () => void;
|
||||
/** A mouse or the focus came to the button: a press may follow, the microphone can open. */
|
||||
onPrepare?: () => void;
|
||||
/** It left again without a press. */
|
||||
onPrepareEnd?: () => void;
|
||||
}
|
||||
|
||||
let {
|
||||
action = VOICE_MODE_MESSAGE.voiceMessage,
|
||||
class: className = '',
|
||||
disabled = false,
|
||||
hasAudioModality = false,
|
||||
filled = false,
|
||||
hint = null,
|
||||
isLoading = false,
|
||||
isRecording = false,
|
||||
micWarning = null,
|
||||
onMicClick
|
||||
onMicClick,
|
||||
onPrepare,
|
||||
onPrepareEnd,
|
||||
secondary = false,
|
||||
unavailableReason = null
|
||||
}: Props = $props();
|
||||
|
||||
// a model without audio input disables the button; a microphone that can't record keeps it
|
||||
// no model to take the recording disables the button; a microphone that can't record keeps it
|
||||
// tappable, so a tap can say why
|
||||
const showWarning = $derived(hasAudioModality && !isRecording && !!micWarning);
|
||||
const showWarning = $derived(!unavailableReason && !isRecording && !!micWarning);
|
||||
// a mouse starts recording on press, not on release, so the first words are not lost;
|
||||
// a touch or a key goes through the click, which is when the browser allows it
|
||||
let pressedAt = -Infinity;
|
||||
|
||||
function handlePointerDown(event: PointerEvent) {
|
||||
if (isRecording || disabled || isLoading || unavailableReason) return;
|
||||
|
||||
if (event.pointerType !== PointerType.MOUSE || event.button !== MouseButton.PRIMARY) return;
|
||||
|
||||
pressedAt = performance.now();
|
||||
onMicClick?.();
|
||||
}
|
||||
|
||||
// a touch presses at once, so only a mouse, a pen or the keyboard come ahead of the press
|
||||
function handlePointerEnter(event: PointerEvent) {
|
||||
if (event.pointerType !== PointerType.TOUCH && !isRecording) onPrepare?.();
|
||||
}
|
||||
|
||||
function handlePointerLeave(event: PointerEvent) {
|
||||
if (event.pointerType !== PointerType.TOUCH) onPrepareEnd?.();
|
||||
}
|
||||
|
||||
function handleClick() {
|
||||
// the click that ends the press that already started it; the recording bar may have
|
||||
// taken the button's place meanwhile, so this one can also never come
|
||||
if (performance.now() - pressedAt < VOICE_PRESS_CLICK_MS) {
|
||||
pressedAt = -Infinity;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
onMicClick?.();
|
||||
}
|
||||
|
||||
const tooltip = $derived(
|
||||
!hasAudioModality ? 'Current model does not support audio' : showWarning ? micWarning : null
|
||||
isRecording
|
||||
? null
|
||||
: (unavailableReason ??
|
||||
(showWarning ? micWarning : [action, hint].filter(Boolean).join('. ')))
|
||||
);
|
||||
</script>
|
||||
|
||||
@@ -37,28 +98,41 @@
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Button
|
||||
class="size-8 max-md:size-9 rounded-full p-0 {isRecording
|
||||
? 'matte-orb orb-danger animate-pulse text-white! hover:bg-transparent'
|
||||
class="group size-8 max-md:size-9 rounded-full p-0 {isRecording
|
||||
? 'matte-orb orb-danger text-white! hover:bg-transparent'
|
||||
: showWarning
|
||||
? 'bg-surface-warning text-fg-warning hover:bg-surface-warning'
|
||||
: ''}"
|
||||
disabled={disabled || isLoading || !hasAudioModality}
|
||||
onclick={onMicClick}
|
||||
disabled={disabled || isLoading || !!unavailableReason}
|
||||
onblur={() => onPrepareEnd?.()}
|
||||
onclick={handleClick}
|
||||
onfocus={() => !isRecording && onPrepare?.()}
|
||||
onpointerdown={handlePointerDown}
|
||||
onpointerenter={handlePointerEnter}
|
||||
onpointerleave={handlePointerLeave}
|
||||
type="button"
|
||||
variant={isRecording || showWarning ? 'ghost' : 'default'}
|
||||
variant={isRecording || showWarning
|
||||
? 'ghost'
|
||||
: secondary
|
||||
? filled
|
||||
? 'secondary'
|
||||
: 'tertiary'
|
||||
: 'default'}
|
||||
>
|
||||
<span class="sr-only"
|
||||
>{isRecording
|
||||
? 'Stop recording'
|
||||
: showWarning
|
||||
? `Recording unavailable: ${micWarning}`
|
||||
: 'Start recording'}</span
|
||||
: action}</span
|
||||
>
|
||||
|
||||
{#if isRecording}
|
||||
<Square class="{ICON_CLASS_DEFAULT} animate-pulse fill-white" />
|
||||
<Square class="{ICON_CLASS_DEFAULT} fill-white" />
|
||||
{:else if showWarning}
|
||||
<MicOff class={ICON_CLASS_DEFAULT} />
|
||||
{:else if secondary}
|
||||
<Mic class="{ICON_CLASS_DEFAULT} stroke-muted-foreground group-hover:stroke-foreground" />
|
||||
{:else}
|
||||
<Mic class={ICON_CLASS_DEFAULT} />
|
||||
{/if}
|
||||
|
||||
+15
-3
@@ -1,16 +1,24 @@
|
||||
<script lang="ts">
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { ArrowUp } from '@lucide/svelte';
|
||||
import { ArrowUp, LoaderCircle } from '@lucide/svelte';
|
||||
|
||||
interface Props {
|
||||
canSend?: boolean;
|
||||
disabled?: boolean;
|
||||
/** The send is on its way: it waits for something, e.g. dictated text being refined. */
|
||||
loading?: boolean;
|
||||
showErrorState?: boolean;
|
||||
tooltipLabel?: string;
|
||||
}
|
||||
|
||||
let { canSend = false, disabled = false, showErrorState = false, tooltipLabel }: Props = $props();
|
||||
let {
|
||||
canSend = false,
|
||||
disabled = false,
|
||||
loading = false,
|
||||
showErrorState = false,
|
||||
tooltipLabel
|
||||
}: Props = $props();
|
||||
|
||||
let isDisabled = $derived(!canSend || disabled);
|
||||
</script>
|
||||
@@ -28,7 +36,11 @@
|
||||
>
|
||||
<span class="sr-only">Send</span>
|
||||
|
||||
<ArrowUp class="size-icon-md" />
|
||||
{#if loading}
|
||||
<LoaderCircle class="size-icon-md animate-spin" />
|
||||
{:else}
|
||||
<ArrowUp class="size-icon-md" />
|
||||
{/if}
|
||||
</Button>
|
||||
{/snippet}
|
||||
|
||||
|
||||
+74
-24
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { Button } from '@ggml-org/llama-ui/ui/button';
|
||||
import { SkipForward, Square } from '@lucide/svelte';
|
||||
import { LoaderCircle, SkipForward, Square } from '@lucide/svelte';
|
||||
import {
|
||||
ChatFormActionRecord,
|
||||
ChatFormActionsAdd,
|
||||
@@ -8,14 +8,17 @@
|
||||
ChatFormContextGauge,
|
||||
ModelsSelector
|
||||
} from '#lib/components/app/index.ts';
|
||||
import { ICON_CLASS_DEFAULT, SET_WORKING_DIRECTORY_LABEL } from '#lib/constants/index.ts';
|
||||
import {
|
||||
ICON_CLASS_DEFAULT,
|
||||
SET_WORKING_DIRECTORY_LABEL,
|
||||
VOICE_MODE_MESSAGE
|
||||
} from '#lib/constants/index.ts';
|
||||
import { setChatFormActionsContext } from '#lib/contexts/index.ts';
|
||||
import { FileTypeCategory } from '#lib/enums/index.ts';
|
||||
import { useChatFormModel } from '#lib/hooks/use-chat-form-model.svelte.ts';
|
||||
import { useShowContextGauge } from '#lib/hooks/use-context-gauge.svelte.ts';
|
||||
import { ChatService } from '#lib/services/index.ts';
|
||||
import { conversationsStore, deviceStore, settingsStore, toolsStore } from '#lib/stores/index.ts';
|
||||
import { abbreviateWorkingDir, getFileTypeCategory } from '#lib/utils/index.ts';
|
||||
import { abbreviateWorkingDir } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
canSend?: boolean;
|
||||
@@ -25,12 +28,25 @@
|
||||
inline?: boolean;
|
||||
isLoading?: boolean;
|
||||
isReasoning?: boolean;
|
||||
isRecording?: boolean;
|
||||
/** A send was asked for and waits for the dictated text to be refined. */
|
||||
isSendWaiting?: boolean;
|
||||
/** The dictated text in the message is still being refined. */
|
||||
isVoiceRefining?: boolean;
|
||||
/** Why the microphone can't record, shown on its button. */
|
||||
micWarning?: string | null;
|
||||
/** Voice input: whether the microphone shows, what it does, and why it can't. */
|
||||
voice?: {
|
||||
action: string;
|
||||
hint: string | null;
|
||||
show: boolean;
|
||||
unavailableReason: string | null;
|
||||
/** The pointer or the focus came to the microphone button, a press may follow */
|
||||
prepare?: () => void;
|
||||
/** It left without a press */
|
||||
unprepare?: () => void;
|
||||
};
|
||||
showAddButton?: boolean;
|
||||
showModelSelector?: boolean;
|
||||
uploadedFiles?: ChatUploadedFile[];
|
||||
/** The chat's working directory, which the + menu names. */
|
||||
workingDirectory?: string | null;
|
||||
onFileUpload?: () => void;
|
||||
@@ -47,7 +63,8 @@
|
||||
inline = false,
|
||||
isLoading = false,
|
||||
isReasoning = false,
|
||||
isRecording = false,
|
||||
isSendWaiting = false,
|
||||
isVoiceRefining = false,
|
||||
micWarning = null,
|
||||
onFileUpload,
|
||||
onMicClick,
|
||||
@@ -56,7 +73,7 @@
|
||||
onWorkingDirectoryClick,
|
||||
showAddButton = true,
|
||||
showModelSelector = true,
|
||||
uploadedFiles = [],
|
||||
voice,
|
||||
workingDirectory = null
|
||||
}: Props = $props();
|
||||
|
||||
@@ -77,12 +94,17 @@
|
||||
return '';
|
||||
});
|
||||
|
||||
let hasAudioAttachments = $derived(
|
||||
uploadedFiles.some((file) => getFileTypeCategory(file.type) === FileTypeCategory.AUDIO)
|
||||
);
|
||||
let shouldShowRecordButton = $derived(
|
||||
hasAudioModality && !canSend && !hasAudioAttachments && currentConfig.autoMicOnEmpty
|
||||
);
|
||||
// the microphone sits beside the send or stop button: on an empty message when the setting
|
||||
// asks for it (disabled, saying why, when no model can take a recording), and while a reply
|
||||
// streams or the message has text or a recording, so more can be said without clearing it
|
||||
let showStop = $derived(isLoading && !canSend);
|
||||
let showRecord = $derived.by(() => {
|
||||
if (!voice?.show) return false;
|
||||
|
||||
if (showStop || canSend) return !voice.unavailableReason;
|
||||
|
||||
return currentConfig.autoMicOnEmpty;
|
||||
});
|
||||
|
||||
let selectorModelRef: ModelsSelector | undefined = $state(undefined);
|
||||
|
||||
@@ -159,6 +181,18 @@
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if isVoiceRefining}
|
||||
<span
|
||||
class="flex items-center gap-1.5 text-caption whitespace-nowrap text-muted-foreground"
|
||||
data-slot="voice-refining"
|
||||
role="status"
|
||||
>
|
||||
<LoaderCircle class="size-icon-sm animate-spin" />
|
||||
|
||||
{isSendWaiting ? VOICE_MODE_MESSAGE.sendAfterRefining : VOICE_MODE_MESSAGE.refining}
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
{#if isReasoning && !canSend}
|
||||
<Button
|
||||
class="group size-8 rounded-full p-0 max-md:size-9"
|
||||
@@ -176,7 +210,26 @@
|
||||
</Button>
|
||||
{/if}
|
||||
|
||||
{#if isLoading && !canSend}
|
||||
{#snippet record(filled: boolean)}
|
||||
<ChatFormActionRecord
|
||||
action={voice?.action}
|
||||
{disabled}
|
||||
{filled}
|
||||
hint={voice?.hint}
|
||||
{micWarning}
|
||||
{onMicClick}
|
||||
onPrepare={voice?.prepare}
|
||||
onPrepareEnd={voice?.unprepare}
|
||||
secondary
|
||||
unavailableReason={voice?.unavailableReason}
|
||||
/>
|
||||
{/snippet}
|
||||
|
||||
{#if showRecord && !showStop}
|
||||
{@render record(false)}
|
||||
{/if}
|
||||
|
||||
{#if showStop}
|
||||
<Button
|
||||
class="group size-8 rounded-full p-0 max-md:size-9 hover:bg-destructive/10!"
|
||||
onclick={onStop}
|
||||
@@ -189,21 +242,18 @@
|
||||
class="size-icon-md fill-muted-foreground stroke-muted-foreground group-hover:fill-destructive group-hover:stroke-destructive hover:fill-destructive hover:stroke-destructive"
|
||||
/>
|
||||
</Button>
|
||||
{:else if shouldShowRecordButton}
|
||||
<ChatFormActionRecord
|
||||
{disabled}
|
||||
{hasAudioModality}
|
||||
{isLoading}
|
||||
{isRecording}
|
||||
{micWarning}
|
||||
{onMicClick}
|
||||
/>
|
||||
{:else}
|
||||
<ChatFormActionSubmit
|
||||
canSend={canSend && (showModelSelector ? hasModelSelected && isSelectedModelInCache : true)}
|
||||
{disabled}
|
||||
loading={isSendWaiting}
|
||||
showErrorState={showModelSelector && hasModelSelected && !isSelectedModelInCache}
|
||||
tooltipLabel={submitTooltip}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- while a reply streams, the microphone is the last button, filled like stop -->
|
||||
{#if showRecord && showStop}
|
||||
{@render record(true)}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
<!--
|
||||
The chat form's actions row while it records: Llama UI's VoiceRecordingBar with the form's labels and its
|
||||
microphone button as the stop button. A voice message's text, when the model streams it, runs under the waveform;
|
||||
dictated text goes into the message instead. After a dictation that streamed nothing, it waits for the text in
|
||||
place of the waveform. Speech that stays too quiet even boosted is said inside the row, with the advice in a tooltip above it, so the form does not jump.
|
||||
-->
|
||||
<script lang="ts">
|
||||
import { VoiceRecordingBar } from '@ggml-org/llama-ui/voice';
|
||||
import { ChatFormActionRecord } from '#lib/components/app/index.ts';
|
||||
import { VOICE_MODE_MESSAGE, VOICE_RECORDING_BAR_STATUS } from '#lib/constants/index.ts';
|
||||
import { type ChatFormVoiceState, VoiceMode } from '#lib/enums/index.ts';
|
||||
|
||||
interface Props {
|
||||
class?: string;
|
||||
elapsedMs: number;
|
||||
levels: number[];
|
||||
/** Text streaming while a voice message is recorded. */
|
||||
liveCaption?: string;
|
||||
mode: VoiceMode;
|
||||
/** The speech is too quiet even at the highest gain: the row says so, without changing its height. */
|
||||
quiet?: boolean;
|
||||
state: ChatFormVoiceState;
|
||||
onCancel: () => void;
|
||||
onStop: () => void;
|
||||
/** Stops and sends; left out, there is no send button. */
|
||||
onSend?: () => void;
|
||||
}
|
||||
|
||||
let {
|
||||
class: className = '',
|
||||
elapsedMs,
|
||||
levels,
|
||||
liveCaption = '',
|
||||
mode,
|
||||
onCancel,
|
||||
onSend,
|
||||
onStop,
|
||||
quiet = false,
|
||||
state
|
||||
}: Props = $props();
|
||||
|
||||
const LABELS = {
|
||||
starting: VOICE_MODE_MESSAGE.starting,
|
||||
waiting: VOICE_MODE_MESSAGE.transcribing
|
||||
};
|
||||
</script>
|
||||
|
||||
<VoiceRecordingBar
|
||||
caption={liveCaption}
|
||||
class={className}
|
||||
{elapsedMs}
|
||||
label={mode === VoiceMode.DICTATION ? VOICE_MODE_MESSAGE.dictating : VOICE_MODE_MESSAGE.recording}
|
||||
labels={LABELS}
|
||||
{levels}
|
||||
{onCancel}
|
||||
{onSend}
|
||||
status={VOICE_RECORDING_BAR_STATUS[state]}
|
||||
warning={quiet ? VOICE_MODE_MESSAGE.tooQuiet : null}
|
||||
>
|
||||
{#snippet stop({ disabled })}
|
||||
<ChatFormActionRecord class="shrink-0" {disabled} isRecording onMicClick={onStop} />
|
||||
{/snippet}
|
||||
</VoiceRecordingBar>
|
||||
@@ -3,7 +3,7 @@
|
||||
import { SETTINGS_KEYS } from '#lib/constants/index.ts';
|
||||
import { useDraftMessages } from '#lib/hooks/use-draft-messages.svelte.ts';
|
||||
import { deviceStore, settingsStore } from '#lib/stores/index.ts';
|
||||
import { hapticTap } from '#lib/utils/index.ts';
|
||||
import { hapticTap, shouldTypeToFocus } from '#lib/utils/index.ts';
|
||||
import { afterNavigate } from '$app/navigation';
|
||||
import { page } from '$app/state';
|
||||
import { onMount, tick } from 'svelte';
|
||||
@@ -127,6 +127,14 @@
|
||||
chatFormRef?.focus();
|
||||
}
|
||||
|
||||
// Typing while nothing takes text starts the message: focused during keydown, the
|
||||
// character the key types lands in the form
|
||||
function handleWindowKeydown(event: KeyboardEvent) {
|
||||
if (disabled || deviceStore.isMobile || !shouldTypeToFocus(event)) return;
|
||||
|
||||
chatFormRef?.focus();
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
if (!deviceStore.isMobile) {
|
||||
setTimeout(focusFormUnlessCaptured, 100);
|
||||
@@ -148,6 +156,8 @@
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:window onkeydown={handleWindowKeydown} />
|
||||
|
||||
<div bind:this={formWrapperEl} class="chat-screen-form-wrapper">
|
||||
<ChatForm
|
||||
bind:this={chatFormRef}
|
||||
|
||||
@@ -79,6 +79,14 @@ export { default as ChatAttachmentsListItemMcpResource } from './ChatAttachments
|
||||
*/
|
||||
export { default as ChatAttachmentsListItemThumbnailFile } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte';
|
||||
|
||||
/**
|
||||
* An audio attachment drawn as a clip: a playable waveform (peaks computed from
|
||||
* the audio when it was stored without them) and, for a voice message, its
|
||||
* transcript under it on hover, focus or a tap, with the transcribing and failed
|
||||
* states. `sm` in the chat form, removable; `md` in a sent message.
|
||||
*/
|
||||
export { default as ChatAttachmentsListItemVoice } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte';
|
||||
|
||||
/**
|
||||
* Thumbnail for image attachments with lazy loading and error fallback.
|
||||
* Displays image preview with configurable dimensions. Falls back to placeholder
|
||||
@@ -170,12 +178,20 @@ export { default as ChatForm } from './ChatForm/ChatForm.svelte';
|
||||
export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte';
|
||||
|
||||
/**
|
||||
* Audio recording button with real-time recording indicator. Records audio
|
||||
* and converts to WAV format for upload. Only visible when the active model
|
||||
* supports audio modality and setting for automatic audio input is enabled. Shows recording duration while active.
|
||||
* Microphone button: starts a voice message or a dictation (its label says which),
|
||||
* or, disabled, says why no model can take a recording; while recording it is the
|
||||
* stop button. Shown on an empty message when voice input is available and the
|
||||
* setting for the microphone on empty input is on.
|
||||
*/
|
||||
export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
|
||||
|
||||
/**
|
||||
* The actions row while the chat form records: cancel, the live waveform and the
|
||||
* elapsed time, and stop, with a voice message's streaming text under the waveform
|
||||
* and a waiting state while dictated text is transcribed after recording.
|
||||
*/
|
||||
export { default as ChatFormActionsRecording } from './ChatForm/ChatFormActions/ChatFormActionsRecording.svelte';
|
||||
|
||||
/**
|
||||
* Container for chat form action buttons. Arranges file attachment, audio record,
|
||||
* and submit/stop buttons in a horizontal layout. Handles conditional visibility
|
||||
|
||||
@@ -13,6 +13,7 @@ import { rehypeEnhanceMermaidBlocks } from './plugins/rehype/enhance-mermaid-blo
|
||||
import { rehypeEnhanceSvgBlocks } from './plugins/rehype/enhance-svg-blocks';
|
||||
import { rehypeEnhanceTables } from './plugins/rehype/enhance-tables';
|
||||
import { rehypeFileBadge } from './plugins/rehype/file-badge';
|
||||
import { rehypeLazyHighlight } from './plugins/rehype/lazy-highlight';
|
||||
import { rehypeLazyKatex } from './plugins/rehype/lazy-katex';
|
||||
import { rehypeMermaidPre } from './plugins/rehype/mermaid-pre';
|
||||
import { rehypeRtlSupport } from './plugins/rehype/rehype-rtl-support';
|
||||
@@ -23,9 +24,7 @@ import { remarkLiteralHtml } from './plugins/remark/literal-html';
|
||||
import { FileTypeText } from '#lib/enums/files.enums.ts';
|
||||
import type { DatabaseMessageExtra } from '#lib/types/database.js';
|
||||
import type { Root as HastRoot } from 'hast';
|
||||
import { all as lowlightAll } from 'lowlight';
|
||||
import type { Root as MdastRoot } from 'mdast';
|
||||
import rehypeHighlight from 'rehype-highlight';
|
||||
import rehypeStringify from 'rehype-stringify';
|
||||
import { remark } from 'remark';
|
||||
import remarkBreaks from 'remark-breaks';
|
||||
@@ -75,10 +74,9 @@ function buildPipeline({
|
||||
}
|
||||
|
||||
const pipeline = proc
|
||||
.use(rehypeHighlight, {
|
||||
aliases: { [FileTypeText.XML]: [FileTypeText.SVELTE, FileTypeText.VUE] },
|
||||
languages: lowlightAll
|
||||
}) // Add syntax highlighting
|
||||
.use(rehypeLazyHighlight, {
|
||||
aliases: { [FileTypeText.XML]: [FileTypeText.SVELTE, FileTypeText.VUE] }
|
||||
}) // Add syntax highlighting, loading highlight.js with the first message that has a code block
|
||||
.use(rehypeRestoreTableHtml) // Restore limited HTML (e.g. <br>, <ul>) inside Markdown tables
|
||||
.use(rehypeEnhanceTables) // Wrap tables in a horizontal scroll container
|
||||
.use(rehypeEnhanceLinks) // Add target="_blank" to links
|
||||
|
||||
+4
-1
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
import {
|
||||
CODE_BLOCK,
|
||||
CODE_BLOCK_CLASS,
|
||||
CODE_ICON_SVG,
|
||||
COPY_ICON_SVG,
|
||||
@@ -100,7 +101,9 @@ export function createSourceView(
|
||||
): Element {
|
||||
const code: Element = codeElement ?? {
|
||||
children: [{ type: 'text', value: source }],
|
||||
properties: { className: ['hljs', `language-${language}`] },
|
||||
properties: {
|
||||
className: [CODE_BLOCK_CLASS.HLJS, `${CODE_BLOCK.LANGUAGE_CLASS_PREFIX}${language}`]
|
||||
},
|
||||
tagName: 'code',
|
||||
type: 'element'
|
||||
};
|
||||
|
||||
+3
-3
@@ -17,7 +17,7 @@ import {
|
||||
createWrapper,
|
||||
generateBlockId
|
||||
} from './code-block-utils';
|
||||
import { CODE_BLOCK_CLASS, MARKDOWN_DATA_ATTRS } from '#lib/constants/index.ts';
|
||||
import { CODE_BLOCK, CODE_BLOCK_CLASS, MARKDOWN_DATA_ATTRS } from '#lib/constants/index.ts';
|
||||
import type { Element, ElementContent, Root } from 'hast';
|
||||
import type { Plugin } from 'unified';
|
||||
import { visit } from 'unist-util-visit';
|
||||
@@ -34,8 +34,8 @@ function extractLanguage(codeElement: Element): string {
|
||||
if (!Array.isArray(className)) return 'text';
|
||||
|
||||
for (const cls of className) {
|
||||
if (typeof cls === 'string' && cls.startsWith('language-')) {
|
||||
return cls.replace('language-', '');
|
||||
if (typeof cls === 'string' && cls.startsWith(CODE_BLOCK.LANGUAGE_CLASS_PREFIX)) {
|
||||
return cls.slice(CODE_BLOCK.LANGUAGE_CLASS_PREFIX.length);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* Rehype plugin that highlights code blocks like rehype-highlight, loading highlight.js and each language only when a
|
||||
* message has a block in it.
|
||||
*
|
||||
* Only `<pre><code>` blocks with a language class (`language-ts`, `lang-ts`) are highlighted; `no-highlight` opts a
|
||||
* block out, and a language highlight.js lacks is left plain. rehype-highlight itself is not used: it bundles
|
||||
* lowlight's common languages, which would load all at once.
|
||||
*/
|
||||
|
||||
import { CODE_BLOCK, CODE_BLOCK_CLASS } from '#lib/constants/index.ts';
|
||||
import {
|
||||
getLoadedHighlightLanguages,
|
||||
loadHighlightLanguages,
|
||||
resolveHighlightLanguage
|
||||
} from '#lib/utils/code.ts';
|
||||
import type { Element, ElementContent, Root } from 'hast';
|
||||
import type { Plugin } from 'unified';
|
||||
import { visit } from 'unist-util-visit';
|
||||
|
||||
type Lowlight = ReturnType<(typeof import('./lowlight-core'))['createLowlight']>;
|
||||
|
||||
export interface LazyHighlightOptions {
|
||||
/** Extra aliases per language, as rehype-highlight takes them: `{ xml: ['svelte', 'vue'] }`. */
|
||||
aliases?: Record<string, string | readonly string[]>;
|
||||
}
|
||||
|
||||
interface CodeBlock {
|
||||
node: Element;
|
||||
/** The highlight.js language to use, or undefined for one it lacks. */
|
||||
name: string | undefined;
|
||||
}
|
||||
|
||||
let lowlight: Lowlight | undefined;
|
||||
let lowlightLoading: Promise<Lowlight> | undefined;
|
||||
|
||||
const registered = new Set<string>();
|
||||
|
||||
// the language a code element's class names, false for `no-highlight`, as rehype-highlight reads it
|
||||
function classLanguage(node: Element): string | false | undefined {
|
||||
const list = node.properties?.className;
|
||||
|
||||
if (!Array.isArray(list)) return undefined;
|
||||
|
||||
let name: string | undefined;
|
||||
|
||||
for (const item of list) {
|
||||
const value = String(item);
|
||||
|
||||
if ((CODE_BLOCK.NO_HIGHLIGHT_CLASSES as readonly string[]).includes(value)) return false;
|
||||
|
||||
for (const prefix of [CODE_BLOCK.LANG_CLASS_PREFIX, CODE_BLOCK.LANGUAGE_CLASS_PREFIX]) {
|
||||
if (!name && value.startsWith(prefix)) name = value.slice(prefix.length);
|
||||
}
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
function textContent(node: ElementContent | Element): string {
|
||||
if (node.type === 'text') return node.value;
|
||||
|
||||
if (node.type !== 'element') return '';
|
||||
|
||||
return node.children.map(textContent).join('');
|
||||
}
|
||||
|
||||
function loadLowlight(): Promise<Lowlight> {
|
||||
lowlightLoading ??= import('./lowlight-core').then(
|
||||
({ createLowlight }) => (lowlight = createLowlight()),
|
||||
(error) => {
|
||||
lowlightLoading = undefined;
|
||||
|
||||
throw error;
|
||||
}
|
||||
);
|
||||
|
||||
return lowlightLoading;
|
||||
}
|
||||
|
||||
// registers the languages highlightCode's loader has loaded with lowlight, as highlight.js ships them
|
||||
function registerLoadedLanguages(target: Lowlight): void {
|
||||
for (const [name, definition] of Object.entries(getLoadedHighlightLanguages())) {
|
||||
if (registered.has(name)) continue;
|
||||
|
||||
target.register(name, definition);
|
||||
registered.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
function highlightBlocks(target: Lowlight, blocks: CodeBlock[]): void {
|
||||
for (const { name, node } of blocks) {
|
||||
const className = Array.isArray(node.properties.className)
|
||||
? node.properties.className
|
||||
: (node.properties.className = []);
|
||||
|
||||
if (!className.includes(CODE_BLOCK_CLASS.HLJS)) className.unshift(CODE_BLOCK_CLASS.HLJS);
|
||||
|
||||
if (!name || !registered.has(name)) continue;
|
||||
|
||||
const result = target.highlight(name, textContent(node));
|
||||
|
||||
if (result.children.length > 0) node.children = result.children as ElementContent[];
|
||||
}
|
||||
}
|
||||
|
||||
// synchronous when there is no code block or its languages have loaded, so a message waits only for a language it
|
||||
// is the first to use, and every other one renders in the same pass as before
|
||||
export const rehypeLazyHighlight: Plugin<[LazyHighlightOptions?], Root> = (options = {}) => {
|
||||
const extraAliases = new Map<string, string>();
|
||||
|
||||
for (const [language, aliases] of Object.entries(options.aliases ?? {})) {
|
||||
for (const alias of [aliases].flat()) extraAliases.set(alias.toLowerCase(), language);
|
||||
}
|
||||
|
||||
return (tree) => {
|
||||
const blocks: CodeBlock[] = [];
|
||||
|
||||
visit(tree, 'element', (node: Element, _, parent) => {
|
||||
if (node.tagName !== 'code' || parent?.type !== 'element' || parent.tagName !== 'pre') {
|
||||
return;
|
||||
}
|
||||
|
||||
const language = classLanguage(node);
|
||||
|
||||
if (!language) return;
|
||||
|
||||
const key = language.toLowerCase();
|
||||
|
||||
blocks.push({ name: resolveHighlightLanguage(extraAliases.get(key) ?? key), node });
|
||||
});
|
||||
|
||||
if (!blocks.length) return;
|
||||
|
||||
const names = [
|
||||
...new Set(blocks.map((block) => block.name).filter((name) => name !== undefined))
|
||||
];
|
||||
|
||||
if (lowlight && names.every((name) => registered.has(name))) {
|
||||
highlightBlocks(lowlight, blocks);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
return Promise.all([loadLowlight(), loadHighlightLanguages(names)]).then(([target]) => {
|
||||
registerLoadedLanguages(target);
|
||||
highlightBlocks(target, blocks);
|
||||
});
|
||||
};
|
||||
};
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
// lowlight without its bundled language sets, for lazy-highlight.ts to load with the first code block: a static
|
||||
// re-export of createLowlight alone lets the build drop lowlight's `common` and `all`
|
||||
export { createLowlight } from 'lowlight';
|
||||
+2
-1
@@ -1,3 +1,4 @@
|
||||
import { CODE_BLOCK } from '#lib/constants/index.ts';
|
||||
import type { Element, ElementContent, Root, Text } from 'hast';
|
||||
import type { Plugin } from 'unified';
|
||||
import { visit } from 'unist-util-visit';
|
||||
@@ -50,7 +51,7 @@ export function createPreTransform(
|
||||
contentGuard?: (text: string) => boolean
|
||||
): Plugin<[], Root> {
|
||||
const codeClasses = (Array.isArray(languages) ? languages : [languages]).map(
|
||||
(language) => `language-${language}`
|
||||
(language) => `${CODE_BLOCK.LANGUAGE_CLASS_PREFIX}${language}`
|
||||
);
|
||||
|
||||
return () => {
|
||||
|
||||
@@ -1,8 +1,23 @@
|
||||
<script lang="ts">
|
||||
import * as Dialog from '@ggml-org/llama-ui/ui/dialog';
|
||||
import { ArrowDown, ArrowUp, CornerDownLeft, MessageSquare, Pin, X } from '@lucide/svelte';
|
||||
import { ActionIcon, KeyboardShortcutInfo } from '#lib/components/app/index.ts';
|
||||
import {
|
||||
ArrowDown,
|
||||
ArrowUp,
|
||||
CornerDownLeft,
|
||||
Loader2,
|
||||
MessageSquare,
|
||||
Pin,
|
||||
X
|
||||
} from '@lucide/svelte';
|
||||
import { ActionIcon, KeyboardShortcutInfo, ModelOrgAvatar } from '#lib/components/app/index.ts';
|
||||
import {
|
||||
CONVERSATION_SEARCH_CONTENT_DEBOUNCE_MS,
|
||||
CONVERSATION_SEARCH_CONTENT_LIMIT,
|
||||
CONVERSATION_SEARCH_HUB_DEBOUNCE_MS,
|
||||
CONVERSATION_SEARCH_HUB_LIMIT,
|
||||
CONVERSATION_SEARCH_MATCH_LIMIT,
|
||||
CONVERSATION_SEARCH_RECENT_LIMIT,
|
||||
DISCOVER_ICON,
|
||||
ICON_CLASS_DEFAULT,
|
||||
ICON_CLASS_SM,
|
||||
ICON_CLASS_XS,
|
||||
@@ -10,11 +25,25 @@
|
||||
SIDEBAR_FOOTER_ITEMS
|
||||
} from '#lib/constants/index.ts';
|
||||
import { KeyboardKey, SidebarAction } from '#lib/enums/index.ts';
|
||||
import { useDebouncedSearch } from '#lib/hooks/use-debounced-search.svelte.ts';
|
||||
import {
|
||||
type ConversationContentMatch,
|
||||
ConversationSearchService,
|
||||
HuggingFaceService
|
||||
} from '#lib/services/index.ts';
|
||||
import { RouterService } from '#lib/services/router.service.ts';
|
||||
import { conversationsStore, deviceStore, uiStore } from '#lib/stores/index.ts';
|
||||
import { formatRelativeTime, isIMEComposing } from '#lib/utils/index.ts';
|
||||
import {
|
||||
conversationsStore,
|
||||
deviceStore,
|
||||
modelsDiscoverStore,
|
||||
uiStore
|
||||
} from '#lib/stores/index.ts';
|
||||
import type { DatabaseConversation } from '#lib/types/database.js';
|
||||
import type { HfModelInfo } from '#lib/types/huggingface.js';
|
||||
import type { ConversationSearchSnippet } from '#lib/utils/conversation-search.ts';
|
||||
import { formatRelativeTime, isIMEComposing, orgOf } from '#lib/utils/index.ts';
|
||||
import { goto } from '$app/navigation';
|
||||
import type { Component } from 'svelte';
|
||||
import { type Component, untrack } from 'svelte';
|
||||
|
||||
interface Props {
|
||||
open: boolean;
|
||||
@@ -30,46 +59,192 @@
|
||||
keys?: string[];
|
||||
label: string;
|
||||
meta?: string;
|
||||
/** Org whose avatar stands in for the icon, e.g. a Hub model's base org. */
|
||||
org?: string;
|
||||
pinned?: boolean;
|
||||
/** Where the query matched, shown under the label */
|
||||
snippet?: ConversationSearchSnippet;
|
||||
run: () => void;
|
||||
}
|
||||
|
||||
interface SearchSection {
|
||||
entries: SearchEntry[];
|
||||
/** Results still on their way; the section shows while they load. */
|
||||
loading?: boolean;
|
||||
title: string;
|
||||
}
|
||||
|
||||
/** Recent chats shown before typing, so the actions stay in view below them. */
|
||||
const RECENT_LIMIT = 5;
|
||||
/** Matches kept while typing; the rest stays reachable by typing more. */
|
||||
const MATCH_LIMIT = 50;
|
||||
|
||||
let query = $state('');
|
||||
let highlighted = $state(0);
|
||||
let inputRef = $state<HTMLInputElement | null>(null);
|
||||
|
||||
const needle = $derived(query.trim().toLowerCase());
|
||||
const limit = $derived(needle ? MATCH_LIMIT : RECENT_LIMIT);
|
||||
const limit = $derived(
|
||||
needle ? CONVERSATION_SEARCH_MATCH_LIMIT : CONVERSATION_SEARCH_RECENT_LIMIT
|
||||
);
|
||||
|
||||
function matches(label: string): boolean {
|
||||
return !needle || label.toLowerCase().includes(needle);
|
||||
}
|
||||
|
||||
const chatEntries = $derived.by((): SearchEntry[] =>
|
||||
conversationsStore.conversations
|
||||
.filter((conversation) => matches(conversation.name))
|
||||
.sort((a, b) => b.lastModified - a.lastModified)
|
||||
.slice(0, limit)
|
||||
.map((conversation) => ({
|
||||
icon: MessageSquare,
|
||||
id: `chat-${conversation.id}`,
|
||||
label: conversation.name,
|
||||
meta: formatRelativeTime(conversation.lastModified),
|
||||
pinned: conversation.pinned,
|
||||
run: () => close(() => goto(RouterService.chat(conversation.id)))
|
||||
}))
|
||||
function chatEntry(
|
||||
conversation: DatabaseConversation,
|
||||
snippet?: ConversationSearchSnippet
|
||||
): SearchEntry {
|
||||
return {
|
||||
icon: MessageSquare,
|
||||
id: `chat-${conversation.id}`,
|
||||
label: conversation.name,
|
||||
meta: formatRelativeTime(conversation.lastModified),
|
||||
pinned: conversation.pinned,
|
||||
run: () => close(() => goto(RouterService.chat(conversation.id))),
|
||||
snippet
|
||||
};
|
||||
}
|
||||
|
||||
const recentConversations = $derived(
|
||||
[...conversationsStore.conversations].sort((a, b) => b.lastModified - a.lastModified)
|
||||
);
|
||||
|
||||
const titleMatches = $derived(
|
||||
recentConversations.filter((conversation) => matches(conversation.name))
|
||||
);
|
||||
|
||||
const chatEntries = $derived(
|
||||
titleMatches.slice(0, limit).map((conversation) => chatEntry(conversation))
|
||||
);
|
||||
|
||||
// chats found by what was said in them, not by their title; scanned once typing settles
|
||||
let contentMatches = $state.raw<ConversationContentMatch[]>([]);
|
||||
|
||||
const contentSearch = useDebouncedSearch({
|
||||
canRun: () => open,
|
||||
debounceMs: CONVERSATION_SEARCH_CONTENT_DEBOUNCE_MS,
|
||||
getQuery: () => query.trim(),
|
||||
run: async (term, _signal, isCurrent) => {
|
||||
// a chat already listed by its title is not scanned again
|
||||
const byTitle = new Set(titleMatches.map((conversation) => conversation.id));
|
||||
|
||||
try {
|
||||
const found = await ConversationSearchService.searchContent(
|
||||
recentConversations.filter((conversation) => !byTitle.has(conversation.id)),
|
||||
term,
|
||||
{ isCurrent, limit: CONVERSATION_SEARCH_CONTENT_LIMIT }
|
||||
);
|
||||
|
||||
if (isCurrent()) contentMatches = found;
|
||||
} catch {
|
||||
if (isCurrent()) contentMatches = [];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const contentEntries = $derived(
|
||||
needle
|
||||
? contentMatches.map(({ conversation, snippet }) => ({
|
||||
...chatEntry(conversation, snippet),
|
||||
id: `message-${conversation.id}`
|
||||
}))
|
||||
: []
|
||||
);
|
||||
|
||||
// Discover data: the catalog suggestions, and Hugging Face matches once typing settles
|
||||
const isDiscoverEnabled = $derived(uiStore.isDiscoverEnabled);
|
||||
|
||||
// the catalog loads once a session; opening the search asks for it if it has not yet.
|
||||
// untracked: fetch() toggles its loading flag synchronously
|
||||
$effect(() => {
|
||||
if (open && isDiscoverEnabled) untrack(() => void modelsDiscoverStore.fetch());
|
||||
});
|
||||
|
||||
/** A Hub repo as a result; picking it opens Discover on that repo. */
|
||||
function modelEntry(model: HfModelInfo, section: string): SearchEntry {
|
||||
const range = modelsDiscoverStore.cachedSizeRangeFor(model.id);
|
||||
|
||||
return {
|
||||
icon: DISCOVER_ICON,
|
||||
id: `${section}-${model.id}`,
|
||||
label: model.id,
|
||||
meta: range
|
||||
? HuggingFaceService.formatSizeRange(range.min, range.max)
|
||||
: `${HuggingFaceService.formatDownloads(model.downloads)} downloads`,
|
||||
// the model's own org, the way the model cards show it
|
||||
org: orgOf(HuggingFaceService.getBaseModels(model)[0]) || orgOf(model.id),
|
||||
run: () => close(() => uiStore.openModelsDiscover(model.id))
|
||||
};
|
||||
}
|
||||
|
||||
const suggestedEntries = $derived.by((): SearchEntry[] =>
|
||||
isDiscoverEnabled && needle
|
||||
? modelsDiscoverStore.catalogModels
|
||||
.filter((model) => matches(model.id))
|
||||
.slice(0, limit)
|
||||
.map((model) => modelEntry(model, 'suggested'))
|
||||
: []
|
||||
);
|
||||
|
||||
// the dialog keeps its own results: the Discover search stays as it was
|
||||
let hubResults = $state.raw<HfModelInfo[]>([]);
|
||||
|
||||
const hubSearch = useDebouncedSearch({
|
||||
canRun: () => open && isDiscoverEnabled,
|
||||
debounceMs: CONVERSATION_SEARCH_HUB_DEBOUNCE_MS,
|
||||
getQuery: () => query.trim(),
|
||||
run: async (term, _signal, isCurrent) => {
|
||||
try {
|
||||
const results = await HuggingFaceService.searchByQuery(term, {
|
||||
limit: CONVERSATION_SEARCH_HUB_LIMIT
|
||||
});
|
||||
|
||||
if (isCurrent()) hubResults = modelsDiscoverStore.baseOrgFirst(results);
|
||||
} catch {
|
||||
// offline or rate limited: the local results stand on their own
|
||||
if (isCurrent()) hubResults = [];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
const term = query.trim();
|
||||
const isOpen = open;
|
||||
const withHub = isDiscoverEnabled;
|
||||
|
||||
untrack(() => {
|
||||
if (!isOpen || !term) {
|
||||
contentSearch.cancel();
|
||||
hubSearch.cancel();
|
||||
contentMatches = [];
|
||||
hubResults = [];
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
contentSearch.setLoading(true);
|
||||
contentSearch.run(term);
|
||||
|
||||
if (!withHub) {
|
||||
hubSearch.cancel();
|
||||
hubResults = [];
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
hubSearch.setLoading(true);
|
||||
hubSearch.run(term);
|
||||
});
|
||||
});
|
||||
|
||||
// a suggestion that matches is listed once, under the suggestions
|
||||
const hubEntries = $derived.by((): SearchEntry[] => {
|
||||
if (!needle) return [];
|
||||
|
||||
const suggested = new Set(suggestedEntries.map((entry) => entry.label));
|
||||
|
||||
return hubResults
|
||||
.filter((model) => !suggested.has(model.id))
|
||||
.map((model) => modelEntry(model, 'hub'));
|
||||
});
|
||||
|
||||
const actionEntries = $derived.by((): SearchEntry[] =>
|
||||
// every sidebar action the sidebar shows but Search itself, which is what opened this
|
||||
// dialog, and the models manager on a phone, which picks its models in the selector
|
||||
@@ -91,11 +266,23 @@
|
||||
.filter((entry) => matches(entry.label))
|
||||
);
|
||||
|
||||
// your own chats first; the Hub answers last, so it comes last and never moves a row above it
|
||||
const sections = $derived<SearchSection[]>(
|
||||
[
|
||||
{ entries: chatEntries, title: needle ? 'Chats' : 'Recents' },
|
||||
{ entries: actionEntries, title: 'Actions' }
|
||||
].filter((section) => section.entries.length > 0)
|
||||
{
|
||||
entries: contentEntries,
|
||||
loading: contentSearch.isSearching && !!needle,
|
||||
title: 'In messages'
|
||||
},
|
||||
{ entries: suggestedEntries, title: 'Suggested models' },
|
||||
{ entries: actionEntries, title: 'Actions' },
|
||||
{
|
||||
entries: hubEntries,
|
||||
loading: hubSearch.isSearching && !!needle,
|
||||
title: 'On Hugging Face'
|
||||
}
|
||||
].filter((section) => section.entries.length > 0 || section.loading)
|
||||
);
|
||||
|
||||
const entries = $derived(sections.flatMap((section) => section.entries));
|
||||
@@ -197,8 +384,12 @@
|
||||
.reduce((total, previous) => total + previous.entries.length, 0)}
|
||||
|
||||
<div class="flex flex-col gap-0.5 {sectionIndex > 0 ? 'mt-3' : ''}" role="group">
|
||||
<p class="px-2.5 pt-1 pb-1.5 text-label-sm text-muted-foreground">
|
||||
<p class="flex items-center gap-1.5 px-2.5 pt-1 pb-1.5 text-label-sm text-muted-foreground">
|
||||
{section.title}
|
||||
|
||||
{#if section.loading}
|
||||
<Loader2 aria-label="Searching" class="{ICON_CLASS_XS} animate-spin" />
|
||||
{/if}
|
||||
</p>
|
||||
|
||||
{#each section.entries as entry, entryIndex (entry.id)}
|
||||
@@ -218,9 +409,25 @@
|
||||
tabindex={-1}
|
||||
type="button"
|
||||
>
|
||||
<entry.icon class="{ICON_CLASS_DEFAULT} shrink-0 text-muted-foreground" />
|
||||
{#if entry.org && HuggingFaceService.isEnabled()}
|
||||
<ModelOrgAvatar class="shrink-0" org={entry.org} size="size-icon-md" />
|
||||
{:else}
|
||||
<entry.icon class="{ICON_CLASS_DEFAULT} shrink-0 text-muted-foreground" />
|
||||
{/if}
|
||||
|
||||
<span class="min-w-0 flex-1 truncate">{entry.label}</span>
|
||||
{#if entry.snippet}
|
||||
<span class="flex min-w-0 flex-1 flex-col">
|
||||
<span class="truncate">{entry.label}</span>
|
||||
|
||||
<span class="truncate text-caption text-muted-foreground">
|
||||
{entry.snippet.before}<mark class="bg-transparent font-medium text-foreground"
|
||||
>{entry.snippet.match}</mark
|
||||
>{entry.snippet.after}
|
||||
</span>
|
||||
</span>
|
||||
{:else}
|
||||
<span class="min-w-0 flex-1 truncate">{entry.label}</span>
|
||||
{/if}
|
||||
|
||||
{#if entry.pinned}
|
||||
<Pin aria-label="Pinned" class="{ICON_CLASS_SM} shrink-0 text-muted-foreground" />
|
||||
@@ -263,7 +470,7 @@
|
||||
<!-- like every dialog, a phone shows it full screen -->
|
||||
<Dialog.Root onOpenChange={handleOpenChange} {open}>
|
||||
<Dialog.Content
|
||||
class="top-[12%] flex h-[min(36rem,76dvh)]! w-full max-w-2xl! translate-y-0 flex-col gap-0 overflow-hidden rounded-xl p-0"
|
||||
class="top-[12%] flex h-[min(36rem,76dvh)]! w-full max-w-2xl! md:w-[calc(100%-2rem)] translate-y-0 flex-col gap-0 overflow-hidden rounded-xl p-0"
|
||||
onOpenAutoFocus={(event) => {
|
||||
event.preventDefault();
|
||||
inputRef?.focus();
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
<script lang="ts">
|
||||
import { Heart, HeartOff, Info } from '@lucide/svelte';
|
||||
import { Heart, HeartOff, Info, Mic } from '@lucide/svelte';
|
||||
import { ActionIcon } from '#lib/components/app/index.ts';
|
||||
import { modelsStore, uiStore } from '#lib/stores/index.ts';
|
||||
import { VOICE_DICTATION_MODEL_LABEL } from '#lib/constants/index.ts';
|
||||
import { modelsStore, serverStore, uiStore, voiceStore } from '#lib/stores/index.ts';
|
||||
import type { ModelOption } from '#lib/types/models.js';
|
||||
|
||||
interface Props {
|
||||
@@ -13,7 +14,19 @@
|
||||
|
||||
let { isFav, option, revealOnHover = true }: Props = $props();
|
||||
|
||||
// the favorite heart stays visible, so the reveal applies to the other icons
|
||||
// the favorite heart and the dictation model's microphone stay visible, so the reveal
|
||||
// applies to the other icons
|
||||
// any router model that hears can be the one that transcribes dictation
|
||||
let canDictate = $derived(serverStore.isRouterMode && voiceStore.roleOf(option.model) !== null);
|
||||
let dictates = $derived(voiceStore.isDictationModel(option.model));
|
||||
let dictationLabel = $derived.by(() => {
|
||||
if (!dictates) return VOICE_DICTATION_MODEL_LABEL.use;
|
||||
|
||||
return voiceStore.dictationIsAutomatic
|
||||
? VOICE_DICTATION_MODEL_LABEL.automatic
|
||||
: VOICE_DICTATION_MODEL_LABEL.chosen;
|
||||
});
|
||||
|
||||
const revealClass =
|
||||
'pointer-events-none opacity-0 group-hover:pointer-events-auto group-hover:opacity-100 [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100';
|
||||
</script>
|
||||
@@ -35,6 +48,26 @@
|
||||
/>
|
||||
</span>
|
||||
|
||||
{#if canDictate}
|
||||
<!-- like the heart: lit on the model that dictates, among the hover actions on the others -->
|
||||
<span
|
||||
class={!dictates && revealOnHover ? revealClass : ''}
|
||||
data-dictates={dictates || undefined}
|
||||
data-slot="model-dictation-action"
|
||||
>
|
||||
<ActionIcon
|
||||
class={dictates
|
||||
? 'size-icon-lg text-destructive hover:text-destructive'
|
||||
: 'size-icon-lg hover:text-foreground'}
|
||||
icon={Mic}
|
||||
iconSize="size-icon-md"
|
||||
onclick={() => voiceStore.toggleDictation(option.model)}
|
||||
tooltip={dictationLabel}
|
||||
tooltipAsTitle
|
||||
/>
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
{#if isFav}
|
||||
<span class="flex size-5 items-center justify-center">
|
||||
<span class="flex group-hover:hidden [@media(pointer:coarse)]:hidden">
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
<!--
|
||||
What kind of model with audio input a model is, as a small badge beside its name in the model picker and the Models
|
||||
manager: a speech-to-text model that is for dictation rather than chat, or a chat model that hears and gets voice
|
||||
messages. A model that cannot hear shows nothing. The tooltip says what the kind means. Which model transcribes
|
||||
dictation is shown by the microphone among the row's actions instead (ModelRowActions).
|
||||
-->
|
||||
<script lang="ts">
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import { AudioLines, Captions } from '@lucide/svelte';
|
||||
import { VOICE_MODEL_ROLE_DESCRIPTION, VOICE_MODEL_ROLE_LABEL } from '#lib/constants/index.ts';
|
||||
import { VoiceModelRole } from '#lib/enums/index.ts';
|
||||
import { voiceStore } from '#lib/stores/index.ts';
|
||||
|
||||
interface Props {
|
||||
model: string;
|
||||
class?: string;
|
||||
}
|
||||
|
||||
let { class: className = '', model }: Props = $props();
|
||||
|
||||
const ICONS = {
|
||||
[VoiceModelRole.TRANSCRIPTION]: Captions,
|
||||
[VoiceModelRole.VOICE]: AudioLines
|
||||
};
|
||||
|
||||
let role = $derived(voiceStore.roleOf(model));
|
||||
let description = $derived(
|
||||
role === VoiceModelRole.TRANSCRIPTION
|
||||
? VOICE_MODEL_ROLE_DESCRIPTION.transcription
|
||||
: VOICE_MODEL_ROLE_DESCRIPTION.voice
|
||||
);
|
||||
</script>
|
||||
|
||||
{#if role}
|
||||
{@const Icon = ICONS[role]}
|
||||
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
{#snippet child({ props })}
|
||||
<span
|
||||
{...props}
|
||||
class={[
|
||||
'inline-flex h-4 shrink-0 items-center gap-0.5 rounded-md border border-border/60 px-1 text-caption-sm leading-none font-medium whitespace-nowrap text-muted-foreground',
|
||||
className
|
||||
]}
|
||||
data-role={role}
|
||||
data-slot="model-voice-role"
|
||||
>
|
||||
<Icon aria-hidden="true" class="size-2.5" />
|
||||
|
||||
{VOICE_MODEL_ROLE_LABEL[role]}
|
||||
</span>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content class="max-w-64">
|
||||
<p>{description}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
+18
-3
@@ -4,15 +4,20 @@
|
||||
import ModelContext from '../ModelContext.svelte';
|
||||
import ModelDraftSubline from '../ModelDraftSubline.svelte';
|
||||
import ModelId from '../ModelId.svelte';
|
||||
import ModelVoiceRole from '../ModelVoiceRole.svelte';
|
||||
import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte';
|
||||
import { modelRowActions } from './row-actions';
|
||||
import { configuredContext, downloadProgressFor, isRowControlClick } from './utils';
|
||||
import { ModelDownloadProgressBar } from '@ggml-org/llama-ui/models';
|
||||
import { Heart, MoreHorizontal } from '@lucide/svelte';
|
||||
import { Heart, Mic, MoreHorizontal } from '@lucide/svelte';
|
||||
import { ContextMenuActions, DropdownMenuActions } from '#lib/components/app/index.ts';
|
||||
import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '#lib/constants/index.ts';
|
||||
import {
|
||||
MODEL_ROW_GRID_CLASS,
|
||||
MODEL_ROW_TRAILING_CELL_CLASS,
|
||||
VOICE_DICTATION_MODEL_LABEL
|
||||
} from '#lib/constants/index.ts';
|
||||
import { ModelRowDownloadState } from '#lib/enums/index.ts';
|
||||
import { deviceStore, modelsStore } from '#lib/stores/index.ts';
|
||||
import { deviceStore, modelsStore, voiceStore } from '#lib/stores/index.ts';
|
||||
import type { ModelOption, ModelSidecarBadge } from '#lib/types/models.js';
|
||||
|
||||
interface Props {
|
||||
@@ -38,6 +43,7 @@
|
||||
}: Props = $props();
|
||||
|
||||
let favorite = $derived(isFavorite(option));
|
||||
let dictates = $derived(voiceStore.isDictationModel(option.model));
|
||||
let isHidden = $derived(modelsStore.isHidden(option.id));
|
||||
// live while the download runs, frozen while it is paused
|
||||
let downloadProgress = $derived(downloadProgressFor(option.model));
|
||||
@@ -101,9 +107,18 @@
|
||||
title={option.model}
|
||||
/>
|
||||
|
||||
<ModelVoiceRole model={option.model} />
|
||||
|
||||
<!-- a phone has no width for the modality icons, the id needs it more -->
|
||||
<ModelCapabilities hideModalities={deviceStore.isMobile} {option} />
|
||||
|
||||
{#if dictates}
|
||||
<!-- decorative like the heart: the text carries the state -->
|
||||
<Mic aria-hidden="true" class="size-icon-sm shrink-0 text-destructive" />
|
||||
|
||||
<span class="sr-only">{VOICE_DICTATION_MODEL_LABEL.current}</span>
|
||||
{/if}
|
||||
|
||||
{#if favorite}
|
||||
<!-- the heart is decorative, the button's text carries the state -->
|
||||
<Heart aria-hidden="true" class="size-icon-sm shrink-0 text-rose-500" />
|
||||
|
||||
+24
-2
@@ -16,7 +16,7 @@
|
||||
SETTINGS_KEYS
|
||||
} from '#lib/constants/index.ts';
|
||||
import { ModelDownloadConfirmAction } from '#lib/enums/index.ts';
|
||||
import { modelsStore, settingsStore } from '#lib/stores/index.ts';
|
||||
import { modelsStore, settingsStore, voiceStore } from '#lib/stores/index.ts';
|
||||
import {
|
||||
type GroupedModelOptions,
|
||||
groupModelFamilies,
|
||||
@@ -53,6 +53,12 @@
|
||||
showOrgName = true
|
||||
}: Props = $props();
|
||||
let render = $derived(renderOption ?? defaultOption);
|
||||
// a loaded model that only transcribes dictation is listed apart from the chat models;
|
||||
// the conversation's own model stays with them, even when it also dictates
|
||||
const isDictationOnly = (item: ModelItem) =>
|
||||
item.option.model === voiceStore.dictationModel && item.option.model !== currentModel;
|
||||
let loadedForDictation = $derived(loaded.filter(isDictationOnly));
|
||||
let loadedForChat = $derived(loaded.filter((item) => !isDictationOnly(item)));
|
||||
// a large catalog mounts a window at a time: the sentinel at the end of the list
|
||||
// grows the window when it scrolls into view
|
||||
let visibleCount = $state(MODEL_ROW_WINDOW);
|
||||
@@ -180,9 +186,25 @@
|
||||
<Power class="size-icon-sm shrink-0" />
|
||||
{/snippet}
|
||||
|
||||
{#each loaded as item (item.option.id)}
|
||||
{#each loadedForChat as item (item.option.id)}
|
||||
{@render render(item, !showOrgName)}
|
||||
{/each}
|
||||
|
||||
{#if loadedForDictation.length > 0}
|
||||
<!-- the model that transcribes dictation, apart from the ones to chat with -->
|
||||
{#if loadedForChat.length > 0}
|
||||
<p
|
||||
class="m-0 px-2 pt-2 pb-1 text-label-sm text-muted-foreground/70 select-none"
|
||||
data-slot="models-loaded-dictation-heading"
|
||||
>
|
||||
For dictation
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
{#each loadedForDictation as item (item.option.id)}
|
||||
{@render render(item, !showOrgName)}
|
||||
{/each}
|
||||
{/if}
|
||||
</ModelsSection>
|
||||
{/if}
|
||||
|
||||
|
||||
+4
-1
@@ -5,7 +5,8 @@
|
||||
ModelCapabilities,
|
||||
ModelId,
|
||||
ModelLoadControl,
|
||||
ModelRowActions
|
||||
ModelRowActions,
|
||||
ModelVoiceRole
|
||||
} from '#lib/components/app/index.ts';
|
||||
import { SETTINGS_KEYS } from '#lib/constants/index.ts';
|
||||
import { ServerModelStatus } from '#lib/enums/index.ts';
|
||||
@@ -89,6 +90,8 @@
|
||||
title={option.model}
|
||||
/>
|
||||
|
||||
<ModelVoiceRole model={option.model} />
|
||||
|
||||
{#if showCapabilities}
|
||||
<ModelCapabilities {option} />
|
||||
{/if}
|
||||
|
||||
@@ -53,6 +53,8 @@ export { default as ModelLoadControl } from './ModelLoadControl.svelte';
|
||||
export { default as ModelOrgAvatar } from './ModelOrgAvatar.svelte';
|
||||
export { default as ModelRowActions } from './ModelRowActions.svelte';
|
||||
export { default as ModelsSection } from './ModelsSection.svelte';
|
||||
/** **ModelVoiceRole** - a model's part in voice input (dictation, speech to text, voice), as a badge. */
|
||||
export { default as ModelVoiceRole } from './ModelVoiceRole.svelte';
|
||||
|
||||
/** Capability and modality icon row, shared by every model-id surface. */
|
||||
export { ModelCapabilityIcons } from '@ggml-org/llama-ui/models';
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
SettingsChatImportExportTab,
|
||||
SettingsChatMobileHeader,
|
||||
SettingsChatToolsTab,
|
||||
SettingsChatVoiceTab,
|
||||
SettingsFooter,
|
||||
SettingsResetButton
|
||||
} from '#lib/components/app/settings/index.ts';
|
||||
@@ -185,6 +186,12 @@
|
||||
<SettingsChatToolsTab />
|
||||
{:else if currentSection.slug === SETTINGS_SECTION_SLUGS.IMPORT_EXPORT}
|
||||
<SettingsChatImportExportTab />
|
||||
{:else if currentSection.slug === SETTINGS_SECTION_SLUGS.VOICE && currentSection.fields}
|
||||
<SettingsChatVoiceTab
|
||||
fields={currentSection.fields}
|
||||
{localConfig}
|
||||
onConfigChange={handleConfigChange}
|
||||
/>
|
||||
{:else if currentSection.fields}
|
||||
<div class="space-y-6">
|
||||
<SettingsChatFields
|
||||
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
<!--
|
||||
Voice section of the settings: the registry's fields, with the microphone and the transcription model between them.
|
||||
Those list this device's microphones and the server's models with audio input, which a static field cannot, so they
|
||||
are drawn here.
|
||||
-->
|
||||
<script lang="ts">
|
||||
import { Label } from '@ggml-org/llama-ui/ui/label';
|
||||
import * as Select from '@ggml-org/llama-ui/ui/select';
|
||||
import {
|
||||
listMicrophones,
|
||||
type Microphone,
|
||||
VOICE_MICROPHONE_AUTO,
|
||||
VOICE_MICROPHONE_DEFAULT
|
||||
} from '@ggml-org/llama-voice';
|
||||
import { SettingsChatFields } from '#lib/components/app/settings/index.ts';
|
||||
import {
|
||||
SETTING_CONFIG_INFO,
|
||||
SETTINGS_KEYS,
|
||||
VOICE_GGUF_EXTENSION_PATTERN,
|
||||
VOICE_SETTINGS_LABEL,
|
||||
VOICE_TRANSCRIPTION_MODEL_AUTO
|
||||
} from '#lib/constants/index.ts';
|
||||
import { modelsStore, serverStore } from '#lib/stores/index.ts';
|
||||
import { isTranscriptionOnlyModel } from '#lib/utils/voice-mode.ts';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
interface Props {
|
||||
fields: SettingsFieldConfig[];
|
||||
localConfig: SettingsConfigType;
|
||||
onConfigChange: (key: string, value: string | boolean) => void;
|
||||
}
|
||||
|
||||
let { fields, localConfig, onConfigChange }: Props = $props();
|
||||
|
||||
const KEY = SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL;
|
||||
|
||||
// the picker sits right after the Voice input field
|
||||
let splitAt = $derived(fields.findIndex((f) => f.key === SETTINGS_KEYS.VOICE_INPUT_MODE) + 1);
|
||||
let isRouter = $derived(serverStore.isRouterMode);
|
||||
let current = $derived(String(localConfig[KEY] ?? VOICE_TRANSCRIPTION_MODEL_AUTO));
|
||||
|
||||
let audioModels = $derived.by(() => {
|
||||
void modelsStore.props.cacheVersion;
|
||||
|
||||
return modelsStore
|
||||
.getVisibleModels()
|
||||
.filter((option) => modelsStore.props.modelSupportsAudio(option.model));
|
||||
});
|
||||
|
||||
let options = $derived.by(() => {
|
||||
const list = audioModels.map((option) => ({ label: option.name, value: option.model }));
|
||||
|
||||
// a model picked earlier that the server no longer lists stays visible as the choice
|
||||
if (current && !list.some((option) => option.value === current)) {
|
||||
list.push({ label: current, value: current });
|
||||
}
|
||||
|
||||
return [
|
||||
{ label: VOICE_SETTINGS_LABEL.automatic, value: VOICE_TRANSCRIPTION_MODEL_AUTO },
|
||||
...list
|
||||
];
|
||||
});
|
||||
|
||||
// speech-to-text models first: they are made for dictation, the chat models that hear can do it too
|
||||
let optionGroups = $derived.by(() => {
|
||||
const models = options.filter((option) => option.value !== VOICE_TRANSCRIPTION_MODEL_AUTO);
|
||||
|
||||
return [
|
||||
{
|
||||
heading: VOICE_SETTINGS_LABEL.speechToText,
|
||||
items: models.filter((option) => isTranscriptionOnlyModel(option.value))
|
||||
},
|
||||
{
|
||||
heading: VOICE_SETTINGS_LABEL.chatModels,
|
||||
items: models.filter((option) => !isTranscriptionOnlyModel(option.value))
|
||||
}
|
||||
].filter((group) => group.items.length > 0);
|
||||
});
|
||||
|
||||
let selectedLabel = $derived(
|
||||
options.find((option) => option.value === current)?.label ?? VOICE_SETTINGS_LABEL.automatic
|
||||
);
|
||||
|
||||
const MIC_KEY = SETTINGS_KEYS.VOICE_MICROPHONE;
|
||||
|
||||
let microphones = $state<Microphone[]>([]);
|
||||
let micCurrent = $derived(String(localConfig[MIC_KEY] ?? VOICE_MICROPHONE_AUTO));
|
||||
// the browser names its microphones only once the page may use one
|
||||
let micNamesHidden = $derived(microphones.some((mic) => !mic.label));
|
||||
let micOptions = $derived.by(() => {
|
||||
const list = microphones.map((mic, index) => ({
|
||||
label: mic.label || VOICE_SETTINGS_LABEL.microphone(index + 1),
|
||||
value: mic.deviceId
|
||||
}));
|
||||
const fixed = [VOICE_MICROPHONE_AUTO, VOICE_MICROPHONE_DEFAULT];
|
||||
|
||||
// a microphone picked earlier that is not plugged in now stays visible as the choice
|
||||
if (!fixed.includes(micCurrent) && !list.some((option) => option.value === micCurrent)) {
|
||||
list.push({ label: VOICE_SETTINGS_LABEL.disconnectedMicrophone, value: micCurrent });
|
||||
}
|
||||
|
||||
return [
|
||||
{ label: VOICE_SETTINGS_LABEL.automatic, value: VOICE_MICROPHONE_AUTO },
|
||||
{ label: VOICE_SETTINGS_LABEL.systemDefault, value: VOICE_MICROPHONE_DEFAULT },
|
||||
...list.filter((option) => option.value)
|
||||
];
|
||||
});
|
||||
let micLabel = $derived(
|
||||
micOptions.find((option) => option.value === micCurrent)?.label ??
|
||||
VOICE_SETTINGS_LABEL.automatic
|
||||
);
|
||||
|
||||
onMount(() => {
|
||||
const devices = navigator.mediaDevices;
|
||||
const refresh = () => {
|
||||
listMicrophones().then(
|
||||
(list) => (microphones = list),
|
||||
() => (microphones = [])
|
||||
);
|
||||
};
|
||||
|
||||
refresh();
|
||||
devices?.addEventListener('devicechange', refresh);
|
||||
|
||||
return () => devices?.removeEventListener('devicechange', refresh);
|
||||
});
|
||||
|
||||
let singleModelId = $derived(modelsStore.activeModelId);
|
||||
// the file's name, not the GGUF path a single-model server reports
|
||||
let singleModel = $derived(
|
||||
singleModelId
|
||||
? (singleModelId.split('/').pop() ?? singleModelId).replace(VOICE_GGUF_EXTENSION_PATTERN, '')
|
||||
: null
|
||||
);
|
||||
let singleModelHasAudio = $derived(
|
||||
singleModelId ? modelsStore.props.modelSupportsAudio(singleModelId) : false
|
||||
);
|
||||
</script>
|
||||
|
||||
<div class="space-y-6">
|
||||
<SettingsChatFields fields={fields.slice(0, splitAt)} {localConfig} {onConfigChange} />
|
||||
|
||||
<div class="space-y-2" data-slot="voice-microphone">
|
||||
<Label class="text-label" for={MIC_KEY}>Microphone</Label>
|
||||
|
||||
<Select.Root
|
||||
onValueChange={(value) => onConfigChange(MIC_KEY, value)}
|
||||
type="single"
|
||||
value={micCurrent}
|
||||
>
|
||||
<Select.Trigger class="w-full" id={MIC_KEY}>{micLabel}</Select.Trigger>
|
||||
|
||||
<Select.Content>
|
||||
{#each micOptions as option (option.value)}
|
||||
<Select.Item label={option.label} value={option.value}>{option.label}</Select.Item>
|
||||
{/each}
|
||||
</Select.Content>
|
||||
</Select.Root>
|
||||
|
||||
<p class="mt-1 text-caption text-muted-foreground">{SETTING_CONFIG_INFO[MIC_KEY]}</p>
|
||||
|
||||
{#if micNamesHidden}
|
||||
<p class="text-caption text-muted-foreground">
|
||||
The browser names your microphones once you have allowed one, e.g. after your first
|
||||
recording.
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<div class="space-y-2" data-slot="voice-transcription-model">
|
||||
<Label class="text-label" for={KEY}>Transcription model</Label>
|
||||
|
||||
{#if isRouter}
|
||||
<Select.Root
|
||||
onValueChange={(value) => onConfigChange(KEY, value)}
|
||||
type="single"
|
||||
value={current}
|
||||
>
|
||||
<Select.Trigger class="w-full" id={KEY}>{selectedLabel}</Select.Trigger>
|
||||
|
||||
<Select.Content>
|
||||
<Select.Item
|
||||
label={VOICE_SETTINGS_LABEL.automatic}
|
||||
value={VOICE_TRANSCRIPTION_MODEL_AUTO}
|
||||
>
|
||||
{VOICE_SETTINGS_LABEL.automatic}
|
||||
</Select.Item>
|
||||
|
||||
{#each optionGroups as group (group.heading)}
|
||||
<Select.Group>
|
||||
<Select.GroupHeading>{group.heading}</Select.GroupHeading>
|
||||
|
||||
{#each group.items as option (option.value)}
|
||||
<Select.Item label={option.label} value={option.value}>{option.label}</Select.Item>
|
||||
{/each}
|
||||
</Select.Group>
|
||||
{/each}
|
||||
</Select.Content>
|
||||
</Select.Root>
|
||||
|
||||
<p class="mt-1 text-caption text-muted-foreground">
|
||||
{SETTING_CONFIG_INFO[KEY]} Only models with audio input are listed.
|
||||
</p>
|
||||
|
||||
<p class="text-caption text-muted-foreground">
|
||||
A model you pick here is loaded on the server when you dictate, alongside the chat model, if
|
||||
it isn't loaded already. A server that keeps one model at a time (<code>--models-max 1</code
|
||||
>) unloads the chat model to do it.
|
||||
</p>
|
||||
{:else}
|
||||
<p class="text-body-sm text-muted-foreground">
|
||||
{#if singleModel}
|
||||
This server runs one model, <span class="font-medium text-foreground">{singleModel}</span
|
||||
>.
|
||||
{singleModelHasAudio
|
||||
? 'It has audio input, so it transcribes your voice.'
|
||||
: 'It has no audio input, so voice input is not available on this server.'}
|
||||
{:else}
|
||||
This server runs one model, which transcribes your voice when it has audio input.
|
||||
{/if}
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<SettingsChatFields fields={fields.slice(splitAt)} {localConfig} {onConfigChange} />
|
||||
</div>
|
||||
@@ -52,6 +52,15 @@ export { default as SettingsResetButton } from './SettingsResetButton.svelte';
|
||||
*/
|
||||
export { default as SettingsChatImportExportTab } from './SettingsChat/SettingsChatImportExportTab.svelte';
|
||||
|
||||
/**
|
||||
* **SettingsChatVoiceTab** - Voice section of the settings
|
||||
*
|
||||
* The registry's voice fields, with the transcription model picker between them:
|
||||
* in router mode it lists the server's models with audio input, on a single-model
|
||||
* server it says whether that model can transcribe.
|
||||
*/
|
||||
export { default as SettingsChatVoiceTab } from './SettingsChat/SettingsChatVoiceTab.svelte';
|
||||
|
||||
/**
|
||||
* Section wrapper for import/export sections. Displays a title, description,
|
||||
* icon button, and optional summary of recent actions.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Search dialog constants: how many results each section keeps and how long typing settles
|
||||
* before the slower searches (message content, Hugging Face) run.
|
||||
*/
|
||||
|
||||
/** Recent chats shown before typing, so the actions stay in view below them. */
|
||||
export const CONVERSATION_SEARCH_RECENT_LIMIT = 5;
|
||||
|
||||
/** Title matches kept while typing; the rest stays reachable by typing more. */
|
||||
export const CONVERSATION_SEARCH_MATCH_LIMIT = 50;
|
||||
|
||||
/** Chats matched by their messages; the scan stops once this many are found, newest first. */
|
||||
export const CONVERSATION_SEARCH_CONTENT_LIMIT = 20;
|
||||
|
||||
/** Pause after typing before the messages are scanned. */
|
||||
export const CONVERSATION_SEARCH_CONTENT_DEBOUNCE_MS = 150;
|
||||
|
||||
/** Characters kept before the match in a snippet. */
|
||||
export const CONVERSATION_SEARCH_SNIPPET_LEAD = 24;
|
||||
|
||||
/** Snippet length, the match included. */
|
||||
export const CONVERSATION_SEARCH_SNIPPET_MAX_CHARS = 120;
|
||||
|
||||
/** Hugging Face matches kept: Discover lists the rest. */
|
||||
export const CONVERSATION_SEARCH_HUB_LIMIT = 8;
|
||||
|
||||
/** Pause after typing before Hugging Face is asked, as long as the Discover search waits. */
|
||||
export const CONVERSATION_SEARCH_HUB_DEBOUNCE_MS = 300;
|
||||
@@ -8,12 +8,13 @@ import LoadFailedIcon from '@lucide/svelte/icons/circle-alert';
|
||||
import DiscoverIcon from '@lucide/svelte/icons/compass';
|
||||
import UnloadIcon from '@lucide/svelte/icons/eject';
|
||||
import FileIcon from '@lucide/svelte/icons/file';
|
||||
// not a microphone: that one is for recording and dictation
|
||||
import AudioIcon from '@lucide/svelte/icons/file-headphone';
|
||||
import FileTextIcon from '@lucide/svelte/icons/file-text';
|
||||
import CompactCommandIcon from '@lucide/svelte/icons/fold-vertical';
|
||||
import CwdCommandIcon from '@lucide/svelte/icons/folder-open';
|
||||
import ImageIcon from '@lucide/svelte/icons/image';
|
||||
import LoadingIcon from '@lucide/svelte/icons/loader-2';
|
||||
import AudioIcon from '@lucide/svelte/icons/mic';
|
||||
import ModelIcon from '@lucide/svelte/icons/package';
|
||||
import PauseDownloadIcon from '@lucide/svelte/icons/pause';
|
||||
import ResumeDownloadIcon from '@lucide/svelte/icons/play';
|
||||
|
||||
@@ -16,6 +16,7 @@ export * from './auto-scroll.constants';
|
||||
export * from './context-gauge-dial.constants';
|
||||
export * from './context-gauge-popup.constants';
|
||||
export * from './conversation-import.constants';
|
||||
export * from './conversation-search.constants';
|
||||
export * from './binary-detection.constants';
|
||||
export * from './content-detection.constants';
|
||||
export * from './tool-call-args.constants';
|
||||
@@ -46,7 +47,6 @@ export * from './mcp-form.constants';
|
||||
export * from './mcp-resource.constants';
|
||||
export * from './mention-badge.constants';
|
||||
export * from './message-export.constants';
|
||||
export * from './microphone.constants';
|
||||
export * from './path-display.constants';
|
||||
export * from './model-id.constants';
|
||||
export * from './model-loading.constants';
|
||||
@@ -74,3 +74,4 @@ export * from './read-media';
|
||||
export * from './get-datetime';
|
||||
export * from './browser-info';
|
||||
export * from './search-chats';
|
||||
export * from './voice.constants';
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
/** Why the chat form's microphone can't record, as its button and toast tell the user. */
|
||||
export const MICROPHONE_UNAVAILABLE_MESSAGE = {
|
||||
busy: 'The microphone is in use by another app.',
|
||||
denied: "Microphone access was denied. Allow it in the browser's site settings to record.",
|
||||
failed: "The microphone couldn't start.",
|
||||
insecure:
|
||||
'The microphone only works over HTTPS or on localhost. Open this page over HTTPS to record.',
|
||||
notFound: 'No microphone was found.',
|
||||
unsupported: "This browser can't record audio."
|
||||
} as const;
|
||||
@@ -87,6 +87,17 @@ export const SETTINGS_KEYS = {
|
||||
TYP_P: 'typ_p',
|
||||
// General
|
||||
USE_HUGGING_FACE_HUB: 'useHuggingFaceHub',
|
||||
// Voice
|
||||
VOICE_AUTO_GAIN: 'voiceAutoGain',
|
||||
VOICE_BOOST: 'voiceBoost',
|
||||
VOICE_COLLAPSE_TRANSCRIPTS: 'voiceCollapseTranscripts',
|
||||
VOICE_ECHO_CANCELLATION: 'voiceEchoCancellation',
|
||||
VOICE_INPUT_MODE: 'voiceInputMode',
|
||||
VOICE_MICROPHONE: 'voiceMicrophone',
|
||||
VOICE_NOISE_SUPPRESSION: 'voiceNoiseSuppression',
|
||||
VOICE_TRANSCRIBE_MESSAGES: 'voiceTranscribeMessages',
|
||||
VOICE_TRANSCRIPTION_LANGUAGE: 'voiceTranscriptionLanguage',
|
||||
VOICE_TRANSCRIPTION_MODEL: 'voiceTranscriptionModel',
|
||||
XTC_PROBABILITY: 'xtc_probability',
|
||||
XTC_THRESHOLD: 'xtc_threshold'
|
||||
} as const;
|
||||
|
||||
@@ -4,19 +4,24 @@ import { COMPACTION } from './compaction.constants';
|
||||
import { DEFAULT_MCP_CONFIG } from './mcp.constants';
|
||||
import { SETTINGS_KEYS } from './settings-keys.constants';
|
||||
import { TITLE_GENERATION } from './title-generation.constants';
|
||||
import { VOICE_TRANSCRIPTION_MODEL_AUTO } from './voice.constants';
|
||||
import { FILE_GLOB_SEARCH_PICKERS } from './working-directory.constants';
|
||||
import { VOICE_MICROPHONE_AUTO } from '@ggml-org/llama-voice';
|
||||
import AudioLines from '@lucide/svelte/icons/audio-lines';
|
||||
import Code from '@lucide/svelte/icons/code';
|
||||
import Database from '@lucide/svelte/icons/database';
|
||||
import File from '@lucide/svelte/icons/file';
|
||||
import Funnel from '@lucide/svelte/icons/funnel';
|
||||
import GlassWater from '@lucide/svelte/icons/glass-water';
|
||||
import ListRestart from '@lucide/svelte/icons/list-restart';
|
||||
import Mic from '@lucide/svelte/icons/mic';
|
||||
import Monitor from '@lucide/svelte/icons/monitor';
|
||||
import Moon from '@lucide/svelte/icons/moon';
|
||||
import PencilRuler from '@lucide/svelte/icons/pencil-ruler';
|
||||
import SlidersVertical from '@lucide/svelte/icons/sliders-vertical';
|
||||
import Sun from '@lucide/svelte/icons/sun';
|
||||
import { SurfaceTheme, SyncableParameterType } from '#lib/enums/index.ts';
|
||||
import TextCursorInput from '@lucide/svelte/icons/text-cursor-input';
|
||||
import { SurfaceTheme, SyncableParameterType, VoiceInputMode } from '#lib/enums/index.ts';
|
||||
import { SettingsFieldType } from '#lib/enums/settings.enums.ts';
|
||||
import { ColorMode } from '#lib/enums/ui.enums.ts';
|
||||
import type {
|
||||
@@ -35,7 +40,8 @@ export const SETTINGS_SECTIONS = {
|
||||
GENERAL: { slug: 'general', title: 'General' },
|
||||
IMPORT_EXPORT: { slug: 'import-export', title: 'Import/Export' },
|
||||
SAMPLING_PENALTIES: { slug: 'sampling-penalties', title: 'Sampling & Penalties' },
|
||||
TOOLS: { slug: 'tools', title: 'Tools' }
|
||||
TOOLS: { slug: 'tools', title: 'Tools' },
|
||||
VOICE: { slug: 'voice', title: 'Voice' }
|
||||
} as const;
|
||||
|
||||
export const SETTINGS_SECTION_SLUGS = {
|
||||
@@ -45,7 +51,8 @@ export const SETTINGS_SECTION_SLUGS = {
|
||||
GENERAL: SETTINGS_SECTIONS.GENERAL.slug,
|
||||
IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.slug,
|
||||
SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.slug,
|
||||
TOOLS: SETTINGS_SECTIONS.TOOLS.slug
|
||||
TOOLS: SETTINGS_SECTIONS.TOOLS.slug,
|
||||
VOICE: SETTINGS_SECTIONS.VOICE.slug
|
||||
} as const;
|
||||
|
||||
export const SETTINGS_SECTION_TITLES = {
|
||||
@@ -55,7 +62,8 @@ export const SETTINGS_SECTION_TITLES = {
|
||||
GENERAL: SETTINGS_SECTIONS.GENERAL.title,
|
||||
IMPORT_EXPORT: SETTINGS_SECTIONS.IMPORT_EXPORT.title,
|
||||
SAMPLING_PENALTIES: SETTINGS_SECTIONS.SAMPLING_PENALTIES.title,
|
||||
TOOLS: SETTINGS_SECTIONS.TOOLS.title
|
||||
TOOLS: SETTINGS_SECTIONS.TOOLS.title,
|
||||
VOICE: SETTINGS_SECTIONS.VOICE.title
|
||||
} as const;
|
||||
|
||||
export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
|
||||
@@ -132,7 +140,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
|
||||
},
|
||||
{
|
||||
defaultValue: true,
|
||||
help: 'Automatically show microphone button instead of send button when textarea is empty for models with audio modality support.',
|
||||
help: 'Show the microphone beside the send button while the message is empty, whenever voice input is available (see Voice).',
|
||||
key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY,
|
||||
label: 'Show microphone on empty input',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
@@ -405,6 +413,91 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
|
||||
slug: SETTINGS_SECTION_SLUGS.DISPLAY,
|
||||
title: SETTINGS_SECTION_TITLES.DISPLAY
|
||||
},
|
||||
// Voice: the transcription model picker lists the server's models (SettingsChatVoiceTab)
|
||||
{
|
||||
icon: Mic,
|
||||
settings: [
|
||||
{
|
||||
defaultValue: VoiceInputMode.AUTO,
|
||||
help: 'Automatic sends your voice to a model that understands audio, as a voice message with its transcript, and turns it into text in the message input for any other model. Always dictate turns it into text for every model, so only the text reaches the chat.',
|
||||
key: SETTINGS_KEYS.VOICE_INPUT_MODE,
|
||||
label: 'Voice input',
|
||||
options: [
|
||||
{ icon: AudioLines, label: 'Automatic', value: VoiceInputMode.AUTO },
|
||||
{ icon: TextCursorInput, label: 'Always dictate', value: VoiceInputMode.DICTATION }
|
||||
],
|
||||
type: SettingsFieldType.SELECT
|
||||
},
|
||||
{
|
||||
defaultValue: VOICE_TRANSCRIPTION_MODEL_AUTO,
|
||||
help: 'The model that turns speech into text when the chat model has no audio input. Automatic uses a model with audio input that is already loaded, so dictating loads nothing.',
|
||||
key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL,
|
||||
label: 'Transcription model',
|
||||
standaloneField: false,
|
||||
type: SettingsFieldType.SELECT
|
||||
},
|
||||
{
|
||||
defaultValue: VOICE_MICROPHONE_AUTO,
|
||||
help: "The microphone you record from. Automatic takes the system's default one, unless that is a Bluetooth headset's: recording from it switches the headset to its call mode, which plays music in low quality, so the built-in microphone is used instead.",
|
||||
key: SETTINGS_KEYS.VOICE_MICROPHONE,
|
||||
label: 'Microphone',
|
||||
standaloneField: false,
|
||||
type: SettingsFieldType.SELECT
|
||||
},
|
||||
{
|
||||
defaultValue: true,
|
||||
help: 'Write down what was said in each voice message, to read under its waveform. The chat model writes it, and a model without audio input gets it in place of the audio.',
|
||||
key: SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES,
|
||||
label: 'Transcribe voice messages',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: false,
|
||||
help: 'Show the transcript of a voice message folded under its waveform, to open on hover or with a tap, instead of open.',
|
||||
key: SETTINGS_KEYS.VOICE_COLLAPSE_TRANSCRIPTS,
|
||||
label: 'Collapse transcripts',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: true,
|
||||
help: 'Raise quiet speech to a level the model hears well, up to eight times louder, so a quiet microphone or a voice from afar is still understood. Only speech is measured, never the silence around it, and loud speech is left as it is.',
|
||||
key: SETTINGS_KEYS.VOICE_BOOST,
|
||||
label: 'Boost quiet speech',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: false,
|
||||
help: 'Let the browser filter background noise out of what the microphone records. Off, the model gets the raw signal, which speech models take well.',
|
||||
key: SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION,
|
||||
label: 'Noise suppression',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: false,
|
||||
help: 'Let the browser remove what your speakers play from the recording. It makes the system treat a recording as a call, which can lower or pause music and other sound while you record.',
|
||||
key: SETTINGS_KEYS.VOICE_ECHO_CANCELLATION,
|
||||
label: 'Echo cancellation',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: false,
|
||||
help: 'Let the browser even out how loud you sound. It fades the microphone in, so the first words of a recording can come out quiet.',
|
||||
key: SETTINGS_KEYS.VOICE_AUTO_GAIN,
|
||||
label: 'Automatic gain',
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: '',
|
||||
help: 'The language you speak, as a two-letter code such as en or de, passed to the transcription. Leave it empty to let the model tell.',
|
||||
key: SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE,
|
||||
label: 'Transcription language',
|
||||
placeholder: 'Automatic',
|
||||
type: SettingsFieldType.INPUT
|
||||
}
|
||||
],
|
||||
slug: SETTINGS_SECTION_SLUGS.VOICE,
|
||||
title: SETTINGS_SECTION_TITLES.VOICE
|
||||
},
|
||||
// MCP Servers (non-UI config object)
|
||||
{
|
||||
icon: PencilRuler,
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { STORAGE_APP_NAME } from './storage.constants';
|
||||
import { VoiceRecordingBarStatus } from '@ggml-org/llama-ui/voice';
|
||||
import { ChatFormVoiceState } from '#lib/enums/voice.enums.ts';
|
||||
|
||||
/** What the chat form's microphone says it will do, or why it can't, on its button. */
|
||||
export const VOICE_MODE_MESSAGE = {
|
||||
dictate: 'Dictate',
|
||||
dictating: 'Dictating',
|
||||
realtimeOnlyBehindKey:
|
||||
'This model only transcribes live, over a connection a browser cannot open on a server with an API key.',
|
||||
recording: 'Recording a voice message',
|
||||
refining: 'Refining transcript...',
|
||||
sendAfterRefining: 'Sending once the transcript is refined',
|
||||
separateModelHint:
|
||||
'The transcription model is loaded on the server alongside the chat model, which a server that keeps one model at a time unloads.',
|
||||
starting: 'Starting...',
|
||||
tooQuiet:
|
||||
'Your microphone sounds quiet. Move closer, or turn its input level up in your system settings.',
|
||||
transcribing: 'Transcribing...',
|
||||
transcriptionFailed: 'Transcription failed',
|
||||
unavailable:
|
||||
'No model with audio input is available. In router mode, load one or pick a transcription model in Settings > Voice.',
|
||||
voiceMessage: 'Send a voice message'
|
||||
} as const;
|
||||
|
||||
/** The status of a server that has no such route for the model, e.g. a live-only model asked over HTTP. */
|
||||
export const VOICE_NOT_SUPPORTED_STATUS = 501;
|
||||
|
||||
/** Where the chat form remembers that it showed the hint about a separate transcription model. */
|
||||
export const VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.voiceSeparateModelHintSeen`;
|
||||
|
||||
/** Name prefix of a voice message's WAV file; the rest makes it unique per recording. */
|
||||
export const VOICE_CLIP_NAME_PREFIX = 'voice-';
|
||||
|
||||
/** File extension of a voice message. */
|
||||
export const VOICE_CLIP_EXTENSION = '.wav';
|
||||
|
||||
/** Characters of a random id in a voice message's file name, after its time. */
|
||||
export const VOICE_CLIP_ID_LENGTH = 8;
|
||||
|
||||
/** How often the recording's elapsed time is updated. */
|
||||
export const VOICE_ELAPSED_TICK_MS = 200;
|
||||
|
||||
/**
|
||||
* A mouse starts recording on press, so the click that ends that press comes within this
|
||||
* long and is not another tap.
|
||||
*/
|
||||
export const VOICE_PRESS_CLICK_MS = 1000;
|
||||
|
||||
/** How long a transcript that came back keeps looking for the message it belongs to. */
|
||||
export const VOICE_TRANSCRIPT_APPLY_RETRY_DELAYS_MS = [500, 1500, 4000] as const;
|
||||
|
||||
/** A string that is already a URL (data, blob, http...), which an <audio> plays as it is. */
|
||||
export const VOICE_URL_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
|
||||
|
||||
/** A URL whose audio has to be fetched before its peaks can be drawn. */
|
||||
export const VOICE_FETCHABLE_URL_PATTERN = /^(blob|https?):/i;
|
||||
|
||||
/** Number of characters at a stored clip's end that, with its name and length, tell clips apart. */
|
||||
export const VOICE_PEAKS_CACHE_TAIL_LENGTH = 32;
|
||||
|
||||
/** The engine's server routes are relative (`./v1/...`); the app's base takes them from the root. */
|
||||
export const VOICE_RELATIVE_PATH_PATTERN = /^\./;
|
||||
|
||||
/** A model file's extension, left out where a single model's name is shown. */
|
||||
export const VOICE_GGUF_EXTENSION_PATTERN = /\.gguf$/i;
|
||||
|
||||
/** Element of the message box's editor: a tap there edits the live text of a voice message. */
|
||||
export const VOICE_EDITOR_SELECTOR = '.cm-editor';
|
||||
|
||||
/** What Settings > Voice says beside its pickers. */
|
||||
export const VOICE_SETTINGS_LABEL = {
|
||||
automatic: 'Automatic',
|
||||
chatModels: 'Chat models with audio input',
|
||||
disconnectedMicrophone: 'A microphone that is not connected',
|
||||
microphone: (index: number) => `Microphone ${index}`,
|
||||
speechToText: 'Speech to text',
|
||||
systemDefault: 'System default'
|
||||
} as const;
|
||||
|
||||
/** What a voice message attachment is called, in place of its file name. */
|
||||
export const VOICE_MESSAGE_LABEL = 'Voice message';
|
||||
|
||||
/**
|
||||
* The Transcription model setting's Automatic choice: the chat model when it hears, else a
|
||||
* model with audio input that is already loaded, so dictating loads nothing.
|
||||
*/
|
||||
export const VOICE_TRANSCRIPTION_MODEL_AUTO = '';
|
||||
|
||||
/**
|
||||
* Names of speech-to-text models, which write down what is said and are not meant to chat
|
||||
* (Qwen3-ASR, Whisper, Parakeet...). The server tells them apart by their chat template and
|
||||
* does not say so, so the app goes by the name.
|
||||
*/
|
||||
export const VOICE_TRANSCRIPTION_ONLY_PATTERN =
|
||||
/(^|[^a-z0-9])(asr|whisper|parakeet|canary|moonshine|sensevoice|stt)([^a-z0-9]|$)/i;
|
||||
|
||||
/** How the model picker and the Models manager name a kind of model with audio input. */
|
||||
export const VOICE_MODEL_ROLE_LABEL = {
|
||||
transcription: 'Speech to text',
|
||||
voice: 'Voice'
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* What the microphone on a model's row says: on the model that transcribes dictation for
|
||||
* chat models that cannot hear, whether it was chosen or picked automatically (a loaded
|
||||
* model with audio input), and on any other model that hears, that it can be the one.
|
||||
*/
|
||||
export const VOICE_DICTATION_MODEL_LABEL = {
|
||||
automatic: 'Transcribes dictation, picked automatically. Click to keep it',
|
||||
chosen: 'Transcribes dictation. Click to pick automatically',
|
||||
current: 'transcribes dictation',
|
||||
use: 'Use for dictation'
|
||||
} as const;
|
||||
|
||||
/** What each kind means, in the badge's tooltip. */
|
||||
export const VOICE_MODEL_ROLE_DESCRIPTION = {
|
||||
transcription:
|
||||
'A speech-to-text model: it writes down what is said, for dictation, and is not meant to chat.',
|
||||
voice:
|
||||
'Hears audio: as the chat model it gets your voice messages, and it can also transcribe dictation.'
|
||||
} as const;
|
||||
|
||||
/** What the recording bar shows for each state of the chat form's recording. */
|
||||
export const VOICE_RECORDING_BAR_STATUS: Record<ChatFormVoiceState, VoiceRecordingBarStatus> = {
|
||||
[ChatFormVoiceState.IDLE]: VoiceRecordingBarStatus.RECORDING,
|
||||
[ChatFormVoiceState.RECORDING]: VoiceRecordingBarStatus.RECORDING,
|
||||
[ChatFormVoiceState.STARTING]: VoiceRecordingBarStatus.STARTING,
|
||||
[ChatFormVoiceState.TRANSCRIBING]: VoiceRecordingBarStatus.WAITING
|
||||
};
|
||||
@@ -50,5 +50,6 @@ export enum AttachmentLabel {
|
||||
FILE = 'File',
|
||||
MCP_PROMPT = 'MCP Prompt',
|
||||
MCP_RESOURCE = 'MCP Resource',
|
||||
PDF_FILE = 'PDF File'
|
||||
PDF_FILE = 'PDF File',
|
||||
VOICE_TRANSCRIPT = 'Voice message transcript'
|
||||
}
|
||||
|
||||
@@ -100,6 +100,8 @@ export {
|
||||
HtmlInputType,
|
||||
McpPromptVariant,
|
||||
MenuKind,
|
||||
MouseButton,
|
||||
PointerType,
|
||||
ScrollCarouselVariant,
|
||||
SidebarAction,
|
||||
SidebarSectionKind,
|
||||
@@ -121,3 +123,5 @@ export {
|
||||
} from './tools.enums';
|
||||
|
||||
export { SplashOrientation } from './splash.enums';
|
||||
|
||||
export { ChatFormVoiceState, VoiceInputMode, VoiceMode, VoiceModelRole } from './voice.enums';
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export { ColorMode, TooltipSide } from '@ggml-org/llama-ui/enums';
|
||||
export { ColorMode, MouseButton, PointerType, TooltipSide } from '@ggml-org/llama-ui/enums';
|
||||
|
||||
/** The light surface theme: Paper (white panels on a faint grey page) or Matte (grey glass on white). */
|
||||
export enum SurfaceTheme {
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/** Where the chat form's recording is: off, waiting for the microphone, recording, or waiting for dictated text. */
|
||||
export enum ChatFormVoiceState {
|
||||
IDLE = 'idle',
|
||||
RECORDING = 'recording',
|
||||
STARTING = 'starting',
|
||||
TRANSCRIBING = 'transcribing'
|
||||
}
|
||||
|
||||
/** The Voice input setting: what the microphone does with a model that understands audio. */
|
||||
export enum VoiceInputMode {
|
||||
/** Send the voice to a model with audio input, dictate otherwise. */
|
||||
AUTO = 'auto',
|
||||
/** Always turn speech into text; only the text reaches the chat. */
|
||||
DICTATION = 'dictation'
|
||||
}
|
||||
|
||||
/** What a recording becomes for the conversation's model, as the chat form resolves it. */
|
||||
export enum VoiceMode {
|
||||
/** Speech becomes text in the input; no audio is attached. */
|
||||
DICTATION = 'dictation',
|
||||
/** No model can take audio, so the microphone has nothing to do. */
|
||||
UNAVAILABLE = 'unavailable',
|
||||
/** The clip is attached and sent as audio, with a transcript to read. */
|
||||
VOICE_MESSAGE = 'voice-message'
|
||||
}
|
||||
|
||||
/**
|
||||
* What kind of model with audio input a model is, as the model picker and the Models manager
|
||||
* badge it. Which of them transcribes dictation is told apart from it, by a microphone.
|
||||
*/
|
||||
export enum VoiceModelRole {
|
||||
/** A speech-to-text model: for dictation, not for chat */
|
||||
TRANSCRIPTION = 'transcription',
|
||||
/** A chat model with audio input: it gets voice messages */
|
||||
VOICE = 'voice'
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
/**
|
||||
* Voice input for the chat form.
|
||||
*
|
||||
* Resolves what the microphone does for the conversation's model (a voice message,
|
||||
* dictation, or nothing, see resolveVoiceMode) and runs a recording: live levels and
|
||||
* elapsed time, and on stop either a voice message attachment, whose transcript follows,
|
||||
* or the dictated text. Writing the text into the message is createDictation()'s; this
|
||||
* adds what is the chat form's own around it.
|
||||
* A voice message's live text shows as the empty message box's placeholder; focusing
|
||||
* the box makes it editable text there until the recording stops, when it goes with
|
||||
* the clip (edited, as its transcript).
|
||||
*/
|
||||
|
||||
import {
|
||||
createDictation,
|
||||
describeMicrophoneError,
|
||||
MicrophoneWarmup,
|
||||
VoiceCaptureError,
|
||||
type VoiceRecorderStatus,
|
||||
type VoiceSessionOptions,
|
||||
type VoiceSessionResult
|
||||
} from '@ggml-org/llama-voice';
|
||||
import {
|
||||
SETTINGS_KEYS,
|
||||
VOICE_CLIP_EXTENSION,
|
||||
VOICE_CLIP_ID_LENGTH,
|
||||
VOICE_CLIP_NAME_PREFIX,
|
||||
VOICE_ELAPSED_TICK_MS,
|
||||
VOICE_MODE_MESSAGE,
|
||||
VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY
|
||||
} from '#lib/constants/index.ts';
|
||||
import { BooleanString, ChatFormVoiceState, MimeTypeAudio, VoiceMode } from '#lib/enums/index.ts';
|
||||
import { VoiceService } from '#lib/services/voice.service.ts';
|
||||
import {
|
||||
serverStore,
|
||||
settingsStore,
|
||||
voiceStore,
|
||||
voiceTranscriptsStore
|
||||
} from '#lib/stores/index.ts';
|
||||
import type { ChatUploadedFile } from '#lib/types/index.ts';
|
||||
import { uuid } from '#lib/utils/index.ts';
|
||||
import { voiceModeLabel } from '#lib/utils/voice-mode.ts';
|
||||
import { onDestroy } from 'svelte';
|
||||
import { toast } from 'svelte-sonner';
|
||||
|
||||
/** What the form needs of a VoiceRecorder; tests and stories pass a fake. */
|
||||
export interface VoiceRecorderLike {
|
||||
readonly levels: number[];
|
||||
readonly live: boolean;
|
||||
readonly liveText: string;
|
||||
/** The speech stays quiet even at the highest gain; a recorder that cannot tell leaves it out */
|
||||
readonly quiet?: boolean;
|
||||
readonly status: VoiceRecorderStatus;
|
||||
start(options?: VoiceSessionOptions): Promise<void>;
|
||||
stop(): Promise<VoiceSessionResult>;
|
||||
cancel(): void;
|
||||
}
|
||||
|
||||
interface UseChatFormVoiceOptions {
|
||||
recorder: VoiceRecorderLike;
|
||||
getValue: () => string;
|
||||
/** Writes the message as dictation changes it; not a user edit. */
|
||||
setValue: (value: string) => void;
|
||||
getCaretOffset: () => number;
|
||||
/** Puts the caret after the dictated text once it is final. */
|
||||
setCaretOffset: (offset: number) => void;
|
||||
addVoiceFile: (file: ChatUploadedFile) => void;
|
||||
/** Why the last start failed, for the microphone button; null once one worked. */
|
||||
onMicError: (message: string | null) => void;
|
||||
}
|
||||
|
||||
function readHintSeen(): boolean {
|
||||
try {
|
||||
return localStorage.getItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY) === BooleanString.TRUE;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function writeHintSeen(): void {
|
||||
try {
|
||||
localStorage.setItem(VOICE_SEPARATE_MODEL_HINT_LOCALSTORAGE_KEY, BooleanString.TRUE);
|
||||
} catch {
|
||||
// private mode: the hint shows again next time
|
||||
}
|
||||
}
|
||||
|
||||
function readAsDataUrl(blob: Blob): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
|
||||
reader.onload = () => resolve(reader.result as string);
|
||||
reader.onerror = () => reject(reader.error);
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
export function useChatFormVoice(options: UseChatFormVoiceOptions) {
|
||||
const config = $derived(settingsStore.config);
|
||||
const isRouter = $derived(serverStore.isRouterMode);
|
||||
// what the microphone does for the conversation's model, shared with the model lists
|
||||
const resolution = $derived(voiceStore.resolution);
|
||||
// a router with a model that hears keeps the button, so it can say where to pick it
|
||||
const anyAudioModel = $derived(voiceStore.anyAudioModel);
|
||||
|
||||
let hintSeen = $state(readHintSeen());
|
||||
let state = $state<ChatFormVoiceState>(ChatFormVoiceState.IDLE);
|
||||
let elapsedMs = $state(0);
|
||||
// the mode of the recording under way, fixed when it starts
|
||||
let recordingMode = $state<VoiceMode>(VoiceMode.UNAVAILABLE);
|
||||
let ticker: ReturnType<typeof setInterval> | undefined;
|
||||
// bumped when the dictation changes, so the getters that read it follow
|
||||
let dictationVersion = $state(0);
|
||||
// bumped by cancel, so a recording still starting or stopping is dropped
|
||||
let generation = 0;
|
||||
// the pointer or the focus is on the microphone button, see prepare
|
||||
let preparing = false;
|
||||
|
||||
const warmup = new MicrophoneWarmup();
|
||||
const dictation = createDictation({
|
||||
getCaretOffset: options.getCaretOffset,
|
||||
getValue: options.getValue,
|
||||
onChange: () => dictationVersion++,
|
||||
onError: (error) =>
|
||||
toast.error(
|
||||
VoiceService.failureReason(error) ??
|
||||
(error instanceof Error ? error.message : VOICE_MODE_MESSAGE.transcriptionFailed)
|
||||
),
|
||||
setCaretOffset: options.setCaretOffset,
|
||||
setValue: options.setValue
|
||||
});
|
||||
const recorder = options.recorder;
|
||||
const unavailableReason = $derived(
|
||||
resolution.mode === VoiceMode.UNAVAILABLE ? resolution.reason : null
|
||||
);
|
||||
const hint = $derived(
|
||||
resolution.mode === VoiceMode.DICTATION && resolution.separateModel && !hintSeen
|
||||
? VOICE_MODE_MESSAGE.separateModelHint
|
||||
: null
|
||||
);
|
||||
|
||||
function startTicker() {
|
||||
const startedAt = Date.now();
|
||||
|
||||
elapsedMs = 0;
|
||||
ticker = setInterval(() => (elapsedMs = Date.now() - startedAt), VOICE_ELAPSED_TICK_MS);
|
||||
}
|
||||
|
||||
function stopTicker() {
|
||||
clearInterval(ticker);
|
||||
ticker = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A voice message's live text moves from the placeholder into the message, as text the
|
||||
* user can edit, and what is said next follows it there.
|
||||
*/
|
||||
function adoptLiveText() {
|
||||
if (recordingMode !== VoiceMode.VOICE_MESSAGE || dictation.active) return;
|
||||
|
||||
if (state !== ChatFormVoiceState.STARTING && state !== ChatFormVoiceState.RECORDING) return;
|
||||
|
||||
dictation.begin({ at: options.getValue().length, keepsEdits: true });
|
||||
dictation.write(recorder?.liveText ?? '');
|
||||
options.setCaretOffset(dictation.end);
|
||||
}
|
||||
|
||||
/** The session options of a recording started now: the settings, the server and the mode. */
|
||||
function wiringFor(mode: VoiceMode, model: string | null) {
|
||||
const transcribe =
|
||||
mode === VoiceMode.DICTATION || config[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES] !== false;
|
||||
|
||||
return VoiceService.sessionOptions(config, { isRouter, model, transcribe });
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the microphone while the pointer or the focus is on its button, if the page may
|
||||
* already use it: by the click it has settled, so the first words are not quiet, and the
|
||||
* recording starts at once. Let go shortly after the pointer leaves (unprepare).
|
||||
*/
|
||||
async function prepare() {
|
||||
if (!recorder || state !== ChatFormVoiceState.IDLE || resolution.mode === VoiceMode.UNAVAILABLE)
|
||||
return;
|
||||
|
||||
preparing = true;
|
||||
|
||||
const wiring = await wiringFor(resolution.mode, resolution.transcriptionModel);
|
||||
|
||||
// the pointer may have left meanwhile, and a press asks for its own microphone
|
||||
if (preparing && state === ChatFormVoiceState.IDLE) warmup.open(wiring.capture);
|
||||
}
|
||||
|
||||
function unprepare() {
|
||||
preparing = false;
|
||||
warmup.release();
|
||||
}
|
||||
|
||||
async function start() {
|
||||
if (!recorder || state !== ChatFormVoiceState.IDLE || resolution.mode === VoiceMode.UNAVAILABLE)
|
||||
return;
|
||||
|
||||
const mode = resolution.mode;
|
||||
const model = resolution.transcriptionModel;
|
||||
|
||||
if (hint) {
|
||||
hintSeen = true;
|
||||
writeHintSeen();
|
||||
}
|
||||
|
||||
recordingMode = mode;
|
||||
state = ChatFormVoiceState.STARTING;
|
||||
|
||||
if (mode === VoiceMode.DICTATION) dictation.begin();
|
||||
|
||||
const run = ++generation;
|
||||
|
||||
try {
|
||||
const wiring = await wiringFor(mode, model);
|
||||
// the microphone the hover opened, when it is open for the same options
|
||||
const stream = await warmup.take(wiring.capture);
|
||||
|
||||
// cancelled while the microphone opened
|
||||
if (run !== generation) {
|
||||
for (const track of stream?.getTracks() ?? []) track.stop();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await recorder.start({
|
||||
...wiring,
|
||||
capture: { ...wiring.capture, ...(stream ? { stream } : {}) },
|
||||
// a voice message's text reaches the message only once the user made it editable
|
||||
onText: (text) => dictation.write(text)
|
||||
});
|
||||
} catch (error) {
|
||||
if (run !== generation) return;
|
||||
|
||||
state = ChatFormVoiceState.IDLE;
|
||||
dictation.cancel();
|
||||
|
||||
const message =
|
||||
error instanceof VoiceCaptureError ? error.message : describeMicrophoneError(error);
|
||||
|
||||
options.onMicError(message);
|
||||
toast.error(message);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// cancelled meanwhile: the recorder dropped its own session
|
||||
if (run !== generation) return;
|
||||
|
||||
options.onMicError(null);
|
||||
state = ChatFormVoiceState.RECORDING;
|
||||
startTicker();
|
||||
}
|
||||
|
||||
async function stop() {
|
||||
if (!recorder || state !== ChatFormVoiceState.RECORDING) return;
|
||||
|
||||
const run = generation;
|
||||
const mode = recordingMode;
|
||||
|
||||
stopTicker();
|
||||
|
||||
let result: VoiceSessionResult;
|
||||
|
||||
try {
|
||||
result = await recorder.stop();
|
||||
} catch (error) {
|
||||
console.error('Failed to stop the recording:', error);
|
||||
dictation.cancel();
|
||||
state = ChatFormVoiceState.IDLE;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (run !== generation) return;
|
||||
|
||||
if (mode === VoiceMode.VOICE_MESSAGE) {
|
||||
// the clip carries the text from here: what the user edited is its transcript
|
||||
const taken = dictation.take();
|
||||
|
||||
state = ChatFormVoiceState.IDLE;
|
||||
await attachVoiceMessage(result, taken?.userEdited ? taken.text : null);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// with live text the message already shows it and can be edited, while the whole
|
||||
// recording is transcribed once more to refine it; otherwise wait for the text
|
||||
state =
|
||||
result.transcript && !result.liveText
|
||||
? ChatFormVoiceState.TRANSCRIBING
|
||||
: ChatFormVoiceState.IDLE;
|
||||
await dictation.finish(result);
|
||||
|
||||
if (run === generation) state = ChatFormVoiceState.IDLE;
|
||||
}
|
||||
|
||||
async function attachVoiceMessage(
|
||||
{ clip, transcript }: VoiceSessionResult,
|
||||
edited: string | null = null
|
||||
) {
|
||||
const name = `${VOICE_CLIP_NAME_PREFIX}${Date.now()}-${uuid().slice(0, VOICE_CLIP_ID_LENGTH)}${VOICE_CLIP_EXTENSION}`;
|
||||
const file = new File([clip.blob], name, { type: MimeTypeAudio.WAV });
|
||||
|
||||
if (transcript) voiceTranscriptsStore.track(name, transcript);
|
||||
|
||||
if (edited !== null) voiceTranscriptsStore.setTranscript(name, edited);
|
||||
|
||||
options.addVoiceFile({
|
||||
file,
|
||||
id: uuid(),
|
||||
name,
|
||||
preview: await readAsDataUrl(clip.blob),
|
||||
size: file.size,
|
||||
type: MimeTypeAudio.WAV,
|
||||
voice: { durationMs: clip.durationMs, peaks: clip.peaks }
|
||||
});
|
||||
}
|
||||
|
||||
/** Drops the recording and any text it dictated. */
|
||||
function cancel() {
|
||||
generation++;
|
||||
stopTicker();
|
||||
recorder?.cancel();
|
||||
dictation.cancel();
|
||||
state = ChatFormVoiceState.IDLE;
|
||||
}
|
||||
|
||||
/** Follows an edit the user typed while dictation writes into the message. */
|
||||
function handleUserInput() {
|
||||
dictation.edited();
|
||||
}
|
||||
|
||||
onDestroy(() => {
|
||||
preparing = false;
|
||||
warmup.drop();
|
||||
|
||||
if (state !== ChatFormVoiceState.IDLE) cancel();
|
||||
});
|
||||
|
||||
return {
|
||||
adoptLiveText,
|
||||
cancel,
|
||||
get elapsedMs() {
|
||||
return elapsedMs;
|
||||
},
|
||||
handleUserInput,
|
||||
get hint() {
|
||||
return hint;
|
||||
},
|
||||
/** A recording (or its dictated text) is under way: the form shows the recording bar. */
|
||||
get isActive() {
|
||||
return state !== ChatFormVoiceState.IDLE;
|
||||
},
|
||||
/** The dictated text shown is still being refined by a pass over the whole recording. */
|
||||
get isRefining() {
|
||||
void dictationVersion;
|
||||
|
||||
return dictation.refining;
|
||||
},
|
||||
/** What the microphone button says it will do. */
|
||||
get label() {
|
||||
return voiceModeLabel(resolution);
|
||||
},
|
||||
get levels() {
|
||||
return recorder?.levels ?? [];
|
||||
},
|
||||
/** Text streaming while a voice message is recorded, shown under the waveform. */
|
||||
get liveCaption() {
|
||||
void dictationVersion;
|
||||
|
||||
// once made editable, the text is in the message instead
|
||||
return recordingMode === VoiceMode.VOICE_MESSAGE && !dictation.active
|
||||
? (recorder?.liveText ?? '')
|
||||
: '';
|
||||
},
|
||||
get mode() {
|
||||
return state === ChatFormVoiceState.IDLE ? resolution.mode : recordingMode;
|
||||
},
|
||||
/** The pointer or the focus is on the microphone button: see prepare(). */
|
||||
prepare,
|
||||
/** The recording's speech is too quiet even at the highest gain: the form says so. */
|
||||
get quiet() {
|
||||
return state === ChatFormVoiceState.RECORDING && (recorder?.quiet ?? false);
|
||||
},
|
||||
get resolution() {
|
||||
return resolution;
|
||||
},
|
||||
/** Resolves once the text in the message is final, at once when nothing is refining it. */
|
||||
settled(): Promise<void> {
|
||||
return dictation.settled();
|
||||
},
|
||||
/** Whether the form shows a microphone button at all. */
|
||||
get showButton() {
|
||||
return resolution.mode !== VoiceMode.UNAVAILABLE || anyAudioModel;
|
||||
},
|
||||
start,
|
||||
get state() {
|
||||
return state;
|
||||
},
|
||||
stop,
|
||||
get unavailableReason() {
|
||||
return unavailableReason;
|
||||
},
|
||||
/** The pointer or the focus left the microphone button without a press. */
|
||||
unprepare
|
||||
};
|
||||
}
|
||||
@@ -50,6 +50,12 @@ import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts';
|
||||
import { formatAttachmentText } from '#lib/utils/formatters.ts';
|
||||
import { streamIdentity } from '#lib/utils/stream-identity.ts';
|
||||
|
||||
/** How stored messages are shaped for the model a request goes to. */
|
||||
export interface ApiMessageConversionOptions {
|
||||
/** False when the model has no audio input: voice messages go as their transcript. */
|
||||
audioInput?: boolean;
|
||||
}
|
||||
|
||||
interface ResumableStreamState {
|
||||
bytesReceived: number;
|
||||
updatedAt: number;
|
||||
@@ -133,13 +139,24 @@ export class ChatService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* How messages are shaped for `model`: whether it takes audio, known only once its
|
||||
* modalities are. Unknown modalities keep the audio, as before voice messages.
|
||||
*/
|
||||
static conversionOptionsFor(model?: string | null): ApiMessageConversionOptions {
|
||||
const modalities = modelsStore.props.getModelModalities(model ?? '');
|
||||
|
||||
return modalities ? { audioInput: modalities.audio } : {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a database message with attachments to API chat message format.
|
||||
* Processes various attachment types (images, text files, PDFs) and formats them
|
||||
* as content parts suitable for the chat completion API.
|
||||
*/
|
||||
static async convertDbMessageToApiChatMessageData(
|
||||
message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
message: DatabaseMessage & { extra?: DatabaseMessageExtra[] },
|
||||
options: ApiMessageConversionOptions = {}
|
||||
): Promise<ApiChatMessageData> {
|
||||
// Handle tool result messages (role: 'tool')
|
||||
if (message.role === MessageRole.TOOL && message.toolCallId) {
|
||||
@@ -231,6 +248,20 @@ export class ChatService {
|
||||
);
|
||||
|
||||
for (const audio of audioFiles) {
|
||||
// a model that cannot hear reads what was said instead; one that can gets the audio alone
|
||||
if (options.audioInput === false && audio.transcript) {
|
||||
contentParts.push({
|
||||
text: formatAttachmentText(
|
||||
AttachmentLabel.VOICE_TRANSCRIPT,
|
||||
audio.name,
|
||||
audio.transcript
|
||||
),
|
||||
type: ContentPartType.TEXT
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
contentParts.push({
|
||||
input_audio: {
|
||||
data: audio.base64Data,
|
||||
@@ -816,14 +847,18 @@ export class ChatService {
|
||||
* messages. Shared by sendMessage, preEncode and the agentic flow.
|
||||
*/
|
||||
static async normalizeMessagesForApi(
|
||||
messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[]
|
||||
messages: ApiChatMessageData[] | (DatabaseMessage & { extra?: DatabaseMessageExtra[] })[],
|
||||
model?: string | null
|
||||
): Promise<ApiChatMessageData[]> {
|
||||
const options = ChatService.conversionOptionsFor(model);
|
||||
|
||||
return (
|
||||
await Promise.all(
|
||||
messages.map((msg) => {
|
||||
if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
|
||||
return ChatService.convertDbMessageToApiChatMessageData(
|
||||
msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
|
||||
msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] },
|
||||
options
|
||||
);
|
||||
}
|
||||
|
||||
@@ -855,8 +890,10 @@ export class ChatService {
|
||||
signal?: AbortSignal,
|
||||
tools?: OpenAIToolDefinition[]
|
||||
): Promise<ChatMessageTimings | null> {
|
||||
const normalizedMessages: ApiChatMessageData[] =
|
||||
await ChatService.normalizeMessagesForApi(messages);
|
||||
const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(
|
||||
messages,
|
||||
model
|
||||
);
|
||||
const requestBody: Record<string, unknown> = {
|
||||
messages: normalizedMessages.map((msg: ApiChatMessageData) => {
|
||||
const mapped: Record<string, unknown> = {
|
||||
@@ -1106,8 +1143,10 @@ export class ChatService {
|
||||
xtc_probability,
|
||||
xtc_threshold
|
||||
} = options;
|
||||
const normalizedMessages: ApiChatMessageData[] =
|
||||
await ChatService.normalizeMessagesForApi(messages);
|
||||
const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(
|
||||
messages,
|
||||
options.model
|
||||
);
|
||||
|
||||
// Filter out image attachments if the model doesn't support vision
|
||||
if (options.model && !modelsStore.props.modelSupportsVision(options.model)) {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* ConversationSearchService - the search dialog's message search
|
||||
*
|
||||
* Scans the messages of each conversation, newest conversation first, and stops at the first
|
||||
* matching message of a conversation and once enough conversations matched, so a common word
|
||||
* does not read the whole database. What a message offers to the search lives in utils/conversation-search.
|
||||
*/
|
||||
|
||||
import { DatabaseService } from './database.service';
|
||||
import type { DatabaseConversation } from '#lib/types/database.js';
|
||||
import {
|
||||
type ConversationSearchSnippet,
|
||||
findMessageMatch,
|
||||
matchSnippet
|
||||
} from '#lib/utils/conversation-search.ts';
|
||||
|
||||
export interface ConversationContentMatch {
|
||||
conversation: DatabaseConversation;
|
||||
messageId: string;
|
||||
snippet: ConversationSearchSnippet;
|
||||
}
|
||||
|
||||
export interface ConversationContentSearchOptions {
|
||||
/** Matches to find before the scan stops */
|
||||
limit: number;
|
||||
/** Checked between conversations; false stops the scan, its result is then dropped */
|
||||
isCurrent?: () => boolean;
|
||||
}
|
||||
|
||||
export class ConversationSearchService {
|
||||
/**
|
||||
* Conversations whose messages contain `query`, case-insensitively, in the order given.
|
||||
* Pass `conversations` newest first: the scan stops after `limit` matches.
|
||||
*/
|
||||
static async searchContent(
|
||||
conversations: DatabaseConversation[],
|
||||
query: string,
|
||||
{ isCurrent = () => true, limit }: ConversationContentSearchOptions
|
||||
): Promise<ConversationContentMatch[]> {
|
||||
const needle = query.trim().toLowerCase();
|
||||
|
||||
if (!needle || limit <= 0) return [];
|
||||
|
||||
return DatabaseService.readOnly(async ({ messages }) => {
|
||||
const found: ConversationContentMatch[] = [];
|
||||
|
||||
for (const conversation of conversations) {
|
||||
if (found.length >= limit || !isCurrent()) break;
|
||||
|
||||
const message = await messages
|
||||
.where('convId')
|
||||
.equals(conversation.id)
|
||||
.filter((row) => findMessageMatch(row, needle) !== null)
|
||||
.first();
|
||||
// only the one message that matched is read twice
|
||||
const match = message && findMessageMatch(message, needle);
|
||||
|
||||
if (!message || !match) continue;
|
||||
|
||||
found.push({
|
||||
conversation,
|
||||
messageId: message.id,
|
||||
snippet: matchSnippet(match.text, match.at, needle.length)
|
||||
});
|
||||
}
|
||||
|
||||
return found;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -572,6 +572,16 @@ export class DatabaseService {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the messages written at or after `timestamp`, in any conversation.
|
||||
*
|
||||
* @param timestamp - Epoch ms
|
||||
* @returns Messages, oldest first
|
||||
*/
|
||||
static async getMessagesSince(timestamp: number): Promise<DatabaseMessage[]> {
|
||||
return await db[IDXDB_TABLES.messages].where('timestamp').aboveOrEqual(timestamp).toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports multiple conversations and their messages.
|
||||
* Skips conversations that already exist.
|
||||
|
||||
@@ -307,6 +307,10 @@ export { SandboxService } from './sandbox.service';
|
||||
* @see utils/search-chats - parameter parsing, matching and formatting
|
||||
*/
|
||||
export { SearchChatsService } from './search-chats.service';
|
||||
export {
|
||||
type ConversationContentMatch,
|
||||
ConversationSearchService
|
||||
} from './conversation-search.service';
|
||||
|
||||
/**
|
||||
* **RouterService** — Dynamic route URL construction utility
|
||||
@@ -367,3 +371,15 @@ export { MigrationService } from './migration.service';
|
||||
* @see settingsStore in stores/settings/index.svelte.ts - reactive state + business logic
|
||||
*/
|
||||
export { SettingsService } from './settings.service';
|
||||
|
||||
/**
|
||||
* **VoiceService** - Wiring between the voice engine and this app
|
||||
*
|
||||
* Builds what the engine (@ggml-org/llama-voice) is handed from outside: how it reaches the active
|
||||
* server (routes, API key, whether realtime can be used) and the capture and transcribe
|
||||
* options the Voice settings ask for. Every feature that records starts its session with
|
||||
* these, so they all reach the server the same way and follow the same settings.
|
||||
*
|
||||
* @see voice.service.ts
|
||||
*/
|
||||
export { VoiceService } from './voice.service';
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
import {
|
||||
resolveMicrophone,
|
||||
VOICE_MICROPHONE_AUTO,
|
||||
type VoiceCaptureOptions,
|
||||
type VoiceServerContext,
|
||||
type VoiceSessionOptions,
|
||||
VoiceTranscriptionError
|
||||
} from '@ggml-org/llama-voice';
|
||||
import {
|
||||
HEADERS,
|
||||
SETTINGS_KEYS,
|
||||
VOICE_MODE_MESSAGE,
|
||||
VOICE_NOT_SUPPORTED_STATUS,
|
||||
VOICE_RELATIVE_PATH_PATTERN
|
||||
} from '#lib/constants/index.ts';
|
||||
import type { SettingsConfigType } from '#lib/types/index.ts';
|
||||
import { getAuthHeaders } from '#lib/utils/api-headers.ts';
|
||||
import { withBase } from '#lib/utils/base-path.ts';
|
||||
|
||||
/** What a feature asks of the voice wiring, besides the settings. */
|
||||
export interface VoiceSessionRequest {
|
||||
/** The model that transcribes, from resolveVoiceMode(); null leaves it to the server */
|
||||
model: string | null;
|
||||
/** Router mode: the model goes in the request, as the router routes on it */
|
||||
isRouter: boolean;
|
||||
/** False records without transcribing, e.g. a voice message when transcripts are off */
|
||||
transcribe?: boolean;
|
||||
/**
|
||||
* The microphone processing to ask for, in place of what the Voice settings switch on,
|
||||
* e.g. echo cancellation for a spoken conversation that plays replies aloud
|
||||
*/
|
||||
constraints?: MediaTrackConstraints;
|
||||
}
|
||||
|
||||
/**
|
||||
* The session options voice features share: where to capture from, and how to transcribe.
|
||||
* The capture is always plain options, so a feature can also open a VoiceCapture of its own
|
||||
* with them.
|
||||
*/
|
||||
export type VoiceSessionWiring = Pick<VoiceSessionOptions, 'realtime' | 'transcribe'> & {
|
||||
capture: VoiceCaptureOptions;
|
||||
};
|
||||
|
||||
/**
|
||||
* **VoiceService** - How the voice engine (@ggml-org/llama-voice) reaches this app's server and settings
|
||||
*
|
||||
* The engine imports no app code: it is handed a VoiceServerContext and plain capture
|
||||
* options. This builds both from the active server and the Voice settings, once for every
|
||||
* feature that records (the chat form's dictation and voice messages, voice mode), so they
|
||||
* reach the server the same way and follow the same settings.
|
||||
*/
|
||||
export class VoiceService {
|
||||
/**
|
||||
* Why a transcription failed, where the app can say more than the server did, else null.
|
||||
* A model that only transcribes live (Voxtral Realtime) has no HTTP route, and the
|
||||
* realtime socket is left out for a server with an API key (see serverContext), so
|
||||
* there it cannot transcribe at all.
|
||||
*/
|
||||
static failureReason(error: unknown): string | null {
|
||||
const liveOnly =
|
||||
error instanceof VoiceTranscriptionError && error.status === VOICE_NOT_SUPPORTED_STATUS;
|
||||
|
||||
return liveOnly && !VoiceService.serverContext().realtime
|
||||
? VOICE_MODE_MESSAGE.realtimeOnlyBehindKey
|
||||
: null;
|
||||
}
|
||||
|
||||
/** The transcription language from the Voice settings, undefined to let the model tell. */
|
||||
static language(config: SettingsConfigType): string | undefined {
|
||||
return String(config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE] ?? '').trim() || undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The microphone processing the Voice settings switch on, laid over the engine's raw
|
||||
* signal; undefined when none is.
|
||||
*/
|
||||
static processing(config: SettingsConfigType): MediaTrackConstraints | undefined {
|
||||
const processing: MediaTrackConstraints = {
|
||||
...(config[SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION] === true ? { noiseSuppression: true } : {}),
|
||||
...(config[SETTINGS_KEYS.VOICE_ECHO_CANCELLATION] === true ? { echoCancellation: true } : {}),
|
||||
...(config[SETTINGS_KEYS.VOICE_AUTO_GAIN] === true ? { autoGainControl: true } : {})
|
||||
};
|
||||
|
||||
return Object.keys(processing).length > 0 ? processing : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The server's routes and auth for the engine, and whether its realtime socket can be
|
||||
* used: a browser websocket can not send the API key, so a server that needs one gets
|
||||
* HTTP only.
|
||||
*/
|
||||
static serverContext(): { context: VoiceServerContext; realtime: boolean } {
|
||||
const hasAuth = Object.keys(getAuthHeaders()).some(
|
||||
(key) => key.toLowerCase() === HEADERS.AUTHORIZATION.toLowerCase()
|
||||
);
|
||||
|
||||
return {
|
||||
context: {
|
||||
getHeaders: () => getAuthHeaders(),
|
||||
resolveUrl: (path) => withBase(path.replace(VOICE_RELATIVE_PATH_PATTERN, ''))
|
||||
},
|
||||
realtime: !hasAuth
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The capture, realtime and transcribe options of a voice session, from the Voice
|
||||
* settings (microphone, processing, boost, language) and the active server. Async:
|
||||
* picking the microphone may list the devices.
|
||||
*/
|
||||
static async sessionOptions(
|
||||
config: SettingsConfigType,
|
||||
request: VoiceSessionRequest
|
||||
): Promise<VoiceSessionWiring> {
|
||||
const { context, realtime } = VoiceService.serverContext();
|
||||
const deviceId = await resolveMicrophone(
|
||||
String(config[SETTINGS_KEYS.VOICE_MICROPHONE] ?? VOICE_MICROPHONE_AUTO)
|
||||
);
|
||||
const constraints = request.constraints ?? VoiceService.processing(config);
|
||||
|
||||
return {
|
||||
capture: {
|
||||
boost: config[SETTINGS_KEYS.VOICE_BOOST] !== false,
|
||||
...(deviceId ? { deviceId } : {}),
|
||||
...(constraints ? { constraints } : {})
|
||||
},
|
||||
realtime,
|
||||
transcribe:
|
||||
request.transcribe === false
|
||||
? undefined
|
||||
: {
|
||||
context,
|
||||
language: VoiceService.language(config),
|
||||
model: request.isRouter && request.model ? request.model : undefined
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -347,8 +347,10 @@ class AgenticStore {
|
||||
|
||||
console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`);
|
||||
|
||||
const normalizedMessages: ApiChatMessageData[] =
|
||||
await ChatService.normalizeMessagesForApi(messages);
|
||||
const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi(
|
||||
messages,
|
||||
options.model
|
||||
);
|
||||
|
||||
this.updateSession(conversationId, {
|
||||
currentTurn: 0,
|
||||
|
||||
@@ -1480,7 +1480,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
|
||||
const allMessages = await conversationsStore.getConversationMessages(convId);
|
||||
const branchPath = [...filterByLeafNodeId(allMessages, leafId, false)];
|
||||
const request = await agenticStore.requestFor(
|
||||
await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath)),
|
||||
await ChatService.normalizeMessagesForApi(sliceAtLastCompaction(branchPath), model),
|
||||
this.toolPolicy()
|
||||
);
|
||||
|
||||
@@ -1646,7 +1646,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
|
||||
// message shaping, cut before the preserved steps: the server
|
||||
// reuses the prefix it cached and only the instruction is new.
|
||||
const request = await agenticStore.requestFor(
|
||||
await ChatService.normalizeMessagesForApi(source.slice(0, cut)),
|
||||
await ChatService.normalizeMessagesForApi(source.slice(0, cut), effectiveModel),
|
||||
this.toolPolicy()
|
||||
);
|
||||
const configValue = settingsStore.config;
|
||||
|
||||
@@ -61,6 +61,10 @@ export { permissionsStore } from './permissions.svelte';
|
||||
// TOOLS
|
||||
export { toolsStore } from './tools.svelte';
|
||||
|
||||
// VOICE (which model does what, and transcripts of voice messages on their way)
|
||||
export { voiceStore } from './voice.svelte';
|
||||
export { voiceTranscriptsStore } from './voice-transcripts.svelte';
|
||||
|
||||
// ENVIRONMENT / META
|
||||
export { versionStore } from './version.svelte';
|
||||
|
||||
|
||||
@@ -36,12 +36,35 @@ class ModelsDiscoverStore {
|
||||
private catalog: HfCatalogEntry[] = [];
|
||||
/** Repo id -> size range, for catalog rows and lazily measured search rows. */
|
||||
private catalogSizeRanges = new SvelteMap<string, ModelsDiscoverSizeRange>();
|
||||
private defaultModels: HfModelInfo[] = [];
|
||||
private defaultModels = $state.raw<HfModelInfo[]>([]);
|
||||
private fetched = false;
|
||||
private searchRequestId = 0;
|
||||
/** In-flight `sizeRange()` lookups, keyed by repo id. */
|
||||
private sizeRangePending = new Map<string, Promise<ModelsDiscoverSizeRange | undefined>>();
|
||||
|
||||
/** The curated catalog repos as they land, whatever a search shows right now. */
|
||||
get catalogModels(): HfModelInfo[] {
|
||||
return this.defaultModels;
|
||||
}
|
||||
|
||||
/**
|
||||
* Repos published by their base model's own org first (e.g. `Qwen/Qwen3-8B-GGUF`
|
||||
* ahead of third-party quants of `Qwen/Qwen3-8B`); the hub's order holds otherwise.
|
||||
* Every Hub search ranks its results this way.
|
||||
*/
|
||||
baseOrgFirst(models: HfModelInfo[]): HfModelInfo[] {
|
||||
// org names are case-insensitive on the hub, and base_model tags are not always canonical
|
||||
const isBaseOrg = (model: HfModelInfo) => {
|
||||
const org = orgOf(model.id).toLowerCase();
|
||||
|
||||
return HuggingFaceService.getBaseModels(model).some(
|
||||
(base) => orgOf(base).toLowerCase() === org
|
||||
);
|
||||
};
|
||||
|
||||
return [...models.filter(isBaseOrg), ...models.filter((model) => !isBaseOrg(model))];
|
||||
}
|
||||
|
||||
/**
|
||||
* Cached size range for a repo, without measuring: the synchronous part of
|
||||
* `sizeRange()`, for rendering a row before its measurement resolves.
|
||||
@@ -102,7 +125,10 @@ class ModelsDiscoverStore {
|
||||
this.catalogSizeRanges.set(build.repo, this.sizeRangeFor(build, tree));
|
||||
|
||||
// the catalog repo id, not the HF response `id`, drives selection
|
||||
this.defaultModels.push({ ...info, id: build.repo, modelId: build.repo });
|
||||
this.defaultModels = [
|
||||
...this.defaultModels,
|
||||
{ ...info, id: build.repo, modelId: build.repo }
|
||||
];
|
||||
}
|
||||
|
||||
// show what has landed, unless a search owns the list right now
|
||||
@@ -181,23 +207,6 @@ class ModelsDiscoverStore {
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* Repos published by their base model's own org first (e.g. `Qwen/Qwen3-8B-GGUF`
|
||||
* ahead of third-party quants of `Qwen/Qwen3-8B`); the hub's order holds otherwise.
|
||||
*/
|
||||
private baseOrgFirst(models: HfModelInfo[]): HfModelInfo[] {
|
||||
// org names are case-insensitive on the hub, and base_model tags are not always canonical
|
||||
const isBaseOrg = (model: HfModelInfo) => {
|
||||
const org = orgOf(model.id).toLowerCase();
|
||||
|
||||
return HuggingFaceService.getBaseModels(model).some(
|
||||
(base) => orgOf(base).toLowerCase() === org
|
||||
);
|
||||
};
|
||||
|
||||
return [...models.filter(isBaseOrg), ...models.filter((model) => !isBaseOrg(model))];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sizes the catalog lists under this repo. Never empty: an entry with no parsable
|
||||
* size contributes the build's own.
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* voiceTranscriptsStore - Transcripts of voice messages that are on their way
|
||||
*
|
||||
* A voice message is attached as soon as the recording stops, while its transcript
|
||||
* can take a few seconds more. This keeps each pending transcript by the clip's name
|
||||
* (unique per recording), so the chip in the form and the sent message show its
|
||||
* state, a send picks it up once it is there, and a message sent before it arrived
|
||||
* gets it written into its stored attachment afterwards. Kept in memory only.
|
||||
*/
|
||||
|
||||
import { VoiceTranscriptStatus } from '@ggml-org/llama-ui/voice';
|
||||
import { VOICE_TRANSCRIPT_APPLY_RETRY_DELAYS_MS } from '#lib/constants/index.ts';
|
||||
import { AttachmentType } from '#lib/enums/index.ts';
|
||||
import { DatabaseService } from '#lib/services/database.service.ts';
|
||||
import { VoiceService } from '#lib/services/voice.service.ts';
|
||||
// direct imports between stores, not via the barrel, to avoid circular deps
|
||||
import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts';
|
||||
import type {
|
||||
DatabaseMessage,
|
||||
DatabaseMessageExtra,
|
||||
DatabaseMessageExtraAudioFile
|
||||
} from '#lib/types/index.ts';
|
||||
import { SvelteMap } from 'svelte/reactivity';
|
||||
|
||||
export type VoiceTranscriptState =
|
||||
| { status: VoiceTranscriptStatus.TRANSCRIBING }
|
||||
| { status: VoiceTranscriptStatus.DONE; transcript: string }
|
||||
| {
|
||||
status: VoiceTranscriptStatus.ERROR;
|
||||
message: string;
|
||||
/** Why, in words for the user, when the app can tell (VoiceService.failureReason) */
|
||||
reason: string | null;
|
||||
};
|
||||
|
||||
function isPendingClip(extra: DatabaseMessageExtra, name: string): boolean {
|
||||
return (
|
||||
extra.type === AttachmentType.AUDIO &&
|
||||
extra.name === name &&
|
||||
extra.voice === true &&
|
||||
extra.transcript === undefined
|
||||
);
|
||||
}
|
||||
|
||||
/** The extras with the clip's transcript filled in, or null when none of them is that clip. */
|
||||
export function withTranscript(
|
||||
extras: DatabaseMessageExtra[] | undefined,
|
||||
name: string,
|
||||
transcript: string
|
||||
): DatabaseMessageExtra[] | null {
|
||||
if (!extras?.some((extra) => isPendingClip(extra, name))) return null;
|
||||
|
||||
return extras.map((extra) =>
|
||||
isPendingClip(extra, name)
|
||||
? ({ ...(extra as DatabaseMessageExtraAudioFile), transcript } as DatabaseMessageExtra)
|
||||
: extra
|
||||
);
|
||||
}
|
||||
|
||||
class VoiceTranscriptsStore {
|
||||
// clips whose transcript the user wrote while recording: the model's does not replace it
|
||||
private edited = new Set<string>();
|
||||
private entries = new SvelteMap<string, VoiceTranscriptState>();
|
||||
private recordedAt = new Map<string, number>();
|
||||
|
||||
/** Drops a clip removed from the form before it was sent, or whose message has its transcript. */
|
||||
forget(name: string): void {
|
||||
this.entries.delete(name);
|
||||
this.recordedAt.delete(name);
|
||||
this.edited.delete(name);
|
||||
}
|
||||
|
||||
/** The state of a clip's transcript, undefined when none was asked for in this session. */
|
||||
get(name: string): VoiceTranscriptState | undefined {
|
||||
return this.entries.get(name);
|
||||
}
|
||||
|
||||
/** Sets a clip's transcript to the text the user edited while recording; the model's is ignored. */
|
||||
setTranscript(name: string, transcript: string): void {
|
||||
this.edited.add(name);
|
||||
this.recordedAt.set(name, this.recordedAt.get(name) ?? Date.now());
|
||||
this.entries.set(name, { status: VoiceTranscriptStatus.DONE, transcript });
|
||||
}
|
||||
|
||||
/** A clip's transcript as it is shown: the one stored with it, else the one on its way. */
|
||||
shown(
|
||||
name: string,
|
||||
stored?: string
|
||||
): { status: VoiceTranscriptStatus; transcript?: string; errorReason?: string | null } {
|
||||
const transcript = stored ?? this.transcriptOf(name);
|
||||
|
||||
if (transcript !== undefined) return { status: VoiceTranscriptStatus.DONE, transcript };
|
||||
|
||||
const entry = this.entries.get(name);
|
||||
|
||||
return {
|
||||
errorReason: entry?.status === VoiceTranscriptStatus.ERROR ? entry.reason : null,
|
||||
status: entry?.status ?? VoiceTranscriptStatus.IDLE
|
||||
};
|
||||
}
|
||||
|
||||
/** Follows a clip's transcript, and writes it into the message it went out with. */
|
||||
track(name: string, transcript: Promise<string>): void {
|
||||
this.entries.set(name, { status: VoiceTranscriptStatus.TRANSCRIBING });
|
||||
this.recordedAt.set(name, Date.now());
|
||||
|
||||
transcript.then(
|
||||
(text) => {
|
||||
if (!this.entries.has(name) || this.edited.has(name)) return;
|
||||
|
||||
this.entries.set(name, { status: VoiceTranscriptStatus.DONE, transcript: text });
|
||||
void this.applyToMessage(name, text);
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (!this.entries.has(name) || this.edited.has(name)) return;
|
||||
|
||||
console.warn('Voice message transcription failed:', error);
|
||||
this.entries.set(name, {
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
reason: VoiceService.failureReason(error),
|
||||
status: VoiceTranscriptStatus.ERROR
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/** The transcript once it is there. */
|
||||
transcriptOf(name: string): string | undefined {
|
||||
const entry = this.entries.get(name);
|
||||
|
||||
return entry?.status === VoiceTranscriptStatus.DONE ? entry.transcript : undefined;
|
||||
}
|
||||
|
||||
private async applyToMessage(name: string, transcript: string): Promise<void> {
|
||||
for (const delay of [0, ...VOICE_TRANSCRIPT_APPLY_RETRY_DELAYS_MS]) {
|
||||
if (delay) await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
|
||||
// in its message now, which shows it from there
|
||||
if (await this.writeTranscript(name, transcript)) return this.forget(name);
|
||||
}
|
||||
// not sent (still in the form, or dropped): a send reads it from here
|
||||
}
|
||||
|
||||
private async writeTranscript(name: string, transcript: string): Promise<boolean> {
|
||||
const active = (conversationsStore.activeMessages as DatabaseMessage[]).find((message) =>
|
||||
message.extra?.some((extra) => isPendingClip(extra, name))
|
||||
);
|
||||
|
||||
if (active) {
|
||||
// plain objects: IndexedDB cannot clone the state proxies
|
||||
const extra = withTranscript($state.snapshot(active.extra), name, transcript)!;
|
||||
|
||||
await DatabaseService.updateMessage(active.id, { extra });
|
||||
conversationsStore.updateMessageAtIndex(conversationsStore.findMessageIndex(active.id), {
|
||||
extra
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// the chat was left before the transcript came back: the message is newer than the clip
|
||||
const since = this.recordedAt.get(name) ?? Date.now();
|
||||
const stored = (await DatabaseService.getMessagesSince(since)).find(
|
||||
(message: DatabaseMessage) => message.extra?.some((extra) => isPendingClip(extra, name))
|
||||
);
|
||||
|
||||
if (!stored) return false;
|
||||
|
||||
await DatabaseService.updateMessage(stored.id, {
|
||||
extra: withTranscript(stored.extra, name, transcript)!
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export const voiceTranscriptsStore = new VoiceTranscriptsStore();
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* voiceStore - Which model does what in voice input
|
||||
*
|
||||
* One place for what the microphone does with the conversation's model (a voice message,
|
||||
* dictation, or nothing, see resolveVoiceMode) and for the model that transcribes
|
||||
* dictation, so the chat form, the model picker and the Models manager agree on it: where
|
||||
* models are listed, a badge says what kind of hearing model each is (roleOf) and a
|
||||
* microphone which one transcribes dictation (isDictationModel), and picks it.
|
||||
*/
|
||||
|
||||
import { SETTINGS_KEYS, VOICE_TRANSCRIPTION_MODEL_AUTO } from '#lib/constants/index.ts';
|
||||
import { VoiceInputMode, VoiceModelRole } from '#lib/enums/index.ts';
|
||||
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
|
||||
// direct imports between stores, not via the barrel, to avoid circular deps
|
||||
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
|
||||
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
|
||||
import { dictationModelFor, resolveVoiceMode, voiceModelRole } from '#lib/utils/voice-mode.ts';
|
||||
|
||||
class VoiceStore {
|
||||
/** A router with a model that hears: the microphone stays, so it can say where to pick one. */
|
||||
readonly anyAudioModel = $derived.by(() => {
|
||||
void modelsStore.props.cacheVersion;
|
||||
|
||||
return (
|
||||
serverStore.isRouterMode && modelsStore.models.some((option) => this.hasAudio(option.model))
|
||||
);
|
||||
});
|
||||
private readonly config = $derived(settingsStore.config);
|
||||
|
||||
private readonly transcriptionSetting = $derived(
|
||||
String(this.config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL] ?? VOICE_TRANSCRIPTION_MODEL_AUTO)
|
||||
);
|
||||
|
||||
/** The dictation model is picked automatically, not chosen in Settings > Voice. */
|
||||
readonly dictationIsAutomatic = $derived(
|
||||
this.transcriptionSetting.trim() === VOICE_TRANSCRIPTION_MODEL_AUTO
|
||||
);
|
||||
|
||||
/** Router mode: what Automatic transcribes with when the chat model can't hear, a loaded model with audio input. */
|
||||
readonly loadedAudioModel = $derived.by(() => {
|
||||
void modelsStore.props.cacheVersion;
|
||||
|
||||
if (!serverStore.isRouterMode) return null;
|
||||
|
||||
return (
|
||||
modelsStore.models.find(
|
||||
(option) => this.hasAudio(option.model) && modelsStore.isModelLoaded(option.model)
|
||||
)?.model ?? null
|
||||
);
|
||||
});
|
||||
|
||||
/** The model that transcribes dictation for chat models that can't hear, null when none can. */
|
||||
readonly dictationModel = $derived.by(() => {
|
||||
const singleModel = serverStore.isRouterMode ? null : modelsStore.activeModelId;
|
||||
|
||||
return dictationModelFor({
|
||||
isRouter: serverStore.isRouterMode,
|
||||
loadedAudioModel: this.loadedAudioModel,
|
||||
singleModel,
|
||||
singleModelHasAudio: this.hasAudio(singleModel),
|
||||
transcriptionModel: this.transcriptionSetting,
|
||||
transcriptionModelHasAudio: this.hasAudio(this.transcriptionSetting || null)
|
||||
});
|
||||
});
|
||||
|
||||
/** What the microphone does for the conversation's model. */
|
||||
readonly resolution = $derived.by(() => {
|
||||
const chatModel = modelsStore.activeModelId;
|
||||
|
||||
return resolveVoiceMode({
|
||||
chatModel,
|
||||
chatModelHasAudio: this.hasAudio(chatModel),
|
||||
isRouter: serverStore.isRouterMode,
|
||||
loadedAudioModel: this.loadedAudioModel,
|
||||
setting: String(this.config[SETTINGS_KEYS.VOICE_INPUT_MODE] ?? VoiceInputMode.AUTO),
|
||||
transcriptionModel: this.transcriptionSetting,
|
||||
transcriptionModelHasAudio: this.hasAudio(this.transcriptionSetting || null)
|
||||
});
|
||||
});
|
||||
|
||||
/** Whether a model has audio input, as far as its props say. */
|
||||
hasAudio(model: string | null): boolean {
|
||||
void modelsStore.props.cacheVersion;
|
||||
|
||||
return model ? modelsStore.props.modelSupportsAudio(model) : false;
|
||||
}
|
||||
|
||||
/** Whether a model is the one that transcribes dictation for chat models that can't hear. */
|
||||
isDictationModel(model: string): boolean {
|
||||
return this.dictationModel === model;
|
||||
}
|
||||
|
||||
/** What kind of model with audio input a model is, null when it cannot hear. */
|
||||
roleOf(model: string): VoiceModelRole | null {
|
||||
return voiceModelRole(model, this.hasAudio(model));
|
||||
}
|
||||
|
||||
/**
|
||||
* What a click on a model's microphone does: makes it the model that transcribes
|
||||
* dictation, or, on the one already chosen, hands the choice back to Automatic.
|
||||
*/
|
||||
toggleDictation(model: string): void {
|
||||
const chosen = !this.dictationIsAutomatic && this.transcriptionSetting.trim() === model;
|
||||
|
||||
settingsStore.updateConfig(
|
||||
SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL,
|
||||
chosen ? VOICE_TRANSCRIPTION_MODEL_AUTO : model
|
||||
);
|
||||
}
|
||||
|
||||
/** Makes a model the one that transcribes dictation (Settings > Voice > Transcription model). */
|
||||
useForDictation(model: string): void {
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL, model);
|
||||
}
|
||||
}
|
||||
|
||||
export const voiceStore = new VoiceStore();
|
||||
Vendored
+8
@@ -48,6 +48,14 @@ export interface ChatUploadedFile {
|
||||
};
|
||||
isLoading?: boolean;
|
||||
loadError?: string;
|
||||
/** A voice message recorded in the chat form. */
|
||||
voice?: ChatUploadedVoice;
|
||||
}
|
||||
|
||||
/** What a voice message recorded in the chat form carries besides its WAV file. */
|
||||
export interface ChatUploadedVoice {
|
||||
durationMs: number;
|
||||
peaks: number[];
|
||||
}
|
||||
|
||||
export interface ChatAttachmentDisplayItem {
|
||||
|
||||
+7
@@ -38,6 +38,13 @@ export interface DatabaseMessageExtraAudioFile {
|
||||
size?: number;
|
||||
base64Data: string;
|
||||
mimeType: string;
|
||||
/** Recorded in the app as a voice message, as opposed to an uploaded audio file. */
|
||||
voice?: true;
|
||||
/** What was said, for reading; a model without audio input gets it in place of the audio. */
|
||||
transcript?: string;
|
||||
/** Normalised 0..1 waveform of the clip; computed from the audio when left out. */
|
||||
peaks?: number[];
|
||||
durationMs?: number;
|
||||
}
|
||||
|
||||
export interface DatabaseMessageExtraVideoFile {
|
||||
|
||||
@@ -57,6 +57,7 @@ export type {
|
||||
export type {
|
||||
AttachmentMenuItem,
|
||||
ChatUploadedFile,
|
||||
ChatUploadedVoice,
|
||||
ChatAttachmentDisplayItem,
|
||||
ChatMessageSiblingInfo,
|
||||
ChatMessageActions,
|
||||
|
||||
@@ -1,158 +1,5 @@
|
||||
import { MICROPHONE_UNAVAILABLE_MESSAGE } from '#lib/constants/index.ts';
|
||||
import { MimeTypeAudio } from '#lib/enums/index.ts';
|
||||
|
||||
/**
|
||||
* AudioRecorder - Browser-based audio recording with MediaRecorder API
|
||||
*
|
||||
* This class provides a complete audio recording solution using the browser's MediaRecorder API.
|
||||
* It handles microphone access, recording state management, and audio format optimization.
|
||||
*
|
||||
* **Features:**
|
||||
* - Automatic microphone permission handling
|
||||
* - Audio enhancement (echo cancellation, noise suppression, auto gain)
|
||||
* - Multiple format support with fallback (WAV, WebM, MP4, AAC)
|
||||
* - Real-time recording state tracking
|
||||
* - Proper cleanup and resource management
|
||||
*/
|
||||
export class AudioRecorder {
|
||||
private audioChunks: Blob[] = [];
|
||||
private mediaRecorder: MediaRecorder | null = null;
|
||||
private recordingState: boolean = false;
|
||||
private stream: MediaStream | null = null;
|
||||
|
||||
cancelRecording(): void {
|
||||
const recorder = this.mediaRecorder;
|
||||
const stream = this.stream;
|
||||
|
||||
this.mediaRecorder = null;
|
||||
this.audioChunks = [];
|
||||
this.stream = null;
|
||||
this.recordingState = false;
|
||||
|
||||
if (recorder && recorder.state !== 'inactive') {
|
||||
// Drop the original handlers so the pending stop event does not touch the instance
|
||||
recorder.onstop = null;
|
||||
recorder.onerror = null;
|
||||
recorder.stop();
|
||||
}
|
||||
|
||||
if (stream) {
|
||||
for (const track of stream.getTracks()) {
|
||||
track.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isRecording(): boolean {
|
||||
return this.recordingState;
|
||||
}
|
||||
|
||||
async startRecording(): Promise<void> {
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true
|
||||
}
|
||||
});
|
||||
|
||||
this.initializeRecorder(this.stream);
|
||||
|
||||
this.audioChunks = [];
|
||||
// Start recording with a small timeslice to ensure we get data
|
||||
this.mediaRecorder!.start(100);
|
||||
this.recordingState = true;
|
||||
} catch (error) {
|
||||
console.error('Failed to start recording:', error);
|
||||
|
||||
// the browser's own error, which says why: see describeMicrophoneError
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async stopRecording(): Promise<Blob> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const recorder = this.mediaRecorder;
|
||||
const chunks = this.audioChunks;
|
||||
const stream = this.stream;
|
||||
|
||||
if (!recorder || recorder.state === 'inactive') {
|
||||
reject(new Error('No active recording to stop'));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Detach instance state right away so a new startRecording can take over without race
|
||||
this.mediaRecorder = null;
|
||||
this.audioChunks = [];
|
||||
this.stream = null;
|
||||
this.recordingState = false;
|
||||
|
||||
recorder.onstop = () => {
|
||||
const audioBlob = new Blob(chunks, {
|
||||
type: recorder.mimeType || MimeTypeAudio.WAV
|
||||
});
|
||||
|
||||
if (stream) {
|
||||
for (const track of stream.getTracks()) {
|
||||
track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
resolve(audioBlob);
|
||||
};
|
||||
|
||||
recorder.onerror = (event) => {
|
||||
console.error('Recording error:', event);
|
||||
|
||||
if (stream) {
|
||||
for (const track of stream.getTracks()) {
|
||||
track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
reject(new Error('Recording failed'));
|
||||
};
|
||||
|
||||
recorder.stop();
|
||||
});
|
||||
}
|
||||
|
||||
private initializeRecorder(stream: MediaStream): void {
|
||||
const options: MediaRecorderOptions = {};
|
||||
|
||||
if (MediaRecorder.isTypeSupported(MimeTypeAudio.WAV)) {
|
||||
options.mimeType = MimeTypeAudio.WAV;
|
||||
} else if (MediaRecorder.isTypeSupported(MimeTypeAudio.WEBM_OPUS)) {
|
||||
options.mimeType = MimeTypeAudio.WEBM_OPUS;
|
||||
} else if (MediaRecorder.isTypeSupported(MimeTypeAudio.WEBM)) {
|
||||
options.mimeType = MimeTypeAudio.WEBM;
|
||||
} else if (MediaRecorder.isTypeSupported(MimeTypeAudio.MP4)) {
|
||||
options.mimeType = MimeTypeAudio.MP4;
|
||||
} else {
|
||||
console.warn('No preferred audio format supported, using default');
|
||||
}
|
||||
|
||||
this.mediaRecorder = new MediaRecorder(stream, options);
|
||||
|
||||
this.mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
this.audioChunks.push(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
this.mediaRecorder.onstop = () => {
|
||||
this.recordingState = false;
|
||||
};
|
||||
|
||||
this.mediaRecorder.onerror = (event) => {
|
||||
console.error('MediaRecorder error:', event);
|
||||
this.recordingState = false;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function convertToWav(audioBlob: Blob): Promise<Blob> {
|
||||
try {
|
||||
if (audioBlob.type.includes('wav')) {
|
||||
@@ -249,51 +96,3 @@ export function createAudioFile(audioBlob: Blob, filename?: string): File {
|
||||
type: audioBlob.type
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if audio recording is supported in the current browser
|
||||
* @returns True if MediaRecorder and getUserMedia are available
|
||||
*/
|
||||
export function isAudioRecordingSupported(): boolean {
|
||||
return !!(
|
||||
typeof navigator !== 'undefined' &&
|
||||
navigator.mediaDevices &&
|
||||
typeof navigator.mediaDevices.getUserMedia === 'function' &&
|
||||
typeof window !== 'undefined' &&
|
||||
window.MediaRecorder
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Why this page can't record at all, or null when it can: a reason no tap will fix. Browsers
|
||||
* only expose the microphone to a secure context, HTTPS or localhost, so a llama-server
|
||||
* opened over plain HTTP on the local network or over Tailscale can't record.
|
||||
*/
|
||||
export function microphoneUnavailableReason(): string | null {
|
||||
if (typeof window === 'undefined') return null;
|
||||
|
||||
if (!window.isSecureContext) return MICROPHONE_UNAVAILABLE_MESSAGE.insecure;
|
||||
|
||||
if (!isAudioRecordingSupported()) return MICROPHONE_UNAVAILABLE_MESSAGE.unsupported;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Why the microphone would not start, from the error the browser raised. */
|
||||
export function describeMicrophoneError(error: unknown): string {
|
||||
switch (error instanceof DOMException ? error.name : '') {
|
||||
case 'NotAllowedError':
|
||||
case 'SecurityError':
|
||||
return MICROPHONE_UNAVAILABLE_MESSAGE.denied;
|
||||
case 'NotFoundError':
|
||||
case 'OverconstrainedError':
|
||||
return MICROPHONE_UNAVAILABLE_MESSAGE.notFound;
|
||||
case 'AbortError':
|
||||
case 'NotReadableError':
|
||||
return MICROPHONE_UNAVAILABLE_MESSAGE.busy;
|
||||
default:
|
||||
return error instanceof Error && error.message
|
||||
? `${MICROPHONE_UNAVAILABLE_MESSAGE.failed} ${error.message}`
|
||||
: MICROPHONE_UNAVAILABLE_MESSAGE.failed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,15 +6,8 @@
|
||||
* only in client-side code or components that are not server-rendered.
|
||||
*/
|
||||
|
||||
// Audio utilities (MediaRecorder API)
|
||||
export {
|
||||
AudioRecorder,
|
||||
convertToWav,
|
||||
createAudioFile,
|
||||
describeMicrophoneError,
|
||||
isAudioRecordingSupported,
|
||||
microphoneUnavailableReason
|
||||
} from './audio-recording';
|
||||
// Audio file utilities (Web Audio decoding); microphone support lives in @ggml-org/llama-voice
|
||||
export { convertToWav, createAudioFile } from './audio-recording';
|
||||
|
||||
// PDF processing utilities (pdfjs-dist with DOMMatrix)
|
||||
export {
|
||||
|
||||
@@ -110,6 +110,16 @@ export function formatMessageForClipboard(
|
||||
extras?: DatabaseMessageExtra[],
|
||||
asPlainText: boolean = false
|
||||
): string {
|
||||
// what was said in a voice message reads as the message's text
|
||||
const transcripts =
|
||||
extras?.flatMap((extra) =>
|
||||
extra.type === AttachmentType.AUDIO && extra.transcript ? [extra.transcript] : []
|
||||
) ?? [];
|
||||
|
||||
if (transcripts.length > 0) {
|
||||
content = [...transcripts, content].filter((part) => part.trim()).join('\n\n');
|
||||
}
|
||||
|
||||
// Filter text-like attachments (TEXT, LEGACY_CONTEXT, MCP_PROMPT, and MCP_RESOURCE types)
|
||||
const textAttachments =
|
||||
extras?.filter(
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { CODE_BLOCK, NEWLINE } from '#lib/constants/index.ts';
|
||||
|
||||
export { highlightCode, trimCodePadding } from '@ggml-org/llama-ui/code';
|
||||
export {
|
||||
getLoadedHighlightLanguages,
|
||||
highlightCode,
|
||||
isHighlightLanguageLoaded,
|
||||
loadHighlightLanguages,
|
||||
resolveHighlightLanguage,
|
||||
trimCodePadding
|
||||
} from '@ggml-org/llama-ui/code';
|
||||
|
||||
export interface IncompleteCodeBlock {
|
||||
language: string;
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Matching for the search dialog's message search: which text of a message is searched and the
|
||||
* snippet shown for a match. No IndexedDB access here, the service feeds it messages.
|
||||
*/
|
||||
|
||||
import {
|
||||
CONVERSATION_SEARCH_SNIPPET_LEAD,
|
||||
CONVERSATION_SEARCH_SNIPPET_MAX_CHARS
|
||||
} from '#lib/constants/index.ts';
|
||||
import type { DatabaseMessage } from '#lib/types/database.js';
|
||||
|
||||
export interface ConversationSearchSnippet {
|
||||
/** Text before the match, with a leading `...` when cut */
|
||||
before: string;
|
||||
match: string;
|
||||
/** Text after the match, with a trailing `...` when cut */
|
||||
after: string;
|
||||
}
|
||||
|
||||
/** Tool call names and arguments, without the call ids */
|
||||
function toolCallTexts(toolCalls: string): string[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(toolCalls);
|
||||
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
return parsed.flatMap((call) => {
|
||||
const fn = call?.function;
|
||||
|
||||
if (!fn) return [];
|
||||
|
||||
return [fn.name, fn.arguments].filter(
|
||||
(value): value is string => typeof value === 'string' && value !== ''
|
||||
);
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every text a person could remember from a message, in reading order: reasoning, the content,
|
||||
* tool calls and the names and text of attachments. Ids, timestamps and base64 payloads are left out.
|
||||
*/
|
||||
export function searchableTexts(message: DatabaseMessage): string[] {
|
||||
const texts: string[] = [];
|
||||
|
||||
if (message.isSynthetic) return texts;
|
||||
|
||||
if (message.reasoningContent) texts.push(message.reasoningContent);
|
||||
else if (message.thinking) texts.push(message.thinking);
|
||||
|
||||
if (typeof message.content === 'string' && message.content) texts.push(message.content);
|
||||
|
||||
if (message.toolCalls) texts.push(...toolCallTexts(message.toolCalls));
|
||||
|
||||
for (const extra of message.extra ?? []) {
|
||||
if (extra.name) texts.push(extra.name);
|
||||
|
||||
if ('content' in extra && typeof extra.content === 'string' && extra.content) {
|
||||
texts.push(extra.content);
|
||||
}
|
||||
}
|
||||
|
||||
return texts;
|
||||
}
|
||||
|
||||
/** First match of `needle` (already lowercased) in the message's text, or null */
|
||||
export function findMessageMatch(
|
||||
message: DatabaseMessage,
|
||||
needle: string
|
||||
): { at: number; text: string } | null {
|
||||
if (!needle) return null;
|
||||
|
||||
for (const text of searchableTexts(message)) {
|
||||
const lower = text.toLowerCase();
|
||||
const at = lower.indexOf(needle);
|
||||
|
||||
// lowercasing can change the length (e.g. U+0130), then the offset does not map back
|
||||
if (at !== -1) return { at: lower.length === text.length ? at : -1, text };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** A one-line excerpt around the match, cut to `maxChars` with the match in it */
|
||||
export function matchSnippet(
|
||||
text: string,
|
||||
at: number,
|
||||
length: number,
|
||||
maxChars: number = CONVERSATION_SEARCH_SNIPPET_MAX_CHARS
|
||||
): ConversationSearchSnippet {
|
||||
const flat = (value: string) => value.replace(/\s+/g, ' ');
|
||||
|
||||
if (at < 0) {
|
||||
const whole = flat(text).trim();
|
||||
|
||||
return {
|
||||
after: whole.length > maxChars ? `${whole.slice(0, maxChars)}...` : whole,
|
||||
before: '',
|
||||
match: ''
|
||||
};
|
||||
}
|
||||
|
||||
const lead = Math.min(at, CONVERSATION_SEARCH_SNIPPET_LEAD);
|
||||
const start = at - lead;
|
||||
const end = Math.min(text.length, start + Math.max(maxChars, lead + length));
|
||||
const before = flat(text.slice(start, at)).trimStart();
|
||||
const after = flat(text.slice(at + length, end)).trimEnd();
|
||||
|
||||
return {
|
||||
after: end < text.length ? `${after}...` : after,
|
||||
before: start > 0 ? `...${before}` : before,
|
||||
match: flat(text.slice(at, at + length))
|
||||
};
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { SETTINGS_KEYS } from '#lib/constants/index.ts';
|
||||
import { AttachmentType, FileTypeCategory, SpecialFileType } from '#lib/enums/index.ts';
|
||||
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
|
||||
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
|
||||
import { voiceTranscriptsStore } from '#lib/stores/voice-transcripts.svelte.ts';
|
||||
import type {
|
||||
ChatUploadedFile,
|
||||
DatabaseMessageExtra,
|
||||
@@ -83,13 +84,21 @@ export async function parseFilesToMessageExtras(
|
||||
// Process audio files (MP3 and WAV)
|
||||
try {
|
||||
const base64Data = await readFileAsBase64(file.file);
|
||||
// a voice message keeps its waveform, and its transcript once it is there
|
||||
const transcript = file.voice ? voiceTranscriptsStore.transcriptOf(file.name) : undefined;
|
||||
|
||||
extras.push({
|
||||
base64Data: base64Data,
|
||||
mimeType: file.type,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: AttachmentType.AUDIO
|
||||
type: AttachmentType.AUDIO,
|
||||
...(file.voice && {
|
||||
durationMs: file.voice.durationMs,
|
||||
peaks: file.voice.peaks,
|
||||
voice: true as const,
|
||||
...(transcript !== undefined && { transcript })
|
||||
})
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`Failed to process audio file ${file.name}:`, error);
|
||||
|
||||
@@ -33,6 +33,8 @@ export {
|
||||
// Code
|
||||
export {
|
||||
highlightCode,
|
||||
isHighlightLanguageLoaded,
|
||||
loadHighlightLanguages,
|
||||
detectIncompleteCodeBlock,
|
||||
splitGluedClosingCodeFences,
|
||||
trimCodePadding,
|
||||
@@ -108,6 +110,7 @@ export {
|
||||
|
||||
// IME utilities
|
||||
export { isIMEComposing } from './is-ime-composing';
|
||||
export { shouldTypeToFocus } from './type-to-focus';
|
||||
|
||||
// LaTeX utilities
|
||||
export { maskInlineLaTeX, preprocessLaTeX } from './latex-protection';
|
||||
@@ -384,3 +387,15 @@ export { groupModelFamilies, modelFamilyKey, type ModelFamilyGroup } from './mod
|
||||
|
||||
// Svelte actions
|
||||
export { nearViewport } from './near-viewport';
|
||||
|
||||
// Voice: what a recording becomes for the conversation's model
|
||||
export {
|
||||
type DictationModelInput,
|
||||
dictationModelFor,
|
||||
isTranscriptionOnlyModel,
|
||||
resolveVoiceMode,
|
||||
voiceModeLabel,
|
||||
type VoiceModeInput,
|
||||
voiceModelRole,
|
||||
type VoiceModeResolution
|
||||
} from './voice-mode';
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { isIMEComposing } from './is-ime-composing';
|
||||
|
||||
const EDITABLE_SELECTOR =
|
||||
'input, textarea, select, [contenteditable]:not([contenteditable="false"])';
|
||||
// widgets that take typed letters themselves (typeahead) or trap focus
|
||||
const KEY_CAPTURING_SELECTOR =
|
||||
'[role="dialog"], [role="alertdialog"], [role="menu"], [role="listbox"]';
|
||||
|
||||
/**
|
||||
* Whether a keypress should move focus to the chat form so the character lands there:
|
||||
* a printable key typed while nothing that takes text has focus.
|
||||
*/
|
||||
export function shouldTypeToFocus(event: KeyboardEvent): boolean {
|
||||
if (event.defaultPrevented || event.ctrlKey || event.metaKey || event.altKey) return false;
|
||||
|
||||
if (isIMEComposing(event)) return false;
|
||||
|
||||
// a single character, not a named key; whitespace keeps scrolling or pressing the focused button
|
||||
if (event.key.length !== 1 || event.key.trim() === '') return false;
|
||||
|
||||
const active = document.activeElement;
|
||||
|
||||
if (active instanceof HTMLElement && active !== document.body) {
|
||||
if (active.closest(EDITABLE_SELECTOR) || active.closest(KEY_CAPTURING_SELECTOR)) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { base64ToBytes, computePeaksFromAudio } from '@ggml-org/llama-voice';
|
||||
import {
|
||||
VOICE_FETCHABLE_URL_PATTERN,
|
||||
VOICE_PEAKS_CACHE_TAIL_LENGTH,
|
||||
VOICE_URL_PATTERN
|
||||
} from '#lib/constants/index.ts';
|
||||
import { AttachmentType, MimeTypeAudio } from '#lib/enums/index.ts';
|
||||
import type { ChatAttachmentDisplayItem, DatabaseMessageExtraAudioFile } from '#lib/types/index.ts';
|
||||
|
||||
// peaks of clips stored without them, kept for the session only
|
||||
const peaksCache = new Map<string, Promise<number[]>>();
|
||||
|
||||
/** What a list item needs to draw an audio attachment as a clip. */
|
||||
export interface VoiceAttachmentSource {
|
||||
/** Base64 or a URL of the audio, or the file itself. */
|
||||
audio: string | Blob | null;
|
||||
durationMs?: number;
|
||||
mimeType: string;
|
||||
name: string;
|
||||
peaks?: number[];
|
||||
/** Recorded in the app, as opposed to an uploaded file. */
|
||||
voice: boolean;
|
||||
transcript?: string;
|
||||
}
|
||||
|
||||
/** The clip behind an audio item of the form or of a message. */
|
||||
export function voiceAttachmentSource(item: ChatAttachmentDisplayItem): VoiceAttachmentSource {
|
||||
const uploaded = item.uploadedFile;
|
||||
|
||||
if (uploaded) {
|
||||
return {
|
||||
audio: uploaded.preview ?? (uploaded.file.size > 0 ? uploaded.file : null),
|
||||
durationMs: uploaded.voice?.durationMs,
|
||||
mimeType: uploaded.type,
|
||||
name: uploaded.name,
|
||||
peaks: uploaded.voice?.peaks,
|
||||
voice: !!uploaded.voice
|
||||
};
|
||||
}
|
||||
|
||||
const attachment =
|
||||
item.attachment?.type === AttachmentType.AUDIO
|
||||
? (item.attachment as DatabaseMessageExtraAudioFile)
|
||||
: null;
|
||||
|
||||
return {
|
||||
audio: attachment?.base64Data || null,
|
||||
durationMs: attachment?.durationMs,
|
||||
mimeType: attachment?.mimeType ?? '',
|
||||
name: item.name,
|
||||
peaks: attachment?.peaks,
|
||||
transcript: attachment?.transcript,
|
||||
voice: attachment?.voice === true
|
||||
};
|
||||
}
|
||||
|
||||
/** A URL an <audio> can play: URLs as they are, base64 and files as object URLs; null for bad data. */
|
||||
export function audioUrl(
|
||||
audio: string | Blob,
|
||||
mimeType: string
|
||||
): { url: string; revoke: boolean } | null {
|
||||
// a data, blob or http URL plays as it is
|
||||
if (typeof audio === 'string' && VOICE_URL_PATTERN.test(audio)) {
|
||||
return { revoke: false, url: audio };
|
||||
}
|
||||
|
||||
if (typeof audio !== 'string') return { revoke: true, url: URL.createObjectURL(audio) };
|
||||
|
||||
let bytes: Uint8Array<ArrayBuffer>;
|
||||
|
||||
try {
|
||||
bytes = base64ToBytes(audio);
|
||||
} catch {
|
||||
// not base64: nothing to play
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
revoke: true,
|
||||
url: URL.createObjectURL(new Blob([bytes], { type: mimeType || MimeTypeAudio.WAV }))
|
||||
};
|
||||
}
|
||||
|
||||
/** Peaks of a clip stored without them, computed once per session. */
|
||||
export function lazyPeaks(name: string, audio: string | Blob): Promise<number[]> {
|
||||
const key =
|
||||
typeof audio === 'string'
|
||||
? `${name}:${audio.length}:${audio.slice(-VOICE_PEAKS_CACHE_TAIL_LENGTH)}`
|
||||
: `${name}:${audio.size}`;
|
||||
|
||||
let peaks = peaksCache.get(key);
|
||||
|
||||
if (!peaks) {
|
||||
const isUrl = typeof audio === 'string' && VOICE_FETCHABLE_URL_PATTERN.test(audio);
|
||||
const bytes = isUrl ? fetch(audio as string).then((r) => r.blob()) : Promise.resolve(audio);
|
||||
|
||||
peaks = bytes
|
||||
.then((source) => computePeaksFromAudio(source))
|
||||
.catch((error) => {
|
||||
console.warn(`Could not draw the waveform of ${name}:`, error);
|
||||
peaksCache.delete(key);
|
||||
|
||||
return [];
|
||||
});
|
||||
peaksCache.set(key, peaks);
|
||||
}
|
||||
|
||||
return peaks;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import {
|
||||
VOICE_MODE_MESSAGE,
|
||||
VOICE_TRANSCRIPTION_MODEL_AUTO,
|
||||
VOICE_TRANSCRIPTION_ONLY_PATTERN
|
||||
} from '#lib/constants/voice.constants.ts';
|
||||
import { VoiceInputMode, VoiceMode, VoiceModelRole } from '#lib/enums/index.ts';
|
||||
|
||||
export interface VoiceModeInput extends Omit<
|
||||
DictationModelInput,
|
||||
'singleModel' | 'singleModelHasAudio'
|
||||
> {
|
||||
/** The Voice input setting. */
|
||||
setting: VoiceInputMode | string;
|
||||
/** The conversation's effective model, null when none is picked. */
|
||||
chatModel: string | null;
|
||||
chatModelHasAudio: boolean;
|
||||
}
|
||||
|
||||
export type VoiceModeResolution =
|
||||
| {
|
||||
mode: VoiceMode.VOICE_MESSAGE;
|
||||
/** The chat model: it writes the transcript too, so a router swaps no model. */
|
||||
transcriptionModel: string | null;
|
||||
separateModel: false;
|
||||
}
|
||||
| {
|
||||
mode: VoiceMode.DICTATION;
|
||||
transcriptionModel: string | null;
|
||||
/** A router model other than the chat model, which transcribing may load. */
|
||||
separateModel: boolean;
|
||||
}
|
||||
| { mode: VoiceMode.UNAVAILABLE; reason: string };
|
||||
|
||||
/**
|
||||
* What a recording becomes for the conversation's model. Automatic sends the voice to a
|
||||
* chat model with audio input and dictates otherwise, also with a speech-to-text model
|
||||
* picked for the conversation: it writes down what is said and has nothing to answer a
|
||||
* voice message with. Dictation transcribes with the chat model when it can hear, else
|
||||
* (router mode) with the chosen transcription model or, on Automatic, a model with audio
|
||||
* input that is already loaded.
|
||||
*/
|
||||
export function resolveVoiceMode(input: VoiceModeInput): VoiceModeResolution {
|
||||
const { chatModel, chatModelHasAudio, isRouter, setting } = input;
|
||||
const sendsVoice =
|
||||
setting !== VoiceInputMode.DICTATION && !(chatModel && isTranscriptionOnlyModel(chatModel));
|
||||
|
||||
if (sendsVoice && chatModelHasAudio) {
|
||||
return { mode: VoiceMode.VOICE_MESSAGE, separateModel: false, transcriptionModel: chatModel };
|
||||
}
|
||||
|
||||
if (chatModelHasAudio) {
|
||||
return { mode: VoiceMode.DICTATION, separateModel: false, transcriptionModel: chatModel };
|
||||
}
|
||||
|
||||
// a single-model server has no other model to transcribe with
|
||||
if (!isRouter) return { mode: VoiceMode.UNAVAILABLE, reason: VOICE_MODE_MESSAGE.unavailable };
|
||||
|
||||
const transcriptionModel = dictationModelFor(input);
|
||||
|
||||
if (transcriptionModel) {
|
||||
return {
|
||||
mode: VoiceMode.DICTATION,
|
||||
// on Automatic it is already loaded, so transcribing swaps no model in or out
|
||||
separateModel:
|
||||
input.transcriptionModel.trim() !== VOICE_TRANSCRIPTION_MODEL_AUTO &&
|
||||
transcriptionModel !== chatModel,
|
||||
transcriptionModel
|
||||
};
|
||||
}
|
||||
|
||||
return { mode: VoiceMode.UNAVAILABLE, reason: VOICE_MODE_MESSAGE.unavailable };
|
||||
}
|
||||
|
||||
/** What the microphone button says it will do, or why it can't. */
|
||||
export function voiceModeLabel(resolution: VoiceModeResolution): string {
|
||||
switch (resolution.mode) {
|
||||
case VoiceMode.VOICE_MESSAGE:
|
||||
return VOICE_MODE_MESSAGE.voiceMessage;
|
||||
case VoiceMode.DICTATION:
|
||||
return VOICE_MODE_MESSAGE.dictate;
|
||||
default:
|
||||
return resolution.reason;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a model is a speech-to-text one, by its name: for dictation, not for chat. */
|
||||
export function isTranscriptionOnlyModel(modelId: string): boolean {
|
||||
return VOICE_TRANSCRIPTION_ONLY_PATTERN.test(modelId);
|
||||
}
|
||||
|
||||
export interface DictationModelInput {
|
||||
isRouter: boolean;
|
||||
/** The Transcription model setting; VOICE_TRANSCRIPTION_MODEL_AUTO picks one. */
|
||||
transcriptionModel: string;
|
||||
transcriptionModelHasAudio: boolean;
|
||||
/** Router mode: a loaded model with audio input, which Automatic falls back to. */
|
||||
loadedAudioModel?: string | null;
|
||||
/** Single-model mode: the server's model, and whether it hears. */
|
||||
singleModel?: string | null;
|
||||
singleModelHasAudio?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* The model that transcribes dictation for chat models that cannot hear, as
|
||||
* resolveVoiceMode() picks it, or null when none can: the chosen transcription model, or on
|
||||
* Automatic a loaded model with audio input; a single-model server's model when it hears.
|
||||
*/
|
||||
export function dictationModelFor(input: DictationModelInput): string | null {
|
||||
if (!input.isRouter) return input.singleModelHasAudio ? (input.singleModel ?? null) : null;
|
||||
|
||||
const chosen = input.transcriptionModel.trim();
|
||||
|
||||
if (chosen !== VOICE_TRANSCRIPTION_MODEL_AUTO) {
|
||||
return input.transcriptionModelHasAudio ? chosen : null;
|
||||
}
|
||||
|
||||
return input.loadedAudioModel ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* What kind of model with audio input a model is: a speech-to-text model, a chat model
|
||||
* that hears, or none (it cannot hear).
|
||||
*/
|
||||
export function voiceModelRole(modelId: string, hasAudio: boolean): VoiceModelRole | null {
|
||||
if (!hasAudio) return null;
|
||||
|
||||
return isTranscriptionOnlyModel(modelId) ? VoiceModelRole.TRANSCRIPTION : VoiceModelRole.VOICE;
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
// Guards voice input in the chat form, with a fake recorder in place of the microphone:
|
||||
// what the microphone offers per model, the recording row, dictated text written into the
|
||||
// message around what the user types, and a voice message whose transcript comes later.
|
||||
|
||||
import { modelServerProps, seedStores } from '../stories/chat/form/fixtures';
|
||||
import { deferred, FakeVoiceRecorder } from '../stories/voice/fake-voice-recorder.svelte';
|
||||
import ChatFormVoiceTestWrapper from './components/ChatFormVoiceTestWrapper.svelte';
|
||||
import { EditorView } from '@codemirror/view';
|
||||
import { VoiceTranscriptStatus } from '@ggml-org/llama-ui/voice';
|
||||
import { VoiceRecorderStatus } from '@ggml-org/llama-voice';
|
||||
import { SETTINGS_KEYS, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
|
||||
import { ServerRole, VoiceInputMode } from '#lib/enums/index.ts';
|
||||
import { voiceTranscriptsStore } from '#lib/stores/index.ts';
|
||||
import { parseFilesToMessageExtras } from '#lib/utils/browser-only.ts';
|
||||
import { tick } from 'svelte';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { userEvent } from 'vitest/browser';
|
||||
import { render } from 'vitest-browser-svelte';
|
||||
|
||||
let restore: (() => void) | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
restore?.();
|
||||
restore = undefined;
|
||||
});
|
||||
|
||||
function seed(
|
||||
audio: boolean,
|
||||
inputMode: VoiceInputMode = VoiceInputMode.AUTO,
|
||||
config: Record<string, unknown> = {}
|
||||
) {
|
||||
restore = seedStores({
|
||||
config: {
|
||||
...config,
|
||||
autoMicOnEmpty: true,
|
||||
[SETTINGS_KEYS.VOICE_INPUT_MODE]: inputMode,
|
||||
[SETTINGS_KEYS.VOICE_TRANSCRIBE_MESSAGES]: true
|
||||
},
|
||||
serverProps: modelServerProps({ modalities: { audio, video: false, vision: false } }),
|
||||
serverRole: ServerRole.MODEL
|
||||
});
|
||||
}
|
||||
|
||||
async function renderForm({ isLoading = false } = {}) {
|
||||
const recorder = new FakeVoiceRecorder();
|
||||
const state = $state({ files: [] as ChatUploadedFile[], sent: 0, value: '' });
|
||||
const screen = render(ChatFormVoiceTestWrapper, { isLoading, state, voiceRecorder: recorder });
|
||||
|
||||
await tick();
|
||||
|
||||
const content = screen.container.querySelector<HTMLElement>('.cm-content');
|
||||
const editor = content && EditorView.findFromDOM(content);
|
||||
|
||||
if (!editor) throw new Error('editor not rendered');
|
||||
|
||||
// an edit as typing makes it: reported to the form like a keystroke
|
||||
const type = (at: number, text: string) =>
|
||||
editor.dispatch({ changes: { from: at, insert: text }, userEvent: 'input.type' });
|
||||
const recordingRow = () => screen.container.querySelector('[data-slot="voice-recording-bar"]');
|
||||
|
||||
return { editor, recorder, recordingRow, screen, state, type };
|
||||
}
|
||||
|
||||
// the tooltip's trigger wraps the button and takes its name, so the button is the inner one
|
||||
describe('the microphone', () => {
|
||||
it('sends a voice message to a model with audio input', async () => {
|
||||
seed(true);
|
||||
|
||||
const { screen } = await renderForm();
|
||||
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last())
|
||||
.toBeEnabled();
|
||||
});
|
||||
|
||||
it('stands before the send button on an empty message, a quieter button than it', async () => {
|
||||
seed(true);
|
||||
|
||||
const { screen } = await renderForm();
|
||||
// the send button keeps its place, waiting for a message
|
||||
const send = screen.getByRole('button', { exact: true, name: 'Send' });
|
||||
|
||||
await expect.element(send).toBeDisabled();
|
||||
|
||||
const mic = screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last();
|
||||
|
||||
await expect.element(mic).toBeEnabled();
|
||||
expect(mic.element().getAttribute('data-variant')).toBe('tertiary');
|
||||
expect(
|
||||
mic.element().compareDocumentPosition(send.element()) & Node.DOCUMENT_POSITION_FOLLOWING
|
||||
).toBeTruthy();
|
||||
});
|
||||
|
||||
it('dictates when the user always dictates', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { screen } = await renderForm();
|
||||
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last())
|
||||
.toBeEnabled();
|
||||
});
|
||||
|
||||
it('stays beside the send button once the message has text, and dictates more', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state, type } = await renderForm();
|
||||
|
||||
type(0, 'Hello');
|
||||
await expect.poll(() => state.value).toBe('Hello');
|
||||
await expect.element(screen.getByRole('button', { name: 'Send' })).toBeEnabled();
|
||||
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
recorder.say('there');
|
||||
await expect.poll(() => state.value).toContain('there');
|
||||
expect(state.value).toContain('Hello');
|
||||
});
|
||||
|
||||
it('stays after the stop button while a reply streams, looking like it', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen } = await renderForm({ isLoading: true });
|
||||
|
||||
await expect.element(screen.getByRole('button', { exact: true, name: 'Stop' })).toBeEnabled();
|
||||
|
||||
const mic = screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last();
|
||||
|
||||
await expect.element(mic).toBeEnabled();
|
||||
|
||||
// the last button of the row, filled like the stop button before it
|
||||
const stop = screen.getByRole('button', { exact: true, name: 'Stop' }).element();
|
||||
const micButton = mic.element();
|
||||
|
||||
expect(stop.compareDocumentPosition(micButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
|
||||
expect(micButton.getAttribute('data-variant')).toBe(stop.getAttribute('data-variant'));
|
||||
|
||||
await mic.click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
});
|
||||
|
||||
it('leaves the send button on a single-model server without audio input', async () => {
|
||||
seed(false);
|
||||
|
||||
const { screen } = await renderForm();
|
||||
|
||||
await expect.element(screen.getByRole('button', { name: 'Send' })).toBeInTheDocument();
|
||||
expect(screen.container.innerHTML).not.toContain(VOICE_MODE_MESSAGE.voiceMessage);
|
||||
});
|
||||
|
||||
it('does not stand beside the send button without voice input', async () => {
|
||||
seed(false);
|
||||
|
||||
const { screen, state, type } = await renderForm();
|
||||
|
||||
type(0, 'Hello');
|
||||
await expect.poll(() => state.value).toBe('Hello');
|
||||
await expect.element(screen.getByRole('button', { name: 'Send' })).toBeEnabled();
|
||||
expect(screen.container.innerHTML).not.toContain(VOICE_MODE_MESSAGE.voiceMessage);
|
||||
expect(screen.container.innerHTML).not.toContain(VOICE_MODE_MESSAGE.dictate);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a pointer on the microphone', () => {
|
||||
/** Lets the page use the microphone without asking, with a live stream as the device. */
|
||||
function grantMicrophone() {
|
||||
const context = new AudioContext();
|
||||
const stream = context.createMediaStreamDestination().stream;
|
||||
const query = vi
|
||||
.spyOn(navigator.permissions, 'query')
|
||||
.mockResolvedValue({ state: 'granted' } as PermissionStatus);
|
||||
const getUserMedia = vi.spyOn(navigator.mediaDevices, 'getUserMedia').mockResolvedValue(stream);
|
||||
const previous = restore;
|
||||
|
||||
restore = () => {
|
||||
previous?.();
|
||||
query.mockRestore();
|
||||
getUserMedia.mockRestore();
|
||||
void context.close();
|
||||
};
|
||||
|
||||
return { getUserMedia, stream };
|
||||
}
|
||||
|
||||
it('opens the microphone ahead of the press, and the recording takes it', async () => {
|
||||
seed(true);
|
||||
|
||||
const { getUserMedia, stream } = grantMicrophone();
|
||||
const { recorder, screen } = await renderForm();
|
||||
const mic = screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last();
|
||||
|
||||
await mic.hover();
|
||||
await expect.poll(() => getUserMedia.mock.calls.length).toBe(1);
|
||||
|
||||
await mic.click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect((recorder.options?.capture as { stream?: MediaStream }).stream).toBe(stream);
|
||||
// handed over, not asked for again
|
||||
expect(getUserMedia).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('lets the microphone go when the pointer leaves without a press', async () => {
|
||||
seed(true);
|
||||
|
||||
const { getUserMedia, stream } = grantMicrophone();
|
||||
const { screen } = await renderForm();
|
||||
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().hover();
|
||||
await expect.poll(() => getUserMedia.mock.calls.length).toBe(1);
|
||||
|
||||
await userEvent.unhover(
|
||||
screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last()
|
||||
);
|
||||
await expect.poll(() => stream.getAudioTracks()[0].readyState, { timeout: 4000 }).toBe('ended');
|
||||
});
|
||||
|
||||
it('lets the microphone go when the recording is cancelled while it opens', async () => {
|
||||
seed(true);
|
||||
|
||||
const { getUserMedia, stream } = grantMicrophone();
|
||||
const opening = deferred<MediaStream>();
|
||||
|
||||
getUserMedia.mockReturnValueOnce(opening.promise);
|
||||
|
||||
const { recorder, screen } = await renderForm();
|
||||
const mic = screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last();
|
||||
|
||||
await mic.hover();
|
||||
await expect.poll(() => getUserMedia.mock.calls.length).toBe(1);
|
||||
await mic.click();
|
||||
await userEvent.keyboard('{Escape}');
|
||||
|
||||
opening.resolve(stream);
|
||||
|
||||
await expect.poll(() => stream.getAudioTracks()[0].readyState).toBe('ended');
|
||||
expect(recorder.status).toBe(VoiceRecorderStatus.IDLE);
|
||||
});
|
||||
});
|
||||
|
||||
describe('while a reply streams', () => {
|
||||
it('offers to send what is dictated as soon as its first words are in the message', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state } = await renderForm({ isLoading: true });
|
||||
|
||||
recorder.transcript = Promise.resolve('And one more thing');
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
|
||||
// nothing said yet: nothing to send
|
||||
expect(screen.container.innerHTML).not.toContain('Stop and send');
|
||||
|
||||
recorder.say('And one more thing');
|
||||
await expect.poll(() => state.value).toBe('And one more thing');
|
||||
|
||||
await screen.getByRole('button', { name: 'Stop and send' }).click();
|
||||
await expect.poll(() => state.sent).toBe(1);
|
||||
});
|
||||
|
||||
it('offers to send a voice message once its live text shows', async () => {
|
||||
seed(true);
|
||||
|
||||
const { recorder, screen } = await renderForm({ isLoading: true });
|
||||
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect(screen.container.innerHTML).not.toContain('Stop and send');
|
||||
|
||||
recorder.say('What about tomorrow');
|
||||
await expect.element(screen.getByRole('button', { name: 'Stop and send' })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('keeps the microphone beside the send button once a voice message is attached', async () => {
|
||||
seed(true);
|
||||
|
||||
const { recorder, screen, state } = await renderForm({ isLoading: true });
|
||||
|
||||
recorder.transcript = Promise.resolve('What about tomorrow');
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
recorder.say('What about tomorrow');
|
||||
await screen.getByRole('button', { name: 'Stop recording' }).last().click();
|
||||
await expect.poll(() => state.files.length).toBe(1);
|
||||
|
||||
await expect.element(screen.getByRole('button', { exact: true, name: 'Send' })).toBeEnabled();
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last())
|
||||
.toBeEnabled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('recording', () => {
|
||||
it('asks for the raw microphone, and for the processing that is switched on', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const raw = await renderForm();
|
||||
|
||||
await raw.screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => raw.recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect(raw.recorder.options?.capture).not.toHaveProperty('constraints');
|
||||
raw.screen.unmount();
|
||||
restore?.();
|
||||
|
||||
seed(true, VoiceInputMode.DICTATION, {
|
||||
[SETTINGS_KEYS.VOICE_ECHO_CANCELLATION]: true,
|
||||
[SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION]: true
|
||||
});
|
||||
|
||||
const filtered = await renderForm();
|
||||
|
||||
await filtered.screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => filtered.recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect(filtered.recorder.options?.capture).toEqual({
|
||||
boost: true,
|
||||
constraints: { echoCancellation: true, noiseSuppression: true }
|
||||
});
|
||||
});
|
||||
|
||||
it('boosts quiet speech unless the setting is off', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const boosted = await renderForm();
|
||||
|
||||
await boosted.screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => boosted.recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect(boosted.recorder.options?.capture).toMatchObject({ boost: true });
|
||||
boosted.screen.unmount();
|
||||
restore?.();
|
||||
|
||||
seed(true, VoiceInputMode.DICTATION, { [SETTINGS_KEYS.VOICE_BOOST]: false });
|
||||
|
||||
const plain = await renderForm();
|
||||
|
||||
await plain.screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => plain.recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect(plain.recorder.options?.capture).toMatchObject({ boost: false });
|
||||
});
|
||||
|
||||
it('says when the microphone is too quiet without moving the form, and stops once it is not', async () => {
|
||||
seed(true);
|
||||
|
||||
const { recorder, screen } = await renderForm();
|
||||
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
expect(screen.container.innerHTML).not.toContain(VOICE_MODE_MESSAGE.tooQuiet);
|
||||
|
||||
recorder.setQuiet(true);
|
||||
// inside the recording row, so the form keeps its height: read out, and shown above it
|
||||
await expect
|
||||
.element(screen.getByRole('status').filter({ hasText: VOICE_MODE_MESSAGE.tooQuiet }))
|
||||
.toBeInTheDocument();
|
||||
await expect
|
||||
.poll(() => document.querySelector('[data-slot="tooltip-content"]')?.textContent ?? '')
|
||||
.toContain(VOICE_MODE_MESSAGE.tooQuiet);
|
||||
|
||||
recorder.setQuiet(false);
|
||||
await expect
|
||||
.poll(() => screen.container.querySelector('[data-slot="voice-recording-bar-warning"]'))
|
||||
.toBeNull();
|
||||
});
|
||||
|
||||
it('shows the live waveform with cancel and stop, and Escape drops it', async () => {
|
||||
seed(true);
|
||||
|
||||
const { recorder, recordingRow, screen } = await renderForm();
|
||||
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().click();
|
||||
await expect.poll(recordingRow).toBeTruthy();
|
||||
|
||||
recorder.setLevels(Array.from({ length: 48 }, (_, i) => (i % 5) / 5));
|
||||
await tick();
|
||||
|
||||
expect(recordingRow()?.querySelector('[data-slot="voice-waveform-live"]')).toBeTruthy();
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: 'Cancel recording' }))
|
||||
.toBeInTheDocument();
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: 'Stop recording' }).last())
|
||||
.toBeEnabled();
|
||||
|
||||
await userEvent.keyboard('{Escape}');
|
||||
await expect.poll(recordingRow).toBeNull();
|
||||
expect(recorder.cancelled).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dictation', () => {
|
||||
it('writes the live text into the message, keeping what is typed around it', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state, type } = await renderForm();
|
||||
const transcript = deferred<string>();
|
||||
|
||||
recorder.transcript = transcript.promise;
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
|
||||
recorder.say('one');
|
||||
await expect.poll(() => state.value).toBe('one');
|
||||
|
||||
// typed before and after the dictated text while it streams
|
||||
type(0, 'Note: ');
|
||||
await expect.poll(() => state.value).toBe('Note: one');
|
||||
type(state.value.length, ' (end)');
|
||||
recorder.say('one two');
|
||||
await expect.poll(() => state.value).toBe('Note: one two (end)');
|
||||
|
||||
await screen.getByRole('button', { name: 'Stop recording' }).last().click();
|
||||
transcript.resolve('One, two.');
|
||||
|
||||
await expect.poll(() => state.value).toBe('Note: One, two. (end)');
|
||||
expect(state.files).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('waits for the text after recording when none streamed, then inserts it', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state } = await renderForm();
|
||||
const transcript = deferred<string>();
|
||||
|
||||
recorder.transcript = transcript.promise;
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
await screen.getByRole('button', { name: 'Stop recording' }).last().click();
|
||||
|
||||
await expect.element(screen.getByText(VOICE_MODE_MESSAGE.transcribing)).toBeInTheDocument();
|
||||
|
||||
transcript.resolve('Hello there');
|
||||
await expect.poll(() => state.value).toBe('Hello there');
|
||||
expect(state.files).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('says the live text is being refined, and a send waits for the refined text', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state } = await renderForm();
|
||||
const transcript = deferred<string>();
|
||||
|
||||
recorder.transcript = transcript.promise;
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
recorder.say('I went. To the store.');
|
||||
await expect.poll(() => state.value).toBe('I went. To the store.');
|
||||
await screen.getByRole('button', { name: 'Stop recording' }).last().click();
|
||||
|
||||
await expect.element(screen.getByText(VOICE_MODE_MESSAGE.refining)).toBeInTheDocument();
|
||||
|
||||
await screen.getByRole('button', { name: 'Send' }).click();
|
||||
await expect
|
||||
.element(screen.getByText(VOICE_MODE_MESSAGE.sendAfterRefining))
|
||||
.toBeInTheDocument();
|
||||
expect(state.sent).toBe(0);
|
||||
|
||||
transcript.resolve('I went to the store.');
|
||||
await expect.poll(() => state.sent).toBe(1);
|
||||
expect(state.value).toBe('I went to the store.');
|
||||
expect(screen.container.querySelector('[data-slot="voice-refining"]')).toBeNull();
|
||||
});
|
||||
|
||||
it('stops and sends the dictated text in one go', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state } = await renderForm();
|
||||
|
||||
recorder.transcript = Promise.resolve('Hello there');
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
recorder.say('Hello there');
|
||||
|
||||
await screen.getByRole('button', { name: 'Stop and send' }).click();
|
||||
await expect.poll(() => state.sent).toBe(1);
|
||||
expect(state.value).toBe('Hello there');
|
||||
});
|
||||
|
||||
it('takes the dictated text back out on cancel', async () => {
|
||||
seed(true, VoiceInputMode.DICTATION);
|
||||
|
||||
const { recorder, screen, state } = await renderForm();
|
||||
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.dictate }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
recorder.say('never mind');
|
||||
await expect.poll(() => state.value).toBe('never mind');
|
||||
|
||||
await screen.getByRole('button', { name: 'Cancel recording' }).click();
|
||||
await expect.poll(() => state.value).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('a voice message', () => {
|
||||
it('turns its live text into text to edit on focus, which goes with the clip on stop', async () => {
|
||||
seed(true);
|
||||
|
||||
const { editor, recorder, screen, state } = await renderForm();
|
||||
|
||||
recorder.transcript = Promise.resolve('model transcript');
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
recorder.say('Hello wrold');
|
||||
|
||||
editor.focus();
|
||||
await expect.poll(() => state.value).toBe('Hello wrold');
|
||||
|
||||
// fixed by hand, and what is said next follows it
|
||||
editor.dispatch({ changes: { from: 7, insert: 'or', to: 9 }, userEvent: 'input.type' });
|
||||
await expect.poll(() => state.value).toBe('Hello world');
|
||||
recorder.say('Hello wrold, how are you');
|
||||
await expect.poll(() => state.value).toBe('Hello world, how are you');
|
||||
|
||||
await screen.getByRole('button', { name: 'Stop recording' }).last().click();
|
||||
await expect.poll(() => state.files.length).toBe(1);
|
||||
expect(state.value).toBe('');
|
||||
expect(voiceTranscriptsStore.transcriptOf(state.files[0].name)).toBe(
|
||||
'Hello world, how are you'
|
||||
);
|
||||
});
|
||||
|
||||
it('is attached as a voice clip that gets its transcript later', async () => {
|
||||
seed(true);
|
||||
|
||||
const { recorder, screen, state } = await renderForm();
|
||||
const transcript = deferred<string>();
|
||||
|
||||
recorder.transcript = transcript.promise;
|
||||
await screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last().click();
|
||||
await expect.poll(() => recorder.status).toBe(VoiceRecorderStatus.RECORDING);
|
||||
|
||||
// streamed text shows in the empty message box as its placeholder, not in the message
|
||||
const placeholder = () => screen.container.querySelector('.cm-placeholder')?.textContent;
|
||||
|
||||
recorder.say('What is the weather on Mars');
|
||||
await expect.poll(placeholder).toBe('What is the weather on Mars');
|
||||
expect(state.value).toBe('');
|
||||
|
||||
await screen.getByRole('button', { name: 'Stop recording' }).last().click();
|
||||
await expect.poll(() => state.files.length).toBe(1);
|
||||
// the text now goes with the clip
|
||||
await expect.poll(placeholder).not.toBe('What is the weather on Mars');
|
||||
|
||||
const [file] = state.files;
|
||||
|
||||
expect(file.voice).toMatchObject({ durationMs: 2400 });
|
||||
expect(file.voice?.peaks).toHaveLength(64);
|
||||
expect(state.value).toBe('');
|
||||
expect(voiceTranscriptsStore.get(file.name)?.status).toBe(VoiceTranscriptStatus.TRANSCRIBING);
|
||||
|
||||
// drawn as a voice chip, its transcript on the way
|
||||
const reveal = () =>
|
||||
screen.container.querySelector('[data-slot="chat-attachment-voice"] [data-status]');
|
||||
|
||||
await expect
|
||||
.poll(() => reveal()?.getAttribute('data-status'))
|
||||
.toBe(VoiceTranscriptStatus.TRANSCRIBING);
|
||||
|
||||
transcript.resolve('What is the weather on Mars?');
|
||||
await expect
|
||||
.poll(() => voiceTranscriptsStore.transcriptOf(file.name))
|
||||
.toBe('What is the weather on Mars?');
|
||||
|
||||
await expect.poll(() => reveal()?.getAttribute('data-status')).toBe(VoiceTranscriptStatus.DONE);
|
||||
|
||||
const { extras } = await parseFilesToMessageExtras($state.snapshot(state.files));
|
||||
|
||||
expect(extras[0]).toMatchObject({
|
||||
durationMs: 2400,
|
||||
transcript: 'What is the weather on Mars?',
|
||||
voice: true
|
||||
});
|
||||
|
||||
voiceTranscriptsStore.forget(file.name);
|
||||
});
|
||||
});
|
||||
@@ -3,12 +3,16 @@
|
||||
import ChatFormActionRecord from '#lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
|
||||
|
||||
let {
|
||||
hasAudioModality = true,
|
||||
micWarning = null,
|
||||
onMicClick
|
||||
}: { hasAudioModality?: boolean; micWarning?: string | null; onMicClick?: () => void } = $props();
|
||||
onMicClick,
|
||||
unavailableReason = null
|
||||
}: {
|
||||
micWarning?: string | null;
|
||||
onMicClick?: () => void;
|
||||
unavailableReason?: string | null;
|
||||
} = $props();
|
||||
</script>
|
||||
|
||||
<Tooltip.Provider>
|
||||
<ChatFormActionRecord {hasAudioModality} {micWarning} {onMicClick} />
|
||||
<ChatFormActionRecord {micWarning} {onMicClick} {unavailableReason} />
|
||||
</Tooltip.Provider>
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
<script lang="ts">
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import ChatForm from '#lib/components/app/chat/ChatForm/ChatForm.svelte';
|
||||
import type { VoiceRecorderLike } from '#lib/hooks/use-chat-form-voice.svelte.ts';
|
||||
|
||||
interface Props {
|
||||
/** A reply is streaming: the form offers stop in place of send. */
|
||||
isLoading?: boolean;
|
||||
/** The form's message and files, kept in step with it for the test to read, and how often it sent. */
|
||||
state: { files: ChatUploadedFile[]; sent?: number; value: string };
|
||||
voiceRecorder: VoiceRecorderLike;
|
||||
}
|
||||
|
||||
let { isLoading = false, state, voiceRecorder }: Props = $props();
|
||||
</script>
|
||||
|
||||
<Tooltip.Provider>
|
||||
<ChatForm
|
||||
bind:uploadedFiles={state.files}
|
||||
bind:value={state.value}
|
||||
{isLoading}
|
||||
onSubmit={() => (state.sent = (state.sent ?? 0) + 1)}
|
||||
{voiceRecorder}
|
||||
/>
|
||||
</Tooltip.Provider>
|
||||
@@ -0,0 +1,18 @@
|
||||
<script lang="ts">
|
||||
import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip';
|
||||
import ModelsSelectorList from '#lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte';
|
||||
import type { GroupedModelOptions, ModelItem } from '#lib/utils/index.ts';
|
||||
|
||||
interface Props {
|
||||
currentModel: string | null;
|
||||
groups: GroupedModelOptions;
|
||||
loaded?: ModelItem[];
|
||||
}
|
||||
|
||||
let { currentModel, groups, loaded = [] }: Props = $props();
|
||||
</script>
|
||||
|
||||
<!-- The rows mount tooltips, which need the provider +layout.svelte supplies. -->
|
||||
<Tooltip.Provider>
|
||||
<ModelsSelectorList activeId={null} {currentModel} {groups} {loaded} onSelect={() => {}} />
|
||||
</Tooltip.Provider>
|
||||
@@ -0,0 +1,92 @@
|
||||
// IndexedDB side of the search dialog's message search: order, the stop at `limit` and staleness.
|
||||
// Matching and snippets are covered by tests/unit/conversation-search.test.ts.
|
||||
|
||||
import { MessageRole, MessageType } from '#lib/enums/index.ts';
|
||||
import { ConversationSearchService } from '#lib/services/conversation-search.service.ts';
|
||||
import { DatabaseService } from '#lib/services/database.service.ts';
|
||||
import type { DatabaseConversation, ExportedConversation } from '#lib/types/database.js';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
function makeConversation(
|
||||
id: string,
|
||||
turns: [MessageRole, string][],
|
||||
lastModified: number
|
||||
): ExportedConversation {
|
||||
const messages = turns.map(([role, content], i) => ({
|
||||
children: i + 1 < turns.length ? [`${id}-m${i + 1}`] : [],
|
||||
content,
|
||||
convId: id,
|
||||
id: `${id}-m${i}`,
|
||||
parent: i === 0 ? null : `${id}-m${i - 1}`,
|
||||
role,
|
||||
timestamp: lastModified + i,
|
||||
type: MessageType.TEXT
|
||||
}));
|
||||
|
||||
return {
|
||||
conv: { currNode: messages.at(-1)!.id, id, lastModified, name: id },
|
||||
messages
|
||||
} as unknown as ExportedConversation;
|
||||
}
|
||||
|
||||
let newestFirst: DatabaseConversation[] = [];
|
||||
|
||||
beforeEach(async () => {
|
||||
await DatabaseService.importConversations([
|
||||
makeConversation(
|
||||
'old',
|
||||
[
|
||||
[MessageRole.USER, 'what is the weather?'],
|
||||
[MessageRole.TOOL, 'Lisbon: sunny'],
|
||||
[MessageRole.ASSISTANT, 'It is sunny in Lisbon.']
|
||||
],
|
||||
1000
|
||||
),
|
||||
makeConversation('new', [[MessageRole.USER, 'Is it sunny tomorrow?']], 2000),
|
||||
makeConversation('other', [[MessageRole.USER, 'recipe for bread']], 3000)
|
||||
]);
|
||||
|
||||
newestFirst = (await DatabaseService.getAllConversations()).sort(
|
||||
(a, b) => b.lastModified - a.lastModified
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const conversations = await DatabaseService.getAllConversations();
|
||||
|
||||
await DatabaseService.bulkDeleteConversations(conversations.map((conv) => conv.id));
|
||||
});
|
||||
|
||||
describe('ConversationSearchService.searchContent', () => {
|
||||
it('returns each matching chat once, in the order given', async () => {
|
||||
const found = await ConversationSearchService.searchContent(newestFirst, 'SUNNY', {
|
||||
limit: 10
|
||||
});
|
||||
|
||||
expect(found.map((match) => match.conversation.id)).toEqual(['new', 'old']);
|
||||
expect(found[0].snippet.match).toBe('sunny');
|
||||
});
|
||||
|
||||
it('stops at the limit', async () => {
|
||||
const found = await ConversationSearchService.searchContent(newestFirst, 'sunny', { limit: 1 });
|
||||
|
||||
expect(found.map((match) => match.conversation.id)).toEqual(['new']);
|
||||
});
|
||||
|
||||
it('stops once the search is stale', async () => {
|
||||
const found = await ConversationSearchService.searchContent(newestFirst, 'sunny', {
|
||||
isCurrent: () => false,
|
||||
limit: 10
|
||||
});
|
||||
|
||||
expect(found).toEqual([]);
|
||||
});
|
||||
|
||||
it('finds tool output', async () => {
|
||||
const found = await ConversationSearchService.searchContent(newestFirst, 'lisbon: sun', {
|
||||
limit: 10
|
||||
});
|
||||
|
||||
expect(found.map((match) => match.conversation.id)).toEqual(['old']);
|
||||
});
|
||||
});
|
||||
@@ -4,8 +4,12 @@
|
||||
// while staying tappable, so a tap on a phone, which shows no tooltip, can explain it.
|
||||
|
||||
import ChatFormActionRecordWrapper from './components/ChatFormActionRecordWrapper.svelte';
|
||||
import { MICROPHONE_UNAVAILABLE_MESSAGE } from '#lib/constants/index.ts';
|
||||
import { describeMicrophoneError, microphoneUnavailableReason } from '#lib/utils/browser-only.ts';
|
||||
import {
|
||||
describeMicrophoneError,
|
||||
microphoneUnavailableReason,
|
||||
VOICE_MICROPHONE_MESSAGE
|
||||
} from '@ggml-org/llama-voice';
|
||||
import { VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { render } from 'vitest-browser-svelte';
|
||||
|
||||
@@ -17,14 +21,14 @@ describe('microphoneUnavailableReason', () => {
|
||||
it('names HTTPS when the page is not a secure context', () => {
|
||||
vi.stubGlobal('isSecureContext', false);
|
||||
|
||||
expect(microphoneUnavailableReason()).toBe(MICROPHONE_UNAVAILABLE_MESSAGE.insecure);
|
||||
expect(microphoneUnavailableReason()).toBe(VOICE_MICROPHONE_MESSAGE.insecure);
|
||||
});
|
||||
|
||||
it('names the browser when it has no recorder', () => {
|
||||
it('names the browser when it cannot capture audio', () => {
|
||||
vi.stubGlobal('isSecureContext', true);
|
||||
vi.stubGlobal('MediaRecorder', undefined);
|
||||
vi.stubGlobal('AudioContext', undefined);
|
||||
|
||||
expect(microphoneUnavailableReason()).toBe(MICROPHONE_UNAVAILABLE_MESSAGE.unsupported);
|
||||
expect(microphoneUnavailableReason()).toBe(VOICE_MICROPHONE_MESSAGE.unsupported);
|
||||
});
|
||||
|
||||
it('has nothing to say when the page can record', () => {
|
||||
@@ -36,17 +40,17 @@ describe('microphoneUnavailableReason', () => {
|
||||
|
||||
describe('describeMicrophoneError', () => {
|
||||
it.each([
|
||||
['NotAllowedError', MICROPHONE_UNAVAILABLE_MESSAGE.denied],
|
||||
['SecurityError', MICROPHONE_UNAVAILABLE_MESSAGE.denied],
|
||||
['NotFoundError', MICROPHONE_UNAVAILABLE_MESSAGE.notFound],
|
||||
['NotReadableError', MICROPHONE_UNAVAILABLE_MESSAGE.busy]
|
||||
['NotAllowedError', VOICE_MICROPHONE_MESSAGE.denied],
|
||||
['SecurityError', VOICE_MICROPHONE_MESSAGE.denied],
|
||||
['NotFoundError', VOICE_MICROPHONE_MESSAGE.notFound],
|
||||
['NotReadableError', VOICE_MICROPHONE_MESSAGE.busy]
|
||||
])('names a %s', (name, message) => {
|
||||
expect(describeMicrophoneError(new DOMException('', name))).toBe(message);
|
||||
});
|
||||
|
||||
it("keeps an unknown error's own message", () => {
|
||||
expect(describeMicrophoneError(new Error('boom'))).toBe(
|
||||
`${MICROPHONE_UNAVAILABLE_MESSAGE.failed} boom`
|
||||
`${VOICE_MICROPHONE_MESSAGE.failed} boom`
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -56,7 +60,7 @@ describe('the record button', () => {
|
||||
it('shows the reason and stays tappable when the microphone cannot record', async () => {
|
||||
const onMicClick = vi.fn();
|
||||
const screen = render(ChatFormActionRecordWrapper, {
|
||||
micWarning: MICROPHONE_UNAVAILABLE_MESSAGE.insecure,
|
||||
micWarning: VOICE_MICROPHONE_MESSAGE.insecure,
|
||||
onMicClick
|
||||
});
|
||||
const button = screen.getByRole('button', { name: /^Recording unavailable: / }).last();
|
||||
@@ -71,18 +75,18 @@ describe('the record button', () => {
|
||||
const screen = render(ChatFormActionRecordWrapper, {});
|
||||
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: 'Start recording' }).last())
|
||||
.element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last())
|
||||
.toBeEnabled();
|
||||
});
|
||||
|
||||
it('stays disabled for a model without audio, whatever the microphone', async () => {
|
||||
it('stays disabled when no model can take audio, whatever the microphone', async () => {
|
||||
const screen = render(ChatFormActionRecordWrapper, {
|
||||
hasAudioModality: false,
|
||||
micWarning: MICROPHONE_UNAVAILABLE_MESSAGE.insecure
|
||||
micWarning: VOICE_MICROPHONE_MESSAGE.insecure,
|
||||
unavailableReason: VOICE_MODE_MESSAGE.unavailable
|
||||
});
|
||||
|
||||
await expect
|
||||
.element(screen.getByRole('button', { name: 'Start recording' }).last())
|
||||
.element(screen.getByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage }).last())
|
||||
.toBeDisabled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
// Guards how the model picker shows each model's part in voice input: a badge for a
|
||||
// speech-to-text model and for a chat model that hears, nothing for one that can't, and a
|
||||
// microphone among the row's actions that is lit on the model that transcribes dictation
|
||||
// and picks it on any other model that hears; and that a loaded model kept only for
|
||||
// dictation sits apart from the chat models, under its own heading.
|
||||
|
||||
import ModelsSelectorListWrapper from './components/ModelsSelectorListWrapper.svelte';
|
||||
import { SETTINGS_KEYS, VOICE_TRANSCRIPTION_MODEL_AUTO } from '#lib/constants/index.ts';
|
||||
import { ServerModelStatus, ServerRole } from '#lib/enums/index.ts';
|
||||
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
|
||||
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
|
||||
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
|
||||
import type { ApiModelDataEntry, ModelOption } from '#lib/types/index.ts';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { render } from 'vitest-browser-svelte';
|
||||
|
||||
function option(model: string, audio: boolean): ModelOption {
|
||||
return {
|
||||
capabilities: [],
|
||||
id: model,
|
||||
modalities: { audio, video: false, vision: false },
|
||||
model,
|
||||
name: model
|
||||
};
|
||||
}
|
||||
|
||||
function loadedEntry(id: string): ApiModelDataEntry {
|
||||
return {
|
||||
created: 0,
|
||||
id,
|
||||
in_cache: true,
|
||||
object: 'model',
|
||||
owned_by: 'llamacpp',
|
||||
path: `/models/${id}`,
|
||||
status: { value: ServerModelStatus.LOADED }
|
||||
};
|
||||
}
|
||||
|
||||
const CHAT = option('org/chat-text:Q4_K_M', false);
|
||||
const OMNI = option('org/omni-2B:Q8_0', true);
|
||||
const ASR = option('org/Qwen3-ASR-0.6B:Q8_0', true);
|
||||
|
||||
let previousConfig: typeof settingsStore.config;
|
||||
|
||||
beforeEach(() => {
|
||||
previousConfig = settingsStore.config;
|
||||
modelsStore.models = [CHAT, OMNI, ASR];
|
||||
modelsStore.routerModels = [loadedEntry(CHAT.model), loadedEntry(ASR.model)];
|
||||
serverStore.role = ServerRole.ROUTER;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
settingsStore.config = previousConfig;
|
||||
});
|
||||
|
||||
function useTranscriptionModel(model: string) {
|
||||
settingsStore.config = {
|
||||
...settingsStore.config,
|
||||
[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]: model
|
||||
};
|
||||
}
|
||||
|
||||
function renderList(currentModel: string) {
|
||||
const items = [CHAT, OMNI, ASR].map((entry) => ({ option: entry }));
|
||||
|
||||
return render(ModelsSelectorListWrapper, {
|
||||
currentModel,
|
||||
groups: { available: [{ items, orgName: null }] },
|
||||
loaded: [{ option: CHAT }, { option: ASR }]
|
||||
});
|
||||
}
|
||||
|
||||
/** The role badge of each listed row, by model, as the rows show them. */
|
||||
function roles(container: HTMLElement): Record<string, string | null> {
|
||||
const out: Record<string, string | null> = {};
|
||||
|
||||
for (const row of container.querySelectorAll<HTMLElement>('[role="option"]')) {
|
||||
const model = [CHAT, OMNI, ASR].find((entry) => row.querySelector(`[title="${entry.model}"]`));
|
||||
|
||||
if (model)
|
||||
out[model.model] =
|
||||
row.querySelector<HTMLElement>('[data-slot="model-voice-role"]')?.dataset.role ?? null;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The microphone action of a model's row, null when the row has none. */
|
||||
function microphone(container: HTMLElement, model: string): HTMLElement | null {
|
||||
for (const row of container.querySelectorAll<HTMLElement>('[role="option"]')) {
|
||||
if (row.querySelector(`[title="${model}"]`)) {
|
||||
return row.querySelector<HTMLElement>('[data-slot="model-dictation-action"]');
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The models whose microphone is lit, as the ones that transcribe dictation. */
|
||||
function dictating(container: HTMLElement): string[] {
|
||||
return [CHAT, OMNI, ASR]
|
||||
.map((entry) => entry.model)
|
||||
.filter((model) => microphone(container, model)?.dataset.dictates !== undefined);
|
||||
}
|
||||
|
||||
describe('voice roles in the model picker', () => {
|
||||
it('badges a speech-to-text model and a chat model that hears', () => {
|
||||
useTranscriptionModel(OMNI.model);
|
||||
|
||||
const screen = renderList(CHAT.model);
|
||||
|
||||
expect(roles(screen.container)).toEqual({
|
||||
[ASR.model]: 'transcription',
|
||||
[CHAT.model]: null,
|
||||
[OMNI.model]: 'voice'
|
||||
});
|
||||
});
|
||||
|
||||
it('lights the microphone of the dictation model, and offers it on the others that hear', () => {
|
||||
useTranscriptionModel(OMNI.model);
|
||||
|
||||
const screen = renderList(CHAT.model);
|
||||
|
||||
expect(dictating(screen.container)).toEqual([OMNI.model]);
|
||||
expect(microphone(screen.container, ASR.model)).not.toBeNull();
|
||||
// a model that cannot hear has no microphone
|
||||
expect(microphone(screen.container, CHAT.model)).toBeNull();
|
||||
});
|
||||
|
||||
it('picks the dictation model with a click, and hands it back to Automatic with another', async () => {
|
||||
useTranscriptionModel(OMNI.model);
|
||||
|
||||
const screen = renderList(CHAT.model);
|
||||
const click = (model: string) =>
|
||||
microphone(screen.container, model)!.querySelector('button')!.click();
|
||||
|
||||
click(ASR.model);
|
||||
await expect.poll(() => dictating(screen.container)).toEqual([ASR.model]);
|
||||
expect(settingsStore.config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL]).toBe(ASR.model);
|
||||
|
||||
click(ASR.model);
|
||||
await expect
|
||||
.poll(() => settingsStore.config[SETTINGS_KEYS.VOICE_TRANSCRIPTION_MODEL])
|
||||
.toBe(VOICE_TRANSCRIPTION_MODEL_AUTO);
|
||||
});
|
||||
|
||||
it('lists a loaded model kept for dictation apart from the chat models', () => {
|
||||
useTranscriptionModel(ASR.model);
|
||||
|
||||
const screen = renderList(CHAT.model);
|
||||
const heading = screen.container.querySelector('[data-slot="models-loaded-dictation-heading"]');
|
||||
|
||||
expect(heading?.textContent?.trim()).toBe('For dictation');
|
||||
// the dictation model comes after the heading, the chat model before it
|
||||
const rows = [...screen.container.querySelectorAll('[role="option"]')].slice(0, 2);
|
||||
|
||||
expect(rows[0].textContent).toContain('chat-text');
|
||||
expect(rows[1].textContent).toContain('Qwen3-ASR');
|
||||
expect(heading?.compareDocumentPosition(rows[1]) ?? 0).toBe(Node.DOCUMENT_POSITION_FOLLOWING);
|
||||
});
|
||||
|
||||
it('keeps the conversation model with the chat models, even when it dictates', () => {
|
||||
useTranscriptionModel(ASR.model);
|
||||
|
||||
const screen = renderList(ASR.model);
|
||||
|
||||
expect(
|
||||
screen.container.querySelector('[data-slot="models-loaded-dictation-heading"]')
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('on Automatic, gives dictation to the loaded model that hears', () => {
|
||||
useTranscriptionModel(VOICE_TRANSCRIPTION_MODEL_AUTO);
|
||||
|
||||
const screen = renderList(CHAT.model);
|
||||
|
||||
expect(dictating(screen.container)).toEqual([ASR.model]);
|
||||
expect(roles(screen.container)[OMNI.model]).toBe('voice');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { shouldTypeToFocus } from '#lib/utils/type-to-focus.ts';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
function key(key: string, init: KeyboardEventInit = {}) {
|
||||
return new KeyboardEvent('keydown', { bubbles: true, cancelable: true, key, ...init });
|
||||
}
|
||||
|
||||
function mount(html: string) {
|
||||
const host = document.createElement('div');
|
||||
|
||||
host.innerHTML = html;
|
||||
document.body.appendChild(host);
|
||||
|
||||
return host;
|
||||
}
|
||||
|
||||
describe('shouldTypeToFocus', () => {
|
||||
afterEach(() => {
|
||||
(document.activeElement as HTMLElement | null)?.blur();
|
||||
document.body.innerHTML = '';
|
||||
});
|
||||
|
||||
it('takes a printable key when nothing has focus', () => {
|
||||
expect(shouldTypeToFocus(key('a'))).toBe(true);
|
||||
expect(shouldTypeToFocus(key('A', { shiftKey: true }))).toBe(true);
|
||||
expect(shouldTypeToFocus(key('?'))).toBe(true);
|
||||
});
|
||||
|
||||
it('takes a printable key while a button has focus', () => {
|
||||
mount('<button>Copy</button>').querySelector('button')!.focus();
|
||||
|
||||
expect(shouldTypeToFocus(key('h'))).toBe(true);
|
||||
});
|
||||
|
||||
it('leaves named keys, whitespace and shortcuts alone', () => {
|
||||
for (const name of ['Enter', 'Escape', 'Tab', 'ArrowDown', 'F5', ' ']) {
|
||||
expect(shouldTypeToFocus(key(name))).toBe(false);
|
||||
}
|
||||
|
||||
expect(shouldTypeToFocus(key('k', { metaKey: true }))).toBe(false);
|
||||
expect(shouldTypeToFocus(key('k', { ctrlKey: true }))).toBe(false);
|
||||
expect(shouldTypeToFocus(key('k', { altKey: true }))).toBe(false);
|
||||
});
|
||||
|
||||
it('leaves keys another handler already took', () => {
|
||||
const event = key('a');
|
||||
|
||||
event.preventDefault();
|
||||
|
||||
expect(shouldTypeToFocus(event)).toBe(false);
|
||||
});
|
||||
|
||||
it('leaves typing in a field or editor where it is', () => {
|
||||
const host = mount('<input /><textarea></textarea><div contenteditable="true">x</div>');
|
||||
|
||||
for (const el of host.querySelectorAll<HTMLElement>('input, textarea, [contenteditable]')) {
|
||||
el.focus();
|
||||
expect(shouldTypeToFocus(key('a'))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves keys inside dialogs and menus', () => {
|
||||
const host = mount(
|
||||
'<div role="dialog"><button>OK</button></div><div role="menu"><button role="menuitem">Rename</button></div>'
|
||||
);
|
||||
|
||||
for (const button of host.querySelectorAll('button')) {
|
||||
button.focus();
|
||||
expect(shouldTypeToFocus(key('r'))).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,184 @@
|
||||
// Guards VoiceCapture in a real browser, fed by a synthesized microphone (an oscillator
|
||||
// into a MediaStream, since the fake capture device flags are not available here): the
|
||||
// levels move, the PCM worklet loads from its Blob URL and streams chunks, stop() returns
|
||||
// a WAV clip with a plausible duration and peaks, and every path stops the tracks.
|
||||
|
||||
import {
|
||||
VOICE_MICROPHONE_MESSAGE,
|
||||
VoiceCapture,
|
||||
VoiceCaptureError,
|
||||
VoiceCaptureState
|
||||
} from '@ggml-org/llama-voice';
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { userEvent } from 'vitest/browser';
|
||||
|
||||
const sources: AudioContext[] = [];
|
||||
|
||||
/** A 440 Hz tone as a microphone stream. */
|
||||
function toneStream(): MediaStream {
|
||||
const context = new AudioContext();
|
||||
const oscillator = context.createOscillator();
|
||||
const destination = context.createMediaStreamDestination();
|
||||
|
||||
oscillator.frequency.value = 440;
|
||||
oscillator.connect(destination);
|
||||
oscillator.start();
|
||||
sources.push(context);
|
||||
|
||||
return destination.stream;
|
||||
}
|
||||
|
||||
// audio contexts only run after a user gesture, as the mic button gives in the app
|
||||
beforeAll(async () => {
|
||||
const button = document.createElement('button');
|
||||
|
||||
button.textContent = 'Record';
|
||||
document.body.append(button);
|
||||
await userEvent.click(button);
|
||||
button.remove();
|
||||
});
|
||||
|
||||
const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** Waits for `done`, polling: a loaded test run can start the audio graph late. */
|
||||
async function waitFor(done: () => boolean, timeoutMs = 5000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
|
||||
while (!done()) {
|
||||
if (Date.now() > deadline) throw new Error('timed out waiting for audio');
|
||||
|
||||
await wait(20);
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
|
||||
for (const context of sources.splice(0)) await context.close();
|
||||
});
|
||||
|
||||
describe('VoiceCapture', () => {
|
||||
it('streams levels and PCM chunks, and stops with a WAV clip', async () => {
|
||||
const warn = vi.spyOn(console, 'warn');
|
||||
const stream = toneStream();
|
||||
const capture = new VoiceCapture({ levelsIntervalMs: 20, levelsLength: 16, stream });
|
||||
const levels: (readonly number[])[] = [];
|
||||
const chunks: Int16Array[] = [];
|
||||
|
||||
capture.onLevels((value) => levels.push(value));
|
||||
capture.onPcm((chunk) => chunks.push(chunk));
|
||||
|
||||
await capture.start();
|
||||
expect(capture.state).toBe(VoiceCaptureState.RECORDING);
|
||||
|
||||
// at least 600 ms of tone recorded, however long the audio graph took to start
|
||||
await waitFor(() => chunks.length >= 6 && levels.length > 5);
|
||||
|
||||
const clip = await capture.stop();
|
||||
|
||||
// the worklet loaded from its Blob URL under COOP/COEP, as llama-server serves the UI, no fallback
|
||||
expect(crossOriginIsolated).toBe(true);
|
||||
expect(warn).not.toHaveBeenCalledWith(
|
||||
expect.stringContaining('AudioWorklet unavailable'),
|
||||
expect.anything()
|
||||
);
|
||||
|
||||
expect(levels.length).toBeGreaterThan(5);
|
||||
expect(levels.at(-1)).toHaveLength(16);
|
||||
expect(Math.max(...levels.at(-1)!)).toBeGreaterThan(0.5);
|
||||
|
||||
// 100 ms chunks at 16 kHz, the last one may be shorter
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(chunks[0]).toHaveLength(1600);
|
||||
|
||||
expect(clip.sampleRate).toBe(16_000);
|
||||
expect(clip.durationMs).toBeGreaterThan(500);
|
||||
expect(clip.durationMs).toBeLessThan(5000);
|
||||
expect(clip.peaks).toHaveLength(64);
|
||||
expect(Math.max(...clip.peaks)).toBe(1);
|
||||
// a steady tone: once it reaches the capture, no span of the clip is silent. The capture
|
||||
// starts with its first audio block, which can come before the tone stream's first sound
|
||||
const toneFrom = clip.peaks.findIndex((peak) => peak > 0.5);
|
||||
|
||||
expect(toneFrom).toBeGreaterThanOrEqual(0);
|
||||
expect(clip.peaks.slice(toneFrom).every((peak) => peak > 0.5)).toBe(true);
|
||||
expect(clip.peaks.length - toneFrom).toBeGreaterThan(16);
|
||||
|
||||
const header = new DataView(await clip.blob.slice(0, 44).arrayBuffer());
|
||||
const pcmBytes = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
|
||||
|
||||
expect(clip.blob.type).toBe('audio/wav');
|
||||
expect(header.getUint16(22, true)).toBe(1);
|
||||
expect(header.getUint32(24, true)).toBe(16_000);
|
||||
expect(header.getUint16(34, true)).toBe(16);
|
||||
expect(header.getUint32(40, true)).toBe(pcmBytes);
|
||||
|
||||
// the clip decodes as audio
|
||||
const decoded = await new OfflineAudioContext(1, 1, 16_000).decodeAudioData(
|
||||
await clip.blob.arrayBuffer()
|
||||
);
|
||||
|
||||
expect(decoded.duration * 1000).toBeCloseTo(clip.durationMs, 0);
|
||||
expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true);
|
||||
expect(capture.state).toBe(VoiceCaptureState.STOPPED);
|
||||
});
|
||||
|
||||
it('cancel stops the tracks and drops the recording', async () => {
|
||||
const stream = toneStream();
|
||||
const capture = new VoiceCapture({ stream });
|
||||
|
||||
await capture.start();
|
||||
await wait(200);
|
||||
capture.cancel();
|
||||
|
||||
expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true);
|
||||
expect(capture.state).toBe(VoiceCaptureState.IDLE);
|
||||
await expect(capture.stop()).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('cancel during start releases the microphone once it arrives', async () => {
|
||||
const stream = toneStream();
|
||||
|
||||
let grant!: (value: MediaStream) => void;
|
||||
|
||||
vi.spyOn(navigator.mediaDevices, 'getUserMedia').mockImplementation(
|
||||
() => new Promise((resolve) => (grant = resolve))
|
||||
);
|
||||
|
||||
const capture = new VoiceCapture();
|
||||
const started = capture.start().catch((e) => e);
|
||||
|
||||
capture.cancel();
|
||||
grant(stream);
|
||||
|
||||
expect(await started).toBeInstanceOf(DOMException);
|
||||
expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true);
|
||||
});
|
||||
|
||||
it('a page hide tears the capture down', async () => {
|
||||
const stream = toneStream();
|
||||
const capture = new VoiceCapture({ stream });
|
||||
|
||||
await capture.start();
|
||||
dispatchEvent(new PageTransitionEvent('pagehide'));
|
||||
|
||||
expect(capture.state).toBe(VoiceCaptureState.IDLE);
|
||||
expect(stream.getTracks().every((track) => track.readyState === 'ended')).toBe(true);
|
||||
});
|
||||
|
||||
it('asks for the raw microphone signal and names a refusal', async () => {
|
||||
const getUserMedia = vi
|
||||
.spyOn(navigator.mediaDevices, 'getUserMedia')
|
||||
.mockRejectedValue(new DOMException('denied', 'NotAllowedError'));
|
||||
const capture = new VoiceCapture();
|
||||
const error = await capture.start().catch((e) => e);
|
||||
|
||||
expect(getUserMedia).toHaveBeenCalledWith({
|
||||
audio: { autoGainControl: false, echoCancellation: false, noiseSuppression: false }
|
||||
});
|
||||
expect(error).toBeInstanceOf(VoiceCaptureError);
|
||||
expect(error.message).toBe(VOICE_MICROPHONE_MESSAGE.denied);
|
||||
expect(capture.state).toBe(VoiceCaptureState.IDLE);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
// Guards the wiring every voice feature starts its session with: the Voice settings become
|
||||
// capture options (boost, processing, a caller's own constraints), the language and the
|
||||
// transcription model go where a router needs them, and an API key keeps the realtime
|
||||
// socket off, since a browser websocket can not send it, which the app says of a model that
|
||||
// only transcribes over that socket.
|
||||
|
||||
import { seedStores } from '../stories/chat/form/fixtures';
|
||||
import { VOICE_MICROPHONE_DEFAULT, VoiceTranscriptionError } from '@ggml-org/llama-voice';
|
||||
import { SETTINGS_KEYS, VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
|
||||
import { SETTING_CONFIG_DEFAULT } from '#lib/constants/settings.constants.ts';
|
||||
import { VoiceService } from '#lib/services/voice.service.ts';
|
||||
import type { SettingsConfigType } from '#lib/types/index.ts';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
let restore: (() => void) | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
restore?.();
|
||||
restore = undefined;
|
||||
});
|
||||
|
||||
/** The default settings with `changes`, on the system microphone so no device is listed. */
|
||||
function settings(changes: Record<string, unknown> = {}): SettingsConfigType {
|
||||
return {
|
||||
...SETTING_CONFIG_DEFAULT,
|
||||
[SETTINGS_KEYS.VOICE_MICROPHONE]: VOICE_MICROPHONE_DEFAULT,
|
||||
...changes
|
||||
} as SettingsConfigType;
|
||||
}
|
||||
|
||||
describe('VoiceService.sessionOptions', () => {
|
||||
it('boosts the raw signal by default, and transcribes with the model a router routes on', async () => {
|
||||
const options = await VoiceService.sessionOptions(settings(), {
|
||||
isRouter: true,
|
||||
model: 'org/hearing-model'
|
||||
});
|
||||
|
||||
expect(options.capture).toEqual({ boost: true });
|
||||
expect(options.transcribe).toMatchObject({ language: undefined, model: 'org/hearing-model' });
|
||||
});
|
||||
|
||||
it('leaves the model out for a single-model server', async () => {
|
||||
const options = await VoiceService.sessionOptions(settings(), {
|
||||
isRouter: false,
|
||||
model: 'org/hearing-model'
|
||||
});
|
||||
|
||||
expect(options.transcribe?.model).toBeUndefined();
|
||||
});
|
||||
|
||||
it('follows the processing, boost and language settings', async () => {
|
||||
const options = await VoiceService.sessionOptions(
|
||||
settings({
|
||||
[SETTINGS_KEYS.VOICE_AUTO_GAIN]: true,
|
||||
[SETTINGS_KEYS.VOICE_BOOST]: false,
|
||||
[SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION]: true,
|
||||
[SETTINGS_KEYS.VOICE_TRANSCRIPTION_LANGUAGE]: ' de '
|
||||
}),
|
||||
{ isRouter: false, model: null }
|
||||
);
|
||||
|
||||
expect(options.capture).toEqual({
|
||||
boost: false,
|
||||
constraints: { autoGainControl: true, noiseSuppression: true }
|
||||
});
|
||||
expect(options.transcribe?.language).toBe('de');
|
||||
});
|
||||
|
||||
it("takes a caller's own constraints over the processing settings", async () => {
|
||||
const options = await VoiceService.sessionOptions(
|
||||
settings({ [SETTINGS_KEYS.VOICE_NOISE_SUPPRESSION]: true }),
|
||||
{ constraints: { echoCancellation: true }, isRouter: false, model: null }
|
||||
);
|
||||
|
||||
expect(options.capture).toMatchObject({ constraints: { echoCancellation: true } });
|
||||
});
|
||||
|
||||
it('records without transcribing when asked to', async () => {
|
||||
const options = await VoiceService.sessionOptions(settings(), {
|
||||
isRouter: false,
|
||||
model: null,
|
||||
transcribe: false
|
||||
});
|
||||
|
||||
expect(options.transcribe).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('VoiceService.serverContext', () => {
|
||||
it('uses the realtime socket when no API key is set', () => {
|
||||
restore = seedStores({ config: { apiKey: '' } });
|
||||
|
||||
const { context, realtime } = VoiceService.serverContext();
|
||||
|
||||
expect(realtime).toBe(true);
|
||||
expect(context.getHeaders?.()).toEqual({});
|
||||
expect(context.resolveUrl('./v1/realtime')).toMatch(/\/v1\/realtime$/);
|
||||
});
|
||||
|
||||
it('keeps to HTTP, with the key, when the server needs one', () => {
|
||||
restore = seedStores({ config: { apiKey: 'secret' } });
|
||||
|
||||
const { context, realtime } = VoiceService.serverContext();
|
||||
|
||||
expect(realtime).toBe(false);
|
||||
expect(context.getHeaders?.()).toEqual({ Authorization: 'Bearer secret' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('VoiceService.failureReason', () => {
|
||||
const notSupported = new VoiceTranscriptionError(
|
||||
'this server does not support chat/completions',
|
||||
501
|
||||
);
|
||||
|
||||
it('says a live-only model cannot transcribe on a server with an API key', () => {
|
||||
restore = seedStores({ config: { apiKey: 'secret' } });
|
||||
|
||||
expect(VoiceService.failureReason(notSupported)).toBe(VOICE_MODE_MESSAGE.realtimeOnlyBehindKey);
|
||||
});
|
||||
|
||||
it("leaves other failures to the server's own words", () => {
|
||||
restore = seedStores({ config: { apiKey: 'secret' } });
|
||||
|
||||
expect(
|
||||
VoiceService.failureReason(new VoiceTranscriptionError('Loading model', 503))
|
||||
).toBeNull();
|
||||
expect(VoiceService.failureReason(new Error('Failed to fetch'))).toBeNull();
|
||||
|
||||
restore();
|
||||
restore = seedStores({ config: { apiKey: '' } });
|
||||
|
||||
// without a key the realtime socket was tried first: the model simply cannot hear
|
||||
expect(VoiceService.failureReason(notSupported)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
// Guards VoiceWaveform's playback with real clicks, which a browser needs before it plays audio (the stories' clicks
|
||||
// are scripted, so their play functions cover the rest): play and pause toggle, and one clip plays at a time.
|
||||
|
||||
import { demoClip } from '../stories/voice/fixtures';
|
||||
import { VoiceWaveform } from '@ggml-org/llama-ui/voice';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { page, userEvent } from 'vitest/browser';
|
||||
import { render } from 'vitest-browser-svelte';
|
||||
|
||||
describe('VoiceWaveform playback', () => {
|
||||
it('toggles between play and pause', async () => {
|
||||
const clip = demoClip(3, 1);
|
||||
|
||||
render(VoiceWaveform, { durationMs: clip.durationMs, peaks: clip.peaks, src: clip.src });
|
||||
|
||||
await userEvent.click(page.getByRole('button', { name: 'Play voice message' }));
|
||||
await expect.element(page.getByRole('button', { name: 'Pause voice message' })).toBeVisible();
|
||||
|
||||
await userEvent.click(page.getByRole('button', { name: 'Pause voice message' }));
|
||||
await expect.element(page.getByRole('button', { name: 'Play voice message' })).toBeVisible();
|
||||
});
|
||||
|
||||
it('pauses one clip when another starts', async () => {
|
||||
const first = demoClip(3, 2);
|
||||
const second = demoClip(3, 3);
|
||||
|
||||
render(VoiceWaveform, { label: 'First clip', peaks: first.peaks, src: first.src });
|
||||
render(VoiceWaveform, { label: 'Second clip', peaks: second.peaks, src: second.src });
|
||||
|
||||
await userEvent.click(page.getByRole('button', { name: 'Play first clip' }));
|
||||
await expect.element(page.getByRole('button', { name: 'Pause first clip' })).toBeVisible();
|
||||
|
||||
await userEvent.click(page.getByRole('button', { name: 'Play second clip' }));
|
||||
await expect.element(page.getByRole('button', { name: 'Pause second clip' })).toBeVisible();
|
||||
await expect.element(page.getByRole('button', { name: 'Play first clip' })).toBeVisible();
|
||||
|
||||
await userEvent.click(page.getByRole('button', { name: 'Pause second clip' }));
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,10 @@
|
||||
<script lang="ts" module>
|
||||
import { defineMeta } from '@storybook/addon-svelte-csf';
|
||||
import ChatFormActionRecord from '#lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
|
||||
import { VOICE_MODE_MESSAGE } from '#lib/constants/index.ts';
|
||||
import StateGallery from '$llama-ui-stories/helpers/StateGallery.svelte';
|
||||
import StateGalleryItem from '$llama-ui-stories/helpers/StateGalleryItem.svelte';
|
||||
import { expect, screen } from 'storybook/test';
|
||||
|
||||
const { Story } = defineMeta({
|
||||
component: ChatFormActionRecord,
|
||||
@@ -14,35 +16,59 @@
|
||||
</script>
|
||||
|
||||
<Story asChild name="States">
|
||||
<StateGallery columns={5}>
|
||||
<StateGalleryItem label="Idle">
|
||||
<ChatFormActionRecord hasAudioModality />
|
||||
<StateGallery columns={6}>
|
||||
<StateGalleryItem label="Voice message">
|
||||
<ChatFormActionRecord action={VOICE_MODE_MESSAGE.voiceMessage} />
|
||||
</StateGalleryItem>
|
||||
|
||||
<StateGalleryItem label="Recording">
|
||||
<ChatFormActionRecord hasAudioModality isRecording />
|
||||
<StateGalleryItem label="Dictate">
|
||||
<ChatFormActionRecord action={VOICE_MODE_MESSAGE.dictate} />
|
||||
</StateGalleryItem>
|
||||
|
||||
<StateGalleryItem label="Recording" note="the stop button">
|
||||
<ChatFormActionRecord isRecording />
|
||||
</StateGalleryItem>
|
||||
|
||||
<StateGalleryItem label="Loading">
|
||||
<ChatFormActionRecord hasAudioModality isLoading />
|
||||
<ChatFormActionRecord isLoading />
|
||||
</StateGalleryItem>
|
||||
|
||||
<StateGalleryItem label="Disabled">
|
||||
<ChatFormActionRecord disabled hasAudioModality />
|
||||
<ChatFormActionRecord disabled />
|
||||
</StateGalleryItem>
|
||||
|
||||
<StateGalleryItem label="No audio" note="model lacks audio">
|
||||
<ChatFormActionRecord hasAudioModality={false} />
|
||||
<StateGalleryItem label="Unavailable" note="no model with audio">
|
||||
<ChatFormActionRecord unavailableReason={VOICE_MODE_MESSAGE.unavailable} />
|
||||
</StateGalleryItem>
|
||||
</StateGallery>
|
||||
</Story>
|
||||
|
||||
<!-- separate: play hovers the button to open its tooltip -->
|
||||
<Story
|
||||
args={{ hasAudioModality: false }}
|
||||
name="NoAudioModalityTooltip"
|
||||
args={{ unavailableReason: VOICE_MODE_MESSAGE.unavailable }}
|
||||
name="UnavailableTooltip"
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
const button = canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.voiceMessage });
|
||||
|
||||
await expect(button.at(-1)).toBeDisabled();
|
||||
// the tooltip trigger wraps the disabled button, hover the trigger
|
||||
await userEvent.hover(canvas.getAllByRole('button', { name: 'Start recording' })[0]);
|
||||
await userEvent.hover(button[0]);
|
||||
await expect(
|
||||
(await screen.findAllByText(VOICE_MODE_MESSAGE.unavailable)).length
|
||||
).toBeGreaterThan(0);
|
||||
}}
|
||||
/>
|
||||
|
||||
<Story
|
||||
args={{
|
||||
action: VOICE_MODE_MESSAGE.dictate,
|
||||
hint: VOICE_MODE_MESSAGE.separateModelHint
|
||||
}}
|
||||
name="DictateWithAnotherModelTooltip"
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
await userEvent.hover(canvas.getAllByRole('button', { name: VOICE_MODE_MESSAGE.dictate })[0]);
|
||||
await expect((await screen.findAllByText(/alongside the chat model/)).length).toBeGreaterThan(
|
||||
0
|
||||
);
|
||||
}}
|
||||
/>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user