Implement inspectable RAG ingestion pipeline
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
# Project Thoth RAG ingestion - Oracle embedding configuration
|
||||
THOTH_ORACLE_BASE_URL=
|
||||
THOTH_EMBEDDING_MODEL=
|
||||
THOTH_EMBEDDING_DIMENSION=
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
.env
|
||||
|
||||
+25
-19
@@ -1,31 +1,37 @@
|
||||
Current Version
|
||||
|
||||
0.0.1
|
||||
0.0.2
|
||||
|
||||
Current Focus
|
||||
|
||||
ChatGPT Capture Connector
|
||||
Project Thoth Librarian MVP
|
||||
|
||||
Completed
|
||||
|
||||
✓ Reference Architecture
|
||||
✓ Core ADRs
|
||||
✓ Source Metadata Specification
|
||||
✓ Source Metadata Generator
|
||||
✓ Conversation Manifest Specification
|
||||
✓ Conversation Manifest Generator
|
||||
✓ Manual ChatGPT capture workflow validated
|
||||
✓ Manual web article/post capture workflow identified
|
||||
✓ Manual email capture workflow identified
|
||||
|
||||
✓ ADR-001
|
||||
|
||||
✓ ADR-002
|
||||
|
||||
✓ ADR-003
|
||||
|
||||
✓ Metadata Generator
|
||||
Shelved
|
||||
• ChatGPT Capture Connector MVP
|
||||
• Browser extension capture implementation
|
||||
• Automated capture framework
|
||||
|
||||
In Progress
|
||||
|
||||
• Capture Service
|
||||
|
||||
Blocked
|
||||
|
||||
• Browser Extension
|
||||
• Librarian architecture
|
||||
• Inbox processing workflow
|
||||
• Canonical artifact package definition
|
||||
• Processor orchestration contract
|
||||
|
||||
Next Milestone
|
||||
Catalog one ChatGPT conversation from /vault/inbox/
|
||||
into a validated canonical Vault artifact package
|
||||
|
||||
Capture MVP
|
||||
Capture Boundary
|
||||
Capture occurs manually outside Project Thoth.
|
||||
|
||||
Librarian Boundary
|
||||
Project Thoth processing begins when a source file is placed in /vault/inbox/.
|
||||
+62
-26
@@ -60,56 +60,92 @@ Project architecture is stable and documented.
|
||||
|
||||
---
|
||||
|
||||
# Phase 2 — Capture MVP
|
||||
# Phase 2 — Manual Capture Workflow
|
||||
|
||||
## Objective
|
||||
|
||||
Capture conversations from ChatGPT into canonical Project Thoth artifacts.
|
||||
Establish reliable manual capture conventions using external tools.
|
||||
|
||||
### Deliverables
|
||||
|
||||
- Browser extension
|
||||
- ChatGPT connector
|
||||
- Conversation discovery
|
||||
- DOM-to-Markdown transformation
|
||||
- Markdown serialization
|
||||
- Download support
|
||||
- Basic user feedback
|
||||
- Tampermonkey ChatGPT capture;
|
||||
- MarkDownload article/post capture;
|
||||
- native email save;
|
||||
- documented manual movement from Downloads to Inbox.
|
||||
|
||||
### Success Criteria
|
||||
|
||||
Conversations can be captured with high fidelity into `conversation.md`.
|
||||
Files are saved into the Project Thoth Inbox for cataloging.
|
||||
|
||||
---
|
||||
|
||||
# Phase 3 — Capture Framework
|
||||
# Phase 3 — Librarian
|
||||
|
||||
## Objective
|
||||
|
||||
Generalize the ChatGPT implementation into a reusable connector framework.
|
||||
Convert loose Inbox files into complete canonical artifact packages.
|
||||
|
||||
### Deliverables
|
||||
|
||||
- Capture Connector interface
|
||||
- Canonical Conversation Model
|
||||
- Shared HTML-to-Markdown engine
|
||||
- Shared Markdown serializer
|
||||
- Connector testing framework
|
||||
|
||||
### Target Connectors
|
||||
|
||||
- ChatGPT
|
||||
- Claude
|
||||
- Gemini
|
||||
- Microsoft Copilot
|
||||
- Open WebUI
|
||||
- Inbox listing;
|
||||
- file selection;
|
||||
- source classification;
|
||||
- source identity derivation;
|
||||
- duplicate handling;
|
||||
- Source Metadata generation;
|
||||
- Manifest generation;
|
||||
- package validation;
|
||||
- transactional promotion to canonical Vault location.
|
||||
|
||||
### Success Criteria
|
||||
|
||||
New connectors primarily require implementation of platform-specific discovery.
|
||||
A working Librarian function in the Project Thoth application.
|
||||
|
||||
---
|
||||
|
||||
# Phase 4 - Librarian Source Expansion
|
||||
|
||||
## Objective
|
||||
|
||||
Add generators and specifications for articles, posts, and email formats.
|
||||
|
||||
## Deliverables
|
||||
|
||||
- article-manifest-generator.md
|
||||
- post-manifest-generator.md
|
||||
- email-manifest-generator.md
|
||||
- article-metadata-specification.md
|
||||
- post-metadata-specification.md
|
||||
- email-metadata-specification.md
|
||||
|
||||
### Success Criteria
|
||||
|
||||
All supported sources are catalogued with metadata and manifest into required Vault structures.
|
||||
|
||||
|
||||
# Phase 5 — Archive and Index Management
|
||||
package repair;
|
||||
version tracking;
|
||||
catalog index;
|
||||
integrity checks;
|
||||
search preparation.
|
||||
|
||||
# Phase 6 — Harvest and corpus assembly
|
||||
|
||||
Only after cataloging is routine.
|
||||
|
||||
# Deferred capture framework
|
||||
|
||||
The existing connector framework should remain in the roadmap, but much later:
|
||||
|
||||
automated source connectors;
|
||||
direct application submission;
|
||||
background synchronization;
|
||||
API-based capture;
|
||||
incremental updates.
|
||||
|
||||
# NOTE: The following phases are numbered out of sequence and are maintained here for reference purposes.
|
||||
|
||||
# Phase 4 — Processor Framework
|
||||
|
||||
## Objective
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
# Work Order 0001 — Establish the RAG Ingestion Foundation
|
||||
|
||||
## Status
|
||||
|
||||
Planned
|
||||
|
||||
## Work Stream
|
||||
|
||||
RAG Ingestion
|
||||
|
||||
## Application
|
||||
|
||||
`rag-ingestion`
|
||||
|
||||
## Location
|
||||
|
||||
Work order:
|
||||
|
||||
`./codex/rag-ingestion/0001-establish-rag-ingestion-foundation.md`
|
||||
|
||||
Implementation package:
|
||||
|
||||
`./processors/rag_ingestion/`
|
||||
|
||||
---
|
||||
|
||||
## Objective
|
||||
|
||||
Establish the first working slice of the Project Thoth RAG ingestion pipeline.
|
||||
|
||||
This work order should create a small, explicit, inspectable Python application that can:
|
||||
|
||||
1. Accept the path to a single Markdown source file.
|
||||
2. Read the source file without modifying it.
|
||||
3. Capture basic mechanical source information from the filesystem.
|
||||
4. Assign a deterministic document identifier.
|
||||
5. Produce an in-memory document object representing the source.
|
||||
6. Display the resulting document record in a human-readable form for inspection.
|
||||
|
||||
This work order intentionally stops before chunking, embeddings, PostgreSQL, pgvector, or recursive folder ingestion.
|
||||
|
||||
The purpose is to establish and understand the ingestion boundary before adding downstream RAG behavior.
|
||||
|
||||
---
|
||||
|
||||
## Learning Objective
|
||||
|
||||
This work order is not only an implementation task.
|
||||
|
||||
The implementation must make the ingestion process understandable enough that the developer can explain:
|
||||
|
||||
- what "ingestion" means in a RAG pipeline;
|
||||
- what information exists before any AI processing occurs;
|
||||
- which source properties can be obtained deterministically from the filesystem;
|
||||
- why a stable document identifier is needed;
|
||||
- what information belongs to the original source versus a derived ingestion record;
|
||||
- where ingestion ends and chunking begins.
|
||||
|
||||
Codex should prefer explicit code over framework abstractions when the abstraction would hide these concepts.
|
||||
|
||||
The finished implementation should be easy to read in VS Code and suitable for walking through line by line.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Context
|
||||
|
||||
Project Thoth preserves Primary Sources as authoritative records.
|
||||
|
||||
The RAG ingestion pipeline is a consumer of those sources. It must not modify, rewrite, summarize, enrich, or replace them.
|
||||
|
||||
For this baseline RAG implementation, distinguish between:
|
||||
|
||||
### Mechanical Metadata
|
||||
|
||||
Information obtained directly from the file or filesystem, such as:
|
||||
|
||||
- source path;
|
||||
- filename;
|
||||
- file size;
|
||||
- modified timestamp;
|
||||
- source format.
|
||||
|
||||
### Knowledge Metadata
|
||||
|
||||
Information that requires classification or interpretation, such as:
|
||||
|
||||
- primary topics;
|
||||
- reasoning contexts;
|
||||
- content types;
|
||||
- entities;
|
||||
- controlled vocabulary;
|
||||
- relationships.
|
||||
|
||||
Knowledge Metadata is explicitly out of scope for this work order.
|
||||
|
||||
Project Thoth already has separate Source Metadata and Conversation Manifest processors. This baseline RAG pipeline must not silently reproduce their responsibilities.
|
||||
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
The Project Thoth vault already contains many saved conversations as Markdown files.
|
||||
|
||||
Before those files can participate in a RAG pipeline, the system needs a deterministic way to discover and represent a source document.
|
||||
|
||||
Most RAG frameworks combine loading, parsing, chunking, metadata extraction, and indexing behind high-level abstractions. That is useful for rapid application development, but it can obscure the mechanics that this implementation is intended to teach.
|
||||
|
||||
The first step should therefore be implemented directly in Python with minimal dependencies.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Create a dedicated Python package at:
|
||||
|
||||
`./processors/rag_ingestion/`
|
||||
|
||||
The package will initially contain only the code necessary for single-file Markdown ingestion.
|
||||
|
||||
Use Python package naming conventions (`rag_ingestion`) even though the application and work-order directory use the name `rag-ingestion`.
|
||||
|
||||
The source Markdown file remains untouched.
|
||||
|
||||
The ingestion result is a derived in-memory representation.
|
||||
|
||||
Do not introduce LangChain, LlamaIndex, Haystack, or another RAG framework in this work order.
|
||||
|
||||
---
|
||||
|
||||
## Proposed Package Structure
|
||||
|
||||
```text
|
||||
processors/
|
||||
└── rag_ingestion/
|
||||
├── __init__.py
|
||||
├── models.py
|
||||
├── loader.py
|
||||
└── ingest.py
|
||||
```
|
||||
|
||||
The exact internal organization may vary slightly if repository conventions require it, but responsibilities should remain separated.
|
||||
|
||||
### `models.py`
|
||||
|
||||
Defines the document representation used by the ingestion layer.
|
||||
|
||||
### `loader.py`
|
||||
|
||||
Contains the logic for reading a Markdown source and gathering filesystem metadata.
|
||||
|
||||
### `ingest.py`
|
||||
|
||||
Provides the command-line entry point for ingesting one Markdown file and displaying the resulting document record.
|
||||
|
||||
---
|
||||
|
||||
## Document Record
|
||||
|
||||
The initial document representation should include, at minimum:
|
||||
|
||||
- `document_id`
|
||||
- `source_path`
|
||||
- `filename`
|
||||
- `source_format`
|
||||
- `file_size`
|
||||
- `modified_at`
|
||||
- `raw_text`
|
||||
|
||||
The implementation may add a small number of additional mechanical fields if required by existing project conventions.
|
||||
|
||||
Do not add semantic or AI-generated fields.
|
||||
|
||||
---
|
||||
|
||||
## Deterministic Document Identifier
|
||||
|
||||
The `document_id` must be deterministic.
|
||||
|
||||
Running ingestion repeatedly against the same logical source should produce the same identifier.
|
||||
|
||||
Do not use a random UUID.
|
||||
|
||||
Select and document a simple deterministic strategy.
|
||||
|
||||
Examples of acceptable inputs to the identifier calculation include:
|
||||
|
||||
- normalized source path;
|
||||
- repository-relative or configured corpus-relative path.
|
||||
|
||||
Do not base the identifier solely on file contents because editing a source should not automatically turn the same logical source into a different document identity.
|
||||
|
||||
The implementation should make the chosen identity rule obvious in the code.
|
||||
|
||||
---
|
||||
|
||||
## Command-Line Behavior
|
||||
|
||||
Provide a simple invocation for one file.
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest /path/to/conversation.md
|
||||
```
|
||||
|
||||
The exact argument syntax may follow existing repository conventions.
|
||||
|
||||
The command should:
|
||||
|
||||
1. validate that the path exists;
|
||||
2. validate that the source is a Markdown file;
|
||||
3. load the source;
|
||||
4. construct the document record;
|
||||
5. print the document record in a readable form;
|
||||
6. exit successfully.
|
||||
|
||||
Invalid inputs should fail clearly with an actionable error.
|
||||
|
||||
---
|
||||
|
||||
## Inspectability Requirement
|
||||
|
||||
The output must make it possible to verify what the ingestion process learned from the source.
|
||||
|
||||
At minimum, the displayed output should make the following visible:
|
||||
|
||||
```text
|
||||
Document ID:
|
||||
Source Path:
|
||||
Filename:
|
||||
Source Format:
|
||||
File Size:
|
||||
Modified At:
|
||||
Raw Text Length:
|
||||
```
|
||||
|
||||
It is not necessary to print the entire Markdown document by default.
|
||||
|
||||
The implementation should allow the developer to inspect the raw text easily in code or during debugging.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
1. Create the `rag_ingestion` Python package under `./processors`.
|
||||
2. Implement ingestion for one Markdown file only.
|
||||
3. Preserve the source file exactly as it exists.
|
||||
4. Capture only deterministic/mechanical metadata.
|
||||
5. Generate a deterministic document ID.
|
||||
6. Keep source loading separate from the document model.
|
||||
7. Provide a CLI entry point.
|
||||
8. Produce readable inspection output.
|
||||
9. Use clear type annotations.
|
||||
10. Include docstrings where they improve understanding.
|
||||
11. Keep dependencies minimal.
|
||||
12. Do not hide the ingestion process behind a RAG framework.
|
||||
13. Follow existing Project Thoth repository conventions.
|
||||
14. Do not perform opportunistic refactoring outside this work order.
|
||||
|
||||
---
|
||||
|
||||
## Non-Goals
|
||||
|
||||
Do not implement any of the following:
|
||||
|
||||
- recursive directory traversal;
|
||||
- Nextcloud integration;
|
||||
- Markdown semantic parsing;
|
||||
- conversation-specific parsing;
|
||||
- chunking;
|
||||
- token counting for chunking;
|
||||
- embeddings;
|
||||
- embedding models;
|
||||
- vector databases;
|
||||
- PostgreSQL;
|
||||
- pgvector;
|
||||
- lexical indexing;
|
||||
- retrieval;
|
||||
- reranking;
|
||||
- prompt construction;
|
||||
- Oracle communication;
|
||||
- LLM inference;
|
||||
- Source Metadata generation;
|
||||
- Conversation Manifest generation;
|
||||
- knowledge extraction;
|
||||
- taxonomy;
|
||||
- relationship extraction;
|
||||
- web UI;
|
||||
- background watchers;
|
||||
- automatic re-ingestion.
|
||||
|
||||
These belong to later work orders.
|
||||
|
||||
---
|
||||
|
||||
## Manual Test Plan
|
||||
|
||||
Use at least one real Markdown conversation from the Project Thoth archive for the manual test.
|
||||
|
||||
### Test 1 — Valid Markdown source
|
||||
|
||||
Run the ingestion command against a real `.md` conversation.
|
||||
|
||||
Verify:
|
||||
|
||||
- the file loads successfully;
|
||||
- the document ID is produced;
|
||||
- source path is correct;
|
||||
- filename is correct;
|
||||
- source format identifies Markdown;
|
||||
- file size corresponds to the filesystem;
|
||||
- modified timestamp is captured;
|
||||
- raw text length is greater than zero.
|
||||
|
||||
### Test 2 — Deterministic identity
|
||||
|
||||
Run the command against the same source multiple times.
|
||||
|
||||
Verify:
|
||||
|
||||
- `document_id` is identical on each run.
|
||||
|
||||
### Test 3 — Different source
|
||||
|
||||
Run the command against a second Markdown file.
|
||||
|
||||
Verify:
|
||||
|
||||
- the second source receives a different `document_id`.
|
||||
|
||||
### Test 4 — Missing source
|
||||
|
||||
Run the command against a path that does not exist.
|
||||
|
||||
Verify:
|
||||
|
||||
- the program fails clearly;
|
||||
- the error identifies the missing path.
|
||||
|
||||
### Test 5 — Wrong file type
|
||||
|
||||
Run the command against a non-Markdown file.
|
||||
|
||||
Verify:
|
||||
|
||||
- the program rejects the input;
|
||||
- the error explains that this ingestion path currently supports Markdown only.
|
||||
|
||||
### Test 6 — Source preservation
|
||||
|
||||
Record the source file hash before and after ingestion.
|
||||
|
||||
Verify:
|
||||
|
||||
- the source file hash is unchanged.
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
This work order is complete when:
|
||||
|
||||
- `./processors/rag_ingestion/` exists as an importable Python package;
|
||||
- a developer can invoke the ingestion pipeline against one Markdown file;
|
||||
- the source is represented by a typed document model;
|
||||
- all required mechanical fields are populated;
|
||||
- the document ID is deterministic;
|
||||
- the source file is not modified;
|
||||
- no chunking or AI processing occurs;
|
||||
- the implementation contains no RAG orchestration framework;
|
||||
- all manual tests pass;
|
||||
- the developer can trace the execution from CLI input to document record without encountering a hidden framework pipeline.
|
||||
|
||||
---
|
||||
|
||||
## Learning Verification
|
||||
|
||||
After implementation, Codex should include a short implementation report that answers the following questions using the actual code that was built:
|
||||
|
||||
1. Where does ingestion begin in this implementation?
|
||||
2. Which values come directly from the filesystem?
|
||||
3. Which value is generated by the ingestion system?
|
||||
4. How is document identity made deterministic?
|
||||
5. Why is `raw_text` retained even though later stages will operate on chunks?
|
||||
6. What information has deliberately not been inferred from the document?
|
||||
7. At what exact point will the next work order begin adding derived retrieval units?
|
||||
|
||||
The answers should reference the relevant files/functions in the implementation.
|
||||
|
||||
This section is required because Project Thoth work orders are being used both to build the system and to develop a deep technical understanding of RAG.
|
||||
|
||||
---
|
||||
|
||||
## Definition of Done
|
||||
|
||||
The first RAG ingestion boundary exists and is operational.
|
||||
|
||||
A real Project Thoth Markdown conversation can be passed into the application and converted into an inspectable deterministic document record without modifying the Primary Source or performing any semantic processing.
|
||||
|
||||
The implementation is sufficiently transparent that the developer can explain every field in the record, where it came from, and why it exists.
|
||||
|
||||
The next work order can begin with chunking from this known document representation rather than combining source ingestion and retrieval processing into a single opaque step.
|
||||
|
||||
---
|
||||
|
||||
## Codex Execution Guidance
|
||||
|
||||
Before making changes:
|
||||
|
||||
1. Read the repository bootstrap and architectural guidance.
|
||||
2. Inspect existing `./processors` conventions.
|
||||
3. Inspect existing Python project/dependency configuration.
|
||||
4. Do not assume a new dependency manager or application layout if the repository already defines one.
|
||||
5. Plan the narrowest set of files required for this work order.
|
||||
|
||||
During implementation:
|
||||
|
||||
1. Keep the code explicit and educational.
|
||||
2. Prefer standard-library functionality where practical.
|
||||
3. Preserve separation between source loading and the document model.
|
||||
4. Do not implement future work-order functionality "while already in the code."
|
||||
5. Add only the tests or supporting files necessary to validate this work order.
|
||||
|
||||
At completion, report:
|
||||
|
||||
- root cause / need addressed;
|
||||
- files created or changed;
|
||||
- implementation decisions;
|
||||
- deterministic ID strategy;
|
||||
- validation performed;
|
||||
- manual test results;
|
||||
- learning verification answers;
|
||||
- anything that could not be verified.
|
||||
|
||||
@@ -0,0 +1,705 @@
|
||||
# Work Order 0002 — Add Inspectable Chunking
|
||||
|
||||
## Status
|
||||
|
||||
Planned
|
||||
|
||||
## Work Stream
|
||||
|
||||
RAG Ingestion
|
||||
|
||||
## Application
|
||||
|
||||
`rag-ingestion`
|
||||
|
||||
## Location
|
||||
|
||||
Work order:
|
||||
|
||||
`./codex/rag-ingestion/0002-add-inspectable-chunking.md`
|
||||
|
||||
Implementation package:
|
||||
|
||||
`./processors/rag_ingestion/`
|
||||
|
||||
---
|
||||
|
||||
## Objective
|
||||
|
||||
Extend the existing `rag-ingestion` application so that a successfully ingested Markdown document can be divided into deterministic, inspectable retrieval units.
|
||||
|
||||
This work order begins where Work Order 0001 ended.
|
||||
|
||||
The existing ingestion pipeline already produces a document record containing the Primary Source text and mechanical metadata. This work order must add a chunking stage that accepts that document record and produces an ordered collection of chunk records.
|
||||
|
||||
The implementation must make chunk boundaries visible and understandable.
|
||||
|
||||
This work order intentionally stops before embeddings, vector storage, PostgreSQL, pgvector, retrieval, reranking, prompt construction, or Oracle integration.
|
||||
|
||||
---
|
||||
|
||||
## Learning Objective
|
||||
|
||||
This work order is both an implementation task and a technical learning exercise.
|
||||
|
||||
The developer should finish the work order able to explain:
|
||||
|
||||
- why RAG systems usually divide documents into chunks;
|
||||
- what a retrieval unit is;
|
||||
- how chunk size changes the amount of context contained in each retrieval unit;
|
||||
- why overlap is commonly introduced between adjacent chunks;
|
||||
- what information is duplicated by overlap;
|
||||
- how chunk boundaries can separate ideas that were coherent in the original source;
|
||||
- why chunking is a derived transformation rather than a property of the Primary Source;
|
||||
- why deterministic chunking is important for debugging, testing, indexing, and re-ingestion;
|
||||
- the difference between character-based, token-based, structural, and semantic chunking;
|
||||
- which chunking strategy this implementation uses and why it was selected as the baseline.
|
||||
|
||||
Codex must prefer transparent implementation over high-level framework abstractions.
|
||||
|
||||
The developer should be able to trace, line by line, how `raw_text` becomes a sequence of chunks.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Context
|
||||
|
||||
Project Thoth treats the source Markdown file as the authoritative Primary Source.
|
||||
|
||||
Chunk records are derived retrieval artifacts.
|
||||
|
||||
They do not replace the Primary Source.
|
||||
|
||||
Conceptually:
|
||||
|
||||
```text
|
||||
Primary Source
|
||||
↓
|
||||
Document Loader
|
||||
↓
|
||||
Document Record
|
||||
↓
|
||||
Chunker
|
||||
↓
|
||||
Chunk Records
|
||||
```
|
||||
|
||||
Work Order 0001 established the first three stages.
|
||||
|
||||
This work order adds only the fourth.
|
||||
|
||||
No AI interpretation should occur during chunking.
|
||||
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
A complete conversation may be too large to retrieve and insert into an LLM context for every query.
|
||||
|
||||
RAG systems therefore typically divide documents into smaller units that can be indexed and retrieved independently.
|
||||
|
||||
This introduces an architectural tradeoff.
|
||||
|
||||
If chunks are too large:
|
||||
|
||||
- retrieval becomes coarse;
|
||||
- irrelevant text may accompany relevant text;
|
||||
- fewer distinct sources may fit into a context window.
|
||||
|
||||
If chunks are too small:
|
||||
|
||||
- concepts may lose surrounding context;
|
||||
- relationships between statements may be separated;
|
||||
- retrieved fragments may become ambiguous;
|
||||
- a single idea may require multiple chunks to reconstruct.
|
||||
|
||||
Overlap can mitigate some boundary loss, but overlap also duplicates content and increases index size.
|
||||
|
||||
The goal of this work order is not to find the "perfect" chunking strategy.
|
||||
|
||||
The goal is to implement a clear baseline and make its behavior observable against real Project Thoth conversations.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a simple deterministic chunker directly in Python.
|
||||
|
||||
Do not introduce LangChain, LlamaIndex, Haystack, or another RAG framework.
|
||||
|
||||
Use a baseline chunking strategy that is easy to inspect and reason about.
|
||||
|
||||
### Baseline Strategy
|
||||
|
||||
Use token-aware chunking if a tokenizer is already available in the repository with minimal dependency impact.
|
||||
|
||||
If no suitable tokenizer exists, use a clearly documented character-based approximation for this work order rather than introducing a large dependency solely for token counting.
|
||||
|
||||
The selected strategy must support:
|
||||
|
||||
- configurable target chunk size;
|
||||
- configurable overlap;
|
||||
- deterministic output;
|
||||
- ordered chunks;
|
||||
- stable chunk identifiers.
|
||||
|
||||
The implementation must document whether size and overlap are measured in tokens or characters.
|
||||
|
||||
Do not describe character counts as tokens.
|
||||
|
||||
---
|
||||
|
||||
## Default Configuration
|
||||
|
||||
Choose conservative baseline defaults suitable for long Markdown conversations.
|
||||
|
||||
Recommended starting values if using tokens:
|
||||
|
||||
```text
|
||||
chunk_size = 800 tokens
|
||||
chunk_overlap = 120 tokens
|
||||
```
|
||||
|
||||
If using characters, choose an approximate equivalent and document the rationale.
|
||||
|
||||
Defaults must be configurable from the CLI.
|
||||
|
||||
The exact values may be adjusted if existing repository conventions or tokenizer behavior justify a different baseline.
|
||||
|
||||
The purpose of the defaults is to provide a starting point for observation, not to assert that they are optimal.
|
||||
|
||||
---
|
||||
|
||||
## Proposed Package Structure
|
||||
|
||||
Extend the existing package approximately as follows:
|
||||
|
||||
```text
|
||||
processors/
|
||||
└── rag_ingestion/
|
||||
├── __init__.py
|
||||
├── models.py
|
||||
├── loader.py
|
||||
├── chunking.py
|
||||
└── ingest.py
|
||||
```
|
||||
|
||||
Existing files from Work Order 0001 should be modified only where necessary.
|
||||
|
||||
### `models.py`
|
||||
|
||||
Extend the existing model definitions with a chunk representation.
|
||||
|
||||
### `chunking.py`
|
||||
|
||||
Contain the chunking logic and chunk configuration.
|
||||
|
||||
### `ingest.py`
|
||||
|
||||
Extend the existing CLI so that, after loading the document, it can chunk the source and print an inspectable summary.
|
||||
|
||||
Do not collapse loader and chunker responsibilities into one function.
|
||||
|
||||
---
|
||||
|
||||
## Chunk Record
|
||||
|
||||
Each chunk should include, at minimum:
|
||||
|
||||
- `chunk_id`
|
||||
- `document_id`
|
||||
- `chunk_number`
|
||||
- `text`
|
||||
- `start_offset`
|
||||
- `end_offset`
|
||||
- `size`
|
||||
- `size_unit`
|
||||
|
||||
Where practical, include:
|
||||
|
||||
- `overlap_with_previous`
|
||||
- `overlap_with_next`
|
||||
|
||||
The exact representation may vary to fit existing project conventions.
|
||||
|
||||
### Field Intent
|
||||
|
||||
`chunk_id`
|
||||
|
||||
A deterministic identifier for this retrieval unit.
|
||||
|
||||
`document_id`
|
||||
|
||||
Links the chunk back to the source document record created in Work Order 0001.
|
||||
|
||||
`chunk_number`
|
||||
|
||||
Zero-based or one-based ordered position in the source. Choose one convention and document it.
|
||||
|
||||
`text`
|
||||
|
||||
The exact derived text included in this chunk.
|
||||
|
||||
`start_offset` and `end_offset`
|
||||
|
||||
Allow the developer to understand where the chunk came from in the original document.
|
||||
|
||||
Offsets must use a clearly documented unit.
|
||||
|
||||
Character offsets are acceptable even when chunk size is token-based.
|
||||
|
||||
`size`
|
||||
|
||||
The measured size of the chunk.
|
||||
|
||||
`size_unit`
|
||||
|
||||
Must explicitly state whether the size is in `tokens` or `characters`.
|
||||
|
||||
---
|
||||
|
||||
## Deterministic Chunk Identifier
|
||||
|
||||
Chunk IDs must be deterministic.
|
||||
|
||||
The same source document processed with the same chunking configuration must produce the same chunk IDs.
|
||||
|
||||
A reasonable strategy is to derive the chunk ID from:
|
||||
|
||||
- `document_id`;
|
||||
- chunk number;
|
||||
- chunking configuration.
|
||||
|
||||
Do not use random UUIDs.
|
||||
|
||||
If changing the chunking configuration changes the boundaries, the resulting chunk identity should be allowed to change.
|
||||
|
||||
Document the exact strategy in the implementation.
|
||||
|
||||
---
|
||||
|
||||
## Chunking Behavior
|
||||
|
||||
The chunker must:
|
||||
|
||||
1. accept the document record from Work Order 0001;
|
||||
2. operate on the document's `raw_text`;
|
||||
3. produce chunks in source order;
|
||||
4. preserve the text exactly within each selected region;
|
||||
5. implement configured overlap;
|
||||
6. avoid emitting empty chunks;
|
||||
7. handle documents smaller than the configured chunk size;
|
||||
8. behave deterministically.
|
||||
|
||||
Do not summarize, clean, rewrite, normalize, classify, or semantically interpret the source text.
|
||||
|
||||
Do not strip Markdown merely because it is Markdown.
|
||||
|
||||
For this baseline, Markdown syntax is part of the text being chunked.
|
||||
|
||||
---
|
||||
|
||||
## Inspectability Requirement
|
||||
|
||||
The console output must make chunk boundaries easy to inspect.
|
||||
|
||||
Do not print every complete chunk by default if doing so makes normal output unusable.
|
||||
|
||||
For each chunk, display at minimum:
|
||||
|
||||
```text
|
||||
Chunk:
|
||||
Chunk ID:
|
||||
Chunk Number:
|
||||
Size:
|
||||
Offsets:
|
||||
Starts With:
|
||||
Ends With:
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
```text
|
||||
Chunk 3
|
||||
Chunk ID: ...
|
||||
Size: 792 tokens
|
||||
Offsets: 10422-14691
|
||||
Starts With: "The problem with conventional RAG..."
|
||||
Ends With: "...before retrieval ever begins."
|
||||
```
|
||||
|
||||
Preview lengths should be long enough to reveal whether a coherent thought was split.
|
||||
|
||||
Provide an optional CLI flag that prints the complete chunk text for detailed inspection.
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest conversation.md --show-chunks
|
||||
```
|
||||
|
||||
The exact flag name may follow repository conventions.
|
||||
|
||||
---
|
||||
|
||||
## CLI Configuration
|
||||
|
||||
Extend the CLI to accept chunking parameters.
|
||||
|
||||
Conceptually:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest \
|
||||
/path/to/conversation.md \
|
||||
--chunk-size 800 \
|
||||
--chunk-overlap 120
|
||||
```
|
||||
|
||||
The implementation must validate configuration.
|
||||
|
||||
At minimum:
|
||||
|
||||
- chunk size must be greater than zero;
|
||||
- overlap must not be negative;
|
||||
- overlap must be smaller than chunk size.
|
||||
|
||||
Invalid configuration should fail with a clear message.
|
||||
|
||||
---
|
||||
|
||||
## Comparison Mode
|
||||
|
||||
Because this work order has a learning objective, provide a simple way to run the same source with different chunk configurations without editing source code.
|
||||
|
||||
This does not need to be a sophisticated benchmarking system.
|
||||
|
||||
The CLI parameters themselves may satisfy this requirement if the resulting summary clearly reports:
|
||||
|
||||
- source size;
|
||||
- configured chunk size;
|
||||
- configured overlap;
|
||||
- resulting number of chunks;
|
||||
- minimum chunk size;
|
||||
- maximum chunk size;
|
||||
- average chunk size.
|
||||
|
||||
This should allow the developer to run, for example:
|
||||
|
||||
```text
|
||||
800 / 120
|
||||
400 / 60
|
||||
1200 / 180
|
||||
```
|
||||
|
||||
and observe how the same conversation is transformed.
|
||||
|
||||
Do not automatically select a "best" configuration.
|
||||
|
||||
The purpose is observation.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
1. Build on the document model created in Work Order 0001.
|
||||
2. Add a dedicated chunk model.
|
||||
3. Add a dedicated chunking module.
|
||||
4. Keep chunking separate from source loading.
|
||||
5. Implement deterministic chunking.
|
||||
6. Implement configurable chunk size.
|
||||
7. Implement configurable overlap.
|
||||
8. Report the unit used for chunk size.
|
||||
9. Generate deterministic chunk IDs.
|
||||
10. Preserve ordered linkage from chunks to the source document.
|
||||
11. Provide source offsets or another direct way to trace a chunk back into the Primary Source.
|
||||
12. Provide readable chunk previews.
|
||||
13. Provide an option to display complete chunk text.
|
||||
14. Report aggregate chunk statistics.
|
||||
15. Reject invalid chunk configuration.
|
||||
16. Add type annotations.
|
||||
17. Add tests for chunking behavior.
|
||||
18. Keep dependencies minimal.
|
||||
19. Do not introduce a RAG orchestration framework.
|
||||
20. Do not perform opportunistic refactoring outside this work order.
|
||||
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
Add automated tests appropriate to the existing repository conventions.
|
||||
|
||||
At minimum test:
|
||||
|
||||
### Small document
|
||||
|
||||
A document smaller than the chunk size should produce one chunk.
|
||||
|
||||
### Multiple chunks
|
||||
|
||||
A document larger than the chunk size should produce more than one ordered chunk.
|
||||
|
||||
### Overlap
|
||||
|
||||
Adjacent chunks should contain the configured amount of overlap, subject to the selected sizing strategy.
|
||||
|
||||
### No overlap
|
||||
|
||||
A configuration with zero overlap should produce adjacent non-overlapping chunks.
|
||||
|
||||
### Invalid overlap
|
||||
|
||||
Overlap equal to or greater than chunk size should be rejected.
|
||||
|
||||
### Invalid chunk size
|
||||
|
||||
Zero or negative chunk size should be rejected.
|
||||
|
||||
### Determinism
|
||||
|
||||
The same document and same configuration should produce identical chunk boundaries and chunk IDs across repeated runs.
|
||||
|
||||
### Configuration sensitivity
|
||||
|
||||
Changing chunk size or overlap should change chunk boundaries and/or chunk IDs when appropriate.
|
||||
|
||||
### Source preservation
|
||||
|
||||
Chunking must not modify the document's `raw_text` or the source file.
|
||||
|
||||
### Empty input
|
||||
|
||||
Define and test expected behavior for an empty Markdown file.
|
||||
|
||||
Prefer a clear explicit result or validation error rather than accidental behavior.
|
||||
|
||||
---
|
||||
|
||||
## Manual Test Plan
|
||||
|
||||
Use at least one real Project Thoth conversation from the Vault.
|
||||
|
||||
Prefer a conversation the developer knows well enough to recognize topic boundaries.
|
||||
|
||||
### Test 1 — Baseline configuration
|
||||
|
||||
Run with the default chunk configuration.
|
||||
|
||||
Record:
|
||||
|
||||
- source size;
|
||||
- number of chunks;
|
||||
- average chunk size;
|
||||
- first and last chunk previews.
|
||||
|
||||
Inspect several internal chunk boundaries.
|
||||
|
||||
Ask:
|
||||
|
||||
- Does a paragraph get split?
|
||||
- Does a speaker turn get split?
|
||||
- Does a single argument cross multiple chunks?
|
||||
- Does the overlap preserve enough context to understand the continuation?
|
||||
|
||||
Do not change the algorithm merely because a split looks awkward. Record the observation first.
|
||||
|
||||
### Test 2 — Smaller chunks
|
||||
|
||||
Run the same source with approximately half the default chunk size.
|
||||
|
||||
Compare:
|
||||
|
||||
- number of chunks;
|
||||
- boundary frequency;
|
||||
- amount of duplicated text;
|
||||
- apparent loss of local context.
|
||||
|
||||
### Test 3 — Larger chunks
|
||||
|
||||
Run the same source with a larger chunk size.
|
||||
|
||||
Compare:
|
||||
|
||||
- number of chunks;
|
||||
- breadth of context in each chunk;
|
||||
- likelihood that unrelated material appears in one retrieval unit.
|
||||
|
||||
### Test 4 — No overlap
|
||||
|
||||
Run with overlap set to zero.
|
||||
|
||||
Inspect boundaries where an idea spans chunks.
|
||||
|
||||
Compare against the baseline run.
|
||||
|
||||
### Test 5 — Repeatability
|
||||
|
||||
Run the same source twice with identical settings.
|
||||
|
||||
Verify:
|
||||
|
||||
- identical chunk count;
|
||||
- identical boundaries;
|
||||
- identical chunk IDs.
|
||||
|
||||
---
|
||||
|
||||
## Observation Notes
|
||||
|
||||
At the end of the manual test, record a short set of observations in the implementation report.
|
||||
|
||||
These are observations, not architecture decisions.
|
||||
|
||||
Examples of useful observations:
|
||||
|
||||
- "A speaker response was divided between chunks 6 and 7."
|
||||
- "120 tokens of overlap repeated approximately one paragraph in this source."
|
||||
- "The smaller configuration produced significantly more fragments around list structures."
|
||||
- "The larger chunks kept complete exchanges together but also combined two distinct topics."
|
||||
|
||||
Do not add semantic chunking or special Markdown rules in response to these observations during this work order.
|
||||
|
||||
Those findings are evidence for later design decisions.
|
||||
|
||||
---
|
||||
|
||||
## Non-Goals
|
||||
|
||||
Do not implement:
|
||||
|
||||
- embeddings;
|
||||
- embedding model selection;
|
||||
- vector generation;
|
||||
- PostgreSQL;
|
||||
- pgvector;
|
||||
- vector indexing;
|
||||
- BM25 or lexical indexing;
|
||||
- retrieval;
|
||||
- hybrid search;
|
||||
- reranking;
|
||||
- query processing;
|
||||
- prompt construction;
|
||||
- Oracle communication;
|
||||
- LLM inference;
|
||||
- recursive Vault ingestion;
|
||||
- directory watching;
|
||||
- automatic re-ingestion;
|
||||
- Source Metadata generation;
|
||||
- Conversation Manifest generation;
|
||||
- semantic chunking;
|
||||
- LLM-based chunking;
|
||||
- Markdown heading-aware chunking;
|
||||
- speaker-turn-aware chunking;
|
||||
- knowledge graph processing;
|
||||
- relationship extraction;
|
||||
- web UI.
|
||||
|
||||
These may be evaluated in later work orders.
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
This work order is complete when:
|
||||
|
||||
- the existing document ingestion flow still works;
|
||||
- a loaded document can be passed to a dedicated chunker;
|
||||
- multiple deterministic chunk records are created when appropriate;
|
||||
- each chunk links back to its source document;
|
||||
- each chunk can be traced to a location in the original source;
|
||||
- chunk size and overlap are configurable;
|
||||
- the sizing unit is explicitly reported;
|
||||
- chunk previews make boundaries visible;
|
||||
- complete chunks can be displayed on request;
|
||||
- aggregate chunk statistics are displayed;
|
||||
- automated tests pass;
|
||||
- a real Vault conversation has been tested with multiple configurations;
|
||||
- repeat runs with the same configuration produce identical results;
|
||||
- no embeddings, persistence, or AI processing have been added.
|
||||
|
||||
---
|
||||
|
||||
## Learning Verification
|
||||
|
||||
At completion, Codex must provide a short implementation report answering the following questions from the actual implementation:
|
||||
|
||||
1. What exactly is a chunk in this implementation?
|
||||
2. What determines where one chunk ends and the next begins?
|
||||
3. Is chunk size measured in tokens or characters, and why?
|
||||
4. What does overlap do?
|
||||
5. What content is duplicated because of overlap?
|
||||
6. How can a chunk be traced back to the Primary Source?
|
||||
7. What makes chunk IDs deterministic?
|
||||
8. What happens to chunk identity when the chunk configuration changes?
|
||||
9. What information from the original document is lost by creating chunks?
|
||||
10. What information is duplicated?
|
||||
11. What information is not represented in the chunk model?
|
||||
12. Where does chunking end and embedding begin?
|
||||
13. Based on the real Vault test, identify at least two examples where the mechanical chunk boundaries did not correspond cleanly to conceptual boundaries.
|
||||
14. Explain why those observations are not being "fixed" in this work order.
|
||||
|
||||
The answers should reference actual modules, classes, functions, or tests.
|
||||
|
||||
---
|
||||
|
||||
## Definition of Done
|
||||
|
||||
Project Thoth now has an inspectable baseline chunking stage.
|
||||
|
||||
A real Markdown conversation can move through:
|
||||
|
||||
```text
|
||||
Primary Source
|
||||
↓
|
||||
Document Record
|
||||
↓
|
||||
Chunk Records
|
||||
```
|
||||
|
||||
without modifying the Primary Source and without introducing semantic interpretation.
|
||||
|
||||
The developer can inspect exactly where every chunk begins and ends, understand why the boundaries occur, see what overlap duplicates, and compare how configuration changes transform the same source.
|
||||
|
||||
The implementation is intentionally mechanical.
|
||||
|
||||
The next work order can begin with embeddings using known, observable retrieval units rather than hiding chunk creation inside an embedding or indexing framework.
|
||||
|
||||
---
|
||||
|
||||
## Codex Execution Guidance
|
||||
|
||||
Before changing code:
|
||||
|
||||
1. Read the complete Work Order 0001 implementation and completion report.
|
||||
2. Read this work order completely.
|
||||
3. Inspect the current `./processors/rag_ingestion/` package.
|
||||
4. Inspect existing tests and dependency configuration.
|
||||
5. Identify whether a tokenizer is already available.
|
||||
6. Decide whether the baseline will use tokens or characters.
|
||||
7. Explain that choice briefly before editing code.
|
||||
8. Summarize the narrow implementation plan.
|
||||
|
||||
During implementation:
|
||||
|
||||
1. Preserve the existing ingestion behavior.
|
||||
2. Keep the chunker independent of the loader.
|
||||
3. Make boundary calculations explicit.
|
||||
4. Keep configuration visible and easy to modify.
|
||||
5. Prefer simple functions/classes that can be inspected in VS Code.
|
||||
6. Do not introduce future pipeline stages.
|
||||
7. Do not optimize based on imagined future requirements.
|
||||
8. Do not "fix" awkward semantic boundaries encountered during testing.
|
||||
|
||||
At completion, report:
|
||||
|
||||
- files created or changed;
|
||||
- selected chunking strategy;
|
||||
- sizing unit;
|
||||
- default chunk size and overlap;
|
||||
- deterministic chunk ID strategy;
|
||||
- automated test results;
|
||||
- manual test configurations and results;
|
||||
- notable boundary observations from the real Vault conversation;
|
||||
- Learning Verification answers;
|
||||
- assumptions or limitations;
|
||||
- anything that could not be validated.
|
||||
|
||||
Do not begin Work Order 0003.
|
||||
@@ -0,0 +1,760 @@
|
||||
# Work Order 0003 — Add Inspectable Embeddings and Direct Similarity
|
||||
|
||||
## Status
|
||||
|
||||
Planned
|
||||
|
||||
## Work Stream
|
||||
|
||||
RAG Ingestion
|
||||
|
||||
## Application
|
||||
|
||||
`rag-ingestion`
|
||||
|
||||
## Location
|
||||
|
||||
Work order:
|
||||
|
||||
`./codex/rag-ingestion/0003-add-inspectable-embeddings.md`
|
||||
|
||||
Implementation package:
|
||||
|
||||
`./processors/rag_ingestion/`
|
||||
|
||||
---
|
||||
|
||||
## Objective
|
||||
|
||||
Extend the existing `rag-ingestion` application with an embedding stage that converts the deterministic chunk records created in Work Order 0002 into numerical vector representations.
|
||||
|
||||
This work order must make embeddings observable and testable before any vector database is introduced.
|
||||
|
||||
The implementation should allow the developer to:
|
||||
|
||||
1. ingest a real Project Thoth Markdown source;
|
||||
2. chunk it using the existing deterministic chunking stage;
|
||||
3. select one or more chunks;
|
||||
4. generate an embedding vector for each selected chunk;
|
||||
5. inspect basic properties of each embedding;
|
||||
6. calculate cosine similarity directly between selected embeddings;
|
||||
7. compare semantically related and unrelated chunks using actual numerical scores.
|
||||
|
||||
This work order intentionally stops before PostgreSQL, pgvector, persistent indexing, corpus-wide vector search, retrieval, reranking, prompt assembly, or Oracle generation.
|
||||
|
||||
---
|
||||
|
||||
## Learning Objective
|
||||
|
||||
This work order is both an implementation task and a technical learning exercise.
|
||||
|
||||
At completion, the developer should be able to explain:
|
||||
|
||||
- what an embedding is in practical terms;
|
||||
- what input is sent to an embedding model;
|
||||
- what the model returns;
|
||||
- what vector dimensionality means;
|
||||
- why an embedding vector is not human-readable metadata;
|
||||
- why two chunks can be compared mathematically even though their wording differs;
|
||||
- how cosine similarity is calculated and interpreted;
|
||||
- why similarity is not the same as relevance;
|
||||
- why every valid input, including weak or structurally poor chunks, can still receive a vector;
|
||||
- how changing the embedding model changes the representation space;
|
||||
- why embeddings generated by different models or model versions generally cannot be mixed safely in one vector index;
|
||||
- where embedding ends and vector indexing/retrieval begins.
|
||||
|
||||
Codex must keep the implementation explicit enough that the developer can trace a chunk's text into the embedding call and then trace the returned vector into the similarity calculation.
|
||||
|
||||
Do not hide embedding behavior behind a RAG orchestration framework.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Context
|
||||
|
||||
The current baseline pipeline is:
|
||||
|
||||
```text
|
||||
Primary Source
|
||||
↓
|
||||
Document Loader
|
||||
↓
|
||||
Document Record
|
||||
↓
|
||||
Deterministic Chunker
|
||||
↓
|
||||
Chunk Records
|
||||
```
|
||||
|
||||
This work order adds:
|
||||
|
||||
```text
|
||||
Chunk Record
|
||||
↓
|
||||
Embedding Provider
|
||||
↓
|
||||
Embedding Record
|
||||
```
|
||||
|
||||
For direct comparison:
|
||||
|
||||
```text
|
||||
Embedding A ─┐
|
||||
├── Cosine Similarity ──> Score
|
||||
Embedding B ─┘
|
||||
```
|
||||
|
||||
The embedding is a derived retrieval artifact.
|
||||
|
||||
It is not part of the authoritative Primary Source and does not replace the chunk text.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Boundary
|
||||
|
||||
Project Thoth currently separates Gateway and Oracle responsibilities.
|
||||
|
||||
For this work order:
|
||||
|
||||
- do not move RAG orchestration or storage responsibilities to Oracle;
|
||||
- do not add generative LLM calls;
|
||||
- do not use Oracle as a hidden all-purpose RAG service;
|
||||
- prefer running embedding generation on Gateway or through a clearly defined embedding service reachable from the development environment.
|
||||
|
||||
If the current environment makes local embedding generation on Gateway impractical during development, Codex may implement the provider boundary so the embedding endpoint is configurable.
|
||||
|
||||
Any deviation from the intended Gateway-local embedding role must be documented rather than silently introduced.
|
||||
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
After chunking, conventional RAG systems commonly convert each retrieval unit into an embedding.
|
||||
|
||||
An embedding model maps text into a fixed-length numerical vector. Retrieval systems can then compare vectors to estimate semantic similarity.
|
||||
|
||||
This transformation is frequently treated as a black box:
|
||||
|
||||
```text
|
||||
text → embedding → vector database
|
||||
```
|
||||
|
||||
That is insufficient for the learning objective of this project.
|
||||
|
||||
Before introducing pgvector, the implementation must expose:
|
||||
|
||||
- the source chunk;
|
||||
- the embedding model;
|
||||
- the returned vector dimension;
|
||||
- enough of the vector to verify that it is numerical data;
|
||||
- the vector magnitude/norm;
|
||||
- the direct similarity score between known chunks.
|
||||
|
||||
The purpose is to understand what will eventually be stored and searched.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Implement a dedicated embedding provider abstraction and one concrete local embedding implementation.
|
||||
|
||||
Do not introduce LangChain, LlamaIndex, Haystack, or another RAG orchestration framework.
|
||||
|
||||
The embedding implementation should use a model specifically intended for text embeddings.
|
||||
|
||||
Do not use the generative model merely because it is already available.
|
||||
|
||||
### Model Selection
|
||||
|
||||
Before implementation, inspect the current Gateway/development environment and repository configuration.
|
||||
|
||||
Select a practical local embedding model with these characteristics:
|
||||
|
||||
- intended for semantic retrieval or text embeddings;
|
||||
- modest enough to run comfortably in the home-lab RAG architecture;
|
||||
- available through a stable local interface;
|
||||
- suitable for English technical and conversational content;
|
||||
- produces a fixed-dimensional dense vector;
|
||||
- does not require a paid cloud API.
|
||||
|
||||
Reasonable candidates may include models from the Nomic, BGE, E5, or similar embedding families.
|
||||
|
||||
Do not choose a model solely because it is popular.
|
||||
|
||||
Document:
|
||||
|
||||
- exact model name;
|
||||
- model version/tag where available;
|
||||
- provider/runtime;
|
||||
- vector dimension;
|
||||
- why it was selected for this baseline.
|
||||
|
||||
Do not add multiple embedding models in this work order.
|
||||
|
||||
---
|
||||
|
||||
## Provider Design
|
||||
|
||||
Create a small embedding interface so downstream code does not depend directly on one runtime.
|
||||
|
||||
Conceptually:
|
||||
|
||||
```python
|
||||
class EmbeddingProvider:
|
||||
def embed(self, text: str) -> list[float]:
|
||||
...
|
||||
```
|
||||
|
||||
The exact form may use a protocol, abstract base class, or another simple repository-consistent pattern.
|
||||
|
||||
The provider boundary should expose the fact that:
|
||||
|
||||
```text
|
||||
text in
|
||||
vector out
|
||||
```
|
||||
|
||||
Do not create an unnecessarily generic framework.
|
||||
|
||||
---
|
||||
|
||||
## Proposed Package Structure
|
||||
|
||||
Extend the current package approximately as follows:
|
||||
|
||||
```text
|
||||
processors/
|
||||
└── rag_ingestion/
|
||||
├── __init__.py
|
||||
├── models.py
|
||||
├── loader.py
|
||||
├── chunking.py
|
||||
├── embeddings.py
|
||||
├── similarity.py
|
||||
└── ingest.py
|
||||
```
|
||||
|
||||
Existing organization may be adapted to repository conventions.
|
||||
|
||||
### `models.py`
|
||||
|
||||
Add an embedding record if appropriate.
|
||||
|
||||
### `embeddings.py`
|
||||
|
||||
Define the provider interface and concrete local provider.
|
||||
|
||||
### `similarity.py`
|
||||
|
||||
Implement direct cosine similarity explicitly.
|
||||
|
||||
### `ingest.py`
|
||||
|
||||
Extend the CLI with opt-in commands/options for embedding selected chunks and comparing them.
|
||||
|
||||
Do not make every normal ingestion run automatically embed thousands of chunks.
|
||||
|
||||
Embedding should be explicit during this work order so experiments remain controlled and understandable.
|
||||
|
||||
---
|
||||
|
||||
## Embedding Record
|
||||
|
||||
The derived embedding representation should include, at minimum:
|
||||
|
||||
- `chunk_id`
|
||||
- `document_id`
|
||||
- `embedding_model`
|
||||
- `embedding_dimension`
|
||||
- `vector`
|
||||
|
||||
Where practical, also include:
|
||||
|
||||
- provider/runtime name;
|
||||
- model version/tag;
|
||||
- generation timestamp for diagnostics.
|
||||
|
||||
Do not add semantic labels inferred by an LLM.
|
||||
|
||||
The complete vector should remain available in memory, but the default console output should not dump hundreds or thousands of floating-point values.
|
||||
|
||||
---
|
||||
|
||||
## CLI Behavior
|
||||
|
||||
Preserve the Work Order 0001 and 0002 behavior.
|
||||
|
||||
Add an explicit way to embed selected chunks.
|
||||
|
||||
The exact CLI design may follow repository conventions, but it should support behavior conceptually similar to:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest conversation.md \
|
||||
--embed-chunk 12
|
||||
```
|
||||
|
||||
Multiple selections should be possible:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest conversation.md \
|
||||
--embed-chunk 12 \
|
||||
--embed-chunk 37
|
||||
```
|
||||
|
||||
Provide a way to compare two selected chunks:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest conversation.md \
|
||||
--compare-chunks 12 37
|
||||
```
|
||||
|
||||
The exact syntax may differ if a subcommand design is cleaner.
|
||||
|
||||
Do not automatically embed the complete Jellyfin conversation merely to demonstrate embeddings.
|
||||
|
||||
---
|
||||
|
||||
## Embedding Inspection Output
|
||||
|
||||
For each embedded chunk, display at minimum:
|
||||
|
||||
```text
|
||||
Chunk ID:
|
||||
Chunk Number:
|
||||
Model:
|
||||
Dimension:
|
||||
Vector Norm:
|
||||
Vector Preview:
|
||||
Text Preview:
|
||||
```
|
||||
|
||||
`Vector Preview` should show only a small number of leading values, for example:
|
||||
|
||||
```text
|
||||
[0.0214, -0.0871, 0.0032, ...]
|
||||
```
|
||||
|
||||
The output must make clear that the preview is incomplete.
|
||||
|
||||
When two chunks are compared, display:
|
||||
|
||||
```text
|
||||
Chunk A:
|
||||
Chunk B:
|
||||
Cosine Similarity:
|
||||
```
|
||||
|
||||
Use enough decimal precision to make repeated experiments useful.
|
||||
|
||||
---
|
||||
|
||||
## Cosine Similarity
|
||||
|
||||
Implement cosine similarity directly in project code for this work order.
|
||||
|
||||
Do not rely on a vector database or opaque retrieval library.
|
||||
|
||||
The implementation should make the calculation understandable:
|
||||
|
||||
```text
|
||||
similarity(A, B) = (A · B) / (||A|| ||B||)
|
||||
```
|
||||
|
||||
The implementation must:
|
||||
|
||||
- verify equal dimensions;
|
||||
- handle zero-length or zero-norm vectors safely;
|
||||
- return a numerical similarity score;
|
||||
- include tests with known vectors where the expected result is obvious.
|
||||
|
||||
Do not present cosine similarity as a probability or percentage.
|
||||
|
||||
Document the expected score behavior for the chosen embedding model.
|
||||
|
||||
---
|
||||
|
||||
## Controlled Similarity Experiments
|
||||
|
||||
The manual test must go beyond "the model returned a vector."
|
||||
|
||||
Use the real Jellyfin conversation that has been used for Work Orders 0001 and 0002 when practical.
|
||||
|
||||
Select known chunks representing at least:
|
||||
|
||||
### Pair A — Closely related content
|
||||
|
||||
Two chunks discussing substantially the same technical subject.
|
||||
|
||||
Expected observation:
|
||||
|
||||
The score should generally indicate stronger similarity than clearly unrelated content.
|
||||
|
||||
### Pair B — Related topic, different wording
|
||||
|
||||
Two chunks discussing the same broader concept with different wording.
|
||||
|
||||
This is intended to demonstrate why embeddings can retrieve semantically related text without exact keyword matches.
|
||||
|
||||
### Pair C — Unrelated content
|
||||
|
||||
Compare two chunks discussing clearly different subjects.
|
||||
|
||||
### Pair D — Structural or low-value content
|
||||
|
||||
If the Jellyfin export contains a chunk dominated by encoded, machine-oriented, or otherwise low-semantic-value content, compare it against meaningful prose.
|
||||
|
||||
Do not assume in advance what score it will receive.
|
||||
|
||||
Observe the result.
|
||||
|
||||
### Pair E — Self comparison
|
||||
|
||||
Compare a chunk with itself.
|
||||
|
||||
This should provide a sanity check for the cosine implementation.
|
||||
|
||||
---
|
||||
|
||||
## Experimental Output
|
||||
|
||||
For manual experiments, record a small table similar to:
|
||||
|
||||
```text
|
||||
Chunk A | Chunk B | Relationship expected | Cosine similarity | Observation
|
||||
```
|
||||
|
||||
The purpose is not to prove the embedding model is "correct."
|
||||
|
||||
The purpose is to observe how numerical similarity behaves against content the developer already understands.
|
||||
|
||||
---
|
||||
|
||||
## Critical Learning Distinction
|
||||
|
||||
The implementation report must explicitly distinguish:
|
||||
|
||||
### Semantic Similarity
|
||||
|
||||
"These two vector representations are near one another according to this embedding model."
|
||||
|
||||
from:
|
||||
|
||||
### Retrieval Relevance
|
||||
|
||||
"This chunk is useful evidence for answering this particular query in this particular business or reasoning context."
|
||||
|
||||
The two are related but are not equivalent.
|
||||
|
||||
This distinction is central to the Project Thoth RAG investigation.
|
||||
|
||||
Do not add metadata filtering, taxonomy, reranking, or relationship logic yet to improve weak similarity results.
|
||||
|
||||
Weak or surprising results are findings for later work orders.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
1. Preserve Work Orders 0001 and 0002 behavior.
|
||||
2. Add a dedicated embedding provider boundary.
|
||||
3. Add one concrete local embedding implementation.
|
||||
4. Use a model designed for embeddings.
|
||||
5. Keep the model configurable.
|
||||
6. Record exact model identity.
|
||||
7. Add an embedding record/model where useful.
|
||||
8. Generate embeddings only from chunk text.
|
||||
9. Preserve linkage from embedding to chunk and document.
|
||||
10. Display embedding dimensionality.
|
||||
11. Display vector norm.
|
||||
12. Display a small vector preview.
|
||||
13. Do not dump complete vectors by default.
|
||||
14. Implement cosine similarity directly.
|
||||
15. Allow selected chunks to be compared.
|
||||
16. Validate dimension mismatches.
|
||||
17. Add tests for provider behavior where practical.
|
||||
18. Add deterministic tests for cosine similarity independent of the external model.
|
||||
19. Keep dependencies minimal.
|
||||
20. Do not introduce a RAG orchestration framework.
|
||||
21. Do not introduce persistent vector storage.
|
||||
22. Do not perform opportunistic refactoring outside this work order.
|
||||
|
||||
---
|
||||
|
||||
## Automated Tests
|
||||
|
||||
Add tests appropriate to repository conventions.
|
||||
|
||||
At minimum test:
|
||||
|
||||
### Cosine identity
|
||||
|
||||
A vector compared with itself should produce the expected identity similarity.
|
||||
|
||||
### Orthogonal vectors
|
||||
|
||||
Known orthogonal vectors should produce the expected similarity.
|
||||
|
||||
### Opposite vectors
|
||||
|
||||
Known opposite vectors should produce the expected similarity.
|
||||
|
||||
### Dimension mismatch
|
||||
|
||||
Vectors of different dimensions should fail clearly.
|
||||
|
||||
### Zero vector handling
|
||||
|
||||
Define and test safe behavior.
|
||||
|
||||
### Embedding result shape
|
||||
|
||||
A real or mocked provider result must contain the expected vector dimension.
|
||||
|
||||
### Chunk linkage
|
||||
|
||||
The embedding record must preserve the correct `chunk_id` and `document_id`.
|
||||
|
||||
### Model identity
|
||||
|
||||
The embedding record must identify the model used.
|
||||
|
||||
### Source preservation
|
||||
|
||||
Embedding must not modify:
|
||||
|
||||
- source file;
|
||||
- document raw text;
|
||||
- chunk text.
|
||||
|
||||
### Existing regression suite
|
||||
|
||||
All Work Order 0001 and Work Order 0002 tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## Manual Test Plan
|
||||
|
||||
Use:
|
||||
|
||||
`research/chatgpt/ChatGPT-Jellyfin_Server_Setup_Options.md`
|
||||
|
||||
or the same canonical Jellyfin conversation source used in prior work orders.
|
||||
|
||||
### Test 1 — Embed one known chunk
|
||||
|
||||
Select a prose chunk.
|
||||
|
||||
Verify:
|
||||
|
||||
- vector is returned;
|
||||
- dimension is reported;
|
||||
- values are numerical;
|
||||
- norm is reported;
|
||||
- model identity is reported;
|
||||
- chunk text is unchanged.
|
||||
|
||||
### Test 2 — Repeat the same embedding
|
||||
|
||||
Embed the same chunk twice with the same model.
|
||||
|
||||
Observe whether the returned vectors are identical or practically identical.
|
||||
|
||||
Document the actual behavior.
|
||||
|
||||
Do not invent a determinism guarantee if the runtime/model does not provide one.
|
||||
|
||||
### Test 3 — Compare related chunks
|
||||
|
||||
Select two clearly related technical chunks.
|
||||
|
||||
Record cosine similarity.
|
||||
|
||||
### Test 4 — Compare differently worded related chunks
|
||||
|
||||
Select two chunks discussing the same topic without relying on identical phrasing.
|
||||
|
||||
Record cosine similarity.
|
||||
|
||||
### Test 5 — Compare unrelated chunks
|
||||
|
||||
Select clearly unrelated chunks.
|
||||
|
||||
Record cosine similarity.
|
||||
|
||||
### Test 6 — Compare a chunk with itself
|
||||
|
||||
Confirm the expected sanity-check result.
|
||||
|
||||
### Test 7 — Inspect low-value content
|
||||
|
||||
If an appropriate chunk exists, compare machine-oriented or encoded content with prose.
|
||||
|
||||
Record what actually happens.
|
||||
|
||||
Do not filter it out.
|
||||
|
||||
---
|
||||
|
||||
## Observation Notes
|
||||
|
||||
Record observations from the real-source experiment without immediately changing the architecture.
|
||||
|
||||
Useful questions include:
|
||||
|
||||
- Did related chunks actually score closer together?
|
||||
- Did lexical overlap appear to dominate any comparison?
|
||||
- Did differently worded material remain semantically close?
|
||||
- Did an apparently unrelated pair receive a surprisingly high score?
|
||||
- What happened to encoded or machine-oriented text?
|
||||
- Does a high similarity score necessarily indicate that one chunk would answer a question about the other?
|
||||
- How easy is it for a human to interpret the embedding vector itself?
|
||||
|
||||
These observations become evidence for later retrieval and Knowledge Architecture work.
|
||||
|
||||
---
|
||||
|
||||
## Non-Goals
|
||||
|
||||
Do not implement:
|
||||
|
||||
- PostgreSQL;
|
||||
- pgvector;
|
||||
- Qdrant;
|
||||
- Chroma;
|
||||
- FAISS indexing;
|
||||
- persistent embedding storage;
|
||||
- corpus-wide vector search;
|
||||
- query embedding and retrieval;
|
||||
- nearest-neighbor search;
|
||||
- BM25;
|
||||
- hybrid retrieval;
|
||||
- reranking;
|
||||
- metadata filtering;
|
||||
- corpus routing;
|
||||
- Source Metadata integration;
|
||||
- Conversation Manifest integration;
|
||||
- taxonomy;
|
||||
- relationship traversal;
|
||||
- prompt construction;
|
||||
- Oracle generation;
|
||||
- LLM answering;
|
||||
- recursive Vault ingestion;
|
||||
- automatic embedding of the entire Vault;
|
||||
- semantic chunking;
|
||||
- chunking changes intended to improve embedding quality;
|
||||
- web UI.
|
||||
|
||||
These belong to later work orders.
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
This work order is complete when:
|
||||
|
||||
- Work Orders 0001 and 0002 still function unchanged;
|
||||
- a selected chunk can be sent to a real local embedding model;
|
||||
- a numerical embedding vector is returned;
|
||||
- the model and dimension are visible;
|
||||
- the vector can be inspected without dumping the full vector by default;
|
||||
- two chunk vectors can be compared using project-owned cosine similarity code;
|
||||
- known mathematical similarity tests pass;
|
||||
- real Jellyfin chunks have been used for controlled comparisons;
|
||||
- at least one related, one unrelated, and one self-comparison has been recorded;
|
||||
- surprising or weak results are documented rather than silently corrected;
|
||||
- no vector database, persistent index, or retrieval system has been added.
|
||||
|
||||
---
|
||||
|
||||
## Learning Verification
|
||||
|
||||
At completion, Codex must provide a short implementation report answering the following questions from the actual implementation:
|
||||
|
||||
1. What exact text is sent to the embedding model?
|
||||
2. What exact type of object does the embedding provider return?
|
||||
3. What is the vector dimension of the selected model?
|
||||
4. What does one individual number in that vector mean to us as application developers?
|
||||
5. Why can two vectors be compared even when the source text uses different words?
|
||||
6. How is cosine similarity calculated in this implementation?
|
||||
7. What does a higher cosine similarity score tell us?
|
||||
8. What does it *not* tell us?
|
||||
9. Why is cosine similarity not equivalent to business relevance or answer usefulness?
|
||||
10. What model identity must eventually be stored alongside persisted vectors, and why?
|
||||
11. What would happen if vectors produced by two different embedding models were mixed in one index?
|
||||
12. Does embedding preserve source structure such as Markdown headings, speaker turns, document hierarchy, or relationships?
|
||||
13. What happened when the real Jellyfin chunks with related content were compared?
|
||||
14. What happened with unrelated content?
|
||||
15. What happened with any low-value or encoded content tested?
|
||||
16. Were repeated embeddings of the same input identical? Report observed behavior rather than assumption.
|
||||
17. Where does embedding end and vector indexing begin?
|
||||
18. Based on this implementation, explain in plain language what a vector database will need to store in the next stage.
|
||||
|
||||
Answers should reference actual files, functions, model configuration, tests, and observed scores.
|
||||
|
||||
---
|
||||
|
||||
## Definition of Done
|
||||
|
||||
Project Thoth now has an inspectable embedding stage.
|
||||
|
||||
A real source can move through:
|
||||
|
||||
```text
|
||||
Primary Source
|
||||
↓
|
||||
Document Record
|
||||
↓
|
||||
Chunk Records
|
||||
↓
|
||||
Embedding Vector
|
||||
```
|
||||
|
||||
and selected vectors can be compared directly without introducing a vector database.
|
||||
|
||||
The developer can explain what text was embedded, which model generated the vector, how large the vector is, how cosine similarity is calculated, and why semantic similarity is not the same thing as retrieval relevance.
|
||||
|
||||
The next work order can introduce persistence and vector indexing using objects that have already been examined and understood directly.
|
||||
|
||||
---
|
||||
|
||||
## Codex Execution Guidance
|
||||
|
||||
Before changing code:
|
||||
|
||||
1. Read the Work Order 0001 and 0002 implementations and completion notes.
|
||||
2. Read this work order completely.
|
||||
3. Inspect the current `./processors/rag_ingestion/` package.
|
||||
4. Inspect repository dependency/configuration conventions.
|
||||
5. Inspect the available Gateway/development embedding runtimes.
|
||||
6. Identify candidate local embedding models.
|
||||
7. Select one baseline model and briefly justify it.
|
||||
8. State its expected vector dimension if documented by the runtime/model.
|
||||
9. Summarize the narrow implementation plan before editing files.
|
||||
|
||||
During implementation:
|
||||
|
||||
1. Preserve existing ingestion and chunking behavior.
|
||||
2. Keep embedding separate from chunking.
|
||||
3. Keep similarity math separate from the embedding provider.
|
||||
4. Keep model identity visible.
|
||||
5. Do not embed the entire Vault.
|
||||
6. Do not introduce persistence.
|
||||
7. Do not interpret surprising scores as bugs without evidence.
|
||||
8. Do not modify chunking to make the embedding experiment look better.
|
||||
9. Keep code explicit enough for line-by-line review.
|
||||
|
||||
At completion, report:
|
||||
|
||||
- files created or changed;
|
||||
- selected embedding model and runtime;
|
||||
- why the model was selected;
|
||||
- vector dimension;
|
||||
- provider design;
|
||||
- cosine similarity implementation;
|
||||
- automated test results;
|
||||
- real Jellyfin comparison table;
|
||||
- notable observations;
|
||||
- Learning Verification answers;
|
||||
- assumptions or limitations;
|
||||
- anything that could not be validated.
|
||||
|
||||
Do not begin Work Order 0004.
|
||||
@@ -0,0 +1,508 @@
|
||||
# Work Order 0004 — Add Environment Configuration for Oracle Embeddings
|
||||
|
||||
## Status
|
||||
Planned
|
||||
|
||||
## Work Stream
|
||||
RAG Ingestion
|
||||
|
||||
## Application
|
||||
`rag-ingestion`
|
||||
|
||||
## Location
|
||||
Work order: `./codex/rag-ingestion/0004-add-environment-configuration.md`
|
||||
|
||||
Implementation package: `./processors/rag_ingestion/`
|
||||
|
||||
Repository configuration: `./.env`, `./.env.example`
|
||||
|
||||
## Objective
|
||||
|
||||
Add environment-based runtime configuration for the `rag-ingestion` application so Oracle is the explicitly configured inference host for embedding operations.
|
||||
|
||||
Work Order 0003 established the embedding provider and direct similarity behavior. This work order must remove any remaining localhost- or model-specific assumptions from application code and move deployment-specific values into environment configuration.
|
||||
|
||||
Initial runtime configuration:
|
||||
|
||||
```env
|
||||
THOTH_ORACLE_BASE_URL=http://192.168.5.52:11434
|
||||
THOTH_EMBEDDING_MODEL=gemma4:e4b
|
||||
THOTH_EMBEDDING_DIMENSION=
|
||||
```
|
||||
|
||||
`THOTH_EMBEDDING_DIMENSION` remains optional until the actual model response has been validated.
|
||||
|
||||
This work order is limited to configuration and validation. Do not introduce persistence, pgvector, retrieval, reranking, or Work Order 0005 functionality.
|
||||
|
||||
## Learning Objective
|
||||
|
||||
At completion, the developer should be able to explain:
|
||||
|
||||
- why runtime/deployment configuration should not be hard-coded in application logic;
|
||||
- the difference between application configuration and architectural documentation;
|
||||
- why `.env` is appropriate for machine- or deployment-specific values;
|
||||
- why `.env.example` should be committed while `.env` should not;
|
||||
- how environment variables enter the Python application;
|
||||
- how missing configuration differs from invalid configuration;
|
||||
- why Oracle endpoint, embedding model, and embedding dimension are separate settings;
|
||||
- why embedding dimension is useful as a validation constraint;
|
||||
- why the dimension can remain unset until empirically discovered;
|
||||
- how unit tests remain independent of the live Oracle service.
|
||||
|
||||
The developer should be able to trace a value from `.env` through configuration loading to the embedding provider.
|
||||
|
||||
## Architectural Context
|
||||
|
||||
Project Thoth uses this responsibility boundary:
|
||||
|
||||
```text
|
||||
Gateway / Application Side
|
||||
- ingestion
|
||||
- chunking
|
||||
- configuration
|
||||
- provider orchestration
|
||||
- similarity
|
||||
- later storage and retrieval
|
||||
|
||||
HTTP
|
||||
↓
|
||||
|
||||
Oracle
|
||||
- model inference
|
||||
- embedding inference
|
||||
```
|
||||
|
||||
Oracle is the dedicated inference host. The application must not assume inference runs on `localhost`.
|
||||
|
||||
## Decision
|
||||
|
||||
Use environment variables for runtime inference configuration.
|
||||
|
||||
Commit `.env.example` as the configuration template. Keep the real `.env` local and uncommitted.
|
||||
|
||||
Use a small centralized configuration layer. Do not scatter `os.getenv()` calls throughout the application.
|
||||
|
||||
## Required Environment Variables
|
||||
|
||||
### `THOTH_ORACLE_BASE_URL`
|
||||
|
||||
Purpose: identifies the base HTTP URL of Oracle's model API.
|
||||
|
||||
Initial development value:
|
||||
|
||||
```env
|
||||
THOTH_ORACLE_BASE_URL=http://192.168.5.52:11434
|
||||
```
|
||||
|
||||
Requirements:
|
||||
|
||||
- include the URL scheme;
|
||||
- trailing slash optional;
|
||||
- no silent localhost fallback;
|
||||
- missing value produces a clear error when live embedding is requested.
|
||||
|
||||
### `THOTH_EMBEDDING_MODEL`
|
||||
|
||||
Purpose: identifies the exact model requested from Oracle for embedding generation.
|
||||
|
||||
Initial development value:
|
||||
|
||||
```env
|
||||
THOTH_EMBEDDING_MODEL=gemma4:e4b
|
||||
```
|
||||
|
||||
Requirements:
|
||||
|
||||
- pass explicitly to Oracle;
|
||||
- no hidden application-code default;
|
||||
- missing value produces a clear error when live embedding is requested;
|
||||
- do not assume the model is embedding-capable merely because it exists on Oracle.
|
||||
|
||||
### `THOTH_EMBEDDING_DIMENSION`
|
||||
|
||||
Purpose: optionally defines the expected number of numeric values in the returned embedding vector.
|
||||
|
||||
Initial development value:
|
||||
|
||||
```env
|
||||
THOTH_EMBEDDING_DIMENSION=
|
||||
```
|
||||
|
||||
Requirements:
|
||||
|
||||
- optional;
|
||||
- blank means "discover from actual model response";
|
||||
- when configured, parse as a positive integer;
|
||||
- when configured, validate returned vector length;
|
||||
- mismatch must fail clearly.
|
||||
|
||||
Matching dimensions do not imply model compatibility.
|
||||
|
||||
## Proposed Configuration Layer
|
||||
|
||||
Add a small configuration module, for example:
|
||||
|
||||
```text
|
||||
processors/
|
||||
└── rag_ingestion/
|
||||
├── config.py
|
||||
├── embeddings.py
|
||||
└── ...
|
||||
```
|
||||
|
||||
Conceptually:
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class EmbeddingConfig:
|
||||
oracle_base_url: str
|
||||
embedding_model: str
|
||||
embedding_dimension: int | None
|
||||
```
|
||||
|
||||
The exact pattern may follow existing repository conventions. Configuration loading and validation must happen in one explicit place.
|
||||
|
||||
## `.env.example`
|
||||
|
||||
Create or update the repository-level `.env.example` with:
|
||||
|
||||
```env
|
||||
# Project Thoth RAG ingestion — Oracle embedding configuration
|
||||
THOTH_ORACLE_BASE_URL=
|
||||
THOTH_EMBEDDING_MODEL=
|
||||
THOTH_EMBEDDING_DIMENSION=
|
||||
```
|
||||
|
||||
Do not put environment-specific values into `.env.example`.
|
||||
|
||||
## `.env`
|
||||
|
||||
Create a local `.env` for the current development environment containing:
|
||||
|
||||
```env
|
||||
THOTH_ORACLE_BASE_URL=http://192.168.5.52:11434
|
||||
THOTH_EMBEDDING_MODEL=gemma4:e4b
|
||||
THOTH_EMBEDDING_DIMENSION=
|
||||
```
|
||||
|
||||
The real `.env` must not be committed.
|
||||
|
||||
## `.gitignore`
|
||||
|
||||
Inspect the existing `.gitignore` and ensure `.env` is ignored while `.env.example` remains trackable. Do not make unrelated changes.
|
||||
|
||||
## Environment Loading
|
||||
|
||||
Inspect repository dependency configuration first.
|
||||
|
||||
If a dotenv library already exists, use the existing convention. If none exists, choose the smallest practical approach and document the decision.
|
||||
|
||||
Normal operating-system environment variables must remain supported. Document the actual precedence behavior between OS environment variables and `.env` values.
|
||||
|
||||
## Provider Integration
|
||||
|
||||
Update the Work Order 0003 embedding provider so that:
|
||||
|
||||
- endpoint comes from `THOTH_ORACLE_BASE_URL`;
|
||||
- model comes from `THOTH_EMBEDDING_MODEL`;
|
||||
- expected dimension comes from `THOTH_EMBEDDING_DIMENSION`;
|
||||
- no localhost inference default remains;
|
||||
- no Nomic-specific or other temporary model default remains;
|
||||
- the provider can be instantiated from validated configuration.
|
||||
|
||||
Prefer:
|
||||
|
||||
```text
|
||||
environment
|
||||
↓
|
||||
configuration layer
|
||||
↓
|
||||
embedding provider
|
||||
```
|
||||
|
||||
rather than direct environment lookups inside the provider.
|
||||
|
||||
## Configuration Validation
|
||||
|
||||
Validate:
|
||||
|
||||
### Oracle base URL
|
||||
|
||||
Reject or clearly report:
|
||||
|
||||
- missing value when live embeddings are requested;
|
||||
- malformed URL;
|
||||
- missing scheme.
|
||||
|
||||
Do not silently prepend `http://`.
|
||||
|
||||
### Embedding model
|
||||
|
||||
Reject or clearly report a missing or blank model when live embeddings are requested.
|
||||
|
||||
### Embedding dimension
|
||||
|
||||
Accept blank as `None`.
|
||||
|
||||
Reject:
|
||||
|
||||
- zero;
|
||||
- negative numbers;
|
||||
- non-integer values.
|
||||
|
||||
When configured, compare expected dimension to actual returned vector length.
|
||||
|
||||
## Oracle Connectivity Check
|
||||
|
||||
Add a narrow explicit validation path for Oracle, either a CLI option or small diagnostic function.
|
||||
|
||||
Conceptually:
|
||||
|
||||
```bash
|
||||
python -m processors.rag_ingestion.ingest --check-embedding-config
|
||||
```
|
||||
|
||||
The diagnostic should report:
|
||||
|
||||
```text
|
||||
Oracle Base URL:
|
||||
Embedding Model:
|
||||
Configured Dimension:
|
||||
Connection:
|
||||
Embedding Request:
|
||||
Returned Dimension:
|
||||
```
|
||||
|
||||
It should make a minimal real embedding request using a small known string such as:
|
||||
|
||||
```text
|
||||
Project Thoth embedding configuration test.
|
||||
```
|
||||
|
||||
The purpose is to verify:
|
||||
|
||||
1. Oracle can be reached;
|
||||
2. the configured API route works;
|
||||
3. the configured model accepts an embedding request;
|
||||
4. a numerical vector is returned;
|
||||
5. actual dimension can be observed;
|
||||
6. configured dimension validation works when enabled.
|
||||
|
||||
## Important Model Validation
|
||||
|
||||
The initial configured model is:
|
||||
|
||||
```env
|
||||
THOTH_EMBEDDING_MODEL=gemma4:e4b
|
||||
```
|
||||
|
||||
Do not assume it supports embedding generation.
|
||||
|
||||
During the live Oracle diagnostic:
|
||||
|
||||
- attempt the embedding request using the configured model;
|
||||
- if successful, record the returned dimension;
|
||||
- if it fails because the model is not embedding-capable, stop and report that fact clearly;
|
||||
- do not silently replace the configured model;
|
||||
- do not download or install another model without explicit authorization.
|
||||
|
||||
This work order is about configuration and validation, not automatic model provisioning.
|
||||
|
||||
## Unit Test Boundary
|
||||
|
||||
Unit tests must not require Oracle.
|
||||
|
||||
Use environment fixtures and stub/mock providers where necessary. Live Oracle checks belong to manual or explicitly marked integration tests.
|
||||
|
||||
The default automated suite must remain runnable when Oracle is offline.
|
||||
|
||||
## Automated Tests
|
||||
|
||||
At minimum test:
|
||||
|
||||
- valid complete configuration;
|
||||
- blank dimension becomes `None`;
|
||||
- missing URL fails clearly for live provider use;
|
||||
- missing model fails clearly;
|
||||
- non-numeric, zero, and negative dimensions fail;
|
||||
- matching configured dimension succeeds against a stub vector;
|
||||
- mismatched configured dimension fails;
|
||||
- provider receives configured URL and model;
|
||||
- absence of config does not silently produce localhost;
|
||||
- Work Orders 0001 through 0003 regression suite still passes.
|
||||
|
||||
## Manual Test Plan
|
||||
|
||||
### Test 1 — Local `.env`
|
||||
|
||||
Create:
|
||||
|
||||
```env
|
||||
THOTH_ORACLE_BASE_URL=http://192.168.5.52:11434
|
||||
THOTH_EMBEDDING_MODEL=gemma4:e4b
|
||||
THOTH_EMBEDDING_DIMENSION=
|
||||
```
|
||||
|
||||
Verify the application loads the expected non-sensitive configuration.
|
||||
|
||||
### Test 2 — Oracle connectivity
|
||||
|
||||
Run the configuration diagnostic and verify whether Oracle responds.
|
||||
|
||||
### Test 3 — Model capability
|
||||
|
||||
Attempt the minimal embedding request and record:
|
||||
|
||||
- success/failure;
|
||||
- exact model identity;
|
||||
- actual vector dimension if successful.
|
||||
|
||||
### Test 4 — Dimension discovery
|
||||
|
||||
With dimension blank, generate one embedding and verify the returned dimension is reported without error.
|
||||
|
||||
### Test 5 — Dimension validation
|
||||
|
||||
If actual dimension is known, temporarily configure it and verify success. Then configure an intentionally incorrect dimension and verify clear failure. Do not commit either local value.
|
||||
|
||||
### Test 6 — Git protection
|
||||
|
||||
Verify `git status` does not show `.env` as a trackable unignored file and that `.env.example` remains trackable.
|
||||
|
||||
## Requirements
|
||||
|
||||
1. Add centralized environment-based configuration.
|
||||
2. Add or update `.env.example`.
|
||||
3. Ensure `.env` is ignored by Git.
|
||||
4. Create local `.env` only for the development environment.
|
||||
5. Configure Oracle base URL through environment.
|
||||
6. Configure embedding model through environment.
|
||||
7. Make embedding dimension optional.
|
||||
8. Validate dimension when supplied.
|
||||
9. Remove localhost inference defaults.
|
||||
10. Remove temporary hard-coded embedding model defaults.
|
||||
11. Keep configuration loading separate from provider behavior.
|
||||
12. Keep unit tests independent of Oracle.
|
||||
13. Add a narrow Oracle/model validation path.
|
||||
14. Do not silently replace an invalid embedding model.
|
||||
15. Do not install models automatically.
|
||||
16. Preserve Work Orders 0001 through 0003 behavior.
|
||||
17. Do not implement persistence or retrieval.
|
||||
18. Do not perform unrelated refactoring.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
Do not implement:
|
||||
|
||||
- PostgreSQL;
|
||||
- pgvector;
|
||||
- persistent document, chunk, or embedding storage;
|
||||
- vector indexing;
|
||||
- corpus-wide ingestion;
|
||||
- query embedding and retrieval;
|
||||
- BM25 or hybrid search;
|
||||
- reranking;
|
||||
- metadata filtering;
|
||||
- Source Metadata or Conversation Manifest integration;
|
||||
- taxonomy or relationships;
|
||||
- prompt construction;
|
||||
- Oracle text generation;
|
||||
- UI work;
|
||||
- automatic model installation or selection.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
This work order is complete when:
|
||||
|
||||
- `.env.example` documents all three variables;
|
||||
- real `.env` is excluded from Git;
|
||||
- Oracle endpoint is no longer hard-coded;
|
||||
- embedding model is no longer hard-coded;
|
||||
- embedding dimension is optional and validated when present;
|
||||
- configuration is loaded through a centralized layer;
|
||||
- Work Order 0003 provider behavior consumes validated configuration;
|
||||
- unit tests do not depend on Oracle;
|
||||
- all prior automated tests pass;
|
||||
- the application can perform an explicit Oracle embedding configuration check;
|
||||
- the check reports whether `gemma4:e4b` actually supports embedding requests;
|
||||
- if successful, actual vector dimension is recorded;
|
||||
- if unsuccessful, capability failure is reported without silently provisioning another model.
|
||||
|
||||
## Learning Verification
|
||||
|
||||
At completion, Codex must answer from the actual implementation:
|
||||
|
||||
1. Where does the application load `.env` values?
|
||||
2. How does a value move from `.env` into the embedding provider?
|
||||
3. What happens if `THOTH_ORACLE_BASE_URL` is absent?
|
||||
4. Why is there no localhost default?
|
||||
5. What happens if `THOTH_EMBEDDING_MODEL` is absent?
|
||||
6. What does a blank `THOTH_EMBEDDING_DIMENSION` mean?
|
||||
7. What happens if a configured dimension differs from Oracle's returned vector length?
|
||||
8. Why store vector dimension separately from model name?
|
||||
9. Why does matching dimension not prove two models are compatible?
|
||||
10. Why is `.env` ignored while `.env.example` is committed?
|
||||
11. What environment-variable precedence rules are actually used?
|
||||
12. Which tests run without Oracle?
|
||||
13. Which validation requires live Oracle access?
|
||||
14. Did `gemma4:e4b` accept a real embedding request?
|
||||
15. If yes, what dimension did it return?
|
||||
16. If no, what exact failure occurred?
|
||||
17. Why should application code not automatically download a replacement model?
|
||||
18. Where does configuration responsibility end and embedding-provider responsibility begin?
|
||||
|
||||
## Definition of Done
|
||||
|
||||
Project Thoth has an explicit runtime configuration boundary for embedding inference.
|
||||
|
||||
The application can be deployed without changing Python source code to specify:
|
||||
|
||||
```text
|
||||
where Oracle lives
|
||||
which embedding model Oracle should use
|
||||
what vector dimension is expected, if known
|
||||
```
|
||||
|
||||
A local `.env` supplies deployment values, while `.env.example` documents the contract without committing machine-specific configuration.
|
||||
|
||||
The application can validate the Oracle embedding path before persistence or vector indexing is introduced.
|
||||
|
||||
## Codex Execution Guidance
|
||||
|
||||
Before changing code:
|
||||
|
||||
1. Read Work Orders 0001 through 0003 and their completion notes.
|
||||
2. Read this work order completely.
|
||||
3. Inspect current `rag_ingestion` configuration and provider code.
|
||||
4. Identify all remaining localhost or hard-coded model defaults.
|
||||
5. Inspect dependency configuration for dotenv support.
|
||||
6. Inspect `.gitignore` and any existing `.env.example`.
|
||||
7. Summarize the narrow implementation plan before editing.
|
||||
|
||||
During implementation:
|
||||
|
||||
1. Keep configuration centralized.
|
||||
2. Do not hard-code environment-specific values into Python.
|
||||
3. Preserve provider injection/testability.
|
||||
4. Keep Oracle out of unit-test dependencies.
|
||||
5. Do not install or remove Ollama or models.
|
||||
6. Do not begin persistence work.
|
||||
7. Do not silently repair invalid deployment configuration.
|
||||
|
||||
At completion, report:
|
||||
|
||||
- files created or changed;
|
||||
- environment loading mechanism;
|
||||
- `.gitignore` changes;
|
||||
- `.env.example` contents;
|
||||
- configuration validation behavior;
|
||||
- removal of old defaults;
|
||||
- automated test results;
|
||||
- Oracle connectivity result;
|
||||
- `gemma4:e4b` embedding capability result;
|
||||
- actual vector dimension if discovered;
|
||||
- Learning Verification answers;
|
||||
- assumptions or limitations;
|
||||
- anything that could not be validated.
|
||||
|
||||
Do not begin Work Order 0005.
|
||||
@@ -0,0 +1,26 @@
|
||||
I agree that the presentation layer should be a self-hosted web application.
|
||||
|
||||
I agree with using an n-tier architecture where the functions exist in the business logic layer and interfaces to external services (like NextCloud) exist in the data layer.
|
||||
|
||||
The iterations of the MVP should be much smaller than proposed:
|
||||
|
||||
Iteration 0:
|
||||
- establish CI/CD pipeline to use with project development
|
||||
- create docker compose to deploy self-hosted application
|
||||
- display home page that says "Project Thoth"
|
||||
|
||||
Iteration 1:
|
||||
- create application shell
|
||||
- navigation bar with placeholders including link to the Vault
|
||||
- header title and menu bar
|
||||
- placeholder for profile icon, and identity management menu
|
||||
- footer content
|
||||
- informational home page with boiler plate content
|
||||
|
||||
Iteration 2:
|
||||
- a Vault page to show nextcloud content
|
||||
- nextcloud interface that will successfully connect to an instance
|
||||
- display top level nextcloud folder structure in Vault page
|
||||
|
||||
Iteration 3:
|
||||
- allow user to browse through the Vault
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Inspectable Markdown ingestion and character chunking for Project Thoth."""
|
||||
|
||||
from .chunking import ChunkConfiguration, chunk_document
|
||||
from .config import EmbeddingConfig, load_embedding_config
|
||||
from .embeddings import (
|
||||
EmbeddingProvider,
|
||||
OllamaEmbeddingProvider,
|
||||
embed_chunk,
|
||||
provider_from_config,
|
||||
)
|
||||
from .loader import load_markdown_document
|
||||
from .models import Chunk, Document, Embedding
|
||||
from .similarity import cosine_similarity
|
||||
|
||||
__all__ = [
|
||||
"Chunk",
|
||||
"ChunkConfiguration",
|
||||
"Document",
|
||||
"Embedding",
|
||||
"EmbeddingConfig",
|
||||
"EmbeddingProvider",
|
||||
"OllamaEmbeddingProvider",
|
||||
"chunk_document",
|
||||
"cosine_similarity",
|
||||
"embed_chunk",
|
||||
"load_embedding_config",
|
||||
"load_markdown_document",
|
||||
"provider_from_config",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,95 @@
|
||||
"""Deterministic, fixed-width character chunking for ingested documents."""
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .models import Chunk, Document
|
||||
|
||||
# A rough four-characters-per-token approximation of the work order's suggested
|
||||
# 800-token chunks with 120-token overlap; these values remain character counts.
|
||||
DEFAULT_CHUNK_SIZE = 3_200
|
||||
DEFAULT_CHUNK_OVERLAP = 480
|
||||
SIZE_UNIT = "characters"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChunkConfiguration:
|
||||
"""Visible settings that determine mechanical chunk boundaries."""
|
||||
|
||||
chunk_size: int = DEFAULT_CHUNK_SIZE
|
||||
chunk_overlap: int = DEFAULT_CHUNK_OVERLAP
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.chunk_size <= 0:
|
||||
raise ValueError("chunk size must be greater than zero")
|
||||
if self.chunk_overlap < 0:
|
||||
raise ValueError("chunk overlap must not be negative")
|
||||
if self.chunk_overlap >= self.chunk_size:
|
||||
raise ValueError("chunk overlap must be smaller than chunk size")
|
||||
|
||||
|
||||
def create_chunk_id(
|
||||
document_id: str,
|
||||
chunk_number: int,
|
||||
configuration: ChunkConfiguration,
|
||||
) -> str:
|
||||
"""Hash source identity, ordinal position, and boundary configuration."""
|
||||
|
||||
identity = (
|
||||
f"{document_id}:characters:"
|
||||
f"{configuration.chunk_size}:{configuration.chunk_overlap}:{chunk_number}"
|
||||
)
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def chunk_document(
|
||||
document: Document,
|
||||
configuration: ChunkConfiguration | None = None,
|
||||
) -> list[Chunk]:
|
||||
"""Divide ``document.raw_text`` into ordered character ranges.
|
||||
|
||||
The next range starts ``chunk_size - chunk_overlap`` characters after the
|
||||
current range. Empty input intentionally produces no retrieval units.
|
||||
"""
|
||||
|
||||
settings = configuration or ChunkConfiguration()
|
||||
source_text = document.raw_text
|
||||
if not source_text:
|
||||
return []
|
||||
|
||||
step_size = settings.chunk_size - settings.chunk_overlap
|
||||
boundaries: list[tuple[int, int]] = []
|
||||
start_offset = 0
|
||||
|
||||
while start_offset < len(source_text):
|
||||
end_offset = min(start_offset + settings.chunk_size, len(source_text))
|
||||
boundaries.append((start_offset, end_offset))
|
||||
if end_offset == len(source_text):
|
||||
break
|
||||
start_offset += step_size
|
||||
|
||||
chunks: list[Chunk] = []
|
||||
for chunk_number, (start_offset, end_offset) in enumerate(boundaries):
|
||||
previous_end = boundaries[chunk_number - 1][1] if chunk_number > 0 else 0
|
||||
next_start = (
|
||||
boundaries[chunk_number + 1][0]
|
||||
if chunk_number + 1 < len(boundaries)
|
||||
else end_offset
|
||||
)
|
||||
text = source_text[start_offset:end_offset]
|
||||
chunks.append(
|
||||
Chunk(
|
||||
chunk_id=create_chunk_id(document.document_id, chunk_number, settings),
|
||||
document_id=document.document_id,
|
||||
chunk_number=chunk_number,
|
||||
text=text,
|
||||
start_offset=start_offset,
|
||||
end_offset=end_offset,
|
||||
size=len(text),
|
||||
size_unit=SIZE_UNIT,
|
||||
overlap_with_previous=max(0, previous_end - start_offset),
|
||||
overlap_with_next=max(0, end_offset - next_start),
|
||||
)
|
||||
)
|
||||
|
||||
return chunks
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Centralized runtime configuration for Oracle embedding inference."""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ORACLE_BASE_URL_VARIABLE = "THOTH_ORACLE_BASE_URL"
|
||||
EMBEDDING_MODEL_VARIABLE = "THOTH_EMBEDDING_MODEL"
|
||||
EMBEDDING_DIMENSION_VARIABLE = "THOTH_EMBEDDING_DIMENSION"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EmbeddingConfig:
|
||||
"""Deployment values used to construct an Oracle embedding provider."""
|
||||
|
||||
oracle_base_url: str | None
|
||||
embedding_model: str | None
|
||||
embedding_dimension: int | None
|
||||
|
||||
def require_live_embedding(self) -> "EmbeddingConfig":
|
||||
"""Validate values required before making a live inference request."""
|
||||
|
||||
if not self.oracle_base_url:
|
||||
raise ValueError(f"{ORACLE_BASE_URL_VARIABLE} is required for embeddings")
|
||||
parsed_url = urlsplit(self.oracle_base_url)
|
||||
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
|
||||
raise ValueError(
|
||||
f"{ORACLE_BASE_URL_VARIABLE} must be an explicit HTTP(S) URL "
|
||||
"including its scheme"
|
||||
)
|
||||
if parsed_url.query or parsed_url.fragment:
|
||||
raise ValueError(
|
||||
f"{ORACLE_BASE_URL_VARIABLE} must not contain a query or fragment"
|
||||
)
|
||||
if not self.embedding_model:
|
||||
raise ValueError(f"{EMBEDDING_MODEL_VARIABLE} is required for embeddings")
|
||||
return self
|
||||
|
||||
|
||||
def read_env_file(path: Path) -> dict[str, str]:
|
||||
"""Read simple KEY=VALUE entries from a local dotenv file."""
|
||||
|
||||
if not path.exists():
|
||||
return {}
|
||||
|
||||
values: dict[str, str] = {}
|
||||
for line_number, original_line in enumerate(
|
||||
path.read_text(encoding="utf-8").splitlines(), start=1
|
||||
):
|
||||
line = original_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
raise ValueError(f"invalid .env entry at {path}:{line_number}")
|
||||
name, value = line.split("=", 1)
|
||||
name = name.strip()
|
||||
value = value.strip()
|
||||
if value[:1] == value[-1:] and value[:1] in {'"', "'"}:
|
||||
value = value[1:-1]
|
||||
values[name] = value
|
||||
return values
|
||||
|
||||
|
||||
def load_embedding_config(
|
||||
environ: Mapping[str, str] | None = None,
|
||||
env_path: str | Path = ".env",
|
||||
) -> EmbeddingConfig:
|
||||
"""Load `.env`, then override it with operating-system environment values."""
|
||||
|
||||
values = read_env_file(Path(env_path))
|
||||
values.update(os.environ if environ is None else environ)
|
||||
|
||||
base_url = values.get(ORACLE_BASE_URL_VARIABLE, "").strip() or None
|
||||
model = values.get(EMBEDDING_MODEL_VARIABLE, "").strip() or None
|
||||
dimension = parse_optional_dimension(values.get(EMBEDDING_DIMENSION_VARIABLE))
|
||||
return EmbeddingConfig(base_url, model, dimension)
|
||||
|
||||
|
||||
def parse_optional_dimension(value: str | None) -> int | None:
|
||||
"""Interpret a blank dimension as unknown and validate configured values."""
|
||||
|
||||
if value is None or not value.strip():
|
||||
return None
|
||||
try:
|
||||
dimension = int(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
f"{EMBEDDING_DIMENSION_VARIABLE} must be a positive integer or blank"
|
||||
) from error
|
||||
if dimension <= 0:
|
||||
raise ValueError(
|
||||
f"{EMBEDDING_DIMENSION_VARIABLE} must be a positive integer or blank"
|
||||
)
|
||||
return dimension
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Explicit provider boundary for locally generated text embeddings."""
|
||||
|
||||
import json
|
||||
import math
|
||||
from typing import Protocol
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from .config import EmbeddingConfig
|
||||
from .models import Chunk, Embedding
|
||||
|
||||
|
||||
class EmbeddingConnectionError(RuntimeError):
|
||||
"""The configured Oracle embedding service could not be reached."""
|
||||
|
||||
|
||||
class EmbeddingRequestError(RuntimeError):
|
||||
"""Oracle was reached but rejected or malformed the embedding request."""
|
||||
|
||||
|
||||
class EmbeddingProvider(Protocol):
|
||||
"""The narrow text-in, vector-out boundary used by Project Thoth."""
|
||||
|
||||
model_name: str
|
||||
provider_name: str
|
||||
expected_dimension: int | None
|
||||
|
||||
def embed(self, text: str) -> list[float]:
|
||||
"""Return one numerical vector for the supplied text."""
|
||||
|
||||
|
||||
class OllamaEmbeddingProvider:
|
||||
"""Generate embeddings through Oracle's Ollama-compatible HTTP API."""
|
||||
|
||||
provider_name = "Ollama"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
base_url: str,
|
||||
expected_dimension: int | None,
|
||||
timeout_seconds: float = 60.0,
|
||||
) -> None:
|
||||
self.model_name = model_name
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.expected_dimension = expected_dimension
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def embed(self, text: str) -> list[float]:
|
||||
"""Send exactly ``text`` to Ollama and return its single vector."""
|
||||
|
||||
payload = json.dumps(
|
||||
{
|
||||
"model": self.model_name,
|
||||
"input": text,
|
||||
# An oversized input should fail visibly rather than be changed
|
||||
# without the developer knowing what the model received.
|
||||
"truncate": False,
|
||||
}
|
||||
).encode("utf-8")
|
||||
request = Request(
|
||||
f"{self.base_url}/api/embed",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=self.timeout_seconds) as response:
|
||||
result = json.load(response)
|
||||
except HTTPError as error:
|
||||
details = error.read().decode("utf-8", errors="replace")
|
||||
raise EmbeddingRequestError(
|
||||
f"Ollama embedding request failed with HTTP {error.code}: {details}"
|
||||
) from error
|
||||
except URLError as error:
|
||||
raise EmbeddingConnectionError(
|
||||
f"Cannot reach Ollama embedding service at {self.base_url}: "
|
||||
f"{error.reason}"
|
||||
) from error
|
||||
|
||||
embeddings = result.get("embeddings")
|
||||
if not isinstance(embeddings, list) or len(embeddings) != 1:
|
||||
raise EmbeddingRequestError(
|
||||
"Ollama response did not contain exactly one embedding"
|
||||
)
|
||||
|
||||
vector = embeddings[0]
|
||||
if not isinstance(vector, list) or not vector:
|
||||
raise EmbeddingRequestError(
|
||||
"Ollama returned an empty or invalid embedding vector"
|
||||
)
|
||||
if not all(isinstance(value, (int, float)) for value in vector):
|
||||
raise EmbeddingRequestError(
|
||||
"Ollama embedding vector contains non-numerical values"
|
||||
)
|
||||
|
||||
return [float(value) for value in vector]
|
||||
|
||||
|
||||
def embed_chunk(chunk: Chunk, provider: EmbeddingProvider) -> Embedding:
|
||||
"""Embed only ``chunk.text`` and retain its source linkage."""
|
||||
|
||||
vector = provider.embed(chunk.text)
|
||||
if (
|
||||
provider.expected_dimension is not None
|
||||
and len(vector) != provider.expected_dimension
|
||||
):
|
||||
raise ValueError(
|
||||
f"model {provider.model_name} returned {len(vector)} dimensions; "
|
||||
f"expected {provider.expected_dimension}"
|
||||
)
|
||||
if not all(math.isfinite(value) for value in vector):
|
||||
raise ValueError("embedding vector contains a non-finite value")
|
||||
|
||||
return Embedding(
|
||||
chunk_id=chunk.chunk_id,
|
||||
document_id=chunk.document_id,
|
||||
embedding_model=provider.model_name,
|
||||
embedding_provider=provider.provider_name,
|
||||
embedding_dimension=len(vector),
|
||||
vector=tuple(vector),
|
||||
)
|
||||
|
||||
|
||||
def provider_from_config(config: EmbeddingConfig) -> OllamaEmbeddingProvider:
|
||||
"""Construct the provider only from validated centralized configuration."""
|
||||
|
||||
validated = config.require_live_embedding()
|
||||
assert validated.oracle_base_url is not None
|
||||
assert validated.embedding_model is not None
|
||||
return OllamaEmbeddingProvider(
|
||||
model_name=validated.embedding_model,
|
||||
base_url=validated.oracle_base_url,
|
||||
expected_dimension=validated.embedding_dimension,
|
||||
)
|
||||
@@ -0,0 +1,302 @@
|
||||
"""Command-line entry point for single-file Markdown ingestion."""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
|
||||
from .chunking import (
|
||||
DEFAULT_CHUNK_OVERLAP,
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
ChunkConfiguration,
|
||||
chunk_document,
|
||||
)
|
||||
from .config import EmbeddingConfig, load_embedding_config
|
||||
from .embeddings import (
|
||||
EmbeddingConnectionError,
|
||||
EmbeddingRequestError,
|
||||
embed_chunk,
|
||||
provider_from_config,
|
||||
)
|
||||
from .loader import load_markdown_document
|
||||
from .models import Chunk, Document, Embedding
|
||||
from .similarity import cosine_similarity, vector_norm
|
||||
|
||||
PREVIEW_LENGTH = 120
|
||||
VECTOR_PREVIEW_LENGTH = 8
|
||||
CONFIGURATION_TEST_TEXT = "Project Thoth embedding configuration test."
|
||||
|
||||
|
||||
def format_document(document: Document) -> str:
|
||||
"""Format the mechanical document record for human inspection."""
|
||||
|
||||
return "\n".join(
|
||||
(
|
||||
f"Document ID: {document.document_id}",
|
||||
f"Source Path: {document.source_path}",
|
||||
f"Filename: {document.filename}",
|
||||
f"Source Format: {document.source_format}",
|
||||
f"File Size: {document.file_size} bytes",
|
||||
f"Modified At: {document.modified_at.isoformat()}",
|
||||
f"Raw Text Length: {len(document.raw_text)} characters",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def preview(text: str) -> str:
|
||||
"""Make a chunk boundary visible on one console line."""
|
||||
|
||||
return text.replace("\r", "\\r").replace("\n", "\\n")
|
||||
|
||||
|
||||
def format_chunks(
|
||||
document: Document,
|
||||
chunks: list[Chunk],
|
||||
configuration: ChunkConfiguration,
|
||||
show_chunks: bool = False,
|
||||
) -> str:
|
||||
"""Format aggregate statistics and inspectable chunk boundaries."""
|
||||
|
||||
sizes = [chunk.size for chunk in chunks]
|
||||
minimum_size = min(sizes, default=0)
|
||||
maximum_size = max(sizes, default=0)
|
||||
average_size = sum(sizes) / len(sizes) if sizes else 0.0
|
||||
lines = [
|
||||
"",
|
||||
"Chunking Summary",
|
||||
f"Source Size: {len(document.raw_text)} characters",
|
||||
f"Chunk Size: {configuration.chunk_size} characters",
|
||||
f"Chunk Overlap: {configuration.chunk_overlap} characters",
|
||||
f"Chunk Count: {len(chunks)}",
|
||||
f"Minimum Chunk Size: {minimum_size} characters",
|
||||
f"Maximum Chunk Size: {maximum_size} characters",
|
||||
f"Average Chunk Size: {average_size:.2f} characters",
|
||||
]
|
||||
|
||||
for chunk in chunks:
|
||||
lines.extend(
|
||||
(
|
||||
"",
|
||||
f"Chunk {chunk.chunk_number}",
|
||||
f" Chunk ID: {chunk.chunk_id}",
|
||||
f" Chunk Number: {chunk.chunk_number}",
|
||||
f" Size: {chunk.size} {chunk.size_unit}",
|
||||
f" Offsets: [{chunk.start_offset}, {chunk.end_offset}) characters",
|
||||
f" Overlap With Previous: {chunk.overlap_with_previous} characters",
|
||||
f" Overlap With Next: {chunk.overlap_with_next} characters",
|
||||
f' Starts With: "{preview(chunk.text[:PREVIEW_LENGTH])}"',
|
||||
f' Ends With: "{preview(chunk.text[-PREVIEW_LENGTH:])}"',
|
||||
)
|
||||
)
|
||||
if show_chunks:
|
||||
lines.extend((" Complete Text:", chunk.text))
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_embedding(chunk: Chunk, embedding: Embedding) -> str:
|
||||
"""Format inspectable properties without dumping the complete vector."""
|
||||
|
||||
leading_values = ", ".join(
|
||||
f"{value:.6f}" for value in embedding.vector[:VECTOR_PREVIEW_LENGTH]
|
||||
)
|
||||
return "\n".join(
|
||||
(
|
||||
"",
|
||||
f"Embedding for Chunk {chunk.chunk_number}",
|
||||
f" Chunk ID: {embedding.chunk_id}",
|
||||
f" Chunk Number: {chunk.chunk_number}",
|
||||
f" Provider: {embedding.embedding_provider}",
|
||||
f" Model: {embedding.embedding_model}",
|
||||
f" Dimension: {embedding.embedding_dimension}",
|
||||
f" Vector Norm: {vector_norm(embedding.vector):.8f}",
|
||||
f" Vector Preview: [{leading_values}, ...] (incomplete)",
|
||||
f' Text Preview: "{preview(chunk.text[:PREVIEW_LENGTH])}"',
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def selected_chunk(chunks: list[Chunk], chunk_number: int) -> Chunk:
|
||||
"""Return a zero-based chunk selection or fail with an actionable error."""
|
||||
|
||||
if chunk_number < 0 or chunk_number >= len(chunks):
|
||||
raise ValueError(
|
||||
f"chunk number {chunk_number} is out of range; "
|
||||
f"valid range is 0-{len(chunks) - 1}"
|
||||
)
|
||||
return chunks[chunk_number]
|
||||
|
||||
|
||||
def check_embedding_config(config: EmbeddingConfig) -> tuple[str, bool]:
|
||||
"""Make one explicit live request and report configuration diagnostics."""
|
||||
|
||||
lines = [
|
||||
"Oracle Embedding Configuration",
|
||||
f"Oracle Base URL: {config.oracle_base_url or '(missing)'}",
|
||||
f"Embedding Model: {config.embedding_model or '(missing)'}",
|
||||
"Configured Dimension: "
|
||||
+ (str(config.embedding_dimension) if config.embedding_dimension else "unknown"),
|
||||
]
|
||||
try:
|
||||
provider = provider_from_config(config)
|
||||
vector = provider.embed(CONFIGURATION_TEST_TEXT)
|
||||
if (
|
||||
config.embedding_dimension is not None
|
||||
and len(vector) != config.embedding_dimension
|
||||
):
|
||||
raise ValueError(
|
||||
f"model {provider.model_name} returned {len(vector)} dimensions; "
|
||||
f"expected {config.embedding_dimension}"
|
||||
)
|
||||
except EmbeddingConnectionError as error:
|
||||
lines.extend(
|
||||
(
|
||||
"Connection: failed",
|
||||
f"Embedding Request: failed - {error}",
|
||||
"Returned Dimension: unavailable",
|
||||
)
|
||||
)
|
||||
return "\n".join(lines), False
|
||||
except (EmbeddingRequestError, ValueError) as error:
|
||||
lines.extend(
|
||||
(
|
||||
"Connection: succeeded",
|
||||
f"Embedding Request: failed - {error}",
|
||||
"Returned Dimension: unavailable",
|
||||
)
|
||||
)
|
||||
return "\n".join(lines), False
|
||||
|
||||
lines.extend(
|
||||
(
|
||||
"Connection: succeeded",
|
||||
"Embedding Request: succeeded",
|
||||
f"Returned Dimension: {len(vector)}",
|
||||
)
|
||||
)
|
||||
return "\n".join(lines), True
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Build the command-line parser."""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Ingest one Markdown Primary Source for inspection."
|
||||
)
|
||||
parser.add_argument(
|
||||
"source", nargs="?", help="Path to one Markdown (.md) source file"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chunk-size",
|
||||
type=int,
|
||||
default=DEFAULT_CHUNK_SIZE,
|
||||
help=f"Characters per chunk (default: {DEFAULT_CHUNK_SIZE})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chunk-overlap",
|
||||
type=int,
|
||||
default=DEFAULT_CHUNK_OVERLAP,
|
||||
help=f"Characters repeated between chunks (default: {DEFAULT_CHUNK_OVERLAP})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-chunks",
|
||||
action="store_true",
|
||||
help="Print complete chunk text in addition to boundary previews",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--embed-chunk",
|
||||
type=int,
|
||||
action="append",
|
||||
default=[],
|
||||
help="Embed one zero-based chunk number; may be repeated",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compare-chunks",
|
||||
type=int,
|
||||
nargs=2,
|
||||
metavar=("CHUNK_A", "CHUNK_B"),
|
||||
help="Embed and compare two zero-based chunk numbers",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check-embedding-config",
|
||||
action="store_true",
|
||||
help="Validate configured Oracle embedding inference with one small request",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Run ingestion from command-line arguments."""
|
||||
|
||||
# Vault Markdown may contain characters outside a host console's legacy
|
||||
# code page. UTF-8 keeps previews and --show-chunks faithful to the source.
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
parser = build_parser()
|
||||
arguments = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
embedding_config = load_embedding_config()
|
||||
except ValueError as error:
|
||||
parser.error(str(error))
|
||||
|
||||
if arguments.check_embedding_config:
|
||||
if arguments.source:
|
||||
parser.error("source must be omitted with --check-embedding-config")
|
||||
report, succeeded = check_embedding_config(embedding_config)
|
||||
print(report)
|
||||
return 0 if succeeded else 1
|
||||
if not arguments.source:
|
||||
parser.error("source is required unless --check-embedding-config is used")
|
||||
|
||||
try:
|
||||
document = load_markdown_document(arguments.source)
|
||||
configuration = ChunkConfiguration(
|
||||
chunk_size=arguments.chunk_size,
|
||||
chunk_overlap=arguments.chunk_overlap,
|
||||
)
|
||||
chunks = chunk_document(document, configuration)
|
||||
requested_numbers = list(arguments.embed_chunk)
|
||||
if arguments.compare_chunks:
|
||||
requested_numbers.extend(arguments.compare_chunks)
|
||||
|
||||
embeddings_by_chunk: dict[int, Embedding] = {}
|
||||
if requested_numbers:
|
||||
provider = provider_from_config(embedding_config)
|
||||
for chunk_number in dict.fromkeys(requested_numbers):
|
||||
chunk = selected_chunk(chunks, chunk_number)
|
||||
embeddings_by_chunk[chunk_number] = embed_chunk(chunk, provider)
|
||||
except (FileNotFoundError, RuntimeError, ValueError, OSError, UnicodeError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
print(format_document(document))
|
||||
print(format_chunks(document, chunks, configuration, arguments.show_chunks))
|
||||
for chunk_number in dict.fromkeys(requested_numbers):
|
||||
print(
|
||||
format_embedding(
|
||||
selected_chunk(chunks, chunk_number),
|
||||
embeddings_by_chunk[chunk_number],
|
||||
)
|
||||
)
|
||||
if arguments.compare_chunks:
|
||||
chunk_a, chunk_b = arguments.compare_chunks
|
||||
score = cosine_similarity(
|
||||
embeddings_by_chunk[chunk_a].vector,
|
||||
embeddings_by_chunk[chunk_b].vector,
|
||||
)
|
||||
print(
|
||||
"\n".join(
|
||||
(
|
||||
"",
|
||||
"Chunk Comparison",
|
||||
f" Chunk A: {chunk_a}",
|
||||
f" Chunk B: {chunk_b}",
|
||||
f" Cosine Similarity: {score:.8f}",
|
||||
)
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Read one Markdown Primary Source into an ingestion document record."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .models import Document
|
||||
|
||||
MARKDOWN_SUFFIX = ".md"
|
||||
MARKDOWN_FORMAT = "Markdown"
|
||||
|
||||
|
||||
def normalized_source_identity(source_path: Path) -> str:
|
||||
"""Return the stable path string used to identify a logical source.
|
||||
|
||||
Resolving the path makes relative and absolute references to the same file
|
||||
equivalent. ``normcase`` also follows the host filesystem's case rules.
|
||||
File contents are deliberately excluded so edits retain document identity.
|
||||
"""
|
||||
|
||||
resolved_path = source_path.resolve()
|
||||
return os.path.normcase(str(resolved_path))
|
||||
|
||||
|
||||
def create_document_id(source_path: Path) -> str:
|
||||
"""Create a deterministic SHA-256 identifier from the normalized path."""
|
||||
|
||||
identity = normalized_source_identity(source_path)
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def load_markdown_document(source_path: str | Path) -> Document:
|
||||
"""Validate and ingest one UTF-8 Markdown file without modifying it."""
|
||||
|
||||
path = Path(source_path).expanduser()
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown source does not exist: {path}")
|
||||
if not path.is_file():
|
||||
raise ValueError(f"Markdown source is not a file: {path}")
|
||||
if path.suffix.lower() != MARKDOWN_SUFFIX:
|
||||
raise ValueError(
|
||||
f"Markdown ingestion currently supports .md files only: {path}"
|
||||
)
|
||||
|
||||
resolved_path = path.resolve()
|
||||
# newline="" prevents Python from translating source line endings while
|
||||
# decoding the text, keeping the in-memory source representation faithful.
|
||||
with resolved_path.open("r", encoding="utf-8", newline="") as source_file:
|
||||
raw_text = source_file.read()
|
||||
file_status = resolved_path.stat()
|
||||
|
||||
return Document(
|
||||
document_id=create_document_id(resolved_path),
|
||||
source_path=str(resolved_path),
|
||||
filename=resolved_path.name,
|
||||
source_format=MARKDOWN_FORMAT,
|
||||
file_size=file_status.st_size,
|
||||
modified_at=datetime.fromtimestamp(file_status.st_mtime, tz=timezone.utc),
|
||||
raw_text=raw_text,
|
||||
)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Data models at the RAG ingestion boundary."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Document:
|
||||
"""An immutable, in-memory representation of one source document.
|
||||
|
||||
Every field describes the source mechanically. No semantic classification or
|
||||
AI-generated information is added during ingestion.
|
||||
"""
|
||||
|
||||
document_id: str
|
||||
source_path: str
|
||||
filename: str
|
||||
source_format: str
|
||||
file_size: int
|
||||
modified_at: datetime
|
||||
raw_text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chunk:
|
||||
"""One derived retrieval unit selected from a document's raw text.
|
||||
|
||||
Chunk numbers are zero-based. Offsets are zero-based character positions in
|
||||
``Document.raw_text``; ``start_offset`` is inclusive and ``end_offset`` is
|
||||
exclusive, matching normal Python string slicing.
|
||||
"""
|
||||
|
||||
chunk_id: str
|
||||
document_id: str
|
||||
chunk_number: int
|
||||
text: str
|
||||
start_offset: int
|
||||
end_offset: int
|
||||
size: int
|
||||
size_unit: str
|
||||
overlap_with_previous: int
|
||||
overlap_with_next: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Embedding:
|
||||
"""A derived numerical representation of one chunk's exact text."""
|
||||
|
||||
chunk_id: str
|
||||
document_id: str
|
||||
embedding_model: str
|
||||
embedding_provider: str
|
||||
embedding_dimension: int
|
||||
vector: tuple[float, ...]
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Direct vector mathematics used for inspectable similarity experiments."""
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
def vector_norm(vector: Sequence[float]) -> float:
|
||||
"""Calculate Euclidean (L2) vector magnitude."""
|
||||
|
||||
return math.sqrt(math.fsum(value * value for value in vector))
|
||||
|
||||
|
||||
def cosine_similarity(vector_a: Sequence[float], vector_b: Sequence[float]) -> float:
|
||||
"""Return the cosine of the angle between two equal-dimensional vectors."""
|
||||
|
||||
if len(vector_a) != len(vector_b):
|
||||
raise ValueError(
|
||||
"cosine similarity requires vectors with equal dimensions: "
|
||||
f"received {len(vector_a)} and {len(vector_b)}"
|
||||
)
|
||||
if not vector_a:
|
||||
raise ValueError("cosine similarity requires non-empty vectors")
|
||||
|
||||
dot_product = math.fsum(
|
||||
value_a * value_b for value_a, value_b in zip(vector_a, vector_b)
|
||||
)
|
||||
norm_a = vector_norm(vector_a)
|
||||
norm_b = vector_norm(vector_b)
|
||||
if norm_a == 0.0 or norm_b == 0.0:
|
||||
raise ValueError("cosine similarity is undefined for a zero vector")
|
||||
|
||||
return dot_product / (norm_a * norm_b)
|
||||
File diff suppressed because one or more lines are too long
@@ -1,215 +0,0 @@
|
||||
# ChatGPT Conversation DOM Specification
|
||||
|
||||
## Application Shell
|
||||
|
||||
Purpose
|
||||
|
||||
Contains navigation, sidebar, overlays and application chrome.
|
||||
|
||||
Relevant Entry Point
|
||||
|
||||
main
|
||||
|
||||
Ignored
|
||||
|
||||
- dialogs
|
||||
- scripts
|
||||
- aria-live
|
||||
- overlays
|
||||
|
||||
---
|
||||
|
||||
## Conversation turn structure
|
||||
|
||||
### Turn container
|
||||
|
||||
Observed selector:
|
||||
|
||||
```css
|
||||
section[data-turn]
|
||||
```
|
||||
|
||||
Observed user-turn example:
|
||||
|
||||
```html
|
||||
<section
|
||||
data-turn-id="837e77b6-5763-4f50-887b-573586d0fcda"
|
||||
data-turn-id-container="837e77b6-5763-4f50-887b-573586d0fcda"
|
||||
data-testid="conversation-turn-1"
|
||||
data-turn="user">
|
||||
```
|
||||
|
||||
Purpose:
|
||||
|
||||
- Represents one ordered conversation turn.
|
||||
- Provides the strongest observed semantic boundary for a turn.
|
||||
- Groups the authored message with turn-level interface elements and any auxiliary content.
|
||||
- Carries turn identity, order, and role information.
|
||||
|
||||
Observed semantic attributes:
|
||||
|
||||
- `data-turn`
|
||||
- `data-turn-id`
|
||||
- `data-turn-id-container`
|
||||
- `data-testid`
|
||||
|
||||
The section also carries layout and scrolling classes, but those classes appear incidental to presentation and should not be treated as stable extraction selectors.
|
||||
|
||||
The extractor should enumerate `section[data-turn]` elements in DOM order.
|
||||
|
||||
### Message container
|
||||
|
||||
Observed selector:
|
||||
|
||||
```css
|
||||
[data-message-author-role]
|
||||
```
|
||||
|
||||
Observed user-message example:
|
||||
|
||||
```html
|
||||
<div
|
||||
data-message-author-role="user"
|
||||
data-message-id="837e77b6-5763-4f50-887b-573586d0fcda">
|
||||
```
|
||||
|
||||
Purpose:
|
||||
|
||||
- Represents the authored message region inside a conversation turn.
|
||||
- Provides the author role independently of the outer turn container.
|
||||
- Contains the rendered message payload, but does not necessarily contain all turn-level content.
|
||||
|
||||
Observed semantic attributes:
|
||||
|
||||
- `data-message-author-role`
|
||||
- `data-message-id`
|
||||
|
||||
Relationship to the turn container:
|
||||
|
||||
- The message container is a descendant of `section[data-turn]`.
|
||||
- In the observed user turn, `data-turn-id`, `data-turn-id-container`, and `data-message-id` contain the same UUID.
|
||||
- This correspondence is observed in the sample but should not yet be assumed to hold for every turn type without further verification.
|
||||
|
||||
### Rendered turn-content region
|
||||
|
||||
Observed selector:
|
||||
|
||||
```css
|
||||
[data-conversation-screenshot-content]
|
||||
```
|
||||
|
||||
Observed relationship:
|
||||
|
||||
```text
|
||||
section[data-turn]
|
||||
[data-conversation-screenshot-content]
|
||||
[data-message-author-role]
|
||||
turn-level actions and controls
|
||||
```
|
||||
|
||||
The attribute name suggests that ChatGPT may use this node to define the content included in a conversation screenshot or similar rendered representation.
|
||||
|
||||
Current assessment:
|
||||
|
||||
- Observed: yes
|
||||
- Potentially useful: yes
|
||||
- Stability: unknown
|
||||
- Required by extractor: not yet established
|
||||
|
||||
### Turn-level controls
|
||||
|
||||
The user-turn action controls are descendants of the turn section but siblings of the rendered message-content region:
|
||||
|
||||
```html
|
||||
<div aria-label="Your message actions" role="group">
|
||||
```
|
||||
|
||||
These controls are not conversation content and should be ignored by the extractor.
|
||||
|
||||
Their location confirms that the outer `section[data-turn]` represents the complete rendered turn, while `[data-message-author-role]` represents the authored-message component within that turn.
|
||||
|
||||
## Current extraction model
|
||||
|
||||
```text
|
||||
section[data-turn]
|
||||
Turn boundary, order, identity, and role
|
||||
|
||||
[data-conversation-screenshot-content]
|
||||
Candidate rendered-content boundary
|
||||
|
||||
[data-message-author-role]
|
||||
Authored message region
|
||||
|
||||
Turn-level controls and auxiliary UI
|
||||
Ignore unless later analysis identifies meaningful content
|
||||
```
|
||||
|
||||
Recommended initial traversal:
|
||||
|
||||
```text
|
||||
1. Select all section[data-turn] elements in DOM order.
|
||||
2. Read the turn role and identity from the section attributes.
|
||||
3. Locate the descendant [data-message-author-role] element.
|
||||
4. Extract message content from within that semantic message region.
|
||||
5. Ignore action controls and other interface-only descendants.
|
||||
6. Preserve the outer turn as the unit of extraction so future sibling content can be evaluated.
|
||||
```
|
||||
|
||||
## Conversation materialization evidence
|
||||
|
||||
### Scroll container
|
||||
|
||||
The saved full-page HTML places `main > #thread` inside an ancestor carrying:
|
||||
|
||||
```css
|
||||
[data-scroll-root]
|
||||
```
|
||||
|
||||
That element also carries the vertical scrolling behavior and scroll-state attributes. It is the strongest semantic selector for the conversation viewport. The sidebar has a separate scrolling region and must not be used for conversation materialization.
|
||||
|
||||
Recommended lookup:
|
||||
|
||||
```text
|
||||
#thread
|
||||
closest ancestor [data-scroll-root]
|
||||
```
|
||||
|
||||
### Partial rendering evidence
|
||||
|
||||
The saved Jellyfin snapshot contains 11 `section[data-turn]` elements, while the conversation is known to contain approximately 28 user turns and their corresponding assistant turns. The observed `data-testid="conversation-turn-N"` values contain large sequence gaps. This supports the conclusion that the saved DOM is incomplete.
|
||||
|
||||
### Dynamic behavior requiring live verification
|
||||
|
||||
A static snapshot cannot establish:
|
||||
|
||||
- whether upward scrolling inserts older turns,
|
||||
- whether newer turns are removed during traversal,
|
||||
- whether downward scrolling restores newer turns,
|
||||
- whether all visited turns remain in the DOM,
|
||||
- or which loading indicator, if any, is shown while older turns materialize.
|
||||
|
||||
The capture implementation must therefore retain each observed turn by `data-turn-id` as it traverses, rather than assuming the final DOM contains the complete conversation. These dynamic behaviors must be confirmed against a live long conversation when a browser session is available.
|
||||
|
||||
### Ordering evidence
|
||||
|
||||
Runtime observation confirms that `data-testid="conversation-turn-N"` values are reused within different virtualized windows. They are local presentation indices and must not be used as global conversation order.
|
||||
|
||||
Global ordering must instead be reconstructed from:
|
||||
|
||||
- stable identity from `data-turn-id`,
|
||||
- DOM order inside each rendered window,
|
||||
- and contiguous stable-ID overlap between consecutive windows.
|
||||
|
||||
UUID turn IDs identify turns but do not encode order and must never be sorted lexically.
|
||||
|
||||
### Confirmed virtualization behavior
|
||||
|
||||
Manual runtime observation established that:
|
||||
|
||||
- different scroll positions expose different subsets of the conversation,
|
||||
- observed windows contained 12, 11, and 9 user turns,
|
||||
- turns leave the DOM as other turns enter it,
|
||||
- the complete conversation does not coexist in the DOM,
|
||||
- and the Jellyfin conversation contains 28 user turns with corresponding assistant turns.
|
||||
|
||||
Capture must therefore extract each stable turn immediately and retain ordered window observations. The final live DOM is not a complete source artifact.
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,136 @@
|
||||
"""Tests for the Work Order 0002 mechanical chunking stage."""
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from processors.rag_ingestion.chunking import ChunkConfiguration, chunk_document
|
||||
from processors.rag_ingestion.ingest import format_chunks
|
||||
from processors.rag_ingestion.loader import load_markdown_document
|
||||
from processors.rag_ingestion.models import Document
|
||||
|
||||
|
||||
def document_with_text(text: str) -> Document:
|
||||
return Document(
|
||||
document_id="document-identity",
|
||||
source_path="/vault/conversation.md",
|
||||
filename="conversation.md",
|
||||
source_format="Markdown",
|
||||
file_size=len(text.encode("utf-8")),
|
||||
modified_at=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
raw_text=text,
|
||||
)
|
||||
|
||||
|
||||
class CharacterChunkingTests(unittest.TestCase):
|
||||
def test_small_document_produces_one_traceable_chunk(self) -> None:
|
||||
document = document_with_text("short source")
|
||||
|
||||
chunks = chunk_document(document, ChunkConfiguration(20, 5))
|
||||
|
||||
self.assertEqual(len(chunks), 1)
|
||||
self.assertEqual(chunks[0].document_id, document.document_id)
|
||||
self.assertEqual(chunks[0].chunk_number, 0)
|
||||
self.assertEqual(chunks[0].text, document.raw_text)
|
||||
self.assertEqual((chunks[0].start_offset, chunks[0].end_offset), (0, 12))
|
||||
self.assertEqual(chunks[0].size_unit, "characters")
|
||||
|
||||
def test_multiple_chunks_are_ordered_and_match_source_offsets(self) -> None:
|
||||
document = document_with_text("abcdefghijklmnopqrstuvwxyz")
|
||||
|
||||
chunks = chunk_document(document, ChunkConfiguration(10, 2))
|
||||
|
||||
self.assertGreater(len(chunks), 1)
|
||||
self.assertEqual([chunk.chunk_number for chunk in chunks], list(range(3)))
|
||||
for chunk in chunks:
|
||||
self.assertEqual(
|
||||
chunk.text,
|
||||
document.raw_text[chunk.start_offset : chunk.end_offset],
|
||||
)
|
||||
|
||||
def test_adjacent_chunks_contain_configured_overlap(self) -> None:
|
||||
chunks = chunk_document(
|
||||
document_with_text("abcdefghijklmnopqrstuvwxyz"),
|
||||
ChunkConfiguration(10, 3),
|
||||
)
|
||||
|
||||
for previous, current in zip(chunks, chunks[1:]):
|
||||
self.assertEqual(previous.text[-3:], current.text[:3])
|
||||
self.assertEqual(previous.overlap_with_next, 3)
|
||||
self.assertEqual(current.overlap_with_previous, 3)
|
||||
|
||||
def test_zero_overlap_produces_adjacent_ranges(self) -> None:
|
||||
chunks = chunk_document(
|
||||
document_with_text("abcdefghijklmnopqrst"),
|
||||
ChunkConfiguration(7, 0),
|
||||
)
|
||||
|
||||
for previous, current in zip(chunks, chunks[1:]):
|
||||
self.assertEqual(previous.end_offset, current.start_offset)
|
||||
self.assertEqual(previous.overlap_with_next, 0)
|
||||
self.assertEqual(current.overlap_with_previous, 0)
|
||||
|
||||
def test_invalid_configuration_is_rejected(self) -> None:
|
||||
invalid_settings = ((0, 0), (-1, 0), (10, -1), (10, 10), (10, 11))
|
||||
|
||||
for chunk_size, overlap in invalid_settings:
|
||||
with self.subTest(chunk_size=chunk_size, overlap=overlap):
|
||||
with self.assertRaises(ValueError):
|
||||
ChunkConfiguration(chunk_size, overlap)
|
||||
|
||||
def test_repeated_chunking_is_deterministic(self) -> None:
|
||||
document = document_with_text("0123456789" * 5)
|
||||
configuration = ChunkConfiguration(13, 4)
|
||||
|
||||
first = chunk_document(document, configuration)
|
||||
second = chunk_document(document, configuration)
|
||||
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_configuration_changes_boundaries_and_ids(self) -> None:
|
||||
document = document_with_text("0123456789" * 5)
|
||||
|
||||
first = chunk_document(document, ChunkConfiguration(12, 2))
|
||||
second = chunk_document(document, ChunkConfiguration(15, 2))
|
||||
third = chunk_document(document, ChunkConfiguration(12, 3))
|
||||
|
||||
self.assertNotEqual(first, second)
|
||||
self.assertNotEqual(first, third)
|
||||
self.assertNotEqual(first[0].chunk_id, second[0].chunk_id)
|
||||
self.assertNotEqual(first[0].chunk_id, third[0].chunk_id)
|
||||
|
||||
def test_empty_document_produces_no_chunks(self) -> None:
|
||||
self.assertEqual(chunk_document(document_with_text("")), [])
|
||||
|
||||
def test_inspection_output_can_include_complete_chunk_text(self) -> None:
|
||||
document = document_with_text("complete chunk text")
|
||||
configuration = ChunkConfiguration(50, 0)
|
||||
chunks = chunk_document(document, configuration)
|
||||
|
||||
summary = format_chunks(document, chunks, configuration, show_chunks=False)
|
||||
detailed = format_chunks(document, chunks, configuration, show_chunks=True)
|
||||
|
||||
self.assertIn("Chunk Count: 1", summary)
|
||||
self.assertIn("Offsets: [0, 19) characters", summary)
|
||||
self.assertNotIn("Complete Text:", summary)
|
||||
self.assertIn("Complete Text:\ncomplete chunk text", detailed)
|
||||
|
||||
def test_chunking_preserves_document_and_source_file(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
source = Path(directory) / "conversation.md"
|
||||
source.write_text("Primary Source\n" * 20, encoding="utf-8")
|
||||
before_hash = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
document = load_markdown_document(source)
|
||||
before_text = document.raw_text
|
||||
|
||||
chunk_document(document, ChunkConfiguration(25, 5))
|
||||
|
||||
after_hash = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
self.assertEqual(document.raw_text, before_text)
|
||||
self.assertEqual(before_hash, after_hash)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Offline tests for Work Order 0004 environment configuration."""
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from processors.rag_ingestion.config import EmbeddingConfig, load_embedding_config
|
||||
from processors.rag_ingestion.embeddings import embed_chunk, provider_from_config
|
||||
from processors.rag_ingestion.embeddings import (
|
||||
EmbeddingConnectionError,
|
||||
EmbeddingRequestError,
|
||||
)
|
||||
from processors.rag_ingestion.ingest import check_embedding_config
|
||||
from tests.test_chunking import document_with_text
|
||||
from processors.rag_ingestion.chunking import ChunkConfiguration, chunk_document
|
||||
|
||||
|
||||
class EmbeddingConfigurationTests(unittest.TestCase):
|
||||
def test_valid_configuration_parses(self) -> None:
|
||||
config = self.load(
|
||||
THOTH_ORACLE_BASE_URL="http://oracle.example:11434/",
|
||||
THOTH_EMBEDDING_MODEL="embedding-model:version",
|
||||
THOTH_EMBEDDING_DIMENSION="768",
|
||||
)
|
||||
|
||||
self.assertEqual(config.embedding_dimension, 768)
|
||||
self.assertIs(config.require_live_embedding(), config)
|
||||
|
||||
def test_blank_dimension_means_unknown(self) -> None:
|
||||
config = self.load(THOTH_EMBEDDING_DIMENSION="")
|
||||
self.assertIsNone(config.embedding_dimension)
|
||||
|
||||
def test_missing_url_fails_only_for_live_use(self) -> None:
|
||||
config = self.load(THOTH_EMBEDDING_MODEL="model")
|
||||
with self.assertRaisesRegex(ValueError, "THOTH_ORACLE_BASE_URL"):
|
||||
config.require_live_embedding()
|
||||
|
||||
def test_url_requires_explicit_scheme(self) -> None:
|
||||
config = EmbeddingConfig("oracle:11434", "model", None)
|
||||
with self.assertRaisesRegex(ValueError, "including its scheme"):
|
||||
config.require_live_embedding()
|
||||
|
||||
def test_missing_model_fails_for_live_use(self) -> None:
|
||||
config = self.load(THOTH_ORACLE_BASE_URL="http://oracle:11434")
|
||||
with self.assertRaisesRegex(ValueError, "THOTH_EMBEDDING_MODEL"):
|
||||
config.require_live_embedding()
|
||||
|
||||
def test_invalid_dimensions_fail(self) -> None:
|
||||
for value in ("words", "0", "-1"):
|
||||
with self.subTest(value=value):
|
||||
with self.assertRaisesRegex(ValueError, "positive integer"):
|
||||
self.load(THOTH_EMBEDDING_DIMENSION=value)
|
||||
|
||||
def test_os_environment_overrides_dotenv(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
env_path = Path(directory) / ".env"
|
||||
env_path.write_text(
|
||||
"THOTH_ORACLE_BASE_URL=http://from-file:11434\n"
|
||||
"THOTH_EMBEDDING_MODEL=file-model\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
config = load_embedding_config(
|
||||
{"THOTH_EMBEDDING_MODEL": "os-model"}, env_path
|
||||
)
|
||||
self.assertEqual(config.oracle_base_url, "http://from-file:11434")
|
||||
self.assertEqual(config.embedding_model, "os-model")
|
||||
|
||||
def test_absent_configuration_has_no_localhost_fallback(self) -> None:
|
||||
config = self.load()
|
||||
self.assertIsNone(config.oracle_base_url)
|
||||
self.assertIsNone(config.embedding_model)
|
||||
|
||||
def test_provider_receives_validated_configuration(self) -> None:
|
||||
config = EmbeddingConfig("http://oracle:11434/", "model:v1", None)
|
||||
provider = provider_from_config(config)
|
||||
self.assertEqual(provider.base_url, "http://oracle:11434")
|
||||
self.assertEqual(provider.model_name, "model:v1")
|
||||
self.assertIsNone(provider.expected_dimension)
|
||||
|
||||
def load(self, **values: str) -> EmbeddingConfig:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
return load_embedding_config(values, Path(directory) / ".env")
|
||||
|
||||
|
||||
class OptionalDimensionTests(unittest.TestCase):
|
||||
def test_unknown_dimension_accepts_returned_shape(self) -> None:
|
||||
chunk = chunk_document(
|
||||
document_with_text("source"), ChunkConfiguration(100, 0)
|
||||
)[0]
|
||||
provider = provider_from_config(
|
||||
EmbeddingConfig("http://oracle:11434", "model:v1", None)
|
||||
)
|
||||
with patch.object(provider, "embed", return_value=[1.0, 2.0]):
|
||||
embedding = embed_chunk(chunk, provider)
|
||||
self.assertEqual(embedding.embedding_dimension, 2)
|
||||
|
||||
def test_matching_dimension_succeeds(self) -> None:
|
||||
chunk = chunk_document(
|
||||
document_with_text("source"), ChunkConfiguration(100, 0)
|
||||
)[0]
|
||||
provider = provider_from_config(
|
||||
EmbeddingConfig("http://oracle:11434", "model:v1", 2)
|
||||
)
|
||||
with patch.object(provider, "embed", return_value=[1.0, 2.0]):
|
||||
self.assertEqual(embed_chunk(chunk, provider).embedding_dimension, 2)
|
||||
|
||||
def test_mismatched_dimension_fails(self) -> None:
|
||||
chunk = chunk_document(
|
||||
document_with_text("source"), ChunkConfiguration(100, 0)
|
||||
)[0]
|
||||
provider = provider_from_config(
|
||||
EmbeddingConfig("http://oracle:11434", "model:v1", 3)
|
||||
)
|
||||
with patch.object(provider, "embed", return_value=[1.0, 2.0]):
|
||||
with self.assertRaisesRegex(ValueError, "expected 3"):
|
||||
embed_chunk(chunk, provider)
|
||||
|
||||
|
||||
class ConfigurationDiagnosticTests(unittest.TestCase):
|
||||
def test_request_failure_distinguishes_successful_connection(self) -> None:
|
||||
config = EmbeddingConfig("http://oracle:11434", "model:v1", None)
|
||||
with patch(
|
||||
"processors.rag_ingestion.ingest.provider_from_config"
|
||||
) as provider_factory:
|
||||
provider_factory.return_value.embed.side_effect = EmbeddingRequestError(
|
||||
"model does not support embeddings"
|
||||
)
|
||||
report, succeeded = check_embedding_config(config)
|
||||
self.assertFalse(succeeded)
|
||||
self.assertIn("Connection: succeeded", report)
|
||||
self.assertIn("Embedding Request: failed", report)
|
||||
|
||||
def test_connection_failure_is_reported_separately(self) -> None:
|
||||
config = EmbeddingConfig("http://oracle:11434", "model:v1", None)
|
||||
with patch(
|
||||
"processors.rag_ingestion.ingest.provider_from_config"
|
||||
) as provider_factory:
|
||||
provider_factory.return_value.embed.side_effect = EmbeddingConnectionError(
|
||||
"unreachable"
|
||||
)
|
||||
report, succeeded = check_embedding_config(config)
|
||||
self.assertFalse(succeeded)
|
||||
self.assertIn("Connection: failed", report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Tests for embedding records and provider-independent similarity math."""
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from processors.rag_ingestion.chunking import ChunkConfiguration, chunk_document
|
||||
from processors.rag_ingestion.embeddings import embed_chunk
|
||||
from processors.rag_ingestion.loader import load_markdown_document
|
||||
from processors.rag_ingestion.similarity import cosine_similarity
|
||||
|
||||
|
||||
class FixedEmbeddingProvider:
|
||||
provider_name = "Test Provider"
|
||||
model_name = "test-embedding-model:v1"
|
||||
expected_dimension = 3
|
||||
|
||||
def __init__(self, vector: list[float] | None = None) -> None:
|
||||
self.vector = vector or [0.25, -0.5, 0.75]
|
||||
self.received_text: str | None = None
|
||||
|
||||
def embed(self, text: str) -> list[float]:
|
||||
self.received_text = text
|
||||
return list(self.vector)
|
||||
|
||||
|
||||
class CosineSimilarityTests(unittest.TestCase):
|
||||
def test_identical_vectors_have_similarity_one(self) -> None:
|
||||
self.assertAlmostEqual(cosine_similarity([1.0, 2.0], [1.0, 2.0]), 1.0)
|
||||
|
||||
def test_orthogonal_vectors_have_similarity_zero(self) -> None:
|
||||
self.assertAlmostEqual(cosine_similarity([1.0, 0.0], [0.0, 1.0]), 0.0)
|
||||
|
||||
def test_opposite_vectors_have_similarity_negative_one(self) -> None:
|
||||
self.assertAlmostEqual(cosine_similarity([1.0, 2.0], [-1.0, -2.0]), -1.0)
|
||||
|
||||
def test_dimension_mismatch_is_rejected(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "equal dimensions"):
|
||||
cosine_similarity([1.0], [1.0, 2.0])
|
||||
|
||||
def test_zero_vector_is_rejected(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "zero vector"):
|
||||
cosine_similarity([0.0, 0.0], [1.0, 1.0])
|
||||
|
||||
def test_empty_vectors_are_rejected(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "non-empty"):
|
||||
cosine_similarity([], [])
|
||||
|
||||
|
||||
class EmbeddingRecordTests(unittest.TestCase):
|
||||
def test_embedding_uses_exact_chunk_text_and_preserves_linkage(self) -> None:
|
||||
document = self._load_document("# Conversation\n\nExact chunk text.")
|
||||
chunk = chunk_document(document, ChunkConfiguration(100, 0))[0]
|
||||
provider = FixedEmbeddingProvider()
|
||||
|
||||
embedding = embed_chunk(chunk, provider)
|
||||
|
||||
self.assertEqual(provider.received_text, chunk.text)
|
||||
self.assertEqual(embedding.chunk_id, chunk.chunk_id)
|
||||
self.assertEqual(embedding.document_id, document.document_id)
|
||||
self.assertEqual(embedding.embedding_model, provider.model_name)
|
||||
self.assertEqual(embedding.embedding_provider, provider.provider_name)
|
||||
self.assertEqual(embedding.embedding_dimension, 3)
|
||||
self.assertEqual(embedding.vector, (0.25, -0.5, 0.75))
|
||||
|
||||
def test_unexpected_embedding_dimension_is_rejected(self) -> None:
|
||||
document = self._load_document("source")
|
||||
chunk = chunk_document(document, ChunkConfiguration(100, 0))[0]
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "returned 2 dimensions"):
|
||||
embed_chunk(chunk, FixedEmbeddingProvider([1.0, 2.0]))
|
||||
|
||||
def test_embedding_preserves_source_document_and_chunk_text(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
source = Path(directory) / "conversation.md"
|
||||
source.write_text("Primary Source\n" * 10, encoding="utf-8")
|
||||
source_hash = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
document = load_markdown_document(source)
|
||||
chunk = chunk_document(document, ChunkConfiguration(50, 10))[0]
|
||||
document_text = document.raw_text
|
||||
chunk_text = chunk.text
|
||||
|
||||
embed_chunk(chunk, FixedEmbeddingProvider())
|
||||
|
||||
self.assertEqual(hashlib.sha256(source.read_bytes()).hexdigest(), source_hash)
|
||||
self.assertEqual(document.raw_text, document_text)
|
||||
self.assertEqual(chunk.text, chunk_text)
|
||||
|
||||
def _load_document(self, text: str):
|
||||
temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary_directory.cleanup)
|
||||
source = Path(temporary_directory.name) / "conversation.md"
|
||||
source.write_text(text, encoding="utf-8")
|
||||
return load_markdown_document(source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Tests for the Work Order 0001 ingestion boundary."""
|
||||
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from processors.rag_ingestion.ingest import format_document
|
||||
from processors.rag_ingestion.loader import load_markdown_document
|
||||
|
||||
|
||||
class MarkdownIngestionTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary_directory.cleanup)
|
||||
self.directory = Path(self.temporary_directory.name)
|
||||
|
||||
def write_source(self, name: str, text: str) -> Path:
|
||||
path = self.directory / name
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def test_loads_required_mechanical_fields_and_raw_text(self) -> None:
|
||||
source = self.write_source("conversation.md", "# Conversation\n\nHello.\n")
|
||||
|
||||
document = load_markdown_document(source)
|
||||
|
||||
self.assertEqual(document.source_path, str(source.resolve()))
|
||||
self.assertEqual(document.filename, "conversation.md")
|
||||
self.assertEqual(document.source_format, "Markdown")
|
||||
self.assertEqual(document.file_size, source.stat().st_size)
|
||||
self.assertEqual(document.raw_text, source.read_bytes().decode("utf-8"))
|
||||
self.assertIsNotNone(document.modified_at.tzinfo)
|
||||
|
||||
def test_identity_is_stable_when_source_contents_change(self) -> None:
|
||||
source = self.write_source("same.md", "original")
|
||||
|
||||
first = load_markdown_document(source)
|
||||
source.write_text("edited contents", encoding="utf-8")
|
||||
second = load_markdown_document(source.resolve())
|
||||
|
||||
self.assertEqual(first.document_id, second.document_id)
|
||||
|
||||
def test_different_paths_have_different_identities(self) -> None:
|
||||
first = self.write_source("first.md", "same contents")
|
||||
second = self.write_source("second.md", "same contents")
|
||||
|
||||
self.assertNotEqual(
|
||||
load_markdown_document(first).document_id,
|
||||
load_markdown_document(second).document_id,
|
||||
)
|
||||
|
||||
def test_missing_source_is_rejected_with_path(self) -> None:
|
||||
missing = self.directory / "missing.md"
|
||||
|
||||
with self.assertRaisesRegex(FileNotFoundError, "missing.md"):
|
||||
load_markdown_document(missing)
|
||||
|
||||
def test_non_markdown_source_is_rejected(self) -> None:
|
||||
source = self.write_source("notes.txt", "not Markdown")
|
||||
|
||||
with self.assertRaisesRegex(ValueError, r"supports \.md files only"):
|
||||
load_markdown_document(source)
|
||||
|
||||
def test_loading_does_not_modify_source(self) -> None:
|
||||
source = self.write_source("preserved.md", "Primary Source\n")
|
||||
before = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
|
||||
load_markdown_document(source)
|
||||
|
||||
after = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
self.assertEqual(before, after)
|
||||
|
||||
def test_inspection_output_contains_required_labels(self) -> None:
|
||||
source = self.write_source("inspect.md", "inspect me")
|
||||
|
||||
output = format_document(load_markdown_document(source))
|
||||
|
||||
for label in (
|
||||
"Document ID:",
|
||||
"Source Path:",
|
||||
"Filename:",
|
||||
"Source Format:",
|
||||
"File Size:",
|
||||
"Modified At:",
|
||||
"Raw Text Length:",
|
||||
):
|
||||
self.assertIn(label, output)
|
||||
|
||||
def test_cli_reports_invalid_input(self) -> None:
|
||||
missing = self.directory / "absent.md"
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "processors.rag_ingestion.ingest", str(missing)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(str(missing), result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user