Complete outstanding ingestion and document UX work

engine/tests/test_document_routes.py

@@ -0,0 +1,102 @@
+"""Focused tests for document metadata lookup and aggregation."""
+
+import pytest
+
+from kb import database
+from kb.routes import documents
+
+
+@pytest.fixture
+def document_db(tmp_path, monkeypatch):
+    db_path = tmp_path / "kb.db"
+    conn = database.get_connection(db_path)
+    database.init_schema(conn, 3)
+    conn.execute(
+        """
+        INSERT INTO documents(title, source_path, content_hash, doc_type, original_filename)
+        VALUES ('Vehicle Guide', '/data/staging/random.pdf', 'hash', 'pdf', 'M38T_manual.pdf')
+        """
+    )
+    doc_id = conn.execute("SELECT id FROM documents").fetchone()["id"]
+    conn.execute(
+        "INSERT INTO chunks(document_id, chunk_index, text, enriched_text) VALUES (?, 0, 'body', 'body')",
+        (doc_id,),
+    )
+    conn.commit()
+    conn.close()
+    monkeypatch.setattr(documents.cfg, "data_dir", tmp_path)
+    return doc_id
+
+
+@pytest.mark.asyncio
+async def test_list_filters_title_and_original_filename(document_db):
+    by_title = await documents.list_documents(
+        type=None, tags=None, title="vehicle", filename=None
+    )
+    by_filename = await documents.list_documents(
+        type=None, tags=None, title=None, filename="m38t"
+    )
+
+    assert [item["id"] for item in by_title] == [document_db]
+    assert by_filename[0]["original_filename"] == "M38T_manual.pdf"
+
+
+@pytest.mark.asyncio
+async def test_info_can_omit_chunks_without_losing_count(document_db):
+    result = await documents.get_document(document_db, include_chunks=False)
+
+    assert result["chunk_count"] == 1
+    assert result["chunks"] == []
+
+
+@pytest.mark.asyncio
+async def test_find_aggregates_chunk_hits_by_document(monkeypatch):
+    class Connection:
+        def close(self):
+            pass
+
+    monkeypatch.setattr(documents, "get_connection", lambda _path: Connection())
+    monkeypatch.setattr(
+        documents,
+        "hybrid_search",
+        lambda *_args, **_kwargs: {
+            "results": [
+                {
+                    "document_id": 7,
+                    "title": "Guide",
+                    "doc_type": "pdf",
+                    "source_path": "/staging/file",
+                    "original_filename": "guide.pdf",
+                    "tags": [],
+                    "score": 0.8,
+                    "text": "best hit",
+                },
+                {
+                    "document_id": 7,
+                    "title": "Guide",
+                    "doc_type": "pdf",
+                    "source_path": "/staging/file",
+                    "original_filename": "guide.pdf",
+                    "tags": [],
+                    "score": 0.7,
+                    "text": "second hit",
+                },
+            ]
+        },
+    )
+
+    result = await documents.find_documents(
+        documents.DocumentFindRequest(query="guide")
+    )
+
+    assert result == [{
+        "document_id": 7,
+        "title": "Guide",
+        "doc_type": "pdf",
+        "source_path": "/staging/file",
+        "original_filename": "guide.pdf",
+        "tags": [],
+        "score": 0.8,
+        "hit_count": 2,
+        "top_chunk": "best hit",
+    }]

engine/tests/test_ingest_quality.py

@@ -0,0 +1,19 @@
+"""Tests for filtering noisy OCR fragments."""
+
+from kb.ingest.quality import has_minimum_content
+
+
+def test_short_ocr_fragments_are_rejected():
+    assert not has_minimum_content('"')
+    assert not has_minimum_content("B")
+    assert not has_minimum_content("12")
+
+
+def test_short_identifiers_and_real_text_are_retained():
+    assert has_minimum_content("BID")
+    assert has_minimum_content("00:15")
+    assert has_minimum_content("Useful text")
+
+
+def test_filter_can_be_disabled():
+    assert has_minimum_content("B", min_alnum=0)

engine/tests/test_note_titles.py

@@ -0,0 +1,75 @@
+"""Tests for automatic and backfilled note titles."""
+
+import sqlite3
+
+from kb.ingest.note import auto_title
+from kb.maintenance import backfill_note_titles
+from kb.maintenance.backfill_note_titles import find_repairs
+
+
+def test_auto_title_strips_markdown_and_limits_length():
+    assert auto_title("## Useful heading\nBody") == "Useful heading"
+    assert auto_title("x" * 100) == "x" * 80
+
+
+def test_find_repairs_only_selects_synthetic_note_titles():
+    conn = sqlite3.connect(":memory:")
+    conn.row_factory = sqlite3.Row
+    conn.executescript(
+        """
+        CREATE TABLE documents (
+            id INTEGER PRIMARY KEY, title TEXT, original_filename TEXT, doc_type TEXT
+        );
+        CREATE TABLE chunks (
+            id INTEGER PRIMARY KEY, document_id INTEGER, chunk_index INTEGER,
+            text TEXT, metadata TEXT
+        );
+        INSERT INTO documents VALUES
+          (1, '7dec828a-1234-4567-89ab-123456789abc_note.note',
+           '7dec828a-1234-4567-89ab-123456789abc_note.note', 'note'),
+          (2, 'A deliberate title', 'named.note', 'note');
+        INSERT INTO chunks VALUES
+          (1, 1, 0, '# Derived title\nBody', '{}'),
+          (2, 2, 0, 'Must not replace', '{}');
+        """
+    )
+
+    assert find_repairs(conn) == [{
+        "document_id": 1,
+        "old_title": "7dec828a-1234-4567-89ab-123456789abc_note.note",
+        "new_title": "Derived title",
+        "original_filename": "7dec828a-1234-4567-89ab-123456789abc_note.note",
+    }]
+
+
+def test_apply_repairs_refreshes_title_search_text_and_vector(monkeypatch):
+    conn = sqlite3.connect(":memory:")
+    conn.row_factory = sqlite3.Row
+    conn.executescript(
+        """
+        CREATE TABLE documents (id INTEGER PRIMARY KEY, title TEXT, updated_at TEXT);
+        CREATE TABLE chunks (
+            id INTEGER PRIMARY KEY, document_id INTEGER, chunk_index INTEGER,
+            text TEXT, metadata TEXT, enriched_text TEXT
+        );
+        CREATE TABLE chunks_vec (embedding BLOB, chunk_id INTEGER);
+        INSERT INTO documents VALUES (1, 'old', NULL);
+        INSERT INTO chunks VALUES (3, 1, 0, 'New title\nBody', '{}', 'old text');
+        INSERT INTO chunks_vec VALUES (X'00', 3);
+        """
+    )
+    monkeypatch.setattr(
+        backfill_note_titles.embeddings,
+        "embed_texts",
+        lambda texts: [[1.0, 2.0, 3.0] for _ in texts],
+    )
+
+    backfill_note_titles.apply_repairs(conn, [{
+        "document_id": 1, "new_title": "New title"
+    }])
+
+    assert conn.execute("SELECT title FROM documents").fetchone()[0] == "New title"
+    assert conn.execute("SELECT enriched_text FROM chunks").fetchone()[0] == (
+        "New title\n\nNew title\nBody"
+    )
+    assert len(conn.execute("SELECT embedding FROM chunks_vec").fetchone()[0]) == 12