steve
6dfc13be1d
Implements five of the six enhancements from docs/kb-enhancements-proposal.htm, closing the retrieval-quality gap identified in the qmd review. - Cross-encoder reranking: new kb/reranker.py loads an optional reranking model at startup (KB_RERANK_ENABLED, KB_RERANKER_MODEL, KB_RERANK_CANDIDATES). Search degrades gracefully to plain hybrid retrieval when the model is absent. Exposed via a "rerank" block in /status, a rerank flag on search, and --no-rerank in the CLI. - RRF rank fusion: FTS and vector lists now merge by reciprocal rank fusion with a top-rank bonus, replacing the old score blend. Scores are comparable across queries. - Bench harness and explain traces: kb bench runs a query fixture against each backend and reports precision@k, recall and MRR. --explain returns a per-result score breakdown. - Tag context descriptions: tags carry an optional one-line description (kb tag-describe), returned as tag_contexts with search results. Adds a tags.description column migration. - Structured data ingestion: .json/.yaml/.toml files ingest as text via the new "data" doc type, pretty-printing minified JSON before chunking. Query expansion (proposal item 5) is deliberately left out pending bench results. Requires engine v3.3.0. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
52 lines
1.4 KiB
Python
52 lines
1.4 KiB
Python
"""Detect document type and language from file extension."""
|
|
|
|
from pathlib import Path
|
|
|
|
SUPPORTED_EXTENSIONS: dict[str, tuple[str, str | None]] = {
|
|
".pdf": ("pdf", None),
|
|
".docx": ("pdf", None),
|
|
".html": ("pdf", None),
|
|
".md": ("markdown", None),
|
|
".txt": ("note", None),
|
|
".py": ("code", "python"),
|
|
".sh": ("code", "bash"),
|
|
".go": ("code", "go"),
|
|
".json": ("data", "json"),
|
|
".yaml": ("data", "yaml"),
|
|
".yml": ("data", "yaml"),
|
|
".toml": ("data", "toml"),
|
|
}
|
|
|
|
|
|
def is_supported(path: Path) -> bool:
|
|
"""Check if the file extension is supported for ingestion."""
|
|
return path.suffix.lower() in SUPPORTED_EXTENSIONS
|
|
|
|
|
|
def detect_type(
|
|
path: Path,
|
|
force_type: str | None = None,
|
|
force_language: str | None = None,
|
|
) -> tuple[str, str | None]:
|
|
"""Return (doc_type, language) for the given file path.
|
|
|
|
Uses force_type / force_language when provided, otherwise falls back to
|
|
extension-based lookup. Raises ValueError for unsupported extensions.
|
|
"""
|
|
ext = path.suffix.lower()
|
|
|
|
if ext not in SUPPORTED_EXTENSIONS:
|
|
raise ValueError(
|
|
f"Unsupported file extension '{ext}'. "
|
|
f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
|
|
)
|
|
|
|
doc_type, language = SUPPORTED_EXTENSIONS[ext]
|
|
|
|
if force_type is not None:
|
|
doc_type = force_type
|
|
if force_language is not None:
|
|
language = force_language
|
|
|
|
return doc_type, language
|