steve/kb
1
0
Fork 0
Code Issues 1 Pull Requests Actions Projects Releases 20 Wiki Activity

Complete outstanding ingestion and document UX work

Browse Source
This commit is contained in:
Steve Cliff
2026-08-22 18:54:07 +01:00
parent 932e889ee8
commit 5b5a9ecbc3
24 changed files with 696 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/cmd/add.go
+50 -6
View File
@@ -8,6 +8,7 @@ import (
"path/filepath"
"sort"
"strings"
"time"
"github.com/kb-search/kb/internal/api"
"github.com/kb-search/kb/internal/output"
@@ -54,12 +55,16 @@ var addfileCmd = &cobra.Command{
func init() {
addfileCmd.Flags().String("tags", "", "tags (comma-separated)")
addfileCmd.Flags().BoolP("recursive", "r", false, "recursively add directory contents")
addfileCmd.Flags().Bool("wait", false, "wait for ingestion to finish")
addfileCmd.Flags().Duration("wait-timeout", 10*time.Minute, "maximum time to wait per ingestion job")
rootCmd.AddCommand(addfileCmd)
}
func runAddfile(cmd *cobra.Command, args []string) error {
tags, _ := cmd.Flags().GetString("tags")
recursive, _ := cmd.Flags().GetBool("recursive")
wait, _ := cmd.Flags().GetBool("wait")
timeout, _ := cmd.Flags().GetDuration("wait-timeout")
client := api.NewClient()
@@ -89,12 +94,25 @@ func runAddfile(cmd *cobra.Command, args []string) error {
}
if output.IsJSON() {
output.PrintJSON([]interface{}{result.Raw})
if !wait || result.Duplicate {
output.PrintJSON([]interface{}{result.Raw})
}
} else if result.Duplicate {
fmt.Println(result.duplicateMsg())
} else {
fmt.Printf("Queued: %s\n", filepath.Base(path))
}
if wait && !result.Duplicate {
job, err := waitForJob(client, int(result.JobID), timeout)
if err != nil {
return err
}
if output.IsJSON() {
output.PrintJSON([]interface{}{job})
} else {
fmt.Printf("Ingested: %s (doc ID: %d, chunks: %d)\n", filepath.Base(path), job.DocumentID, job.ChunkCount)
}
}
return nil
}
@@ -122,6 +140,7 @@ func runAddfile(cmd *cobra.Command, args []string) error {
}
var results []interface{}
var pending []*uploadResult
queued := 0
duplicates := 0
for _, f := range files {
@@ -130,7 +149,9 @@ func runAddfile(cmd *cobra.Command, args []string) error {
fmt.Fprintf(os.Stderr, "Error uploading %s: %v\n", f, err)
continue
}
results = append(results, result.Raw)
if !wait || result.Duplicate {
results = append(results, result.Raw)
}
if result.Duplicate {
duplicates++
if !output.IsJSON() {
@@ -138,11 +159,25 @@ func runAddfile(cmd *cobra.Command, args []string) error {
}
} else {
queued++
pending = append(pending, result)
if !output.IsJSON() {
fmt.Printf("Queued: %s\n", filepath.Base(f))
}
}
}
if wait {
for _, result := range pending {
job, err := waitForJob(client, int(result.JobID), timeout)
if err != nil {
return err
}
if output.IsJSON() {
results = append(results, job)
} else {
fmt.Printf("Ingested job %d (doc ID: %d, chunks: %d)\n", int(result.JobID), job.DocumentID, job.ChunkCount)
}
}
}
if output.IsJSON() {
output.PrintJSON(results)
@@ -203,10 +238,19 @@ func uploadFile(client *api.Client, path, tags string) (*uploadResult, error) {
return nil, err
}
var result interface{}
if err := api.DecodeJSON(resp, &result); err != nil {
var raw json.RawMessage
if err := api.DecodeJSON(resp, &raw); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
return &uploadResult{Raw: result}, nil
var queued struct {
JobID float64 `json:"job_id"`
}
if err := json.Unmarshal(raw, &queued); err != nil {
return nil, fmt.Errorf("failed to decode queued job: %w", err)
}
var result interface{}
if err := json.Unmarshal(raw, &result); err != nil {
return nil, fmt.Errorf("failed to decode queued response: %w", err)
}
return &uploadResult{Raw: result, JobID: queued.JobID}, nil
}
client/cmd/addnote.go
+27 -5
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
"os"
"time"
"github.com/kb-search/kb/internal/api"
"github.com/kb-search/kb/internal/output"
@@ -22,21 +23,25 @@ var addnoteCmd = &cobra.Command{
}
return nil
},
RunE: runAddnote,
RunE: runAddnote,
}
func init() {
addnoteCmd.Flags().String("tags", "", "tags (comma-separated)")
addnoteCmd.Flags().Bool("wait", false, "wait for ingestion to finish")
addnoteCmd.Flags().Duration("wait-timeout", 10*time.Minute, "maximum time to wait for ingestion")
rootCmd.AddCommand(addnoteCmd)
}
func runAddnote(cmd *cobra.Command, args []string) error {
tags, _ := cmd.Flags().GetString("tags")
wait, _ := cmd.Flags().GetBool("wait")
timeout, _ := cmd.Flags().GetDuration("wait-timeout")
client := api.NewClient()
return submitNote(client, args[0], tags)
return submitNote(client, args[0], tags, wait, timeout)
}
func submitNote(client *api.Client, note, tags string) error {
func submitNote(client *api.Client, note, tags string, wait bool, timeout time.Duration) error {
fields := map[string]string{
"note": note,
}
@@ -74,15 +79,32 @@ func submitNote(client *api.Client, note, tags string) error {
os.Exit(1)
}
var result interface{}
var result struct {
JobID int `json:"job_id"`
Status string `json:"status"`
Filename string `json:"filename"`
}
if err := api.DecodeJSON(resp, &result); err != nil {
return fmt.Errorf("failed to decode response: %w", err)
}
if output.IsJSON() {
output.PrintJSON(result)
if !wait {
output.PrintJSON(result)
}
} else {
fmt.Println("Queued: note")
}
if wait {
job, err := waitForJob(client, result.JobID, timeout)
if err != nil {
return err
}
if output.IsJSON() {
output.PrintJSON(job)
} else {
fmt.Printf("Ingested: note (doc ID: %d, chunks: %d)\n", job.DocumentID, job.ChunkCount)
}
}
return nil
}
client/cmd/examples.go
+5 -1
View File
@@ -14,21 +14,25 @@ var examplesCmd = &cobra.Command{
fmt.Print(`Add notes:
kb addnote "Remember to update DNS records"
kb addnote "Server room is building 3" --tags ops
kb addnote "Deploy checklist" --wait
Add files:
kb addfile report.pdf
kb addfile ~/docs/ --recursive --tags reference
kb addfile report.pdf --wait
Search:
kb search "how to restart nginx"
kb search "deploy" --tags ops --top 5
kb find "quarterly report" --type pdf
Update notes:
kb updatenote 42 "revised note content"
Manage documents:
kb list --type pdf
kb info 3
kb list --filename report.pdf
kb info 3 --no-chunks
kb tag 3 --add important,ops
kb remove 3 --yes
`)
client/cmd/find.go
+85
View File
@@ -0,0 +1,85 @@
package cmd
import (
"fmt"
"github.com/kb-search/kb/internal/api"
"github.com/kb-search/kb/internal/output"
"github.com/spf13/cobra"
)
var findCmd = &cobra.Command{
Use: "find <query>",
Short: "Find documents by their indexed content",
Args: cobra.ExactArgs(1),
RunE: runFind,
}
func init() {
findCmd.Flags().IntP("top", "n", 10, "number of documents to return")
findCmd.Flags().String("tags", "", "filter by tags (comma-separated)")
findCmd.Flags().String("type", "", "filter by document type")
rootCmd.AddCommand(findCmd)
}
func runFind(cmd *cobra.Command, args []string) error {
top, _ := cmd.Flags().GetInt("top")
tags, _ := cmd.Flags().GetString("tags")
docType, _ := cmd.Flags().GetString("type")
body := map[string]interface{}{"query": args[0], "top": top}
if tags != "" {
body["tags"] = splitTags(tags)
}
if docType != "" {
body["doc_type"] = docType
}
resp, err := api.NewClient().Post("/api/v1/documents/find", body)
if err != nil {
return err
}
if err := api.CheckError(resp); err != nil {
return err
}
if output.IsJSON() {
var raw interface{}
if err := api.DecodeJSON(resp, &raw); err != nil {
return fmt.Errorf("failed to decode response: %w", err)
}
output.PrintJSON(raw)
return nil
}
var docs []struct {
DocumentID int `json:"document_id"`
Title string `json:"title"`
Filename string `json:"original_filename"`
Type string `json:"doc_type"`
Score float64 `json:"score"`
HitCount int `json:"hit_count"`
TopChunk string `json:"top_chunk"`
}
if err := api.DecodeJSON(resp, &docs); err != nil {
return fmt.Errorf("failed to decode response: %w", err)
}
if len(docs) == 0 {
fmt.Println("No documents found.")
return nil
}
for i, doc := range docs {
preview := doc.TopChunk
if len(preview) > 200 {
preview = preview[:200] + "..."
}
fmt.Printf("\n%d. [%.4f] %s (doc:%d, hits:%d)\n", i+1, doc.Score, doc.Title, doc.DocumentID, doc.HitCount)
if doc.Filename != "" {
fmt.Printf(" Filename: %s\n", doc.Filename)
}
if doc.Type != "" {
fmt.Printf(" Type: %s\n", doc.Type)
}
fmt.Printf(" %s\n", preview)
}
fmt.Println()
return nil
}
client/cmd/info.go
+18 -11
View File
@@ -17,12 +17,18 @@ var infoCmd = &cobra.Command{
}
func init() {
infoCmd.Flags().Bool("no-chunks", false, "return document metadata without chunk details")
rootCmd.AddCommand(infoCmd)
}
func runInfo(cmd *cobra.Command, args []string) error {
client := api.NewClient()
resp, err := client.Get("/api/v1/documents/" + args[0])
noChunks, _ := cmd.Flags().GetBool("no-chunks")
path := "/api/v1/documents/" + args[0]
if noChunks {
path += "?include_chunks=false"
}
resp, err := client.Get(path)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
@@ -42,16 +48,17 @@ func runInfo(cmd *cobra.Command, args []string) error {
}
var doc struct {
ID int `json:"id"`
Title string `json:"title"`
Type string `json:"doc_type"`
Tags []string `json:"tags"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
Chunks []struct {
ID int `json:"id"`
ID int `json:"id"`
Title string `json:"title"`
Type string `json:"doc_type"`
Tags []string `json:"tags"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ChunkCount int `json:"chunk_count"`
Chunks []struct {
ID int `json:"id"`
Page interface{} `json:"page"`
Section string `json:"section"`
Section string `json:"section"`
} `json:"chunks"`
}
if err := api.DecodeJSON(resp, &doc); err != nil {
@@ -65,7 +72,7 @@ func runInfo(cmd *cobra.Command, args []string) error {
{"Tags", joinStrings(doc.Tags)},
{"Created", doc.CreatedAt},
{"Updated", doc.UpdatedAt},
{"Chunks", fmt.Sprintf("%d", len(doc.Chunks))},
{"Chunks", fmt.Sprintf("%d", doc.ChunkCount)},
}
output.PrintKeyValue(pairs)
client/cmd/list.go
+18 -6
View File
@@ -13,18 +13,23 @@ import (
var listCmd = &cobra.Command{
Use: "list",
Short: "List documents in the knowledge base",
Args: cobra.NoArgs,
RunE: runList,
}
func init() {
listCmd.Flags().String("type", "", "filter by document type")
listCmd.Flags().String("tags", "", "filter by tags (comma-separated)")
listCmd.Flags().String("title", "", "filter by title substring")
listCmd.Flags().String("filename", "", "filter by original filename substring")
rootCmd.AddCommand(listCmd)
}
func runList(cmd *cobra.Command, args []string) error {
docType, _ := cmd.Flags().GetString("type")
tags, _ := cmd.Flags().GetString("tags")
title, _ := cmd.Flags().GetString("title")
filename, _ := cmd.Flags().GetString("filename")
params := url.Values{}
if docType != "" {
@@ -33,6 +38,12 @@ func runList(cmd *cobra.Command, args []string) error {
if tags != "" {
params.Set("tags", tags)
}
if title != "" {
params.Set("title", title)
}
if filename != "" {
params.Set("filename", filename)
}
path := "/api/v1/documents"
if len(params) > 0 {
@@ -60,10 +71,11 @@ func runList(cmd *cobra.Command, args []string) error {
}
var docs []struct {
ID int `json:"id"`
Title string `json:"title"`
Type string `json:"doc_type"`
Tags []string `json:"tags"`
ID int `json:"id"`
Title string `json:"title"`
Filename string `json:"original_filename"`
Type string `json:"doc_type"`
Tags []string `json:"tags"`
}
if err := api.DecodeJSON(resp, &docs); err != nil {
return fmt.Errorf("failed to decode response: %w", err)
@@ -74,10 +86,10 @@ func runList(cmd *cobra.Command, args []string) error {
return nil
}
headers := []string{"ID", "TITLE", "TYPE", "TAGS"}
headers := []string{"ID", "TITLE", "FILENAME", "TYPE", "TAGS"}
var rows [][]string
for _, d := range docs {
rows = append(rows, []string{fmt.Sprintf("%d", d.ID), d.Title, d.Type, joinStrings(d.Tags)})
rows = append(rows, []string{fmt.Sprintf("%d", d.ID), d.Title, d.Filename, d.Type, joinStrings(d.Tags)})
}
output.PrintTable(headers, rows)
return nil
client/cmd/root_test.go
+13
View File
@@ -67,3 +67,16 @@ func TestAddnoteCmd_TooManyArgs_ReturnsError(t *testing.T) {
t.Errorf("expected 'accepts 1 arg' error, got: %s", errMsg)
}
}
func TestListCmd_PositionalArgReturnsError(t *testing.T) {
rootCmd.SetArgs([]string{"list", "ignored-title"})
err := rootCmd.Execute()
if err == nil {
t.Fatal("expected error for positional list argument, got nil")
}
if !strings.Contains(err.Error(), "unknown command") &&
!strings.Contains(err.Error(), "accepts 0 arg") {
t.Errorf("expected positional argument error, got: %s", err)
}
}
client/cmd/wait.go
+43
View File
@@ -0,0 +1,43 @@
package cmd
import (
"fmt"
"time"
"github.com/kb-search/kb/internal/api"
)
type jobStatus struct {
ID int `json:"id"`
Status string `json:"status"`
DocumentID int `json:"document_id"`
ChunkCount int `json:"chunk_count"`
Error string `json:"error"`
}
func waitForJob(client *api.Client, jobID int, timeout time.Duration) (*jobStatus, error) {
deadline := time.Now().Add(timeout)
for {
resp, err := client.Get(fmt.Sprintf("/api/v1/jobs/%d", jobID))
if err != nil {
return nil, err
}
if err := api.CheckError(resp); err != nil {
return nil, err
}
var job jobStatus
if err := api.DecodeJSON(resp, &job); err != nil {
return nil, fmt.Errorf("failed to decode job status: %w", err)
}
switch job.Status {
case "done", "skipped":
return &job, nil
case "failed":
return nil, fmt.Errorf("ingestion job %d failed: %s", jobID, job.Error)
}
if time.Now().After(deadline) {
return nil, fmt.Errorf("timed out waiting for ingestion job %d", jobID)
}
time.Sleep(time.Second)
}
}