feat: semantic search with Qdrant + Ollama embeddings
- Add SemanticSearchService with embed() + searchQdrant() methods - Add GET /api/search/semantic endpoint (?q=query&k=5) - Wire SemanticSearchService into router and cmd/server/main.go - Add SemanticSearch React page with results + similarity scores - Add 'Semantic Search' nav link in App.tsx - Add unit tests with mocked Ollama + Qdrant HTTP servers (4 tests, all passing) - Add GitHub issue templates (bug report, feature request) - Add pull request template
This commit is contained in:
Christopher Mayor
188
internal/service/semantic_search.go
Normal file
188
internal/service/semantic_search.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SemanticSearchResult struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MediaType string `json:"media_type"`
|
||||
Year *int `json:"year,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
}
|
||||
|
||||
type ollamaEmbedRequest struct {
|
||||
Model string `json:"model"`
|
||||
Prompt string `json:"prompt"`
|
||||
}
|
||||
|
||||
type ollamaEmbedResponse struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
}
|
||||
|
||||
type qdrantSearchRequest struct {
|
||||
Vector []float64 `json:"vector"`
|
||||
Limit int `json:"limit"`
|
||||
WithPayload bool `json:"with_payload"`
|
||||
}
|
||||
|
||||
type qdrantSearchResponse struct {
|
||||
Result []qdrantHit `json:"result"`
|
||||
}
|
||||
|
||||
type qdrantHit struct {
|
||||
ID string `json:"id"`
|
||||
Score float64 `json:"score"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
type qdrantPayload struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MediaType string `json:"media_type"`
|
||||
Year *int `json:"year,omitempty"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
}
|
||||
|
||||
type SemanticSearchService struct {
|
||||
ollamaURL string
|
||||
qdrantURL string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewSemanticSearchService(ollamaURL, qdrantURL string) *SemanticSearchService {
|
||||
return &SemanticSearchService{
|
||||
ollamaURL: ollamaURL,
|
||||
qdrantURL: qdrantURL,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SemanticSearchService) Search(ctx context.Context, query string, k int) ([]SemanticSearchResult, error) {
|
||||
embedding, err := s.embed(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate embedding: %w", err)
|
||||
}
|
||||
|
||||
results, err := s.searchQdrant(ctx, embedding, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search qdrant: %w", err)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *SemanticSearchService) embed(ctx context.Context, text string) ([]float64, error) {
|
||||
reqBody := ollamaEmbedRequest{
|
||||
Model: "nomic-embed-text",
|
||||
Prompt: text,
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal embed request: %w", err)
|
||||
}
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, s.ollamaURL+"/api/embeddings", bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create embed request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ollama request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
slog.Error("ollama embed returned non-200", "status", resp.StatusCode, "body", string(respBody))
|
||||
return nil, fmt.Errorf("ollama returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var embedResp ollamaEmbedResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&embedResp); err != nil {
|
||||
return nil, fmt.Errorf("decode ollama response: %w", err)
|
||||
}
|
||||
|
||||
if len(embedResp.Embedding) == 0 {
|
||||
return nil, fmt.Errorf("ollama returned empty embedding")
|
||||
}
|
||||
|
||||
return embedResp.Embedding, nil
|
||||
}
|
||||
|
||||
func (s *SemanticSearchService) searchQdrant(ctx context.Context, embedding []float64, k int) ([]SemanticSearchResult, error) {
|
||||
reqBody := qdrantSearchRequest{
|
||||
Vector: embedding,
|
||||
Limit: k,
|
||||
WithPayload: true,
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal qdrant request: %w", err)
|
||||
}
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
url := fmt.Sprintf("%s/collections/media/points/search", s.qdrantURL)
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, url, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create qdrant request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("qdrant request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
slog.Error("qdrant search returned non-200", "status", resp.StatusCode, "body", string(respBody))
|
||||
return nil, fmt.Errorf("qdrant returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var searchResp qdrantSearchResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&searchResp); err != nil {
|
||||
return nil, fmt.Errorf("decode qdrant response: %w", err)
|
||||
}
|
||||
|
||||
results := make([]SemanticSearchResult, 0, len(searchResp.Result))
|
||||
for _, hit := range searchResp.Result {
|
||||
var payload qdrantPayload
|
||||
if err := json.Unmarshal(hit.Payload, &payload); err != nil {
|
||||
slog.Error("failed to unmarshal qdrant payload", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, SemanticSearchResult{
|
||||
ID: payload.ID,
|
||||
Title: payload.Title,
|
||||
MediaType: payload.MediaType,
|
||||
Year: payload.Year,
|
||||
Score: hit.Score,
|
||||
Overview: payload.Overview,
|
||||
})
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
143
internal/service/semantic_search_test.go
Normal file
143
internal/service/semantic_search_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSemanticSearchService_Search(t *testing.T) {
|
||||
// Mock Ollama server
|
||||
ollamaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/embeddings" {
|
||||
t.Errorf("expected /api/embeddings, got %s", r.URL.Path)
|
||||
}
|
||||
var req ollamaEmbedRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatalf("failed to decode request: %v", err)
|
||||
}
|
||||
if req.Model != "nomic-embed-text" {
|
||||
t.Errorf("expected model nomic-embed-text, got %s", req.Model)
|
||||
}
|
||||
if req.Prompt == "" {
|
||||
t.Error("expected non-empty prompt")
|
||||
}
|
||||
// Return a 768-dim embedding (truncated for test)
|
||||
embedding := make([]float64, 768)
|
||||
embedding[0] = 0.1
|
||||
embedding[1] = -0.2
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(ollamaEmbedResponse{Embedding: embedding})
|
||||
}))
|
||||
defer ollamaServer.Close()
|
||||
|
||||
// Mock Qdrant server
|
||||
qdrantServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/collections/media/points/search" {
|
||||
t.Errorf("expected /collections/media/points/search, got %s", r.URL.Path)
|
||||
}
|
||||
var req qdrantSearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatalf("failed to decode request: %v", err)
|
||||
}
|
||||
if req.Limit != 5 {
|
||||
t.Errorf("expected limit 5, got %d", req.Limit)
|
||||
}
|
||||
if !req.WithPayload {
|
||||
t.Error("expected WithPayload=true")
|
||||
}
|
||||
payload, _ := json.Marshal(qdrantPayload{
|
||||
ID: 42,
|
||||
Title: "Blade Runner 2049",
|
||||
MediaType: "movie",
|
||||
Year: intPtr(2017),
|
||||
Overview: "A young blade runner's discovery of a long-buried secret leads him to track down former blade runner Rick Deckard.",
|
||||
})
|
||||
resp := qdrantSearchResponse{
|
||||
Result: []qdrantHit{
|
||||
{ID: "42", Score: 0.8712, Payload: payload},
|
||||
},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
}))
|
||||
defer qdrantServer.Close()
|
||||
|
||||
svc := NewSemanticSearchService(ollamaServer.URL, qdrantServer.URL)
|
||||
results, err := svc.Search(context.Background(), "sci-fi noir film", 5)
|
||||
if err != nil {
|
||||
t.Fatalf("Search failed: %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("expected 1 result, got %d", len(results))
|
||||
}
|
||||
if results[0].ID != 42 {
|
||||
t.Errorf("expected ID 42, got %d", results[0].ID)
|
||||
}
|
||||
if results[0].Title != "Blade Runner 2049" {
|
||||
t.Errorf("expected title 'Blade Runner 2049', got %s", results[0].Title)
|
||||
}
|
||||
if results[0].MediaType != "movie" {
|
||||
t.Errorf("expected media_type 'movie', got %s", results[0].MediaType)
|
||||
}
|
||||
if results[0].Score != 0.8712 {
|
||||
t.Errorf("expected score 0.8712, got %f", results[0].Score)
|
||||
}
|
||||
if results[0].Year == nil || *results[0].Year != 2017 {
|
||||
t.Error("expected year 2017")
|
||||
}
|
||||
}
|
||||
|
||||
// Note: empty query validation happens at the API handler level (search.go),
|
||||
// not in the service. The service layer trusts its callers.
|
||||
func TestSemanticSearchService_embed_emptyQuery(t *testing.T) {
|
||||
ollamaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Empty string is passed through to Ollama — Ollama may return an error
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}))
|
||||
defer ollamaServer.Close()
|
||||
|
||||
svc := NewSemanticSearchService(ollamaServer.URL, "http://localhost:6333")
|
||||
_, err := svc.embed(context.Background(), "")
|
||||
if err == nil {
|
||||
t.Error("expected error for empty query passed to Ollama")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticSearchService_qdrantError(t *testing.T) {
|
||||
ollamaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
embedding := make([]float64, 768)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(ollamaEmbedResponse{Embedding: embedding})
|
||||
}))
|
||||
defer ollamaServer.Close()
|
||||
|
||||
qdrantServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer qdrantServer.Close()
|
||||
|
||||
svc := NewSemanticSearchService(ollamaServer.URL, qdrantServer.URL)
|
||||
_, err := svc.Search(context.Background(), "test query", 5)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when Qdrant is unavailable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticSearchService_ollamaError(t *testing.T) {
|
||||
ollamaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer ollamaServer.Close()
|
||||
|
||||
svc := NewSemanticSearchService(ollamaServer.URL, "http://localhost:6333")
|
||||
_, err := svc.Search(context.Background(), "test query", 5)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when Ollama is unavailable")
|
||||
}
|
||||
}
|
||||
|
||||
func intPtr(i int) *int { return &i }
|
||||
Reference in New Issue
Block a user