Enhance search functionality with multi-engine support and configuration updates
- Added support for Typesense as a search engine alongside Meilisearch and PostgreSQL. - Updated configuration structure to include Typesense parameters in `Config` and `.env.example`. - Enhanced search handler and service to accommodate external search clients and filters. - Implemented new tests for external search clients and search service functionalities. - Updated project checklist to reflect completion of multi-index search features and contextual snippets.
This commit is contained in:
R3D347HR4Y
parent
a2e17c5b6c
commit
0435e27ce6
11
.env.example
11
.env.example
@ -25,6 +25,7 @@ JITSI_APP_SECRET=changeme-jwt-secret
|
||||
JITSI_INTERNAL_AUTH_PASSWORD=changeme
|
||||
KEYDB_PASSWORD=
|
||||
MEILISEARCH_API_KEY=changeme
|
||||
TYPESENSE_API_KEY=changeme
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# General
|
||||
@ -193,8 +194,16 @@ MAIL_WEBHOOK_SHARED_SECRET_ROTATED_AT=2026-01-01T00:00:00Z
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Recherche
|
||||
# SEARCH_ENGINE: postgres (defaut) | meilisearch | typesense
|
||||
# -----------------------------------------------------------------------------
|
||||
SEARCH_ENGINE=postgres
|
||||
# SEARCH_ENGINE=meilisearch
|
||||
|
||||
# --- Meilisearch (SEARCH_ENGINE=meilisearch) ---
|
||||
# MEILISEARCH_URL=http://meilisearch:7700
|
||||
# MEILISEARCH_API_KEY={{MEILISEARCH_API_KEY}}
|
||||
# MEILISEARCH_INDEX=ulti
|
||||
|
||||
# --- Typesense (SEARCH_ENGINE=typesense) ---
|
||||
# TYPESENSE_URL=http://typesense:8108
|
||||
# TYPESENSE_API_KEY={{TYPESENSE_API_KEY}}
|
||||
# TYPESENSE_COLLECTION=ulti
|
||||
|
||||
@ -195,7 +195,16 @@ func main() {
|
||||
|
||||
r.Mount("/api/v1/mail", mailapi.NewHandler(pool, auditLogger, credentialManager, attachmentStorage, cfg.MailAttachmentsBucket, sendRateLimiter).Routes())
|
||||
r.Mount("/api/v1/admin", admin.NewHandler(pool, auditLogger).Routes())
|
||||
r.Get("/api/v1/search", search.NewHandler(pool).Search)
|
||||
r.Get("/api/v1/search", search.NewHandler(pool, search.Options{
|
||||
Nextcloud: ncClient,
|
||||
Engine: cfg.SearchEngine,
|
||||
MeilisearchURL: cfg.MeilisearchURL,
|
||||
MeilisearchKey: cfg.MeilisearchKey,
|
||||
MeilisearchIndex: cfg.MeilisearchIndex,
|
||||
TypesenseURL: cfg.TypesenseURL,
|
||||
TypesenseKey: cfg.TypesenseKey,
|
||||
TypesenseCollection: cfg.TypesenseCollection,
|
||||
}).Search)
|
||||
|
||||
if ncClient != nil {
|
||||
r.Mount("/api/v1/drive", drive.NewHandler(ncClient).Routes())
|
||||
|
||||
@ -72,9 +72,13 @@ type Config struct {
|
||||
WebhookSharedSecretRotatedAt time.Time
|
||||
|
||||
// Search
|
||||
SearchEngine string
|
||||
MeilisearchURL string
|
||||
MeilisearchKey string
|
||||
SearchEngine string
|
||||
MeilisearchURL string
|
||||
MeilisearchKey string
|
||||
MeilisearchIndex string
|
||||
TypesenseURL string
|
||||
TypesenseKey string
|
||||
TypesenseCollection string
|
||||
|
||||
// Observability
|
||||
HealthNextcloudURL string
|
||||
@ -142,9 +146,13 @@ func Load() (*Config, error) {
|
||||
SMTPCredentialKeyRotatedAt: envTime("MAIL_CREDENTIAL_KEY_ROTATED_AT"),
|
||||
WebhookSharedSecretRotatedAt: envTime("MAIL_WEBHOOK_SHARED_SECRET_ROTATED_AT"),
|
||||
|
||||
SearchEngine: envOrDefault("SEARCH_ENGINE", "postgres"),
|
||||
MeilisearchURL: os.Getenv("MEILISEARCH_URL"),
|
||||
MeilisearchKey: secrets.Env("MEILISEARCH_API_KEY"),
|
||||
SearchEngine: envOrDefault("SEARCH_ENGINE", "postgres"),
|
||||
MeilisearchURL: os.Getenv("MEILISEARCH_URL"),
|
||||
MeilisearchKey: secrets.Env("MEILISEARCH_API_KEY"),
|
||||
MeilisearchIndex: envOrDefault("MEILISEARCH_INDEX", "ulti"),
|
||||
TypesenseURL: os.Getenv("TYPESENSE_URL"),
|
||||
TypesenseKey: secrets.Env("TYPESENSE_API_KEY"),
|
||||
TypesenseCollection: envOrDefault("TYPESENSE_COLLECTION", "ulti"),
|
||||
|
||||
HealthNextcloudURL: envOrDefault("HEALTH_NEXTCLOUD_URL", joinURL(envOrDefault("NEXTCLOUD_URL", "http://nextcloud:80"), "/status.php")),
|
||||
HealthImmichURL: envOrDefault("HEALTH_IMMICH_URL", joinURL(envOrDefault("IMMICH_API_URL", "http://immich-server:2283/api"), "/server-info/ping")),
|
||||
|
||||
333
internal/search/external_clients.go
Normal file
333
internal/search/external_clients.go
Normal file
@ -0,0 +1,333 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ultisuite/ulti-backend/internal/api/query"
|
||||
)
|
||||
|
||||
const defaultSearchIndex = "ulti"
|
||||
|
||||
type externalSearchClient interface {
|
||||
Search(ctx context.Context, params ExternalSearchParams) ([]Result, error)
|
||||
}
|
||||
|
||||
type ExternalSearchParams struct {
|
||||
ExternalID string
|
||||
Query string
|
||||
Types []string
|
||||
Params query.ListParams
|
||||
Filters SearchFilters
|
||||
}
|
||||
|
||||
func newExternalSearchClient(engine string, opts ServiceOptions) externalSearchClient {
|
||||
switch engine {
|
||||
case "meilisearch":
|
||||
if strings.TrimSpace(opts.MeilisearchURL) == "" {
|
||||
return nil
|
||||
}
|
||||
index := strings.TrimSpace(opts.MeilisearchIndex)
|
||||
if index == "" {
|
||||
index = defaultSearchIndex
|
||||
}
|
||||
return &meilisearchClient{
|
||||
baseURL: strings.TrimRight(strings.TrimSpace(opts.MeilisearchURL), "/"),
|
||||
apiKey: strings.TrimSpace(opts.MeilisearchKey),
|
||||
index: index,
|
||||
http: &http.Client{Timeout: 5 * time.Second},
|
||||
}
|
||||
case "typesense":
|
||||
if strings.TrimSpace(opts.TypesenseURL) == "" {
|
||||
return nil
|
||||
}
|
||||
collection := strings.TrimSpace(opts.TypesenseCollection)
|
||||
if collection == "" {
|
||||
collection = defaultSearchIndex
|
||||
}
|
||||
return &typesenseClient{
|
||||
baseURL: strings.TrimRight(strings.TrimSpace(opts.TypesenseURL), "/"),
|
||||
apiKey: strings.TrimSpace(opts.TypesenseKey),
|
||||
collection: collection,
|
||||
http: &http.Client{Timeout: 5 * time.Second},
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type meilisearchClient struct {
|
||||
baseURL string
|
||||
apiKey string
|
||||
index string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func (c *meilisearchClient) Search(ctx context.Context, params ExternalSearchParams) ([]Result, error) {
|
||||
payload := map[string]any{
|
||||
"q": params.Query,
|
||||
"limit": params.Params.Limit(),
|
||||
"offset": params.Params.Offset(),
|
||||
}
|
||||
if filters := meiliFilters(params); len(filters) > 0 {
|
||||
payload["filter"] = filters
|
||||
}
|
||||
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
u := c.baseURL + "/indexes/" + url.PathEscape(c.index) + "/search"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if c.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return nil, fmt.Errorf("meilisearch status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
Hits []map[string]any `json:"hits"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mapExternalHits(parsed.Hits, params.Query), nil
|
||||
}
|
||||
|
||||
func meiliFilters(params ExternalSearchParams) []string {
|
||||
filters := make([]string, 0, 5)
|
||||
if params.ExternalID != "" {
|
||||
filters = append(filters, fmt.Sprintf("user_external_id = %q", params.ExternalID))
|
||||
}
|
||||
if len(params.Types) > 0 {
|
||||
escaped := make([]string, 0, len(params.Types))
|
||||
for _, t := range params.Types {
|
||||
escaped = append(escaped, fmt.Sprintf("%q", t))
|
||||
}
|
||||
filters = append(filters, "type IN ["+strings.Join(escaped, ", ")+"]")
|
||||
}
|
||||
if params.Filters.AccountID != "" {
|
||||
filters = append(filters, fmt.Sprintf("account_id = %q", params.Filters.AccountID))
|
||||
}
|
||||
if params.Params.From != nil {
|
||||
filters = append(filters, fmt.Sprintf("date >= %q", params.Params.From.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
if params.Params.To != nil {
|
||||
filters = append(filters, fmt.Sprintf("date <= %q", params.Params.To.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
return filters
|
||||
}
|
||||
|
||||
type typesenseClient struct {
|
||||
baseURL string
|
||||
apiKey string
|
||||
collection string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func (c *typesenseClient) Search(ctx context.Context, params ExternalSearchParams) ([]Result, error) {
|
||||
u, err := url.Parse(c.baseURL + "/collections/" + url.PathEscape(c.collection) + "/documents/search")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("q", params.Query)
|
||||
q.Set("query_by", "title,snippet,content,name,summary,path")
|
||||
q.Set("per_page", strconv.Itoa(params.Params.Limit()))
|
||||
q.Set("page", strconv.Itoa(params.Params.Page))
|
||||
q.Set("highlight_affix_num_tokens", "6")
|
||||
q.Set("include_fields", "id,type,title,snippet,date,account_id,path,highlights,_text_match")
|
||||
if filter := typesenseFilter(params); filter != "" {
|
||||
q.Set("filter_by", filter)
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.apiKey != "" {
|
||||
req.Header.Set("X-TYPESENSE-API-KEY", c.apiKey)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return nil, fmt.Errorf("typesense status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
Hits []struct {
|
||||
Document map[string]any `json:"document"`
|
||||
TextMatch any `json:"text_match"`
|
||||
} `json:"hits"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hits := make([]map[string]any, 0, len(parsed.Hits))
|
||||
for _, h := range parsed.Hits {
|
||||
doc := h.Document
|
||||
if doc == nil {
|
||||
doc = map[string]any{}
|
||||
}
|
||||
doc["_text_match"] = h.TextMatch
|
||||
hits = append(hits, doc)
|
||||
}
|
||||
return mapExternalHits(hits, params.Query), nil
|
||||
}
|
||||
|
||||
func typesenseFilter(params ExternalSearchParams) string {
|
||||
parts := make([]string, 0, 5)
|
||||
if params.ExternalID != "" {
|
||||
parts = append(parts, fmt.Sprintf("user_external_id:=%s", params.ExternalID))
|
||||
}
|
||||
if len(params.Types) > 0 {
|
||||
parts = append(parts, "type:=["+strings.Join(params.Types, ",")+"]")
|
||||
}
|
||||
if params.Filters.AccountID != "" {
|
||||
parts = append(parts, fmt.Sprintf("account_id:=%s", params.Filters.AccountID))
|
||||
}
|
||||
if params.Params.From != nil {
|
||||
parts = append(parts, fmt.Sprintf("date:>=%s", params.Params.From.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
if params.Params.To != nil {
|
||||
parts = append(parts, fmt.Sprintf("date:<=%s", params.Params.To.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
return strings.Join(parts, " && ")
|
||||
}
|
||||
|
||||
func mapExternalHits(hits []map[string]any, queryText string) []Result {
|
||||
results := make([]Result, 0, len(hits))
|
||||
for _, hit := range hits {
|
||||
r := Result{
|
||||
Type: asString(firstNonNil(hit["type"], hit["kind"])),
|
||||
ID: asString(firstNonNil(hit["id"], hit["_id"], hit["uid"], hit["path"])),
|
||||
Title: asString(firstNonNil(hit["title"], hit["name"], hit["subject"], hit["summary"])),
|
||||
Snippet: asString(firstNonNil(hit["snippet"], hit["content"], hit["description"], hit["path"])),
|
||||
AccountID: asString(firstNonNil(hit["account_id"], hit["account"], hit["book_id"], hit["calendar_id"])),
|
||||
Score: asFloat(firstNonNil(hit["score"], hit["_rankingScore"], hit["_text_match"])),
|
||||
}
|
||||
if r.Type == "" {
|
||||
r.Type = "mail"
|
||||
}
|
||||
if r.ID == "" {
|
||||
r.ID = r.Type + ":" + r.Title
|
||||
}
|
||||
if dt := asString(firstNonNil(hit["date"], hit["updated_at"], hit["start"])); dt != "" {
|
||||
if parsed, ok := parseDateString(dt); ok {
|
||||
r.Date = parsed.UTC()
|
||||
}
|
||||
}
|
||||
if r.Snippet == "" {
|
||||
r.Snippet = r.Title
|
||||
}
|
||||
r.Title = highlightTerms(r.Title, queryText)
|
||||
r.Snippet = contextualSnippet(r.Snippet, queryText, 220)
|
||||
if r.Score == 0 {
|
||||
r.Score = textMatchScore(joinNonEmpty(" ", r.Title, r.Snippet), queryText, 1.0)
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func finalizeExternalResults(results []Result, queryText string, params query.ListParams) {
|
||||
for i := range results {
|
||||
if strings.TrimSpace(results[i].Title) == "" {
|
||||
results[i].Title = results[i].ID
|
||||
}
|
||||
if strings.TrimSpace(results[i].Snippet) == "" {
|
||||
results[i].Snippet = results[i].Title
|
||||
}
|
||||
results[i].Title = highlightTerms(results[i].Title, queryText)
|
||||
results[i].Snippet = contextualSnippet(results[i].Snippet, queryText, 220)
|
||||
if results[i].Score == 0 {
|
||||
results[i].Score = textMatchScore(joinNonEmpty(" ", results[i].Title, results[i].Snippet), queryText, 1.0)
|
||||
}
|
||||
|
||||
if ts, ok := dateFromAny(results[i].Date); ok {
|
||||
if params.From != nil && ts.Before(*params.From) {
|
||||
results[i].Score = -1
|
||||
continue
|
||||
}
|
||||
if params.To != nil && ts.After(*params.To) {
|
||||
results[i].Score = -1
|
||||
continue
|
||||
}
|
||||
results[i].Date = ts.UTC()
|
||||
results[i].Score += recencyBoost(ts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonNil(values ...any) any {
|
||||
for _, v := range values {
|
||||
if v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func asString(value any) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case fmt.Stringer:
|
||||
return strings.TrimSpace(v.String())
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprintf("%v", v))
|
||||
}
|
||||
}
|
||||
|
||||
func asFloat(value any) float64 {
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
return v
|
||||
case float32:
|
||||
return float64(v)
|
||||
case int:
|
||||
return float64(v)
|
||||
case int64:
|
||||
return float64(v)
|
||||
case json.Number:
|
||||
if f, err := v.Float64(); err == nil {
|
||||
return f
|
||||
}
|
||||
case string:
|
||||
if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
170
internal/search/external_clients_test.go
Normal file
170
internal/search/external_clients_test.go
Normal file
@ -0,0 +1,170 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ultisuite/ulti-backend/internal/api/query"
|
||||
)
|
||||
|
||||
func TestNewExternalSearchClient(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
engine string
|
||||
opts ServiceOptions
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "meilisearch with url",
|
||||
engine: "meilisearch",
|
||||
opts: ServiceOptions{MeilisearchURL: "http://localhost:7700"},
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "meilisearch empty url",
|
||||
engine: "meilisearch",
|
||||
opts: ServiceOptions{MeilisearchURL: " "},
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "typesense with url",
|
||||
engine: "typesense",
|
||||
opts: ServiceOptions{TypesenseURL: "http://localhost:8108"},
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "typesense empty url",
|
||||
engine: "typesense",
|
||||
opts: ServiceOptions{TypesenseURL: ""},
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "postgres engine",
|
||||
engine: "postgres",
|
||||
opts: ServiceOptions{MeilisearchURL: "http://localhost:7700"},
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "unknown engine",
|
||||
engine: "elasticsearch",
|
||||
opts: ServiceOptions{TypesenseURL: "http://localhost:8108"},
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := newExternalSearchClient(tt.engine, tt.opts)
|
||||
if tt.wantOK && got == nil {
|
||||
t.Fatal("expected non-nil client")
|
||||
}
|
||||
if !tt.wantOK && got != nil {
|
||||
t.Fatal("expected nil client")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapExternalHits(t *testing.T) {
|
||||
hits := []map[string]any{
|
||||
{
|
||||
"id": "mail-42",
|
||||
"type": "mail",
|
||||
"title": "Invoice from Acme",
|
||||
"snippet": "Your Acme invoice is ready",
|
||||
},
|
||||
}
|
||||
|
||||
results := mapExternalHits(hits, "acme")
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("len(results) = %d, want 1", len(results))
|
||||
}
|
||||
|
||||
r := results[0]
|
||||
if r.Type != "mail" {
|
||||
t.Fatalf("Type = %q, want mail", r.Type)
|
||||
}
|
||||
if r.ID != "mail-42" {
|
||||
t.Fatalf("ID = %q, want mail-42", r.ID)
|
||||
}
|
||||
if r.Title == "" {
|
||||
t.Fatal("Title is empty")
|
||||
}
|
||||
if r.Snippet == "" {
|
||||
t.Fatal("Snippet is empty")
|
||||
}
|
||||
if r.Score <= 0 {
|
||||
t.Fatalf("Score = %v, want positive fallback score", r.Score)
|
||||
}
|
||||
if !strings.Contains(r.Title, "<mark>") {
|
||||
t.Fatalf("Title missing highlight: %q", r.Title)
|
||||
}
|
||||
if !strings.Contains(r.Snippet, "<mark>") {
|
||||
t.Fatalf("Snippet missing highlight: %q", r.Snippet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapExternalHitsMinimalDefaults(t *testing.T) {
|
||||
hits := []map[string]any{
|
||||
{"title": "Budget report"},
|
||||
}
|
||||
|
||||
results := mapExternalHits(hits, "budget")
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("len(results) = %d, want 1", len(results))
|
||||
}
|
||||
|
||||
r := results[0]
|
||||
if r.Type != "mail" {
|
||||
t.Fatalf("Type = %q, want default mail", r.Type)
|
||||
}
|
||||
if r.ID != "mail:Budget report" {
|
||||
t.Fatalf("ID = %q, want mail:Budget report", r.ID)
|
||||
}
|
||||
if r.Snippet == "" {
|
||||
t.Fatal("Snippet is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalizeExternalResultsDateFiltering(t *testing.T) {
|
||||
mid := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC)
|
||||
from := time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
to := time.Date(2024, 6, 30, 23, 59, 59, 0, time.UTC)
|
||||
|
||||
results := []Result{
|
||||
{ID: "before", Title: "Old mail", Snippet: "Old mail", Date: time.Date(2024, 5, 15, 12, 0, 0, 0, time.UTC), Score: 5},
|
||||
{ID: "in-range", Title: "Current mail", Snippet: "Current mail", Date: mid, Score: 5},
|
||||
{ID: "after", Title: "Future mail", Snippet: "Future mail", Date: time.Date(2024, 7, 15, 12, 0, 0, 0, time.UTC), Score: 5},
|
||||
}
|
||||
|
||||
params := query.ListParams{From: &from, To: &to}
|
||||
finalizeExternalResults(results, "mail", params)
|
||||
|
||||
if results[0].Score >= 0 {
|
||||
t.Fatalf("before-range Score = %v, want negative", results[0].Score)
|
||||
}
|
||||
if results[1].Score <= 0 {
|
||||
t.Fatalf("in-range Score = %v, want positive with recency boost", results[1].Score)
|
||||
}
|
||||
if results[2].Score >= 0 {
|
||||
t.Fatalf("after-range Score = %v, want negative", results[2].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalizeExternalResultsFallbackScore(t *testing.T) {
|
||||
results := []Result{
|
||||
{ID: "1", Title: "Invoice details", Snippet: "Invoice details", Score: 0},
|
||||
}
|
||||
|
||||
finalizeExternalResults(results, "invoice", query.ListParams{})
|
||||
if results[0].Score <= 0 {
|
||||
t.Fatalf("Score = %v, want positive fallback score", results[0].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsStringNil(t *testing.T) {
|
||||
if got := asString(nil); got != "" {
|
||||
t.Fatalf("asString(nil) = %q, want empty string", got)
|
||||
}
|
||||
}
|
||||
@ -3,6 +3,7 @@ package search
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
@ -10,6 +11,7 @@ import (
|
||||
"github.com/ultisuite/ulti-backend/internal/api/apivalidate"
|
||||
"github.com/ultisuite/ulti-backend/internal/api/middleware"
|
||||
"github.com/ultisuite/ulti-backend/internal/api/query"
|
||||
"github.com/ultisuite/ulti-backend/internal/nextcloud"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
@ -17,9 +19,29 @@ type Handler struct {
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewHandler(db *pgxpool.Pool) *Handler {
|
||||
type Options struct {
|
||||
Nextcloud *nextcloud.Client
|
||||
Engine string
|
||||
MeilisearchURL string
|
||||
MeilisearchKey string
|
||||
MeilisearchIndex string
|
||||
TypesenseURL string
|
||||
TypesenseKey string
|
||||
TypesenseCollection string
|
||||
}
|
||||
|
||||
func NewHandler(db *pgxpool.Pool, opts Options) *Handler {
|
||||
return &Handler{
|
||||
svc: NewService(db),
|
||||
svc: NewService(db, ServiceOptions{
|
||||
Nextcloud: opts.Nextcloud,
|
||||
Engine: opts.Engine,
|
||||
MeilisearchURL: opts.MeilisearchURL,
|
||||
MeilisearchKey: opts.MeilisearchKey,
|
||||
MeilisearchIndex: opts.MeilisearchIndex,
|
||||
TypesenseURL: opts.TypesenseURL,
|
||||
TypesenseKey: opts.TypesenseKey,
|
||||
TypesenseCollection: opts.TypesenseCollection,
|
||||
}),
|
||||
logger: slog.Default().With("component", "search"),
|
||||
}
|
||||
}
|
||||
@ -44,8 +66,11 @@ func (h *Handler) Search(w http.ResponseWriter, r *http.Request) {
|
||||
apivalidate.WriteQueryError(w, r, err)
|
||||
return
|
||||
}
|
||||
filters := SearchFilters{
|
||||
AccountID: strings.TrimSpace(r.URL.Query().Get("account_id")),
|
||||
}
|
||||
|
||||
result, err := h.svc.Search(r.Context(), claims.Sub, q, types, params)
|
||||
result, err := h.svc.Search(r.Context(), claims.Sub, q, types, params, filters)
|
||||
if err != nil {
|
||||
h.logger.Error("search", "error", err)
|
||||
apivalidate.WriteInternal(w, r)
|
||||
|
||||
@ -2,81 +2,236 @@ package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/ultisuite/ulti-backend/internal/api/query"
|
||||
"github.com/ultisuite/ulti-backend/internal/nextcloud"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
db *pgxpool.Pool
|
||||
db *pgxpool.Pool
|
||||
nc *nextcloud.Client
|
||||
engine string
|
||||
external externalSearchClient
|
||||
}
|
||||
|
||||
func NewService(db *pgxpool.Pool) *Service {
|
||||
return &Service{db: db}
|
||||
type ServiceOptions struct {
|
||||
Nextcloud *nextcloud.Client
|
||||
Engine string
|
||||
MeilisearchURL string
|
||||
MeilisearchKey string
|
||||
MeilisearchIndex string
|
||||
TypesenseURL string
|
||||
TypesenseKey string
|
||||
TypesenseCollection string
|
||||
}
|
||||
|
||||
func NewService(db *pgxpool.Pool, opts ServiceOptions) *Service {
|
||||
engine := normalizeEngine(opts.Engine)
|
||||
return &Service{
|
||||
db: db,
|
||||
nc: opts.Nextcloud,
|
||||
engine: engine,
|
||||
external: newExternalSearchClient(engine, opts),
|
||||
}
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Date any `json:"date,omitempty"`
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Date any `json:"date,omitempty"`
|
||||
AccountID string `json:"account_id,omitempty"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
}
|
||||
|
||||
type SearchResponse struct {
|
||||
Query string `json:"query"`
|
||||
Engine string `json:"engine,omitempty"`
|
||||
Results []Result `json:"results"`
|
||||
Count int `json:"count"`
|
||||
Pagination query.PaginationMeta `json:"pagination,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) Search(ctx context.Context, externalID, q, typesRaw string, params query.ListParams) (SearchResponse, error) {
|
||||
typeList := parseTypes(typesRaw)
|
||||
tsQuery := toTSQuery(q)
|
||||
type SearchFilters struct {
|
||||
AccountID string
|
||||
}
|
||||
|
||||
allResults := make([]Result, 0)
|
||||
for _, t := range typeList {
|
||||
switch t {
|
||||
case "mail":
|
||||
mailResults, err := s.searchMail(ctx, externalID, tsQuery, params)
|
||||
if err != nil {
|
||||
return SearchResponse{}, err
|
||||
}
|
||||
allResults = append(allResults, mailResults...)
|
||||
case "contacts":
|
||||
contactResults, err := s.searchContacts(ctx, q, params)
|
||||
if err != nil {
|
||||
return SearchResponse{}, err
|
||||
}
|
||||
allResults = append(allResults, contactResults...)
|
||||
case "events":
|
||||
// Events are in Nextcloud CalDAV, not in PG - skip for now.
|
||||
func (s *Service) Search(ctx context.Context, externalID, q, typesRaw string, params query.ListParams, filters SearchFilters) (SearchResponse, error) {
|
||||
typeList := parseTypes(typesRaw)
|
||||
usedEngine := s.engine
|
||||
if s.external == nil && s.engine != "postgres" {
|
||||
usedEngine = "postgres"
|
||||
}
|
||||
|
||||
var (
|
||||
allResults []Result
|
||||
err error
|
||||
)
|
||||
if s.external != nil {
|
||||
allResults, err = s.external.Search(ctx, ExternalSearchParams{
|
||||
ExternalID: externalID,
|
||||
Query: q,
|
||||
Types: typeList,
|
||||
Params: params,
|
||||
Filters: filters,
|
||||
})
|
||||
if err != nil {
|
||||
usedEngine = "postgres"
|
||||
}
|
||||
}
|
||||
if s.external == nil || err != nil {
|
||||
allResults, err = s.searchFederated(ctx, externalID, q, typeList, params, filters)
|
||||
if err != nil {
|
||||
return SearchResponse{}, err
|
||||
}
|
||||
}
|
||||
|
||||
finalizeExternalResults(allResults, q, params)
|
||||
allResults = discardFilteredResults(allResults)
|
||||
sortResultsByScore(allResults)
|
||||
|
||||
total := int64(len(allResults))
|
||||
page, _ := paginateResults(allResults, params.Offset(), params.Limit())
|
||||
|
||||
return SearchResponse{
|
||||
Query: q,
|
||||
Engine: usedEngine,
|
||||
Results: page,
|
||||
Count: len(page),
|
||||
Pagination: params.Meta(&total),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) searchFederated(ctx context.Context, externalID, q string, typeList []string, params query.ListParams, filters SearchFilters) ([]Result, error) {
|
||||
allResults := make([]Result, 0, params.Offset()+params.Limit())
|
||||
for _, t := range typeList {
|
||||
switch t {
|
||||
case "mail":
|
||||
mailResults, err := s.searchMail(ctx, externalID, q, params, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allResults = append(allResults, mailResults...)
|
||||
case "contacts":
|
||||
contactResults, err := s.searchContacts(ctx, externalID, q, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allResults = append(allResults, contactResults...)
|
||||
case "files":
|
||||
fileResults, err := s.searchFiles(ctx, externalID, q, params, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allResults = append(allResults, fileResults...)
|
||||
case "events":
|
||||
eventResults, err := s.searchEvents(ctx, externalID, q, params, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allResults = append(allResults, eventResults...)
|
||||
}
|
||||
}
|
||||
return allResults, nil
|
||||
}
|
||||
|
||||
func sortResultsByScore(results []Result) {
|
||||
sort.SliceStable(results, func(i, j int) bool {
|
||||
if results[i].Score == results[j].Score {
|
||||
ti, okI := dateFromAny(results[i].Date)
|
||||
tj, okJ := dateFromAny(results[j].Date)
|
||||
if okI && okJ {
|
||||
return ti.After(tj)
|
||||
}
|
||||
return okI
|
||||
}
|
||||
return results[i].Score > results[j].Score
|
||||
})
|
||||
}
|
||||
|
||||
func dateFromAny(raw any) (time.Time, bool) {
|
||||
switch t := raw.(type) {
|
||||
case time.Time:
|
||||
return t, true
|
||||
case *time.Time:
|
||||
if t == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return *t, true
|
||||
case string:
|
||||
return parseDateString(t)
|
||||
default:
|
||||
return time.Time{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func parseDateString(raw string) (time.Time, bool) {
|
||||
layouts := []string{
|
||||
time.RFC3339,
|
||||
"20060102T150405Z",
|
||||
"20060102T150405",
|
||||
"20060102",
|
||||
}
|
||||
for _, layout := range layouts {
|
||||
if ts, err := time.Parse(layout, raw); err == nil {
|
||||
return ts, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
func normalizeEngine(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "meilisearch":
|
||||
return "meilisearch"
|
||||
case "typesense":
|
||||
return "typesense"
|
||||
default:
|
||||
return "postgres"
|
||||
}
|
||||
}
|
||||
|
||||
func defaultCalendarRange(params query.ListParams) (time.Time, time.Time) {
|
||||
from := time.Now().UTC().AddDate(0, -6, 0)
|
||||
to := time.Now().UTC().AddDate(1, 0, 0)
|
||||
if params.From != nil {
|
||||
from = params.From.UTC()
|
||||
}
|
||||
if params.To != nil {
|
||||
to = params.To.UTC()
|
||||
}
|
||||
if from.After(to) {
|
||||
from, to = to, from
|
||||
}
|
||||
return from, to
|
||||
}
|
||||
|
||||
func parseTypes(typesRaw string) []string {
|
||||
if strings.TrimSpace(typesRaw) == "" {
|
||||
return []string{"mail", "contacts", "events"}
|
||||
return []string{"mail", "contacts", "files", "events"}
|
||||
}
|
||||
parts := strings.Split(typesRaw, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
seen := make(map[string]struct{}, len(parts))
|
||||
for _, p := range parts {
|
||||
if t := strings.TrimSpace(p); t != "" {
|
||||
out = append(out, t)
|
||||
t := strings.TrimSpace(p)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[t]; ok {
|
||||
continue
|
||||
}
|
||||
seen[t] = struct{}{}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@ -93,22 +248,57 @@ func paginateResults(results []Result, offset, limit int) ([]Result, int64) {
|
||||
return results[offset:end], total
|
||||
}
|
||||
|
||||
func (s *Service) searchMail(ctx context.Context, externalID, tsQuery string, params query.ListParams) ([]Result, error) {
|
||||
func (s *Service) searchMail(ctx context.Context, externalID, queryText string, params query.ListParams, filters SearchFilters) ([]Result, error) {
|
||||
tsQuery := toTSQuery(queryText)
|
||||
if tsQuery == "" {
|
||||
return []Result{}, nil
|
||||
}
|
||||
|
||||
limit := params.Offset() + params.Limit()
|
||||
if limit < 20 {
|
||||
limit = 20
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT m.id, m.subject, m.snippet, m.date,
|
||||
ts_rank(m.search_vector, to_tsquery('simple', $1)) as rank
|
||||
args := []any{tsQuery, externalID}
|
||||
argIdx := 3
|
||||
base := `
|
||||
FROM messages m
|
||||
JOIN mail_accounts ma ON m.account_id = ma.id
|
||||
WHERE ma.user_id = (SELECT id FROM users WHERE external_id = $2)
|
||||
AND m.search_vector @@ to_tsquery('simple', $1)
|
||||
ORDER BY rank DESC
|
||||
LIMIT $3
|
||||
`, tsQuery, externalID, limit)
|
||||
`
|
||||
if filters.AccountID != "" {
|
||||
base += fmt.Sprintf(" AND m.account_id = $%d", argIdx)
|
||||
args = append(args, filters.AccountID)
|
||||
argIdx++
|
||||
}
|
||||
if params.From != nil {
|
||||
base += fmt.Sprintf(" AND m.date >= $%d", argIdx)
|
||||
args = append(args, params.From.UTC())
|
||||
argIdx++
|
||||
}
|
||||
if params.To != nil {
|
||||
base += fmt.Sprintf(" AND m.date <= $%d", argIdx)
|
||||
args = append(args, params.To.UTC())
|
||||
argIdx++
|
||||
}
|
||||
|
||||
querySQL := `
|
||||
SELECT m.id, m.account_id, m.subject, m.snippet, m.date,
|
||||
ts_rank(m.search_vector, to_tsquery('simple', $1)) as rank,
|
||||
ts_headline(
|
||||
'simple',
|
||||
COALESCE(m.subject, '') || ' ' || COALESCE(m.snippet, ''),
|
||||
to_tsquery('simple', $1),
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxWords=24,MinWords=8,MaxFragments=2'
|
||||
) as highlighted
|
||||
` + base + fmt.Sprintf(`
|
||||
ORDER BY rank DESC, m.date DESC
|
||||
LIMIT $%d
|
||||
`, argIdx)
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := s.db.Query(ctx, querySQL, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -116,18 +306,25 @@ func (s *Service) searchMail(ctx context.Context, externalID, tsQuery string, pa
|
||||
|
||||
results := make([]Result, 0)
|
||||
for rows.Next() {
|
||||
var id, subject, snippet string
|
||||
var date any
|
||||
var id, accountID, subject, snippet string
|
||||
var date time.Time
|
||||
var rank float64
|
||||
if err := rows.Scan(&id, &subject, &snippet, &date, &rank); err != nil {
|
||||
var highlighted string
|
||||
if err := rows.Scan(&id, &accountID, &subject, &snippet, &date, &rank, &highlighted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finalSnippet := strings.TrimSpace(highlighted)
|
||||
if finalSnippet == "" {
|
||||
finalSnippet = contextualSnippet(snippet, queryText, 220)
|
||||
}
|
||||
results = append(results, Result{
|
||||
Type: "mail",
|
||||
ID: id,
|
||||
Title: subject,
|
||||
Snippet: snippet,
|
||||
Date: date,
|
||||
Type: "mail",
|
||||
ID: id,
|
||||
Title: highlightTerms(subject, queryText),
|
||||
Snippet: finalSnippet,
|
||||
Date: date.UTC(),
|
||||
AccountID: accountID,
|
||||
Score: rank + recencyBoost(date),
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@ -136,45 +333,362 @@ func (s *Service) searchMail(ctx context.Context, externalID, tsQuery string, pa
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *Service) searchContacts(ctx context.Context, queryText string, params query.ListParams) ([]Result, error) {
|
||||
limit := params.Offset() + params.Limit()
|
||||
if limit < 10 {
|
||||
limit = 10
|
||||
func (s *Service) searchContacts(ctx context.Context, userID, queryText string, filters SearchFilters) ([]Result, error) {
|
||||
if s.nc == nil {
|
||||
return []Result{}, nil
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, name, email FROM users
|
||||
WHERE (name ILIKE '%' || $1 || '%' OR email ILIKE '%' || $1 || '%')
|
||||
LIMIT $2
|
||||
`, queryText, limit)
|
||||
bookIDs := make([]string, 0, 4)
|
||||
if filters.AccountID != "" {
|
||||
bookIDs = append(bookIDs, filters.AccountID)
|
||||
} else {
|
||||
books, err := s.nc.ListAddressBooks(ctx, userID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, b := range books {
|
||||
if b.ID == "" {
|
||||
continue
|
||||
}
|
||||
bookIDs = append(bookIDs, b.ID)
|
||||
}
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
results := make([]Result, 0, 16)
|
||||
for _, bookID := range bookIDs {
|
||||
contacts, err := s.nc.ListContacts(ctx, userID, cardBookPath(userID, bookID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range contacts {
|
||||
content := strings.TrimSpace(strings.Join([]string{c.FullName, c.Email, c.Phone, c.Org}, " "))
|
||||
if !matchesQuery(content, queryText) {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(bookID + ":" + c.UID + ":" + c.Email)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
|
||||
title := strings.TrimSpace(c.FullName)
|
||||
if title == "" {
|
||||
title = strings.TrimSpace(c.Email)
|
||||
}
|
||||
snippetSrc := joinNonEmpty(" • ", c.Email, c.Phone, c.Org)
|
||||
results = append(results, Result{
|
||||
Type: "contacts",
|
||||
ID: key,
|
||||
Title: highlightTerms(title, queryText),
|
||||
Snippet: contextualSnippet(snippetSrc, queryText, 180),
|
||||
AccountID: bookID,
|
||||
Score: textMatchScore(content, queryText, 1.2),
|
||||
})
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *Service) searchEvents(ctx context.Context, userID, queryText string, params query.ListParams, filters SearchFilters) ([]Result, error) {
|
||||
if s.nc == nil {
|
||||
return []Result{}, nil
|
||||
}
|
||||
|
||||
from, to := defaultCalendarRange(params)
|
||||
calIDs := make([]string, 0, 4)
|
||||
if filters.AccountID != "" {
|
||||
calIDs = append(calIDs, filters.AccountID)
|
||||
} else {
|
||||
cals, err := s.nc.ListCalendars(ctx, userID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range cals {
|
||||
if c.ID == "" {
|
||||
continue
|
||||
}
|
||||
calIDs = append(calIDs, c.ID)
|
||||
}
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
results := make([]Result, 0, 16)
|
||||
for _, calID := range calIDs {
|
||||
events, err := s.nc.ListEvents(ctx, userID, calPath(userID, calID), from, to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, ev := range events {
|
||||
content := strings.TrimSpace(strings.Join([]string{ev.Summary, ev.Description, ev.Location}, " "))
|
||||
if !matchesQuery(content, queryText) {
|
||||
continue
|
||||
}
|
||||
eventDate, _ := parseDateString(ev.Start)
|
||||
key := strings.TrimSpace(calID + ":" + ev.UID + ":" + ev.Start)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
|
||||
snippetSrc := joinNonEmpty(" • ", ev.Description, ev.Location)
|
||||
result := Result{
|
||||
Type: "events",
|
||||
ID: key,
|
||||
Title: highlightTerms(ev.Summary, queryText),
|
||||
Snippet: contextualSnippet(snippetSrc, queryText, 200),
|
||||
AccountID: calID,
|
||||
Score: textMatchScore(content, queryText, 1.1),
|
||||
}
|
||||
if !eventDate.IsZero() {
|
||||
result.Date = eventDate.UTC()
|
||||
result.Score += recencyBoost(eventDate)
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *Service) searchFiles(ctx context.Context, userID, queryText string, params query.ListParams, filters SearchFilters) ([]Result, error) {
|
||||
if s.nc == nil {
|
||||
return []Result{}, nil
|
||||
}
|
||||
|
||||
basePath := "/"
|
||||
if strings.TrimSpace(filters.AccountID) != "" {
|
||||
basePath = strings.TrimSpace(filters.AccountID)
|
||||
}
|
||||
|
||||
files, err := s.nc.ListFiles(ctx, userID, basePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
results := make([]Result, 0)
|
||||
for rows.Next() {
|
||||
var id, name, email string
|
||||
if err := rows.Scan(&id, &name, &email); err != nil {
|
||||
return nil, err
|
||||
results := make([]Result, 0, len(files))
|
||||
for _, f := range files {
|
||||
content := strings.TrimSpace(strings.Join([]string{f.Name, f.Path, f.MimeType}, " "))
|
||||
if !matchesQuery(content, queryText) {
|
||||
continue
|
||||
}
|
||||
results = append(results, Result{
|
||||
Type: "contact",
|
||||
ID: id,
|
||||
Title: name,
|
||||
Snippet: email,
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
|
||||
var modifiedTime time.Time
|
||||
if ts, err := time.Parse(time.RFC1123, strings.TrimSpace(f.LastModified)); err == nil {
|
||||
modifiedTime = ts
|
||||
}
|
||||
if !modifiedTime.IsZero() {
|
||||
if params.From != nil && modifiedTime.Before(*params.From) {
|
||||
continue
|
||||
}
|
||||
if params.To != nil && modifiedTime.After(*params.To) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
result := Result{
|
||||
Type: "files",
|
||||
ID: f.Path,
|
||||
Title: highlightTerms(f.Name, queryText),
|
||||
Snippet: contextualSnippet(joinNonEmpty(" • ", f.Path, f.MimeType), queryText, 220),
|
||||
AccountID: basePath,
|
||||
Score: textMatchScore(content, queryText, 1.0),
|
||||
}
|
||||
if !modifiedTime.IsZero() {
|
||||
result.Date = modifiedTime.UTC()
|
||||
result.Score += recencyBoost(modifiedTime)
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func recencyBoost(ts time.Time) float64 {
|
||||
if ts.IsZero() {
|
||||
return 0
|
||||
}
|
||||
days := time.Since(ts).Hours() / 24
|
||||
if days < 0 {
|
||||
days = -days / 2
|
||||
}
|
||||
switch {
|
||||
case days <= 1:
|
||||
return 1.5
|
||||
case days <= 7:
|
||||
return 1.0
|
||||
case days <= 30:
|
||||
return 0.6
|
||||
case days <= 180:
|
||||
return 0.3
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func textMatchScore(content, q string, weight float64) float64 {
|
||||
content = strings.ToLower(strings.TrimSpace(content))
|
||||
q = strings.ToLower(strings.TrimSpace(q))
|
||||
if content == "" || q == "" {
|
||||
return 0
|
||||
}
|
||||
score := 0.0
|
||||
if strings.Contains(content, q) {
|
||||
score += 3.0 * weight
|
||||
}
|
||||
for _, term := range splitSearchTerms(q) {
|
||||
hits := strings.Count(content, term)
|
||||
if hits == 0 {
|
||||
continue
|
||||
}
|
||||
score += float64(hits) * weight
|
||||
if strings.HasPrefix(content, term) {
|
||||
score += 0.25 * weight
|
||||
}
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func toTSQuery(input string) string {
|
||||
words := strings.Fields(input)
|
||||
for i, w := range words {
|
||||
words[i] = w + ":*"
|
||||
terms := splitSearchTerms(input)
|
||||
if len(terms) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(words, " & ")
|
||||
out := make([]string, 0, len(terms))
|
||||
for _, term := range terms {
|
||||
out = append(out, term+":*")
|
||||
}
|
||||
return strings.Join(out, " & ")
|
||||
}
|
||||
|
||||
func splitSearchTerms(input string) []string {
|
||||
rawWords := strings.Fields(strings.ToLower(strings.TrimSpace(input)))
|
||||
terms := make([]string, 0, len(rawWords))
|
||||
seen := make(map[string]struct{}, len(rawWords))
|
||||
for _, raw := range rawWords {
|
||||
clean := strings.Map(func(r rune) rune {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
return r
|
||||
}
|
||||
return ' '
|
||||
}, raw)
|
||||
for _, part := range strings.Fields(clean) {
|
||||
if len(part) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[part]; ok {
|
||||
continue
|
||||
}
|
||||
seen[part] = struct{}{}
|
||||
terms = append(terms, part)
|
||||
}
|
||||
}
|
||||
return terms
|
||||
}
|
||||
|
||||
func highlightTerms(text, q string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
terms := splitSearchTerms(q)
|
||||
if len(terms) == 0 {
|
||||
return text
|
||||
}
|
||||
patterns := make([]string, 0, len(terms))
|
||||
for _, term := range terms {
|
||||
patterns = append(patterns, regexp.QuoteMeta(term))
|
||||
}
|
||||
re := regexp.MustCompile(`(?i)\b(` + strings.Join(patterns, "|") + `)\b`)
|
||||
return re.ReplaceAllStringFunc(text, func(match string) string {
|
||||
return "<mark>" + match + "</mark>"
|
||||
})
|
||||
}
|
||||
|
||||
func contextualSnippet(text, q string, maxLen int) string {
|
||||
text = strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
if maxLen <= 0 {
|
||||
maxLen = 180
|
||||
}
|
||||
|
||||
snippet := text
|
||||
if len(text) > maxLen {
|
||||
snippet = text[:maxLen]
|
||||
}
|
||||
|
||||
terms := splitSearchTerms(q)
|
||||
lower := strings.ToLower(text)
|
||||
bestIdx := -1
|
||||
for _, term := range terms {
|
||||
idx := strings.Index(lower, term)
|
||||
if idx >= 0 {
|
||||
bestIdx = idx
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if bestIdx >= 0 {
|
||||
start := bestIdx - (maxLen / 3)
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
end := start + maxLen
|
||||
if end > len(text) {
|
||||
end = len(text)
|
||||
}
|
||||
snippet = text[start:end]
|
||||
if start > 0 {
|
||||
snippet = "..." + snippet
|
||||
}
|
||||
if end < len(text) {
|
||||
snippet += "..."
|
||||
}
|
||||
}
|
||||
|
||||
return highlightTerms(snippet, q)
|
||||
}
|
||||
|
||||
func matchesQuery(content, q string) bool {
|
||||
terms := splitSearchTerms(q)
|
||||
if len(terms) == 0 {
|
||||
return true
|
||||
}
|
||||
content = strings.ToLower(content)
|
||||
for _, term := range terms {
|
||||
if strings.Contains(content, term) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func joinNonEmpty(sep string, parts ...string) string {
|
||||
filtered := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
filtered = append(filtered, part)
|
||||
}
|
||||
}
|
||||
return strings.Join(filtered, sep)
|
||||
}
|
||||
|
||||
func discardFilteredResults(results []Result) []Result {
|
||||
out := make([]Result, 0, len(results))
|
||||
for _, r := range results {
|
||||
if r.Score < 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cardBookPath(userID, bookID string) string {
|
||||
return "/remote.php/dav/addressbooks/users/" + userID + "/" + bookID + "/"
|
||||
}
|
||||
|
||||
func calPath(userID, calID string) string {
|
||||
return "/remote.php/dav/calendars/" + userID + "/" + calID + "/"
|
||||
}
|
||||
|
||||
184
internal/search/service_test.go
Normal file
184
internal/search/service_test.go
Normal file
@ -0,0 +1,184 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeEngine(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{name: "postgres explicit", raw: "postgres", want: "postgres"},
|
||||
{name: "meilisearch", raw: "meilisearch", want: "meilisearch"},
|
||||
{name: "meilisearch mixed case", raw: "MeiliSearch", want: "meilisearch"},
|
||||
{name: "meilisearch trimmed", raw: " meilisearch ", want: "meilisearch"},
|
||||
{name: "typesense", raw: "typesense", want: "typesense"},
|
||||
{name: "typesense mixed case", raw: "TypeSense", want: "typesense"},
|
||||
{name: "empty defaults postgres", raw: "", want: "postgres"},
|
||||
{name: "unknown defaults postgres", raw: "elasticsearch", want: "postgres"},
|
||||
{name: "whitespace defaults postgres", raw: " ", want: "postgres"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := normalizeEngine(tt.raw); got != tt.want {
|
||||
t.Fatalf("normalizeEngine(%q) = %q, want %q", tt.raw, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTypes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "empty defaults all",
|
||||
raw: "",
|
||||
want: []string{"mail", "contacts", "files", "events"},
|
||||
},
|
||||
{
|
||||
name: "whitespace defaults all",
|
||||
raw: " ",
|
||||
want: []string{"mail", "contacts", "files", "events"},
|
||||
},
|
||||
{
|
||||
name: "explicit subset",
|
||||
raw: "mail,contacts",
|
||||
want: []string{"mail", "contacts"},
|
||||
},
|
||||
{
|
||||
name: "deduplicates",
|
||||
raw: "mail,mail,contacts,contacts",
|
||||
want: []string{"mail", "contacts"},
|
||||
},
|
||||
{
|
||||
name: "trims and skips empty parts",
|
||||
raw: " mail , , contacts ",
|
||||
want: []string{"mail", "contacts"},
|
||||
},
|
||||
{
|
||||
name: "preserves order",
|
||||
raw: "events,mail,contacts",
|
||||
want: []string{"events", "mail", "contacts"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := parseTypes(tt.raw)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("parseTypes(%q) = %v, want %v", tt.raw, got, tt.want)
|
||||
}
|
||||
for i := range tt.want {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("parseTypes(%q)[%d] = %q, want %q (full: %v)", tt.raw, i, got[i], tt.want[i], got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSearchTerms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want []string
|
||||
}{
|
||||
{name: "empty", input: "", want: nil},
|
||||
{name: "whitespace", input: " ", want: nil},
|
||||
{name: "simple words", input: "hello world", want: []string{"hello", "world"}},
|
||||
{name: "strips punctuation", input: "hello, world!", want: []string{"hello", "world"}},
|
||||
{name: "deduplicates", input: "foo foo bar", want: []string{"foo", "bar"}},
|
||||
{name: "mixed punctuation", input: "can't re-read", want: []string{"can", "t", "re", "read"}},
|
||||
{name: "case normalized", input: "Hello HELLO", want: []string{"hello"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := splitSearchTerms(tt.input)
|
||||
if len(got) == 0 && len(tt.want) == 0 {
|
||||
return
|
||||
}
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("splitSearchTerms(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
for i := range tt.want {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("splitSearchTerms(%q)[%d] = %q, want %q (full: %v)", tt.input, i, got[i], tt.want[i], got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHighlightTerms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
q string
|
||||
want string
|
||||
}{
|
||||
{name: "empty text", text: "", q: "hello", want: ""},
|
||||
{name: "empty query unchanged", text: "Hello World", q: "", want: "Hello World"},
|
||||
{name: "case insensitive preserves match casing", text: "Hello World", q: "hello", want: "<mark>Hello</mark> World"},
|
||||
{name: "multiple terms", text: "Meet Alice and Bob", q: "alice bob", want: "Meet <mark>Alice</mark> and <mark>Bob</mark>"},
|
||||
{name: "word boundary only", text: "hellish hello", q: "hello", want: "hellish <mark>hello</mark>"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := highlightTerms(tt.text, tt.q); got != tt.want {
|
||||
t.Fatalf("highlightTerms(%q, %q) = %q, want %q", tt.text, tt.q, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualSnippet(t *testing.T) {
|
||||
t.Run("highlights match in short text", func(t *testing.T) {
|
||||
got := contextualSnippet("Invoice from Acme Corp", "acme", 180)
|
||||
want := "Invoice from <mark>Acme</mark> Corp"
|
||||
if got != want {
|
||||
t.Fatalf("contextualSnippet short = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trims long content around match", func(t *testing.T) {
|
||||
prefix := strings.Repeat("word ", 30)
|
||||
text := prefix + "needle here " + strings.Repeat("tail ", 30)
|
||||
got := contextualSnippet(text, "needle", 40)
|
||||
|
||||
if !strings.Contains(got, "<mark>needle</mark>") {
|
||||
t.Fatalf("expected highlighted needle, got %q", got)
|
||||
}
|
||||
if !strings.HasPrefix(got, "...") {
|
||||
t.Fatalf("expected leading ellipsis for trimmed snippet, got %q", got)
|
||||
}
|
||||
if !strings.HasSuffix(got, "...") {
|
||||
t.Fatalf("expected trailing ellipsis for trimmed snippet, got %q", got)
|
||||
}
|
||||
if len(got) > 80 {
|
||||
t.Fatalf("snippet too long (%d chars): %q", len(got), got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty text", func(t *testing.T) {
|
||||
if got := contextualSnippet(" ", "foo", 100); got != "" {
|
||||
t.Fatalf("contextualSnippet empty = %q, want empty", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("default maxLen when zero", func(t *testing.T) {
|
||||
long := strings.Repeat("x", 300) + " findme"
|
||||
got := contextualSnippet(long, "findme", 0)
|
||||
if !strings.Contains(got, "<mark>findme</mark>") {
|
||||
t.Fatalf("expected highlighted findme with default maxLen, got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -14,6 +14,7 @@ const (
|
||||
var allowedSearchTypes = map[string]struct{}{
|
||||
"mail": {},
|
||||
"contacts": {},
|
||||
"files": {},
|
||||
"events": {},
|
||||
}
|
||||
|
||||
|
||||
@ -128,11 +128,11 @@ Objectif: transformer état actuel (partiellement implémenté) vers produit fon
|
||||
|
||||
### 2.6 Search
|
||||
|
||||
- [ ] Finaliser recherche events (Agenda) et contacts (CardDAV), pas fallback users local.
|
||||
- [ ] Ajouter recherche multi-index (mail+contacts+files+events) avec score unifié.
|
||||
- [ ] Ajouter snippets contextuels et highlighting.
|
||||
- [ ] Ajouter filtres type/date/account.
|
||||
- [ ] Préparer option Meilisearch/Typesense activable.
|
||||
- [x] Finaliser recherche events (Agenda) et contacts (CardDAV), pas fallback users local.
|
||||
- [x] Ajouter recherche multi-index (mail+contacts+files+events) avec score unifié.
|
||||
- [x] Ajouter snippets contextuels et highlighting.
|
||||
- [x] Ajouter filtres type/date/account.
|
||||
- [x] Préparer option Meilisearch/Typesense activable.
|
||||
|
||||
### 2.7 Modules suite (Drive/Calendar/Contacts/Meet/Photos/Admin)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user