package search import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/ultisuite/ulti-backend/internal/api/query" ) const defaultSearchIndex = "ulti" type externalSearchClient interface { Search(ctx context.Context, params ExternalSearchParams) ([]Result, error) } type ExternalSearchParams struct { ExternalID string Query string Types []string Params query.ListParams Filters SearchFilters } func newExternalSearchClient(engine string, opts ServiceOptions) externalSearchClient { switch engine { case "meilisearch": if strings.TrimSpace(opts.MeilisearchURL) == "" { return nil } index := strings.TrimSpace(opts.MeilisearchIndex) if index == "" { index = defaultSearchIndex } return &meilisearchClient{ baseURL: strings.TrimRight(strings.TrimSpace(opts.MeilisearchURL), "/"), apiKey: strings.TrimSpace(opts.MeilisearchKey), index: index, http: &http.Client{Timeout: 5 * time.Second}, } case "typesense": if strings.TrimSpace(opts.TypesenseURL) == "" { return nil } collection := strings.TrimSpace(opts.TypesenseCollection) if collection == "" { collection = defaultSearchIndex } return &typesenseClient{ baseURL: strings.TrimRight(strings.TrimSpace(opts.TypesenseURL), "/"), apiKey: strings.TrimSpace(opts.TypesenseKey), collection: collection, http: &http.Client{Timeout: 5 * time.Second}, } default: return nil } } type meilisearchClient struct { baseURL string apiKey string index string http *http.Client } func (c *meilisearchClient) Search(ctx context.Context, params ExternalSearchParams) ([]Result, error) { payload := map[string]any{ "q": params.Query, "limit": params.Params.Limit(), "offset": params.Params.Offset(), } if filters := meiliFilters(params); len(filters) > 0 { payload["filter"] = filters } data, err := json.Marshal(payload) if err != nil { return nil, err } u := c.baseURL + "/indexes/" + url.PathEscape(c.index) + "/search" req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(data)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") if c.apiKey != "" { req.Header.Set("Authorization", "Bearer "+c.apiKey) } resp, err := c.http.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) return nil, fmt.Errorf("meilisearch status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } var parsed struct { Hits []map[string]any `json:"hits"` } if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { return nil, err } return mapExternalHits(parsed.Hits, params.Query), nil } func meiliFilters(params ExternalSearchParams) []string { filters := make([]string, 0, 5) if params.ExternalID != "" { filters = append(filters, fmt.Sprintf("user_external_id = %q", params.ExternalID)) } if len(params.Types) > 0 { escaped := make([]string, 0, len(params.Types)) for _, t := range params.Types { escaped = append(escaped, fmt.Sprintf("%q", t)) } filters = append(filters, "type IN ["+strings.Join(escaped, ", ")+"]") } if params.Filters.AccountID != "" { filters = append(filters, fmt.Sprintf("account_id = %q", params.Filters.AccountID)) } if params.Params.From != nil { filters = append(filters, fmt.Sprintf("date >= %q", params.Params.From.UTC().Format(time.RFC3339))) } if params.Params.To != nil { filters = append(filters, fmt.Sprintf("date <= %q", params.Params.To.UTC().Format(time.RFC3339))) } return filters } type typesenseClient struct { baseURL string apiKey string collection string http *http.Client } func (c *typesenseClient) Search(ctx context.Context, params ExternalSearchParams) ([]Result, error) { u, err := url.Parse(c.baseURL + "/collections/" + url.PathEscape(c.collection) + "/documents/search") if err != nil { return nil, err } q := u.Query() q.Set("q", params.Query) q.Set("query_by", "title,snippet,content,name,summary,path") q.Set("per_page", strconv.Itoa(params.Params.Limit())) q.Set("page", strconv.Itoa(params.Params.Page)) q.Set("highlight_affix_num_tokens", "6") q.Set("include_fields", "id,type,title,snippet,date,account_id,path,highlights,_text_match") if filter := typesenseFilter(params); filter != "" { q.Set("filter_by", filter) } u.RawQuery = q.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err } if c.apiKey != "" { req.Header.Set("X-TYPESENSE-API-KEY", c.apiKey) } resp, err := c.http.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode >= 300 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) return nil, fmt.Errorf("typesense status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } var parsed struct { Hits []struct { Document map[string]any `json:"document"` TextMatch any `json:"text_match"` } `json:"hits"` } if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { return nil, err } hits := make([]map[string]any, 0, len(parsed.Hits)) for _, h := range parsed.Hits { doc := h.Document if doc == nil { doc = map[string]any{} } doc["_text_match"] = h.TextMatch hits = append(hits, doc) } return mapExternalHits(hits, params.Query), nil } func typesenseFilter(params ExternalSearchParams) string { parts := make([]string, 0, 5) if params.ExternalID != "" { parts = append(parts, fmt.Sprintf("user_external_id:=%s", params.ExternalID)) } if len(params.Types) > 0 { parts = append(parts, "type:=["+strings.Join(params.Types, ",")+"]") } if params.Filters.AccountID != "" { parts = append(parts, fmt.Sprintf("account_id:=%s", params.Filters.AccountID)) } if params.Params.From != nil { parts = append(parts, fmt.Sprintf("date:>=%s", params.Params.From.UTC().Format(time.RFC3339))) } if params.Params.To != nil { parts = append(parts, fmt.Sprintf("date:<=%s", params.Params.To.UTC().Format(time.RFC3339))) } return strings.Join(parts, " && ") } func mapExternalHits(hits []map[string]any, queryText string) []Result { results := make([]Result, 0, len(hits)) for _, hit := range hits { r := Result{ Type: asString(firstNonNil(hit["type"], hit["kind"])), ID: asString(firstNonNil(hit["id"], hit["_id"], hit["uid"], hit["path"])), Title: asString(firstNonNil(hit["title"], hit["name"], hit["subject"], hit["summary"])), Snippet: asString(firstNonNil(hit["snippet"], hit["content"], hit["description"], hit["path"])), AccountID: asString(firstNonNil(hit["account_id"], hit["account"], hit["book_id"], hit["calendar_id"])), Score: asFloat(firstNonNil(hit["score"], hit["_rankingScore"], hit["_text_match"])), } if r.Type == "" { r.Type = "mail" } if r.ID == "" { r.ID = r.Type + ":" + r.Title } if dt := asString(firstNonNil(hit["date"], hit["updated_at"], hit["start"])); dt != "" { if parsed, ok := parseDateString(dt); ok { r.Date = parsed.UTC() } } if r.Snippet == "" { r.Snippet = r.Title } r.Title = highlightTerms(r.Title, queryText) r.Snippet = contextualSnippet(r.Snippet, queryText, 220) if r.Score == 0 { r.Score = textMatchScore(joinNonEmpty(" ", r.Title, r.Snippet), queryText, 1.0) } results = append(results, r) } return results } func finalizeExternalResults(results []Result, queryText string, params query.ListParams) { for i := range results { if strings.TrimSpace(results[i].Title) == "" { results[i].Title = results[i].ID } if strings.TrimSpace(results[i].Snippet) == "" { results[i].Snippet = results[i].Title } results[i].Title = highlightTerms(results[i].Title, queryText) results[i].Snippet = contextualSnippet(results[i].Snippet, queryText, 220) if results[i].Score == 0 { results[i].Score = textMatchScore(joinNonEmpty(" ", results[i].Title, results[i].Snippet), queryText, 1.0) } if ts, ok := dateFromAny(results[i].Date); ok { if params.From != nil && ts.Before(*params.From) { results[i].Score = -1 continue } if params.To != nil && ts.After(*params.To) { results[i].Score = -1 continue } results[i].Date = ts.UTC() results[i].Score += recencyBoost(ts) } } } func firstNonNil(values ...any) any { for _, v := range values { if v != nil { return v } } return nil } func asString(value any) string { if value == nil { return "" } switch v := value.(type) { case string: return strings.TrimSpace(v) case fmt.Stringer: return strings.TrimSpace(v.String()) default: return strings.TrimSpace(fmt.Sprintf("%v", v)) } } func asFloat(value any) float64 { switch v := value.(type) { case float64: return v case float32: return float64(v) case int: return float64(v) case int64: return float64(v) case json.Number: if f, err := v.Float64(); err == nil { return f } case string: if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil { return f } } return 0 }