216 lines
6.2 KiB
Go

package queryserving
import (
"bufio"
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
type SearchDocument struct {
ID string `json:"id"`
ItemID string `json:"item_id"`
Grams []string `json:"grams"`
}
// Grams are hex-encoded Unicode windows, independent of Typesense's tokenizer,
// punctuation/typo/prefix rules. Intersection is only a conservative prefilter;
// the existing Go matcher remains the final authority. No query is truncated.
func Grams(text string) []string {
r := []rune(strings.ToLower(text))
width := min(3, len(r))
if width == 0 {
return nil
}
set := map[string]bool{}
for i := 0; i+width <= len(r); i++ {
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
}
out := make([]string, 0, len(set))
for s := range set {
out = append(out, s)
}
sort.Strings(out)
return out
}
func IndexGrams(text string) []string {
r := []rune(strings.ToLower(text))
set := map[string]bool{}
for width := 1; width <= 3; width++ {
for i := 0; i+width <= len(r); i++ {
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
}
}
out := make([]string, 0, len(set))
for s := range set {
out = append(out, s)
}
sort.Strings(out)
return out
}
type Typesense struct {
base, key string
client *http.Client
}
type HTTPError struct{ Status int }
func (e *HTTPError) Error() string { return fmt.Sprintf("search backend status %d", e.Status) }
func NewTypesense(address, key string) (*Typesense, error) {
u, err := url.Parse(address)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || key == "" {
return nil, errors.New("invalid search configuration")
}
return &Typesense{base: strings.TrimRight(address, "/"), key: key, client: &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil
}
func (t *Typesense) request(ctx context.Context, method, path string, body []byte) ([]byte, error) {
r, err := http.NewRequestWithContext(ctx, method, t.base+path, bytes.NewReader(body))
if err != nil {
return nil, errors.New("search request invalid")
}
r.Header.Set("X-TYPESENSE-API-KEY", t.key)
r.Header.Set("Content-Type", "application/json")
res, err := t.client.Do(r)
if err != nil {
return nil, errors.New("search backend unavailable")
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, &HTTPError{Status: res.StatusCode}
}
b, err := io.ReadAll(io.LimitReader(res.Body, 32<<20+1))
if err != nil || len(b) > 32<<20 {
return nil, errors.New("search response incomplete")
}
return b, nil
}
func validCollection(c string) bool {
if !strings.HasPrefix(c, "dfq_") || len(c) > 120 {
return false
}
for _, r := range c {
if r != '_' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
return false
}
}
return true
}
func (t *Typesense) Create(ctx context.Context, c string) error {
if !validCollection(c) {
return errors.New("invalid search collection")
}
b, _ := json.Marshal(map[string]any{"name": c, "fields": []map[string]any{{"name": "item_id", "type": "string", "index": false}, {"name": "grams", "type": "string[]", "facet": true}}})
_, err := t.request(ctx, "POST", "/collections", b)
return err
}
func (t *Typesense) Import(ctx context.Context, c string, docs []SearchDocument) error {
if !validCollection(c) {
return errors.New("invalid search collection")
}
var data bytes.Buffer
for _, d := range docs {
if err := json.NewEncoder(&data).Encode(d); err != nil {
return err
}
}
b, err := t.request(ctx, "POST", "/collections/"+c+"/documents/import?action=create", data.Bytes())
if err != nil {
return err
}
s := bufio.NewScanner(bytes.NewReader(b))
s.Buffer(make([]byte, 4096), 1<<20)
count := 0
for s.Scan() {
var result struct {
Success bool `json:"success"`
}
if json.Unmarshal(s.Bytes(), &result) != nil || !result.Success {
return errors.New("search import incomplete")
}
count++
}
if s.Err() != nil || count != len(docs) {
return errors.New("search import response incomplete")
}
return nil
}
func (t *Typesense) Candidates(ctx context.Context, c string, terms ...string) ([]string, error) {
if !validCollection(c) {
return nil, errors.New("invalid search collection")
}
filters := []string{}
for _, g := range candidateGrams(terms...) {
filters = append(filters, "grams:=["+g+"]")
}
params := url.Values{"include_fields": {"item_id"}}
if len(filters) > 0 {
params.Set("filter_by", strings.Join(filters, " && "))
}
b, err := t.request(ctx, "GET", "/collections/"+c+"/documents/export?"+params.Encode(), nil)
if err != nil {
return nil, err
}
s := bufio.NewScanner(bytes.NewReader(b))
s.Buffer(make([]byte, 4096), 1<<20)
ids := []string{}
for s.Scan() {
var d SearchDocument
if json.Unmarshal(s.Bytes(), &d) != nil || d.ItemID == "" {
return nil, errors.New("search candidates incomplete")
}
ids = append(ids, d.ItemID)
}
if s.Err() != nil {
return nil, errors.New("search candidates incomplete")
}
return ids, nil
}
func (t *Typesense) Delete(ctx context.Context, c string) error {
if !validCollection(c) {
return errors.New("invalid search collection")
}
_, err := t.request(ctx, "DELETE", "/collections/"+c, nil)
var status *HTTPError
if errors.As(err, &status) && status.Status == 404 {
return nil
}
return err
}
// Typesense's default parser counts both clauses and boolean operators. Keep
// well below its 100-operation limit without changing that server setting.
// Dropping prefilter constraints only adds candidates; the original complete
// query is still matched by Go, and no task/result inventory is truncated.
func candidateGrams(terms ...string) []string {
set := map[string]bool{}
for _, term := range terms {
for _, gram := range Grams(term) {
set[gram] = true
}
}
all := make([]string, 0, len(set))
for gram := range set {
all = append(all, gram)
}
sort.Strings(all)
const limit = 40
if len(all) <= limit {
return all
}
selected := make([]string, limit)
for i := range selected {
selected[i] = all[i*(len(all)-1)/(limit-1)]
}
return selected
}