216 lines
6.2 KiB
Go
216 lines
6.2 KiB
Go
package queryserving
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type SearchDocument struct {
|
|
ID string `json:"id"`
|
|
ItemID string `json:"item_id"`
|
|
Grams []string `json:"grams"`
|
|
}
|
|
|
|
// Grams are hex-encoded Unicode windows, independent of Typesense's tokenizer,
|
|
// punctuation/typo/prefix rules. Intersection is only a conservative prefilter;
|
|
// the existing Go matcher remains the final authority. No query is truncated.
|
|
func Grams(text string) []string {
|
|
r := []rune(strings.ToLower(text))
|
|
width := min(3, len(r))
|
|
if width == 0 {
|
|
return nil
|
|
}
|
|
set := map[string]bool{}
|
|
for i := 0; i+width <= len(r); i++ {
|
|
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
|
|
}
|
|
out := make([]string, 0, len(set))
|
|
for s := range set {
|
|
out = append(out, s)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
func IndexGrams(text string) []string {
|
|
r := []rune(strings.ToLower(text))
|
|
set := map[string]bool{}
|
|
for width := 1; width <= 3; width++ {
|
|
for i := 0; i+width <= len(r); i++ {
|
|
set[hex.EncodeToString([]byte(string(r[i:i+width])))] = true
|
|
}
|
|
}
|
|
out := make([]string, 0, len(set))
|
|
for s := range set {
|
|
out = append(out, s)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
type Typesense struct {
|
|
base, key string
|
|
client *http.Client
|
|
}
|
|
type HTTPError struct{ Status int }
|
|
|
|
func (e *HTTPError) Error() string { return fmt.Sprintf("search backend status %d", e.Status) }
|
|
|
|
func NewTypesense(address, key string) (*Typesense, error) {
|
|
u, err := url.Parse(address)
|
|
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || key == "" {
|
|
return nil, errors.New("invalid search configuration")
|
|
}
|
|
return &Typesense{base: strings.TrimRight(address, "/"), key: key, client: &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil
|
|
}
|
|
func (t *Typesense) request(ctx context.Context, method, path string, body []byte) ([]byte, error) {
|
|
r, err := http.NewRequestWithContext(ctx, method, t.base+path, bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, errors.New("search request invalid")
|
|
}
|
|
r.Header.Set("X-TYPESENSE-API-KEY", t.key)
|
|
r.Header.Set("Content-Type", "application/json")
|
|
res, err := t.client.Do(r)
|
|
if err != nil {
|
|
return nil, errors.New("search backend unavailable")
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
|
return nil, &HTTPError{Status: res.StatusCode}
|
|
}
|
|
b, err := io.ReadAll(io.LimitReader(res.Body, 32<<20+1))
|
|
if err != nil || len(b) > 32<<20 {
|
|
return nil, errors.New("search response incomplete")
|
|
}
|
|
return b, nil
|
|
}
|
|
func validCollection(c string) bool {
|
|
if !strings.HasPrefix(c, "dfq_") || len(c) > 120 {
|
|
return false
|
|
}
|
|
for _, r := range c {
|
|
if r != '_' && (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
func (t *Typesense) Create(ctx context.Context, c string) error {
|
|
if !validCollection(c) {
|
|
return errors.New("invalid search collection")
|
|
}
|
|
b, _ := json.Marshal(map[string]any{"name": c, "fields": []map[string]any{{"name": "item_id", "type": "string", "index": false}, {"name": "grams", "type": "string[]", "facet": true}}})
|
|
_, err := t.request(ctx, "POST", "/collections", b)
|
|
return err
|
|
}
|
|
func (t *Typesense) Import(ctx context.Context, c string, docs []SearchDocument) error {
|
|
if !validCollection(c) {
|
|
return errors.New("invalid search collection")
|
|
}
|
|
var data bytes.Buffer
|
|
for _, d := range docs {
|
|
if err := json.NewEncoder(&data).Encode(d); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
b, err := t.request(ctx, "POST", "/collections/"+c+"/documents/import?action=create", data.Bytes())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s := bufio.NewScanner(bytes.NewReader(b))
|
|
s.Buffer(make([]byte, 4096), 1<<20)
|
|
count := 0
|
|
for s.Scan() {
|
|
var result struct {
|
|
Success bool `json:"success"`
|
|
}
|
|
if json.Unmarshal(s.Bytes(), &result) != nil || !result.Success {
|
|
return errors.New("search import incomplete")
|
|
}
|
|
count++
|
|
}
|
|
if s.Err() != nil || count != len(docs) {
|
|
return errors.New("search import response incomplete")
|
|
}
|
|
return nil
|
|
}
|
|
func (t *Typesense) Candidates(ctx context.Context, c string, terms ...string) ([]string, error) {
|
|
if !validCollection(c) {
|
|
return nil, errors.New("invalid search collection")
|
|
}
|
|
filters := []string{}
|
|
for _, g := range candidateGrams(terms...) {
|
|
filters = append(filters, "grams:=["+g+"]")
|
|
}
|
|
params := url.Values{"include_fields": {"item_id"}}
|
|
if len(filters) > 0 {
|
|
params.Set("filter_by", strings.Join(filters, " && "))
|
|
}
|
|
b, err := t.request(ctx, "GET", "/collections/"+c+"/documents/export?"+params.Encode(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
s := bufio.NewScanner(bytes.NewReader(b))
|
|
s.Buffer(make([]byte, 4096), 1<<20)
|
|
ids := []string{}
|
|
for s.Scan() {
|
|
var d SearchDocument
|
|
if json.Unmarshal(s.Bytes(), &d) != nil || d.ItemID == "" {
|
|
return nil, errors.New("search candidates incomplete")
|
|
}
|
|
ids = append(ids, d.ItemID)
|
|
}
|
|
if s.Err() != nil {
|
|
return nil, errors.New("search candidates incomplete")
|
|
}
|
|
return ids, nil
|
|
}
|
|
func (t *Typesense) Delete(ctx context.Context, c string) error {
|
|
if !validCollection(c) {
|
|
return errors.New("invalid search collection")
|
|
}
|
|
_, err := t.request(ctx, "DELETE", "/collections/"+c, nil)
|
|
var status *HTTPError
|
|
if errors.As(err, &status) && status.Status == 404 {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Typesense's default parser counts both clauses and boolean operators. Keep
|
|
// well below its 100-operation limit without changing that server setting.
|
|
// Dropping prefilter constraints only adds candidates; the original complete
|
|
// query is still matched by Go, and no task/result inventory is truncated.
|
|
func candidateGrams(terms ...string) []string {
|
|
set := map[string]bool{}
|
|
for _, term := range terms {
|
|
for _, gram := range Grams(term) {
|
|
set[gram] = true
|
|
}
|
|
}
|
|
all := make([]string, 0, len(set))
|
|
for gram := range set {
|
|
all = append(all, gram)
|
|
}
|
|
sort.Strings(all)
|
|
const limit = 40
|
|
if len(all) <= limit {
|
|
return all
|
|
}
|
|
selected := make([]string, limit)
|
|
for i := range selected {
|
|
selected[i] = all[i*(len(all)-1)/(limit-1)]
|
|
}
|
|
return selected
|
|
}
|