eintopf.info@v0.13.16/service/search/search_test.go

// Copyright (C) 2022 The Eintopf authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

package search_test

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"testing"
	"time"

	"eintopf.info/service/search"
	"eintopf.info/test"
)

type doc struct {
	ID   string
	A    string    `json:"a"`
	Int  int       `json:"b"`
	C    string    `json:"c"`
	D    time.Time `json:"d"`
	E    string    `json:"e"`
	Bool bool      `json:"bool"`
}

func (d *doc) Identifier() string {
	return d.ID
}

func (d *doc) Type() string {
	return "mytype"
}

func (d *doc) QueryText() string {
	if d.A == "" {
		return "a"
	}
	return d.A
}

func (d *doc) SearchFields() map[string]interface{} {
	return map[string]interface{}{
		"id":   d.ID,
		"a":    d.A,
		"Int":  d.Int,
		"c":    d.C,
		"d":    d.D,
		"e":    d.E,
		"bool": d.Bool,
	}
}

func TestSearchQuery(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name:     "returns nothing without a match",
			docs:     []doc{{ID: "1", A: "something", Int: 42}},
			query:    "nothing",
			wantHits: []doc{},
		}, {
			name:     "empty query returns all documents",
			docs:     []doc{{ID: "1", A: "something", Int: 42}, {ID: "2", A: "nothing", Int: 42}},
			query:    "",
			wantHits: []doc{{ID: "1", A: "something", Int: 42}, {ID: "2", A: "nothing", Int: 42}},
		}, {
			name:     "matches on a word",
			docs:     []doc{{ID: "1", A: "something", Int: 42}},
			query:    "something",
			opts:     nil,
			wantHits: []doc{{ID: "1", A: "something", Int: 42}},
		}, {
			name:     "matches on part of the text",
			docs:     []doc{{ID: "1", A: "from to", Int: 42}},
			query:    "from",
			opts:     nil,
			wantHits: []doc{{ID: "1", A: "from to", Int: 42}},
		}, {
			name:     "matches on multiple words in the query",
			docs:     []doc{{ID: "1", A: "from to", Int: 42}},
			query:    "from to",
			opts:     nil,
			wantHits: []doc{{ID: "1", A: "from to", Int: 42}},
		}, {
			name:     "Fahradtaschen -> Fahradtasche",
			docs:     []doc{{ID: "1", A: "Fahradtaschen", Int: 42}},
			query:    "fahradtasche",
			opts:     nil,
			wantHits: []doc{{ID: "1", A: "Fahradtaschen", Int: 42}},
		}, {
			name: "finds multple docs",
			docs: []doc{
				{ID: "1", A: "something", Int: 42},
				{ID: "2", A: "something", Int: 43},
			},
			query: "something",
			opts:  &search.Options{Sort: "id"},
			wantHits: []doc{
				{ID: "1", A: "something", Int: 42},
				{ID: "2", A: "something", Int: 43},
			},
		}, {
			name: "doesn't always return everything",
			docs: []doc{
				{ID: "1", A: "something", Int: 42},
				{ID: "2", A: "nothing", Int: 43},
			},
			query:    "something",
			opts:     nil,
			wantHits: []doc{{ID: "1", A: "something", Int: 42}},
		},
	})
}

func TestSearchSort(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name:     "sort number field",
			docs:     []doc{{ID: "2", Int: 2}, {ID: "1", Int: 1}},
			query:    "",
			opts:     &search.Options{Sort: "Int"},
			wantHits: []doc{{ID: "1", Int: 1}, {ID: "2", Int: 2}},
		},
		{
			name:     "sort string field",
			docs:     []doc{{ID: "1", A: "foo"}, {ID: "2", A: "bar"}},
			query:    "",
			opts:     &search.Options{Sort: "a"},
			wantHits: []doc{{ID: "2", A: "bar"}, {ID: "1", A: "foo"}},
		},
		{
			name:     "sort string field (ignores upper/lower case)",
			docs:     []doc{{ID: "1", A: "Foo"}, {ID: "2", A: "bar"}},
			query:    "",
			opts:     &search.Options{Sort: "a"},
			wantHits: []doc{{ID: "2", A: "bar"}, {ID: "1", A: "Foo"}},
		},
		{
			name: "sort time field",
			docs: []doc{
				{ID: "1", D: time.Date(2022, 10, 20, 10, 10, 10, 10, time.UTC)},
				{ID: "2", D: time.Date(2022, 10, 10, 10, 10, 10, 10, time.UTC)},
			},
			query: "",
			opts:  &search.Options{Sort: "d"},
			wantHits: []doc{
				{ID: "2", D: time.Date(2022, 10, 10, 10, 10, 10, 10, time.UTC)},
				{ID: "1", D: time.Date(2022, 10, 20, 10, 10, 10, 10, time.UTC)},
			},
		},
		{
			name:     "sort descending",
			docs:     []doc{{ID: "2", Int: 2}, {ID: "1", Int: 1}},
			query:    "",
			opts:     &search.Options{Sort: "b", SortDescending: true},
			wantHits: []doc{{ID: "2", Int: 2}, {ID: "1", Int: 1}},
		},
	})
}

func TestSearchPagination(t *testing.T) {
	three := uint64(3)
	testSearchTestcases(t, []testcase{
		{
			name:      "NoPagination",
			docs:      []doc{{ID: "1"}, {ID: "2"}, {ID: "3"}},
			query:     "",
			opts:      &search.Options{Sort: "id", Page: 0, PageSize: 0},
			wantHits:  []doc{{ID: "1"}, {ID: "2"}, {ID: "3"}},
			wantTotal: &three,
		}, {
			name:      "FirstPage",
			docs:      []doc{{ID: "1"}, {ID: "2"}, {ID: "3"}},
			query:     "",
			opts:      &search.Options{Sort: "id", Page: 0, PageSize: 2},
			wantHits:  []doc{{ID: "1"}, {ID: "2"}},
			wantTotal: &three,
		}, {
			name:      "SecondPage",
			docs:      []doc{{ID: "1"}, {ID: "2"}, {ID: "3"}},
			query:     "",
			opts:      &search.Options{Sort: "id", Page: 1, PageSize: 2},
			wantHits:  []doc{{ID: "3"}},
			wantTotal: &three,
		},
	})
}

func TestSearchFilterTerms(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name: "String1",
			docs: []doc{{ID: "1", C: "foo"}, {ID: "2", C: "bar"}},
			opts: &search.Options{Filters: []search.Filter{
				&search.TermsFilter{
					Field: "c",
					Terms: []string{"foo"},
				},
			}},
			wantHits: []doc{{ID: "1", C: "foo"}},
		}, {
			name: "String2",
			docs: []doc{{ID: "1", C: "foo bar"}, {ID: "2", C: "bar"}},
			opts: &search.Options{Filters: []search.Filter{
				&search.TermsFilter{
					Field: "c",
					Terms: []string{"bar"},
				},
			}},
			wantHits: []doc{{ID: "2", C: "bar"}, {ID: "1", C: "foo bar"}},
		}, {
			name: "UUID",
			docs: []doc{{ID: "aee609c7-4fea-4a1c-8acf-0799a3bebadd", C: "foo"}, {ID: "39396d6e-fe59-4fea-899a-387d716a8968", C: "bar"}},
			opts: &search.Options{Filters: []search.Filter{
				&search.TermsFilter{
					Field: "id",
					Terms: []string{"aee609c7-4fea-4a1c-8acf-0799a3bebadd"},
				},
			}},
			wantHits: []doc{{ID: "aee609c7-4fea-4a1c-8acf-0799a3bebadd", C: "foo"}},
		}, {
			name: "Integer",
			docs: []doc{{ID: "1", Int: 1}, {ID: "2", Int: 2}},
			opts: &search.Options{Filters: []search.Filter{
				&search.TermsFilter{
					Field: "Int",
					Terms: []string{"1"},
				},
			}},
			wantHits: []doc{}, // can't find int via terms filter
		}, {
			name: "MultipleTerms",
			docs: []doc{
				{ID: "1", C: "foo"},
				{ID: "2", C: "bar"},
				{ID: "3", C: "baz"},
			},
			opts: &search.Options{Sort: "c", Filters: []search.Filter{
				&search.TermsFilter{
					Field: "c",
					Terms: []string{"foo", "baz"},
				},
			}},
			wantHits: []doc{
				{ID: "3", C: "baz"},
				{ID: "1", C: "foo"},
			},
		},
	})
}

func TestSearchFilterDateRange(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name: "DateRangeFilter",
			docs: []doc{
				{ID: "1", D: time.Date(2022, 4, 22, 12, 0, 0, 0, time.UTC)},
				{ID: "2", D: time.Date(2022, 4, 22, 14, 0, 0, 0, time.UTC)},
			},
			opts: &search.Options{Filters: []search.Filter{
				&search.DateRangeFilter{
					Field: "d",
					Min:   time.Date(2022, 4, 22, 11, 0, 0, 0, time.UTC),
					Max:   time.Date(2022, 4, 22, 13, 0, 0, 0, time.UTC),
				},
			}},
			wantHits: []doc{{ID: "1", D: time.Date(2022, 4, 22, 12, 0, 0, 0, time.UTC)}},
		}, {
			name: "DateRangeFilter2",
			docs: []doc{
				{ID: "1", D: now().Add(1 * time.Hour)},
				{ID: "2", D: now().Add(-1 * time.Hour)},
			},
			opts: &search.Options{Filters: []search.Filter{
				&search.DateRangeFilter{
					Field: "d",
					Min:   now(),
				},
			}},
			wantHits: []doc{{ID: "1", D: now().Add(1 * time.Hour)}},
		},
	})
}

func TestSearchFilterBool(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name: "True",
			docs: []doc{
				{ID: "1", Bool: true},
				{ID: "2", Bool: false},
				{ID: "3", Bool: true},
			},
			opts: &search.Options{Sort: "id", Filters: []search.Filter{
				&search.BoolFilter{
					Field: "bool",
					Value: true,
				},
			}},
			wantHits: []doc{
				{ID: "1", Bool: true},
				{ID: "3", Bool: true},
			},
		}, {
			name: "False",
			docs: []doc{
				{ID: "1", Bool: true},
				{ID: "2", Bool: false},
				{ID: "3", Bool: true},
			},
			opts: &search.Options{Sort: "id", Filters: []search.Filter{
				&search.BoolFilter{
					Field: "bool",
					Value: false,
				},
			}},
			wantHits: []doc{
				{ID: "2", Bool: false},
			},
		},
	})
}

func TestSearchFilterNumericRange(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name: "Min",
			docs: []doc{
				{ID: "1", Int: 1},
				{ID: "2", Int: 2},
				{ID: "3", Int: 3},
			},
			opts: &search.Options{Sort: "id", Filters: []search.Filter{
				&search.NumericRangeFilter{
					Field: "Int",
					Min:   f64ptr(2),
				},
			}},
			wantHits: []doc{
				{ID: "2", Int: 2},
				{ID: "3", Int: 3},
			},
		},
	})
}

func TestSearchFilterMultiple(t *testing.T) {
	testSearchTestcases(t, []testcase{
		{
			name: "MultipleFilters",
			docs: []doc{
				{ID: "1", A: "f", C: "foo"},
				{ID: "2", A: "b", C: "bar"},
				{ID: "3", A: "a", C: "baz"},
			},
			opts: &search.Options{Filters: []search.Filter{
				&search.TermsFilter{
					Field: "a",
					Terms: []string{"f"},
				},
				&search.TermsFilter{
					Field: "c",
					Terms: []string{"foo"},
				},
			}},
			wantHits: []doc{
				{ID: "1", A: "f", C: "foo"},
			},
		},
	})
}

func TestSearchAggregation(t *testing.T) {
	time1 := time.Date(2019, 1, 12, 20, 0, 0, 0, time.UTC)
	time2 := time.Date(2019, 2, 12, 20, 0, 0, 0, time.UTC)
	testSearchTestcases(t, []testcase{
		{
			name: "Terms",
			docs: []doc{
				{ID: "1", C: "foo"},
				{ID: "2", C: "bar"},
				{ID: "3", C: "bar"},
			},
			opts: &search.Options{Aggregations: map[string]search.Aggregation{
				"foo": {
					Field: "c",
					Type:  search.TermsAggregation,
				},
			}},
			wantBuckets: map[string]search.Bucket{
				"foo": search.TermsBucket([]search.Term{
					{Term: "bar", Count: 2},
					{Term: "foo", Count: 1},
				}),
			},
		}, {
			name: "DateRange",
			docs: []doc{{ID: "1", D: time1}, {ID: "2", D: time2}},
			opts: &search.Options{Aggregations: map[string]search.Aggregation{
				"foo": {
					Field: "d",
					Type:  search.DateRangeAggregation,
				},
			}},
			wantBuckets: map[string]search.Bucket{
				"foo": search.DateRangeBucket{
					Min: time1,
					Max: time2,
				},
			},
		}, {
			name: "WithTermsFilter",
			docs: []doc{{ID: "1", A: "f", C: "foo"}, {ID: "2", A: "b", C: "bar"}},
			opts: &search.Options{Aggregations: map[string]search.Aggregation{
				"foo": {
					Field: "c",
					Type:  search.TermsAggregation,
					Filters: []search.Filter{
						&search.TermsFilter{
							Field: "a",
							Terms: []string{"f"},
						},
					},
				},
			}},
			wantBuckets: map[string]search.Bucket{
				"foo": search.TermsBucket([]search.Term{
					{Term: "foo", Count: 1},
				}),
			},
		},
	})
}

type testcase struct {
	name        string
	docs        []doc
	query       string
	opts        *search.Options
	wantHits    []doc
	wantBuckets map[string]search.Bucket
	wantTotal   *uint64
}

func testSearchTestcases(t *testing.T, testcases []testcase) {
	t.Helper()
	for _, tc := range testcases {
		t.Run(tc.name, func(tt *testing.T) {
			index, cleanupIndex := test.CreateBleveTestIndex(tc.name)
			tt.Cleanup(cleanupIndex)
			s, err := search.NewService(index, time.Second, 1, 1, time.UTC)
			if err != nil {
				tt.Errorf("search.NewService unexpectedly failed: %s", err)
			}
			defer s.Stop()

			for _, doc := range tc.docs {
				err = s.Index(&doc)
				if err != nil {
					tt.Errorf("Index() unexpectedly failed: %s", err)
				}
			}

			if tc.opts == nil {
				tc.opts = &search.Options{}
			}
			tc.opts.Query = tc.query
			result, err := s.Search(context.Background(), tc.opts)
			if err != nil {
				tt.Errorf("Search(%s, opts) unexpectedly failed: %s", tc.query, err)
			}

			if tc.wantHits != nil {
				hits := []doc{}
				for _, rawHit := range result.Hits {
					var hit doc
					err = rawHit.Unmarshal(&hit)
					if err != nil {
						tt.Errorf("hit.Unmarshal unexpectedly failed: %s", err)
					}
					hits = append(hits, hit)
				}
				if fmt.Sprint(hits) != fmt.Sprint(tc.wantHits) {
					tt.Errorf("hits don't match:\nwant: %v\ngot:  %v", tc.wantHits, hits)
				}
			}

			if tc.wantBuckets != nil {
				if fmt.Sprint(result.Buckets) != fmt.Sprint(tc.wantBuckets) {
					tt.Errorf("buckets don't match: \nwant: %v\ngot:  %v", tc.wantBuckets, result.Buckets)
				}
			}

			if tc.wantTotal != nil {
				if *tc.wantTotal != result.Total {
					tt.Errorf("total don't match: want %d, got %d", &tc.wantTotal, result.Total)
				}
			}
		})
	}
}

func TestIndexConcurrently(t *testing.T) {
	index, cleanupIndex := test.CreateBleveTestIndex("TestIndexConcurrently")
	t.Cleanup(cleanupIndex)
	s, err := search.NewService(index, time.Second, 1, 1, time.UTC)
	if err != nil {
		t.Fatalf("search.NewService() failed unexpectedly: %s", err)
	}
	defer s.Stop()

	wg := sync.WaitGroup{}

	for _, d := range makeDocs(10) {
		dd := &d
		wg.Add(1)
		go func(i search.Indexable) {
			s.Index(i)
			wg.Done()
		}(dd)
	}

	wg.Wait()
}

func TestSearchServiceWithExistingIndex(t *testing.T) {
	index, cleanupIndex := test.CreateBleveTestIndex("TestSearchServiceWithExistingIndex")
	t.Cleanup(cleanupIndex)

	s, err := search.NewService(index, time.Second, 1, 1, time.UTC)
	if err != nil {
		t.Fatalf("search.NewService() failed unexpectedly: %s", err)
	}
	s.Stop()

	s, err = search.NewService(index, time.Second, 1, 1, time.UTC)
	if err != nil {
		t.Fatalf("search.NewService() failed with existing index: %s", err)
	}
	s.Stop()
}

func TestSearchCancel(t *testing.T) {
	index, cleanupIndex := test.CreateBleveTestIndex("TestSearchCancel")
	t.Cleanup(cleanupIndex)

	s, err := search.NewService(index, time.Second*10, 1, 1, time.UTC)
	if err != nil {
		t.Fatalf("search.NewService() failed unexpectedly: %s", err)
	}
	t.Cleanup(s.Stop)
	// Add documents to the index, such that a search is not instant.
	for i := 0; i < 100; i++ {
		s.Index(&doc{ID: fmt.Sprintf("%d", i), A: "f", C: "foo"})
	}

	ctx, cancel := context.WithCancel(context.Background())
	go func() { cancel() }()
	_, err = s.Search(ctx, nil)
	if err == nil || err.Error() != "context canceled" {
		t.Errorf("err should be 'context canceled', got: %s", err)
	}
}

func TestSearchCancelTimeout(t *testing.T) {
	index, cleanupIndex := test.CreateBleveTestIndex("TestSearchCancelTimeout")
	t.Cleanup(cleanupIndex)

	s, err := search.NewService(index, time.Nanosecond, 1, 1, time.UTC)
	if err != nil {
		t.Fatalf("search.NewService() failed unexpectedly: %s", err)
	}
	t.Cleanup(s.Stop)

	_, err = s.Search(context.Background(), nil)
	if err == nil || err.Error() != "context deadline exceeded" {
		t.Errorf("err should be 'context deadline exceeded', got: %s", err)
	}
}

func makeDocs(count int) []doc {
	docs := make([]doc, 0, count)
	for i := 0; i < count; i++ {
		docs = append(docs, doc{
			A: "djasd",
			D: time.Unix(0, 0),
		})
	}
	return docs
}

func makeAggregations(count int) map[string]search.Aggregation {
	aggregations := make(map[string]search.Aggregation)
	for i := 0; i < count; i++ {
		aggregations[strconv.Itoa(i)+"_terms"] = search.Aggregation{
			Type:  search.TermsAggregation,
			Field: "a",
		}
		aggregations[strconv.Itoa(i)+"_daterange"] = search.Aggregation{
			Type:  search.DateRangeAggregation,
			Field: "d",
		}
	}
	return aggregations
}

func BenchmarkSearch(b *testing.B) {
	benchcases := []struct {
		name  string
		docs  []doc
		query string
		opts  *search.Options
	}{
		{
			name:  "Normal",
			docs:  makeDocs(1000),
			query: "",
		}, {
			name:  "AggregationSmall",
			docs:  makeDocs(100),
			query: "",
			opts: &search.Options{
				Aggregations: makeAggregations(2),
			},
		}, {
			name:  "AggregationBig",
			docs:  makeDocs(100),
			query: "",
			opts: &search.Options{
				Aggregations: makeAggregations(100),
			},
		},
	}

	for _, bc := range benchcases {
		b.Run(bc.name, func(bb *testing.B) {
			index, cleanupIndex := test.CreateBleveTestIndex(bc.name)
			bb.Cleanup(cleanupIndex)

			s, err := search.NewService(index, time.Second, 1, 1, time.UTC)
			if err != nil {
				bb.Fatalf("search.NewService() failed unexpectedly: %s", err)
			}
			defer s.Stop()

			for _, doc := range bc.docs {
				err = s.Index(&doc)
				if err != nil {
					bb.Fatalf("Index() unexpectedly failed: %s", err)
				}
			}

			bb.ResetTimer()
			for n := 0; n < bb.N; n++ {
				bc.opts.Query = bc.query
				_, err := s.Search(context.Background(), bc.opts)
				if err != nil {
					bb.Fatalf("Search(%s, opts) unexpectedly failed: %s", "foo", err)
				}
			}
		})
	}
}

func now() time.Time {
	n := time.Now()
	return time.Date(n.Year(), n.Month(), n.Hour(), n.Minute(), 0, 0, 0, time.UTC)
}

func f64ptr(f float64) *float64 {
	return &f
}