web/forgejo
9
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-06-21 00:37:13 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Merge pull request 'enh(issue search): sort by score and term based query for fuzzy search' (#5819) from snematoda/enh-issue-search into forgejo

Browse source 
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5819
Reviewed-by: Otto <otto@codeberg.org>
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
This commit is contained in:
Otto 2024-11-10 12:51:55 +00:00
commit ab36ab57e4
11 changed files with 76 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

9
modules/indexer/internal/bleve/query.go
View file

@ -19,6 +19,15 @@ func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
return q
}
// MatchQuery generates a match query for the given phrase, field and analyzer
func MatchQuery(matchTerm, field, analyzer string, fuzziness int) *query.MatchQuery {
q := bleve.NewMatchQuery(matchTerm)
q.FieldVal = field
q.Analyzer = analyzer
q.Fuzziness = fuzziness
return q
}
// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
q := bleve.NewMatchPhraseQuery(matchPhrase)

33
modules/indexer/issues/bleve/bleve.go
View file

@ -35,13 +35,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
})
}
const (
maxBatchSize = 16
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
fuzzyDenominator = 4
// see https://github.com/blevesearch/bleve/issues/1563#issuecomment-786822311
maxFuzziness = 2
)
const maxBatchSize = 16
// IndexerData an update to the issue indexer
type IndexerData internal.IndexerData
@ -162,16 +156,25 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
var queries []query.Query
if options.Keyword != "" {
fuzziness := 0
if options.IsFuzzyKeyword {
fuzziness = min(maxFuzziness, len(options.Keyword)/fuzzyDenominator)
fuzziness := 1
if kl := len(options.Keyword); kl > 3 {
fuzziness = 2
} else if kl < 2 {
fuzziness = 0
}
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
}...))
} else {
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0),
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0),
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0),
}...))
}
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
}...))
}
if len(options.RepoIDs) > 0 || options.AllPublic {

4
modules/indexer/issues/dboptions.go
View file

@ -78,7 +78,9 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
searchOpt.Paginator = opts.Paginator
switch opts.SortType {
case "", "latest":
case "", "relevance":
searchOpt.SortBy = SortByScore
case "latest":
searchOpt.SortBy = SortByCreatedDesc
case "oldest":
searchOpt.SortBy = SortByCreatedAsc

2
modules/indexer/issues/elasticsearch/elasticsearch.go
View file

@ -236,7 +236,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
}
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
options.SortBy = internal.SortByScore
}
sortBy := []elastic.Sorter{
parseSortBy(options.SortBy),

1
modules/indexer/issues/indexer.go
View file

@ -269,6 +269,7 @@ func IsAvailable(ctx context.Context) bool {
type SearchOptions = internal.SearchOptions
const (
SortByScore = internal.SortByScore
SortByCreatedDesc = internal.SortByCreatedDesc
SortByUpdatedDesc = internal.SortByUpdatedDesc
SortByCommentsDesc = internal.SortByCommentsDesc

1
modules/indexer/issues/internal/model.go
View file

@ -127,6 +127,7 @@ func (o *SearchOptions) Copy(edit ...func(options *SearchOptions)) *SearchOption
type SortBy string
const (
SortByScore SortBy = "-_score"
SortByCreatedDesc SortBy = "-created_unix"
SortByUpdatedDesc SortBy = "-updated_unix"
SortByCommentsDesc SortBy = "-comment_count"

20
modules/indexer/issues/internal/tests/tests.go
View file

@ -126,6 +126,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello",
SortBy: internal.SortByCreatedDesc,
},
ExpectedIDs: []int64{1002, 1001, 1000},
ExpectedTotal: 3,
@ -139,6 +140,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello world",
SortBy: internal.SortByCreatedDesc,
IsFuzzyKeyword: true,
},
ExpectedIDs: []int64{1002, 1001, 1000},
@ -157,6 +159,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello",
SortBy: internal.SortByCreatedDesc,
RepoIDs: []int64{1, 4},
},
ExpectedIDs: []int64{1006, 1002, 1001},
@ -175,6 +178,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello",
SortBy: internal.SortByCreatedDesc,
RepoIDs: []int64{1, 4},
AllPublic: true,
},
@ -597,6 +601,22 @@ var cases = []*testIndexerCase{
}
},
},
{
Name: "SortByScore",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptionsAll,
SortBy: internal.SortByScore,
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Equal(t, len(data), len(result.Hits))
assert.Equal(t, len(data), int(result.Total))
for i, v := range result.Hits {
if i < len(result.Hits)-1 {
assert.GreaterOrEqual(t, v.Score, result.Hits[i+1].Score)
}
}
},
},
{
Name: "SortByCreatedAsc",
SearchOptions: &internal.SearchOptions{

18
modules/indexer/issues/meilisearch/meilisearch.go
View file

@ -208,12 +208,18 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
query.And(inner_meilisearch.NewFilterLte("updated_unix", options.UpdatedBeforeUnix.Value()))
}
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
}
sortBy := []string{
parseSortBy(options.SortBy),
"id:desc",
var sortBy []string
switch options.SortBy {
// sort by relevancy (no explicit sorting)
case internal.SortByScore:
fallthrough
case "":
sortBy = []string{}
default:
sortBy = []string{
parseSortBy(options.SortBy),
"id:desc",
}
}
skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits)

4
modules/templates/util_string.go
View file

@ -19,6 +19,10 @@ func NewStringUtils() *StringUtils {
return &stringUtils
}
func (su *StringUtils) Make(arr ...string) []string {
return arr
}
func (su *StringUtils) HasPrefix(s any, prefix string) bool {
switch v := s.(type) {
case string: