github.com/keybase/client/go@v0.0.0-20240309051027-028f7c731f8b/chat/search/utils.go

package search

import (
	"context"
	"fmt"
	"regexp"
	"strings"

	"github.com/araddon/dateparse"
	mapset "github.com/deckarep/golang-set"
	"github.com/keybase/client/go/chat/globals"
	"github.com/keybase/client/go/chat/utils"
	"github.com/keybase/client/go/protocol/chat1"
	"github.com/keybase/client/go/protocol/gregor1"
	porterstemmer "github.com/keybase/go-porterstemmer"
)

// Split on whitespace, punctuation, code and quote markdown separators
var splitExpr = regexp.MustCompile(`[\s\.,\?!]`)

// Strip the following separators to create tokens
var stripSeps = []string{
	// groupings
	"<", ">",
	"\\(", "\\)",
	"\\[", "\\]",
	"\\{", "\\}",
	"\"",
	"'",
	// phone number delimiter
	"-",
	// mentions
	"@",
	"#",
	// markdown
	"\\*",
	"_",
	"~",
	"`",
}
var stripExpr = regexp.MustCompile(strings.Join(stripSeps, "|"))

func prefixes(token string) (res []string) {
	if len(token) < MinTokenLength {
		return nil
	}
	for i := range token {
		if i < MinTokenLength {
			continue
		}
		// Skip any prefixes longer than `maxPrefixLength` to limit the index size.
		if i > maxPrefixLength {
			break
		}
		res = append(res, token[:i])
	}
	return res
}

type tokenMap map[string]map[string]chat1.EmptyStruct

// getIndexTokens splits the content of the given message on whitespace and
// special characters returning a map of tokens to aliases  normalized to lowercase.
func tokenize(msgText string) tokenMap {
	if msgText == "" {
		return nil
	}

	// split the message text up on basic punctuation/spaces
	tokens := splitExpr.Split(msgText, -1)
	tokenMap := tokenMap{}
	for _, token := range tokens {
		if len(token) < MinTokenLength {
			continue
		}

		token = strings.ToLower(token)
		if _, ok := tokenMap[token]; !ok {
			tokenMap[token] = map[string]chat1.EmptyStruct{}
		}

		// strip separators to raw tokens which we count as an alias to the
		// original token
		stripped := stripExpr.Split(token, -1)
		for _, s := range stripped {
			if s == "" {
				continue
			}
			tokenMap[token][s] = chat1.EmptyStruct{}

			// add the stem as an alias
			stemmed := porterstemmer.StemWithoutLowerCasing([]rune(s))
			tokenMap[token][string(stemmed)] = chat1.EmptyStruct{}

			// calculate prefixes to alias to the token
			for _, prefix := range prefixes(s) {
				tokenMap[token][prefix] = chat1.EmptyStruct{}
			}
		}
		// drop the original token from the set of aliases
		delete(tokenMap[token], token)
	}
	return tokenMap
}

func tokensFromMsg(msg chat1.MessageUnboxed) tokenMap {
	return tokenize(msg.SearchableText())
}

func msgIDsFromSet(set mapset.Set) []chat1.MessageID {
	if set == nil {
		return nil
	}
	msgIDSlice := []chat1.MessageID{}
	for _, el := range set.ToSlice() {
		msgID, ok := el.(chat1.MessageID)
		if ok {
			msgIDSlice = append(msgIDSlice, msgID)
		}
	}
	return msgIDSlice
}

func searchMatches(msg chat1.MessageUnboxed, queryRe *regexp.Regexp) (validMatches []chat1.ChatSearchMatch) {
	msgText := msg.SearchableText()
	matches := queryRe.FindAllStringIndex(msgText, -1)
	for _, m := range matches {
		if len(m) != 2 {
			// sanity check but regex package should always return a two
			// element slice
			continue
		}
		startIndex := m[0]
		endIndex := m[1]
		if startIndex != endIndex {
			validMatches = append(validMatches, chat1.ChatSearchMatch{
				StartIndex: startIndex,
				EndIndex:   endIndex,
				Match:      msgText[startIndex:endIndex],
			})
		}
	}
	return validMatches
}

// Order messages ascending by ID for presentation
func getUIMsgs(ctx context.Context, g *globals.Context, convID chat1.ConversationID,
	uid gregor1.UID, msgs []chat1.MessageUnboxed) (uiMsgs []chat1.UIMessage) {
	for i := len(msgs) - 1; i >= 0; i-- {
		msg := msgs[i]
		uiMsg := utils.PresentMessageUnboxed(ctx, g, msg, uid, convID)
		uiMsgs = append(uiMsgs, uiMsg)
	}
	return uiMsgs
}

const beforeFilter = "before:"
const afterFilter = "after:"
const fromFilter = "from:"
const toFilter = "to:"

var senderRegex = regexp.MustCompile(fmt.Sprintf(
	"(%s|%s)(@?[a-z0-9][a-z0-9_]+)", fromFilter, toFilter))
var dateRangeRegex = regexp.MustCompile(fmt.Sprintf(
	`(%s|%s)(\d{1,4}[-/\.]+\d{1,2}[-/\.]+\d{1,4})`, beforeFilter, afterFilter))

func UpgradeSearchOptsFromQuery(query string, opts chat1.SearchOpts, username string) (string, chat1.SearchOpts) {
	query = strings.Trim(query, " ")
	var hasQueryOpts bool

	// To/From
	matches := senderRegex.FindAllStringSubmatch(query, 2)
	for _, match := range matches {
		// [fullMatch, filter, sender]
		if len(match) != 3 {
			continue
		}
		hasQueryOpts = true
		query = strings.TrimSpace(strings.ReplaceAll(query, match[0], ""))
		sender := strings.TrimSpace(strings.ReplaceAll(match[2], "@", ""))
		if sender == "me" {
			sender = username
		}
		switch match[1] {
		case fromFilter:
			opts.SentBy = sender
		case toFilter:
			opts.SentTo = sender
		}
	}
	if opts.SentTo == username {
		opts.MatchMentions = true
	}

	matches = dateRangeRegex.FindAllStringSubmatch(query, 2)
	for _, match := range matches {
		// [fullMatch, filter, dateRange]
		if len(match) != 3 {
			continue
		}
		hasQueryOpts = true
		query = strings.TrimSpace(strings.Replace(query, match[0], "", 1))
		time, err := dateparse.ParseAny(strings.TrimSpace(match[2]))
		if err != nil {
			continue
		}

		gtime := gregor1.ToTime(time)
		switch match[1] {
		case beforeFilter:
			opts.SentBefore = gtime
		case afterFilter:
			opts.SentAfter = gtime
		}
	}

	if hasQueryOpts && len(query) == 0 {
		query = "/.*/"
	}
	// IsRegex
	if len(query) > 2 && query[0] == '/' && query[len(query)-1] == '/' {
		query = query[1 : len(query)-1]
		opts.IsRegex = true
	}
	return query, opts
}

func MinMaxIDs(conv chat1.Conversation) (min, max chat1.MessageID) {
	// lowest msgID we care about
	min = conv.GetMaxDeletedUpTo()
	if min == 0 {
		min = 1
	}
	// highest msgID we care about
	max = conv.GetMaxMessageID()
	return min, max
}