upstream-mattermost/server/channels/app/mention_parser_standard_test.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.

package app

import (
	"fmt"
	"maps"
	"strings"
	"testing"

	"github.com/mattermost/mattermost/server/public/model"
	"github.com/stretchr/testify/assert"
)

// withFillerKeywords returns the given keywords along with count additional single-byte keywords
// that are not expected to match anything.
func withFillerKeywords(keywords map[string][]string, count int) map[string][]string {
	result := make(map[string][]string, len(keywords)+count)
	maps.Copy(result, keywords)

	filler := model.NewId()
	for i := range count {
		result[fmt.Sprintf("keyword%d", i)] = []string{filler}
	}

	return result
}

func TestMakeStandardMentionParser(t *testing.T) {
	mainHelper.Parallel(t)

	t.Run("should precompute only the multibyte keywords", func(t *testing.T) {
		id := model.NewId()
		p := makeStandardMentionParser(mapsToMentionKeywords(map[string][]string{
			"apple":  {id},
			"banana": {id},
			"番茄":     {id},
			"世界":     {id},
			"café":   {id},
		}, nil))

		assert.ElementsMatch(t, []string{"番茄", "世界", "café"}, p.multibyteKeywords)
	})

	t.Run("should precompute nothing when no keyword is multibyte", func(t *testing.T) {
		id := model.NewId()
		p := makeStandardMentionParser(mapsToMentionKeywords(map[string][]string{
			"apple":  {id},
			"banana": {id},
		}, nil))

		assert.Empty(t, p.multibyteKeywords)
	})
}

func TestIsKeywordMultibyte(t *testing.T) {
	mainHelper.Parallel(t)
	id1 := model.NewId()
	id2 := model.NewId()

	for name, tc := range map[string]struct {
		Message     string
		Attachments []*model.MessageAttachment
		Keywords    map[string][]string
		Groups      map[string]*model.Group
		Expected    *MentionResults
	}{
		"MultibyteCharacter": {
			Message:  "My name is 萌",
			Keywords: map[string][]string{"萌": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"MultibyteCharacterWithNoUser": {
			Message:  "My name is 萌",
			Keywords: map[string][]string{"萌": {}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"MultibyteCharacterAtBeginningOfSentence": {
			Message:  "이메일을 보내다.",
			Keywords: map[string][]string{"이메일": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"MultibyteCharacterAtBeginningOfSentenceWithNoUser": {
			Message:  "이메일을 보내다.",
			Keywords: map[string][]string{"이메일": {}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"MultibyteCharacterInPartOfSentence": {
			Message:  "我爱吃番茄炒饭",
			Keywords: map[string][]string{"番茄": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"MultibyteCharacterInPartOfSentenceWithNoUser": {
			Message:  "我爱吃番茄炒饭",
			Keywords: map[string][]string{"番茄": {}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"MultibyteCharacterAtEndOfSentence": {
			Message:  "こんにちは、世界",
			Keywords: map[string][]string{"世界": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"MultibyteCharacterAtEndOfSentenceWithNoUser": {
			Message:  "こんにちは、世界",
			Keywords: map[string][]string{"世界": {}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"MultibyteCharacterTwiceInSentence": {
			Message:  "石橋さんが石橋を渡る",
			Keywords: map[string][]string{"石橋": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"MultibyteCharacterTwiceInSentenceWithNoUser": {
			Message:  "石橋さんが石橋を渡る",
			Keywords: map[string][]string{"石橋": {}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"MultibyteCharacterAlongsideManySingleByteKeywords": {
			Message:  "我爱吃番茄炒饭",
			Keywords: withFillerKeywords(map[string][]string{"番茄": {id1}}, 500),
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"MultibyteCharacterWithOnlySingleByteKeywords": {
			Message:  "我爱吃番茄炒饭",
			Keywords: map[string][]string{"tomato": {id1}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"SingleByteWordWithOnlyMultibyteKeywords": {
			Message:  "the quick brown fox",
			Keywords: map[string][]string{"番茄": {id1}},
			Expected: &MentionResults{
				Mentions: nil,
			},
		},
		"MultipleMultibyteKeywordsWhereOnlyOneMatches": {
			Message:  "我爱吃番茄炒饭",
			Keywords: map[string][]string{"番茄": {id1}, "世界": {id2}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
	} {
		t.Run(name, func(t *testing.T) {
			post := &model.Post{
				Message: tc.Message,
				Props: model.StringInterface{
					model.PostPropsAttachments: tc.Attachments,
				},
			}

			m := getExplicitMentions(post, mapsToMentionKeywords(tc.Keywords, tc.Groups), true)
			assert.EqualValues(t, tc.Expected, m)
		})
	}
}

func TestCheckForMentionUsers(t *testing.T) {
	mainHelper.Parallel(t)
	id1 := model.NewId()
	id2 := model.NewId()

	for name, tc := range map[string]struct {
		Word        string
		Attachments []*model.MessageAttachment
		Keywords    map[string][]string
		Expected    *MentionResults
	}{
		"Nobody": {
			Word:     "nothing",
			Keywords: map[string][]string{},
			Expected: &MentionResults{},
		},
		"UppercaseUser1": {
			Word:     "@User",
			Keywords: map[string][]string{"@user": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"LowercaseUser1": {
			Word:     "@user",
			Keywords: map[string][]string{"@user": {id1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id1: KeywordMention,
				},
			},
		},
		"LowercaseUser2": {
			Word:     "@user2",
			Keywords: map[string][]string{"@user2": {id2}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id2: KeywordMention,
				},
			},
		},
		"UppercaseUser2": {
			Word:     "@UsEr2",
			Keywords: map[string][]string{"@user2": {id2}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					id2: KeywordMention,
				},
			},
		},
		"HereMention": {
			Word: "@here",
			Expected: &MentionResults{
				HereMentioned: true,
			},
		},
		"ChannelMention": {
			Word: "@channel",
			Expected: &MentionResults{
				ChannelMentioned: true,
			},
		},
		"AllMention": {
			Word: "@all",
			Expected: &MentionResults{
				AllMentioned: true,
			},
		},
		"UppercaseHere": {
			Word: "@HeRe",
			Expected: &MentionResults{
				HereMentioned: true,
			},
		},
		"UppercaseChannel": {
			Word: "@ChaNNel",
			Expected: &MentionResults{
				ChannelMentioned: true,
			},
		},
		"UppercaseAll": {
			Word: "@ALL",
			Expected: &MentionResults{
				AllMentioned: true,
			},
		},
	} {
		t.Run(name, func(t *testing.T) {
			p := makeStandardMentionParser(mapsToMentionKeywords(tc.Keywords, nil))
			p.checkForMention(tc.Word)

			assert.EqualValues(t, tc.Expected, p.Results())
		})
	}
}

func TestCheckForMentionGroups(t *testing.T) {
	mainHelper.Parallel(t)
	groupID1 := model.NewId()
	groupID2 := model.NewId()

	for name, tc := range map[string]struct {
		Word     string
		Groups   map[string]*model.Group
		Expected *MentionResults
	}{
		"No groups": {
			Word:     "nothing",
			Groups:   map[string]*model.Group{},
			Expected: &MentionResults{},
		},
		"No matching groups": {
			Word: "nothing",
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{},
		},
		"matching group with no @": {
			Word: "engineering",
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{},
		},
		"matching group with preceding @": {
			Word: "@engineering",
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				GroupMentions: map[string]MentionType{
					groupID1: GroupMention,
				},
			},
		},
		"matching upper case group with preceding @": {
			Word: "@Engineering",
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				GroupMentions: map[string]MentionType{
					groupID1: GroupMention,
				},
			},
		},
	} {
		t.Run(name, func(t *testing.T) {
			p := makeStandardMentionParser(mapsToMentionKeywords(nil, tc.Groups))
			p.checkForMention(tc.Word)

			mr := p.Results()

			assert.EqualValues(t, tc.Expected, mr)
		})
	}
}

func TestProcessText(t *testing.T) {
	mainHelper.Parallel(t)
	userID1 := model.NewId()

	groupID1 := model.NewId()
	groupID2 := model.NewId()

	for name, tc := range map[string]struct {
		Text     string
		Keywords map[string][]string
		Groups   map[string]*model.Group
		Expected *MentionResults
	}{
		"Mention user in text": {
			Text:     "hello user @user1",
			Keywords: map[string][]string{"@user1": {userID1}},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					userID1: KeywordMention,
				},
			},
		},
		"Mention user after ending a sentence with full stop": {
			Text:     "hello user.@user1",
			Keywords: map[string][]string{"@user1": {userID1}},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					userID1: KeywordMention,
				},
			},
		},
		"Mention user after hyphen": {
			Text:     "hello user-@user1",
			Keywords: map[string][]string{"@user1": {userID1}},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					userID1: KeywordMention,
				},
			},
		},
		"Mention user after colon": {
			Text:     "hello user:@user1",
			Keywords: map[string][]string{"@user1": {userID1}},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					userID1: KeywordMention,
				},
			},
		},
		"Mention here after colon": {
			Text:     "hello all:@here",
			Keywords: map[string][]string{},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				HereMentioned: true,
			},
		},
		"Mention all after hyphen": {
			Text:     "hello all-@all",
			Keywords: map[string][]string{},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				AllMentioned: true,
			},
		},
		"Mention channel after full stop": {
			Text:     "hello channel.@channel",
			Keywords: map[string][]string{},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				ChannelMentioned: true,
			},
		},
		"Mention other potential users or system calls": {
			Text:     "hello @potentialuser and @otherpotentialuser",
			Keywords: map[string][]string{},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				OtherPotentialMentions: []string{"potentialuser", "otherpotentialuser"},
			},
		},
		"Mention a real user and another potential user": {
			Text:     "@user1, you can use @systembot to get help",
			Keywords: map[string][]string{"@user1": {userID1}},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					userID1: KeywordMention,
				},
				OtherPotentialMentions: []string{"systembot"},
			},
		},
		"Mention a group": {
			Text:     "@engineering",
			Keywords: map[string][]string{"@user1": {userID1}},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				GroupMentions: map[string]MentionType{groupID1: GroupMention},
			},
		},
		"Mention a real user and another potential user and a group": {
			Text:     "@engineering @user1, you can use @systembot to get help from",
			Keywords: map[string][]string{"@user1": {userID1}},
			Groups: map[string]*model.Group{
				groupID1: {Id: groupID1, Name: new("engineering")},
				groupID2: {Id: groupID2, Name: new("developers")},
			},
			Expected: &MentionResults{
				Mentions: map[string]MentionType{
					userID1: KeywordMention,
				},
				GroupMentions:          map[string]MentionType{groupID1: GroupMention},
				OtherPotentialMentions: []string{"systembot"},
			},
		},
	} {
		t.Run(name, func(t *testing.T) {
			p := makeStandardMentionParser(mapsToMentionKeywords(tc.Keywords, tc.Groups))
			p.ProcessText(tc.Text)

			assert.EqualValues(t, tc.Expected, p.Results())
		})
	}
}

// makeBenchmarkKeywords returns count single-byte keywords, optionally alongside one multibyte
// keyword, all belonging to a single user.
func makeBenchmarkKeywords(count int, multibyteKeyword string) MentionKeywords {
	keywords := make(MentionKeywords, count+1)

	id := mentionableUserID(model.NewId())
	for i := range count {
		keywords[fmt.Sprintf("keyword%d", i)] = []MentionableID{id}
	}

	if multibyteKeyword != "" {
		keywords[multibyteKeyword] = []MentionableID{id}
	}

	return keywords
}

func BenchmarkGetExplicitMentions(b *testing.B) {
	// Roughly 1800 words, matching the scale of a long message in a busy channel.
	asciiMessage := strings.Repeat("the quick brown fox jumps over the lazy dog ", 200)
	multibyteMessage := strings.Repeat("こんにちは、世界 the quick brown fox ", 200)

	for _, tc := range []struct {
		name             string
		message          string
		multibyteKeyword string
	}{
		{name: "ascii post, no multibyte keywords", message: asciiMessage},
		{name: "ascii post, one multibyte keyword", message: asciiMessage, multibyteKeyword: "世界"},
		{name: "multibyte post, one multibyte keyword", message: multibyteMessage, multibyteKeyword: "世界"},
	} {
		for _, numKeywords := range []int{10, 1000, 10000, 55000} {
			b.Run(fmt.Sprintf("%s/keywords=%d", tc.name, numKeywords), func(b *testing.B) {
				keywords := makeBenchmarkKeywords(numKeywords, tc.multibyteKeyword)
				post := &model.Post{Message: tc.message}

				b.ReportAllocs()
				for b.Loop() {
					getExplicitMentions(post, keywords, true)
				}
			})
		}
	}
}