// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. package app import ( "fmt" "maps" "strings" "testing" "github.com/mattermost/mattermost/server/public/model" "github.com/stretchr/testify/assert" ) // withFillerKeywords returns the given keywords along with count additional single-byte keywords // that are not expected to match anything. func withFillerKeywords(keywords map[string][]string, count int) map[string][]string { result := make(map[string][]string, len(keywords)+count) maps.Copy(result, keywords) filler := model.NewId() for i := range count { result[fmt.Sprintf("keyword%d", i)] = []string{filler} } return result } func TestMakeStandardMentionParser(t *testing.T) { mainHelper.Parallel(t) t.Run("should precompute only the multibyte keywords", func(t *testing.T) { id := model.NewId() p := makeStandardMentionParser(mapsToMentionKeywords(map[string][]string{ "apple": {id}, "banana": {id}, "番茄": {id}, "世界": {id}, "café": {id}, }, nil)) assert.ElementsMatch(t, []string{"番茄", "世界", "café"}, p.multibyteKeywords) }) t.Run("should precompute nothing when no keyword is multibyte", func(t *testing.T) { id := model.NewId() p := makeStandardMentionParser(mapsToMentionKeywords(map[string][]string{ "apple": {id}, "banana": {id}, }, nil)) assert.Empty(t, p.multibyteKeywords) }) } func TestIsKeywordMultibyte(t *testing.T) { mainHelper.Parallel(t) id1 := model.NewId() id2 := model.NewId() for name, tc := range map[string]struct { Message string Attachments []*model.MessageAttachment Keywords map[string][]string Groups map[string]*model.Group Expected *MentionResults }{ "MultibyteCharacter": { Message: "My name is 萌", Keywords: map[string][]string{"萌": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "MultibyteCharacterWithNoUser": { Message: "My name is 萌", Keywords: map[string][]string{"萌": {}}, Expected: &MentionResults{ Mentions: nil, }, }, "MultibyteCharacterAtBeginningOfSentence": { Message: "이메일을 보내다.", Keywords: map[string][]string{"이메일": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "MultibyteCharacterAtBeginningOfSentenceWithNoUser": { Message: "이메일을 보내다.", Keywords: map[string][]string{"이메일": {}}, Expected: &MentionResults{ Mentions: nil, }, }, "MultibyteCharacterInPartOfSentence": { Message: "我爱吃番茄炒饭", Keywords: map[string][]string{"番茄": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "MultibyteCharacterInPartOfSentenceWithNoUser": { Message: "我爱吃番茄炒饭", Keywords: map[string][]string{"番茄": {}}, Expected: &MentionResults{ Mentions: nil, }, }, "MultibyteCharacterAtEndOfSentence": { Message: "こんにちは、世界", Keywords: map[string][]string{"世界": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "MultibyteCharacterAtEndOfSentenceWithNoUser": { Message: "こんにちは、世界", Keywords: map[string][]string{"世界": {}}, Expected: &MentionResults{ Mentions: nil, }, }, "MultibyteCharacterTwiceInSentence": { Message: "石橋さんが石橋を渡る", Keywords: map[string][]string{"石橋": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "MultibyteCharacterTwiceInSentenceWithNoUser": { Message: "石橋さんが石橋を渡る", Keywords: map[string][]string{"石橋": {}}, Expected: &MentionResults{ Mentions: nil, }, }, "MultibyteCharacterAlongsideManySingleByteKeywords": { Message: "我爱吃番茄炒饭", Keywords: withFillerKeywords(map[string][]string{"番茄": {id1}}, 500), Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "MultibyteCharacterWithOnlySingleByteKeywords": { Message: "我爱吃番茄炒饭", Keywords: map[string][]string{"tomato": {id1}}, Expected: &MentionResults{ Mentions: nil, }, }, "SingleByteWordWithOnlyMultibyteKeywords": { Message: "the quick brown fox", Keywords: map[string][]string{"番茄": {id1}}, Expected: &MentionResults{ Mentions: nil, }, }, "MultipleMultibyteKeywordsWhereOnlyOneMatches": { Message: "我爱吃番茄炒饭", Keywords: map[string][]string{"番茄": {id1}, "世界": {id2}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, } { t.Run(name, func(t *testing.T) { post := &model.Post{ Message: tc.Message, Props: model.StringInterface{ model.PostPropsAttachments: tc.Attachments, }, } m := getExplicitMentions(post, mapsToMentionKeywords(tc.Keywords, tc.Groups), true) assert.EqualValues(t, tc.Expected, m) }) } } func TestCheckForMentionUsers(t *testing.T) { mainHelper.Parallel(t) id1 := model.NewId() id2 := model.NewId() for name, tc := range map[string]struct { Word string Attachments []*model.MessageAttachment Keywords map[string][]string Expected *MentionResults }{ "Nobody": { Word: "nothing", Keywords: map[string][]string{}, Expected: &MentionResults{}, }, "UppercaseUser1": { Word: "@User", Keywords: map[string][]string{"@user": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "LowercaseUser1": { Word: "@user", Keywords: map[string][]string{"@user": {id1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id1: KeywordMention, }, }, }, "LowercaseUser2": { Word: "@user2", Keywords: map[string][]string{"@user2": {id2}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id2: KeywordMention, }, }, }, "UppercaseUser2": { Word: "@UsEr2", Keywords: map[string][]string{"@user2": {id2}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ id2: KeywordMention, }, }, }, "HereMention": { Word: "@here", Expected: &MentionResults{ HereMentioned: true, }, }, "ChannelMention": { Word: "@channel", Expected: &MentionResults{ ChannelMentioned: true, }, }, "AllMention": { Word: "@all", Expected: &MentionResults{ AllMentioned: true, }, }, "UppercaseHere": { Word: "@HeRe", Expected: &MentionResults{ HereMentioned: true, }, }, "UppercaseChannel": { Word: "@ChaNNel", Expected: &MentionResults{ ChannelMentioned: true, }, }, "UppercaseAll": { Word: "@ALL", Expected: &MentionResults{ AllMentioned: true, }, }, } { t.Run(name, func(t *testing.T) { p := makeStandardMentionParser(mapsToMentionKeywords(tc.Keywords, nil)) p.checkForMention(tc.Word) assert.EqualValues(t, tc.Expected, p.Results()) }) } } func TestCheckForMentionGroups(t *testing.T) { mainHelper.Parallel(t) groupID1 := model.NewId() groupID2 := model.NewId() for name, tc := range map[string]struct { Word string Groups map[string]*model.Group Expected *MentionResults }{ "No groups": { Word: "nothing", Groups: map[string]*model.Group{}, Expected: &MentionResults{}, }, "No matching groups": { Word: "nothing", Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{}, }, "matching group with no @": { Word: "engineering", Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{}, }, "matching group with preceding @": { Word: "@engineering", Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ GroupMentions: map[string]MentionType{ groupID1: GroupMention, }, }, }, "matching upper case group with preceding @": { Word: "@Engineering", Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ GroupMentions: map[string]MentionType{ groupID1: GroupMention, }, }, }, } { t.Run(name, func(t *testing.T) { p := makeStandardMentionParser(mapsToMentionKeywords(nil, tc.Groups)) p.checkForMention(tc.Word) mr := p.Results() assert.EqualValues(t, tc.Expected, mr) }) } } func TestProcessText(t *testing.T) { mainHelper.Parallel(t) userID1 := model.NewId() groupID1 := model.NewId() groupID2 := model.NewId() for name, tc := range map[string]struct { Text string Keywords map[string][]string Groups map[string]*model.Group Expected *MentionResults }{ "Mention user in text": { Text: "hello user @user1", Keywords: map[string][]string{"@user1": {userID1}}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ Mentions: map[string]MentionType{ userID1: KeywordMention, }, }, }, "Mention user after ending a sentence with full stop": { Text: "hello user.@user1", Keywords: map[string][]string{"@user1": {userID1}}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ Mentions: map[string]MentionType{ userID1: KeywordMention, }, }, }, "Mention user after hyphen": { Text: "hello user-@user1", Keywords: map[string][]string{"@user1": {userID1}}, Expected: &MentionResults{ Mentions: map[string]MentionType{ userID1: KeywordMention, }, }, }, "Mention user after colon": { Text: "hello user:@user1", Keywords: map[string][]string{"@user1": {userID1}}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ Mentions: map[string]MentionType{ userID1: KeywordMention, }, }, }, "Mention here after colon": { Text: "hello all:@here", Keywords: map[string][]string{}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ HereMentioned: true, }, }, "Mention all after hyphen": { Text: "hello all-@all", Keywords: map[string][]string{}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ AllMentioned: true, }, }, "Mention channel after full stop": { Text: "hello channel.@channel", Keywords: map[string][]string{}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ ChannelMentioned: true, }, }, "Mention other potential users or system calls": { Text: "hello @potentialuser and @otherpotentialuser", Keywords: map[string][]string{}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ OtherPotentialMentions: []string{"potentialuser", "otherpotentialuser"}, }, }, "Mention a real user and another potential user": { Text: "@user1, you can use @systembot to get help", Keywords: map[string][]string{"@user1": {userID1}}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ Mentions: map[string]MentionType{ userID1: KeywordMention, }, OtherPotentialMentions: []string{"systembot"}, }, }, "Mention a group": { Text: "@engineering", Keywords: map[string][]string{"@user1": {userID1}}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ GroupMentions: map[string]MentionType{groupID1: GroupMention}, }, }, "Mention a real user and another potential user and a group": { Text: "@engineering @user1, you can use @systembot to get help from", Keywords: map[string][]string{"@user1": {userID1}}, Groups: map[string]*model.Group{ groupID1: {Id: groupID1, Name: new("engineering")}, groupID2: {Id: groupID2, Name: new("developers")}, }, Expected: &MentionResults{ Mentions: map[string]MentionType{ userID1: KeywordMention, }, GroupMentions: map[string]MentionType{groupID1: GroupMention}, OtherPotentialMentions: []string{"systembot"}, }, }, } { t.Run(name, func(t *testing.T) { p := makeStandardMentionParser(mapsToMentionKeywords(tc.Keywords, tc.Groups)) p.ProcessText(tc.Text) assert.EqualValues(t, tc.Expected, p.Results()) }) } } // makeBenchmarkKeywords returns count single-byte keywords, optionally alongside one multibyte // keyword, all belonging to a single user. func makeBenchmarkKeywords(count int, multibyteKeyword string) MentionKeywords { keywords := make(MentionKeywords, count+1) id := mentionableUserID(model.NewId()) for i := range count { keywords[fmt.Sprintf("keyword%d", i)] = []MentionableID{id} } if multibyteKeyword != "" { keywords[multibyteKeyword] = []MentionableID{id} } return keywords } func BenchmarkGetExplicitMentions(b *testing.B) { // Roughly 1800 words, matching the scale of a long message in a busy channel. asciiMessage := strings.Repeat("the quick brown fox jumps over the lazy dog ", 200) multibyteMessage := strings.Repeat("こんにちは、世界 the quick brown fox ", 200) for _, tc := range []struct { name string message string multibyteKeyword string }{ {name: "ascii post, no multibyte keywords", message: asciiMessage}, {name: "ascii post, one multibyte keyword", message: asciiMessage, multibyteKeyword: "世界"}, {name: "multibyte post, one multibyte keyword", message: multibyteMessage, multibyteKeyword: "世界"}, } { for _, numKeywords := range []int{10, 1000, 10000, 55000} { b.Run(fmt.Sprintf("%s/keywords=%d", tc.name, numKeywords), func(b *testing.B) { keywords := makeBenchmarkKeywords(numKeywords, tc.multibyteKeyword) post := &model.Post{Message: tc.message} b.ReportAllocs() for b.Loop() { getExplicitMentions(post, keywords, true) } }) } } }