snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 1s
Server CI Nightly Race / Build Environment (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-16 00:29:55 +05:00
commit 240db4e2a7
15309 changed files with 2173406 additions and 0 deletions

View File

@@ -0,0 +1,10 @@
with-expecter: false
packages:
github.com/mattermost/mattermost/server/v8/channels/store:
config:
include-regex: ".*Store"
dir: "channels/store/storetest/mocks"
filename: "{{.InterfaceName}}.go"
mockname: "{{.InterfaceName}}"
outpkg: mocks
note: "Regenerate this file using `make store-mocks`."

View File

@@ -0,0 +1,15 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package store
const (
ChannelExistsError = "store.sql_channel.save_channel.exists.app_error"
UserSearchOptionNamesOnly = "names_only"
UserSearchOptionNamesOnlyNoFullName = "names_only_no_full_name"
UserSearchOptionAllNoFullName = "all_no_full_name"
UserSearchOptionAllowInactive = "allow_inactive"
FeatureTogglePrefix = "feature_enabled_"
)

View File

@@ -0,0 +1,38 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package store
import (
"context"
"github.com/mattermost/mattermost/server/public/shared/request"
)
// storeContextKey is the base type for all context keys for the store.
type storeContextKey string
// contextValue is a type to hold some pre-determined context values.
type contextValue string
// Different possible values of contextValue.
const (
useMaster contextValue = "useMaster"
)
// RequestContextWithMaster adds the context value that master DB should be selected for this request.
func RequestContextWithMaster(rctx request.CTX) request.CTX {
ctx := context.WithValue(rctx.Context(), storeContextKey(useMaster), true)
rctx = rctx.WithContext(ctx)
return rctx
}
// HasMaster is a helper function to check whether master DB should be selected or not.
func HasMaster(ctx context.Context) bool {
if v := ctx.Value(storeContextKey(useMaster)); v != nil {
if res, ok := v.(bool); ok && res {
return true
}
}
return false
}

View File

@@ -0,0 +1,47 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package store
import (
"testing"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/stretchr/testify/assert"
)
func TestRequestContextWithMaster(t *testing.T) {
t.Run("set and get", func(t *testing.T) {
var rctx request.CTX = request.TestContext(t)
rctx = RequestContextWithMaster(rctx)
assert.True(t, HasMaster(rctx.Context()))
})
t.Run("values get copied from original context", func(t *testing.T) {
var rctx request.CTX = request.TestContext(t)
rctx = RequestContextWithMaster(rctx)
rctxCopy := rctx
assert.True(t, HasMaster(rctx.Context()))
assert.True(t, HasMaster(rctxCopy.Context()))
})
t.Run("directly assigning does not cause the copy to alter the original context", func(t *testing.T) {
var rctx request.CTX = request.TestContext(t)
rctxCopy := rctx
rctxCopy = RequestContextWithMaster(rctxCopy)
assert.False(t, HasMaster(rctx.Context()))
assert.True(t, HasMaster(rctxCopy.Context()))
})
t.Run("directly assigning does not cause the original context to alter the copy", func(t *testing.T) {
var rctx request.CTX = request.TestContext(t)
rctxCopy := rctx
rctx = RequestContextWithMaster(rctx)
assert.True(t, HasMaster(rctx.Context()))
assert.False(t, HasMaster(rctxCopy.Context()))
})
}

View File

@@ -0,0 +1,34 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package store
import "time"
// DatabaseDiagnostics is a snapshot of database health and pool state.
// Pointer fields are nil when the underlying metric is unavailable
// (non-Postgres driver, query failure, or no matching row).
type DatabaseDiagnostics struct {
MasterConnectionsInUse int
MasterConnectionsIdle int
MasterPoolWaitCount int64
MasterPoolWaitDurationMs int64
MasterConnectionsClosedMaxIdle int64
MasterConnectionsClosedMaxLifetime int64
ReplicaConnectionsInUse int
ReplicaConnectionsIdle int
ReplicaPoolWaitCount int64
ReplicaPoolWaitDurationMs int64
ReplicaConnectionsClosedMaxIdle int64
ReplicaConnectionsClosedMaxLifetime int64
CacheHitRatio *float64
Deadlocks *int64
TempFiles *int64
TempBytesMB *float64
Rollbacks *int64
IdleInTransactionCount *int64
LongestQueryDurationSeconds *float64
WaitingForLockCount *int64
PostsDeadTuples *int64
PostsLastAutovacuum *time.Time
}

View File

@@ -0,0 +1,132 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
/*
Package store provides the data persistence layer for Mattermost channels.
The store package implements a layered architecture that abstracts database operations
and provides a clean interface for data access. It follows the repository pattern with advanced features
like caching, search indexing, retry logic, and metrics collection.
# Architecture Overview
The store package uses a multi-layered architecture where each layer adds specific
functionality:
Application Layer (app/)
↓
Store Interface (store.Store)
↓
Timer Layer (metrics/timing)
↓
Retry Layer (deadlock/error handling)
↓
Local Cache Layer (in-memory caching)
↓
Search Layer (search indexing)
↓
SQL Store (database operations)
Each layer wraps the layer below it, following the decorator pattern to add
cross-cutting concerns without modifying the core business logic.
# Store Interface
The main Store interface provides access to all domain-specific stores:
type Store interface {
Team() TeamStore
Channel() ChannelStore
Post() PostStore
User() UserStore
// ... and many more
}
Each domain store interface defines operations for a specific entity type,
following CRUD patterns and domain-specific operations.
# Key Components
## SQL Store (sqlstore/)
The foundation layer that handles direct database operations:
- Connection management with master/replica support
- Query building using Squirrel
- Transaction handling
- Database migration support
- Schema management
## Local Cache Layer (localcachelayer/)
Provides in-memory caching for frequently accessed data:
- LRU cache implementation
- Cache invalidation strategies
- Cluster-aware cache invalidation
- Configurable cache sizes and TTLs
- Metrics collection for hit/miss ratios
## Search Layer (searchlayer/)
Integrates with search engines for full-text search:
- Elasticsearch integration
- Bleve search engine support
- Automatic indexing of content
- Search result ranking and filtering
## Retry Layer (retrylayer/)
Handles transient failures and database deadlocks:
- Automatic retry logic for database deadlocks
- Configurable retry policies
- Dead letter queue handling
## Timer Layer (timerlayer/)
Provides performance monitoring and metrics:
- Method execution timing
- Database query performance metrics
- Integration with Prometheus metrics
- Performance bottleneck identification
## Store Test Framework (storetest/)
Comprehensive testing infrastructure:
- Mock implementations for all store interfaces
- Test data factories and builders
- Database setup and teardown utilities
- Integration test helpers
## Cache Operations
The cache layer is transparent to callers but provides significant performance
benefits for read-heavy operations:
// This call may be served from cache
user, err := store.User().Get(rctx, userID)
// Cache is automatically invalidated on updates
updatedUser, err := store.User().Update(ctx, user, allowRoleUpdate)
## Error Handling
The store package defines custom error types for common scenarios:
// Check for specific error types
if err != nil {
var notFoundErr *store.ErrNotFound
if errors.As(err, &notFoundErr) {
// Handle not found case
}
var conflictErr *store.ErrConflict
if errors.As(err, &conflictErr) {
// Handle conflict case
}
}
# Migration and Schema
The store package includes a robust migration system:
- Version-controlled schema migrations
- Automatic migration execution
- Rollback support
- Schema validation
For detailed information about specific store implementations, see the
documentation in the respective subdirectories (sqlstore/, localcachelayer/, etc.).
*/
package store

View File

@@ -0,0 +1,205 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package store
import (
"fmt"
"strings"
)
// ErrInvalidInput indicates an error that has occurred due to an invalid input.
type ErrInvalidInput struct {
Entity string // The entity which was sent as the input.
Field string // The field of the entity which was invalid.
Value any // The actual value of the field.
wrapped error // The original error
}
func NewErrInvalidInput(entity, field string, value any) *ErrInvalidInput {
return &ErrInvalidInput{
Entity: entity,
Field: field,
Value: value,
}
}
func (e *ErrInvalidInput) Error() string {
if e.wrapped != nil {
return fmt.Sprintf("invalid input: entity: %s field: %s value: %s error: %s", e.Entity, e.Field, e.Value, e.wrapped)
}
return fmt.Sprintf("invalid input: entity: %s field: %s value: %s", e.Entity, e.Field, e.Value)
}
func (e *ErrInvalidInput) Wrap(err error) *ErrInvalidInput {
e.wrapped = err
return e
}
func (e *ErrInvalidInput) Unwrap() error {
return e.wrapped
}
func (e *ErrInvalidInput) InvalidInputInfo() (entity string, field string, value any) {
entity = e.Entity
field = e.Field
value = e.Value
return
}
// ErrLimitExceeded indicates an error that has occurred because some value exceeded a limit.
type ErrLimitExceeded struct {
What string // What was the object that exceeded.
Count int // The value of the object.
meta string // Any additional metadata.
}
func NewErrLimitExceeded(what string, count int, meta string) *ErrLimitExceeded {
return &ErrLimitExceeded{
What: what,
Count: count,
meta: meta,
}
}
func (e *ErrLimitExceeded) Error() string {
return fmt.Sprintf("limit exceeded: what: %s count: %d metadata: %s", e.What, e.Count, e.meta)
}
// ErrConflict indicates a conflict that occurred.
type ErrConflict struct {
Resource string // The resource which created the conflict.
err error // Internal error.
meta string // Any additional metadata.
}
func NewErrConflict(resource string, err error, meta string) *ErrConflict {
return &ErrConflict{
Resource: resource,
err: err,
meta: meta,
}
}
func (e *ErrConflict) Error() string {
msg := e.Resource + "exists " + e.meta
if e.err != nil {
msg += " " + e.err.Error()
}
return msg
}
func (e *ErrConflict) Unwrap() error {
return e.err
}
// IsErrConflict allows easy type assertion without adding store as a dependency.
func (e *ErrConflict) IsErrConflict() bool {
return true
}
// ErrNotFound indicates that a resource was not found
type ErrNotFound struct {
resource string
ID string
wrapped error
}
func NewErrNotFound(resource, id string) *ErrNotFound {
return &ErrNotFound{
resource: resource,
ID: id,
}
}
func (e *ErrNotFound) Wrap(err error) *ErrNotFound {
e.wrapped = err
return e
}
func (e *ErrNotFound) Error() string {
if e.wrapped != nil {
return fmt.Sprintf("resource %q not found, id: %s, error: %s", e.resource, e.ID, e.wrapped)
}
return fmt.Sprintf("resource %q not found, id: %s", e.resource, e.ID)
}
// IsErrNotFound allows easy type assertion without adding store as a dependency.
func (e *ErrNotFound) IsErrNotFound() bool {
return true
}
// ErrOutOfBounds indicates that the requested total numbers of rows
// was greater than the allowed limit.
type ErrOutOfBounds struct {
value int
}
func (e *ErrOutOfBounds) Error() string {
return fmt.Sprintf("invalid limit parameter: %d", e.value)
}
func NewErrOutOfBounds(value int) *ErrOutOfBounds {
return &ErrOutOfBounds{value: value}
}
// ErrNotImplemented indicates that some feature or requirement is not implemented yet.
type ErrNotImplemented struct {
detail string
}
func (e *ErrNotImplemented) Error() string {
return e.detail
}
func NewErrNotImplemented(detail string) *ErrNotImplemented {
return &ErrNotImplemented{detail: detail}
}
type ErrUniqueConstraint struct {
Columns []string
}
// NewErrUniqueConstraint creates a uniqueness constraint error for the given column(s).
//
// Examples:
//
// store.NewErrUniqueConstraint("DisplayName") // single column constraint
// store.NewErrUniqueConstraint("Name", "Source") // multi-column constraint
func NewErrUniqueConstraint(columns ...string) *ErrUniqueConstraint {
return &ErrUniqueConstraint{
Columns: columns,
}
}
func (e *ErrUniqueConstraint) Error() string {
var tmpl string
if len(e.Columns) > 1 {
tmpl = "unique constraint: (%s)"
} else {
tmpl = "unique constraint: %s"
}
return fmt.Sprintf(tmpl, strings.Join(e.Columns, ","))
}
func IsErrNotFound(err error) bool {
_, ok := err.(*ErrNotFound)
return ok
}
// ErrResultsMismatch indicates that a batch lookup returned fewer results
// than the number of IDs requested, meaning some IDs were not found.
type ErrResultsMismatch struct {
Got int
Expected int
}
func NewErrResultsMismatch(got, expected int) *ErrResultsMismatch {
return &ErrResultsMismatch{Got: got, Expected: expected}
}
func (e *ErrResultsMismatch) Error() string {
return fmt.Sprintf("results mismatch: got %d results of the %d ids passed", e.Got, e.Expected)
}

View File

@@ -0,0 +1,28 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package store
import (
"errors"
"testing"
"github.com/mattermost/mattermost/server/public/model"
"github.com/stretchr/testify/assert"
)
func TestErrNotFound(t *testing.T) {
id := model.NewId()
t.Run("plain", func(t *testing.T) {
err := NewErrNotFound("channel", id)
assert.EqualError(t, err, "resource \"channel\" not found, id: "+id)
})
t.Run("with wrapped error", func(t *testing.T) {
err := NewErrNotFound("channel", id)
err = err.Wrap(errors.New("some error"))
assert.EqualError(t, err, "resource \"channel\" not found, id: "+id+", error: some error")
})
}

View File

@@ -0,0 +1,277 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package main
import (
"bytes"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"io"
"log"
"os"
"path"
"slices"
"strings"
"text/template"
)
const (
ErrorType = "error"
)
func isError(typeName string) bool {
return strings.Contains(typeName, ErrorType)
}
func main() {
if err := buildTimerLayer(); err != nil {
log.Fatal(err)
}
if err := buildRetryLayer(); err != nil {
log.Fatal(err)
}
}
func buildRetryLayer() error {
code, err := generateLayer("RetryLayer", "retry_layer.go.tmpl")
if err != nil {
return err
}
formatedCode, err := format.Source(code)
if err != nil {
return err
}
return os.WriteFile(path.Join("retrylayer/retrylayer.go"), formatedCode, 0644)
}
func buildTimerLayer() error {
code, err := generateLayer("TimerLayer", "timer_layer.go.tmpl")
if err != nil {
return err
}
formatedCode, err := format.Source(code)
if err != nil {
return err
}
return os.WriteFile(path.Join("timerlayer", "timerlayer.go"), formatedCode, 0644)
}
type methodParam struct {
Name string
Type string
}
type methodData struct {
Params []methodParam
Results []string
}
type subStore struct {
Methods map[string]methodData
}
type storeMetadata struct {
Name string
SubStores map[string]subStore
Methods map[string]methodData
}
func extractMethodMetadata(method *ast.Field, src []byte) methodData {
params := []methodParam{}
results := []string{}
ast.Inspect(method.Type, func(expr ast.Node) bool {
switch e := expr.(type) {
case *ast.FuncType:
if e.Params != nil {
for _, param := range e.Params.List {
for _, paramName := range param.Names {
params = append(params, methodParam{Name: paramName.Name, Type: string(src[param.Type.Pos()-1 : param.Type.End()-1])})
}
}
}
if e.Results != nil {
for _, result := range e.Results.List {
results = append(results, string(src[result.Type.Pos()-1:result.Type.End()-1]))
}
}
}
return true
})
return methodData{Params: params, Results: results}
}
func extractStoreMetadata() (*storeMetadata, error) {
// Create the AST by parsing src.
fset := token.NewFileSet() // positions are relative to fset
file, err := os.Open("store.go")
if err != nil {
return nil, fmt.Errorf("unable to open store/store.go file: %w", err)
}
src, err := io.ReadAll(file)
if err != nil {
return nil, err
}
file.Close()
f, err := parser.ParseFile(fset, "", src, parser.AllErrors|parser.ParseComments)
if err != nil {
return nil, err
}
topLevelFunctions := map[string]bool{
"MarkSystemRanUnitTests": false,
"Close": false,
"LockToMaster": false,
"UnlockFromMaster": false,
"DropAllTables": false,
"TotalMasterDbConnections": true,
"TotalReadDbConnections": true,
"SetContext": true,
"TotalSearchDbConnections": true,
"GetCurrentSchemaVersion": true,
}
metadata := storeMetadata{Methods: map[string]methodData{}, SubStores: map[string]subStore{}}
ast.Inspect(f, func(n ast.Node) bool {
switch x := n.(type) {
case *ast.TypeSpec:
if x.Name.Name == "Store" {
for _, method := range x.Type.(*ast.InterfaceType).Methods.List {
methodName := method.Names[0].Name
if _, ok := topLevelFunctions[methodName]; ok {
metadata.Methods[methodName] = extractMethodMetadata(method, src)
}
}
} else if subStoreName, ok := strings.CutSuffix(x.Name.Name, "Store"); ok {
metadata.SubStores[subStoreName] = subStore{Methods: map[string]methodData{}}
for _, method := range x.Type.(*ast.InterfaceType).Methods.List {
methodName := method.Names[0].Name
metadata.SubStores[subStoreName].Methods[methodName] = extractMethodMetadata(method, src)
}
}
}
return true
})
return &metadata, nil
}
func generateLayer(name, templateFile string) ([]byte, error) {
out := bytes.NewBufferString("")
metadata, err := extractStoreMetadata()
if err != nil {
return nil, err
}
metadata.Name = name
myFuncs := template.FuncMap{
"joinResults": func(results []string) string {
return strings.Join(results, ", ")
},
"joinResultsForSignature": func(results []string) string {
if len(results) == 0 {
return ""
}
returns := []string{}
for _, result := range results {
switch result {
case "*PostReminderMetadata":
returns = append(returns, fmt.Sprintf("*store.%s", strings.TrimPrefix(result, "*")))
case "[]*ChannelGuard":
returns = append(returns, fmt.Sprintf("[]*store.%s", strings.TrimPrefix(result, "[]*")))
default:
returns = append(returns, result)
}
}
if len(returns) == 1 {
return strings.Join(returns, ", ")
}
return fmt.Sprintf("(%s)", strings.Join(returns, ", "))
},
"genResultsVars": func(results []string, withNilError bool) string {
vars := []string{}
for i, typeName := range results {
if isError(typeName) {
if withNilError {
vars = append(vars, "nil")
} else {
vars = append(vars, "err")
}
} else if i == 0 {
vars = append(vars, "result")
} else {
vars = append(vars, fmt.Sprintf("resultVar%d", i))
}
}
return strings.Join(vars, ", ")
},
"errorToBoolean": func(results []string) string {
if slices.ContainsFunc(results, isError) {
return "err == nil"
}
return "true"
},
"errorPresent": func(results []string) bool {
return slices.ContainsFunc(results, isError)
},
"errorVar": func(results []string) string {
if slices.ContainsFunc(results, isError) {
return "err"
}
return ""
},
"joinParams": func(params []methodParam) string {
paramsNames := make([]string, 0, len(params))
for _, param := range params {
tParams := ""
if strings.HasPrefix(param.Type, "...") {
tParams = "..."
}
paramsNames = append(paramsNames, param.Name+tParams)
}
return strings.Join(paramsNames, ", ")
},
"joinParamsWithType": func(params []methodParam) string {
paramsWithType := []string{}
for _, param := range params {
switch param.Type {
case "ChannelSearchOpts", "UserGetByIdsOpts", "ThreadMembershipOpts", "GetPolicyOptions":
paramsWithType = append(paramsWithType, fmt.Sprintf("%s store.%s", param.Name, param.Type))
case "*UserGetByIdsOpts", "*SidebarCategorySearchOpts", "*ChannelGuard":
paramsWithType = append(paramsWithType, fmt.Sprintf("%s *store.%s", param.Name, strings.TrimPrefix(param.Type, "*")))
default:
paramsWithType = append(paramsWithType, fmt.Sprintf("%s %s", param.Name, param.Type))
}
}
return strings.Join(paramsWithType, ", ")
},
"joinParamsWithTypeOutsideStore": func(params []methodParam) string {
paramsWithType := []string{}
for _, param := range params {
switch param.Type {
case "ChannelSearchOpts", "UserGetByIdsOpts", "ThreadMembershipOpts", "GetPolicyOptions":
paramsWithType = append(paramsWithType, fmt.Sprintf("%s store.%s", param.Name, param.Type))
case "*UserGetByIdsOpts", "*SidebarCategorySearchOpts", "*ChannelGuard":
paramsWithType = append(paramsWithType, fmt.Sprintf("%s *store.%s", param.Name, strings.TrimPrefix(param.Type, "*")))
default:
paramsWithType = append(paramsWithType, fmt.Sprintf("%s %s", param.Name, param.Type))
}
}
return strings.Join(paramsWithType, ", ")
},
}
t := template.Must(template.New(templateFile).Funcs(myFuncs).ParseFiles("layer_generators/" + templateFile))
if err = t.Execute(out, metadata); err != nil {
return nil, err
}
return out.Bytes(), nil
}

View File

@@ -0,0 +1,95 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// Code generated by "make store-layers"
// DO NOT EDIT
package retrylayer
import (
timepkg "time"
"github.com/lib/pq"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type {{.Name}} struct {
store.Store
{{range $index, $element := .SubStores}} {{$index}}Store store.{{$index}}Store
{{end}}
}
{{range $index, $element := .SubStores}}func (s *{{$.Name}}) {{$index}}() store.{{$index}}Store {
return s.{{$index}}Store
}
{{end}}
{{range $index, $element := .SubStores}}type {{$.Name}}{{$index}}Store struct {
store.{{$index}}Store
Root *{{$.Name}}
}
{{end}}
func isRepeatableError(err error) bool {
var pqErr *pq.Error
switch {
case errors.As(errors.Cause(err), &pqErr):
if pqErr.Code == "40001" || pqErr.Code == "40P01" {
return true
}
}
return false
}
{{range $substoreName, $substore := .SubStores}}
{{range $index, $element := $substore.Methods}}
func (s *{{$.Name}}{{$substoreName}}Store) {{$index}}({{$element.Params | joinParamsWithTypeOutsideStore}}) {{$element.Results | joinResultsForSignature}} {
{{if $element.Results | len | eq 0}}
s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
{{else}}
{{if $element.Results | errorPresent}}
tries := 0
for {
{{genResultsVars $element.Results false }} := s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
if {{$element.Results | errorVar}} == nil {
return {{genResultsVars $element.Results true }}
}
if !isRepeatableError({{$element.Results | errorVar}}) {
return {{genResultsVars $element.Results false }}
}
tries++
if tries >= 3 {
{{$element.Results | errorVar}} = errors.Wrap({{$element.Results | errorVar}}, "giving up after 3 consecutive repeatable transaction failures")
return {{genResultsVars $element.Results false }}
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
{{else}}
return s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
{{end}}
{{end}}
}
{{end}}
{{end}}
{{range $index, $element := .Methods}}
func (s *{{$.Name}}) {{$index}}({{$element.Params | joinParamsWithTypeOutsideStore}}) {{$element.Results | joinResultsForSignature}} {
{{if $element.Results | len | eq 0}}s.Store.{{$index}}({{$element.Params | joinParams}})
{{else}}return s.Store.{{$index}}({{$element.Params | joinParams}})
{{end}}}
{{end}}
func New(childStore store.Store) *{{.Name}} {
newStore := {{.Name}}{
Store: childStore,
}
{{range $substoreName, $substore := .SubStores}}
newStore.{{$substoreName}}Store = &{{$.Name}}{{$substoreName}}Store{{"{"}}{{$substoreName}}Store: childStore.{{$substoreName}}(), Root: &newStore}{{end}}
return &newStore
}

View File

@@ -0,0 +1,79 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// Code generated by "make store-layers"
// DO NOT EDIT
package timerlayer
import (
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/einterfaces"
)
type {{.Name}} struct {
store.Store
Metrics einterfaces.MetricsInterface
{{range $index, $element := .SubStores}} {{$index}}Store store.{{$index}}Store
{{end}}
}
{{range $index, $element := .SubStores}}func (s *{{$.Name}}) {{$index}}() store.{{$index}}Store {
return s.{{$index}}Store
}
{{end}}
{{range $index, $element := .SubStores}}type {{$.Name}}{{$index}}Store struct {
store.{{$index}}Store
Root *{{$.Name}}
}
{{end}}
{{range $substoreName, $substore := .SubStores}}
{{range $index, $element := $substore.Methods}}
func (s *{{$.Name}}{{$substoreName}}Store) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
start := time.Now()
{{if $element.Results | len | eq 0}}
s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
{{else}}
{{genResultsVars $element.Results false }} := s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
{{end}}
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if {{$element.Results | errorToBoolean}} {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("{{$substoreName}}Store.{{$index}}", success, elapsed)
{{ with (genResultsVars $element.Results false ) -}}
}
return {{ . }}
{{- else -}}
}
{{- end }}
}
{{end}}
{{end}}
{{range $index, $element := .Methods}}
func (s *{{$.Name}}) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
{{if $element.Results | len | eq 0}}s.Store.{{$index}}({{$element.Params | joinParams}})
{{else}}return s.Store.{{$index}}({{$element.Params | joinParams}})
{{end}}}
{{end}}
func New(childStore store.Store, metrics einterfaces.MetricsInterface) *{{.Name}} {
newStore := {{.Name}}{
Store: childStore,
Metrics: metrics,
}
{{range $substoreName, $substore := .SubStores}}
newStore.{{$substoreName}}Store = &{{$.Name}}{{$substoreName}}Store{{"{"}}{{$substoreName}}Store: childStore.{{$substoreName}}(), Root: &newStore}{{end}}
return &newStore
}

View File

@@ -0,0 +1,60 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheAccessControlPolicyStore struct {
store.AccessControlPolicyStore
rootStore *LocalCacheStore
}
func (s *LocalCacheAccessControlPolicyStore) handleClusterInvalidateAccessControlPolicyEtag(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
if err := s.rootStore.accessControlPolicyEtagCache.Purge(); err != nil {
s.rootStore.logger.Warn("failed to purge access control policy etag cache", mlog.Err(err))
}
} else if err := s.rootStore.accessControlPolicyEtagCache.Remove(string(msg.Data)); err != nil {
s.rootStore.logger.Warn("failed to remove access control policy etag cache entry", mlog.Err(err))
}
}
// Keyed by channel ID, with the empty string as the system-scoped-only bucket. Every channel's
// epoch folds in the permission policies, so a permission-policy change has to clear the whole
// cache; a single channel's change only invalidates its own key.
func (s LocalCacheAccessControlPolicyStore) GetEtagEpoch(rctx request.CTX, channelID string) (string, error) {
var epoch string
if err := s.rootStore.doStandardReadCache(s.rootStore.accessControlPolicyEtagCache, channelID, &epoch); err == nil {
return epoch, nil
}
epoch, err := s.AccessControlPolicyStore.GetEtagEpoch(rctx, channelID)
if err != nil {
return "", err
}
s.rootStore.doStandardAddToCache(s.rootStore.accessControlPolicyEtagCache, channelID, epoch)
return epoch, nil
}
func (s LocalCacheAccessControlPolicyStore) InvalidateEtagForChannel(channelID string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.accessControlPolicyEtagCache, channelID, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.accessControlPolicyEtagCache.Name())
}
}
func (s LocalCacheAccessControlPolicyStore) ClearEtagCache() {
s.rootStore.doClearCacheCluster(s.rootStore.accessControlPolicyEtagCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.accessControlPolicyEtagCache.Name())
}
}

View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestAccessControlPolicyStoreCache(t *testing.T) {
channelID := "channel-id"
logger := mlog.CreateConsoleTestLogger(t)
rctx := request.TestContext(t)
t.Run("GetEtagEpoch cached on second call", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
epoch, err := cachedStore.AccessControlPolicy().GetEtagEpoch(rctx, channelID)
require.NoError(t, err)
assert.Equal(t, "100-1", epoch)
mockStore.AccessControlPolicy().(*mocks.AccessControlPolicyStore).AssertNumberOfCalls(t, "GetEtagEpoch", 1)
epoch, err = cachedStore.AccessControlPolicy().GetEtagEpoch(rctx, channelID)
require.NoError(t, err)
assert.Equal(t, "100-1", epoch)
mockStore.AccessControlPolicy().(*mocks.AccessControlPolicyStore).AssertNumberOfCalls(t, "GetEtagEpoch", 1)
})
t.Run("InvalidateEtagForChannel forces a re-query", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
_, err = cachedStore.AccessControlPolicy().GetEtagEpoch(rctx, channelID)
require.NoError(t, err)
mockStore.AccessControlPolicy().(*mocks.AccessControlPolicyStore).AssertNumberOfCalls(t, "GetEtagEpoch", 1)
cachedStore.AccessControlPolicy().InvalidateEtagForChannel(channelID)
_, err = cachedStore.AccessControlPolicy().GetEtagEpoch(rctx, channelID)
require.NoError(t, err)
mockStore.AccessControlPolicy().(*mocks.AccessControlPolicyStore).AssertNumberOfCalls(t, "GetEtagEpoch", 2)
})
t.Run("ClearEtagCache forces a re-query", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
_, err = cachedStore.AccessControlPolicy().GetEtagEpoch(rctx, channelID)
require.NoError(t, err)
mockStore.AccessControlPolicy().(*mocks.AccessControlPolicyStore).AssertNumberOfCalls(t, "GetEtagEpoch", 1)
cachedStore.AccessControlPolicy().ClearEtagCache()
_, err = cachedStore.AccessControlPolicy().GetEtagEpoch(rctx, channelID)
require.NoError(t, err)
mockStore.AccessControlPolicy().(*mocks.AccessControlPolicyStore).AssertNumberOfCalls(t, "GetEtagEpoch", 2)
})
}

View File

@@ -0,0 +1,59 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheAttributesStore struct {
store.AttributesStore
rootStore *LocalCacheStore
}
func (s *LocalCacheAttributesStore) handleClusterInvalidateUserPropertyValuesEpoch(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
if err := s.rootStore.userPropertyValuesEpochCache.Purge(); err != nil {
s.rootStore.logger.Warn("failed to purge user property values epoch cache", mlog.Err(err))
}
} else if err := s.rootStore.userPropertyValuesEpochCache.Remove(string(msg.Data)); err != nil {
s.rootStore.logger.Warn("failed to remove user property values epoch cache entry", mlog.Err(err))
}
}
// Keyed by user ID, so the ABAC-aware post-list ETag doesn't hit PropertyValues on every post GET.
// The App layer invalidates the key on property-value writes.
func (s LocalCacheAttributesStore) GetUserPropertyValuesEpoch(rctx request.CTX, userID string) (string, error) {
var epoch string
if err := s.rootStore.doStandardReadCache(s.rootStore.userPropertyValuesEpochCache, userID, &epoch); err == nil {
return epoch, nil
}
epoch, err := s.AttributesStore.GetUserPropertyValuesEpoch(rctx, userID)
if err != nil {
return "", err
}
s.rootStore.doStandardAddToCache(s.rootStore.userPropertyValuesEpochCache, userID, epoch)
return epoch, nil
}
func (s LocalCacheAttributesStore) InvalidateUserPropertyValuesEpoch(userID string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.userPropertyValuesEpochCache, userID, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userPropertyValuesEpochCache.Name())
}
}
func (s LocalCacheAttributesStore) ClearUserPropertyValuesEpochCache() {
s.rootStore.doClearCacheCluster(s.rootStore.userPropertyValuesEpochCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userPropertyValuesEpochCache.Name())
}
}

View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestAttributesStoreCache(t *testing.T) {
userID := "user-id"
logger := mlog.CreateConsoleTestLogger(t)
rctx := request.TestContext(t)
t.Run("GetUserPropertyValuesEpoch cached on second call", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
epoch, err := cachedStore.Attributes().GetUserPropertyValuesEpoch(rctx, userID)
require.NoError(t, err)
assert.Equal(t, "200-1", epoch)
mockStore.Attributes().(*mocks.AttributesStore).AssertNumberOfCalls(t, "GetUserPropertyValuesEpoch", 1)
epoch, err = cachedStore.Attributes().GetUserPropertyValuesEpoch(rctx, userID)
require.NoError(t, err)
assert.Equal(t, "200-1", epoch)
mockStore.Attributes().(*mocks.AttributesStore).AssertNumberOfCalls(t, "GetUserPropertyValuesEpoch", 1)
})
t.Run("InvalidateUserPropertyValuesEpoch forces a re-query", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
_, err = cachedStore.Attributes().GetUserPropertyValuesEpoch(rctx, userID)
require.NoError(t, err)
mockStore.Attributes().(*mocks.AttributesStore).AssertNumberOfCalls(t, "GetUserPropertyValuesEpoch", 1)
cachedStore.Attributes().InvalidateUserPropertyValuesEpoch(userID)
_, err = cachedStore.Attributes().GetUserPropertyValuesEpoch(rctx, userID)
require.NoError(t, err)
mockStore.Attributes().(*mocks.AttributesStore).AssertNumberOfCalls(t, "GetUserPropertyValuesEpoch", 2)
})
t.Run("ClearUserPropertyValuesEpochCache forces a re-query", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
_, err = cachedStore.Attributes().GetUserPropertyValuesEpoch(rctx, userID)
require.NoError(t, err)
mockStore.Attributes().(*mocks.AttributesStore).AssertNumberOfCalls(t, "GetUserPropertyValuesEpoch", 1)
cachedStore.Attributes().ClearUserPropertyValuesEpochCache()
_, err = cachedStore.Attributes().GetUserPropertyValuesEpoch(rctx, userID)
require.NoError(t, err)
mockStore.Attributes().(*mocks.AttributesStore).AssertNumberOfCalls(t, "GetUserPropertyValuesEpoch", 2)
})
}

View File

@@ -0,0 +1,195 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"fmt"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheAutoTranslationStore struct {
store.AutoTranslationStore
rootStore *LocalCacheStore
}
// Cache key generators
func userAutoTranslationKey(userID, channelID string) string {
return fmt.Sprintf("user:%s:%s", userID, channelID)
}
func userLanguageKey(userID, channelID string) string {
return fmt.Sprintf("lang:%s:%s", userID, channelID)
}
func postTranslationEtagKey(channelID string) string {
return fmt.Sprintf("etag:%s", channelID)
}
// Cluster invalidation handler for user auto-translation cache
func (s *LocalCacheAutoTranslationStore) handleClusterInvalidateUserAutoTranslation(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.userAutoTranslationCache.Purge()
} else {
s.rootStore.userAutoTranslationCache.Remove(string(msg.Data))
}
}
// Cluster invalidation handler for post translation etag cache
func (s *LocalCacheAutoTranslationStore) handleClusterInvalidatePostTranslationEtag(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.postTranslationEtagCache.Purge()
} else {
s.rootStore.postTranslationEtagCache.Remove(string(msg.Data))
}
}
// ClearCaches purges all auto-translation caches
func (s LocalCacheAutoTranslationStore) ClearCaches() {
s.rootStore.doClearCacheCluster(s.rootStore.userAutoTranslationCache)
s.rootStore.doClearCacheCluster(s.rootStore.postTranslationEtagCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userAutoTranslationCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postTranslationEtagCache.Name())
}
}
// IsUserEnabled checks if auto-translation is enabled for a user in a channel (with caching)
func (s LocalCacheAutoTranslationStore) IsUserEnabled(userID, channelID string) (bool, error) {
key := userAutoTranslationKey(userID, channelID)
var enabled bool
if err := s.rootStore.doStandardReadCache(s.rootStore.userAutoTranslationCache, key, &enabled); err == nil {
return enabled, nil
}
enabled, err := s.AutoTranslationStore.IsUserEnabled(userID, channelID)
if err != nil {
return false, err
}
s.rootStore.doStandardAddToCache(s.rootStore.userAutoTranslationCache, key, enabled)
return enabled, nil
}
// GetUserLanguage gets the user's language preference for a channel (with caching)
func (s LocalCacheAutoTranslationStore) GetUserLanguage(userID, channelID string) (string, error) {
key := userLanguageKey(userID, channelID)
var language string
if err := s.rootStore.doStandardReadCache(s.rootStore.userAutoTranslationCache, key, &language); err == nil {
return language, nil
}
language, err := s.AutoTranslationStore.GetUserLanguage(userID, channelID)
if err != nil {
return "", err
}
// Only cache non-empty results
if language != "" {
s.rootStore.doStandardAddToCache(s.rootStore.userAutoTranslationCache, key, language)
}
return language, nil
}
// InvalidateUserAutoTranslation invalidates all user auto-translation caches for a specific user+channel
// This should be called when a user is removed from a channel
func (s LocalCacheAutoTranslationStore) InvalidateUserAutoTranslation(userID, channelID string) {
// Invalidate user auto-translation enabled cache
userKey := userAutoTranslationKey(userID, channelID)
s.rootStore.doInvalidateCacheCluster(s.rootStore.userAutoTranslationCache, userKey, nil)
// Invalidate user language cache
langKey := userLanguageKey(userID, channelID)
s.rootStore.doInvalidateCacheCluster(s.rootStore.userAutoTranslationCache, langKey, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userAutoTranslationCache.Name())
}
}
// InvalidateUserLocaleCache invalidates all language caches for a user across all channels
// This should be called when a user changes their locale
func (s LocalCacheAutoTranslationStore) InvalidateUserLocaleCache(userID string) {
// Scan the cache and find all keys that match "lang:{userID}:*"
prefix := fmt.Sprintf("lang:%s:", userID)
var toDelete []string
err := s.rootStore.userAutoTranslationCache.Scan(func(keys []string) error {
for _, key := range keys {
if len(key) > len(prefix) && key[:len(prefix)] == prefix {
toDelete = append(toDelete, key)
}
}
return nil
})
if err != nil {
// If scan fails, just purge the entire cache as fallback
s.rootStore.doClearCacheCluster(s.rootStore.userAutoTranslationCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userAutoTranslationCache.Name())
}
return
}
// Invalidate all matching keys
for _, key := range toDelete {
s.rootStore.doInvalidateCacheCluster(s.rootStore.userAutoTranslationCache, key, nil)
}
if s.rootStore.metrics != nil && len(toDelete) > 0 {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userAutoTranslationCache.Name())
}
}
// GetLatestPostUpdateAtForChannel returns the most recent updateAt timestamp for post translations
// in the given channel (across all locales, with caching)
func (s LocalCacheAutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) {
key := postTranslationEtagKey(channelID)
var updateAt int64
if err := s.rootStore.doStandardReadCache(s.rootStore.postTranslationEtagCache, key, &updateAt); err == nil {
return updateAt, nil
}
updateAt, err := s.AutoTranslationStore.GetLatestPostUpdateAtForChannel(channelID)
if err != nil {
return 0, err
}
s.rootStore.doStandardAddToCache(s.rootStore.postTranslationEtagCache, key, updateAt)
return updateAt, nil
}
// InvalidatePostTranslationEtag invalidates the cached post translation etag for a channel
// This should be called after saving a new post translation
func (s LocalCacheAutoTranslationStore) InvalidatePostTranslationEtag(channelID string) {
key := postTranslationEtagKey(channelID)
s.rootStore.doInvalidateCacheCluster(s.rootStore.postTranslationEtagCache, key, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postTranslationEtagCache.Name())
}
}
// Save wraps the underlying Save and invalidates the post translation etag cache for post translations
func (s LocalCacheAutoTranslationStore) Save(translation *model.Translation) error {
err := s.AutoTranslationStore.Save(translation)
if err != nil {
return err
}
// Invalidate post translation etag cache only for post translations
if translation.ChannelID != "" && translation.ObjectType == model.TranslationObjectTypePost {
s.InvalidatePostTranslationEtag(translation.ChannelID)
}
return nil
}

View File

@@ -0,0 +1,568 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"strings"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
type LocalCacheChannelStore struct {
store.ChannelStore
rootStore *LocalCacheStore
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelMemberCounts(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelMemberCountsCache.Purge()
} else {
s.rootStore.channelMemberCountsCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelPinnedPostCount(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelPinnedPostCountsCache.Purge()
} else {
s.rootStore.channelPinnedPostCountsCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelGuestCounts(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelGuestCountCache.Purge()
} else {
s.rootStore.channelGuestCountCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelById(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelByIdCache.Purge()
s.rootStore.deliveryTracking.purgeChannels()
} else {
s.rootStore.channelByIdCache.Remove(string(msg.Data))
s.rootStore.deliveryTracking.invalidateChannel(string(msg.Data))
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelForUser(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelMembersForUserCache.Purge()
} else {
s.rootStore.channelMembersForUserCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelMembersNotifyProps(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelMembersNotifyPropsCache.Purge()
} else {
s.rootStore.channelMembersNotifyPropsCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelByName(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.channelByNameCache.Purge()
} else {
s.rootStore.channelByNameCache.Remove(string(msg.Data))
}
}
func (s LocalCacheChannelStore) ClearMembersForUserCache() {
s.rootStore.doClearCacheCluster(s.rootStore.channelMembersForUserCache)
}
func (s LocalCacheChannelStore) ClearCaches() {
s.rootStore.doClearCacheCluster(s.rootStore.channelMemberCountsCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelPinnedPostCountsCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelGuestCountCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelByIdCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelMembersForUserCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelMembersNotifyPropsCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelByNameCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelMemberCountsCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelPinnedPostCountsCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelGuestCountCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelByIdCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelMembersForUserCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelMembersNotifyPropsCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelByNameCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidatePinnedPostCount(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelPinnedPostCountsCache, channelId, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelPinnedPostCountsCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidateMemberCount(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelMemberCountsCache, channelId, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelMemberCountsCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidateGuestCount(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelGuestCountCache, channelId, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelGuestCountCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidateChannel(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelByIdCache, channelId, nil)
s.rootStore.deliveryTracking.invalidateChannel(channelId)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelByIdCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidateAllChannelMembersForUser(userId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelMembersForUserCache, userId, nil)
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelMembersForUserCache, userId+"_deleted", nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelMembersForUserCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidateCacheForChannelMembersNotifyProps(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelMembersNotifyPropsCache, channelId, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelMembersNotifyPropsCache.Name())
}
}
func (s LocalCacheChannelStore) InvalidateChannelByName(teamId, name string) {
props := make(map[string]string)
props["name"] = name
if teamId == "" {
props["id"] = "dm"
} else {
props["id"] = teamId
}
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelByNameCache, teamId+name, props)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.channelByNameCache.Name())
}
}
func (s LocalCacheChannelStore) GetMemberCount(channelId string, allowFromCache bool) (int64, error) {
if allowFromCache {
var count int64
if err := s.rootStore.doStandardReadCache(s.rootStore.channelMemberCountsCache, channelId, &count); err == nil {
return count, nil
}
}
count, err := s.ChannelStore.GetMemberCount(channelId, allowFromCache)
if allowFromCache && err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.channelMemberCountsCache, channelId, count)
}
return count, err
}
func (s LocalCacheChannelStore) GetGuestCount(channelId string, allowFromCache bool) (int64, error) {
if allowFromCache {
var count int64
if err := s.rootStore.doStandardReadCache(s.rootStore.channelGuestCountCache, channelId, &count); err == nil {
return count, nil
}
}
count, err := s.ChannelStore.GetGuestCount(channelId, allowFromCache)
if allowFromCache && err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.channelGuestCountCache, channelId, count)
}
return count, err
}
func (s LocalCacheChannelStore) GetMemberCountFromCache(channelId string) int64 {
var count int64
if err := s.rootStore.doStandardReadCache(s.rootStore.channelMemberCountsCache, channelId, &count); err == nil {
return count
}
count, err := s.GetMemberCount(channelId, true)
if err != nil {
return 0
}
return count
}
func (s LocalCacheChannelStore) GetPinnedPostCount(channelId string, allowFromCache bool) (int64, error) {
if allowFromCache {
var count int64
if err := s.rootStore.doStandardReadCache(s.rootStore.channelPinnedPostCountsCache, channelId, &count); err == nil {
return count, nil
}
}
count, err := s.ChannelStore.GetPinnedPostCount(channelId, allowFromCache)
if err != nil {
return 0, err
}
if allowFromCache {
s.rootStore.doStandardAddToCache(s.rootStore.channelPinnedPostCountsCache, channelId, count)
}
return count, nil
}
func (s LocalCacheChannelStore) Save(rctx request.CTX, channel *model.Channel, maxChannelsPerTeam int64, channelOptions ...model.ChannelOption) (*model.Channel, error) {
newChannel, err := s.ChannelStore.Save(rctx, channel, maxChannelsPerTeam, channelOptions...)
// Space backing channels are excluded from the generic by-id Get/GetMany (SQL) and resolve
// only through GetChannelOfType; caching them here would let the generic cached lookups
// return them and defeat that exclusion.
if err == nil && !newChannel.IsSpace() {
s.rootStore.doStandardAddToCache(s.rootStore.channelByIdCache, newChannel.Id, newChannel)
}
return newChannel, err
}
func (s LocalCacheChannelStore) Update(rctx request.CTX, channel *model.Channel) (*model.Channel, error) {
updatedChannel, err := s.ChannelStore.Update(rctx, channel)
if err == nil {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelByIdCache, channel.Id, nil)
}
return updatedChannel, err
}
func (s LocalCacheChannelStore) Get(id string, allowFromCache bool) (*model.Channel, error) {
if allowFromCache {
var cacheItem *model.Channel
if err := s.rootStore.doStandardReadCache(s.rootStore.channelByIdCache, id, &cacheItem); err == nil {
return cacheItem, nil
}
}
ch, err := s.ChannelStore.Get(id, allowFromCache)
if allowFromCache && err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.channelByIdCache, id, ch)
}
return ch, err
}
func (s LocalCacheChannelStore) GetMany(ids []string, allowFromCache bool) (model.ChannelList, error) {
var foundChannels []*model.Channel
var channelsToQuery []string
if !allowFromCache {
return s.ChannelStore.GetMany(ids, allowFromCache)
}
toPass := allocateCacheTargets[*model.Channel](len(ids))
errs := s.rootStore.doMultiReadCache(s.rootStore.channelByIdCache, ids, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
s.rootStore.logger.Warn("Error in Channelstore.GetMany: ", mlog.Err(err))
}
channelsToQuery = append(channelsToQuery, ids[i])
} else {
gotChannel := *(toPass[i].(**model.Channel))
if gotChannel != nil {
foundChannels = append(foundChannels, gotChannel)
} else {
s.rootStore.logger.Warn("Found nil channel in GetMany. This is not expected")
}
}
}
if len(channelsToQuery) == 0 {
return foundChannels, nil
}
channels, err := s.ChannelStore.GetMany(channelsToQuery, allowFromCache)
if err != nil {
return nil, err
}
for _, ch := range channels {
s.rootStore.doStandardAddToCache(s.rootStore.channelByIdCache, ch.Id, ch)
}
return append(foundChannels, channels...), nil
}
func (s LocalCacheChannelStore) GetAllChannelMembersForUser(rctx request.CTX, userId string, allowFromCache bool, includeDeleted bool) (map[string]string, error) {
cache_key := userId
if includeDeleted {
cache_key += "_deleted"
}
if allowFromCache {
var ids model.StringMap
if err := s.rootStore.doStandardReadCache(s.rootStore.channelMembersForUserCache, cache_key, &ids); err == nil {
return ids, nil
}
}
ids, err := s.ChannelStore.GetAllChannelMembersForUser(rctx, userId, allowFromCache, includeDeleted)
if err != nil {
return nil, err
}
if allowFromCache {
s.rootStore.doStandardAddToCache(s.rootStore.channelMembersForUserCache, cache_key, ids)
}
return ids, nil
}
func (s LocalCacheChannelStore) GetAllChannelMembersNotifyPropsForChannel(channelId string, allowFromCache bool) (map[string]model.StringMap, error) {
if allowFromCache {
var cacheItem map[string]model.StringMap
if err := s.rootStore.doStandardReadCache(s.rootStore.channelMembersNotifyPropsCache, channelId, &cacheItem); err == nil {
return cacheItem, nil
}
}
props, err := s.ChannelStore.GetAllChannelMembersNotifyPropsForChannel(channelId, allowFromCache)
if err != nil {
return nil, err
}
if allowFromCache {
s.rootStore.doStandardAddToCache(s.rootStore.channelMembersNotifyPropsCache, channelId, props)
}
return props, nil
}
func (s LocalCacheChannelStore) GetByNamesIncludeDeleted(teamId string, names []string, allowFromCache bool) ([]*model.Channel, error) {
return s.getByNames(teamId, names, allowFromCache, true)
}
func (s LocalCacheChannelStore) GetByNames(teamId string, names []string, allowFromCache bool) ([]*model.Channel, error) {
return s.getByNames(teamId, names, allowFromCache, false)
}
func (s LocalCacheChannelStore) getByNames(teamId string, names []string, allowFromCache, includeArchivedChannels bool) ([]*model.Channel, error) {
var channels []*model.Channel
if allowFromCache {
var misses []string
visited := make(map[string]struct{})
var newKeys []string
for _, name := range names {
if _, ok := visited[name]; ok {
continue
}
visited[name] = struct{}{}
newKeys = append(newKeys, teamId+name)
}
toPass := allocateCacheTargets[*model.Channel](len(newKeys))
errs := s.rootStore.doMultiReadCache(s.rootStore.channelByNameCache, newKeys, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
s.rootStore.logger.Warn("Error in Channelstore.GetByNames: ", mlog.Err(err))
}
misses = append(misses, strings.TrimPrefix(newKeys[i], teamId))
} else {
gotChannel := *(toPass[i].(**model.Channel))
if (gotChannel != nil) && (includeArchivedChannels || gotChannel.DeleteAt == 0) {
channels = append(channels, gotChannel)
} else if gotChannel == nil {
s.rootStore.logger.Warn("Found nil channel in getByNames. This is not expected")
}
}
}
names = misses
}
if len(names) > 0 {
var dbChannels []*model.Channel
var err error
if includeArchivedChannels {
dbChannels, err = s.ChannelStore.GetByNamesIncludeDeleted(teamId, names, allowFromCache)
} else {
dbChannels, err = s.ChannelStore.GetByNames(teamId, names, allowFromCache)
}
if err != nil {
return nil, err
}
for _, channel := range dbChannels {
if allowFromCache {
s.rootStore.doStandardAddToCache(s.rootStore.channelByNameCache, teamId+channel.Name, channel)
}
channels = append(channels, channel)
}
}
return channels, nil
}
func (s LocalCacheChannelStore) GetByNameIncludeDeleted(teamId string, name string, allowFromCache bool) (*model.Channel, error) {
return s.getByName(teamId, name, allowFromCache, true)
}
func (s LocalCacheChannelStore) GetByName(teamId string, name string, allowFromCache bool) (*model.Channel, error) {
return s.getByName(teamId, name, allowFromCache, false)
}
func (s LocalCacheChannelStore) getByName(teamId string, name string, allowFromCache, includeArchivedChannels bool) (*model.Channel, error) {
var channel *model.Channel
if allowFromCache {
if err := s.rootStore.doStandardReadCache(s.rootStore.channelByNameCache, teamId+name, &channel); err == nil {
if includeArchivedChannels || channel.DeleteAt == 0 {
return channel, nil
}
}
}
var err error
if includeArchivedChannels {
channel, err = s.ChannelStore.GetByNameIncludeDeleted(teamId, name, allowFromCache)
} else {
channel, err = s.ChannelStore.GetByName(teamId, name, allowFromCache)
}
if err != nil {
return nil, err
}
if allowFromCache {
s.rootStore.doStandardAddToCache(s.rootStore.channelByNameCache, teamId+name, channel)
}
return channel, nil
}
func (s LocalCacheChannelStore) SaveMember(rctx request.CTX, member *model.ChannelMember) (*model.ChannelMember, error) {
member, err := s.ChannelStore.SaveMember(rctx, member)
if err != nil {
return nil, err
}
// For redis, directly increment member count.
if externalCache, ok := s.rootStore.channelMemberCountsCache.(cache.ExternalCache); ok {
s.rootStore.doIncrementCache(externalCache, member.ChannelId, 1)
} else {
s.InvalidateMemberCount(member.ChannelId)
}
return member, nil
}
func (s LocalCacheChannelStore) SaveMultipleMembers(members []*model.ChannelMember) ([]*model.ChannelMember, error) {
members, err := s.ChannelStore.SaveMultipleMembers(members)
if err != nil {
return nil, err
}
for _, member := range members {
// For redis, directly increment member count.
// It should be possible to group the members from the slice
// by channelID and increment it once per channel. But it depends
// on whether all members are part of the same channel or not.
if externalCache, ok := s.rootStore.channelMemberCountsCache.(cache.ExternalCache); ok {
s.rootStore.doIncrementCache(externalCache, member.ChannelId, 1)
} else {
s.InvalidateMemberCount(member.ChannelId)
}
}
return members, nil
}
func (s LocalCacheChannelStore) GetChannelsMemberCount(channelIDs []string) (_ map[string]int64, err error) {
counts := make(map[string]int64)
remainingChannels := make([]string, 0)
toPass := allocateCacheTargets[int64](len(channelIDs))
errs := s.rootStore.doMultiReadCache(s.rootStore.channelMemberCountsCache, channelIDs, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
s.rootStore.logger.Warn("Error in Channelstore.GetChannelsMemberCount: ", mlog.Err(err))
}
remainingChannels = append(remainingChannels, channelIDs[i])
} else {
gotCount := *(toPass[i].(*int64))
if gotCount != 0 {
counts[channelIDs[i]] = gotCount
}
}
}
if len(remainingChannels) > 0 {
remainingChannels, err := s.ChannelStore.GetChannelsMemberCount(remainingChannels)
if err != nil {
return nil, err
}
for id, count := range remainingChannels {
s.rootStore.doStandardAddToCache(s.rootStore.channelMemberCountsCache, id, count)
counts[id] = count
}
}
return counts, nil
}
func (s LocalCacheChannelStore) UpdateMember(rctx request.CTX, member *model.ChannelMember) (*model.ChannelMember, error) {
member, err := s.ChannelStore.UpdateMember(rctx, member)
if err != nil {
return nil, err
}
s.InvalidateMemberCount(member.ChannelId)
return member, nil
}
func (s LocalCacheChannelStore) UpdateMultipleMembers(members []*model.ChannelMember) ([]*model.ChannelMember, error) {
members, err := s.ChannelStore.UpdateMultipleMembers(members)
if err != nil {
return nil, err
}
for _, member := range members {
s.InvalidateMemberCount(member.ChannelId)
}
return members, nil
}
func (s LocalCacheChannelStore) RemoveMember(rctx request.CTX, channelId, userId string) error {
err := s.ChannelStore.RemoveMember(rctx, channelId, userId)
if err != nil {
return err
}
// For redis, directly decrement member count.
if externalCache, ok := s.rootStore.channelMemberCountsCache.(cache.ExternalCache); ok {
s.rootStore.doDecrementCache(externalCache, channelId, 1)
} else {
s.InvalidateMemberCount(channelId)
}
return nil
}
func (s LocalCacheChannelStore) RemoveMembers(rctx request.CTX, channelId string, userIds []string) error {
err := s.ChannelStore.RemoveMembers(rctx, channelId, userIds)
if err != nil {
return err
}
// For redis, directly decrement member count.
if externalCache, ok := s.rootStore.channelMemberCountsCache.(cache.ExternalCache); ok {
s.rootStore.doDecrementCache(externalCache, channelId, len(userIds))
} else {
s.InvalidateMemberCount(channelId)
}
return nil
}

View File

@@ -0,0 +1,538 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
cmocks "github.com/mattermost/mattermost/server/v8/platform/services/cache/mocks"
)
func TestChannelStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestReactionStore)
}
func TestChannelStoreChannelMemberCountsCache(t *testing.T) {
countResult := int64(10)
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
count, err := cachedStore.Channel().GetMemberCount("id", true)
require.NoError(t, err)
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
count, err = cachedStore.Channel().GetMemberCount("id", true)
require.NoError(t, err)
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
cachedStore.Channel().GetMemberCount("id", false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 2)
})
t.Run("first call force not cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetMemberCount("id", false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 2)
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 2)
})
t.Run("first call with GetMemberCountFromCache not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
count := cachedStore.Channel().GetMemberCountFromCache("id")
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
count = cachedStore.Channel().GetMemberCountFromCache("id")
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
cachedStore.Channel().ClearCaches()
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 2)
})
t.Run("first call not cached, invalidate cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 1)
cachedStore.Channel().InvalidateMemberCount("id")
cachedStore.Channel().GetMemberCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMemberCount", 2)
})
}
func TestChannelStoreChannelsMemberCountCache(t *testing.T) {
channelsCountResult := map[string]int64{
"channel1": 10,
"channel2": 20,
}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
channelsCount, err := cachedStore.Channel().GetChannelsMemberCount([]string{"channel1", "channel2"})
require.NoError(t, err)
assert.Equal(t, channelsCount, channelsCountResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetChannelsMemberCount", 1)
channelsCount, err = cachedStore.Channel().GetChannelsMemberCount([]string{"channel1", "channel2"})
require.NoError(t, err)
assert.Equal(t, channelsCount, channelsCountResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetChannelsMemberCount", 1)
})
t.Run("first call not cached, invalidate cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetChannelsMemberCount([]string{"channel1", "channel2"})
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetChannelsMemberCount", 1)
cachedStore.Channel().InvalidateMemberCount("channel1")
cachedStore.Channel().InvalidateMemberCount("channel2")
cachedStore.Channel().GetChannelsMemberCount([]string{"channel1", "channel2"})
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetChannelsMemberCount", 2)
})
}
func TestChannelStoreChannelPinnedPostsCountsCache(t *testing.T) {
countResult := int64(10)
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
count, err := cachedStore.Channel().GetPinnedPostCount("id", true)
require.NoError(t, err)
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 1)
count, err = cachedStore.Channel().GetPinnedPostCount("id", true)
require.NoError(t, err)
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 1)
cachedStore.Channel().GetPinnedPostCount("id", false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 2)
})
t.Run("first call force not cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetPinnedPostCount("id", false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 1)
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 2)
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 2)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 1)
cachedStore.Channel().ClearCaches()
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 2)
})
t.Run("first call not cached, invalidate cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 1)
cachedStore.Channel().InvalidatePinnedPostCount("id")
cachedStore.Channel().GetPinnedPostCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetPinnedPostCount", 2)
})
}
func TestChannelStoreGuestCountCache(t *testing.T) {
countResult := int64(12)
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
count, err := cachedStore.Channel().GetGuestCount("id", true)
require.NoError(t, err)
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 1)
count, err = cachedStore.Channel().GetGuestCount("id", true)
require.NoError(t, err)
assert.Equal(t, count, countResult)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 1)
cachedStore.Channel().GetGuestCount("id", false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 2)
})
t.Run("first call force not cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetGuestCount("id", false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 1)
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 2)
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 2)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 1)
cachedStore.Channel().ClearCaches()
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 2)
})
t.Run("first call not cached, invalidate cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 1)
cachedStore.Channel().InvalidateGuestCount("id")
cachedStore.Channel().GetGuestCount("id", true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 2)
})
}
func TestChannelStoreChannel(t *testing.T) {
channelId := "channel1"
fakeChannel := model.Channel{Id: channelId, Name: "channel1-name"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call by id not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
channel, err := cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
assert.Equal(t, channel, &fakeChannel)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
channel, err = cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
assert.Equal(t, channel, &fakeChannel)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().Get(channelId, false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call force no cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().Get(channelId, false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().ClearCaches()
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, invalidate cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().InvalidateChannel(channelId)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
}
func TestChannelStoreSavePopulatesCache(t *testing.T) {
channelId := "channel1"
fakeChannel := model.Channel{Id: channelId, Name: "channel1-name"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("save populates cache so subsequent get does not hit store", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.Channel().Save(request.TestContext(t), &fakeChannel, int64(0))
require.NoError(t, err)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Save", 1)
// Get should be served from cache without hitting the underlying store
channel, err := cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
assert.Equal(t, &fakeChannel, channel)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 0)
})
t.Run("save populates cache, invalidate removes it, get hits store", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.Channel().Save(request.TestContext(t), &fakeChannel, int64(0))
require.NoError(t, err)
cachedStore.Channel().InvalidateChannel(channelId)
_, err = cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
})
}
func TestChannelStoreUpdateInvalidatesCache(t *testing.T) {
channelId := "channel1"
fakeChannel := model.Channel{Id: channelId, Name: "channel1-name"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("update after save invalidates cache, get hits store", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// Save populates cache
_, err = cachedStore.Channel().Save(request.TestContext(t), &fakeChannel, int64(0))
require.NoError(t, err)
// Update invalidates cache
_, err = cachedStore.Channel().Update(request.TestContext(t), &fakeChannel)
require.NoError(t, err)
// Get should miss cache and hit the store
_, err = cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("update after get invalidates cache, next get hits store", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First Get populates cache
_, err = cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
// Update invalidates cache
_, err = cachedStore.Channel().Update(request.TestContext(t), &fakeChannel)
require.NoError(t, err)
// Second Get should miss cache and hit the store again
_, err = cachedStore.Channel().Get(channelId, true)
require.NoError(t, err)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
}
func TestChannelStoreGetManyCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
fakeChannel := model.Channel{Id: "channel1", Name: "channel1-name"}
fakeChannel2 := model.Channel{Id: "channel2", Name: "channel2-name"}
channels, err := cachedStore.Channel().GetMany([]string{fakeChannel.Id}, true)
require.NoError(t, err)
assert.ElementsMatch(t, model.ChannelList{&fakeChannel}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMany", 1)
channels, err = cachedStore.Channel().GetMany([]string{fakeChannel.Id}, true)
require.NoError(t, err)
assert.ElementsMatch(t, model.ChannelList{&fakeChannel}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMany", 1)
channels, err = cachedStore.Channel().GetMany([]string{fakeChannel.Id, fakeChannel2.Id}, true)
require.NoError(t, err)
assert.ElementsMatch(t, model.ChannelList{&fakeChannel, &fakeChannel2}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMany", 2)
})
t.Run("passing allowCache=false should bypass cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
fakeChannel := model.Channel{Id: "channel1", Name: "channel1-name"}
channels, err := cachedStore.Channel().GetMany([]string{fakeChannel.Id}, false)
require.NoError(t, err)
assert.ElementsMatch(t, model.ChannelList{&fakeChannel}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetMany", 1)
})
}
func TestChannelStoreGetByNamesCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
fakeChannel := model.Channel{Id: "channel1", Name: "channel1-name"}
fakeChannel2 := model.Channel{Id: "channel2", Name: "channel2-name"}
channels, err := cachedStore.Channel().GetByNames("team1", []string{fakeChannel.Name}, true)
require.NoError(t, err)
assert.ElementsMatch(t, []*model.Channel{&fakeChannel}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetByNames", 1)
channels, err = cachedStore.Channel().GetByNames("team1", []string{fakeChannel.Name}, true)
require.NoError(t, err)
assert.ElementsMatch(t, []*model.Channel{&fakeChannel}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetByNames", 1)
channels, err = cachedStore.Channel().GetByNames("team1", []string{fakeChannel.Name, fakeChannel2.Name}, true)
require.NoError(t, err)
assert.ElementsMatch(t, []*model.Channel{&fakeChannel, &fakeChannel2}, channels)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetByNames", 2)
})
}
func TestChannelStoreGetAllChannelMembersForUser(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cmock := cmocks.NewCache(t)
cmock.On("Get", "u1", mock.AnythingOfType("*model.StringMap")).Return(nil)
cachedStore.channel.rootStore.channelMembersForUserCache = cmock
_, err = cachedStore.Channel().GetAllChannelMembersForUser(request.TestContext(t), "u1", true, false)
require.NoError(t, err)
}

View File

@@ -0,0 +1,54 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
const (
CACHE_KEY_REVIEWER_SETTINGS = "reviewer_settings"
)
type LocalCacheContentFlaggingStore struct {
store.ContentFlaggingStore
rootStore *LocalCacheStore
}
func (s *LocalCacheContentFlaggingStore) handleClusterInvalidateContentFlagging(msg *model.ClusterMessage) {
if err := s.rootStore.contentFlaggingCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge content flagging cache", mlog.Err(err))
}
}
func (s LocalCacheContentFlaggingStore) ClearCaches() {
if err := s.rootStore.contentFlaggingCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge content flagging cache", mlog.Err(err))
}
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.contentFlaggingCache.Name())
}
}
func (s LocalCacheContentFlaggingStore) GetReviewerSettings() (*model.ReviewerIDsSettings, error) {
var reviewerSettings *model.ReviewerIDsSettings
err := s.rootStore.doStandardReadCache(s.rootStore.contentFlaggingCache, CACHE_KEY_REVIEWER_SETTINGS, &reviewerSettings)
if err == nil {
return reviewerSettings, nil
}
reviewerSettings, err = s.ContentFlaggingStore.GetReviewerSettings()
if err != nil {
return nil, err
}
if reviewerSettings != nil {
s.rootStore.doStandardAddToCache(s.rootStore.contentFlaggingCache, CACHE_KEY_REVIEWER_SETTINGS, reviewerSettings)
}
return reviewerSettings, nil
}

View File

@@ -0,0 +1,99 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestContentFlaggingStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestContentFlaggingStore)
}
func TestContentFlaggingStoreGetReviewerSettingsCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
setBasicMock := func(mockStore *mocks.Store, cachedStore LocalCacheStore) {
reviewerSettings := &model.ReviewerIDsSettings{
CommonReviewerIds: []string{"user1", "user2", "user3"},
}
mockContentFlaggingStore := mockStore.ContentFlagging().(*mocks.ContentFlaggingStore)
mockContentFlaggingStore.On("GetReviewerSettings").Return(reviewerSettings, nil)
cachedStore.contentFlagging.ContentFlaggingStore = mockContentFlaggingStore
}
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
setBasicMock(mockStore, cachedStore)
settings, err := cachedStore.ContentFlagging().GetReviewerSettings()
require.NoError(t, err)
require.Len(t, settings.CommonReviewerIds, 3)
require.Contains(t, settings.CommonReviewerIds, "user1")
require.Contains(t, settings.CommonReviewerIds, "user2")
require.Contains(t, settings.CommonReviewerIds, "user3")
mockStore.ContentFlagging().(*mocks.ContentFlaggingStore).AssertNumberOfCalls(t, "GetReviewerSettings", 1)
settings, err = cachedStore.ContentFlagging().GetReviewerSettings()
require.NoError(t, err)
require.Len(t, settings.CommonReviewerIds, 3)
require.Contains(t, settings.CommonReviewerIds, "user1")
require.Contains(t, settings.CommonReviewerIds, "user2")
require.Contains(t, settings.CommonReviewerIds, "user3")
mockStore.ContentFlagging().(*mocks.ContentFlaggingStore).AssertNumberOfCalls(t, "GetReviewerSettings", 1)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
setBasicMock(mockStore, cachedStore)
_, err = cachedStore.ContentFlagging().GetReviewerSettings()
require.NoError(t, err)
mockStore.ContentFlagging().(*mocks.ContentFlaggingStore).AssertNumberOfCalls(t, "GetReviewerSettings", 1)
cachedStore.ContentFlagging().ClearCaches()
_, err = cachedStore.ContentFlagging().GetReviewerSettings()
require.NoError(t, err)
mockStore.ContentFlagging().(*mocks.ContentFlaggingStore).AssertNumberOfCalls(t, "GetReviewerSettings", 2)
})
t.Run("cluster invalidation clears cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
setBasicMock(mockStore, cachedStore)
// First call to populate cache
_, err = cachedStore.ContentFlagging().GetReviewerSettings()
require.NoError(t, err)
mockStore.ContentFlagging().(*mocks.ContentFlaggingStore).AssertNumberOfCalls(t, "GetReviewerSettings", 1)
// Simulate cluster invalidation message
msg := &model.ClusterMessage{
Event: model.ClusterEventInvalidateCacheForContentFlagging,
}
cachedStore.contentFlagging.handleClusterInvalidateContentFlagging(msg)
// Next call should hit the store again
_, err = cachedStore.ContentFlagging().GetReviewerSettings()
require.NoError(t, err)
mockStore.ContentFlagging().(*mocks.ContentFlaggingStore).AssertNumberOfCalls(t, "GetReviewerSettings", 2)
})
}

View File

@@ -0,0 +1,136 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"errors"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
const cacheKeyTrackedChannelIDs = "tracked_channel_ids"
// LocalCacheDeliveryTrackingStore memoizes the per-channel eligibility answers that post
// delivery audit logging consults once per recorded delivery.
//
// The two channel caches hold plain bools rather than going through cache.Cache, which
// marshals to msgpack on write and unmarshals on every read. On this path that would allocate
// per record. They are populated lazily as channel ids are seen, so there is no startup load
// and a failed read only affects the channel that triggered it.
type LocalCacheDeliveryTrackingStore struct {
store.DeliveryTrackingStore
rootStore *LocalCacheStore
// trackedChannels memoizes explicit allow-list membership, which changes only when an
// admin saves the list.
trackedChannels *lru.Cache[string, bool]
// trackableChannels memoizes whether a channel is neither a DM nor a GM. Channel types are
// mutable (ConvertGroupMessageToChannel rewrites one in place), so entries are dropped
// alongside channelByIdCache — see LocalCacheChannelStore.InvalidateChannel.
trackableChannels *lru.Cache[string, bool]
}
func (s *LocalCacheDeliveryTrackingStore) handleClusterInvalidateDeliveryTracking(msg *model.ClusterMessage) {
s.purge()
}
func (s *LocalCacheDeliveryTrackingStore) ClearCaches() {
s.purge()
s.purgeChannels()
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.deliveryTrackingCache.Name())
}
}
func (s *LocalCacheDeliveryTrackingStore) purge() {
if err := s.rootStore.deliveryTrackingCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge delivery tracking cache", mlog.Err(err))
}
s.trackedChannels.Purge()
}
func (s *LocalCacheDeliveryTrackingStore) SaveTrackedChannelIDs(rctx request.CTX, channelIDs []string) error {
if err := s.DeliveryTrackingStore.SaveTrackedChannelIDs(rctx, channelIDs); err != nil {
return err
}
s.purge()
s.rootStore.doClearCacheCluster(s.rootStore.deliveryTrackingCache)
return nil
}
func (s *LocalCacheDeliveryTrackingStore) GetTrackedChannelIDs(rctx request.CTX) ([]string, error) {
var channelIDs []string
if err := s.rootStore.doStandardReadCache(s.rootStore.deliveryTrackingCache, cacheKeyTrackedChannelIDs, &channelIDs); err == nil {
return channelIDs, nil
}
channelIDs, err := s.DeliveryTrackingStore.GetTrackedChannelIDs(rctx)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.deliveryTrackingCache, cacheKeyTrackedChannelIDs, channelIDs)
return channelIDs, nil
}
func (s *LocalCacheDeliveryTrackingStore) IsChannelTracked(rctx request.CTX, channelID string) (bool, error) {
if cached, ok := s.trackedChannels.Get(channelID); ok {
return cached, nil
}
tracked, err := s.DeliveryTrackingStore.IsChannelTracked(rctx, channelID)
if err != nil {
return false, err
}
s.trackedChannels.Add(channelID, tracked)
return tracked, nil
}
func (s *LocalCacheDeliveryTrackingStore) IsChannelTrackable(rctx request.CTX, channelID string) (bool, error) {
if cached, ok := s.trackableChannels.Get(channelID); ok {
return cached, nil
}
channel, err := s.rootStore.Channel().Get(channelID, true)
if err != nil {
var nfErr *store.ErrNotFound
if !errors.As(err, &nfErr) {
return false, err
}
// Not cached: a channel missing now — replica lag, or a type this store does not
// return — may resolve on a later read, and caching would exclude it permanently.
return false, nil
}
trackable := !channel.IsGroupOrDirect()
s.trackableChannels.Add(channelID, trackable)
return trackable, nil
}
// invalidateChannel and purgeChannels are called from the channel store, which is built before
// this one, so they tolerate a nil receiver.
func (s *LocalCacheDeliveryTrackingStore) invalidateChannel(channelID string) {
if s == nil {
return
}
s.trackableChannels.Remove(channelID)
}
func (s *LocalCacheDeliveryTrackingStore) purgeChannels() {
if s == nil {
return
}
s.trackableChannels.Purge()
}

View File

@@ -0,0 +1,198 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"errors"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestDeliveryTrackingStoreGetTrackedChannelIDsCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
rctx := request.TestContext(t)
setBasicMock := func(mockStore *mocks.Store, cachedStore LocalCacheStore) *mocks.DeliveryTrackingStore {
mockDeliveryTrackingStore := mockStore.DeliveryTracking().(*mocks.DeliveryTrackingStore)
mockDeliveryTrackingStore.On("GetTrackedChannelIDs", mock.Anything).Return([]string{"channel1", "channel2"}, nil)
mockDeliveryTrackingStore.On("SaveTrackedChannelIDs", mock.Anything, mock.Anything).Return(nil)
cachedStore.deliveryTracking.DeliveryTrackingStore = mockDeliveryTrackingStore
return mockDeliveryTrackingStore
}
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
mockDeliveryTrackingStore := setBasicMock(mockStore, cachedStore)
channelIDs, err := cachedStore.DeliveryTracking().GetTrackedChannelIDs(rctx)
require.NoError(t, err)
require.Equal(t, []string{"channel1", "channel2"}, channelIDs)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "GetTrackedChannelIDs", 1)
channelIDs, err = cachedStore.DeliveryTracking().GetTrackedChannelIDs(rctx)
require.NoError(t, err)
require.Equal(t, []string{"channel1", "channel2"}, channelIDs)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "GetTrackedChannelIDs", 1)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
mockDeliveryTrackingStore := setBasicMock(mockStore, cachedStore)
_, err = cachedStore.DeliveryTracking().GetTrackedChannelIDs(rctx)
require.NoError(t, err)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "GetTrackedChannelIDs", 1)
cachedStore.DeliveryTracking().ClearCaches()
_, err = cachedStore.DeliveryTracking().GetTrackedChannelIDs(rctx)
require.NoError(t, err)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "GetTrackedChannelIDs", 2)
})
t.Run("saving the list invalidates the cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
mockDeliveryTrackingStore := setBasicMock(mockStore, cachedStore)
_, err = cachedStore.DeliveryTracking().GetTrackedChannelIDs(rctx)
require.NoError(t, err)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "GetTrackedChannelIDs", 1)
require.NoError(t, cachedStore.DeliveryTracking().SaveTrackedChannelIDs(rctx, []string{"channel3"}))
_, err = cachedStore.DeliveryTracking().GetTrackedChannelIDs(rctx)
require.NoError(t, err)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "GetTrackedChannelIDs", 2)
})
}
func TestDeliveryTrackingStoreChannelEligibilityCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
rctx := request.TestContext(t)
openChannel := model.Channel{Id: "channel1", Type: model.ChannelTypeOpen}
dmChannel := model.Channel{Id: "channel2", Type: model.ChannelTypeDirect}
newCachedStore := func(t *testing.T) (LocalCacheStore, *mocks.DeliveryTrackingStore, *mocks.ChannelStore) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
mockDeliveryTrackingStore := mockStore.DeliveryTracking().(*mocks.DeliveryTrackingStore)
mockDeliveryTrackingStore.On("IsChannelTracked", mock.Anything, mock.Anything).Return(true, nil)
mockDeliveryTrackingStore.On("SaveTrackedChannelIDs", mock.Anything, mock.Anything).Return(nil)
cachedStore.deliveryTracking.DeliveryTrackingStore = mockDeliveryTrackingStore
mockChannelStore := mockStore.Channel().(*mocks.ChannelStore)
mockChannelStore.On("Get", openChannel.Id, true).Return(&openChannel, nil)
mockChannelStore.On("Get", dmChannel.Id, true).Return(&dmChannel, nil)
mockChannelStore.On("Get", "missing", true).Return(nil, store.NewErrNotFound("Channel", "missing"))
return cachedStore, mockDeliveryTrackingStore, mockChannelStore
}
t.Run("IsChannelTracked is resolved once per channel", func(t *testing.T) {
cachedStore, mockDeliveryTrackingStore, _ := newCachedStore(t)
for range 3 {
tracked, err := cachedStore.DeliveryTracking().IsChannelTracked(rctx, "channel1")
require.NoError(t, err)
require.True(t, tracked)
}
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "IsChannelTracked", 1)
_, err := cachedStore.DeliveryTracking().IsChannelTracked(rctx, "channel2")
require.NoError(t, err)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "IsChannelTracked", 2)
})
t.Run("IsChannelTrackable resolves through the channel store once per channel", func(t *testing.T) {
cachedStore, mockDeliveryTrackingStore, mockChannelStore := newCachedStore(t)
for range 3 {
trackable, err := cachedStore.DeliveryTracking().IsChannelTrackable(rctx, openChannel.Id)
require.NoError(t, err)
require.True(t, trackable)
}
mockChannelStore.AssertNumberOfCalls(t, "Get", 1)
// The dedicated store method is never consulted; the channel store is the source.
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "IsChannelTrackable", 0)
})
t.Run("DMs are not trackable", func(t *testing.T) {
cachedStore, _, _ := newCachedStore(t)
trackable, err := cachedStore.DeliveryTracking().IsChannelTrackable(rctx, dmChannel.Id)
require.NoError(t, err)
require.False(t, trackable)
})
t.Run("unknown channels are not trackable and are re-resolved", func(t *testing.T) {
cachedStore, _, mockChannelStore := newCachedStore(t)
for range 2 {
trackable, err := cachedStore.DeliveryTracking().IsChannelTrackable(rctx, "missing")
require.NoError(t, err)
require.False(t, trackable)
}
// Not-found is never memoized: the channel may exist on a later read, and a transient
// miss must not exclude it from auditing for the life of the process.
mockChannelStore.AssertNumberOfCalls(t, "Get", 2)
})
t.Run("saving the list re-resolves tracked but not trackable", func(t *testing.T) {
cachedStore, mockDeliveryTrackingStore, mockChannelStore := newCachedStore(t)
_, err := cachedStore.DeliveryTracking().IsChannelTracked(rctx, openChannel.Id)
require.NoError(t, err)
_, err = cachedStore.DeliveryTracking().IsChannelTrackable(rctx, openChannel.Id)
require.NoError(t, err)
require.NoError(t, cachedStore.DeliveryTracking().SaveTrackedChannelIDs(rctx, []string{"channel3"}))
_, err = cachedStore.DeliveryTracking().IsChannelTracked(rctx, openChannel.Id)
require.NoError(t, err)
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "IsChannelTracked", 2)
// Saving the allow-list purges trackableChannels too, but channelByIdCache still holds
// the channel, so no additional store read is needed to re-resolve it.
_, err = cachedStore.DeliveryTracking().IsChannelTrackable(rctx, openChannel.Id)
require.NoError(t, err)
mockChannelStore.AssertNumberOfCalls(t, "Get", 1)
})
t.Run("errors are not cached", func(t *testing.T) {
mockStore := getMockStore(t)
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, getMockCacheProvider(), logger)
require.NoError(t, err)
mockDeliveryTrackingStore := mockStore.DeliveryTracking().(*mocks.DeliveryTrackingStore)
mockDeliveryTrackingStore.On("IsChannelTracked", mock.Anything, mock.Anything).Return(false, errors.New("boom"))
cachedStore.deliveryTracking.DeliveryTrackingStore = mockDeliveryTrackingStore
for range 2 {
_, err = cachedStore.DeliveryTracking().IsChannelTracked(rctx, "channel1")
require.Error(t, err)
}
mockDeliveryTrackingStore.AssertNumberOfCalls(t, "IsChannelTracked", 2)
})
}

View File

@@ -0,0 +1,222 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"sync"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
)
// emojiNotFoundSentinel is stored in emojiIdCacheByName for names known to be
// absent from the DB; real emoji ids are always 26 characters, so no collision.
const emojiNotFoundSentinel = "-"
type LocalCacheEmojiStore struct {
store.EmojiStore
rootStore *LocalCacheStore
emojiByIdMut sync.Mutex
emojiByIdInvalidations map[string]bool
emojiByNameMut sync.Mutex
emojiByNameInvalidations map[string]bool
}
func (es *LocalCacheEmojiStore) Save(emoji *model.Emoji) (*model.Emoji, error) {
savedEmoji, err := es.EmojiStore.Save(emoji)
if err == nil {
// A negative cache entry may exist for this name if it was looked up
// before the emoji was created, so it needs to be invalidated.
es.emojiByNameMut.Lock()
es.emojiByNameInvalidations[savedEmoji.Name] = true
es.emojiByNameMut.Unlock()
es.rootStore.doInvalidateCacheCluster(es.rootStore.emojiIdCacheByName, savedEmoji.Name, nil)
}
return savedEmoji, err
}
func (es *LocalCacheEmojiStore) handleClusterInvalidateEmojiById(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
es.rootStore.emojiCacheById.Purge()
} else {
es.emojiByIdMut.Lock()
es.emojiByIdInvalidations[string(msg.Data)] = true
es.emojiByIdMut.Unlock()
es.rootStore.emojiCacheById.Remove(string(msg.Data))
}
}
func (es *LocalCacheEmojiStore) handleClusterInvalidateEmojiIdByName(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
es.rootStore.emojiIdCacheByName.Purge()
} else {
es.emojiByNameMut.Lock()
es.emojiByNameInvalidations[string(msg.Data)] = true
es.emojiByNameMut.Unlock()
es.rootStore.emojiIdCacheByName.Remove(string(msg.Data))
}
}
func (es *LocalCacheEmojiStore) Get(rctx request.CTX, id string, allowFromCache bool) (*model.Emoji, error) {
if allowFromCache {
if emoji, ok := es.getFromCacheById(id); ok {
return emoji, nil
}
}
// If it was invalidated, then we need to query master.
es.emojiByIdMut.Lock()
if es.emojiByIdInvalidations[id] {
// And then remove the key from the map.
rctx = sqlstore.RequestContextWithMaster(rctx)
delete(es.emojiByIdInvalidations, id)
}
es.emojiByIdMut.Unlock()
emoji, err := es.EmojiStore.Get(rctx, id, allowFromCache)
if allowFromCache && err == nil {
es.addToCache(emoji)
}
return emoji, err
}
func (es *LocalCacheEmojiStore) GetByName(rctx request.CTX, name string, allowFromCache bool) (*model.Emoji, error) {
if id, ok := model.GetSystemEmojiId(name); ok {
return es.Get(rctx, id, allowFromCache)
}
if allowFromCache {
if emoji, ok := es.getFromCacheByName(name); ok {
return emoji, nil
}
}
// If it was invalidated, then we need to query master.
es.emojiByNameMut.Lock()
if es.emojiByNameInvalidations[name] {
rctx = sqlstore.RequestContextWithMaster(rctx)
// And then remove the key from the map.
delete(es.emojiByNameInvalidations, name)
}
es.emojiByNameMut.Unlock()
emoji, err := es.EmojiStore.GetByName(rctx, name, allowFromCache)
if err != nil {
return nil, err
}
if allowFromCache {
es.addToCache(emoji)
}
return emoji, nil
}
func (es *LocalCacheEmojiStore) GetMultipleByName(rctx request.CTX, names []string) ([]*model.Emoji, error) {
emojis := []*model.Emoji{}
remainingEmojiNames := make([]string, 0)
for _, name := range names {
var emojiId string
if err := es.rootStore.doStandardReadCache(es.rootStore.emojiIdCacheByName, name, &emojiId); err == nil {
if emojiId == emojiNotFoundSentinel {
continue
}
if emoji, ok := es.getFromCacheById(emojiId); ok {
emojis = append(emojis, emoji)
continue
}
}
// If it was invalidated, then we need to query master.
es.emojiByNameMut.Lock()
if es.emojiByNameInvalidations[name] {
rctx = sqlstore.RequestContextWithMaster(rctx)
// And then remove the key from the map.
delete(es.emojiByNameInvalidations, name)
}
es.emojiByNameMut.Unlock()
remainingEmojiNames = append(remainingEmojiNames, name)
}
if len(remainingEmojiNames) > 0 {
remainingEmojis, err := es.EmojiStore.GetMultipleByName(rctx, remainingEmojiNames)
if err != nil {
return nil, err
}
foundNames := make(map[string]bool, len(remainingEmojis))
for _, emoji := range remainingEmojis {
es.addToCache(emoji)
emojis = append(emojis, emoji)
foundNames[emoji.Name] = true
}
// Cache the names that don't exist in the database with a short TTL,
// so they don't trigger a DB query on every subsequent lookup. Save
// invalidates these entries when an emoji with such a name is
// created. Names longer than the maximum can never exist and are
// skipped to avoid flooding the cache.
for _, name := range remainingEmojiNames {
if !foundNames[name] && len(name) <= model.EmojiNameMaxLength {
es.rootStore.doStandardAddToCacheWithExpiry(es.rootStore.emojiIdCacheByName, name, emojiNotFoundSentinel, EmojiNotFoundCacheSec*time.Second)
}
}
}
return emojis, nil
}
func (es *LocalCacheEmojiStore) Delete(emoji *model.Emoji, time int64) error {
err := es.EmojiStore.Delete(emoji, time)
if err == nil {
es.removeFromCache(emoji)
}
return err
}
func (es *LocalCacheEmojiStore) addToCache(emoji *model.Emoji) {
es.rootStore.doStandardAddToCache(es.rootStore.emojiCacheById, emoji.Id, emoji)
es.rootStore.doStandardAddToCache(es.rootStore.emojiIdCacheByName, emoji.Name, emoji.Id)
}
func (es *LocalCacheEmojiStore) getFromCacheById(id string) (*model.Emoji, bool) {
var emoji *model.Emoji
if err := es.rootStore.doStandardReadCache(es.rootStore.emojiCacheById, id, &emoji); err == nil {
return emoji, true
}
return nil, false
}
func (es *LocalCacheEmojiStore) getFromCacheByName(name string) (*model.Emoji, bool) {
var emojiId string
if err := es.rootStore.doStandardReadCache(es.rootStore.emojiIdCacheByName, name, &emojiId); err == nil {
if emojiId == emojiNotFoundSentinel {
return nil, false
}
return es.getFromCacheById(emojiId)
}
return nil, false
}
func (es *LocalCacheEmojiStore) removeFromCache(emoji *model.Emoji) {
es.emojiByIdMut.Lock()
es.emojiByIdInvalidations[emoji.Id] = true
es.emojiByIdMut.Unlock()
es.rootStore.doInvalidateCacheCluster(es.rootStore.emojiCacheById, emoji.Id, nil)
es.emojiByNameMut.Lock()
es.emojiByNameInvalidations[emoji.Name] = true
es.emojiByNameMut.Unlock()
es.rootStore.doInvalidateCacheCluster(es.rootStore.emojiIdCacheByName, emoji.Name, nil)
}

View File

@@ -0,0 +1,332 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestEmojiStore(t *testing.T) {
StoreTest(t, storetest.TestEmojiStore)
}
func TestEmojiStoreCache(t *testing.T) {
rctx := request.TestContext(t)
logger := mlog.CreateConsoleTestLogger(t)
fakeEmoji := model.Emoji{Id: "123", Name: "name123"}
fakeEmoji2 := model.Emoji{Id: "321", Name: "name321"}
ctxEmoji := model.Emoji{Id: "master", Name: "name123"}
t.Run("first call by id not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emoji, err := cachedStore.Emoji().Get(rctx, "123", true)
require.NoError(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
emoji, err = cachedStore.Emoji().Get(rctx, "123", true)
require.NoError(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("GetByName: first call by name not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emoji, err := cachedStore.Emoji().GetByName(rctx, "name123", true)
require.NoError(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
emoji, err = cachedStore.Emoji().GetByName(rctx, "name123", true)
require.NoError(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
})
t.Run("GetMultipleByName: first call by name not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123"})
require.NoError(t, err)
require.Len(t, emojis, 1)
assert.Equal(t, emojis[0], &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123"})
require.NoError(t, err)
require.Len(t, emojis, 1)
assert.Equal(t, emojis[0], &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
})
t.Run("GetMultipleByName: multiple elements", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123", "name321"})
require.NoError(t, err)
require.Len(t, emojis, 2)
assert.Equal(t, emojis[0], &fakeEmoji)
assert.Equal(t, emojis[1], &fakeEmoji2)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123"})
require.NoError(t, err)
require.Len(t, emojis, 1)
assert.Equal(t, emojis[0], &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
})
t.Run("GetMultipleByName: missing name cached negatively, second call skips store", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 0)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 0)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
})
t.Run("GetMultipleByName: mix of existing and missing names fully cached after first call", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123", "missing"})
require.NoError(t, err)
require.Len(t, emojis, 1)
assert.Equal(t, emojis[0], &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123", "missing"})
require.NoError(t, err)
require.Len(t, emojis, 1)
assert.Equal(t, emojis[0], &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
})
t.Run("GetMultipleByName: Save invalidates negative cache entry, emoji becomes visible via master", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 0)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
missingEmoji := model.Emoji{Id: "456", Name: "missing"}
_, err = cachedStore.Emoji().Save(&missingEmoji)
require.NoError(t, err)
// The mock returns the emoji only for a master-routed context, so this
// also pins that Save marks the name for master routing.
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 1)
assert.Equal(t, emojis[0], &missingEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 2)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 1)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 2)
})
t.Run("GetByName: negative cache entry does not short-circuit GetByName and is repaired by it", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 0)
missingEmoji := model.Emoji{Id: "456", Name: "missing"}
emoji, err := cachedStore.Emoji().GetByName(rctx, "missing", true)
require.NoError(t, err)
assert.Equal(t, emoji, &missingEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{"missing"})
require.NoError(t, err)
require.Len(t, emojis, 1)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
})
t.Run("GetMultipleByName: names longer than the maximum are not cached negatively", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
tooLongName := strings.Repeat("z", model.EmojiNameMaxLength+1)
emojis, err := cachedStore.Emoji().GetMultipleByName(rctx, []string{tooLongName})
require.NoError(t, err)
require.Len(t, emojis, 0)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 1)
emojis, err = cachedStore.Emoji().GetMultipleByName(rctx, []string{tooLongName})
require.NoError(t, err)
require.Len(t, emojis, 0)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 2)
})
t.Run("first call by id not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Get(rctx, "123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call by name not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().GetByName(rctx, "name123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call by id force not cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().Get(rctx, "123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call by name force not cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().GetByName(rctx, "name123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call by id, second call by name and GetMultipleByName cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 0)
cachedStore.Emoji().GetMultipleByName(rctx, []string{"name123"})
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetMultipleByName", 0)
})
t.Run("first call by name, second call by id cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 0)
})
t.Run("first call by id not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Delete(&fakeEmoji, 0)
cachedStore.Emoji().Get(rctx, "123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("call by id, use master", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().Get(rctx, "master", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Delete(&ctxEmoji, 0)
cachedStore.Emoji().Get(rctx, "master", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call by name not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().Delete(&fakeEmoji, 0)
cachedStore.Emoji().GetByName(rctx, "name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("call by name, use master", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Emoji().GetByName(rctx, "master", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().Delete(&ctxEmoji, 0)
cachedStore.Emoji().GetByName(rctx, "master", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
}

View File

@@ -0,0 +1,139 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"fmt"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheFileInfoStore struct {
store.FileInfoStore
rootStore *LocalCacheStore
}
func (s *LocalCacheFileInfoStore) handleClusterInvalidateFileInfo(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.fileInfoCache.Purge()
return
}
s.rootStore.fileInfoCache.Remove(string(msg.Data))
}
func (s LocalCacheFileInfoStore) GetForPost(postId string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error) {
if !allowFromCache {
return s.FileInfoStore.GetForPost(postId, readFromMaster, includeDeleted, allowFromCache)
}
cacheKey := postId
if includeDeleted {
cacheKey += "_deleted"
}
var fileInfo []*model.FileInfo
if err := s.rootStore.doStandardReadCache(s.rootStore.fileInfoCache, cacheKey, &fileInfo); err == nil {
return fileInfo, nil
}
fileInfos, err := s.FileInfoStore.GetForPost(postId, readFromMaster, includeDeleted, allowFromCache)
if err != nil {
return nil, err
}
if len(fileInfos) > 0 {
s.rootStore.doStandardAddToCache(s.rootStore.fileInfoCache, cacheKey, fileInfos)
}
return fileInfos, nil
}
func (s LocalCacheFileInfoStore) GetByIds(ids []string, includeDeleted, allowFromCache, readFromMaster bool) ([]*model.FileInfo, error) {
if readFromMaster {
return s.FileInfoStore.GetByIds(ids, includeDeleted, false, readFromMaster)
}
if !allowFromCache {
return s.FileInfoStore.GetByIds(ids, includeDeleted, allowFromCache, readFromMaster)
}
var fileIdsToFetch []string
var fileInfos []*model.FileInfo
for _, fileId := range ids {
cacheKey := fmt.Sprintf("%s_%t", fileId, includeDeleted)
var fileInfo *model.FileInfo
if err := s.rootStore.doStandardReadCache(s.rootStore.fileInfoCache, cacheKey, &fileInfo); err == nil {
fileInfos = append(fileInfos, fileInfo)
} else {
fileIdsToFetch = append(fileIdsToFetch, fileId)
}
}
if len(fileIdsToFetch) > 0 {
fetchedFileInfos, err := s.FileInfoStore.GetByIds(fileIdsToFetch, includeDeleted, false, readFromMaster)
if err != nil {
return nil, err
}
for _, fileInfo := range fetchedFileInfos {
cacheKey := fmt.Sprintf("%s_%t", fileInfo.Id, includeDeleted)
s.rootStore.doStandardAddToCache(s.rootStore.fileInfoCache, cacheKey, fileInfo)
fileInfos = append(fileInfos, fileInfo)
}
}
return fileInfos, nil
}
func (s LocalCacheFileInfoStore) ClearCaches() {
s.rootStore.fileInfoCache.Purge()
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.fileInfoCache.Name())
}
}
func (s LocalCacheFileInfoStore) InvalidateFileInfosForPostCache(postId string, deleted bool) {
cacheKey := postId
if deleted {
cacheKey += "_deleted"
}
s.rootStore.doInvalidateCacheCluster(s.rootStore.fileInfoCache, cacheKey, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.fileInfoCache.Name())
}
}
func (s LocalCacheFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) {
storageUsageKey := "storage_usage"
if includeDeleted {
storageUsageKey += "_deleted"
}
if !allowFromCache {
usage, err := s.FileInfoStore.GetStorageUsage(allowFromCache, includeDeleted)
if err != nil {
return 0, err
}
s.rootStore.doStandardAddToCache(s.rootStore.fileInfoCache, storageUsageKey, usage)
return usage, nil
}
var usage int64
if err := s.rootStore.doStandardReadCache(s.rootStore.fileInfoCache, storageUsageKey, &usage); err == nil {
return usage, nil
}
usage, err := s.FileInfoStore.GetStorageUsage(allowFromCache, includeDeleted)
if err != nil {
return 0, err
}
s.rootStore.doStandardAddToCache(s.rootStore.fileInfoCache, storageUsageKey, usage)
return usage, nil
}

View File

@@ -0,0 +1,82 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestFileInfoStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestFileInfoStore)
}
func TestFileInfoStoreCache(t *testing.T) {
fakeFileInfo := model.FileInfo{Id: "123", PostId: "123"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
fileInfos, err := cachedStore.FileInfo().GetForPost("123", true, true, true)
require.NoError(t, err)
assert.Equal(t, fileInfos, []*model.FileInfo{&fakeFileInfo})
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetForPost", 1)
assert.Equal(t, fileInfos, []*model.FileInfo{&fakeFileInfo})
cachedStore.FileInfo().GetForPost("123", true, true, true)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetForPost", 1)
})
t.Run("first call not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.FileInfo().GetForPost("123", true, true, true)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetForPost", 1)
cachedStore.FileInfo().GetForPost("123", true, true, false)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetForPost", 2)
})
t.Run("first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.FileInfo().GetForPost("123", true, true, true)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetForPost", 1)
cachedStore.FileInfo().InvalidateFileInfosForPostCache("123", true)
cachedStore.FileInfo().GetForPost("123", true, true, true)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetForPost", 2)
})
t.Run("GetByIds cache test", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
fileInfos, err := cachedStore.FileInfo().GetByIds([]string{"123"}, true, true, false)
require.NoError(t, err)
assert.Equal(t, []*model.FileInfo{&fakeFileInfo}, fileInfos)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetByIds", 1)
fileInfosCached, err := cachedStore.FileInfo().GetByIds([]string{"123"}, true, true, false)
require.NoError(t, err)
assert.Equal(t, []*model.FileInfo{&fakeFileInfo}, fileInfosCached)
mockStore.FileInfo().(*mocks.FileInfoStore).AssertNumberOfCalls(t, "GetByIds", 1)
})
}

View File

@@ -0,0 +1,836 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"runtime"
"time"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/einterfaces"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
const (
ReactionCacheSize = 20000
ReactionCacheSec = 30 * 60
RoleCacheSize = 20000
RoleCacheSec = 30 * 60
SchemeCacheSize = 20000
SchemeCacheSec = 30 * 60
FileInfoCacheSize = 25000
FileInfoCacheSec = 30 * 60
ChannelGuestCountCacheSize = model.ChannelCacheSize
ChannelGuestCountCacheSec = 30 * 60
WebhookCacheSize = 25000
WebhookCacheSec = 15 * 60
EmojiCacheSize = 5000
EmojiCacheSec = 30 * 60
// Kept short so that a lost cluster invalidation or a stale replica read
// cannot hide a newly created emoji for longer than this.
EmojiNotFoundCacheSec = 60
ChannelPinnedPostsCountsCacheSize = model.ChannelCacheSize
ChannelPinnedPostsCountsCacheSec = 30 * 60
AllChannelMembersForUserCacheSize = model.SessionCacheSize
AllChannelMembersForUserCacheDuration = 15 * time.Minute
AllChannelMembersNotifyPropsForChannelCacheSize = model.SessionCacheSize
AllChannelMembersNotifyPropsForChannelCacheDuration = 30 * time.Minute
ChannelCacheDuration = 15 * time.Minute
ChannelMembersCountsCacheSize = model.ChannelCacheSize
ChannelMembersCountsCacheSec = 30 * 60
LastPostsCacheSize = 20000
LastPostsCacheSec = 30 * 60
PostsUsageCacheSize = 1
PostsUsageCacheSec = 30 * 60
TermsOfServiceCacheSize = 20000
TermsOfServiceCacheSec = 30 * 60
LastPostTimeCacheSize = 25000
LastPostTimeCacheSec = 15 * 60
UserProfileByIDCacheSize = 20000
UserProfileByIDSec = 30 * 60
ProfilesInChannelCacheSize = model.ChannelCacheSize
ProfilesInChannelCacheSec = 15 * 60
TeamCacheSize = 20000
TeamCacheSec = 30 * 60
ChannelCacheSec = 15 * 60 // 15 mins
// Auto-translation caches
UserAutoTranslationCacheSize = 50000 // User+channel combos
UserAutoTranslationCacheSec = 15 * 60
PostTranslationEtagCacheSize = 25000 // Channel etags for post translations
PostTranslationEtagCacheSec = 15 * 60
ContentFlaggingCacheSize = 100
DeliveryTrackingCacheSize = 100
DeliveryTrackingChannelCacheSize = 20000
ReadReceiptCacheSize = 50000
TemporaryPostCacheSize = 10000
TemporaryPostCacheMinutes = 60
SessionAttributeCacheSize = model.SessionCacheSize
PropertyFieldCacheSize = 100
PropertyFieldCacheSec = 30 * 60
AccessControlPolicyEtagCacheSize = 25000 // Per-channel render-ETag epochs
AccessControlPolicyEtagCacheSec = 15 * 60
UserPropertyValuesEpochCacheSize = 25000 // Per-user CPA epochs
UserPropertyValuesEpochCacheSec = 15 * 60
)
var clearCacheMessageData = []byte("")
type LocalCacheStore struct {
store.Store
cacheType string
metrics einterfaces.MetricsInterface
cluster einterfaces.ClusterInterface
logger mlog.LoggerIFace
reaction LocalCacheReactionStore
reactionCache cache.Cache
fileInfo LocalCacheFileInfoStore
fileInfoCache cache.Cache
role LocalCacheRoleStore
roleCache cache.Cache
rolePermissionsCache cache.Cache
scheme LocalCacheSchemeStore
schemeCache cache.Cache
emoji *LocalCacheEmojiStore
emojiCacheById cache.Cache
emojiIdCacheByName cache.Cache
channel LocalCacheChannelStore
channelMemberCountsCache cache.Cache
channelGuestCountCache cache.Cache
channelPinnedPostCountsCache cache.Cache
channelByIdCache cache.Cache
channelMembersForUserCache cache.Cache
channelMembersNotifyPropsCache cache.Cache
channelByNameCache cache.Cache
webhook LocalCacheWebhookStore
webhookCache cache.Cache
post LocalCachePostStore
postLastPostsCache cache.Cache
lastPostTimeCache cache.Cache
postsUsageCache cache.Cache
user *LocalCacheUserStore
allUserCache cache.Cache
userProfileByIdsCache cache.Cache
profilesInChannelCache cache.Cache
team LocalCacheTeamStore
teamAllTeamIdsForUserCache cache.Cache
termsOfService LocalCacheTermsOfServiceStore
termsOfServiceCache cache.Cache
autotranslation LocalCacheAutoTranslationStore
userAutoTranslationCache cache.Cache
postTranslationEtagCache cache.Cache
contentFlagging LocalCacheContentFlaggingStore
contentFlaggingCache cache.Cache
deliveryTracking *LocalCacheDeliveryTrackingStore
deliveryTrackingCache cache.Cache
readReceipt LocalCacheReadReceiptStore
readReceiptCache cache.Cache
readReceiptPostReadersCache cache.Cache
readReceiptPostUnreadCountCache cache.Cache
temporaryPost LocalCacheTemporaryPostStore
temporaryPostCache cache.Cache
sessionAttribute LocalCacheSessionAttributeStore
sessionAttributeCache cache.Cache
propertyField LocalCachePropertyFieldStore
propertyFieldCache cache.Cache
accessControlPolicy LocalCacheAccessControlPolicyStore
accessControlPolicyEtagCache cache.Cache
attributes LocalCacheAttributesStore
userPropertyValuesEpochCache cache.Cache
}
func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterface, cluster einterfaces.ClusterInterface, cacheProvider cache.Provider, logger mlog.LoggerIFace) (localCacheStore LocalCacheStore, err error) {
localCacheStore = LocalCacheStore{
Store: baseStore,
cluster: cluster,
metrics: metrics,
cacheType: cacheProvider.Type(),
logger: logger,
}
// Reactions
if localCacheStore.reactionCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ReactionCacheSize,
Name: "Reaction",
DefaultExpiry: ReactionCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForReactions,
}); err != nil {
return
}
localCacheStore.reaction = LocalCacheReactionStore{ReactionStore: baseStore.Reaction(), rootStore: &localCacheStore}
// Roles
if localCacheStore.roleCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: RoleCacheSize,
Name: "Role",
DefaultExpiry: RoleCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForRoles,
Striped: true,
StripedBuckets: max(runtime.NumCPU()-1, 1),
}); err != nil {
return
}
if localCacheStore.rolePermissionsCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: RoleCacheSize,
Name: "RolePermission",
DefaultExpiry: RoleCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForRolePermissions,
}); err != nil {
return
}
localCacheStore.role = LocalCacheRoleStore{RoleStore: baseStore.Role(), rootStore: &localCacheStore}
// Schemes
if localCacheStore.schemeCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: SchemeCacheSize,
Name: "Scheme",
DefaultExpiry: SchemeCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForSchemes,
}); err != nil {
return
}
localCacheStore.scheme = LocalCacheSchemeStore{SchemeStore: baseStore.Scheme(), rootStore: &localCacheStore}
// FileInfo
if localCacheStore.fileInfoCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: FileInfoCacheSize,
Name: "FileInfo",
DefaultExpiry: FileInfoCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForFileInfos,
}); err != nil {
return
}
localCacheStore.fileInfo = LocalCacheFileInfoStore{FileInfoStore: baseStore.FileInfo(), rootStore: &localCacheStore}
// Webhooks
if localCacheStore.webhookCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: WebhookCacheSize,
Name: "Webhook",
DefaultExpiry: WebhookCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForWebhooks,
}); err != nil {
return
}
localCacheStore.webhook = LocalCacheWebhookStore{WebhookStore: baseStore.Webhook(), rootStore: &localCacheStore}
// Emojis
if localCacheStore.emojiCacheById, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: EmojiCacheSize,
Name: "EmojiById",
DefaultExpiry: EmojiCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForEmojisById,
}); err != nil {
return
}
if localCacheStore.emojiIdCacheByName, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: EmojiCacheSize,
Name: "EmojiByName",
DefaultExpiry: EmojiCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForEmojisIdByName,
}); err != nil {
return
}
localCacheStore.emoji = &LocalCacheEmojiStore{
EmojiStore: baseStore.Emoji(),
rootStore: &localCacheStore,
emojiByIdInvalidations: make(map[string]bool),
emojiByNameInvalidations: make(map[string]bool),
}
// Channels
if localCacheStore.channelPinnedPostCountsCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ChannelPinnedPostsCountsCacheSize,
Name: "ChannelPinnedPostsCounts",
DefaultExpiry: ChannelPinnedPostsCountsCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForChannelPinnedpostsCounts,
}); err != nil {
return
}
if localCacheStore.channelMemberCountsCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ChannelMembersCountsCacheSize,
Name: "ChannelMemberCounts",
DefaultExpiry: ChannelMembersCountsCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForChannelMemberCounts,
}); err != nil {
return
}
if localCacheStore.channelGuestCountCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ChannelGuestCountCacheSize,
Name: "ChannelGuestsCount",
DefaultExpiry: ChannelGuestCountCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForChannelGuestCount,
}); err != nil {
return
}
if localCacheStore.channelByIdCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: model.ChannelCacheSize,
Name: "channelById",
DefaultExpiry: ChannelCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForChannel,
}); err != nil {
return
}
if localCacheStore.channelMembersForUserCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: AllChannelMembersForUserCacheSize,
Name: "ChannelMembersForUser",
DefaultExpiry: AllChannelMembersForUserCacheDuration,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForUser,
}); err != nil {
return
}
if localCacheStore.channelMembersNotifyPropsCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: AllChannelMembersNotifyPropsForChannelCacheSize,
Name: "ChannnelMembersNotifyProps",
DefaultExpiry: AllChannelMembersNotifyPropsForChannelCacheDuration,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForChannelMembersNotifyProps,
}); err != nil {
return
}
if localCacheStore.channelByNameCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: model.ChannelCacheSize,
Name: "ChannelByName",
DefaultExpiry: ChannelCacheDuration,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForChannelByName,
}); err != nil {
return
}
localCacheStore.channel = LocalCacheChannelStore{ChannelStore: baseStore.Channel(), rootStore: &localCacheStore}
// Posts
if localCacheStore.postLastPostsCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: LastPostsCacheSize,
Name: "LastPost",
DefaultExpiry: LastPostsCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForLastPosts,
}); err != nil {
return
}
if localCacheStore.lastPostTimeCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: LastPostTimeCacheSize,
Name: "LastPostTime",
DefaultExpiry: LastPostTimeCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForLastPostTime,
}); err != nil {
return
}
if localCacheStore.postsUsageCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: PostsUsageCacheSize,
Name: "PostsUsage",
DefaultExpiry: PostsUsageCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForPostsUsage,
}); err != nil {
return
}
localCacheStore.post = LocalCachePostStore{PostStore: baseStore.Post(), rootStore: &localCacheStore}
// TOS
if localCacheStore.termsOfServiceCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: TermsOfServiceCacheSize,
Name: "TermsOfService",
DefaultExpiry: TermsOfServiceCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForTermsOfService,
}); err != nil {
return
}
localCacheStore.termsOfService = LocalCacheTermsOfServiceStore{TermsOfServiceStore: baseStore.TermsOfService(), rootStore: &localCacheStore}
// Users
if localCacheStore.allUserCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: 1,
Name: "AllUserProfiles",
DefaultExpiry: UserProfileByIDSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForAllProfiles,
}); err != nil {
return
}
if localCacheStore.userProfileByIdsCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: UserProfileByIDCacheSize,
Name: "UserProfileByIds",
DefaultExpiry: UserProfileByIDSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForProfileByIds,
Striped: true,
StripedBuckets: max(runtime.NumCPU()-1, 1),
}); err != nil {
return
}
// Hardcoding this to LRU because of the volume of SCAN calls in case of Redis.
if localCacheStore.profilesInChannelCache, err = cache.NewProvider().NewCache(&cache.CacheOptions{
Size: ProfilesInChannelCacheSize,
Name: "ProfilesInChannel",
DefaultExpiry: ProfilesInChannelCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForProfileInChannel,
}); err != nil {
return
}
localCacheStore.user = &LocalCacheUserStore{
UserStore: baseStore.User(),
rootStore: &localCacheStore,
userProfileByIdsInvalidations: make(map[string]bool),
}
// Teams
if localCacheStore.teamAllTeamIdsForUserCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: TeamCacheSize,
Name: "Team",
DefaultExpiry: TeamCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForTeams,
}); err != nil {
return
}
localCacheStore.team = LocalCacheTeamStore{TeamStore: baseStore.Team(), rootStore: &localCacheStore}
// Auto-translation
if localCacheStore.userAutoTranslationCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: UserAutoTranslationCacheSize,
Name: "UserAutoTranslation",
DefaultExpiry: UserAutoTranslationCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForUserAutoTranslation,
}); err != nil {
return
}
if localCacheStore.postTranslationEtagCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: PostTranslationEtagCacheSize,
Name: "PostTranslationEtag",
DefaultExpiry: PostTranslationEtagCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForPostTranslationEtag,
}); err != nil {
return
}
localCacheStore.autotranslation = LocalCacheAutoTranslationStore{AutoTranslationStore: baseStore.AutoTranslation(), rootStore: &localCacheStore}
if localCacheStore.contentFlaggingCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ContentFlaggingCacheSize,
Name: "ContentFlagging",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForContentFlagging,
}); err != nil {
return
}
localCacheStore.contentFlagging = LocalCacheContentFlaggingStore{ContentFlaggingStore: baseStore.ContentFlagging(), rootStore: &localCacheStore}
if localCacheStore.deliveryTrackingCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: DeliveryTrackingCacheSize,
Name: "DeliveryTracking",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForDeliveryTracking,
}); err != nil {
return
}
var trackedChannels, trackableChannels *lru.Cache[string, bool]
if trackedChannels, err = lru.New[string, bool](DeliveryTrackingChannelCacheSize); err != nil {
return
}
if trackableChannels, err = lru.New[string, bool](DeliveryTrackingChannelCacheSize); err != nil {
return
}
localCacheStore.deliveryTracking = &LocalCacheDeliveryTrackingStore{
DeliveryTrackingStore: baseStore.DeliveryTracking(),
rootStore: &localCacheStore,
trackedChannels: trackedChannels,
trackableChannels: trackableChannels,
}
// Read Receipts
if localCacheStore.readReceiptCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ReadReceiptCacheSize,
Name: "ReadReceipt",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForReadReceipts,
}); err != nil {
return
}
if localCacheStore.readReceiptPostReadersCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ReadReceiptCacheSize,
Name: "ReadReceiptPostReaders",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForReadReceipts,
}); err != nil {
return
}
if localCacheStore.readReceiptPostUnreadCountCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: ReadReceiptCacheSize,
Name: "ReadReceiptPostUnreadCount",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForReadReceipts,
}); err != nil {
return
}
localCacheStore.readReceipt = LocalCacheReadReceiptStore{ReadReceiptStore: baseStore.ReadReceipt(), rootStore: &localCacheStore}
// Temporary Posts
if localCacheStore.temporaryPostCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: TemporaryPostCacheSize,
Name: "TemporaryPost",
DefaultExpiry: TemporaryPostCacheMinutes * time.Minute,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForTemporaryPosts,
}); err != nil {
return
}
localCacheStore.temporaryPost = LocalCacheTemporaryPostStore{TemporaryPostStore: baseStore.TemporaryPost(), rootStore: &localCacheStore}
// Session Attributes
if localCacheStore.sessionAttributeCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: SessionAttributeCacheSize,
Name: "SessionAttribute",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForSessionAttributes,
}); err != nil {
return
}
localCacheStore.sessionAttribute = LocalCacheSessionAttributeStore{SessionAttributeStore: baseStore.SessionAttribute(), rootStore: &localCacheStore}
// Property Fields
if localCacheStore.propertyFieldCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: PropertyFieldCacheSize,
Name: "PropertyField",
DefaultExpiry: PropertyFieldCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForPropertyFields,
}); err != nil {
return
}
localCacheStore.propertyField = LocalCachePropertyFieldStore{PropertyFieldStore: baseStore.PropertyField(), rootStore: &localCacheStore}
// Access Control Policy render-ETag epochs
if localCacheStore.accessControlPolicyEtagCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: AccessControlPolicyEtagCacheSize,
Name: "AccessControlPolicyEtag",
DefaultExpiry: AccessControlPolicyEtagCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForAccessControlPolicyEtag,
}); err != nil {
return
}
localCacheStore.accessControlPolicy = LocalCacheAccessControlPolicyStore{AccessControlPolicyStore: baseStore.AccessControlPolicy(), rootStore: &localCacheStore}
// User property-values (CPA) epochs
if localCacheStore.userPropertyValuesEpochCache, err = cacheProvider.NewCache(&cache.CacheOptions{
Size: UserPropertyValuesEpochCacheSize,
Name: "UserPropertyValuesEpoch",
DefaultExpiry: UserPropertyValuesEpochCacheSec * time.Second,
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForUserPropertyValuesEpoch,
}); err != nil {
return
}
localCacheStore.attributes = LocalCacheAttributesStore{AttributesStore: baseStore.Attributes(), rootStore: &localCacheStore}
if cluster != nil {
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForReactions, localCacheStore.reaction.handleClusterInvalidateReaction)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForRoles, localCacheStore.role.handleClusterInvalidateRole)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForRolePermissions, localCacheStore.role.handleClusterInvalidateRolePermissions)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForSchemes, localCacheStore.scheme.handleClusterInvalidateScheme)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForFileInfos, localCacheStore.fileInfo.handleClusterInvalidateFileInfo)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForLastPostTime, localCacheStore.post.handleClusterInvalidateLastPostTime)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForPostsUsage, localCacheStore.post.handleClusterInvalidatePostsUsage)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForWebhooks, localCacheStore.webhook.handleClusterInvalidateWebhook)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForEmojisById, localCacheStore.emoji.handleClusterInvalidateEmojiById)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForEmojisIdByName, localCacheStore.emoji.handleClusterInvalidateEmojiIdByName)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelPinnedpostsCounts, localCacheStore.channel.handleClusterInvalidateChannelPinnedPostCount)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelMemberCounts, localCacheStore.channel.handleClusterInvalidateChannelMemberCounts)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelGuestCount, localCacheStore.channel.handleClusterInvalidateChannelGuestCounts)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannel, localCacheStore.channel.handleClusterInvalidateChannelById)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForUser, localCacheStore.channel.handleClusterInvalidateChannelForUser)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelMembersNotifyProps, localCacheStore.channel.handleClusterInvalidateChannelMembersNotifyProps)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelByName, localCacheStore.channel.handleClusterInvalidateChannelByName)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForLastPosts, localCacheStore.post.handleClusterInvalidateLastPosts)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForTermsOfService, localCacheStore.termsOfService.handleClusterInvalidateTermsOfService)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForProfileByIds, localCacheStore.user.handleClusterInvalidateScheme)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForProfileInChannel, localCacheStore.user.handleClusterInvalidateProfilesInChannel)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForAllProfiles, localCacheStore.user.handleClusterInvalidateAllProfiles)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForTeams, localCacheStore.team.handleClusterInvalidateTeam)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForUserAutoTranslation, localCacheStore.autotranslation.handleClusterInvalidateUserAutoTranslation)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForPostTranslationEtag, localCacheStore.autotranslation.handleClusterInvalidatePostTranslationEtag)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForContentFlagging, localCacheStore.contentFlagging.handleClusterInvalidateContentFlagging)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForDeliveryTracking, localCacheStore.deliveryTracking.handleClusterInvalidateDeliveryTracking)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForReadReceipts, localCacheStore.readReceipt.handleClusterInvalidateReadReceipts)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForTemporaryPosts, localCacheStore.temporaryPost.handleClusterInvalidateTemporaryPosts)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForSessionAttributes, localCacheStore.sessionAttribute.handleClusterInvalidateSessionAttributes)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForPropertyFields, localCacheStore.propertyField.handleClusterInvalidatePropertyField)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForAccessControlPolicyEtag, localCacheStore.accessControlPolicy.handleClusterInvalidateAccessControlPolicyEtag)
cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForUserPropertyValuesEpoch, localCacheStore.attributes.handleClusterInvalidateUserPropertyValuesEpoch)
}
return
}
func (s LocalCacheStore) Reaction() store.ReactionStore {
return s.reaction
}
func (s LocalCacheStore) Role() store.RoleStore {
return s.role
}
func (s LocalCacheStore) Scheme() store.SchemeStore {
return s.scheme
}
func (s LocalCacheStore) FileInfo() store.FileInfoStore {
return s.fileInfo
}
func (s LocalCacheStore) Webhook() store.WebhookStore {
return s.webhook
}
func (s LocalCacheStore) Emoji() store.EmojiStore {
return s.emoji
}
func (s LocalCacheStore) Channel() store.ChannelStore {
return s.channel
}
func (s LocalCacheStore) Post() store.PostStore {
return s.post
}
func (s LocalCacheStore) TermsOfService() store.TermsOfServiceStore {
return s.termsOfService
}
func (s LocalCacheStore) User() store.UserStore {
return s.user
}
func (s LocalCacheStore) Team() store.TeamStore {
return s.team
}
func (s LocalCacheStore) AutoTranslation() store.AutoTranslationStore {
return s.autotranslation
}
func (s LocalCacheStore) ContentFlagging() store.ContentFlaggingStore {
return s.contentFlagging
}
func (s LocalCacheStore) DeliveryTracking() store.DeliveryTrackingStore {
return s.deliveryTracking
}
func (s LocalCacheStore) ReadReceipt() store.ReadReceiptStore {
return s.readReceipt
}
func (s LocalCacheStore) TemporaryPost() store.TemporaryPostStore {
return s.temporaryPost
}
func (s LocalCacheStore) SessionAttribute() store.SessionAttributeStore {
return &s.sessionAttribute
}
func (s LocalCacheStore) PropertyField() store.PropertyFieldStore {
return s.propertyField
}
func (s LocalCacheStore) AccessControlPolicy() store.AccessControlPolicyStore {
return s.accessControlPolicy
}
func (s LocalCacheStore) Attributes() store.AttributesStore {
return s.attributes
}
func (s LocalCacheStore) DropAllTables() {
s.Invalidate()
s.Store.DropAllTables()
}
func (s *LocalCacheStore) doInvalidateCacheCluster(cache cache.Cache, key string, props map[string]string) {
err := cache.Remove(key)
if err != nil {
s.logger.Warn("Error while removing cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
if s.cluster != nil && cache.GetInvalidateClusterEvent() != model.ClusterEventNone {
msg := &model.ClusterMessage{
Event: cache.GetInvalidateClusterEvent(),
SendType: model.ClusterSendBestEffort,
Data: []byte(key),
}
if props != nil {
msg.Props = props
}
s.cluster.SendClusterMessage(msg)
}
}
func (s *LocalCacheStore) doMultiInvalidateCacheCluster(cache cache.Cache, keys []string, props map[string]string) {
err := cache.RemoveMulti(keys)
if err != nil {
s.logger.Warn("Error while removing cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
if s.cluster != nil && cache.GetInvalidateClusterEvent() != model.ClusterEventNone {
for _, key := range keys {
msg := &model.ClusterMessage{
Event: cache.GetInvalidateClusterEvent(),
SendType: model.ClusterSendBestEffort,
Data: []byte(key),
}
if props != nil {
msg.Props = props
}
s.cluster.SendClusterMessage(msg)
}
}
}
func (s *LocalCacheStore) doStandardAddToCache(cache cache.Cache, key string, value any) {
err := cache.SetWithDefaultExpiry(key, value)
if err != nil {
s.logger.Warn("Error while setting cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
}
func (s *LocalCacheStore) doStandardAddToCacheWithExpiry(cache cache.Cache, key string, value any, ttl time.Duration) {
err := cache.SetWithExpiry(key, value, ttl)
if err != nil {
s.logger.Warn("Error while setting cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
}
func (s *LocalCacheStore) doStandardReadCache(c cache.Cache, key string, value any) error {
err := c.Get(key, value)
if err == nil {
if s.metrics != nil {
s.metrics.IncrementMemCacheHitCounter(c.Name())
}
return nil
}
if err != cache.ErrKeyNotFound {
s.logger.Warn("Error while reading from cache", mlog.Err(err), mlog.String("cache_name", c.Name()))
}
if s.metrics != nil {
s.metrics.IncrementMemCacheMissCounter(c.Name())
}
return err
}
func (s *LocalCacheStore) doMultiReadCache(cache cache.Cache, keys []string, values []any) []error {
errs := cache.GetMulti(keys, values)
if s.metrics != nil {
for _, err := range errs {
if err == nil {
s.metrics.IncrementMemCacheHitCounter(cache.Name())
continue
}
s.metrics.IncrementMemCacheMissCounter(cache.Name())
}
}
return errs
}
func (s *LocalCacheStore) doIncrementCache(cache cache.ExternalCache, key string, val int) {
err := cache.Increment(key, val)
if err != nil {
s.logger.Warn("Error while incrementing cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
}
func (s *LocalCacheStore) doDecrementCache(cache cache.ExternalCache, key string, val int) {
err := cache.Decrement(key, val)
if err != nil {
s.logger.Warn("Error while decrementing cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
}
func (s *LocalCacheStore) doClearCacheCluster(cache cache.Cache) {
err := cache.Purge()
if err != nil {
s.logger.Warn("Error while purging cache", mlog.Err(err), mlog.String("cache_name", cache.Name()))
}
if s.cluster != nil && cache.GetInvalidateClusterEvent() != model.ClusterEventNone {
msg := &model.ClusterMessage{
Event: cache.GetInvalidateClusterEvent(),
SendType: model.ClusterSendBestEffort,
Data: clearCacheMessageData,
}
s.cluster.SendClusterMessage(msg)
}
}
func (s *LocalCacheStore) Invalidate() {
s.doClearCacheCluster(s.reactionCache)
s.doClearCacheCluster(s.schemeCache)
s.doClearCacheCluster(s.roleCache)
s.doClearCacheCluster(s.fileInfoCache)
s.doClearCacheCluster(s.webhookCache)
s.doClearCacheCluster(s.emojiCacheById)
s.doClearCacheCluster(s.emojiIdCacheByName)
s.doClearCacheCluster(s.channelMemberCountsCache)
s.doClearCacheCluster(s.channelPinnedPostCountsCache)
s.doClearCacheCluster(s.channelGuestCountCache)
s.doClearCacheCluster(s.channelByIdCache)
s.doClearCacheCluster(s.channelMembersForUserCache)
s.doClearCacheCluster(s.channelMembersNotifyPropsCache)
s.doClearCacheCluster(s.channelByNameCache)
s.doClearCacheCluster(s.postLastPostsCache)
s.doClearCacheCluster(s.termsOfServiceCache)
s.doClearCacheCluster(s.lastPostTimeCache)
s.doClearCacheCluster(s.userProfileByIdsCache)
s.doClearCacheCluster(s.allUserCache)
s.doClearCacheCluster(s.profilesInChannelCache)
s.doClearCacheCluster(s.teamAllTeamIdsForUserCache)
s.doClearCacheCluster(s.rolePermissionsCache)
s.doClearCacheCluster(s.userAutoTranslationCache)
s.doClearCacheCluster(s.postTranslationEtagCache)
s.doClearCacheCluster(s.deliveryTrackingCache)
s.doClearCacheCluster(s.readReceiptCache)
s.doClearCacheCluster(s.readReceiptPostReadersCache)
s.doClearCacheCluster(s.temporaryPostCache)
s.doClearCacheCluster(s.sessionAttributeCache)
s.doClearCacheCluster(s.propertyFieldCache)
s.doClearCacheCluster(s.accessControlPolicyEtagCache)
s.doClearCacheCluster(s.userPropertyValuesEpochCache)
}
// allocateCacheTargets is used to fill target value types
// for getting items from cache.
func allocateCacheTargets[T any](l int) []any {
toPass := make([]any, 0, l)
for range l {
toPass = append(toPass, new(T))
}
return toPass
}

View File

@@ -0,0 +1,167 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"sync"
"testing"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/testlib"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
type storeType struct {
Name string
SqlSettings *model.SqlSettings
SqlStore *sqlstore.SqlStore
Store store.Store
}
var storeTypes []*storeType
func newStoreType(name, driver string) *storeType {
return &storeType{
Name: name,
SqlSettings: storetest.MakeSqlSettings(driver),
}
}
func StoreTest(t *testing.T, f func(*testing.T, request.CTX, store.Store)) {
defer func() {
if err := recover(); err != nil {
tearDownStores()
panic(err)
}
}()
for _, st := range storeTypes {
rctx := request.TestContext(t)
t.Run(st.Name, func(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
f(t, rctx, st.Store)
})
}
}
func StoreTestWithSqlStore(t *testing.T, f func(*testing.T, request.CTX, store.Store, storetest.SqlStore)) {
defer func() {
if err := recover(); err != nil {
tearDownStores()
panic(err)
}
}()
for _, st := range storeTypes {
rctx := request.TestContext(t)
t.Run(st.Name, func(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
f(t, rctx, st.Store, sqlstore.NewStoreTestWrapper(st.SqlStore))
})
}
}
func initStores(logger mlog.LoggerIFace) {
if testing.Short() {
return
}
storeTypes = append(storeTypes, newStoreType("LocalCache+PostgreSQL", model.DatabaseDriverPostgres))
defer func() {
if err := recover(); err != nil {
tearDownStores()
panic(err)
}
}()
var eg errgroup.Group
for _, st := range storeTypes {
eg.Go(func() error {
var err error
st.SqlStore, err = sqlstore.New(*st.SqlSettings, logger, nil, sqlstore.DisableMorphLogging())
if err != nil {
return err
}
st.Store, err = NewLocalCacheLayer(st.SqlStore, nil, nil, cache.NewProvider(), logger)
if err != nil {
return err
}
st.Store.DropAllTables()
st.Store.MarkSystemRanUnitTests()
return nil
})
}
if err := eg.Wait(); err != nil {
panic(err)
}
}
var tearDownStoresOnce sync.Once
func tearDownStores() {
if testing.Short() {
return
}
tearDownStoresOnce.Do(func() {
var wg sync.WaitGroup
wg.Add(len(storeTypes))
for _, st := range storeTypes {
go func() {
if st.Store != nil {
st.Store.Close()
}
if st.SqlSettings != nil {
storetest.CleanupSqlSettings(st.SqlSettings)
}
wg.Done()
}()
}
wg.Wait()
})
}
func TestClearCacheCluster(t *testing.T) {
cluster := &testlib.FakeClusterInterface{}
lc := &LocalCacheStore{
cluster: cluster,
}
c := cache.NewLRU(&cache.CacheOptions{
Size: 10,
Name: "test",
InvalidateClusterEvent: model.ClusterEventInvalidateCacheForRoles,
})
lc.doClearCacheCluster(c)
assert.Len(t, cluster.GetMessages(), 1)
expectedMsg := &model.ClusterMessage{
Event: model.ClusterEventInvalidateCacheForRoles,
SendType: model.ClusterSendBestEffort,
Data: clearCacheMessageData,
}
require.Equal(t, expectedMsg, cluster.GetMessages()[0])
c = cache.NewLRU(&cache.CacheOptions{
Size: 10,
Name: "test",
InvalidateClusterEvent: model.ClusterEventNone,
})
lc.doClearCacheCluster(c)
assert.Len(t, cluster.GetMessages(), 1)
}

View File

@@ -0,0 +1,269 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/mock"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
"github.com/mattermost/mattermost/server/v8/channels/testlib"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
cachemocks "github.com/mattermost/mattermost/server/v8/platform/services/cache/mocks"
)
var mainHelper *testlib.MainHelper
func getMockCacheProvider() cache.Provider {
mockCacheProvider := cachemocks.Provider{}
// Each NewCache call returns a fresh LRU so that caches are isolated
// from each other, matching production behaviour.
call := mockCacheProvider.On("NewCache", mock.Anything)
call.Run(func(args mock.Arguments) {
call.ReturnArguments = mock.Arguments{cache.NewLRU(&cache.CacheOptions{Size: 128}), nil}
})
mockCacheProvider.On("Type").Return("lru")
return &mockCacheProvider
}
func getMockStore(t *testing.T) *mocks.Store {
mockStore := mocks.Store{}
fakeReaction := model.Reaction{PostId: "123"}
mockReactionsStore := mocks.ReactionStore{}
mockReactionsStore.On("Save", &fakeReaction).Return(&model.Reaction{}, nil)
mockReactionsStore.On("Delete", &fakeReaction).Return(&model.Reaction{}, nil)
mockReactionsStore.On("GetForPost", "123", false).Return([]*model.Reaction{&fakeReaction}, nil)
mockReactionsStore.On("GetForPost", "123", true).Return([]*model.Reaction{&fakeReaction}, nil)
mockStore.On("Reaction").Return(&mockReactionsStore)
mockAutoTranslationStore := mocks.AutoTranslationStore{}
// GetLatestPostUpdateAtForChannel now takes only channelID (no locale) since caching is per-channel
mockAutoTranslationStore.On("GetLatestPostUpdateAtForChannel", "channelId").Return(int64(5000), nil)
mockStore.On("AutoTranslation").Return(&mockAutoTranslationStore)
fakeRole := model.Role{Id: "123", Name: "role-name"}
fakeRole2 := model.Role{Id: "456", Name: "role-name2"}
mockRolesStore := mocks.RoleStore{}
mockRolesStore.On("Save", &fakeRole).Return(&model.Role{}, nil)
mockRolesStore.On("SavePreservingUnknownPermissions", &fakeRole).Return(&model.Role{}, nil)
mockRolesStore.On("Delete", "123").Return(&fakeRole, nil)
mockRolesStore.On("GetByName", mock.Anything, "role-name").Return(&fakeRole, nil)
mockRolesStore.On("GetByNames", []string{"role-name"}).Return([]*model.Role{&fakeRole}, nil)
mockRolesStore.On("GetByNames", []string{"role-name2"}).Return([]*model.Role{&fakeRole2}, nil)
mockRolesStore.On("PermanentDeleteAll").Return(nil)
mockStore.On("Role").Return(&mockRolesStore)
fakeScheme := model.Scheme{Id: "123", Name: "scheme-name"}
mockSchemesStore := mocks.SchemeStore{}
mockSchemesStore.On("Save", &fakeScheme).Return(&model.Scheme{}, nil)
mockSchemesStore.On("Delete", "123").Return(&model.Scheme{}, nil)
mockSchemesStore.On("Get", "123").Return(&fakeScheme, nil)
mockSchemesStore.On("PermanentDeleteAll").Return(nil)
mockStore.On("Scheme").Return(&mockSchemesStore)
fakeFileInfo := model.FileInfo{Id: "123", PostId: "123"}
mockFileInfoStore := mocks.FileInfoStore{}
mockFileInfoStore.On("GetForPost", "123", true, true, false).Return([]*model.FileInfo{&fakeFileInfo}, nil)
mockFileInfoStore.On("GetForPost", "123", true, true, true).Return([]*model.FileInfo{&fakeFileInfo}, nil)
mockFileInfoStore.On("GetByIds", []string{"123"}, true, false, false).Return([]*model.FileInfo{&fakeFileInfo}, nil)
mockStore.On("FileInfo").Return(&mockFileInfoStore)
fakeWebhook := model.IncomingWebhook{Id: "123"}
mockWebhookStore := mocks.WebhookStore{}
mockWebhookStore.On("GetIncoming", "123", true).Return(&fakeWebhook, nil)
mockWebhookStore.On("GetIncoming", "123", false).Return(&fakeWebhook, nil)
mockStore.On("Webhook").Return(&mockWebhookStore)
fakeEmoji := model.Emoji{Id: "123", Name: "name123"}
fakeEmoji2 := model.Emoji{Id: "321", Name: "name321"}
ctxEmoji := model.Emoji{Id: "master", Name: "name123"}
mockEmojiStore := mocks.EmojiStore{}
mockEmojiStore.On("Get", mock.Anything, "123", true).Return(&fakeEmoji, nil)
mockEmojiStore.On("Get", mock.Anything, "123", false).Return(&fakeEmoji, nil)
mockEmojiStore.On("Get", mock.IsType(&request.Context{}), "master", true).Return(&ctxEmoji, nil)
mockEmojiStore.On("Get", sqlstore.RequestContextWithMaster(request.TestContext(t)), "master", true).Return(&ctxEmoji, nil)
mockEmojiStore.On("GetByName", mock.Anything, "name123", true).Return(&fakeEmoji, nil)
mockEmojiStore.On("GetByName", mock.Anything, "name123", false).Return(&fakeEmoji, nil)
mockEmojiStore.On("GetMultipleByName", mock.IsType(&request.Context{}), []string{"name123"}).Return([]*model.Emoji{&fakeEmoji}, nil)
mockEmojiStore.On("GetMultipleByName", mock.IsType(&request.Context{}), []string{"name123", "name321"}).Return([]*model.Emoji{&fakeEmoji, &fakeEmoji2}, nil)
missingEmoji := model.Emoji{Id: "456", Name: "missing"}
// Master-routed lookups find the emoji, replica ones don't; registered
// first so it wins over the IsType matcher below.
mockEmojiStore.On("GetMultipleByName", mock.MatchedBy(func(rctx request.CTX) bool {
return sqlstore.HasMaster(rctx.Context())
}), []string{"missing"}).Return([]*model.Emoji{&missingEmoji}, nil)
mockEmojiStore.On("GetMultipleByName", mock.IsType(&request.Context{}), []string{"missing"}).Return([]*model.Emoji{}, nil)
mockEmojiStore.On("GetMultipleByName", mock.IsType(&request.Context{}), []string{"name123", "missing"}).Return([]*model.Emoji{&fakeEmoji}, nil)
mockEmojiStore.On("GetMultipleByName", mock.IsType(&request.Context{}), []string{strings.Repeat("z", model.EmojiNameMaxLength+1)}).Return([]*model.Emoji{}, nil)
mockEmojiStore.On("GetByName", mock.Anything, "missing", true).Return(&missingEmoji, nil)
mockEmojiStore.On("Save", &missingEmoji).Return(&missingEmoji, nil)
mockEmojiStore.On("GetByName", mock.IsType(&request.Context{}), "master", true).Return(&ctxEmoji, nil)
mockEmojiStore.On("GetByName", sqlstore.RequestContextWithMaster(request.TestContext(t)), "master", false).Return(&ctxEmoji, nil)
mockEmojiStore.On("Delete", &fakeEmoji, int64(0)).Return(nil)
mockEmojiStore.On("Delete", &ctxEmoji, int64(0)).Return(nil)
mockStore.On("Emoji").Return(&mockEmojiStore)
mockCount := int64(10)
mockGuestCount := int64(12)
channelId := "channel1"
fakeChannel1 := model.Channel{Id: channelId, Name: "channel1-name"}
fakeChannel2 := model.Channel{Id: "channel2", Name: "channel2-name"}
mockChannelStore := mocks.ChannelStore{}
mockChannelStore.On("ClearCaches").Return()
mockChannelStore.On("Save", mock.IsType(&request.Context{}), &fakeChannel1, int64(0)).Return(&fakeChannel1, nil)
mockChannelStore.On("Update", mock.IsType(&request.Context{}), &fakeChannel1).Return(&fakeChannel1, nil)
mockChannelStore.On("GetMemberCount", "id", true).Return(mockCount, nil)
mockChannelStore.On("GetMemberCount", "id", false).Return(mockCount, nil)
mockChannelStore.On("GetGuestCount", "id", true).Return(mockGuestCount, nil)
mockChannelStore.On("GetGuestCount", "id", false).Return(mockGuestCount, nil)
mockChannelStore.On("Get", channelId, true).Return(&fakeChannel1, nil)
mockChannelStore.On("Get", channelId, false).Return(&fakeChannel1, nil)
mockChannelStore.On("GetMany", []string{channelId}, true).Return(model.ChannelList{&fakeChannel1}, nil)
mockChannelStore.On("GetMany", []string{channelId}, false).Return(model.ChannelList{&fakeChannel1}, nil)
mockChannelStore.On("GetMany", []string{fakeChannel2.Id}, true).Return(model.ChannelList{&fakeChannel2}, nil)
mockChannelStore.On("GetByNames", "team1", []string{fakeChannel1.Name}, true).Return([]*model.Channel{&fakeChannel1}, nil)
mockChannelStore.On("GetByNames", "team1", []string{fakeChannel2.Name}, true).Return([]*model.Channel{&fakeChannel2}, nil)
mockStore.On("Channel").Return(&mockChannelStore)
mockChannelsMemberCount := map[string]int64{
"channel1": 10,
"channel2": 20,
}
mockChannelStore.On("GetChannelsMemberCount", []string{"channel1", "channel2"}).Return(mockChannelsMemberCount, nil)
mockPinnedPostsCount := int64(10)
mockChannelStore.On("GetPinnedPostCount", "id", true).Return(mockPinnedPostsCount, nil)
mockChannelStore.On("GetPinnedPostCount", "id", false).Return(mockPinnedPostsCount, nil)
fakePosts := &model.PostList{}
fakeOptions := model.GetPostsOptions{ChannelId: "123", PerPage: 30}
mockPostStore := mocks.PostStore{}
mockPostStore.On("GetPosts", mock.AnythingOfType("*request.Context"), fakeOptions, true, map[string]bool{}).Return(fakePosts, nil)
mockPostStore.On("GetPosts", mock.AnythingOfType("*request.Context"), fakeOptions, false, map[string]bool{}).Return(fakePosts, nil)
mockPostStore.On("InvalidateLastPostTimeCache", "12360")
mockPostStoreOptions := model.GetPostsSinceOptions{
ChannelId: "channelId",
Time: 1,
SkipFetchThreads: false,
}
mockPostStoreEtagResult := fmt.Sprintf("%v.%v", model.CurrentVersion, 1)
mockPostStore.On("ClearCaches")
mockPostStore.On("InvalidateLastPostTimeCache", "channelId")
mockPostStore.On("GetEtag", "channelId", true, false, false).Return(mockPostStoreEtagResult)
mockPostStore.On("GetEtag", "channelId", false, false, false).Return(mockPostStoreEtagResult)
mockPostStore.On("GetEtag", "channelId", true, false, true).Return(mockPostStoreEtagResult)
mockPostStore.On("GetEtag", "channelId", false, false, true).Return(mockPostStoreEtagResult)
mockPostStore.On("GetPostsSince", mock.AnythingOfType("*request.Context"), mockPostStoreOptions, true, map[string]bool{}).Return(model.NewPostList(), nil)
mockPostStore.On("GetPostsSince", mock.AnythingOfType("*request.Context"), mockPostStoreOptions, false, map[string]bool{}).Return(model.NewPostList(), nil)
mockStore.On("Post").Return(&mockPostStore)
fakeTermsOfService := model.TermsOfService{Id: "123", CreateAt: 11111, UserId: "321", Text: "Terms of service test"}
mockTermsOfServiceStore := mocks.TermsOfServiceStore{}
mockTermsOfServiceStore.On("InvalidateTermsOfService", "123")
mockTermsOfServiceStore.On("Save", &fakeTermsOfService).Return(&fakeTermsOfService, nil)
mockTermsOfServiceStore.On("GetLatest", true).Return(&fakeTermsOfService, nil)
mockTermsOfServiceStore.On("GetLatest", false).Return(&fakeTermsOfService, nil)
mockTermsOfServiceStore.On("Get", "123", true).Return(&fakeTermsOfService, nil)
mockTermsOfServiceStore.On("Get", "123", false).Return(&fakeTermsOfService, nil)
mockStore.On("TermsOfService").Return(&mockTermsOfServiceStore)
fakeUser := []*model.User{{
Id: "123",
AuthData: new("authData"),
AuthService: "authService",
}}
mockUserStore := mocks.UserStore{}
mockUserStore.On("GetProfileByIds", mock.Anything, []string{"123"}, &store.UserGetByIdsOpts{}, true).Return(fakeUser, nil)
mockUserStore.On("GetProfileByIds", mock.Anything, []string{"123"}, &store.UserGetByIdsOpts{}, false).Return(fakeUser, nil)
fakeProfilesInChannelMap := map[string]*model.User{
"456": {Id: "456"},
}
mockUserStore.On("GetAllProfilesInChannel", mock.Anything, "123", true).Return(fakeProfilesInChannelMap, nil)
mockUserStore.On("GetAllProfilesInChannel", mock.Anything, "123", false).Return(fakeProfilesInChannelMap, nil)
mockUserStore.On("GetAllProfiles", mock.AnythingOfType("*model.UserGetOptions")).Return(fakeUser, nil)
mockUserStore.On("Get", mock.Anything, "123").Return(fakeUser[0], nil)
users := []*model.User{
fakeUser[0],
{
Id: "456",
AuthData: new("authData"),
AuthService: "authService",
},
}
mockUserStore.On("GetMany", mock.Anything, []string{"123", "456"}).Return(users, nil)
mockUserStore.On("GetMany", mock.Anything, []string{"123"}).Return(users[0:1], nil)
mockStore.On("User").Return(&mockUserStore)
fakeUserTeamIds := []string{"1", "2", "3"}
mockTeamStore := mocks.TeamStore{}
mockTeamStore.On("GetUserTeamIds", "123", true).Return(fakeUserTeamIds, nil)
mockTeamStore.On("GetUserTeamIds", "123", false).Return(fakeUserTeamIds, nil)
mockStore.On("Team").Return(&mockTeamStore)
mockContentFlaggingStore := mocks.ContentFlaggingStore{}
mockStore.On("ContentFlagging").Return(&mockContentFlaggingStore)
mockDeliveryTrackingStore := mocks.DeliveryTrackingStore{}
mockStore.On("DeliveryTracking").Return(&mockDeliveryTrackingStore)
mockSessionAttributeStore := mocks.SessionAttributeStore{}
mockStore.On("SessionAttribute").Return(&mockSessionAttributeStore)
fakeField := model.PropertyField{ID: "field-id", GroupID: "group-id", Name: "field-name"}
mockPropertyFieldStore := mocks.PropertyFieldStore{}
mockPropertyFieldStore.On("GetForGroup", mock.Anything, "group-id").Return([]*model.PropertyField{&fakeField}, nil)
mockPropertyFieldStore.On("Create", &fakeField).Return(&fakeField, nil)
mockPropertyFieldStore.On("Update", "group-id", []*model.PropertyField{&fakeField}, map[string]int64(nil)).Return([]*model.PropertyField{&fakeField}, nil)
mockPropertyFieldStore.On("Delete", "group-id", "field-id").Return(nil)
mockStore.On("PropertyField").Return(&mockPropertyFieldStore)
mockAccessControlPolicyStore := mocks.AccessControlPolicyStore{}
mockAccessControlPolicyStore.On("GetEtagEpoch", mock.Anything, "channel-id").Return("100-1", nil)
mockStore.On("AccessControlPolicy").Return(&mockAccessControlPolicyStore)
mockAttributesStore := mocks.AttributesStore{}
mockAttributesStore.On("GetUserPropertyValuesEpoch", mock.Anything, "user-id").Return("200-1", nil)
mockStore.On("Attributes").Return(&mockAttributesStore)
mockReadReceiptStore := &mocks.ReadReceiptStore{}
mockStore.On("ReadReceipt").Return(mockReadReceiptStore)
fakeTemporaryPost := model.TemporaryPost{
ID: "123",
Type: model.PostTypeBurnOnRead,
ExpireAt: model.GetMillis() + 300000,
Message: "test message",
FileIDs: []string{"file1"},
}
mockTemporaryPostStore := mocks.TemporaryPostStore{}
mockTemporaryPostStore.On("Save", mock.Anything, mock.AnythingOfType("*model.TemporaryPost")).Return(&fakeTemporaryPost, nil)
mockTemporaryPostStore.On("Delete", mock.Anything, "123").Return(nil)
mockTemporaryPostStore.On("Get", mock.Anything, "123", false).Return(&fakeTemporaryPost, nil)
mockTemporaryPostStore.On("Get", mock.Anything, "123", true).Return(&fakeTemporaryPost, nil)
mockTemporaryPostStore.On("InvalidateTemporaryPost", "123").Return()
mockStore.On("TemporaryPost").Return(&mockTemporaryPostStore)
return &mockStore
}
func TestMain(m *testing.M) {
mainHelper = testlib.NewMainHelperWithOptions(nil)
defer mainHelper.Close()
initStores(mainHelper.Logger)
mainHelper.Main(m)
tearDownStores()
}

View File

@@ -0,0 +1,179 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"fmt"
"strconv"
"strings"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCachePostStore struct {
store.PostStore
rootStore *LocalCacheStore
}
func (s *LocalCachePostStore) handleClusterInvalidateLastPostTime(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.lastPostTimeCache.Purge()
} else {
s.rootStore.lastPostTimeCache.Remove(string(msg.Data))
}
}
func (s *LocalCachePostStore) handleClusterInvalidateLastPosts(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.postLastPostsCache.Purge()
} else {
s.rootStore.postLastPostsCache.Remove(string(msg.Data))
}
}
func (s *LocalCachePostStore) handleClusterInvalidatePostsUsage(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.postsUsageCache.Purge()
} else {
s.rootStore.postsUsageCache.Remove(string(msg.Data))
}
}
func (s LocalCachePostStore) ClearCaches() {
s.rootStore.doClearCacheCluster(s.rootStore.lastPostTimeCache)
s.rootStore.doClearCacheCluster(s.rootStore.postLastPostsCache)
s.rootStore.doClearCacheCluster(s.rootStore.postsUsageCache)
s.PostStore.ClearCaches()
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.lastPostTimeCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postLastPostsCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postsUsageCache.Name())
}
}
func (s LocalCachePostStore) InvalidateLastPostTimeCache(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.lastPostTimeCache, channelId, nil)
// Keys are "{channelid}{limit}" and caching only occurs on limits of 30 and 60
s.rootStore.doInvalidateCacheCluster(s.rootStore.postLastPostsCache, channelId+"30", nil)
s.rootStore.doInvalidateCacheCluster(s.rootStore.postLastPostsCache, channelId+"60", nil)
s.PostStore.InvalidateLastPostTimeCache(channelId)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.lastPostTimeCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postLastPostsCache.Name())
}
}
func (s LocalCachePostStore) GetEtag(channelId string, allowFromCache, collapsedThreads bool, includeTranslations bool) string {
var baseEtag string
if allowFromCache {
var lastTime int64
if err := s.rootStore.doStandardReadCache(s.rootStore.lastPostTimeCache, channelId, &lastTime); err == nil {
baseEtag = fmt.Sprintf("%v.%v", model.CurrentVersion, lastTime)
}
}
if baseEtag == "" {
baseEtag = s.PostStore.GetEtag(channelId, allowFromCache, collapsedThreads, includeTranslations)
splittedResult := strings.Split(baseEtag, ".")
lastTime, _ := strconv.ParseInt((splittedResult[len(splittedResult)-1]), 10, 64)
s.rootStore.doStandardAddToCache(s.rootStore.lastPostTimeCache, channelId, lastTime)
}
// If translations should be included, append translation time
if includeTranslations {
translationTime, err := s.rootStore.AutoTranslation().GetLatestPostUpdateAtForChannel(channelId)
if err == nil && translationTime > 0 {
return fmt.Sprintf("%s_%d", baseEtag, translationTime)
}
}
return baseEtag
}
func (s LocalCachePostStore) GetPostsSince(rctx request.CTX, options model.GetPostsSinceOptions, allowFromCache bool, sanitizeOptions map[string]bool) (*model.PostList, error) {
if allowFromCache {
// If the last post in the channel's time is less than or equal to the time we are getting posts since,
// we can safely return no posts.
var lastTime int64
if err := s.rootStore.doStandardReadCache(s.rootStore.lastPostTimeCache, options.ChannelId, &lastTime); err == nil && lastTime <= options.Time {
list := model.NewPostList()
return list, nil
}
}
list, err := s.PostStore.GetPostsSince(rctx, options, allowFromCache, sanitizeOptions)
latestUpdate := options.Time
if err == nil {
for _, p := range list.ToSlice() {
if latestUpdate < p.UpdateAt {
latestUpdate = p.UpdateAt
}
}
s.rootStore.doStandardAddToCache(s.rootStore.lastPostTimeCache, options.ChannelId, latestUpdate)
}
return list, err
}
func (s LocalCachePostStore) GetPosts(rctx request.CTX, options model.GetPostsOptions, allowFromCache bool, sanitizeOptions map[string]bool) (*model.PostList, error) {
if !allowFromCache {
return s.PostStore.GetPosts(rctx, options, allowFromCache, sanitizeOptions)
}
offset := options.PerPage * options.Page
// Caching only occurs on limits of 30 and 60, the common limits requested by MM clients
if offset == 0 && (options.PerPage == 60 || options.PerPage == 30) {
var cacheItem *model.PostList
if err := s.rootStore.doStandardReadCache(s.rootStore.postLastPostsCache, fmt.Sprintf("%s%v", options.ChannelId, options.PerPage), &cacheItem); err == nil {
return cacheItem, nil
}
}
list, err := s.PostStore.GetPosts(rctx, options, false, sanitizeOptions)
if err != nil {
return nil, err
}
// Caching only occurs on limits of 30 and 60, the common limits requested by MM clients
if offset == 0 && (options.PerPage == 60 || options.PerPage == 30) {
s.rootStore.doStandardAddToCache(s.rootStore.postLastPostsCache, fmt.Sprintf("%s%v", options.ChannelId, options.PerPage), list)
}
return list, err
}
// AnalyticsPostCount looks up cache only when ExcludeDeleted and UsersPostsOnly are true and rest are falsy.
func (s LocalCachePostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) {
if !options.AllowFromCache || options.MustHaveFile || options.MustHaveHashtag || !options.UsersPostsOnly || !options.ExcludeDeleted || options.TeamId != "" {
return s.PostStore.AnalyticsPostCount(options)
}
// Currently cache only for app > usage > GetPostsUsage()
// Other filter combinations can be cached if required
cacheKey := "posts_usage"
var count int64
if err := s.rootStore.doStandardReadCache(s.rootStore.postsUsageCache, cacheKey, &count); err == nil {
return count, nil
}
count, err := s.PostStore.AnalyticsPostCount(options)
if err != nil {
return 0, err
}
s.rootStore.doStandardAddToCache(s.rootStore.postsUsageCache, cacheKey, count)
return count, nil
}

View File

@@ -0,0 +1,256 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestPostStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestPostStore)
}
func TestPostStoreLastPostTimeCache(t *testing.T) {
rctx := request.TestContext(t)
var fakeLastTime int64 = 1
channelId := "channelId"
fakeOptions := model.GetPostsSinceOptions{
ChannelId: channelId,
Time: fakeLastTime,
SkipFetchThreads: false,
}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("GetEtag: first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
expectedResult := fmt.Sprintf("%v.%v", model.CurrentVersion, fakeLastTime)
etag := cachedStore.Post().GetEtag(channelId, true, false, false)
assert.Equal(t, etag, expectedResult)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1)
etag = cachedStore.Post().GetEtag(channelId, true, false, false)
assert.Equal(t, etag, expectedResult)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1)
})
t.Run("GetEtag: first call not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Post().GetEtag(channelId, true, false, false)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1)
cachedStore.Post().GetEtag(channelId, false, false, false)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 2)
})
t.Run("GetEtag: first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Post().GetEtag(channelId, true, false, false)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1)
cachedStore.Post().InvalidateLastPostTimeCache(channelId)
cachedStore.Post().GetEtag(channelId, true, false, false)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 2)
})
t.Run("GetEtag: first call not cached, clear caches, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Post().GetEtag(channelId, true, false, false)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1)
cachedStore.Post().ClearCaches()
cachedStore.Post().GetEtag(channelId, true, false, false)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 2)
})
t.Run("GetPostsSince: first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
expectedResult := model.NewPostList()
list, err := cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
require.NoError(t, err)
assert.Equal(t, list, expectedResult)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 1)
list, err = cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
require.NoError(t, err)
assert.Equal(t, list, expectedResult)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 1)
})
t.Run("GetPostsSince: first call not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 1)
cachedStore.Post().GetPostsSince(rctx, fakeOptions, false, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 2)
})
t.Run("GetPostsSince: first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 1)
cachedStore.Post().InvalidateLastPostTimeCache(channelId)
cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 2)
})
t.Run("GetPostsSince: first call not cached, clear caches, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 1)
cachedStore.Post().ClearCaches()
cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 2)
})
t.Run("GetEtag: with includeTranslations appends translation time", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// Expected: "{version}.{lastTime}.{translationTime}"
expectedResult := fmt.Sprintf("%v.%v_%v", model.CurrentVersion, fakeLastTime, 5000)
etag := cachedStore.Post().GetEtag(channelId, true, false, true)
assert.Equal(t, expectedResult, etag)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1)
mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1)
})
t.Run("GetEtag: translation etag is cached per-channel", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First call with includeTranslations=true - should call GetLatestPostUpdateAtForChannel
expected := fmt.Sprintf("%v.%v_%v", model.CurrentVersion, fakeLastTime, 5000)
etag := cachedStore.Post().GetEtag(channelId, true, false, true)
assert.Equal(t, expected, etag)
mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1)
// Second call - should use cached translation time
etag = cachedStore.Post().GetEtag(channelId, true, false, true)
assert.Equal(t, expected, etag)
mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1)
})
t.Run("GetEtag: invalidate translation etag clears the channel cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// Call GetEtag with includeTranslations=true
expected := fmt.Sprintf("%v.%v_%v", model.CurrentVersion, fakeLastTime, 5000)
etag := cachedStore.Post().GetEtag(channelId, true, false, true)
assert.Equal(t, expected, etag)
mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1)
// Invalidate the channel's post translation etag
cachedStore.AutoTranslation().InvalidatePostTranslationEtag(channelId)
// Call GetEtag again - should re-fetch
etag = cachedStore.Post().GetEtag(channelId, true, false, true)
assert.Equal(t, expected, etag)
mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 2)
})
}
func TestPostStoreCache(t *testing.T) {
rctx := request.TestContext(t)
fakePosts := &model.PostList{}
fakeOptions := model.GetPostsOptions{ChannelId: "123", PerPage: 30}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotPosts, err := cachedStore.Post().GetPosts(rctx, fakeOptions, true, map[string]bool{})
require.NoError(t, err)
assert.Equal(t, fakePosts, gotPosts)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPosts", 1)
_, _ = cachedStore.Post().GetPosts(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPosts", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotPosts, err := cachedStore.Post().GetPosts(rctx, fakeOptions, true, map[string]bool{})
require.NoError(t, err)
assert.Equal(t, fakePosts, gotPosts)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPosts", 1)
_, _ = cachedStore.Post().GetPosts(rctx, fakeOptions, false, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPosts", 2)
})
t.Run("first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotPosts, err := cachedStore.Post().GetPosts(rctx, fakeOptions, true, map[string]bool{})
require.NoError(t, err)
assert.Equal(t, fakePosts, gotPosts)
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPosts", 1)
cachedStore.Post().InvalidateLastPostTimeCache("12360")
_, _ = cachedStore.Post().GetPosts(rctx, fakeOptions, true, map[string]bool{})
mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPosts", 1)
})
}

View File

@@ -0,0 +1,96 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
func (s LocalCachePropertyFieldStore) Create(field *model.PropertyField) (*model.PropertyField, error) {
created, err := s.PropertyFieldStore.Create(field)
if err != nil {
return nil, err
}
s.InvalidateFieldsForGroup(created.GroupID)
return created, nil
}
func (s LocalCachePropertyFieldStore) Update(groupID string, fields []*model.PropertyField, expectedUpdateAts map[string]int64) ([]*model.PropertyField, error) {
updated, err := s.PropertyFieldStore.Update(groupID, fields, expectedUpdateAts)
if err != nil {
return nil, err
}
// The returned slice includes both the requested fields and any linked
// fields the store propagated to. Invalidate each distinct group so all
// affected GetForGroup caches are cleared.
invalidated := make(map[string]bool, len(updated))
for _, field := range updated {
if invalidated[field.GroupID] {
continue
}
invalidated[field.GroupID] = true
s.InvalidateFieldsForGroup(field.GroupID)
}
return updated, nil
}
func (s LocalCachePropertyFieldStore) Delete(groupID string, id string) error {
if err := s.PropertyFieldStore.Delete(groupID, id); err != nil {
return err
}
s.InvalidateFieldsForGroup(groupID)
return nil
}
type LocalCachePropertyFieldStore struct {
store.PropertyFieldStore
rootStore *LocalCacheStore
}
func (s *LocalCachePropertyFieldStore) handleClusterInvalidatePropertyField(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
if err := s.rootStore.propertyFieldCache.Purge(); err != nil {
s.rootStore.logger.Warn("failed to purge property field cache", mlog.Err(err))
}
} else if err := s.rootStore.propertyFieldCache.Remove(string(msg.Data)); err != nil {
s.rootStore.logger.Warn("failed to remove property field cache entry", mlog.Err(err))
}
}
func (s LocalCachePropertyFieldStore) InvalidateFieldsForGroup(groupID string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.propertyFieldCache, groupID, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.propertyFieldCache.Name())
}
}
func (s *LocalCachePropertyFieldStore) getFieldsForGroupFromCache(groupID string) ([]*model.PropertyField, bool) {
var fields []*model.PropertyField
if err := s.rootStore.doStandardReadCache(s.rootStore.propertyFieldCache, groupID, &fields); err == nil {
return fields, true
}
return nil, false
}
func (s LocalCachePropertyFieldStore) GetForGroup(rctx request.CTX, groupID string) ([]*model.PropertyField, error) {
if fields, ok := s.getFieldsForGroupFromCache(groupID); ok {
return fields, nil
}
fields, err := s.PropertyFieldStore.GetForGroup(rctx, groupID)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.propertyFieldCache, groupID, fields)
return fields, nil
}

View File

@@ -0,0 +1,95 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestPropertyFieldStoreCache(t *testing.T) {
groupID := "group-id"
fakeField := model.PropertyField{ID: "field-id", GroupID: groupID, Name: "field-name"}
fakeFields := []*model.PropertyField{&fakeField}
logger := mlog.CreateConsoleTestLogger(t)
rctx := request.TestContext(t)
t.Run("GetForGroup cached on second call", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
fields, err := cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
assert.Equal(t, fakeFields, fields)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 1)
fields, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
assert.Equal(t, fakeFields, fields)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 1)
})
t.Run("Create invalidates the group cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 1)
_, err = cachedStore.PropertyField().Create(&fakeField)
require.NoError(t, err)
_, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 2)
})
t.Run("Update invalidates the group cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 1)
_, err = cachedStore.PropertyField().Update(groupID, fakeFields, nil)
require.NoError(t, err)
_, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 2)
})
t.Run("Delete invalidates the group cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 1)
err = cachedStore.PropertyField().Delete(groupID, fakeField.ID)
require.NoError(t, err)
_, err = cachedStore.PropertyField().GetForGroup(rctx, groupID)
require.NoError(t, err)
mockStore.PropertyField().(*mocks.PropertyFieldStore).AssertNumberOfCalls(t, "GetForGroup", 2)
})
}

View File

@@ -0,0 +1,62 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheReactionStore struct {
store.ReactionStore
rootStore *LocalCacheStore
}
func (s *LocalCacheReactionStore) handleClusterInvalidateReaction(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.reactionCache.Purge()
} else {
s.rootStore.reactionCache.Remove(string(msg.Data))
}
}
func (s LocalCacheReactionStore) Save(reaction *model.Reaction) (*model.Reaction, error) {
defer s.rootStore.doInvalidateCacheCluster(s.rootStore.reactionCache, reaction.PostId, nil)
return s.ReactionStore.Save(reaction)
}
func (s LocalCacheReactionStore) Delete(reaction *model.Reaction) (*model.Reaction, error) {
defer s.rootStore.doInvalidateCacheCluster(s.rootStore.reactionCache, reaction.PostId, nil)
return s.ReactionStore.Delete(reaction)
}
func (s LocalCacheReactionStore) GetForPost(postId string, allowFromCache bool) ([]*model.Reaction, error) {
if !allowFromCache {
return s.ReactionStore.GetForPost(postId, false)
}
var reaction []*model.Reaction
if err := s.rootStore.doStandardReadCache(s.rootStore.reactionCache, postId, &reaction); err == nil {
return reaction, nil
}
reaction, err := s.ReactionStore.GetForPost(postId, false)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.reactionCache, postId, reaction)
return reaction, nil
}
func (s LocalCacheReactionStore) DeleteAllWithEmojiName(rctx request.CTX, emojiName string) error {
// This could be improved. Right now we just clear the whole
// cache because we don't have a way find what post Ids have this emoji name.
defer s.rootStore.doClearCacheCluster(s.rootStore.reactionCache)
return s.ReactionStore.DeleteAllWithEmojiName(rctx, emojiName)
}

View File

@@ -0,0 +1,78 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestReactionStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestReactionStore)
}
func TestReactionStoreCache(t *testing.T) {
fakeReaction := model.Reaction{PostId: "123"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
reaction, err := cachedStore.Reaction().GetForPost("123", true)
require.NoError(t, err)
assert.Equal(t, reaction, []*model.Reaction{&fakeReaction})
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 1)
assert.Equal(t, reaction, []*model.Reaction{&fakeReaction})
cachedStore.Reaction().GetForPost("123", true)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Reaction().GetForPost("123", true)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 1)
cachedStore.Reaction().GetForPost("123", false)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 2)
})
t.Run("first call not cached, save, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Reaction().GetForPost("123", true)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 1)
cachedStore.Reaction().Save(&fakeReaction)
cachedStore.Reaction().GetForPost("123", true)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 2)
})
t.Run("first call not cached, delete, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Reaction().GetForPost("123", true)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 1)
cachedStore.Reaction().Delete(&fakeReaction)
cachedStore.Reaction().GetForPost("123", true)
mockStore.Reaction().(*mocks.ReactionStore).AssertNumberOfCalls(t, "GetForPost", 2)
})
}

View File

@@ -0,0 +1,188 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"fmt"
"slices"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
type LocalCacheReadReceiptStore struct {
store.ReadReceiptStore
rootStore *LocalCacheStore
}
func (s *LocalCacheReadReceiptStore) handleClusterInvalidateReadReceipts(msg *model.ClusterMessage) {
if err := s.rootStore.readReceiptCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge read receipt cache", mlog.Err(err))
}
}
func (s LocalCacheReadReceiptStore) ClearCaches() {
if err := s.rootStore.readReceiptCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge read receipt cache", mlog.Err(err))
}
if err := s.rootStore.readReceiptPostReadersCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge read receipt post readers cache", mlog.Err(err))
}
if err := s.rootStore.readReceiptPostUnreadCountCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge read receipt post unread count cache", mlog.Err(err))
}
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.readReceiptCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.readReceiptPostReadersCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.readReceiptPostUnreadCountCache.Name())
}
}
func (s LocalCacheReadReceiptStore) InvalidateReadReceiptForPostsCache(postID string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.readReceiptPostReadersCache, postID, nil)
s.rootStore.doInvalidateCacheCluster(s.rootStore.readReceiptPostUnreadCountCache, postID, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.readReceiptPostReadersCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.readReceiptPostUnreadCountCache.Name())
}
if externalCache, ok := s.rootStore.readReceiptCache.(cache.ExternalCache); ok {
// For redis, invalidate all keys with pattern "postID:*"
s.rootStore.doInvalidateCacheCluster(externalCache, fmt.Sprintf("%s:*", postID), nil)
} else {
if err := s.rootStore.readReceiptCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge read receipt cache", mlog.Err(err))
return
}
}
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.readReceiptCache.Name())
}
}
func (s LocalCacheReadReceiptStore) Delete(rctx request.CTX, postID, userID string) error {
defer func() {
s.InvalidateReadReceiptForPostsCache(postID)
}()
return s.ReadReceiptStore.Delete(rctx, postID, userID)
}
func (s LocalCacheReadReceiptStore) DeleteByPost(rctx request.CTX, postID string) error {
defer func(postID string) {
s.InvalidateReadReceiptForPostsCache(postID)
}(postID)
return s.ReadReceiptStore.DeleteByPost(rctx, postID)
}
func (s LocalCacheReadReceiptStore) Save(rctx request.CTX, receipt *model.ReadReceipt) (*model.ReadReceipt, error) {
defer func() {
s.rootStore.doInvalidateCacheCluster(s.rootStore.readReceiptCache, fmt.Sprintf("%s:%s", receipt.PostID, receipt.UserID), nil)
s.rootStore.doInvalidateCacheCluster(s.rootStore.readReceiptPostReadersCache, receipt.PostID, nil)
s.rootStore.doInvalidateCacheCluster(s.rootStore.readReceiptPostUnreadCountCache, receipt.PostID, nil)
}()
return s.ReadReceiptStore.Save(rctx, receipt)
}
func (s LocalCacheReadReceiptStore) Update(rctx request.CTX, receipt *model.ReadReceipt) (*model.ReadReceipt, error) {
defer func() {
s.rootStore.doInvalidateCacheCluster(s.rootStore.readReceiptCache, fmt.Sprintf("%s:%s", receipt.PostID, receipt.UserID), nil)
}()
return s.ReadReceiptStore.Update(rctx, receipt)
}
func (s LocalCacheReadReceiptStore) Get(rctx request.CTX, postID, userID string) (*model.ReadReceipt, error) {
// no need to store the entire struct in cache, just the expireAt would be sufficient
// as other two fields are part of the cache key
var expireAt int64
if err := s.rootStore.doStandardReadCache(s.rootStore.readReceiptCache, fmt.Sprintf("%s:%s", postID, userID), &expireAt); err == nil {
return &model.ReadReceipt{
PostID: postID,
UserID: userID,
ExpireAt: expireAt,
}, nil
}
rr, err := s.ReadReceiptStore.Get(rctx, postID, userID)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.readReceiptCache, fmt.Sprintf("%s:%s", postID, userID), rr.ExpireAt)
// Update post readers cache: add this userID to the list if not already present
var existingUserIDs []string
if err := s.rootStore.doStandardReadCache(s.rootStore.readReceiptPostReadersCache, postID, &existingUserIDs); err == nil {
// Cache exists: check if userID is already in the list
if !slices.Contains(existingUserIDs, userID) {
// Add userID to the existing list
existingUserIDs = append(existingUserIDs, userID)
s.rootStore.doStandardAddToCache(s.rootStore.readReceiptPostReadersCache, postID, existingUserIDs)
}
} else {
// Cache doesn't exist: create new entry with just this userID
s.rootStore.doStandardAddToCache(s.rootStore.readReceiptPostReadersCache, postID, []string{userID})
}
return rr, nil
}
func (s LocalCacheReadReceiptStore) GetByPost(rctx request.CTX, postID string) ([]*model.ReadReceipt, error) {
// Try to get cached user IDs for this post
var cachedUserIDs []string
if err := s.rootStore.doStandardReadCache(s.rootStore.readReceiptPostReadersCache, postID, &cachedUserIDs); err == nil {
// Cache hit: reconstruct receipts from cached user IDs and individual receipt caches
receipts := make([]*model.ReadReceipt, 0, len(cachedUserIDs))
for _, userID := range cachedUserIDs {
var expireAt int64
if err := s.rootStore.doStandardReadCache(s.rootStore.readReceiptCache, fmt.Sprintf("%s:%s", postID, userID), &expireAt); err == nil {
receipts = append(receipts, &model.ReadReceipt{
PostID: postID,
UserID: userID,
ExpireAt: expireAt,
})
}
}
// If we got all receipts from cache, return them
if len(receipts) == len(cachedUserIDs) {
return receipts, nil
}
}
// Cache miss or partial cache: fetch from underlying store
receipts, err := s.ReadReceiptStore.GetByPost(rctx, postID)
if err != nil {
return nil, err
}
// Cache the user IDs for this post
userIDs := make([]string, len(receipts))
for i, receipt := range receipts {
userIDs[i] = receipt.UserID
// Also ensure individual receipts are cached
s.rootStore.doStandardAddToCache(s.rootStore.readReceiptCache, fmt.Sprintf("%s:%s", postID, receipt.UserID), receipt.ExpireAt)
}
s.rootStore.doStandardAddToCache(s.rootStore.readReceiptPostReadersCache, postID, userIDs)
return receipts, nil
}
func (s LocalCacheReadReceiptStore) GetUnreadCountForPost(rctx request.CTX, post *model.Post) (int64, error) {
var count int64
if err := s.rootStore.doStandardReadCache(s.rootStore.readReceiptPostUnreadCountCache, post.Id, &count); err == nil {
return count, nil
}
count, err := s.ReadReceiptStore.GetUnreadCountForPost(rctx, post)
if err != nil {
return count, err
}
s.rootStore.doStandardAddToCache(s.rootStore.readReceiptPostUnreadCountCache, post.Id, count)
return count, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestReadReceiptStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestReadReceiptStore)
}

View File

@@ -0,0 +1,140 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"slices"
"strings"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
type LocalCacheRoleStore struct {
store.RoleStore
rootStore *LocalCacheStore
}
func (s *LocalCacheRoleStore) handleClusterInvalidateRole(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.roleCache.Purge()
} else {
s.rootStore.roleCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheRoleStore) handleClusterInvalidateRolePermissions(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.rolePermissionsCache.Purge()
} else {
s.rootStore.rolePermissionsCache.Remove(string(msg.Data))
}
}
func (s LocalCacheRoleStore) Save(role *model.Role) (*model.Role, error) {
if role.Name != "" {
defer s.rootStore.doInvalidateCacheCluster(s.rootStore.roleCache, role.Name, nil)
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
}
return s.RoleStore.Save(role)
}
func (s LocalCacheRoleStore) SavePreservingUnknownPermissions(role *model.Role) (*model.Role, error) {
if role.Name != "" {
defer s.rootStore.doInvalidateCacheCluster(s.rootStore.roleCache, role.Name, nil)
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
}
return s.RoleStore.SavePreservingUnknownPermissions(role)
}
func (s LocalCacheRoleStore) GetByName(rctx request.CTX, name string) (*model.Role, error) {
var role *model.Role
if err := s.rootStore.doStandardReadCache(s.rootStore.roleCache, name, &role); err == nil {
return role, nil
}
role, err := s.RoleStore.GetByName(rctx, name)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.roleCache, name, role)
return role, nil
}
func (s LocalCacheRoleStore) GetByNames(names []string) ([]*model.Role, error) {
var foundRoles []*model.Role
var rolesToQuery []string
toPass := allocateCacheTargets[*model.Role](len(names))
errs := s.rootStore.doMultiReadCache(s.rootStore.roleCache, names, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
s.rootStore.logger.Warn("Error in Rolestore.GetByNames: ", mlog.Err(err))
}
rolesToQuery = append(rolesToQuery, names[i])
} else {
gotRole := *(toPass[i].(**model.Role))
if gotRole != nil {
foundRoles = append(foundRoles, gotRole)
continue
}
s.rootStore.logger.Warn("Found nil role in GetByNames. This is not expected")
}
}
if len(rolesToQuery) == 0 {
return foundRoles, nil
}
roles, err := s.RoleStore.GetByNames(rolesToQuery)
if err != nil {
return nil, err
}
for _, role := range roles {
s.rootStore.doStandardAddToCache(s.rootStore.roleCache, role.Name, role)
}
return append(foundRoles, roles...), nil
}
func (s LocalCacheRoleStore) Delete(roleId string) (*model.Role, error) {
role, err := s.RoleStore.Delete(roleId)
if err == nil {
s.rootStore.doInvalidateCacheCluster(s.rootStore.roleCache, role.Name, nil)
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
}
return role, err
}
func (s LocalCacheRoleStore) PermanentDeleteAll() error {
defer s.rootStore.roleCache.Purge()
defer s.rootStore.doClearCacheCluster(s.rootStore.roleCache)
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
return s.RoleStore.PermanentDeleteAll()
}
func (s LocalCacheRoleStore) ChannelHigherScopedPermissions(roleNames []string) (map[string]*model.RolePermissions, error) {
slices.Sort(roleNames)
cacheKey := strings.Join(roleNames, "/")
var rolePermissionsMap map[string]*model.RolePermissions
if err := s.rootStore.doStandardReadCache(s.rootStore.rolePermissionsCache, cacheKey, &rolePermissionsMap); err == nil {
return rolePermissionsMap, nil
}
rolePermissionsMap, err := s.RoleStore.ChannelHigherScopedPermissions(roleNames)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.rolePermissionsCache, cacheKey, rolePermissionsMap)
return rolePermissionsMap, nil
}

View File

@@ -0,0 +1,131 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestRoleStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestRoleStore)
}
func TestRoleStoreCache(t *testing.T) {
fakeRole := model.Role{Id: "123", Name: "role-name"}
logger := mlog.CreateConsoleTestLogger(t)
rctx := request.TestContext(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
role, err := cachedStore.Role().GetByName(rctx, "role-name")
require.NoError(t, err)
assert.Equal(t, role, &fakeRole)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 1)
require.NoError(t, err)
assert.Equal(t, role, &fakeRole)
cachedStore.Role().GetByName(rctx, "role-name")
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 1)
})
t.Run("first call not cached, save, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.Role().GetByName(rctx, "role-name")
require.NoError(t, err)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 1)
_, err = cachedStore.Role().Save(&fakeRole)
require.NoError(t, err)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "Save", 1)
_, err = cachedStore.Role().GetByName(rctx, "role-name")
require.NoError(t, err)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call not cached, save preserving unknown permissions, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, err = cachedStore.Role().GetByName(rctx, "role-name")
require.NoError(t, err)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 1)
_, err = cachedStore.Role().SavePreservingUnknownPermissions(&fakeRole)
require.NoError(t, err)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "SavePreservingUnknownPermissions", 1)
_, err = cachedStore.Role().GetByName(rctx, "role-name")
require.NoError(t, err)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call not cached, delete, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Role().GetByName(rctx, "role-name")
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Role().Delete("123")
cachedStore.Role().GetByName(rctx, "role-name")
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call not cached, permanent delete all, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Role().GetByName(rctx, "role-name")
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Role().PermanentDeleteAll()
cachedStore.Role().GetByName(rctx, "role-name")
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByName", 2)
})
}
func TestRoleStoreGetByNamesCache(t *testing.T) {
fakeRole := model.Role{Id: "123", Name: "role-name"}
fakeRole2 := model.Role{Id: "456", Name: "role-name2"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
roles, err := cachedStore.Role().GetByNames([]string{"role-name"})
require.NoError(t, err)
assert.Equal(t, roles[0], &fakeRole)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByNames", 1)
roles, err = cachedStore.Role().GetByNames([]string{"role-name"})
require.NoError(t, err)
assert.Equal(t, roles[0], &fakeRole)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByNames", 1)
roles, err = cachedStore.Role().GetByNames([]string{"role-name", "role-name2"})
require.NoError(t, err)
assert.ElementsMatch(t, []*model.Role{&fakeRole, &fakeRole2}, roles)
mockStore.Role().(*mocks.RoleStore).AssertNumberOfCalls(t, "GetByNames", 2)
})
}

View File

@@ -0,0 +1,63 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheSchemeStore struct {
store.SchemeStore
rootStore *LocalCacheStore
}
func (s *LocalCacheSchemeStore) handleClusterInvalidateScheme(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.schemeCache.Purge()
} else {
s.rootStore.schemeCache.Remove(string(msg.Data))
}
}
func (s LocalCacheSchemeStore) Save(scheme *model.Scheme) (*model.Scheme, error) {
if scheme.Id != "" {
defer s.rootStore.doInvalidateCacheCluster(s.rootStore.schemeCache, scheme.Id, nil)
}
return s.SchemeStore.Save(scheme)
}
func (s LocalCacheSchemeStore) Get(schemeId string) (*model.Scheme, error) {
var scheme *model.Scheme
if err := s.rootStore.doStandardReadCache(s.rootStore.schemeCache, schemeId, &scheme); err == nil {
return scheme, nil
}
scheme, err := s.SchemeStore.Get(schemeId)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.schemeCache, schemeId, scheme)
return scheme, nil
}
func (s LocalCacheSchemeStore) Delete(schemeId string) (*model.Scheme, error) {
defer s.rootStore.doInvalidateCacheCluster(s.rootStore.schemeCache, schemeId, nil)
defer s.rootStore.doClearCacheCluster(s.rootStore.roleCache)
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
defer s.rootStore.doClearCacheCluster(s.rootStore.channelByIdCache)
return s.SchemeStore.Delete(schemeId)
}
func (s LocalCacheSchemeStore) PermanentDeleteAll() error {
defer s.rootStore.doClearCacheCluster(s.rootStore.schemeCache)
defer s.rootStore.doClearCacheCluster(s.rootStore.roleCache)
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
defer s.rootStore.doClearCacheCluster(s.rootStore.channelByIdCache)
return s.SchemeStore.PermanentDeleteAll()
}

View File

@@ -0,0 +1,80 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestSchemeStore(t *testing.T) {
StoreTest(t, storetest.TestSchemeStore)
}
func TestSchemeStoreCache(t *testing.T) {
fakeScheme := model.Scheme{Id: "123", Name: "scheme-name"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
scheme, err := cachedStore.Scheme().Get("123")
require.NoError(t, err)
assert.Equal(t, scheme, &fakeScheme)
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 1)
require.NoError(t, err)
assert.Equal(t, scheme, &fakeScheme)
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call not cached, save, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Scheme().Save(&fakeScheme)
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, delete, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Scheme().Delete("123")
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, permanent delete all, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Scheme().PermanentDeleteAll()
cachedStore.Scheme().Get("123")
mockStore.Scheme().(*mocks.SchemeStore).AssertNumberOfCalls(t, "Get", 2)
})
}

View File

@@ -0,0 +1,89 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"maps"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
type sessionAttributeEntry struct {
Attrs map[string]any
Timestamps map[string]int64
}
type LocalCacheSessionAttributeStore struct {
store.SessionAttributeStore
rootStore *LocalCacheStore
}
func (s *LocalCacheSessionAttributeStore) handleClusterInvalidateSessionAttributes(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
err := s.rootStore.sessionAttributeCache.Purge()
if err != nil {
s.rootStore.logger.Warn("Error while purging session attribute cache", mlog.Err(err))
}
} else {
err := s.rootStore.sessionAttributeCache.Remove(string(msg.Data))
if err != nil {
s.rootStore.logger.Warn("Error while removing session attribute cache", mlog.Err(err))
}
}
}
func (s *LocalCacheSessionAttributeStore) Refresh(sessionID string, attrs map[string]any, updatedAt int64) error {
if len(attrs) == 0 {
return nil
}
entry := &sessionAttributeEntry{
Attrs: make(map[string]any, len(attrs)),
Timestamps: make(map[string]int64, len(attrs)),
}
var existing *sessionAttributeEntry
if err := s.rootStore.doStandardReadCache(s.rootStore.sessionAttributeCache, sessionID, &existing); err == nil && existing != nil {
maps.Copy(entry.Attrs, existing.Attrs)
maps.Copy(entry.Timestamps, existing.Timestamps)
}
for key, value := range attrs {
if entry.Timestamps[key] > updatedAt {
continue
}
entry.Attrs[key] = value
entry.Timestamps[key] = updatedAt
}
s.rootStore.doStandardAddToCache(s.rootStore.sessionAttributeCache, sessionID, entry)
return nil
}
func (s *LocalCacheSessionAttributeStore) Invalidate(sessionID string) error {
s.rootStore.doInvalidateCacheCluster(s.rootStore.sessionAttributeCache, sessionID, nil)
return nil
}
func (s *LocalCacheSessionAttributeStore) Clear() error {
s.rootStore.doClearCacheCluster(s.rootStore.sessionAttributeCache)
return nil
}
func (s *LocalCacheSessionAttributeStore) Get(sessionID string) (map[string]any, map[string]int64, error) {
var entry *sessionAttributeEntry
if err := s.rootStore.doStandardReadCache(s.rootStore.sessionAttributeCache, sessionID, &entry); err != nil {
return nil, nil, err
}
if entry == nil {
return nil, nil, cache.ErrKeyNotFound
}
return entry.Attrs, entry.Timestamps, nil
}

View File

@@ -0,0 +1,196 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
func TestSessionAttributeStoreCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("Refresh writes attributes that Get returns", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionID := model.NewId()
err = cachedStore.SessionAttribute().Refresh(sessionID, map[string]any{
model.SessionAttributesPropertyFieldUserAgentBrowserName: "Chrome",
model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10",
}, model.GetMillis())
require.NoError(t, err)
got, _, err := cachedStore.SessionAttribute().Get(sessionID)
require.NoError(t, err)
require.Equal(t, "Chrome", got[model.SessionAttributesPropertyFieldUserAgentBrowserName])
require.Equal(t, "192.0.2.10", got[model.SessionAttributesPropertyFieldIPAddress])
})
t.Run("Refresh merges partial attributes", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionID := model.NewId()
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionID, map[string]any{
model.SessionAttributesPropertyFieldUserAgentBrowserName: "Chrome",
model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionID, map[string]any{
model.SessionAttributesPropertyFieldUserAgentBrowserName: "Firefox",
}, model.GetMillis()))
got, timestamps, err := cachedStore.SessionAttribute().Get(sessionID)
require.NoError(t, err)
require.Equal(t, "Firefox", got[model.SessionAttributesPropertyFieldUserAgentBrowserName])
require.Equal(t, "192.0.2.10", got[model.SessionAttributesPropertyFieldIPAddress])
require.NotZero(t, timestamps[model.SessionAttributesPropertyFieldUserAgentBrowserName])
require.NotZero(t, timestamps[model.SessionAttributesPropertyFieldIPAddress])
})
t.Run("Get on missing session returns ErrKeyNotFound", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
got, _, err := cachedStore.SessionAttribute().Get(model.NewId())
require.ErrorIs(t, err, cache.ErrKeyNotFound)
require.Nil(t, got)
})
t.Run("cluster invalidation with session id removes single entry", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionA := model.NewId()
sessionB := model.NewId()
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionA, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionB, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "203.0.113.42",
}, model.GetMillis()))
cachedStore.sessionAttribute.handleClusterInvalidateSessionAttributes(&model.ClusterMessage{
Event: model.ClusterEventInvalidateCacheForSessionAttributes,
Data: []byte(sessionA),
})
_, _, err = cachedStore.SessionAttribute().Get(sessionA)
require.ErrorIs(t, err, cache.ErrKeyNotFound)
got, _, err := cachedStore.SessionAttribute().Get(sessionB)
require.NoError(t, err)
require.Equal(t, "203.0.113.42", got[model.SessionAttributesPropertyFieldIPAddress])
})
t.Run("Invalidate removes only the given session entry", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionA := model.NewId()
sessionB := model.NewId()
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionA, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionB, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "203.0.113.42",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Invalidate(sessionA))
_, _, err = cachedStore.SessionAttribute().Get(sessionA)
require.ErrorIs(t, err, cache.ErrKeyNotFound)
got, _, err := cachedStore.SessionAttribute().Get(sessionB)
require.NoError(t, err)
require.Equal(t, "203.0.113.42", got[model.SessionAttributesPropertyFieldIPAddress])
})
t.Run("Clear removes every entry", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionA := model.NewId()
sessionB := model.NewId()
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionA, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionB, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "203.0.113.42",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Clear())
_, _, err = cachedStore.SessionAttribute().Get(sessionA)
require.ErrorIs(t, err, cache.ErrKeyNotFound)
_, _, err = cachedStore.SessionAttribute().Get(sessionB)
require.ErrorIs(t, err, cache.ErrKeyNotFound)
})
t.Run("Refresh skips an update with an older timestamp", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionID := model.NewId()
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionID, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "203.0.113.42",
}, model.GetMillis()))
// A stale update (older timestamp) for the same key must not win.
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionID,
map[string]any{model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10"},
int64(1),
))
got, _, err := cachedStore.SessionAttribute().Get(sessionID)
require.NoError(t, err)
require.Equal(t, "203.0.113.42", got[model.SessionAttributesPropertyFieldIPAddress])
})
t.Run("cluster invalidation with clear-cache marker purges every entry", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
sessionA := model.NewId()
sessionB := model.NewId()
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionA, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "192.0.2.10",
}, model.GetMillis()))
require.NoError(t, cachedStore.SessionAttribute().Refresh(sessionB, map[string]any{
model.SessionAttributesPropertyFieldIPAddress: "203.0.113.42",
}, model.GetMillis()))
cachedStore.sessionAttribute.handleClusterInvalidateSessionAttributes(&model.ClusterMessage{
Event: model.ClusterEventInvalidateCacheForSessionAttributes,
Data: clearCacheMessageData,
})
_, _, err = cachedStore.SessionAttribute().Get(sessionA)
require.ErrorIs(t, err, cache.ErrKeyNotFound)
_, _, err = cachedStore.SessionAttribute().Get(sessionB)
require.ErrorIs(t, err, cache.ErrKeyNotFound)
})
}

View File

@@ -0,0 +1,83 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheTeamStore struct {
store.TeamStore
rootStore *LocalCacheStore
}
func (s *LocalCacheTeamStore) handleClusterInvalidateTeam(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.teamAllTeamIdsForUserCache.Purge()
} else {
s.rootStore.teamAllTeamIdsForUserCache.Remove(string(msg.Data))
}
}
func (s LocalCacheTeamStore) ClearCaches() {
s.rootStore.teamAllTeamIdsForUserCache.Purge()
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.teamAllTeamIdsForUserCache.Name())
}
}
func (s LocalCacheTeamStore) InvalidateAllTeamIdsForUser(userId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.teamAllTeamIdsForUserCache, userId, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.teamAllTeamIdsForUserCache.Name())
}
}
func (s LocalCacheTeamStore) GetUserTeamIds(userID string, allowFromCache bool) ([]string, error) {
if !allowFromCache {
return s.TeamStore.GetUserTeamIds(userID, allowFromCache)
}
var userTeamIds []string
if err := s.rootStore.doStandardReadCache(s.rootStore.teamAllTeamIdsForUserCache, userID, &userTeamIds); err == nil {
return userTeamIds, nil
}
userTeamIds, err := s.TeamStore.GetUserTeamIds(userID, allowFromCache)
if err != nil {
return nil, err
}
if len(userTeamIds) > 0 {
s.rootStore.doStandardAddToCache(s.rootStore.teamAllTeamIdsForUserCache, userID, userTeamIds)
}
return userTeamIds, nil
}
func (s LocalCacheTeamStore) Update(team *model.Team) (*model.Team, error) {
var oldTeam *model.Team
var err error
if team.DeleteAt != 0 {
oldTeam, err = s.TeamStore.Get(team.Id)
if err != nil {
return nil, err
}
}
tm, err := s.TeamStore.Update(team)
if err != nil {
return nil, err
}
defer s.rootStore.doClearCacheCluster(s.rootStore.rolePermissionsCache)
if oldTeam != nil && oldTeam.DeleteAt == 0 {
s.rootStore.doClearCacheCluster(s.rootStore.teamAllTeamIdsForUserCache)
}
return tm, err
}

View File

@@ -0,0 +1,78 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestTeamStore(t *testing.T) {
StoreTest(t, storetest.TestTeamStore)
}
func TestTeamStoreCache(t *testing.T) {
fakeUserId := "123"
fakeUserTeamIds := []string{"1", "2", "3"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUserTeamIds, err := cachedStore.Team().GetUserTeamIds(fakeUserId, true)
require.NoError(t, err)
assert.Equal(t, fakeUserTeamIds, gotUserTeamIds)
mockStore.Team().(*mocks.TeamStore).AssertNumberOfCalls(t, "GetUserTeamIds", 1)
gotUserTeamIds, err = cachedStore.Team().GetUserTeamIds(fakeUserId, true)
require.NoError(t, err)
assert.Equal(t, fakeUserTeamIds, gotUserTeamIds)
mockStore.Team().(*mocks.TeamStore).AssertNumberOfCalls(t, "GetUserTeamIds", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUserTeamIds, err := cachedStore.Team().GetUserTeamIds(fakeUserId, true)
require.NoError(t, err)
assert.Equal(t, fakeUserTeamIds, gotUserTeamIds)
mockStore.Team().(*mocks.TeamStore).AssertNumberOfCalls(t, "GetUserTeamIds", 1)
gotUserTeamIds, err = cachedStore.Team().GetUserTeamIds(fakeUserId, false)
require.NoError(t, err)
assert.Equal(t, fakeUserTeamIds, gotUserTeamIds)
mockStore.Team().(*mocks.TeamStore).AssertNumberOfCalls(t, "GetUserTeamIds", 2)
})
t.Run("first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUserTeamIds, err := cachedStore.Team().GetUserTeamIds(fakeUserId, true)
require.NoError(t, err)
assert.Equal(t, fakeUserTeamIds, gotUserTeamIds)
mockStore.Team().(*mocks.TeamStore).AssertNumberOfCalls(t, "GetUserTeamIds", 1)
cachedStore.Team().InvalidateAllTeamIdsForUser(fakeUserId)
gotUserTeamIds, err = cachedStore.Team().GetUserTeamIds(fakeUserId, true)
require.NoError(t, err)
assert.Equal(t, fakeUserTeamIds, gotUserTeamIds)
mockStore.Team().(*mocks.TeamStore).AssertNumberOfCalls(t, "GetUserTeamIds", 2)
})
}

View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheTemporaryPostStore struct {
store.TemporaryPostStore
rootStore *LocalCacheStore
}
func (s *LocalCacheTemporaryPostStore) handleClusterInvalidateTemporaryPosts(msg *model.ClusterMessage) {
if err := s.rootStore.temporaryPostCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge temporary post cache", mlog.Err(err))
}
}
func (s LocalCacheTemporaryPostStore) ClearCaches() {
if err := s.rootStore.temporaryPostCache.Purge(); err != nil {
s.rootStore.logger.Error("failed to purge temporary post cache", mlog.Err(err))
}
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.temporaryPostCache.Name())
}
}
func (s LocalCacheTemporaryPostStore) InvalidateTemporaryPost(id string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.temporaryPostCache, id, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.temporaryPostCache.Name())
}
}
func (s LocalCacheTemporaryPostStore) Save(rctx request.CTX, post *model.TemporaryPost) (*model.TemporaryPost, error) {
defer s.InvalidateTemporaryPost(post.ID)
return s.TemporaryPostStore.Save(rctx, post)
}
func (s LocalCacheTemporaryPostStore) Get(rctx request.CTX, id string, allowFromCache bool) (*model.TemporaryPost, error) {
if !allowFromCache {
return s.TemporaryPostStore.Get(rctx, id, false)
}
var post *model.TemporaryPost
if err := s.rootStore.doStandardReadCache(s.rootStore.temporaryPostCache, id, &post); err == nil {
return post, nil
}
post, err := s.TemporaryPostStore.Get(rctx, id, false)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.temporaryPostCache, id, post)
return post, nil
}
func (s LocalCacheTemporaryPostStore) Delete(rctx request.CTX, id string) error {
defer s.InvalidateTemporaryPost(id)
return s.TemporaryPostStore.Delete(rctx, id)
}

View File

@@ -0,0 +1,157 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestTemporaryPostStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestTemporaryPostStore)
}
func TestTemporaryPostStoreCache(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
temporaryPost, err := cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
assert.Equal(t, "123", temporaryPost.ID)
assert.Equal(t, "test message", temporaryPost.Message)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Second call should be cached
temporaryPost2, err := cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
assert.Equal(t, "123", temporaryPost2.ID)
assert.Equal(t, "test message", temporaryPost2.Message)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call not cached, second with allowFromCache=false bypasses cache", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First call - should cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Second call with allowFromCache=false - should bypass cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", false)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("allowFromCache=false always bypasses cache even when cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First call with caching enabled
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Second call with caching enabled - should use cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Third call with allowFromCache=false - should bypass cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", false)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 2)
// Fourth call with allowFromCache=false again - should bypass cache again
_, err = cachedStore.TemporaryPost().Get(nil, "123", false)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 3)
})
t.Run("first call not cached, save, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First call - should cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Save should invalidate cache
tmpPost := &model.TemporaryPost{
ID: "123",
Type: model.PostTypeBurnOnRead,
ExpireAt: model.GetMillis() + 300000,
Message: "test message",
FileIDs: []string{"file1"},
}
_, err = cachedStore.TemporaryPost().Save(nil, tmpPost)
require.NoError(t, err)
// Next Get should hit the database again
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, delete, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First call - should cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Delete should invalidate cache
err = cachedStore.TemporaryPost().Delete(nil, "123")
require.NoError(t, err)
// Next Get should hit the database again
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("cache invalidation via InvalidateTemporaryPost", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
// First call - should cache
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 1)
// Manual invalidation
cachedStore.TemporaryPost().InvalidateTemporaryPost("123")
// Next Get should hit the database again
_, err = cachedStore.TemporaryPost().Get(nil, "123", true)
require.NoError(t, err)
mockStore.TemporaryPost().(*mocks.TemporaryPostStore).AssertNumberOfCalls(t, "Get", 2)
})
}

View File

@@ -0,0 +1,81 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
const (
LatestKey = "latest"
)
type LocalCacheTermsOfServiceStore struct {
store.TermsOfServiceStore
rootStore *LocalCacheStore
}
func (s *LocalCacheTermsOfServiceStore) handleClusterInvalidateTermsOfService(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.termsOfServiceCache.Purge()
} else {
s.rootStore.termsOfServiceCache.Remove(string(msg.Data))
}
}
func (s LocalCacheTermsOfServiceStore) ClearCaches() {
s.rootStore.doClearCacheCluster(s.rootStore.termsOfServiceCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.termsOfServiceCache.Name())
}
}
func (s LocalCacheTermsOfServiceStore) Save(termsOfService *model.TermsOfService) (*model.TermsOfService, error) {
tos, err := s.TermsOfServiceStore.Save(termsOfService)
if err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.termsOfServiceCache, tos.Id, tos)
s.rootStore.doInvalidateCacheCluster(s.rootStore.termsOfServiceCache, LatestKey, nil)
}
return tos, err
}
func (s LocalCacheTermsOfServiceStore) GetLatest(allowFromCache bool) (*model.TermsOfService, error) {
if allowFromCache {
var cacheItem *model.TermsOfService
if err := s.rootStore.doStandardReadCache(s.rootStore.termsOfServiceCache, LatestKey, &cacheItem); err == nil {
return cacheItem, nil
}
}
termsOfService, err := s.TermsOfServiceStore.GetLatest(allowFromCache)
if allowFromCache && err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.termsOfServiceCache, termsOfService.Id, termsOfService)
s.rootStore.doStandardAddToCache(s.rootStore.termsOfServiceCache, LatestKey, termsOfService)
}
return termsOfService, err
}
func (s LocalCacheTermsOfServiceStore) Get(id string, allowFromCache bool) (*model.TermsOfService, error) {
if allowFromCache {
var cacheItem *model.TermsOfService
if err := s.rootStore.doStandardReadCache(s.rootStore.termsOfServiceCache, id, &cacheItem); err == nil {
return cacheItem, nil
}
}
termsOfService, err := s.TermsOfServiceStore.Get(id, allowFromCache)
if allowFromCache && err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.termsOfServiceCache, termsOfService.Id, termsOfService)
}
return termsOfService, err
}

View File

@@ -0,0 +1,147 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestTermsOfServiceStore(t *testing.T) {
StoreTest(t, storetest.TestTermsOfServiceStore)
}
func TestTermsOfServiceStoreTermsOfServiceCache(t *testing.T) {
fakeTermsOfService := model.TermsOfService{Id: "123", CreateAt: 11111, UserId: "321", Text: "Terms of service test"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call by latest not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
termsOfService, err := cachedStore.TermsOfService().GetLatest(true)
require.NoError(t, err)
assert.Equal(t, termsOfService, &fakeTermsOfService)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 1)
termsOfService, err = cachedStore.TermsOfService().GetLatest(true)
require.NoError(t, err)
assert.Equal(t, termsOfService, &fakeTermsOfService)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 1)
})
t.Run("first call by id not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
termsOfService, err := cachedStore.TermsOfService().Get("123", true)
require.NoError(t, err)
assert.Equal(t, termsOfService, &fakeTermsOfService)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 1)
termsOfService, err = cachedStore.TermsOfService().Get("123", true)
require.NoError(t, err)
assert.Equal(t, termsOfService, &fakeTermsOfService)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call by id not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().Get("123", true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.TermsOfService().Get("123", false)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call latest not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().GetLatest(true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 1)
cachedStore.TermsOfService().GetLatest(false)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 2)
})
t.Run("first call by id force no cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().Get("123", false)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.TermsOfService().Get("123", true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 2)
cachedStore.TermsOfService().Get("123", true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call latest force no cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().GetLatest(false)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 1)
cachedStore.TermsOfService().GetLatest(true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 2)
cachedStore.TermsOfService().GetLatest(true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 2)
})
t.Run("first call latest, second call by id cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().GetLatest(true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 1)
cachedStore.TermsOfService().Get("123", true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 0)
})
t.Run("first call by id not cached, save, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().Get("123", false)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.TermsOfService().Save(&fakeTermsOfService)
cachedStore.TermsOfService().Get("123", false)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first get latest not cached, save new, then get latest, returning different data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.TermsOfService().GetLatest(true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 1)
cachedStore.TermsOfService().Save(&fakeTermsOfService)
cachedStore.TermsOfService().GetLatest(true)
mockStore.TermsOfService().(*mocks.TermsOfServiceStore).AssertNumberOfCalls(t, "GetLatest", 2)
})
}

View File

@@ -0,0 +1,368 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"slices"
"sync"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
)
type LocalCacheUserStore struct {
store.UserStore
rootStore *LocalCacheStore
userProfileByIdsMut sync.Mutex
userProfileByIdsInvalidations map[string]bool
}
const allUserKey = "ALL"
func (s *LocalCacheUserStore) handleClusterInvalidateScheme(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.userProfileByIdsCache.Purge()
} else {
s.userProfileByIdsMut.Lock()
s.userProfileByIdsInvalidations[string(msg.Data)] = true
s.userProfileByIdsMut.Unlock()
s.rootStore.userProfileByIdsCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheUserStore) handleClusterInvalidateProfilesInChannel(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.profilesInChannelCache.Purge()
} else {
s.rootStore.profilesInChannelCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheUserStore) handleClusterInvalidateAllProfiles(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.allUserCache.Purge()
} else {
s.rootStore.allUserCache.Remove(string(msg.Data))
}
}
func (s *LocalCacheUserStore) ClearCaches() {
s.rootStore.userProfileByIdsCache.Purge()
s.rootStore.allUserCache.Purge()
s.rootStore.profilesInChannelCache.Purge()
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userProfileByIdsCache.Name())
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.profilesInChannelCache.Name())
}
}
func (s *LocalCacheUserStore) InvalidateProfileCacheForUser(userId string) {
s.userProfileByIdsMut.Lock()
s.userProfileByIdsInvalidations[userId] = true
s.userProfileByIdsMut.Unlock()
s.rootStore.doInvalidateCacheCluster(s.rootStore.userProfileByIdsCache, userId, nil)
s.rootStore.doInvalidateCacheCluster(s.rootStore.allUserCache, allUserKey, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userProfileByIdsCache.Name())
}
}
func (s *LocalCacheUserStore) InvalidateProfilesInChannelCacheByUser(userId string) {
var toDelete []string
err := s.rootStore.profilesInChannelCache.Scan(func(keys []string) error {
if len(keys) == 0 {
return nil
}
toPass := allocateCacheTargets[model.UserMap](len(keys))
errs := s.rootStore.doMultiReadCache(s.rootStore.profilesInChannelCache, keys, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
return err
}
continue
}
gotMap := *(toPass[i].(*model.UserMap))
if gotMap == nil {
s.rootStore.logger.Warn("Found nil userMap in InvalidateProfilesInChannelCacheByUser. This is not expected")
continue
}
if _, ok := gotMap[userId]; ok {
toDelete = append(toDelete, keys[i])
}
}
return nil
})
if err != nil {
s.rootStore.logger.Warn("Error while scanning in InvalidateProfilesInChannelCacheByUser", mlog.Err(err))
return
}
s.rootStore.doMultiInvalidateCacheCluster(s.rootStore.profilesInChannelCache, toDelete, nil)
}
func (s *LocalCacheUserStore) InvalidateProfilesInChannelCache(channelID string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.profilesInChannelCache, channelID, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.profilesInChannelCache.Name())
}
}
func (s *LocalCacheUserStore) GetAllProfiles(options *model.UserGetOptions) ([]*model.User, error) {
if isEmptyOptions(options) &&
options.Page == 0 && options.PerPage == 100 { // This is hardcoded to the webapp call.
// read from cache
var users []*model.User
if err := s.rootStore.doStandardReadCache(s.rootStore.allUserCache, allUserKey, &users); err == nil {
return users, nil
}
users, err := s.UserStore.GetAllProfiles(options)
if err != nil {
return nil, err
}
// populate the cache only for those options.
s.rootStore.doStandardAddToCache(s.rootStore.allUserCache, allUserKey, users)
return users, nil
}
// For any other case, simply use the store
return s.UserStore.GetAllProfiles(options)
}
func (s *LocalCacheUserStore) GetAllProfilesInChannel(rctx request.CTX, channelId string, allowFromCache bool) (map[string]*model.User, error) {
if allowFromCache {
var cachedMap model.UserMap
if err := s.rootStore.doStandardReadCache(s.rootStore.profilesInChannelCache, channelId, &cachedMap); err == nil {
return cachedMap, nil
}
}
userMap, err := s.UserStore.GetAllProfilesInChannel(rctx, channelId, allowFromCache)
if err != nil {
return nil, err
}
if allowFromCache {
s.rootStore.doStandardAddToCache(s.rootStore.profilesInChannelCache, channelId, model.UserMap(userMap))
}
return userMap, nil
}
func (s *LocalCacheUserStore) GetProfileByIds(rctx request.CTX, userIds []string, options *store.UserGetByIdsOpts, allowFromCache bool) ([]*model.User, error) {
if !allowFromCache {
return s.UserStore.GetProfileByIds(rctx, userIds, options, false)
}
if options == nil {
options = &store.UserGetByIdsOpts{}
}
users := []*model.User{}
remainingUserIds := make([]string, 0)
fromMaster := false
toPass := allocateCacheTargets[model.User](len(userIds))
errs := s.rootStore.doMultiReadCache(s.rootStore.userProfileByIdsCache, userIds, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
s.rootStore.logger.Warn("Error in UserStore.GetProfileByIds: ", mlog.Err(err))
}
// If it was invalidated, then we need to query master.
s.userProfileByIdsMut.Lock()
if s.userProfileByIdsInvalidations[userIds[i]] {
fromMaster = true
// And then remove the key from the map.
delete(s.userProfileByIdsInvalidations, userIds[i])
}
s.userProfileByIdsMut.Unlock()
remainingUserIds = append(remainingUserIds, userIds[i])
} else {
gotUser := toPass[i].(*model.User)
if (gotUser != nil) && (options.Since == 0 || gotUser.UpdateAt > options.Since) {
users = append(users, gotUser)
} else if gotUser == nil {
s.rootStore.logger.Warn("Found nil user in GetProfileByIds. This is not expected")
}
}
}
if len(remainingUserIds) > 0 {
if fromMaster {
rctx = sqlstore.RequestContextWithMaster(rctx)
}
remainingUsers, err := s.UserStore.GetProfileByIds(rctx, remainingUserIds, options, false)
if err != nil {
return nil, err
}
for _, user := range remainingUsers {
s.rootStore.doStandardAddToCache(s.rootStore.userProfileByIdsCache, user.Id, user)
users = append(users, user)
}
}
return users, nil
}
func (s *LocalCacheUserStore) UpdateFailedPasswordAttempts(userID string, attempts int) error {
s.InvalidateProfileCacheForUser(userID)
return s.UserStore.UpdateFailedPasswordAttempts(userID, attempts)
}
func (s *LocalCacheUserStore) TryIncrementFailedPasswordAttempts(userID string, maxAttempts int) (bool, error) {
claimed, err := s.UserStore.TryIncrementFailedPasswordAttempts(userID, maxAttempts)
if err != nil {
return false, err
}
if claimed {
s.InvalidateProfileCacheForUser(userID)
}
return claimed, nil
}
func (s *LocalCacheUserStore) DecrementFailedPasswordAttempts(userID string) error {
if err := s.UserStore.DecrementFailedPasswordAttempts(userID); err != nil {
return err
}
s.InvalidateProfileCacheForUser(userID)
return nil
}
// Get is a cache wrapper around the SqlStore method to get a user profile by id.
// It checks if the user entry is present in the cache, returning the entry from cache
// if it is present. Otherwise, it fetches the entry from the store and stores it in the
// cache.
func (s *LocalCacheUserStore) Get(rctx request.CTX, id string) (*model.User, error) {
var cacheItem model.User
if err := s.rootStore.doStandardReadCache(s.rootStore.userProfileByIdsCache, id, &cacheItem); err == nil {
return &cacheItem, nil
}
// If it was invalidated, then we need to query master.
s.userProfileByIdsMut.Lock()
if s.userProfileByIdsInvalidations[id] {
rctx = sqlstore.RequestContextWithMaster(rctx)
// And then remove the key from the map.
delete(s.userProfileByIdsInvalidations, id)
}
s.userProfileByIdsMut.Unlock()
user, err := s.UserStore.Get(rctx, id)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.userProfileByIdsCache, id, user)
return user, nil
}
// GetMany is a cache wrapper around the SqlStore method to get a user profiles by ids.
// It checks if the user entries are present in the cache, returning the entries from cache
// if it is present. Otherwise, it fetches the entries from the store and stores it in the
// cache.
func (s *LocalCacheUserStore) GetMany(rctx request.CTX, ids []string) ([]*model.User, error) {
// we are doing a loop instead of caching the full set in the cache because the number of permutations that we can have
// in this func is making caching of the total set not beneficial.
var cachedUsers []*model.User
var notCachedUserIds []string
uniqIDs := dedup(ids)
fromMaster := false
toPass := allocateCacheTargets[model.User](len(uniqIDs))
errs := s.rootStore.doMultiReadCache(s.rootStore.userProfileByIdsCache, uniqIDs, toPass)
for i, err := range errs {
if err != nil {
if err != cache.ErrKeyNotFound {
s.rootStore.logger.Warn("Error in UserStore.GetMany: ", mlog.Err(err))
}
// If it was invalidated, then we need to query master.
s.userProfileByIdsMut.Lock()
if s.userProfileByIdsInvalidations[uniqIDs[i]] {
fromMaster = true
// And then remove the key from the map.
delete(s.userProfileByIdsInvalidations, uniqIDs[i])
}
s.userProfileByIdsMut.Unlock()
notCachedUserIds = append(notCachedUserIds, uniqIDs[i])
} else {
gotUser := toPass[i].(*model.User)
if gotUser != nil {
cachedUsers = append(cachedUsers, gotUser)
} else {
s.rootStore.logger.Warn("Found nil user in GetMany. This is not expected")
}
}
}
if len(notCachedUserIds) > 0 {
if fromMaster {
rctx = sqlstore.RequestContextWithMaster(rctx)
}
dbUsers, err := s.UserStore.GetMany(rctx, notCachedUserIds)
if err != nil {
return nil, err
}
for _, user := range dbUsers {
s.rootStore.doStandardAddToCache(s.rootStore.userProfileByIdsCache, user.Id, user)
cachedUsers = append(cachedUsers, user)
}
}
return cachedUsers, nil
}
func dedup(elements []string) []string {
if len(elements) == 0 {
return elements
}
slices.Sort(elements)
j := 0
for i := 1; i < len(elements); i++ {
if elements[j] == elements[i] {
continue
}
j++
// preserve the original data
// in[i], in[j] = in[j], in[i]
// only set what is required
elements[j] = elements[i]
}
return elements[:j+1]
}
func isEmptyOptions(options *model.UserGetOptions) bool {
// We check to see if any of the options are set or not, and then
// use the cache only if none are set, which is the most common case.
// options.WithoutTeam, Sort is unused
if options.InTeamId == "" &&
options.NotInTeamId == "" &&
options.InChannelId == "" &&
options.NotInChannelId == "" &&
options.InGroupId == "" &&
options.NotInGroupId == "" &&
!options.GroupConstrained &&
!options.Inactive &&
!options.Active &&
options.Role == "" &&
len(options.Roles) == 0 &&
len(options.ChannelRoles) == 0 &&
len(options.TeamRoles) == 0 &&
options.ViewRestrictions == nil {
return true
}
return false
}

View File

@@ -0,0 +1,400 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
cmocks "github.com/mattermost/mattermost/server/v8/platform/services/cache/mocks"
)
func TestUserStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestUserStore)
}
func TestUserStoreCache(t *testing.T) {
rctx := request.TestContext(t)
fakeUserIds := []string{"123"}
fakeUser := []*model.User{{
Id: "123",
AuthData: new("authData"),
AuthService: "authService",
}}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUser, err := cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
require.NoError(t, err)
assert.Equal(t, fakeUser, gotUser)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetProfileByIds", 1)
_, _ = cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetProfileByIds", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUser, err := cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
require.NoError(t, err)
assert.Equal(t, fakeUser, gotUser)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetProfileByIds", 1)
_, _ = cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, false)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetProfileByIds", 2)
})
t.Run("first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUser, err := cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
require.NoError(t, err)
assert.Equal(t, fakeUser, gotUser)
cachedStore.User().InvalidateProfileCacheForUser("123")
_, _ = cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetProfileByIds", 2)
})
t.Run("should always return a copy of the stored data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
storedUsers, err := mockStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, false)
require.NoError(t, err)
originalProps := make([]model.StringMap, len(storedUsers))
for i := range storedUsers {
originalProps[i] = storedUsers[i].NotifyProps
storedUsers[i].NotifyProps = map[string]string{}
storedUsers[i].NotifyProps["key"] = "somevalue"
}
cachedUsers, err := cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
require.NoError(t, err)
for i := range storedUsers {
assert.Equal(t, storedUsers[i].Id, cachedUsers[i].Id)
}
cachedUsers, err = cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
require.NoError(t, err)
for i := range storedUsers {
storedUsers[i].Props = model.StringMap{}
storedUsers[i].Timezone = model.StringMap{}
assert.Equal(t, storedUsers[i], cachedUsers[i])
if storedUsers[i] == cachedUsers[i] {
assert.Fail(t, "should be different pointers")
}
cachedUsers[i].NotifyProps["key"] = "othervalue"
assert.NotEqual(t, storedUsers[i], cachedUsers[i])
}
for i := range storedUsers {
storedUsers[i].NotifyProps = originalProps[i]
}
})
t.Run("assert **model.User not passed", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cmock := cmocks.NewCache(t)
cmock.On("GetMulti", []string{"123"}, mock.MatchedBy(func(values []any) bool {
if len(values) != 1 {
return false
}
_, ok := values[0].(*model.User)
return ok
})).Return(nil)
cachedStore.user.rootStore.userProfileByIdsCache = cmock
_, err = cachedStore.User().GetProfileByIds(rctx, fakeUserIds, &store.UserGetByIdsOpts{}, true)
require.NoError(t, err)
})
}
func TestUserStoreGetAllProfiles(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
users, err := cachedStore.User().GetAllProfiles(&model.UserGetOptions{Page: 0, PerPage: 100})
require.NoError(t, err)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfiles", 1)
users2, _ := cachedStore.User().GetAllProfiles(&model.UserGetOptions{Page: 0, PerPage: 100})
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfiles", 1)
assert.Equal(t, users, users2)
_, _ = cachedStore.User().GetAllProfiles(&model.UserGetOptions{Page: 2, PerPage: 1})
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfiles", 2)
assert.Equal(t, users, users2)
})
t.Run("different page sizes aren't cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
_, _ = cachedStore.User().GetAllProfiles(&model.UserGetOptions{Page: 0, PerPage: 100})
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfiles", 1)
_, _ = cachedStore.User().GetAllProfiles(&model.UserGetOptions{Page: 2, PerPage: 1})
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfiles", 2)
_, _ = cachedStore.User().GetAllProfiles(&model.UserGetOptions{Page: 1, PerPage: 2})
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfiles", 3)
})
}
func TestUserStoreProfilesInChannelCache(t *testing.T) {
rctx := request.TestContext(t)
fakeChannelId := "123"
fakeUserId := "456"
fakeMap := map[string]*model.User{
fakeUserId: {Id: "456"},
}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotMap, err := cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
require.NoError(t, err)
assert.Equal(t, fakeMap, gotMap)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 1)
_, _ = cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotMap, err := cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
require.NoError(t, err)
assert.Equal(t, fakeMap, gotMap)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 1)
_, _ = cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, false)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 2)
})
t.Run("first call not cached, invalidate by channel, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotMap, err := cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
require.NoError(t, err)
assert.Equal(t, fakeMap, gotMap)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 1)
cachedStore.User().InvalidateProfilesInChannelCache("123")
_, _ = cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 2)
})
t.Run("first call not cached, invalidate by user, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotMap, err := cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
require.NoError(t, err)
assert.Equal(t, fakeMap, gotMap)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 1)
cachedStore.User().InvalidateProfilesInChannelCacheByUser("456")
_, _ = cachedStore.User().GetAllProfilesInChannel(rctx, fakeChannelId, true)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetAllProfilesInChannel", 2)
})
}
func TestUserStoreGetCache(t *testing.T) {
rctx := request.TestContext(t)
fakeUserId := "123"
fakeUser := &model.User{
Id: "123",
AuthData: new("authData"),
AuthService: "authService",
}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUser, err := cachedStore.User().Get(rctx, fakeUserId)
require.NoError(t, err)
assert.Equal(t, fakeUser, gotUser)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "Get", 1)
_, _ = cachedStore.User().Get(rctx, fakeUserId)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUser, err := cachedStore.User().Get(rctx, fakeUserId)
require.NoError(t, err)
assert.Equal(t, fakeUser, gotUser)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.User().InvalidateProfileCacheForUser("123")
_, _ = cachedStore.User().Get(rctx, fakeUserId)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("should always return a copy of the stored data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
storedUser, err := mockStore.User().Get(rctx, fakeUserId)
require.NoError(t, err)
originalProps := storedUser.NotifyProps
storedUser.NotifyProps = map[string]string{}
storedUser.NotifyProps["key"] = "somevalue"
cachedUser, err := cachedStore.User().Get(rctx, fakeUserId)
require.NoError(t, err)
assert.Equal(t, storedUser, cachedUser)
storedUser.Props = model.StringMap{}
storedUser.Timezone = model.StringMap{}
cachedUser, err = cachedStore.User().Get(rctx, fakeUserId)
require.NoError(t, err)
assert.Equal(t, storedUser, cachedUser)
if storedUser == cachedUser {
assert.Fail(t, "should be different pointers")
}
cachedUser.NotifyProps["key"] = "othervalue"
assert.NotEqual(t, storedUser, cachedUser)
storedUser.NotifyProps = originalProps
})
t.Run("assert **model.User not passed", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cmock := cmocks.NewCache(t)
cmock.On("Get", "123", mock.AnythingOfType("*model.User")).Return(nil)
cachedStore.user.rootStore.userProfileByIdsCache = cmock
_, err = cachedStore.User().Get(rctx, fakeUserId)
require.NoError(t, err)
})
}
func TestUserStoreGetManyCache(t *testing.T) {
rctx := request.TestContext(t)
fakeUser := &model.User{
Id: "123",
AuthData: new("authData"),
AuthService: "authService",
}
otherFakeUser := &model.User{
Id: "456",
AuthData: new("authData"),
AuthService: "authService",
}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUsers, err := cachedStore.User().GetMany(rctx, []string{fakeUser.Id, otherFakeUser.Id})
require.NoError(t, err)
assert.Len(t, gotUsers, 2)
assert.Contains(t, gotUsers, fakeUser)
assert.Contains(t, gotUsers, otherFakeUser)
gotUsers, err = cachedStore.User().GetMany(rctx, []string{fakeUser.Id, otherFakeUser.Id})
require.NoError(t, err)
assert.Len(t, gotUsers, 2)
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetMany", 1)
})
t.Run("first call not cached, invalidate one user, and then check that one is cached and one is fetched from db", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
gotUsers, err := cachedStore.User().GetMany(rctx, []string{fakeUser.Id, otherFakeUser.Id})
require.NoError(t, err)
assert.Len(t, gotUsers, 2)
assert.Contains(t, gotUsers, fakeUser)
assert.Contains(t, gotUsers, otherFakeUser)
cachedStore.User().InvalidateProfileCacheForUser("123")
gotUsers, err = cachedStore.User().GetMany(rctx, []string{fakeUser.Id, otherFakeUser.Id})
require.NoError(t, err)
assert.Len(t, gotUsers, 2)
mockStore.User().(*mocks.UserStore).AssertCalled(t, "GetMany", mock.Anything, []string{"123"})
mockStore.User().(*mocks.UserStore).AssertNumberOfCalls(t, "GetMany", 2)
})
}

View File

@@ -0,0 +1,99 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"bytes"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type LocalCacheWebhookStore struct {
store.WebhookStore
rootStore *LocalCacheStore
}
func (s *LocalCacheWebhookStore) handleClusterInvalidateWebhook(msg *model.ClusterMessage) {
if bytes.Equal(msg.Data, clearCacheMessageData) {
s.rootStore.webhookCache.Purge()
} else {
s.rootStore.webhookCache.Remove(string(msg.Data))
}
}
func (s LocalCacheWebhookStore) ClearCaches() {
s.rootStore.doClearCacheCluster(s.rootStore.webhookCache)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.webhookCache.Name())
}
}
func (s LocalCacheWebhookStore) InvalidateWebhookCache(webhookId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.webhookCache, webhookId, nil)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.webhookCache.Name())
}
}
func (s LocalCacheWebhookStore) GetIncoming(id string, allowFromCache bool) (*model.IncomingWebhook, error) {
if !allowFromCache {
return s.WebhookStore.GetIncoming(id, allowFromCache)
}
var incomingWebhook *model.IncomingWebhook
if err := s.rootStore.doStandardReadCache(s.rootStore.webhookCache, id, &incomingWebhook); err == nil {
return incomingWebhook, nil
}
incomingWebhook, err := s.WebhookStore.GetIncoming(id, allowFromCache)
if err != nil {
return nil, err
}
s.rootStore.doStandardAddToCache(s.rootStore.webhookCache, id, incomingWebhook)
return incomingWebhook, nil
}
func (s LocalCacheWebhookStore) DeleteIncoming(webhookId string, time int64) error {
err := s.WebhookStore.DeleteIncoming(webhookId, time)
if err != nil {
return err
}
s.InvalidateWebhookCache(webhookId)
return nil
}
func (s LocalCacheWebhookStore) PermanentDeleteIncomingByUser(userId string) error {
err := s.WebhookStore.PermanentDeleteIncomingByUser(userId)
if err != nil {
return err
}
s.ClearCaches()
return nil
}
func (s LocalCacheWebhookStore) PermanentDeleteIncomingByChannel(channelId string) error {
err := s.WebhookStore.PermanentDeleteIncomingByChannel(channelId)
if err != nil {
return err
}
s.ClearCaches()
return nil
}
func (s LocalCacheWebhookStore) UpdateIncomingLastUsed(webhookID string, lastUsed int64) error {
err := s.WebhookStore.UpdateIncomingLastUsed(webhookID, lastUsed)
if err != nil {
return err
}
s.InvalidateWebhookCache(webhookID)
return nil
}

View File

@@ -0,0 +1,66 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package localcachelayer
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestWebhookStore(t *testing.T) {
StoreTest(t, storetest.TestWebhookStore)
}
func TestWebhookStoreCache(t *testing.T) {
fakeWebhook := model.IncomingWebhook{Id: "123"}
logger := mlog.CreateConsoleTestLogger(t)
t.Run("first call not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
incomingWebhook, err := cachedStore.Webhook().GetIncoming("123", true)
require.NoError(t, err)
assert.Equal(t, incomingWebhook, &fakeWebhook)
mockStore.Webhook().(*mocks.WebhookStore).AssertNumberOfCalls(t, "GetIncoming", 1)
assert.Equal(t, incomingWebhook, &fakeWebhook)
cachedStore.Webhook().GetIncoming("123", true)
mockStore.Webhook().(*mocks.WebhookStore).AssertNumberOfCalls(t, "GetIncoming", 1)
})
t.Run("first call not cached, second force not cached", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Webhook().GetIncoming("123", true)
mockStore.Webhook().(*mocks.WebhookStore).AssertNumberOfCalls(t, "GetIncoming", 1)
cachedStore.Webhook().GetIncoming("123", false)
mockStore.Webhook().(*mocks.WebhookStore).AssertNumberOfCalls(t, "GetIncoming", 2)
})
t.Run("first call not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore(t)
mockCacheProvider := getMockCacheProvider()
cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger)
require.NoError(t, err)
cachedStore.Webhook().GetIncoming("123", true)
mockStore.Webhook().(*mocks.WebhookStore).AssertNumberOfCalls(t, "GetIncoming", 1)
cachedStore.Webhook().InvalidateWebhookCache("123")
cachedStore.Webhook().GetIncoming("123", true)
mockStore.Webhook().(*mocks.WebhookStore).AssertNumberOfCalls(t, "GetIncoming", 2)
})
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,130 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package retrylayer
import (
"testing"
"github.com/lib/pq"
"github.com/pkg/errors"
"github.com/lib/pq/pqerror"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func genStore() *mocks.Store {
mock := &mocks.Store{}
mock.On("Audit").Return(&mocks.AuditStore{})
mock.On("Bot").Return(&mocks.BotStore{})
mock.On("Channel").Return(&mocks.ChannelStore{})
mock.On("ChannelGuard").Return(&mocks.ChannelGuardStore{})
mock.On("ChannelMemberHistory").Return(&mocks.ChannelMemberHistoryStore{})
mock.On("ChannelBookmark").Return(&mocks.ChannelBookmarkStore{})
mock.On("ClusterDiscovery").Return(&mocks.ClusterDiscoveryStore{})
mock.On("RemoteCluster").Return(&mocks.RemoteClusterStore{})
mock.On("Command").Return(&mocks.CommandStore{})
mock.On("CommandWebhook").Return(&mocks.CommandWebhookStore{})
mock.On("Compliance").Return(&mocks.ComplianceStore{})
mock.On("Emoji").Return(&mocks.EmojiStore{})
mock.On("FileInfo").Return(&mocks.FileInfoStore{})
mock.On("UploadSession").Return(&mocks.UploadSessionStore{})
mock.On("Group").Return(&mocks.GroupStore{})
mock.On("Job").Return(&mocks.JobStore{})
mock.On("License").Return(&mocks.LicenseStore{})
mock.On("LinkMetadata").Return(&mocks.LinkMetadataStore{})
mock.On("SharedChannel").Return(&mocks.SharedChannelStore{})
mock.On("OAuth").Return(&mocks.OAuthStore{})
mock.On("OutgoingOAuthConnection").Return(&mocks.OutgoingOAuthConnectionStore{})
mock.On("Plugin").Return(&mocks.PluginStore{})
mock.On("Post").Return(&mocks.PostStore{})
mock.On("Thread").Return(&mocks.ThreadStore{})
mock.On("Preference").Return(&mocks.PreferenceStore{})
mock.On("ProductNotices").Return(&mocks.ProductNoticesStore{})
mock.On("Reaction").Return(&mocks.ReactionStore{})
mock.On("RetentionPolicy").Return(&mocks.RetentionPolicyStore{})
mock.On("Role").Return(&mocks.RoleStore{})
mock.On("Scheme").Return(&mocks.SchemeStore{})
mock.On("Session").Return(&mocks.SessionStore{})
mock.On("SessionAttribute").Return(&mocks.SessionAttributeStore{})
mock.On("Status").Return(&mocks.StatusStore{})
mock.On("System").Return(&mocks.SystemStore{})
mock.On("Team").Return(&mocks.TeamStore{})
mock.On("TermsOfService").Return(&mocks.TermsOfServiceStore{})
mock.On("Token").Return(&mocks.TokenStore{})
mock.On("User").Return(&mocks.UserStore{})
mock.On("UserAccessToken").Return(&mocks.UserAccessTokenStore{})
mock.On("UserTermsOfService").Return(&mocks.UserTermsOfServiceStore{})
mock.On("Webhook").Return(&mocks.WebhookStore{})
mock.On("NotifyAdmin").Return(&mocks.NotifyAdminStore{})
mock.On("Draft").Return(&mocks.DraftStore{})
mock.On("PostPriority").Return(&mocks.PostPriorityStore{})
mock.On("PostAcknowledgement").Return(&mocks.PostAcknowledgementStore{})
mock.On("PostPersistentNotification").Return(&mocks.PostPersistentNotificationStore{})
mock.On("DesktopTokens").Return(&mocks.DesktopTokensStore{})
mock.On("ChannelBookmark").Return(&mocks.ChannelBookmarkStore{})
mock.On("ScheduledPost").Return(&mocks.ScheduledPostStore{})
mock.On("PropertyField").Return(&mocks.PropertyFieldStore{})
mock.On("PropertyGroup").Return(&mocks.PropertyGroupStore{})
mock.On("PropertyValue").Return(&mocks.PropertyValueStore{})
mock.On("AccessControlPolicy").Return(&mocks.AccessControlPolicyStore{})
mock.On("Attributes").Return(&mocks.AttributesStore{})
mock.On("AutoTranslation").Return(&mocks.AutoTranslationStore{})
mock.On("ContentFlagging").Return(&mocks.ContentFlaggingStore{})
mock.On("DeliveryTracking").Return(&mocks.DeliveryTrackingStore{})
mock.On("ReadReceipt").Return(&mocks.ReadReceiptStore{})
mock.On("Recap").Return(&mocks.RecapStore{})
mock.On("ScheduledRecap").Return(&mocks.ScheduledRecapStore{})
mock.On("TemporaryPost").Return(&mocks.TemporaryPostStore{})
mock.On("View").Return(&mocks.ViewStore{})
mock.On("ChannelJoinRequest").Return(&mocks.ChannelJoinRequestStore{})
return mock
}
func TestRetry(t *testing.T) {
t.Run("on regular error should not retry", func(t *testing.T) {
mock := genStore()
mockBotStore := mock.Bot().(*mocks.BotStore)
mockBotStore.On("Get", "test", false).Return(nil, errors.New("regular error")).Times(1)
mock.On("Bot").Return(&mockBotStore)
layer := New(mock)
layer.Bot().Get("test", false)
mockBotStore.AssertExpectations(t)
})
t.Run("on success should not retry", func(t *testing.T) {
mock := genStore()
mockBotStore := mock.Bot().(*mocks.BotStore)
mockBotStore.On("Get", "test", false).Return(&model.Bot{}, nil).Times(1)
mock.On("Bot").Return(&mockBotStore)
layer := New(mock)
layer.Bot().Get("test", false)
mockBotStore.AssertExpectations(t)
})
t.Run("on postgres repeatable error should retry", func(t *testing.T) {
for _, errCode := range []string{"40001", "40P01"} {
t.Run("error "+errCode, func(t *testing.T) {
mock := genStore()
mockBotStore := mock.Bot().(*mocks.BotStore)
pqErr := pq.Error{Code: pqerror.Code(errCode)}
mockBotStore.On("Get", "test", false).Return(nil, errors.Wrap(&pqErr, "test-error")).Times(3)
mock.On("Bot").Return(&mockBotStore)
layer := New(mock)
layer.Bot().Get("test", false)
mockBotStore.AssertExpectations(t)
})
}
})
t.Run("on postgres not repeatable error should not retry", func(t *testing.T) {
mock := genStore()
mockBotStore := mock.Bot().(*mocks.BotStore)
pqErr := pq.Error{Code: "20000"}
mockBotStore.On("Get", "test", false).Return(nil, errors.Wrap(&pqErr, "test-error")).Times(1)
mock.On("Bot").Return(&mockBotStore)
layer := New(mock)
layer.Bot().Get("test", false)
mockBotStore.AssertExpectations(t)
})
}

View File

@@ -0,0 +1,396 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type SearchChannelStore struct {
store.ChannelStore
rootStore *SearchStore
}
func (c *SearchChannelStore) deleteChannelIndex(rctx request.CTX, channel *model.Channel) {
if channel.Type == model.ChannelTypeOpen {
for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteChannel(channel); err != nil {
rctx.Logger().Warn("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
})
}
}
}
}
func (c *SearchChannelStore) indexChannel(rctx request.CTX, channel *model.Channel) {
// Space backing channels are internal and never surface in channel search.
if channel.IsSpace() {
return
}
var userIDs, teamMemberIDs []string
var err error
if channel.Type == model.ChannelTypePrivate {
userIDs, err = c.GetAllChannelMemberIdsByChannelId(channel.Id)
if err != nil {
rctx.Logger().Warn("Encountered error while indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
return
}
}
teamMemberIDs, err = c.GetTeamMembersForChannel(rctx, channel.Id)
if err != nil {
rctx.Logger().Warn("Encountered error while indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
return
}
for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.IndexChannel(rctx, channel, userIDs, teamMemberIDs); err != nil {
rctx.Logger().Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
})
}
}
}
func (c *SearchChannelStore) bulkIndexChannels(rctx request.CTX, channels []*model.Channel, teamMemberIDs []string) {
// Util function to get userIDs, only for private channels
getUserIDsForPrivateChannel := func(channel *model.Channel) ([]string, error) {
if channel.Type != model.ChannelTypePrivate {
return []string{}, nil
}
return c.GetAllChannelMemberIdsByChannelId(channel.Id)
}
for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
if !engine.IsIndexingEnabled() {
continue
}
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
appErr := engineCopy.SyncBulkIndexChannels(rctx, channels, getUserIDsForPrivateChannel, teamMemberIDs)
if appErr != nil {
rctx.Logger().Error("Failed to synchronously bulk-index channels.", mlog.String("search_engine", engineCopy.GetName()), mlog.Err(appErr))
return
}
})
}
}
func (c *SearchChannelStore) Save(rctx request.CTX, channel *model.Channel, maxChannels int64, channelOptions ...model.ChannelOption) (*model.Channel, error) {
newChannel, err := c.ChannelStore.Save(rctx, channel, maxChannels, channelOptions...)
if err == nil {
c.indexChannel(rctx, newChannel)
}
return newChannel, err
}
func (c *SearchChannelStore) Update(rctx request.CTX, channel *model.Channel) (*model.Channel, error) {
// Fetch existing channel to detect type changes for post reindexing.
oldChannel, getErr := c.ChannelStore.Get(channel.Id, true)
if getErr != nil {
rctx.Logger().Warn("Failed to fetch channel before update; channel type change detection skipped",
mlog.String("channel_id", channel.Id),
mlog.Err(getErr))
}
updatedChannel, err := c.ChannelStore.Update(rctx, channel)
if err == nil {
c.indexChannel(rctx, updatedChannel)
// If channel type changed, reindex all posts so their channel_type field is updated.
if getErr == nil && oldChannel.Type != updatedChannel.Type {
c.reindexChannelPosts(rctx, channel.Id, updatedChannel.Type)
}
}
return updatedChannel, err
}
func (c *SearchChannelStore) reindexChannelPosts(rctx request.CTX, channelID string, channelType model.ChannelType) {
for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
rctx.Logger().Info("Starting reindexChannelPosts",
mlog.String("channel_id", channelID),
mlog.String("channel_type", string(channelType)),
mlog.String("search_engine", engineCopy.GetName()))
if err := engineCopy.UpdatePostsChannelTypeByChannelId(rctx, channelID, string(channelType)); err != nil {
rctx.Logger().Error("Failed to update channel_type on posts in reindexChannelPosts. Consider running a full bulk index",
mlog.String("channel_id", channelID),
mlog.String("channel_type", string(channelType)),
mlog.String("search_engine", engineCopy.GetName()),
mlog.Err(err))
}
})
}
}
}
func (c *SearchChannelStore) UpdateMember(rctx request.CTX, cm *model.ChannelMember) (*model.ChannelMember, error) {
member, err := c.ChannelStore.UpdateMember(rctx, cm)
if err == nil {
c.rootStore.indexUserFromID(rctx, cm.UserId)
channel, channelErr := c.ChannelStore.Get(member.ChannelId, true)
if channelErr != nil {
rctx.Logger().Warn("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(channelErr))
} else {
c.indexChannel(rctx, channel)
c.rootStore.indexUserFromID(rctx, channel.CreatorId)
}
}
return member, err
}
func (c *SearchChannelStore) SaveMember(rctx request.CTX, cm *model.ChannelMember) (*model.ChannelMember, error) {
member, err := c.ChannelStore.SaveMember(rctx, cm)
if err == nil {
c.rootStore.indexUserFromID(rctx, cm.UserId)
channel, channelErr := c.ChannelStore.Get(member.ChannelId, true)
if channelErr != nil {
rctx.Logger().Warn("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(channelErr))
} else {
c.indexChannel(rctx, channel)
c.rootStore.indexUserFromID(rctx, channel.CreatorId)
}
}
return member, err
}
func (c *SearchChannelStore) RemoveMember(rctx request.CTX, channelID, userIdToRemove string) error {
err := c.ChannelStore.RemoveMember(rctx, channelID, userIdToRemove)
if err == nil {
c.rootStore.indexUserFromID(rctx, userIdToRemove)
}
// return the removal result, not the re-index Get's — that Get returns
// not-found for space backing channels (excluded from the generic Get)
// and must not mask a successful removal.
channel, getErr := c.ChannelStore.Get(channelID, true)
if getErr == nil {
c.indexChannel(rctx, channel)
}
return err
}
func (c *SearchChannelStore) RemoveMembers(rctx request.CTX, channelID string, userIds []string) error {
if err := c.ChannelStore.RemoveMembers(rctx, channelID, userIds); err != nil {
return err
}
channel, err := c.ChannelStore.Get(channelID, true)
if err == nil {
c.indexChannel(rctx, channel)
}
for _, uid := range userIds {
c.rootStore.indexUserFromID(rctx, uid)
}
return nil
}
func (c *SearchChannelStore) CreateDirectChannel(rctx request.CTX, user *model.User, otherUser *model.User, channelOptions ...model.ChannelOption) (*model.Channel, error) {
channel, err := c.ChannelStore.CreateDirectChannel(rctx, user, otherUser, channelOptions...)
if err == nil {
c.rootStore.indexUserFromID(rctx, user.Id)
c.rootStore.indexUserFromID(rctx, otherUser.Id)
c.indexChannel(rctx, channel)
}
return channel, err
}
func (c *SearchChannelStore) SaveDirectChannel(rctx request.CTX, directchannel *model.Channel, member1 *model.ChannelMember, member2 *model.ChannelMember) (*model.Channel, error) {
channel, err := c.ChannelStore.SaveDirectChannel(rctx, directchannel, member1, member2)
if err == nil {
c.rootStore.indexUserFromID(rctx, member1.UserId)
c.rootStore.indexUserFromID(rctx, member2.UserId)
c.indexChannel(rctx, channel)
}
return channel, err
}
func (c *SearchChannelStore) SaveBoardChannel(rctx request.CTX, channel *model.Channel, maxChannelsPerTeam int64, view *model.View) (*model.Channel, *model.View, error) {
// Board channels are not indexed — they must stay invisible to channel search/autocomplete.
return c.ChannelStore.SaveBoardChannel(rctx, channel, maxChannelsPerTeam, view)
}
func (c *SearchChannelStore) Autocomplete(rctx request.CTX, userID, term string, includeDeleted, isGuest bool) (model.ChannelListWithTeamData, error) {
var channelList model.ChannelListWithTeamData
var err error
allFailed := true
for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
if engine.IsAutocompletionEnabled() {
channelList, err = c.searchAutocompleteChannelsAllTeams(engine, userID, term, includeDeleted, isGuest)
if err != nil {
rctx.Logger().Warn("Encountered error on AutocompleteChannels through SearchEngine. Falling back to default autocompletion.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
continue
}
allFailed = false
rctx.Logger().Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
break
}
}
if allFailed {
rctx.Logger().Debug("Using database search because no other search engine is available")
channelList, err = c.ChannelStore.Autocomplete(rctx, userID, term, includeDeleted, isGuest)
if err != nil {
return nil, errors.Wrap(err, "Failed to autocomplete channels in team")
}
}
if err != nil {
return channelList, err
}
return channelList, nil
}
func (c *SearchChannelStore) AutocompleteInTeam(rctx request.CTX, teamID, userID, term string, includeDeleted, isGuest bool) (model.ChannelList, error) {
var channelList model.ChannelList
var err error
allFailed := true
for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
if engine.IsAutocompletionEnabled() {
channelList, err = c.searchAutocompleteChannels(engine, teamID, userID, term, includeDeleted, isGuest)
if err != nil {
rctx.Logger().Warn("Encountered error on AutocompleteChannels through SearchEngine. Falling back to default autocompletion.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
continue
}
allFailed = false
rctx.Logger().Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
break
}
}
if allFailed {
rctx.Logger().Debug("Using database search because no other search engine is available")
channelList, err = c.ChannelStore.AutocompleteInTeam(rctx, teamID, userID, term, includeDeleted, isGuest)
if err != nil {
return nil, errors.Wrap(err, "Failed to autocomplete channels in team")
}
}
if err != nil {
return channelList, err
}
return channelList, nil
}
func (c *SearchChannelStore) searchAutocompleteChannels(engine searchengine.SearchEngineInterface, teamId, userID, term string, includeDeleted, isGuest bool) (model.ChannelList, error) {
channelIds, err := engine.SearchChannels(teamId, userID, term, isGuest, includeDeleted)
if err != nil {
return nil, err
}
channelList := model.ChannelList{}
var nErr error
if len(channelIds) > 0 {
channelList, nErr = c.ChannelStore.GetChannelsByIds(channelIds, includeDeleted)
if nErr != nil {
return nil, errors.Wrap(nErr, "Failed to get channels by ids")
}
}
return channelList, nil
}
func (c *SearchChannelStore) searchAutocompleteChannelsAllTeams(engine searchengine.SearchEngineInterface, userID, term string, includeDeleted, isGuest bool) (model.ChannelListWithTeamData, error) {
channelIds, err := engine.SearchChannels("", userID, term, isGuest, includeDeleted)
if err != nil {
return nil, err
}
channelList := model.ChannelListWithTeamData{}
var nErr error
if len(channelIds) > 0 {
channelList, nErr = c.ChannelStore.GetChannelsWithTeamDataByIds(channelIds, includeDeleted)
if nErr != nil {
return nil, errors.Wrap(nErr, "Failed to get channels by ids")
}
}
return channelList, nil
}
func (c *SearchChannelStore) PermanentDeleteMembersByUser(rctx request.CTX, userId string) error {
channels, errGetChannels := c.ChannelStore.GetChannelsByUser(userId, false, 0, -1, "")
if errGetChannels != nil {
rctx.Logger().Warn("Encountered error indexing channel after removing user", mlog.String("user_id", userId), mlog.Err(errGetChannels))
}
err := c.ChannelStore.PermanentDeleteMembersByUser(rctx, userId)
if err == nil {
c.rootStore.indexUserFromID(rctx, userId)
if errGetChannels == nil {
for _, ch := range channels {
c.indexChannel(rctx, ch)
}
}
}
return err
}
func (c *SearchChannelStore) RemoveAllDeactivatedMembers(rctx request.CTX, channelId string) error {
profiles, errProfiles := c.rootStore.User().GetAllProfilesInChannel(rctx, channelId, true)
if errProfiles != nil {
rctx.Logger().Warn("Encountered error indexing users for channel", mlog.String("channel_id", channelId), mlog.Err(errProfiles))
}
err := c.ChannelStore.RemoveAllDeactivatedMembers(rctx, channelId)
if err == nil && errProfiles == nil {
for _, user := range profiles {
if user.DeleteAt != 0 {
c.rootStore.indexUser(rctx, user)
}
}
}
return err
}
func (c *SearchChannelStore) PermanentDeleteMembersByChannel(rctx request.CTX, channelId string) error {
profiles, errProfiles := c.rootStore.User().GetAllProfilesInChannel(rctx, channelId, true)
if errProfiles != nil {
rctx.Logger().Warn("Encountered error indexing users for channel", mlog.String("channel_id", channelId), mlog.Err(errProfiles))
}
err := c.ChannelStore.PermanentDeleteMembersByChannel(rctx, channelId)
if err == nil && errProfiles == nil {
for _, user := range profiles {
c.rootStore.indexUser(rctx, user)
}
}
return err
}
func (c *SearchChannelStore) PermanentDelete(rctx request.CTX, channelId string) error {
channel, channelErr := c.ChannelStore.Get(channelId, true)
if channelErr != nil {
rctx.Logger().Warn("Encountered error deleting channel", mlog.String("channel_id", channelId), mlog.Err(channelErr))
}
err := c.ChannelStore.PermanentDelete(rctx, channelId)
if err == nil && channelErr == nil {
c.deleteChannelIndex(rctx, channel)
}
return err
}

View File

@@ -0,0 +1,221 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"fmt"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type SearchFileInfoStore struct {
store.FileInfoStore
rootStore *SearchStore
}
func (s SearchFileInfoStore) indexFile(rctx request.CTX, file *model.FileInfo) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if file.PostId == "" && file.CreatorId != model.BookmarkFileOwner {
return
}
channelId := file.ChannelId
if channelId == "" {
rctx.Logger().Error("Couldn't associate file with a channel for file for SearchEngine indexing.", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id))
return
}
if err := engineCopy.IndexFile(file, channelId); err != nil {
rctx.Logger().Error("Encountered error indexing file", mlog.String("file_info_id", file.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndex(rctx request.CTX, fileID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteFile(fileID); err != nil {
rctx.Logger().Error("Encountered error deleting file", mlog.String("file_info_id", fileID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexForUser(rctx request.CTX, userID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUserFiles(rctx, userID); err != nil {
rctx.Logger().Error("Encountered error deleting files for user", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed user's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", userID))
})
}
}
}
//nolint:unused // Temporarily unused until the post_id is indexed with the file
func (s SearchFileInfoStore) deleteFileIndexForPost(rctx request.CTX, postID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeletePostFiles(rctx, postID); err != nil {
rctx.Logger().Error("Encountered error deleting files for post", mlog.String("post_id", postID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed post's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", postID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexBatch(rctx request.CTX, endTime, limit int64) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteFilesBatch(rctx, endTime, limit); err != nil {
rctx.Logger().Error("Encountered error deleting a batch of files", mlog.Int("limit", limit), mlog.Int("end_time", endTime), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed batch of files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.Int("end_time", endTime), mlog.Int("limit", limit))
})
}
}
}
func (s SearchFileInfoStore) Save(rctx request.CTX, info *model.FileInfo) (*model.FileInfo, error) {
nfile, err := s.FileInfoStore.Save(rctx, info)
if err == nil {
s.indexFile(rctx, nfile)
}
return nfile, err
}
func (s SearchFileInfoStore) SetContent(rctx request.CTX, fileID, content string) error {
err := s.FileInfoStore.SetContent(rctx, fileID, content)
if err == nil {
nfile, err2 := s.FileInfoStore.GetFromMaster(fileID)
if err2 == nil {
nfile.Content = content
s.indexFile(rctx, nfile)
}
}
return err
}
func (s SearchFileInfoStore) AttachToPost(rctx request.CTX, fileId, postId, channelId, creatorId string) error {
err := s.FileInfoStore.AttachToPost(rctx, fileId, postId, channelId, creatorId)
if err == nil {
nFileInfo, err2 := s.FileInfoStore.GetFromMaster(fileId)
if err2 == nil {
s.indexFile(rctx, nFileInfo)
}
}
return err
}
func (s SearchFileInfoStore) DeleteForPost(rctx request.CTX, postID string) (string, error) {
// temporary workaround because deleteFileIndexForPost is not working due to the post_id not being indexed with the file
files, err := s.FileInfoStore.GetForPost(postID, false, true, true)
if err != nil {
return "", fmt.Errorf("failed to get files for post %s: %w", postID, err)
}
result, err := s.FileInfoStore.DeleteForPost(rctx, postID)
if err == nil {
for _, file := range files {
s.deleteFileIndex(rctx, file.Id)
}
}
return result, err
}
func (s SearchFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postID string) error {
// temporary workaround because deleteFileIndexForPost is not working due to the post_id not being indexed with the file
files, err := s.FileInfoStore.GetForPost(postID, false, true, true)
if err != nil {
return err
}
err = s.FileInfoStore.PermanentDeleteForPost(rctx, postID)
if err == nil {
for _, file := range files {
s.deleteFileIndex(rctx, file.Id)
}
}
return err
}
func (s SearchFileInfoStore) PermanentDelete(rctx request.CTX, fileId string) error {
err := s.FileInfoStore.PermanentDelete(rctx, fileId)
if err == nil {
s.deleteFileIndex(rctx, fileId)
}
return err
}
func (s SearchFileInfoStore) PermanentDeleteBatch(rctx request.CTX, endTime int64, limit int64) (int64, error) {
result, err := s.FileInfoStore.PermanentDeleteBatch(rctx, endTime, limit)
if err == nil {
s.deleteFileIndexBatch(rctx, endTime, limit)
}
return result, err
}
func (s SearchFileInfoStore) PermanentDeleteByUser(rctx request.CTX, userId string) (int64, error) {
result, err := s.FileInfoStore.PermanentDeleteByUser(rctx, userId)
if err == nil {
s.deleteFileIndexForUser(rctx, userId)
}
return result, err
}
func (s SearchFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsSearchEnabled() {
userChannels, nErr := s.rootStore.Channel().GetChannels(teamId, userId, &model.ChannelSearchOpts{
IncludeDeleted: paramsList[0].IncludeDeletedChannels,
LastDeleteAt: 0,
})
if nErr != nil {
return nil, nErr
}
fileIds, appErr := engine.SearchFiles(userChannels, paramsList, page, perPage)
if appErr != nil {
rctx.Logger().Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(appErr))
continue
}
// Get the files
filesList := model.NewFileInfoList()
if len(fileIds) > 0 {
files, nErr := s.FileInfoStore.GetByIds(fileIds, false, true, false)
if nErr != nil {
return nil, nErr
}
for _, f := range files {
filesList.AddFileInfo(f)
filesList.AddOrder(f.Id)
}
}
return filesList, nil
}
}
if *s.rootStore.getConfig().SqlSettings.DisableDatabaseSearch {
return model.NewFileInfoList(), nil
}
return s.FileInfoStore.Search(rctx, paramsList, userId, teamId, page, perPage)
}

View File

@@ -0,0 +1,156 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"sync/atomic"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/public/utils"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type SearchStore struct {
store.Store
searchEngine *searchengine.Broker
user *SearchUserStore
team *SearchTeamStore
channel *SearchChannelStore
post *SearchPostStore
fileInfo *SearchFileInfoStore
configValue atomic.Pointer[model.Config]
}
func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker, cfg *model.Config) *SearchStore {
searchStore := &SearchStore{
Store: baseStore,
searchEngine: searchEngine,
}
searchStore.configValue.Store(cfg)
searchStore.channel = &SearchChannelStore{ChannelStore: baseStore.Channel(), rootStore: searchStore}
searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: searchStore}
searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: searchStore}
searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: searchStore}
searchStore.fileInfo = &SearchFileInfoStore{FileInfoStore: baseStore.FileInfo(), rootStore: searchStore}
return searchStore
}
func (s *SearchStore) UpdateConfig(cfg *model.Config) {
s.configValue.Store(cfg)
}
func (s *SearchStore) getConfig() *model.Config {
return s.configValue.Load()
}
func (s *SearchStore) Channel() store.ChannelStore {
return s.channel
}
func (s *SearchStore) Post() store.PostStore {
return s.post
}
func (s *SearchStore) FileInfo() store.FileInfoStore {
return s.fileInfo
}
func (s *SearchStore) Team() store.TeamStore {
return s.team
}
func (s *SearchStore) User() store.UserStore {
return s.user
}
func (s *SearchStore) indexUserFromID(rctx request.CTX, userId string) {
user, err := s.User().Get(rctx, userId)
if err != nil {
return
}
s.indexUser(rctx, user)
}
func (s *SearchStore) indexUser(rctx request.CTX, user *model.User) {
for _, engine := range s.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
userTeams, nErr := s.Team().GetTeamsByUserId(user.Id)
if nErr != nil {
rctx.Logger().Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(nErr))
return
}
userTeamsIds := []string{}
for _, team := range userTeams {
userTeamsIds = append(userTeamsIds, team.Id)
}
userChannelMembers, err := s.Channel().GetAllChannelMembersForUser(rctx, user.Id, false, true)
if err != nil {
rctx.Logger().Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
userChannelsIds := []string{}
for channelId := range userChannelMembers {
userChannelsIds = append(userChannelsIds, channelId)
}
if err := engineCopy.IndexUser(rctx, user, userTeamsIds, userChannelsIds); err != nil {
rctx.Logger().Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Indexed user in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id))
})
}
}
}
func (s *SearchStore) indexChannelsForTeam(rctx request.CTX, teamID string) {
const perPage = 100
var (
channels []*model.Channel
)
channels, err := utils.Pager(func(page int) ([]*model.Channel, error) {
return s.channel.GetPublicChannelsForTeam(teamID, page*perPage, perPage)
}, perPage)
if err != nil {
rctx.Logger().Warn("Encountered error while retrieving public channels for indexing", mlog.String("team_id", teamID), mlog.Err(err))
return
}
if len(channels) == 0 {
return
}
// Use master context to avoid replica lag issues when reading team members
masterRctx := store.RequestContextWithMaster(rctx)
teamMemberIDs, err := s.channel.GetTeamMembersForChannel(masterRctx, channels[0].Id)
if err != nil {
rctx.Logger().Warn("Encountered error while retrieving team members for channel", mlog.String("channel_id", channels[0].Id), mlog.Err(err))
return
}
s.channel.bulkIndexChannels(rctx, channels, teamMemberIDs)
}
// Runs an indexing function synchronously or asynchronously depending on the engine
func runIndexFn(rctx request.CTX, engine searchengine.SearchEngineInterface, indexFn func(searchengine.SearchEngineInterface)) {
if engine.IsIndexingSync() {
indexFn(engine)
if err := engine.RefreshIndexes(rctx); err != nil {
rctx.Logger().Error("Encountered error refresh the indexes", mlog.Err(err))
}
} else {
go (func(engineCopy searchengine.SearchEngineInterface) {
indexFn(engineCopy)
})(engine)
}
}

View File

@@ -0,0 +1,50 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer_test
import (
"os"
"sync"
"testing"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store/searchlayer"
"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/testlib"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
"github.com/stretchr/testify/require"
)
// Test to verify race condition on UpdateConfig. The test must run with -race flag in order to verify
// that there is no race. Ref: (#MM-30868)
func TestUpdateConfigRace(t *testing.T) {
logger := mlog.CreateTestLogger(t)
driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
if driverName == "" {
driverName = model.DatabaseDriverPostgres
}
settings := storetest.MakeSqlSettings(driverName)
store, err := sqlstore.New(*settings, logger, nil, sqlstore.DisableMorphLogging())
require.NoError(t, err)
cfg := &model.Config{}
cfg.SetDefaults()
cfg.ClusterSettings.GossipPort = new(9999)
searchEngine := searchengine.NewBroker(cfg)
layer := searchlayer.NewSearchLayer(&testlib.TestStore{Store: store}, searchEngine, cfg)
var wg sync.WaitGroup
wg.Add(5)
for range 5 {
go func() {
defer wg.Done()
layer.UpdateConfig(cfg.Clone())
}()
}
wg.Wait()
}

View File

@@ -0,0 +1,211 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type SearchPostStore struct {
store.PostStore
rootStore *SearchStore
}
func (s SearchPostStore) indexPost(rctx request.CTX, post *model.Post) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if post.Type == model.PostTypeBurnOnRead {
return
}
// FIXME(IntegratedBoardMVP): Temporarily excluded
if post.Type == model.PostTypeCard {
return
}
channel, chanErr := s.rootStore.Channel().Get(post.ChannelId, true)
if chanErr != nil {
rctx.Logger().Error("Couldn't get channel for post for SearchEngine indexing.", mlog.String("channel_id", post.ChannelId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id), mlog.Err(chanErr))
return
}
if err := engineCopy.IndexPost(post, channel.TeamId, string(channel.Type)); err != nil {
rctx.Logger().Warn("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
})
}
}
}
func (s SearchPostStore) deletePostIndex(rctx request.CTX, post *model.Post) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeletePost(post); err != nil {
rctx.Logger().Warn("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
})
}
}
}
func (s SearchPostStore) deleteChannelPostsIndex(rctx request.CTX, channelID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteChannelPosts(rctx, channelID); err != nil {
rctx.Logger().Warn("Encountered error deleting channel posts", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed all channel posts from the index in search engine", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()))
})
}
}
}
func (s SearchPostStore) deleteUserPostsIndex(rctx request.CTX, userID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUserPosts(rctx, userID); err != nil {
rctx.Logger().Warn("Encountered error deleting user posts", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed all user posts from the index in search engine", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()))
})
}
}
}
func (s SearchPostStore) Update(rctx request.CTX, newPost, oldPost *model.Post) (*model.Post, error) {
post, err := s.PostStore.Update(rctx, newPost, oldPost)
if err == nil {
s.indexPost(rctx, post)
}
return post, err
}
func (s *SearchPostStore) Overwrite(rctx request.CTX, post *model.Post) (*model.Post, error) {
post, err := s.PostStore.Overwrite(rctx, post)
if err == nil {
s.indexPost(rctx, post)
}
return post, err
}
func (s SearchPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
npost, err := s.PostStore.Save(rctx, post)
if err == nil {
s.indexPost(rctx, npost)
}
return npost, err
}
func (s SearchPostStore) Delete(rctx request.CTX, postId string, date int64, deletedByID string) error {
err := s.PostStore.Delete(rctx, postId, date, deletedByID)
if err != nil {
return err
}
post, err := s.PostStore.GetSingle(rctx, postId, true)
if err != nil {
return err
}
s.deletePostIndex(rctx, post)
return nil
}
func (s SearchPostStore) PermanentDelete(rctx request.CTX, postID string) error {
// Get full post struct for later
post, err := s.PostStore.GetSingle(rctx, postID, true)
if err != nil {
return err
}
err = s.PostStore.PermanentDelete(rctx, postID)
if err != nil {
return err
}
s.deletePostIndex(rctx, post)
return nil
}
func (s SearchPostStore) PermanentDeleteByUser(rctx request.CTX, userID string) error {
err := s.PostStore.PermanentDeleteByUser(rctx, userID)
if err == nil {
s.deleteUserPostsIndex(rctx, userID)
}
return err
}
func (s SearchPostStore) PermanentDeleteByChannel(rctx request.CTX, channelID string) error {
err := s.PostStore.PermanentDeleteByChannel(rctx, channelID)
if err == nil {
s.deleteChannelPostsIndex(rctx, channelID)
}
return err
}
func (s SearchPostStore) searchPostsForUserByEngine(engine searchengine.SearchEngineInterface, paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.PostSearchResults, error) {
if err := model.IsSearchParamsListValid(paramsList); err != nil {
return nil, err
}
// We only allow the user to search in channels they are a member of.
userChannels, err2 := s.rootStore.Channel().GetChannels(teamId, userId,
&model.ChannelSearchOpts{
IncludeDeleted: paramsList[0].IncludeDeletedChannels,
LastDeleteAt: 0,
})
if err2 != nil {
return nil, errors.Wrap(err2, "error getting channel for user")
}
postIds, matches, err := engine.SearchPosts(userChannels, paramsList, page, perPage)
if err != nil {
return nil, err
}
// Get the posts
postList := model.NewPostList()
if len(postIds) > 0 {
posts, err := s.PostStore.GetPostsByIds(postIds)
if err != nil {
return nil, err
}
for _, p := range posts {
if p.DeleteAt == 0 {
postList.AddPost(p)
postList.AddOrder(p.Id)
}
}
}
return model.MakePostSearchResults(postList, matches), nil
}
func (s SearchPostStore) SearchPostsForUser(rctx request.CTX, paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.PostSearchResults, error) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsSearchEnabled() {
results, err := s.searchPostsForUserByEngine(engine, paramsList, userId, teamId, page, perPage)
if err != nil {
rctx.Logger().Warn("Encountered error on SearchPostsInTeamForUser.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
continue
}
return results, err
}
}
if *s.rootStore.getConfig().SqlSettings.DisableDatabaseSearch {
return &model.PostSearchResults{PostList: model.NewPostList(), Matches: model.PostSearchMatches{}}, nil
}
return s.PostStore.SearchPostsForUser(rctx, paramsList, userId, teamId, page, perPage)
}

View File

@@ -0,0 +1,67 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
model "github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
store "github.com/mattermost/mattermost/server/v8/channels/store"
)
type SearchTeamStore struct {
store.TeamStore
rootStore *SearchStore
}
func (s SearchTeamStore) SaveMember(rctx request.CTX, teamMember *model.TeamMember, maxUsersPerTeam int) (*model.TeamMember, error) {
member, err := s.TeamStore.SaveMember(rctx, teamMember, maxUsersPerTeam)
if err == nil {
// Nothing to do if search engine is not active
if s.rootStore.searchEngine.ActiveEngine() != "database" && s.rootStore.searchEngine.ActiveEngine() != "none" {
s.rootStore.indexUserFromID(rctx, member.UserId)
s.rootStore.indexChannelsForTeam(rctx, member.TeamId)
}
}
return member, err
}
func (s SearchTeamStore) UpdateMember(rctx request.CTX, teamMember *model.TeamMember) (*model.TeamMember, error) {
member, err := s.TeamStore.UpdateMember(rctx, teamMember)
if err == nil {
s.rootStore.indexUserFromID(rctx, member.UserId)
}
return member, err
}
func (s SearchTeamStore) RemoveMember(rctx request.CTX, teamId string, userId string) error {
err := s.TeamStore.RemoveMember(rctx, teamId, userId)
if err == nil {
// Nothing to do if search engine is not active
if s.rootStore.searchEngine.ActiveEngine() != "database" && s.rootStore.searchEngine.ActiveEngine() != "none" {
s.rootStore.indexUserFromID(rctx, userId)
s.rootStore.indexChannelsForTeam(rctx, teamId)
}
}
return err
}
func (s SearchTeamStore) RemoveAllMembersByUser(rctx request.CTX, userId string) error {
if s.rootStore.searchEngine.ActiveEngine() != "database" && s.rootStore.searchEngine.ActiveEngine() != "none" {
memberships, err := s.TeamStore.GetTeamsForUser(rctx, userId, "", true)
if err != nil {
return err
}
for _, membership := range memberships {
s.rootStore.indexChannelsForTeam(rctx, membership.TeamId)
}
}
err := s.TeamStore.RemoveAllMembersByUser(rctx, userId)
if err == nil {
if s.rootStore.searchEngine.ActiveEngine() != "database" && s.rootStore.searchEngine.ActiveEngine() != "none" {
s.rootStore.indexUserFromID(rctx, userId)
}
}
return err
}

View File

@@ -0,0 +1,234 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"strings"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type SearchUserStore struct {
store.UserStore
rootStore *SearchStore
}
func (s *SearchUserStore) deleteUserIndex(rctx request.CTX, user *model.User) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUser(user); err != nil {
rctx.Logger().Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
rctx.Logger().Debug("Removed user from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id))
})
}
}
}
func (s *SearchUserStore) Search(rctx request.CTX, teamId, term string, options *model.UserSearchOptions) ([]*model.User, error) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsSearchEnabled() {
listOfAllowedChannels, nErr := s.getListOfAllowedChannels(teamId, "", options.ViewRestrictions)
if nErr != nil {
rctx.Logger().Warn("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(nErr))
continue
}
if listOfAllowedChannels != nil && len(listOfAllowedChannels) == 0 {
return []*model.User{}, nil
}
sanitizedTerm := sanitizeSearchTerm(term)
usersIds, err := engine.SearchUsersInTeam(teamId, listOfAllowedChannels, sanitizedTerm, options)
if err != nil {
rctx.Logger().Warn("Encountered error on Search", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
continue
}
users, nErr := s.UserStore.GetProfileByIds(rctx, usersIds, nil, false)
if nErr != nil {
rctx.Logger().Warn("Encountered error on Search", mlog.String("search_engine", engine.GetName()), mlog.Err(nErr))
continue
}
rctx.Logger().Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
return users, nil
}
}
rctx.Logger().Debug("Using database search because no other search engine is available")
return s.UserStore.Search(rctx, teamId, term, options)
}
func (s *SearchUserStore) Update(rctx request.CTX, user *model.User, trustedUpdateData bool) (*model.UserUpdate, error) {
userUpdate, err := s.UserStore.Update(rctx, user, trustedUpdateData)
if err == nil {
s.rootStore.indexUser(rctx, userUpdate.New)
}
return userUpdate, err
}
func (s *SearchUserStore) Save(rctx request.CTX, user *model.User) (*model.User, error) {
nuser, err := s.UserStore.Save(rctx, user)
if err == nil {
s.rootStore.indexUser(rctx, nuser)
}
return nuser, err
}
func (s *SearchUserStore) PermanentDelete(rctx request.CTX, userId string) error {
user, userErr := s.UserStore.Get(rctx, userId)
if userErr != nil {
rctx.Logger().Warn("Encountered error deleting user", mlog.String("user_id", userId), mlog.Err(userErr))
}
err := s.UserStore.PermanentDelete(rctx, userId)
if err == nil && userErr == nil {
s.deleteUserIndex(rctx, user)
}
return err
}
func (s *SearchUserStore) autocompleteUsersInChannelByEngine(rctx request.CTX, engine searchengine.SearchEngineInterface, teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, error) {
var err *model.AppError
uchanIds := []string{}
nuchanIds := []string{}
sanitizedTerm := sanitizeSearchTerm(term)
if channelId != "" && options.ListOfAllowedChannels != nil && !strings.Contains(strings.Join(options.ListOfAllowedChannels, "."), channelId) {
nuchanIds, err = engine.SearchUsersInTeam(teamId, options.ListOfAllowedChannels, sanitizedTerm, options)
} else {
uchanIds, nuchanIds, err = engine.SearchUsersInChannel(teamId, channelId, options.ListOfAllowedChannels, sanitizedTerm, options)
}
if err != nil {
return nil, err
}
uchan := make(chan store.StoreResult[[]*model.User], 1)
go func() {
users, nErr := s.UserStore.GetProfileByIds(rctx, uchanIds, nil, false)
uchan <- store.StoreResult[[]*model.User]{Data: users, NErr: nErr}
close(uchan)
}()
nuchan := make(chan store.StoreResult[[]*model.User], 1)
go func() {
users, nErr := s.UserStore.GetProfileByIds(rctx, nuchanIds, nil, false)
nuchan <- store.StoreResult[[]*model.User]{Data: users, NErr: nErr}
close(nuchan)
}()
autocomplete := &model.UserAutocompleteInChannel{}
result := <-uchan
if result.NErr != nil {
return nil, errors.Wrap(result.NErr, "failed to get user profiles by ids")
}
autocomplete.InChannel = result.Data
result = <-nuchan
if result.NErr != nil {
return nil, errors.Wrap(result.NErr, "failed to get user profiles by ids")
}
autocomplete.OutOfChannel = result.Data
return autocomplete, nil
}
// getListOfAllowedChannels return the list of allowed channels to search user based on the
//
// next scenarios:
// - If there isn't view restrictions (team or channel) and no team id to filter them, then all
// channels are allowed (nil return)
// - If we receive a team Id and either we don't have view restrictions or the provided team id is included in the
// list of restricted teams, then we return all the team channels
// - If we don't receive team id or the provided team id is not in the list of allowed teams to search of and we
// don't have channel restrictions then we return an empty result because we cannot get channels
// - If we receive channels restrictions we get:
// - If we don't have team id, we get those restricted channels (guest accounts and quick search)
// - If we have a team id then we only return those restricted channels that belongs to that team
func (s *SearchUserStore) getListOfAllowedChannels(teamId, channelId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, error) {
var listOfAllowedChannels []string
if viewRestrictions == nil && teamId == "" {
// nil return without error means all channels are allowed
return nil, nil
}
if teamId != "" && (viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId)) {
channels, err := s.rootStore.Channel().GetTeamChannels(teamId)
if err != nil {
return nil, errors.Wrap(err, "failed to get team channels")
}
for _, channel := range channels {
listOfAllowedChannels = append(listOfAllowedChannels, channel.Id)
}
if channelId != "" {
ch, err := s.rootStore.Channel().Get(channelId, true)
if err != nil {
return nil, errors.Wrapf(err, "failed to get channel with id: %s", channelId)
}
// Check if DM/GM channel, and add to the list.
// This is because GetTeamChannels does not return DM/GM channels.
// And since the channelId is passed from the API layer, it is already
// auth checked to confirm that the user has permission.
if ch.IsGroupOrDirect() {
listOfAllowedChannels = append(listOfAllowedChannels, channelId)
}
}
return listOfAllowedChannels, nil
}
if len(viewRestrictions.Channels) > 0 {
channels, err := s.rootStore.Channel().GetChannelsByIds(viewRestrictions.Channels, false)
if err != nil {
return nil, errors.Wrap(err, "failed to get channels by ids")
}
for _, c := range channels {
if teamId == "" || (teamId != "" && c.TeamId == teamId) {
listOfAllowedChannels = append(listOfAllowedChannels, c.Id)
}
}
return listOfAllowedChannels, nil
}
return []string{}, nil
}
func (s *SearchUserStore) AutocompleteUsersInChannel(rctx request.CTX, teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, error) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsAutocompletionEnabled() {
listOfAllowedChannels, nErr := s.getListOfAllowedChannels(teamId, channelId, options.ViewRestrictions)
if nErr != nil {
rctx.Logger().Warn("Encountered error on AutocompleteUsersInChannel.", mlog.String("search_engine", engine.GetName()), mlog.Err(nErr))
continue
}
if listOfAllowedChannels != nil && len(listOfAllowedChannels) == 0 {
return &model.UserAutocompleteInChannel{}, nil
}
options.ListOfAllowedChannels = listOfAllowedChannels
autocomplete, nErr := s.autocompleteUsersInChannelByEngine(rctx, engine, teamId, channelId, term, options)
if nErr != nil {
rctx.Logger().Warn("Encountered error on AutocompleteUsersInChannel.", mlog.String("search_engine", engine.GetName()), mlog.Err(nErr))
continue
}
rctx.Logger().Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
return autocomplete, nil
}
}
rctx.Logger().Debug("Using database search because no other search engine is available")
return s.UserStore.AutocompleteUsersInChannel(rctx, teamId, channelId, term, options)
}

View File

@@ -0,0 +1,12 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"strings"
)
func sanitizeSearchTerm(term string) string {
return strings.TrimLeft(term, "@")
}

View File

@@ -0,0 +1,235 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchtest
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
var searchChannelStoreTests = []searchTest{
{
Name: "Should be able to autocomplete a channel by name",
Fn: testAutocompleteChannelByName,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a channel by name (Postgres)",
Fn: testAutocompleteChannelByNamePostgres,
Tags: []string{EnginePostgres},
},
{
Name: "Should be able to autocomplete a channel by display name",
Fn: testAutocompleteChannelByDisplayName,
Tags: []string{EngineAll},
},
{
Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by - character",
Fn: testAutocompleteChannelByNameSplittedWithDashChar,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by - character (Postgres)",
Fn: testAutocompleteChannelByNameSplittedWithDashCharPostgres,
Tags: []string{EnginePostgres},
},
{
Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by _ character",
Fn: testAutocompleteChannelByNameSplittedWithUnderscoreChar,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a channel by a part of its display name when has parts splitted by whitespace character",
Fn: testAutocompleteChannelByDisplayNameSplittedByWhitespaces,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete retrieving all channels if the term is empty",
Fn: testAutocompleteAllChannelsIfTermIsEmpty,
Tags: []string{EngineAll},
},
{
Name: "Should be able to autocomplete channels in a case insensitive manner",
Fn: testSearchChannelsInCaseInsensitiveManner,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete channels in a case insensitive manner (Postgres)",
Fn: testSearchChannelsInCaseInsensitiveMannerPostgres,
Tags: []string{EnginePostgres},
},
{
Name: "Should support to autocomplete having a hyphen as the last character",
Fn: testSearchShouldSupportHavingHyphenAsLastCharacter,
Tags: []string{EngineAll},
},
{
Name: "Should support to autocomplete with archived channels",
Fn: testSearchShouldSupportAutocompleteWithArchivedChannels,
Tags: []string{EngineAll},
},
}
func TestSearchChannelStore(t *testing.T, s store.Store, testEngine *SearchTestEngine) {
th := &SearchTestHelper{
Context: request.TestContext(t),
Store: s,
}
err := th.SetupBasicFixtures()
require.NoError(t, err)
defer th.CleanFixtures()
runTestSearch(t, testEngine, searchChannelStoreTests, th)
}
func testAutocompleteChannelByName(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "Channel Alternate", "Channel Alternate", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
private, err := th.createChannel(th.Team.Id, "channel-altprivate", "Channel AltPrivate", "Channel Private", model.ChannelTypePrivate, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(private)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, alternate.Id, private.Id}, res)
res2, err := th.Store.Channel().Autocomplete(th.Context, th.User.Id, "channel-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatchWithTeamData(t, []string{th.ChannelBasic.Id, alternate.Id, private.Id, th.ChannelAnotherTeam.Id}, res2)
}
func testAutocompleteChannelByNamePostgres(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "Channel Alternate", "Channel Alternate", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, th.ChannelPrivate.Id, alternate.Id}, res)
}
func testAutocompleteChannelByDisplayName(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
private, err := th.createChannel(th.Team.Id, "channel-altprivate", "ChannelAltPrivate", "Channel Private", model.ChannelTypePrivate, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(private)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "ChannelA", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, alternate.Id, private.Id}, res)
res2, err := th.Store.Channel().Autocomplete(th.Context, th.User.Id, "ChannelA", false, false)
require.NoError(t, err)
th.checkChannelIdsMatchWithTeamData(t, []string{th.ChannelBasic.Id, alternate.Id, private.Id, th.ChannelAnotherTeam.Id}, res2)
}
func testAutocompleteChannelByNameSplittedWithDashChar(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, alternate.Id}, res)
}
func testAutocompleteChannelByNameSplittedWithDashCharPostgres(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, th.ChannelPrivate.Id, alternate.Id}, res)
}
func testAutocompleteChannelByNameSplittedWithUnderscoreChar(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel_alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel_a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{alternate.Id}, res)
res2, err := th.Store.Channel().Autocomplete(th.Context, th.User.Id, "channel_a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatchWithTeamData(t, []string{alternate.Id}, res2)
}
func testAutocompleteChannelByDisplayNameSplittedByWhitespaces(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "Channel Alternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "Channel A", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{alternate.Id}, res)
}
func testAutocompleteAllChannelsIfTermIsEmpty(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "Channel Alternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
other, err := th.createChannel(th.Team.Id, "other-channel", "Other Channel", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
defer th.deleteChannel(other)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, th.ChannelPrivate.Id, alternate.Id, other.Id}, res)
}
func testSearchChannelsInCaseInsensitiveManner(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channela", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, alternate.Id}, res)
res, err = th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "ChAnNeL-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, alternate.Id}, res)
res2, err := th.Store.Channel().Autocomplete(th.Context, th.User.Id, "channela", false, false)
require.NoError(t, err)
th.checkChannelIdsMatchWithTeamData(t, []string{th.ChannelAnotherTeam.Id, th.ChannelBasic.Id, alternate.Id}, res2)
res2, err = th.Store.Channel().Autocomplete(th.Context, th.User.Id, "ChAnNeL-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatchWithTeamData(t, []string{th.ChannelAnotherTeam.Id, th.ChannelBasic.Id, alternate.Id}, res2)
}
func testSearchChannelsInCaseInsensitiveMannerPostgres(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channela", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, alternate.Id}, res)
res, err = th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "ChAnNeL-a", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, th.ChannelPrivate.Id, alternate.Id}, res)
}
func testSearchShouldSupportHavingHyphenAsLastCharacter(t *testing.T, th *SearchTestHelper) {
alternate, err := th.createChannel(th.Team.Id, "channel-alternate", "ChannelAlternate", "", model.ChannelTypeOpen, th.User, false)
require.NoError(t, err)
defer th.deleteChannel(alternate)
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel-", false, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, th.ChannelPrivate.Id, alternate.Id}, res)
res2, err := th.Store.Channel().Autocomplete(th.Context, th.User.Id, "channel-", false, false)
require.NoError(t, err)
th.checkChannelIdsMatchWithTeamData(t, []string{th.ChannelAnotherTeam.Id, th.ChannelBasic.Id, th.ChannelPrivate.Id, alternate.Id}, res2)
}
func testSearchShouldSupportAutocompleteWithArchivedChannels(t *testing.T, th *SearchTestHelper) {
res, err := th.Store.Channel().AutocompleteInTeam(th.Context, th.Team.Id, th.User.Id, "channel-", true, false)
require.NoError(t, err)
th.checkChannelIdsMatch(t, []string{th.ChannelBasic.Id, th.ChannelPrivate.Id, th.ChannelDeleted.Id}, res)
}

View File

@@ -0,0 +1,253 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchtest
import (
"testing"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/stretchr/testify/require"
)
func TestSearchPostStoreEnabledCJK(t *testing.T, s store.Store) {
th := &SearchTestHelper{
Context: request.TestContext(t),
Store: s,
}
err := th.SetupBasicFixtures()
require.NoError(t, err)
defer th.CleanFixtures()
t.Run("Korean searches using nori analyzer", func(t *testing.T) {
t.Run("should be able to search with wildcard and exact search", func(t *testing.T) {
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "한글", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
p2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "한국", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
// Exact search
params := &model.SearchParams{Terms: "한글"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p1.Id, results.Posts)
// Wildcard search
params = &model.SearchParams{Terms: "한*"}
results, err = th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, p1.Id, results.Posts)
th.checkPostInSearchResults(t, p2.Id, results.Posts)
})
t.Run("should search one word and phrase with Nori segmentation", func(t *testing.T) {
pBul, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "불", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
pBulda, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "불다", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "소고기덮밥", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
p4, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "치킨덮밥", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
// One word "불": Nori segments "불다" into "불"+"다", so "불" matches both posts
params := &model.SearchParams{Terms: "불"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, pBul.Id, results.Posts)
th.checkPostInSearchResults(t, pBulda.Id, results.Posts)
// Unquoted "불다": SimpleQueryString treats one term as OR of analyzed tokens (불 OR 다), so both posts match
params = &model.SearchParams{Terms: "불다"}
results, err = th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, pBul.Id, results.Posts)
th.checkPostInSearchResults(t, pBulda.Id, results.Posts)
// Unquoted 덮밥 should match 소고기덮밥 and 치킨덮밥
params = &model.SearchParams{Terms: "덮밥"}
results, err = th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, p3.Id, results.Posts)
th.checkPostInSearchResults(t, p4.Id, results.Posts)
})
t.Run("should search in mixed Korean and English content", func(t *testing.T) {
p5, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "오늘 회의실 예약 meeting", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
params := &model.SearchParams{Terms: "회의실"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p5.Id, results.Posts)
})
t.Run("should search using phrase search", func(t *testing.T) {
p6, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "오늘 회의실 예약", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
params := &model.SearchParams{Terms: "\"오늘 회의실\""}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p6.Id, results.Posts)
})
})
t.Run("Japanese searches using kuromoji analyzer", func(t *testing.T) {
t.Run("should be able to search using wildcard and exact search", func(t *testing.T) {
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "東京", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
p2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "東北", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
// Exact match
params := &model.SearchParams{Terms: "東京"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p1.Id, results.Posts)
// Wildcard search
params = &model.SearchParams{Terms: "東*"}
results, err = th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, p1.Id, results.Posts)
th.checkPostInSearchResults(t, p2.Id, results.Posts)
})
t.Run("should search in mixed Japanese and English content", func(t *testing.T) {
p4, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "projectの締め切りは来週", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
params := &model.SearchParams{Terms: "締め切り"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p4.Id, results.Posts)
})
t.Run("should search using phrase search", func(t *testing.T) {
p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "今日の会議は中止です", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
params := &model.SearchParams{Terms: "\"今日の会議\""}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p3.Id, results.Posts)
})
t.Run("should find conjugated verb forms when searching infinitive", func(t *testing.T) {
// Create posts with different verb conjugations
pInfinitive, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "食べる", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
pPast, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "昨日ラーメンを食べました", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
pTe, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "食べている", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
pNegative, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "食べない", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
// Search for infinitive form should find all conjugated forms
params := &model.SearchParams{Terms: "食べる"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 4)
th.checkPostInSearchResults(t, pInfinitive.Id, results.Posts)
th.checkPostInSearchResults(t, pPast.Id, results.Posts)
th.checkPostInSearchResults(t, pTe.Id, results.Posts)
th.checkPostInSearchResults(t, pNegative.Id, results.Posts)
})
})
t.Run("Chinese searches using smartcn analyzer", func(t *testing.T) {
t.Run("should be able to search using wildcard and exact search", func(t *testing.T) {
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "电脑", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
p2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "电话", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
// Exact search
params := &model.SearchParams{Terms: "电脑"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p1.Id, results.Posts)
// Wildcard search
params = &model.SearchParams{Terms: "电*"}
results, err = th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, p1.Id, results.Posts)
th.checkPostInSearchResults(t, p2.Id, results.Posts)
})
t.Run("should search one and two characters with SmartCN segmentation (你 / 你好)", func(t *testing.T) {
pNiHao, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "你好", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
pNi, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "你", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
// One character: SmartCN segments "你好" into "你"+"好", so "你" matches both
params := &model.SearchParams{Terms: "你"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, pNi.Id, results.Posts)
th.checkPostInSearchResults(t, pNiHao.Id, results.Posts)
// Two characters (unquoted): SimpleQueryString matches any analyzed token (你 OR 好), so both posts match
params = &model.SearchParams{Terms: "你好"}
results, err = th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 2)
th.checkPostInSearchResults(t, pNi.Id, results.Posts)
th.checkPostInSearchResults(t, pNiHao.Id, results.Posts)
})
t.Run("should search in mixed Chinese and English content", func(t *testing.T) {
p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "this is 今天开会讨论API接口 content", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
params := &model.SearchParams{Terms: "接口"}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p3.Id, results.Posts)
})
t.Run("should search using phrase search", func(t *testing.T) {
p4, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "this is 今天开会讨论API接口 content", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
params := &model.SearchParams{Terms: "\"今天开会\""}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, 1)
th.checkPostInSearchResults(t, p4.Id, results.Posts)
})
})
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,503 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchtest
import (
"strconv"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SearchTestHelper struct {
Context *request.Context
Store store.Store
Team *model.Team
AnotherTeam *model.Team
User *model.User
User2 *model.User
UserAnotherTeam *model.User
ChannelBasic *model.Channel
ChannelPrivate *model.Channel
ChannelAnotherTeam *model.Channel
ChannelDeleted *model.Channel
}
func (th *SearchTestHelper) SetupBasicFixtures() error {
// Create teams
team, err := th.createTeam("searchtest-team", "Searchtest team", model.TeamOpen)
if err != nil {
return err
}
anotherTeam, err := th.createTeam("another-searchtest-team", "Another Searchtest team", model.TeamOpen)
if err != nil {
return err
}
// Create users
user, err := th.createUser("basicusername1", "basicnickname1", "basicfirstname1", "basiclastname1")
if err != nil {
return err
}
user2, err := th.createUser("basicusername2", "basicnickname2", "basicfirstname2", "basiclastname2")
if err != nil {
return err
}
useranother, err := th.createUser("basicusername3", "basicnickname3", "basicfirstname3", "basiclastname3")
if err != nil {
return err
}
// Create channels
channelBasic, err := th.createChannel(team.Id, "channel-a", "ChannelA", "", model.ChannelTypeOpen, nil, false)
if err != nil {
return err
}
channelPrivate, err := th.createChannel(team.Id, "channel-private", "ChannelPrivate", "", model.ChannelTypePrivate, nil, false)
if err != nil {
return err
}
channelDeleted, err := th.createChannel(team.Id, "channel-deleted", "ChannelA (deleted)", "", model.ChannelTypeOpen, nil, true)
if err != nil {
return err
}
channelAnotherTeam, err := th.createChannel(anotherTeam.Id, "channel-a", "ChannelA", "", model.ChannelTypeOpen, nil, false)
if err != nil {
return err
}
err = th.addUserToTeams(user, []string{team.Id, anotherTeam.Id})
if err != nil {
return err
}
err = th.addUserToTeams(user2, []string{team.Id, anotherTeam.Id})
if err != nil {
return err
}
err = th.addUserToTeams(useranother, []string{anotherTeam.Id})
if err != nil {
return err
}
err = th.addUserToChannels(user, []string{channelBasic.Id, channelPrivate.Id, channelDeleted.Id})
if err != nil {
return err
}
err = th.addUserToChannels(user2, []string{channelPrivate.Id, channelDeleted.Id})
if err != nil {
return err
}
err = th.addUserToChannels(useranother, []string{channelAnotherTeam.Id})
if err != nil {
return err
}
th.Team = team
th.AnotherTeam = anotherTeam
th.User = user
th.User2 = user2
th.UserAnotherTeam = useranother
th.ChannelBasic = channelBasic
th.ChannelPrivate = channelPrivate
th.ChannelAnotherTeam = channelAnotherTeam
th.ChannelDeleted = channelDeleted
return nil
}
func (th *SearchTestHelper) CleanFixtures() error {
err := th.deleteChannels([]*model.Channel{
th.ChannelBasic, th.ChannelPrivate, th.ChannelAnotherTeam, th.ChannelDeleted,
})
if err != nil {
return err
}
err = th.deleteTeam(th.Team)
if err != nil {
return err
}
err = th.deleteTeam(th.AnotherTeam)
if err != nil {
return err
}
err = th.cleanAllUsers([]*model.User{th.User, th.User2, th.UserAnotherTeam})
if err != nil {
return err
}
return nil
}
func (th *SearchTestHelper) createTeam(name, displayName, teamType string) (*model.Team, error) {
return th.Store.Team().Save(&model.Team{
Name: name,
DisplayName: displayName,
Type: teamType,
})
}
func (th *SearchTestHelper) deleteTeam(team *model.Team) error {
err := th.Store.Team().RemoveAllMembersByTeam(team.Id)
if err != nil {
return err
}
return th.Store.Team().PermanentDelete(team.Id)
}
func (th *SearchTestHelper) makeEmail() string {
return "success_" + model.NewId() + "@simulator.amazon.com"
}
func (th *SearchTestHelper) createUser(username, nickname, firstName, lastName string) (*model.User, error) {
return th.Store.User().Save(th.Context, &model.User{
Username: username,
Password: model.NewTestPassword(),
Nickname: nickname,
FirstName: firstName,
LastName: lastName,
Email: th.makeEmail(),
})
}
func (th *SearchTestHelper) createGuest(username, nickname, firstName, lastName string) (*model.User, error) {
return th.Store.User().Save(th.Context, &model.User{
Username: username,
Password: model.NewTestPassword(),
Nickname: nickname,
FirstName: firstName,
LastName: lastName,
Email: th.makeEmail(),
Roles: model.SystemGuestRoleId,
})
}
func (th *SearchTestHelper) deleteUser(user *model.User) error {
return th.Store.User().PermanentDelete(th.Context, user.Id)
}
func (th *SearchTestHelper) deleteBotUser(botID string) error {
if err := th.deleteBot(botID); err != nil {
return err
}
return th.Store.User().PermanentDelete(th.Context, botID)
}
func (th *SearchTestHelper) cleanAllUsers(users []*model.User) error {
for _, u := range users {
err := th.deleteUser(u)
if err != nil {
return err
}
}
return nil
}
func (th *SearchTestHelper) createBot(username, displayName, ownerID string) (*model.Bot, error) {
botModel := &model.Bot{
Username: username,
DisplayName: displayName,
OwnerId: ownerID,
}
user, err := th.Store.User().Save(th.Context, model.UserFromBot(botModel))
if err != nil {
return nil, errors.New(err.Error())
}
botModel.UserId = user.Id
bot, err := th.Store.Bot().Save(botModel)
if err != nil {
th.Store.User().PermanentDelete(th.Context, bot.UserId)
return nil, errors.New(err.Error())
}
return bot, nil
}
func (th *SearchTestHelper) deleteBot(botID string) error {
err := th.Store.Bot().PermanentDelete(botID)
if err != nil {
return errors.New(err.Error())
}
return nil
}
func (th *SearchTestHelper) createChannel(teamID, name, displayName, purpose string, channelType model.ChannelType, user *model.User, deleted bool) (*model.Channel, error) {
channel, err := th.Store.Channel().Save(th.Context, &model.Channel{
TeamId: teamID,
DisplayName: displayName,
Name: name,
Type: channelType,
Purpose: purpose,
}, 999)
if err != nil {
return nil, err
}
if user != nil {
err = th.addUserToChannels(user, []string{channel.Id})
if err != nil {
return nil, err
}
}
if deleted {
err := th.Store.Channel().Delete(channel.Id, model.GetMillis())
if err != nil {
return nil, err
}
}
return channel, nil
}
func (th *SearchTestHelper) createDirectChannel(teamID, displayName string, users []*model.User) (*model.Channel, error) {
channel := &model.Channel{
TeamId: teamID,
Name: model.GetDMNameFromIds(users[0].Id, users[1].Id),
DisplayName: displayName,
Type: model.ChannelTypeDirect,
}
m1 := &model.ChannelMember{}
m1.ChannelId = channel.Id
m1.UserId = users[0].Id
m1.NotifyProps = model.GetDefaultChannelNotifyProps()
m2 := &model.ChannelMember{}
m2.ChannelId = channel.Id
m2.UserId = users[1].Id
m2.NotifyProps = model.GetDefaultChannelNotifyProps()
channel, err := th.Store.Channel().SaveDirectChannel(th.Context, channel, m1, m2)
if err != nil {
return nil, err
}
return channel, nil
}
func (th *SearchTestHelper) createGroupChannel(teamID, displayName string, users []*model.User) (*model.Channel, error) {
userIDS := make([]string, 0, len(users))
for _, user := range users {
userIDS = append(userIDS, user.Id)
}
group := &model.Channel{
TeamId: teamID,
Name: model.GetGroupNameFromUserIds(userIDS),
DisplayName: displayName,
Type: model.ChannelTypeGroup,
}
channel, err := th.Store.Channel().Save(th.Context, group, 10000)
if err != nil {
return nil, errors.New(err.Error())
}
for _, user := range users {
err := th.addUserToChannels(user, []string{channel.Id})
if err != nil {
return nil, err
}
}
return channel, nil
}
func (th *SearchTestHelper) deleteChannel(channel *model.Channel) error {
err := th.Store.Channel().PermanentDeleteMembersByChannel(th.Context, channel.Id)
if err != nil {
return err
}
return th.Store.Channel().PermanentDelete(th.Context, channel.Id)
}
func (th *SearchTestHelper) deleteChannels(channels []*model.Channel) error {
for _, channel := range channels {
err := th.deleteChannel(channel)
if err != nil {
return err
}
}
return nil
}
func (th *SearchTestHelper) createPostModel(userID, channelID, message, hashtags, postType string, createAt int64, pinned bool) *model.Post {
return &model.Post{
Message: message,
ChannelId: channelID,
PendingPostId: model.NewId() + ":" + strconv.FormatInt(model.GetMillis(), 10),
UserId: userID,
Hashtags: hashtags,
IsPinned: pinned,
CreateAt: createAt,
Type: postType,
}
}
func (th *SearchTestHelper) createPost(userID, channelID, message, hashtags, postType string, createAt int64, pinned bool) (*model.Post, error) {
var creationTime int64 = 1000000
if createAt > 0 {
creationTime = createAt
}
postModel := th.createPostModel(userID, channelID, message, hashtags, postType, creationTime, pinned)
return th.Store.Post().Save(th.Context, postModel)
}
func (th *SearchTestHelper) createFileInfoModel(creatorID, postID, channelID, name, content, extension, mimeType string, createAt, size int64) *model.FileInfo {
return &model.FileInfo{
CreatorId: creatorID,
PostId: postID,
ChannelId: channelID,
CreateAt: createAt,
UpdateAt: createAt,
DeleteAt: 0,
Name: name,
Content: content,
Path: name,
Extension: extension,
Size: size,
MimeType: mimeType,
}
}
func (th *SearchTestHelper) createFileInfo(creatorID, postID, channelID, name, content, extension, mimeType string, createAt, size int64) (*model.FileInfo, error) {
var creationTime int64 = 1000000
if createAt > 0 {
creationTime = createAt
}
fileInfoModel := th.createFileInfoModel(creatorID, postID, channelID, name, content, extension, mimeType, creationTime, size)
return th.Store.FileInfo().Save(th.Context, fileInfoModel)
}
func (th *SearchTestHelper) createReply(userID, message, hashtags string, parent *model.Post, createAt int64, pinned bool) (*model.Post, error) {
replyModel := th.createPostModel(userID, parent.ChannelId, message, hashtags, parent.Type, createAt, pinned)
replyModel.RootId = parent.Id
return th.Store.Post().Save(th.Context, replyModel)
}
func (th *SearchTestHelper) deleteUserPosts(userID string) error {
err := th.Store.Post().PermanentDeleteByUser(th.Context, userID)
if err != nil {
return errors.New(err.Error())
}
return nil
}
func (th *SearchTestHelper) deleteUserFileInfos(userID string) error {
if _, err := th.Store.FileInfo().PermanentDeleteByUser(th.Context, userID); err != nil {
return errors.New(err.Error())
}
return nil
}
func (th *SearchTestHelper) addUserToTeams(user *model.User, teamIDS []string) error {
for _, teamID := range teamIDS {
_, err := th.Store.Team().SaveMember(th.Context, &model.TeamMember{TeamId: teamID, UserId: user.Id}, -1)
if err != nil {
return errors.New(err.Error())
}
}
return nil
}
func (th *SearchTestHelper) addUserToChannels(user *model.User, channelIDS []string) error {
for _, channelID := range channelIDS {
_, err := th.Store.Channel().SaveMember(th.Context, &model.ChannelMember{
ChannelId: channelID,
UserId: user.Id,
NotifyProps: model.GetDefaultChannelNotifyProps(),
})
if err != nil {
return errors.New(err.Error())
}
}
return nil
}
func (th *SearchTestHelper) assertUsersMatchInAnyOrder(t *testing.T, expected, actual []*model.User) {
expectedUsernames := make([]string, 0, len(expected))
for _, user := range expected {
user.Sanitize(map[string]bool{})
expectedUsernames = append(expectedUsernames, user.Username)
}
actualUsernames := make([]string, 0, len(actual))
for _, user := range actual {
user.Sanitize(map[string]bool{})
actualUsernames = append(actualUsernames, user.Username)
}
if assert.ElementsMatch(t, expectedUsernames, actualUsernames) {
assert.ElementsMatch(t, expected, actual)
}
}
func (th *SearchTestHelper) checkPostInSearchResults(t *testing.T, postID string, searchResults map[string]*model.Post) {
t.Helper()
postIDS := make([]string, 0, len(searchResults))
for ID := range searchResults {
postIDS = append(postIDS, ID)
}
assert.Contains(t, postIDS, postID, "Did not find expected post in search results.")
}
func (th *SearchTestHelper) checkFileInfoInSearchResults(t *testing.T, fileID string, searchResults map[string]*model.FileInfo) {
t.Helper()
fileIDS := make([]string, 0, len(searchResults))
for ID := range searchResults {
fileIDS = append(fileIDS, ID)
}
assert.Contains(t, fileIDS, fileID, "Did not find expected file in search results.")
}
func (th *SearchTestHelper) checkChannelIdsMatch(t *testing.T, expected []string, results model.ChannelList) {
t.Helper()
channelIds := make([]string, len(results))
for i, channel := range results {
channelIds[i] = channel.Id
}
require.ElementsMatch(t, expected, channelIds)
}
func (th *SearchTestHelper) checkChannelIdsMatchWithTeamData(t *testing.T, expected []string, results model.ChannelListWithTeamData) {
t.Helper()
channelIds := make([]string, len(results))
for i, channel := range results {
channelIds[i] = channel.Id
}
require.ElementsMatch(t, expected, channelIds)
}
type ByChannelDisplayName model.ChannelList
func (s ByChannelDisplayName) Len() int { return len(s) }
func (s ByChannelDisplayName) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s ByChannelDisplayName) Less(i, j int) bool {
if s[i].DisplayName != s[j].DisplayName {
return s[i].DisplayName < s[j].DisplayName
}
return s[i].Id < s[j].Id
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,75 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchtest
import (
"slices"
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
const (
EngineAll = "all"
EnginePostgres = "postgres"
EngineElasticSearch = "elasticsearch"
)
type SearchTestEngine struct {
Driver string
BeforeTest func(*testing.T, store.Store)
AfterTest func(*testing.T, store.Store)
}
type searchTest struct {
Name string
Fn func(*testing.T, *SearchTestHelper)
Tags []string
Skip bool
SkipMessage string
}
func filterTestsByTag(tests []searchTest, tags ...string) []searchTest {
filteredTests := []searchTest{}
for _, test := range tests {
if slices.Contains(test.Tags, EngineAll) {
filteredTests = append(filteredTests, test)
continue
}
for _, tag := range tags {
if slices.Contains(test.Tags, tag) {
filteredTests = append(filteredTests, test)
break
}
}
}
return filteredTests
}
func runTestSearch(t *testing.T, testEngine *SearchTestEngine, tests []searchTest, th *SearchTestHelper) {
if testing.Short() {
t.Skip("Skipping advanced search test")
return
}
filteredTests := filterTestsByTag(tests, testEngine.Driver)
for _, test := range filteredTests {
if test.Skip {
t.Log("SKIPPED: " + test.Name + ". Reason: " + test.SkipMessage)
continue
}
if testEngine.BeforeTest != nil {
testEngine.BeforeTest(t, th.Store)
}
testName := test.Name
testFn := test.Fn
t.Run(testName, func(t *testing.T) { testFn(t, th) })
if testEngine.AfterTest != nil {
testEngine.AfterTest(t, th.Store)
}
}
}

View File

@@ -0,0 +1,952 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchtest
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
var searchUserStoreTests = []searchTest{
{
Name: "Should retrieve all users in a channel if the search term is empty",
Fn: testGetAllUsersInChannelWithEmptyTerm,
Tags: []string{EngineAll},
},
{
Name: "Should honor channel restrictions when autocompleting users",
Fn: testHonorChannelRestrictionsAutocompletingUsers,
Tags: []string{EngineAll},
},
{
Name: "Should honor team restrictions when autocompleting users",
Fn: testHonorTeamRestrictionsAutocompletingUsers,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should return nothing if the user can't access the channels of a given search",
Fn: testShouldReturnNothingWithoutProperAccess,
Tags: []string{EngineAll},
Skip: true,
SkipMessage: "Failing when the ListOfAllowedChannels property is empty",
},
{
Name: "Should autocomplete for user using username",
Fn: testAutocompleteUserByUsername,
Tags: []string{EngineAll},
},
{
Name: "Should autocomplete user searching by first name",
Fn: testAutocompleteUserByFirstName,
Tags: []string{EngineAll},
},
{
Name: "Should autocomplete user searching by last name",
Fn: testAutocompleteUserByLastName,
Tags: []string{EngineAll},
},
{
Name: "Should autocomplete for user using nickname",
Fn: testAutocompleteUserByNickName,
Tags: []string{EngineAll},
},
{
Name: "Should autocomplete for user using email",
Fn: testAutocompleteUserByEmail,
Tags: []string{EngineAll},
Skip: true,
SkipMessage: "Failing for multiple different reasons in the engines",
},
{
Name: "Should be able not to match specific queries with mail",
Fn: testShouldNotMatchSpecificQueriesEmail,
Tags: []string{EngineAll},
},
{
Name: "Should be able to autocomplete a user by part of its username splitted by Dot",
Fn: testAutocompleteUserByUsernameWithDot,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a user by part of its username splitted by underscore",
Fn: testAutocompleteUserByUsernameWithUnderscore,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a user by part of its username splitted by hyphen",
Fn: testAutocompleteUserByUsernameWithHyphen,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should escape the percentage character",
Fn: testShouldEscapePercentageCharacter,
Tags: []string{EngineAll},
},
{
Name: "Should escape the dash character",
Fn: testShouldEscapeUnderscoreCharacter,
Tags: []string{EngineAll},
},
{
Name: "Should be able to search inactive users",
Fn: testShouldBeAbleToSearchInactiveUsers,
Tags: []string{EnginePostgres, EngineElasticSearch},
},
{
Name: "Should be able to search filtering by role",
Fn: testShouldBeAbleToSearchFilteringByRole,
Tags: []string{EnginePostgres, EngineElasticSearch},
},
{
Name: "Should ignore leading @ when searching users",
Fn: testShouldIgnoreLeadingAtSymbols,
Tags: []string{EngineAll},
},
{
Name: "Should search users in a case insensitive manner",
Fn: testSearchUsersShouldBeCaseInsensitive,
Tags: []string{EngineAll},
},
{
Name: "Should support one or two character usernames and first/last names in search",
Fn: testSearchOneTwoCharUsernamesAndFirstLastNames,
Tags: []string{EngineAll},
},
{
Name: "Should support Korean characters",
Fn: testShouldSupportKoreanCharacters,
Tags: []string{EngineAll},
},
{
Name: "Should support search with a hyphen at the end of the term",
Fn: testSearchWithHyphenAtTheEndOfTheTerm,
Tags: []string{EngineAll},
},
{
Name: "Should support search all users in a team",
Fn: testSearchUsersInTeam,
Tags: []string{EngineElasticSearch},
},
{
Name: "Should support search users by full name",
Fn: testSearchUsersByFullName,
Tags: []string{EngineAll},
},
{
Name: "Should support search all users in a team with username containing a dot",
Fn: testSearchUsersInTeamUsernameWithDot,
Tags: []string{EngineAll},
},
{
Name: "Should support search all users in a team with username containing a hyphen",
Fn: testSearchUsersInTeamUsernameWithHyphen,
Tags: []string{EngineAll},
},
{
Name: "Should support search all users in a team with username containing a underscore",
Fn: testSearchUsersInTeamUsernameWithUnderscore,
Tags: []string{EngineAll},
},
{
Name: "Should support search all users containing a substring in any name",
Fn: testSearchUserBySubstringInAnyName,
Tags: []string{EngineAll},
},
}
func TestSearchUserStore(t *testing.T, s store.Store, testEngine *SearchTestEngine) {
th := &SearchTestHelper{
Context: request.TestContext(t),
Store: s,
}
err := th.SetupBasicFixtures()
require.NoError(t, err)
defer th.CleanFixtures()
runTestSearch(t, testEngine, searchUserStoreTests, th)
}
func testGetAllUsersInChannelWithEmptyTerm(t *testing.T, th *SearchTestHelper) {
options := &model.UserSearchOptions{
AllowFullNames: true,
Limit: model.UserSearchDefaultLimit,
}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
t.Run("Should be able to correctly honor limit when autocompleting", func(t *testing.T) {
result, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
require.Len(t, result.InChannel, 1)
require.Len(t, result.OutOfChannel, 1)
})
t.Run("Return all users in team", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Return all users in teams even though some of them don't have a team associated", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
userAlternate, err := th.createUser("user-alternate", "user-alternate", "user", "alternate")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
userGuest, err := th.createGuest("user-guest", "user-guest", "user", "guest")
require.NoError(t, err)
defer th.deleteUser(userGuest)
// In case teamId and channelId are empty our current logic goes through Search
users, err := th.Store.User().Search(th.Context, "", "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2, th.UserAnotherTeam,
userAlternate, userGuest}, users)
})
}
func testHonorChannelRestrictionsAutocompletingUsers(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("user-alternate", "user-alternate", "user", "alternate")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
guest, err := th.createGuest("guest", "guest", "guest", "one")
require.NoError(t, err)
err = th.addUserToTeams(guest, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(guest, []string{th.ChannelBasic.Id})
require.NoError(t, err)
defer th.deleteUser(guest)
t.Run("Autocomplete users with channel restrictions", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Channels: []string{th.ChannelBasic.Id}}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, userAlternate, guest}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Autocomplete users with term and channel restrictions", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Channels: []string{th.ChannelBasic.Id}}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alt", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Autocomplete users with all channels restricted", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Teams: []string{}, Channels: []string{}}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Autocomplete users with all channels restricted but with empty team", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Teams: []string{}, Channels: []string{}}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, "", th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Autocomplete users with empty team and channels restricted", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Channels: []string{th.ChannelBasic.Id}}
// In case teamId and channelId are empty our current logic goes through Search
users, err := th.Store.User().Search(th.Context, "", "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate, guest, th.User}, users)
})
}
func testHonorTeamRestrictionsAutocompletingUsers(t *testing.T, th *SearchTestHelper) {
t.Run("Should return results for users in the team", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Teams: []string{th.Team.Id}}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Should return empty because we're filtering all the teams", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Teams: []string{}, Channels: []string{}}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return empty when searching in one team and filtering by another", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Teams: []string{th.AnotherTeam.Id}}
users, err := th.Store.User().Search(th.Context, th.Team.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users)
acusers, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, acusers.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, acusers.OutOfChannel)
})
}
func testShouldReturnNothingWithoutProperAccess(t *testing.T, th *SearchTestHelper) {
t.Run("Should return results users for the defined channel in the list", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ListOfAllowedChannels = []string{th.ChannelBasic.Id}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return empty because we're filtering all the channels", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
options.ListOfAllowedChannels = []string{}
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
}
func testAutocompleteUserByUsername(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternateusername", "alternatenick", "user", "alternate")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
}
func testAutocompleteUserByFirstName(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("user-alternate", "user-alternate", "altfirstname", "lastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should autocomplete users when the first name is unique", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "altfirstname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should autocomplete users for in the channel and out of the channel with the same first name", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "basicfirstname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
}
func testAutocompleteUserByLastName(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("user-alternate", "user-alternate", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should return results when the last name is unique", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "altlastname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results for in the channel and out of the channel with the same last name", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "basiclastname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
}
func testAutocompleteUserByNickName(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternateusername", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should return results when the nickname is unique", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternatenickname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return users that share the same part of the nickname", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "basicnickname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
}
func testAutocompleteUserByEmail(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternateusername", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
userAlternate.Email = "useralt@test.email.com"
_, err = th.Store.User().Update(th.Context, userAlternate, false)
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should autocomplete users when the email is unique", func(t *testing.T) {
options := createDefaultOptions(false, true, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "useralt@test.email.com", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should autocomplete users that share the same email user prefix", func(t *testing.T) {
options := createDefaultOptions(false, true, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "success_", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Should autocomplete users that share the same email domain", func(t *testing.T) {
options := createDefaultOptions(false, true, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "simulator.amazon.com", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Should search users when the email is unique", func(t *testing.T) {
options := createDefaultOptions(false, true, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "useralt@test.email.com", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
t.Run("Should search users that share the same email user prefix", func(t *testing.T) {
options := createDefaultOptions(false, true, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "success_", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users)
})
t.Run("Should search users that share the same email domain", func(t *testing.T) {
options := createDefaultOptions(false, true, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "simulator.amazon.com", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users)
})
}
func testShouldNotMatchSpecificQueriesEmail(t *testing.T, th *SearchTestHelper) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "success_", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
}
func testAutocompleteUserByUsernameWithDot(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate.username", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should return results when searching for the whole username with Dot", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternate.username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results when searching for part of the username including the Dot", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, ".username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results when searching for part of the username not including the Dot", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
}
func testAutocompleteUserByUsernameWithUnderscore(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate_username", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should return results when searching for the whole username with underscore", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternate_username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results when searching for part of the username including the underscore", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "_username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results when searching for part of the username not including the underscore", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
}
func testAutocompleteUserByUsernameWithHyphen(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate-username", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should return results when searching for the whole username with hyphen", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternate-username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results when searching for part of the username including the hyphen", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "-username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should return results when searching for part of the username not including the hyphen", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "username", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
}
func testShouldEscapePercentageCharacter(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternateusername", "alternate%nickname", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should autocomplete users escaping percentage symbol", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternate%", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should search users escaping percentage symbol", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "alternate%", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
}
func testShouldEscapeUnderscoreCharacter(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate_username", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should autocomplete users escaping underscore symbol", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternate_", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should search users escaping underscore symbol", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "alternate_", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
}
func testShouldBeAbleToSearchInactiveUsers(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("basicusernamealternate", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
userAlternate.DeleteAt = model.GetMillis()
_, err = th.Store.User().Update(th.Context, userAlternate, true)
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should autocomplete inactive users if we allow it", func(t *testing.T) {
options := createDefaultOptions(false, false, true)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Should search inactive users if we allow it", func(t *testing.T) {
options := createDefaultOptions(false, false, true)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2, userAlternate}, users)
})
t.Run("Shouldn't autocomplete inactive users if we don't allow it", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Shouldn't search inactive users if we don't allow it", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2}, users)
})
}
func testShouldBeAbleToSearchFilteringByRole(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("basicusernamealternate", "alternatenickname", "firstname", "altlastname")
require.NoError(t, err)
userAlternate.Roles = "system_admin system_user"
_, err = th.Store.User().Update(th.Context, userAlternate, true)
require.NoError(t, err)
defer th.deleteUser(userAlternate)
userAlternate2, err := th.createUser("basicusernamealternate2", "alternatenickname2", "firstname2", "altlastname2")
require.NoError(t, err)
userAlternate2.Roles = "system_user"
_, err = th.Store.User().Update(th.Context, userAlternate2, true)
require.NoError(t, err)
defer th.deleteUser(userAlternate2)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToTeams(userAlternate2, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should autocomplete users filtering by roles", func(t *testing.T) {
options := createDefaultOptions(false, false, true)
options.Role = "system_admin"
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should search users filtering by roles", func(t *testing.T) {
options := createDefaultOptions(false, false, true)
options.Role = "system_admin"
users, err := th.Store.User().Search(th.Context, th.Team.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
}
func testShouldIgnoreLeadingAtSymbols(t *testing.T, th *SearchTestHelper) {
t.Run("Should autocomplete ignoring the @ symbol at the beginning", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "@basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
})
t.Run("Should search ignoring the @ symbol at the beginning", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "@basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2}, users)
})
}
func testSearchUsersShouldBeCaseInsensitive(t *testing.T, th *SearchTestHelper) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "BaSiCUsErNaMe", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User2}, users.OutOfChannel)
}
func testSearchOneTwoCharUsernamesAndFirstLastNames(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("ho", "alternatenickname", "zi", "k")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should support two characters in the full name", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "zi", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should support two characters in the username", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "ho", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
}
func testShouldSupportKoreanCharacters(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate-username", "alternate-nickname", "서강준", "안신원")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
t.Run("Should support hanja korean characters", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "서강준", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
t.Run("Should support hangul korean characters", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "안신원", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
})
}
func testSearchWithHyphenAtTheEndOfTheTerm(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate-username", "alternate-nickname", "altfirst", "altlast")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().AutocompleteUsersInChannel(th.Context, th.Team.Id, th.ChannelBasic.Id, "alternate-", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users.InChannel)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users.OutOfChannel)
}
func testSearchUsersInTeam(t *testing.T, th *SearchTestHelper) {
t.Run("Should return all the team users", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2}, users)
})
t.Run("Should return all the team users with no team id", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, "", "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2, th.UserAnotherTeam}, users)
})
t.Run("Should return all the team users filtered by username", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicusername1", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users)
})
t.Run("Should not return spurious results", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "falseuser", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users)
})
t.Run("Should return all the team users filtered by username and with channel restrictions", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Channels: []string{th.ChannelBasic.Id}}
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicusername", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users)
})
t.Run("Should return all the team users filtered by username and with all channel restricted", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
options.ViewRestrictions = &model.ViewUsersRestrictions{Channels: []string{}}
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicusername1", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users)
})
t.Run("Should honor the limit when searching users in team", func(t *testing.T) {
optionsWithLimit := &model.UserSearchOptions{
Limit: 1,
}
users, err := th.Store.User().Search(th.Context, th.Team.Id, "", optionsWithLimit)
require.NoError(t, err)
require.Len(t, users, 1)
})
}
func testSearchUsersInTeamUsernameWithDot(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate.username", "altnickname", "altfirst", "altlast")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "alternate.", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
}
func testSearchUsersInTeamUsernameWithHyphen(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate-username", "altnickname", "altfirst", "altlast")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "alternate-", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
}
func testSearchUsersInTeamUsernameWithUnderscore(t *testing.T, th *SearchTestHelper) {
userAlternate, err := th.createUser("alternate_username", "altnickname", "altfirst", "altlast")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "alternate_", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
}
func testSearchUsersByFullName(t *testing.T, th *SearchTestHelper) {
t.Run("Should search users by full name", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicfirstname", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User, th.User2}, users)
})
t.Run("Should search user by full name", func(t *testing.T) {
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicfirstname1", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{th.User}, users)
})
t.Run("Should return empty when search by full name and is deactivated", func(t *testing.T) {
options := createDefaultOptions(false, false, false)
users, err := th.Store.User().Search(th.Context, th.Team.Id, "basicfirstname1", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{}, users)
})
}
func testSearchUserBySubstringInAnyName(t *testing.T, th *SearchTestHelper) {
t.Run("Should search users by substring in first name", func(t *testing.T) {
userAlternate, err := th.createUser("user-alternate", "user-alternate", "alternate helloooo first name", "alternate")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
// searching user without specifying team
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, "", "hello", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
// adding user to team to search by team
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options = createDefaultOptions(true, false, false)
users, err = th.Store.User().Search(th.Context, th.Team.Id, "hello", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
t.Run("Should search users by substring in last name name", func(t *testing.T) {
userAlternate, err := th.createUser("user-alternate", "user-alternate", "alternate", "alternate helloooo last name")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, "", "hello", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
// adding user to team to search by team
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options = createDefaultOptions(true, false, false)
users, err = th.Store.User().Search(th.Context, th.Team.Id, "hello", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
t.Run("Should search users by substring in nickname name", func(t *testing.T) {
userAlternate, err := th.createUser("user-alternate", "alternate helloooo nickname", "alternate hello first name", "alternate")
require.NoError(t, err)
defer th.deleteUser(userAlternate)
options := createDefaultOptions(true, false, false)
users, err := th.Store.User().Search(th.Context, "", "hello", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
// adding user to team to search by team
err = th.addUserToTeams(userAlternate, []string{th.Team.Id})
require.NoError(t, err)
err = th.addUserToChannels(userAlternate, []string{th.ChannelBasic.Id})
require.NoError(t, err)
options = createDefaultOptions(true, false, false)
users, err = th.Store.User().Search(th.Context, th.Team.Id, "hello", options)
require.NoError(t, err)
th.assertUsersMatchInAnyOrder(t, []*model.User{userAlternate}, users)
})
}
func createDefaultOptions(allowFullName, allowEmails, allowInactive bool) *model.UserSearchOptions {
return &model.UserSearchOptions{
AllowFullNames: allowFullName,
AllowEmails: allowEmails,
AllowInactive: allowInactive,
Limit: model.UserSearchDefaultLimit,
}
}

View File

@@ -0,0 +1,980 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/einterfaces"
"github.com/pkg/errors"
sq "github.com/mattermost/squirrel"
)
const MaxPerPage = 1000
const DefaultPerPage = 10
// Usually rules are how we define the policy, hence the versioning. For v0.1, we also
// have the imports field which is used to link with the parent policy.
type accessControlPolicyV0_1 struct {
Imports []string `json:"imports"`
Rules []model.AccessControlPolicyRule `json:"rules"`
Roles []string `json:"roles,omitempty"`
Scope string `json:"scope,omitempty"`
ScopeID string `json:"scope_id,omitempty"`
}
// These are the fields that meant to be unchanged with the policy versions.
type storeAccessControlPolicy struct {
ID string
Name string
Type string
Active bool
CreateAt int64
Revision int
Version string
Data []byte
Props []byte
}
// This needs to be updated with the new version of the policy.
// with the new name as this only supports v0.1
func (s *storeAccessControlPolicy) toModel() (*model.AccessControlPolicy, error) {
policy := &model.AccessControlPolicy{
ID: s.ID,
Name: s.Name,
Type: s.Type,
Active: s.Active,
CreateAt: s.CreateAt,
Revision: s.Revision,
Version: s.Version,
}
if len(s.Data) > 0 {
var p accessControlPolicyV0_1
if err := json.Unmarshal(s.Data, &p); err != nil {
return nil, err
}
policy.Imports = p.Imports
policy.Rules = p.Rules
policy.Roles = p.Roles
policy.Scope = p.Scope
policy.ScopeID = p.ScopeID
}
if len(s.Props) > 0 {
if err := json.Unmarshal(s.Props, &policy.Props); err != nil {
return nil, err
}
}
return policy, nil
}
func fromModel(policy *model.AccessControlPolicy) (*storeAccessControlPolicy, error) {
imports := policy.Imports
if imports == nil {
imports = []string{}
}
data, err := json.Marshal(&accessControlPolicyV0_1{
Imports: imports,
Rules: policy.Rules,
Roles: policy.Roles,
Scope: policy.Scope,
ScopeID: policy.ScopeID,
})
if err != nil {
return nil, err
}
props, err := json.Marshal(policy.Props)
if err != nil {
return nil, err
}
return &storeAccessControlPolicy{
ID: policy.ID,
Name: policy.Name,
Type: policy.Type,
Active: policy.Active,
CreateAt: policy.CreateAt,
Revision: policy.Revision,
Version: policy.Version,
Data: data,
Props: props,
}, nil
}
func accessControlPolicySliceColumns(prefix ...string) []string {
var p string
if len(prefix) == 1 {
p = prefix[0] + "."
} else if len(prefix) > 1 {
panic("cannot accept multiple prefixes")
}
return []string{
p + "ID",
p + "Name",
p + "Type",
p + "Active",
p + "CreateAt",
p + "Revision",
p + "Version",
p + "Data",
p + "Props",
}
}
func accessControlPolicyHistorySliceColumns(prefix ...string) []string {
var p string
if len(prefix) == 1 {
p = prefix[0] + "."
} else if len(prefix) > 1 {
panic("cannot accept multiple prefixes")
}
return []string{
p + "ID",
p + "Name",
p + "Type",
p + "CreateAt",
p + "Revision",
p + "Version",
p + "Data",
p + "Props",
}
}
type SqlAccessControlPolicyStore struct {
*SqlStore
metrics einterfaces.MetricsInterface
selectQueryBuilder sq.SelectBuilder
}
// newSqlAccessControlPolicyStore creates an instance of AccessControlPolicyStorea.
func newSqlAccessControlPolicyStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) store.AccessControlPolicyStore {
s := &SqlAccessControlPolicyStore{
SqlStore: sqlStore,
metrics: metrics,
}
s.selectQueryBuilder = s.getQueryBuilder().Select(accessControlPolicySliceColumns()...).From("AccessControlPolicies")
return s
}
func preSaveAccessControlPolicy(policy *storeAccessControlPolicy, existingPolicy *model.AccessControlPolicy) {
// since policies are immutable, we need to create a new revision
// also if it's going to be saved, eventually it will be the new one
// we overwrite createAt to make sure it gets the correct timestamp before saving
// if there is no existing policy, we set the revision to 1
policy.CreateAt = model.GetMillis()
if existingPolicy != nil {
policy.Revision = existingPolicy.Revision + 1
} else {
policy.Revision = 1
}
}
func (s *SqlAccessControlPolicyStore) Save(rctx request.CTX, policy *model.AccessControlPolicy) (*model.AccessControlPolicy, error) {
if err := policy.IsValid(); err != nil {
return nil, err
}
tx, err := s.GetMaster().Begin()
if err != nil {
return nil, errors.Wrap(err, "failed to start transaction")
}
defer finalizeTransactionX(tx, &err)
existingPolicy, err := s.getT(rctx, tx, policy.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, errors.Wrapf(err, "failed to fetch policy with id=%s", policy.ID)
}
storePolicy, err := fromModel(policy)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with Id=%s", policy.ID)
}
data := storePolicy.Data
props := storePolicy.Props
if s.IsBinaryParamEnabled() {
data = AppendBinaryFlag(data)
props = AppendBinaryFlag(props)
}
if existingPolicy != nil {
if existingPolicy.Type != policy.Type {
return nil, errors.New("cannot change type of existing policy")
}
// move existing policy to history
tmp, err2 := fromModel(existingPolicy)
if err2 != nil {
return nil, errors.Wrapf(err2, "failed to parse policy with id=%s", policy.ID)
}
// Check if the policy has actually changed
// We compare data and version fields. Name changes are cosmetic
// and should not create a new revision in history.
if bytes.Equal(storePolicy.Data, tmp.Data) &&
storePolicy.Version == tmp.Version {
// Update name in place if it changed, without bumping revision
if storePolicy.Name != tmp.Name {
nameUpdate := s.getQueryBuilder().
Update("AccessControlPolicies").
Set("Name", storePolicy.Name).
Where(sq.Eq{"ID": policy.ID})
if _, err = tx.ExecBuilder(nameUpdate); err != nil {
if IsUniqueConstraintError(err, []string{"Name", "idx_accesscontrolpolicies_name_type"}) {
return nil, store.NewErrConflict("AccessControlPolicy", err, "name="+policy.Name)
}
return nil, errors.Wrapf(err, "failed to update name for policy with id=%s", policy.ID)
}
existingPolicy.Name = storePolicy.Name
if err = tx.Commit(); err != nil {
return nil, errors.Wrap(err, "commit_transaction")
}
}
return existingPolicy, nil
}
existingData := tmp.Data
existingProps := tmp.Props
if s.IsBinaryParamEnabled() {
existingData = AppendBinaryFlag(existingData)
existingProps = AppendBinaryFlag(existingProps)
}
query := s.getQueryBuilder().
Insert("AccessControlPolicyHistory").
Columns(accessControlPolicyHistorySliceColumns()...).
Values(tmp.ID, tmp.Name, tmp.Type, tmp.CreateAt, tmp.Revision, tmp.Version, existingData, existingProps)
_, err = tx.ExecBuilder(query)
if err != nil {
return nil, errors.Wrapf(err, "failed to save policy with id=%s to history", policy.ID)
}
err = s.deleteT(rctx, tx, existingPolicy.ID)
if err != nil {
return nil, errors.Wrapf(err, "failed to delete policy with id=%s", policy.ID)
}
} else {
// if there is no existing policy, also check the history table
// to make sure we are not overwriting an existing policy
existingPolicy, err = s.getHistoryT(rctx, tx, policy.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, errors.Wrapf(err, "failed to fetch policy with id=%s", policy.ID)
}
}
preSaveAccessControlPolicy(storePolicy, existingPolicy)
query := s.getQueryBuilder().
Insert("AccessControlPolicies").
Columns(accessControlPolicySliceColumns()...).
Values(storePolicy.ID, storePolicy.Name, storePolicy.Type, storePolicy.Active, storePolicy.CreateAt, storePolicy.Revision, storePolicy.Version, data, props)
_, err = tx.ExecBuilder(query)
if err != nil {
if IsUniqueConstraintError(err, []string{"Name", "idx_accesscontrolpolicies_name_type"}) {
return nil, store.NewErrConflict("AccessControlPolicy", err, "name="+policy.Name)
}
return nil, errors.Wrapf(err, "failed to save policy with id=%s", policy.ID)
}
cp, err := storePolicy.toModel()
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with id=%s", policy.ID)
}
if err = tx.Commit(); err != nil {
return nil, errors.Wrap(err, "commit_transaction")
}
return cp, nil
}
func (s *SqlAccessControlPolicyStore) Delete(rctx request.CTX, id string) error {
tx, err := s.GetMaster().Begin()
if err != nil {
return errors.Wrap(err, "failed to start transaction")
}
defer finalizeTransactionX(tx, &err)
existingPolicy, err := s.getT(rctx, tx, id)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return errors.Wrapf(err, "failed to fetch policy with id=%s", id)
}
if existingPolicy != nil {
tmp, err2 := fromModel(existingPolicy)
if err2 != nil {
return errors.Wrapf(err2, "failed to parse policy with id=%s", id)
}
data := tmp.Data
props := tmp.Props
if s.IsBinaryParamEnabled() {
data = AppendBinaryFlag(data)
props = AppendBinaryFlag(props)
}
query := s.getQueryBuilder().
Insert("AccessControlPolicyHistory").
Columns(accessControlPolicyHistorySliceColumns()...).
Values(tmp.ID, tmp.Name, tmp.Type, tmp.CreateAt, tmp.Revision, tmp.Version, data, props)
_, err = tx.ExecBuilder(query)
if err != nil {
return errors.Wrapf(err, "failed to save policy with id=%s to history", id)
}
err = s.deleteT(rctx, tx, existingPolicy.ID)
if err != nil {
return errors.Wrapf(err, "failed to delete policy with id=%s", id)
}
}
if err = tx.Commit(); err != nil {
return errors.Wrap(err, "commit_transaction")
}
return nil
}
func (s *SqlAccessControlPolicyStore) deleteT(_ request.CTX, tx *sqlxTxWrapper, id string) error {
query := s.getQueryBuilder().Delete("AccessControlPolicies").Where(sq.Eq{"ID": id})
_, err := tx.ExecBuilder(query)
if err != nil {
return errors.Wrapf(err, "failed to delete policy with id=%s", id)
}
return nil
}
func (s *SqlAccessControlPolicyStore) SetActiveStatus(rctx request.CTX, id string, active bool) (*model.AccessControlPolicy, error) {
tx, err := s.GetMaster().Begin()
if err != nil {
return nil, errors.Wrap(err, "failed to start transaction")
}
defer finalizeTransactionX(tx, &err)
existingPolicy, err := s.getT(rctx, tx, id)
if err != nil {
return nil, errors.Wrapf(err, "failed to fetch policy with id=%s", id)
} else if errors.Is(err, sql.ErrNoRows) {
return nil, store.NewErrNotFound("AccessControlPolicy", id)
}
// also make sure if the policy is valid before updating active status
// just in case
existingPolicy.Active = active
if appErr := existingPolicy.IsValid(); err != nil {
return nil, appErr
}
query, args, err := s.getQueryBuilder().Update("AccessControlPolicies").Set("Active", active).Where(sq.Eq{"ID": id}).ToSql()
if err != nil {
return nil, errors.Wrapf(err, "failed to build query for policy with id=%s", id)
}
_, err = tx.Exec(query, args...)
if err != nil {
return nil, errors.Wrapf(err, "failed to update policy with id=%s", id)
}
if existingPolicy.Type == model.AccessControlPolicyTypeParent {
// if the policy is a parent, we need to update the child policies
expr := sq.Expr("Data->'imports' @> ?::jsonb", fmt.Sprintf("%q", id))
query, args, err = s.getQueryBuilder().Update("AccessControlPolicies").Set("Active", active).Where(expr).ToSql()
if err != nil {
return nil, errors.Wrapf(err, "failed to build query for policy with id=%s", id)
}
_, err = tx.Exec(query, args...)
if err != nil {
return nil, errors.Wrapf(err, "failed to update child policies with id=%s", id)
}
}
if err = tx.Commit(); err != nil {
return nil, errors.Wrap(err, "commit_transaction")
}
return existingPolicy, nil
}
func (s *SqlAccessControlPolicyStore) SetActiveStatusMultiple(rctx request.CTX, list []model.AccessControlPolicyActiveUpdate) ([]*model.AccessControlPolicy, error) {
tx, err := s.GetMaster().Begin()
if err != nil {
return nil, errors.Wrap(err, "failed to start transaction")
}
defer finalizeTransactionX(tx, &err)
// Group by active status for batch updates
activeTrue := []string{}
activeFalse := []string{}
ids := make([]any, 0, len(list))
for _, entry := range list {
ids = append(ids, entry.ID)
if entry.Active {
activeTrue = append(activeTrue, entry.ID)
continue
}
activeFalse = append(activeFalse, entry.ID)
}
// Update active=true policies
if len(activeTrue) > 0 {
query, args, qbErr := s.getQueryBuilder().
Update("AccessControlPolicies").
Set("Active", true).
Where(sq.Eq{"ID": activeTrue}).
ToSql()
if qbErr != nil {
return nil, errors.Wrap(qbErr, "failed to build active=true update query")
}
_, err = tx.Exec(query, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to update active=true policies")
}
}
// Update active=false policies
if len(activeFalse) > 0 {
query, args, qbErr := s.getQueryBuilder().
Update("AccessControlPolicies").
Set("Active", false).
Where(sq.Eq{"ID": activeFalse}).
ToSql()
if qbErr != nil {
return nil, errors.Wrap(qbErr, "failed to build active=false update query")
}
_, err = tx.Exec(query, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to update active=false policies")
}
}
p := []storeAccessControlPolicy{}
query := s.selectQueryBuilder.Where(sq.Eq{"ID": ids})
err = tx.SelectBuilder(&p, query)
if err != nil {
return nil, errors.Wrapf(err, "failed to find policies with ids=%v", ids)
}
policies := make([]*model.AccessControlPolicy, len(p))
for i := range p {
policies[i], err = p[i].toModel()
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with id=%s", p[i].ID)
}
}
if err = tx.Commit(); err != nil {
return nil, errors.Wrap(err, "commit_transaction")
}
return policies, nil
}
func (s *SqlAccessControlPolicyStore) Get(_ request.CTX, id string) (*model.AccessControlPolicy, error) {
p := storeAccessControlPolicy{}
query := s.selectQueryBuilder.Where(sq.Eq{"ID": id})
err := s.GetMaster().GetBuilder(&p, query)
if err != nil {
if err == sql.ErrNoRows {
return nil, store.NewErrNotFound("AccessControlPolicy", id)
}
return nil, errors.Wrapf(err, "failed to find policy with id=%s", id)
}
policy, err := p.toModel()
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with id=%s", id)
}
return policy, nil
}
func (s *SqlAccessControlPolicyStore) getT(_ request.CTX, tx *sqlxTxWrapper, id string) (*model.AccessControlPolicy, error) {
query := s.getQueryBuilder().
Select(accessControlPolicySliceColumns()...).
From("AccessControlPolicies").
Where(
sq.Eq{"ID": id},
)
sql, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrapf(err, "failed to build query for policy with id=%s", id)
}
var storePolicy storeAccessControlPolicy
err = tx.Get(&storePolicy, sql, args...)
if err != nil {
return nil, err
}
policy, err := storePolicy.toModel()
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with id=%s", id)
}
return policy, nil
}
func (s *SqlAccessControlPolicyStore) getHistoryT(_ request.CTX, tx *sqlxTxWrapper, id string) (*model.AccessControlPolicy, error) {
query := s.getQueryBuilder().
Select(accessControlPolicyHistorySliceColumns()...).
From("AccessControlPolicyHistory").
Where(
sq.Eq{"ID": id},
).OrderBy("Revision DESC").
Limit(1)
sql, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrapf(err, "failed to build query for policy with id=%s", id)
}
var storePolicy storeAccessControlPolicy
err = tx.Get(&storePolicy, sql, args...)
if err != nil {
return nil, err
}
policy, err := storePolicy.toModel()
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with id=%s", id)
}
return policy, nil
}
func (s *SqlAccessControlPolicyStore) GetAll(_ request.CTX, opts model.GetAccessControlPolicyOptions) ([]*model.AccessControlPolicy, model.AccessControlPolicyCursor, error) {
p := []storeAccessControlPolicy{}
query := s.selectQueryBuilder
if opts.ParentID != "" {
query = query.Where(sq.Expr("Data->'imports' @> ?", fmt.Sprintf("%q", opts.ParentID)))
}
if opts.Type != "" {
query = query.Where(sq.Eq{"Type": opts.Type})
}
cursor := opts.Cursor
if !cursor.IsEmpty() {
query = query.Where(sq.Or{
sq.Gt{"Id": cursor.ID},
})
}
limit := uint64(opts.Limit)
if limit < 1 {
limit = DefaultPerPage
} else if limit > MaxPerPage {
limit = MaxPerPage
}
query = query.Limit(limit)
err := s.GetReplica().SelectBuilder(&p, query)
if err != nil {
return nil, cursor, errors.Wrapf(err, "failed to find policies with opts={\"parentID\"=%q, \"resourceType\"=%q", opts.ParentID, opts.Type)
}
policies := make([]*model.AccessControlPolicy, len(p))
for i := range p {
policies[i], err = p[i].toModel()
if err != nil {
return nil, cursor, errors.Wrapf(err, "failed to parse policy with id=%s", p[i].ID)
}
}
if len(policies) != 0 {
cursor.ID = policies[len(policies)-1].ID
}
return policies, cursor, nil
}
func (s *SqlAccessControlPolicyStore) SearchPolicies(rctx request.CTX, opts model.AccessControlPolicySearch) ([]*model.AccessControlPolicy, int64, error) {
type wrapper struct {
storeAccessControlPolicy
ChildIDs json.RawMessage
ChannelCount int
TeamCount int
}
p := []wrapper{}
var query sq.SelectBuilder
if opts.IncludeChildren && opts.ParentID == "" {
columns := accessControlPolicySliceColumns("p")
childIDs := `COALESCE((SELECT JSON_AGG(c.ID)
FROM AccessControlPolicies c
WHERE c.Type != 'parent'
AND c.Data->'imports' @> JSONB_BUILD_ARRAY(p.ID)), '[]'::json) AS ChildIDs`
channelCount := `COALESCE((SELECT COUNT(*)
FROM AccessControlPolicies c
WHERE c.Type = 'channel'
AND c.Data->'imports' @> JSONB_BUILD_ARRAY(p.ID)), 0) AS ChannelCount`
teamCount := `COALESCE((SELECT COUNT(*)
FROM AccessControlPolicies c
WHERE c.Type = 'team'
AND c.Data->'imports' @> JSONB_BUILD_ARRAY(p.ID)), 0) AS TeamCount`
columns = append(columns, childIDs, channelCount, teamCount)
query = s.getQueryBuilder().Select(columns...).From("AccessControlPolicies p")
} else {
query = s.selectQueryBuilder
}
count := s.getQueryBuilder().Select("COUNT(*)").From("AccessControlPolicies")
if opts.Term != "" {
safeTerm := sanitizeSearchTerm(opts.Term, "*")
condition := sq.Expr("LOWER(Name) LIKE LOWER(?) ESCAPE '*'", fmt.Sprintf("%%%s%%", safeTerm))
query = query.Where(condition)
count = count.Where(condition)
}
if opts.Type != "" {
condition := sq.Eq{"Type": opts.Type}
query = query.Where(condition)
count = count.Where(condition)
}
if opts.ParentID != "" {
condition := sq.Expr("Data->'imports' @> ?", fmt.Sprintf("%q", opts.ParentID))
query = query.Where(condition)
count = count.Where(condition)
}
if len(opts.Actions) > 0 {
or := sq.Or{}
for _, action := range opts.Actions {
or = append(or, sq.Expr(
"EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(Data->'rules') = 'array' THEN Data->'rules' ELSE '[]'::jsonb END) AS rule WHERE rule->'actions' @> ?::jsonb)",
fmt.Sprintf("%q", action),
))
}
query = query.Where(or)
count = count.Where(or)
}
if opts.Active {
query = query.Where(sq.Eq{"Active": true})
count = count.Where(sq.Eq{"Active": true})
}
if len(opts.IDs) > 0 {
condition := sq.Eq{"Id": opts.IDs}
query = query.Where(condition)
count = count.Where(condition)
}
// Scope/ScopeID filter policies by their persisted scope in the Data JSONB.
// Both must be provided together — a partial match is not meaningful.
if opts.Scope != "" && opts.ScopeID != "" {
scopeCondition := sq.And{
sq.Expr("Data->>'scope' = ?", opts.Scope),
sq.Expr("Data->>'scope_id' = ?", opts.ScopeID),
}
query = query.Where(scopeCondition)
count = count.Where(scopeCondition)
}
// TeamID filters to parent policies whose assigned channels ALL belong to the
// specified team. Policies with channels spanning multiple teams will NOT match
// even if one channel is in the requested team (COUNT(DISTINCT TeamId) = 1
// enforces single-team scope). This only applies to parent policies; the type
// is forced to 'parent' when TeamID is set.
if opts.TeamID != "" {
if opts.Type == model.AccessControlPolicyTypeChannel {
condition := sq.Expr(`Id IN (SELECT Id FROM Channels WHERE TeamId = ?)`, opts.TeamID)
query = query.Where(condition)
count = count.Where(condition)
} else {
if opts.Type == "" {
parentCondition := sq.Eq{"Type": model.AccessControlPolicyTypeParent}
query = query.Where(parentCondition)
count = count.Where(parentCondition)
}
condition := sq.Expr(`Id IN (
SELECT parent_id FROM (
SELECT ch.TeamId, jsonb_array_elements_text(COALESCE(NULLIF(cp.Data -> 'imports', 'null'::jsonb), '[]'::jsonb)) AS parent_id
FROM AccessControlPolicies cp
JOIN Channels ch ON ch.Id = cp.Id
WHERE cp.Type = 'channel'
) team_children
GROUP BY parent_id
HAVING COUNT(DISTINCT TeamId) = 1 AND MIN(TeamId) = ?)`, opts.TeamID)
query = query.Where(condition)
count = count.Where(condition)
}
}
cursor := opts.Cursor
if !cursor.IsEmpty() {
query = query.Where(sq.Gt{"Id": cursor.ID})
}
limit := uint64(opts.Limit)
if limit < 1 {
limit = DefaultPerPage
} else if limit > MaxPerPage {
limit = MaxPerPage
}
query = query.Limit(limit)
query = query.OrderBy("Id ASC")
err := s.GetReplica().SelectBuilder(&p, query)
if err != nil {
return nil, 0, errors.Wrapf(err, "failed to find policies with opts={\"name\"=%q, \"resourceType\"=%q", opts.Term, opts.Type)
}
policies := make([]*model.AccessControlPolicy, len(p))
for i := range p {
m, err2 := p[i].toModel()
if err2 != nil {
return nil, 0, errors.Wrapf(err2, "failed to parse policy with id=%s", p[i].ID)
}
// Props field is not guaranteed to be persisted correctly, and it shouldn't be.
// This is a field that we want to include metadata, some values may be stored but
// not all of them. For example for the childs, we don't want to update it whenever a
// child policy changes.
if opts.IncludeChildren && opts.ParentID == "" {
if m.Props == nil {
m.Props = make(map[string]any)
}
// Unmarshal the JSON array into a slice of strings
var childIDs []string
if err = json.Unmarshal(p[i].ChildIDs, &childIDs); err != nil {
return nil, 0, errors.Wrapf(err, "failed to unmarshal child IDs for policy with id=%s", p[i].ID)
}
m.Props["child_ids"] = childIDs
m.Props["channel_count"] = p[i].ChannelCount
m.Props["team_count"] = p[i].TeamCount
}
policies[i] = m
}
var total int64
err = s.GetReplica().GetBuilder(&total, count)
if err != nil {
return nil, 0, errors.Wrapf(err, "failed to count policies with opts={\"name\"=%q, \"resourceType\"=%q", opts.Term, opts.Type)
}
if len(policies) != 0 {
cursor.ID = policies[len(policies)-1].ID
}
return policies, total, nil
}
// actionsAggregationSQL is the per-policy action-union expression embedded in
// both GetActionsForPolicy and GetActionsForPolicies. It produces a JSONB
// object whose keys are the distinct action strings declared by the policy's
// own rules and the rules of any policies it imports. The expression is
// table-qualified with `p` so it can be reused as a correlated subquery
// against either a single row or a set of rows.
const actionsAggregationSQL = `COALESCE(
(
SELECT jsonb_object_agg(action, true)
FROM (
SELECT DISTINCT jsonb_array_elements_text(rule->'actions') AS action
FROM jsonb_array_elements(
CASE WHEN jsonb_typeof(p.Data->'rules') = 'array'
THEN p.Data->'rules' ELSE '[]'::jsonb END
) AS rule
UNION
SELECT DISTINCT jsonb_array_elements_text(rule->'actions') AS action
FROM AccessControlPolicies parent
JOIN jsonb_array_elements_text(
CASE WHEN jsonb_typeof(p.Data->'imports') = 'array'
THEN p.Data->'imports' ELSE '[]'::jsonb END
) AS imp(id) ON parent.ID = imp.id
CROSS JOIN LATERAL jsonb_array_elements(
CASE WHEN jsonb_typeof(parent.Data->'rules') = 'array'
THEN parent.Data->'rules' ELSE '[]'::jsonb END
) AS rule
) AS unioned_actions
),
'{}'::jsonb
)`
// GetActionsForPolicy returns the union of action keys declared by the policy's
// own rules and the rules of any policies it imports. The result is always a
// non-nil map (empty when the policy exists but declares no rules). Returns
// store.ErrNotFound when no AccessControlPolicies row exists for policyID.
//
// The query is bounded by a single row's expansion plus its (small) imports
// fan-out. Channels with no attached policy never reach this method because
// the App-layer hydrator short-circuits on PolicyEnforced=false.
func (s *SqlAccessControlPolicyStore) GetActionsForPolicy(_ request.CTX, policyID string) (map[string]bool, error) {
if !model.IsValidId(policyID) {
return nil, store.NewErrInvalidInput("AccessControlPolicy", "policyID", policyID)
}
var raw []byte
query := fmt.Sprintf(`SELECT %s AS actions FROM AccessControlPolicies p WHERE p.ID = $1`, actionsAggregationSQL)
err := s.GetReplica().Get(&raw, query, policyID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, store.NewErrNotFound("AccessControlPolicy", policyID)
}
return nil, errors.Wrapf(err, "failed to load policy actions for id=%s", policyID)
}
actions := map[string]bool{}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &actions); err != nil {
return nil, errors.Wrapf(err, "failed to decode policy actions for id=%s", policyID)
}
}
return actions, nil
}
// GetActionsForPolicies returns the per-policy action union for each ID in
// policyIDs. Missing IDs are absent from the result map (callers can detect
// "policy not found" via `_, ok := result[id]`). An empty input slice returns
// an empty map and fires no SQL. Used by batched hydration on channel-list
// reads to avoid an N+1 against AccessControlPolicies.
func (s *SqlAccessControlPolicyStore) GetActionsForPolicies(_ request.CTX, policyIDs []string) (map[string]map[string]bool, error) {
if len(policyIDs) == 0 {
return map[string]map[string]bool{}, nil
}
for _, id := range policyIDs {
if !model.IsValidId(id) {
return nil, store.NewErrInvalidInput("AccessControlPolicy", "policyID", id)
}
}
query, args, err := s.getQueryBuilder().
Select("p.ID", fmt.Sprintf("%s AS actions", actionsAggregationSQL)).
From("AccessControlPolicies p").
Where(sq.Eq{"p.ID": policyIDs}).
ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to build batched policy actions query")
}
rows, err := s.GetReplica().Query(query, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to load policy actions batch")
}
defer rows.Close()
result := make(map[string]map[string]bool, len(policyIDs))
for rows.Next() {
var id string
var raw []byte
if err := rows.Scan(&id, &raw); err != nil {
return nil, errors.Wrap(err, "failed to scan policy actions row")
}
actions := map[string]bool{}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &actions); err != nil {
return nil, errors.Wrapf(err, "failed to decode policy actions for id=%s", id)
}
}
result[id] = actions
}
if err := rows.Err(); err != nil {
return nil, errors.Wrap(err, "policy actions row iteration failed")
}
return result, nil
}
// GetPoliciesByFieldID finds all policies whose CEL rule expressions reference the
// given property field ID. It performs a text search on the serialized JSONB Data
// column for the pattern "id_<fieldID>", which is how property fields are referenced
// in CEL expressions (e.g., user.attributes.id_<fieldID>).
//
// This is a sequential scan over the policies table. At the expected scale (hundreds
// to low thousands of admin-configured policies) this is well under 1ms.
// If the table grows beyond ~10K rows or this query lands on a hot path, a GIN
// trigram index on (Data::text) can accelerate it with no query changes.
func (s *SqlAccessControlPolicyStore) GetPoliciesByFieldID(_ request.CTX, fieldID string) ([]*model.AccessControlPolicy, error) {
if !model.IsValidId(fieldID) {
return nil, store.NewErrInvalidInput("AccessControlPolicy", "fieldID", fieldID)
}
p := []storeAccessControlPolicy{}
query := s.selectQueryBuilder.Where(sq.Expr("Data::text LIKE ?", fmt.Sprintf("%%id\\_%s%%", fieldID)))
err := s.GetReplica().SelectBuilder(&p, query)
if err != nil {
return nil, errors.Wrapf(err, "failed to find policies referencing field id=%s", fieldID)
}
policies := make([]*model.AccessControlPolicy, len(p))
for i := range p {
policies[i], err = p[i].toModel()
if err != nil {
return nil, errors.Wrapf(err, "failed to parse policy with id=%s", p[i].ID)
}
}
return policies, nil
}
// GetEtagEpoch covers the system-scoped permission policies plus the given channel's own policy
// row (a channel policy's ID equals the channel ID); an empty channelID covers only the former.
//
// MAX(CreateAt), because the table has no UpdateAt and saves are delete-and-reinsert. The count is
// folded in to catch deletes: dropping a policy that isn't the newest leaves the max untouched, so
// the ETag would still match and clients would keep a differently-sanitized cached copy.
//
// Matching the channel by primary key instead of scanning every policy's rules JSON costs a benign
// false positive: a membership-only change to this channel's policy also advances the epoch.
func (s *SqlAccessControlPolicyStore) GetEtagEpoch(rctx request.CTX, channelID string) (string, error) {
query, args, err := s.getQueryBuilder().
Select("COALESCE(MAX(CreateAt), 0) AS MaxCreateAt", "COUNT(*) AS Total").
From("AccessControlPolicies").
Where(sq.Or{
sq.Eq{"Type": model.AccessControlPolicyTypePermission},
sq.Eq{"Id": channelID},
}).
ToSql()
if err != nil {
return "", errors.Wrap(err, "GetEtagEpoch: failed to build query")
}
var epoch struct {
MaxCreateAt int64
Total int64
}
// Master, not replica — see the note on SqlAttributesStore.GetUserPropertyValuesEpoch.
if err := s.GetMaster().Get(&epoch, query, args...); err != nil {
return "", errors.Wrap(err, "GetEtagEpoch: query failed")
}
return fmt.Sprintf("%d-%d", epoch.MaxCreateAt, epoch.Total), nil
}
// No-op at the SQL layer; the render-ETag epoch is cached in and invalidated by the local cache layer.
func (s *SqlAccessControlPolicyStore) InvalidateEtagForChannel(channelID string) {}
func (s *SqlAccessControlPolicyStore) ClearEtagCache() {}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestAccessControlPolicyStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestAccessControlPolicyStore)
}

View File

@@ -0,0 +1,51 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"bytes"
"database/sql/driver"
"strconv"
)
type jsonArray []string
func (a jsonArray) Value() (driver.Value, error) {
var out bytes.Buffer
if err := out.WriteByte('['); err != nil {
return nil, err
}
for i, item := range a {
if _, err := out.WriteString(strconv.Quote(item)); err != nil {
return nil, err
}
// Skip the last element.
if i < len(a)-1 {
if err := out.WriteByte(','); err != nil {
return nil, err
}
}
}
err := out.WriteByte(']')
return out.Bytes(), err
}
type jsonStringVal string
func (str jsonStringVal) Value() (driver.Value, error) {
return strconv.Quote(string(str)), nil
}
type jsonKeyPath string
func (str jsonKeyPath) Value() (driver.Value, error) {
return "{" + string(str) + "}", nil
}
type JSONSerializable interface {
ToJSON() string
}

View File

@@ -0,0 +1,21 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestJSONArray(t *testing.T) {
input := []string{"a", "b"}
out, err := jsonArray(input).Value()
require.NoError(t, err)
outBuf, ok := out.([]byte)
require.True(t, ok)
assert.Equal(t, []byte(`["a","b"]`), outBuf)
}

View File

@@ -0,0 +1,380 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"encoding/json"
"fmt"
"strings"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/einterfaces"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
)
type SqlAttributesStore struct {
*SqlStore
metrics einterfaces.MetricsInterface
selectQueryBuilder sq.SelectBuilder
}
// attributeViewsByObjectType is the single source of truth for the
// object-type-to-materialized-view mapping: both the per-type lookup and the
// refresh sweep read it, so adding an object type cannot leave a new view
// perpetually stale.
var attributeViewsByObjectType = map[string]string{
model.PropertyFieldObjectTypeChannel: "ChannelAttributeView",
model.PropertyFieldObjectTypeUser: "UserAttributeView",
}
// attributeViewFor maps a property-field object type to its materialized view.
// An unrecognized type is an error rather than a fallback to the user view: a
// silent coercion would surface a future miswire as a wrong-but-successful user
// lookup.
func attributeViewFor(objectType string) (string, error) {
view, ok := attributeViewsByObjectType[objectType]
if !ok {
return "", errors.Errorf("unknown object type %q for attribute view", objectType)
}
return view, nil
}
func attributesSliceColumns(prefix ...string) []string {
var p string
if len(prefix) == 1 {
p = prefix[0] + "."
} else if len(prefix) > 1 {
panic("cannot accept multiple prefixes")
}
return []string{
p + "TargetID as ID",
p + "TargetType as Type",
p + "Attributes",
}
}
// qualify prefixes each column with the given table name. The members-to-remove
// queries join Users, whose columns (Roles, DeleteAt, CreateAt) collide with the
// member columns, so the member SELECT must be table-qualified to stay unambiguous.
func qualify(table string, columns []string) []string {
qualified := make([]string, len(columns))
for i, c := range columns {
qualified[i] = table + "." + c
}
return qualified
}
func newSqlAttributesStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) store.AttributesStore {
s := &SqlAttributesStore{
SqlStore: sqlStore,
metrics: metrics,
}
// The From clause is chosen per call by object type (GetSubject), so the
// shared builder carries only the Select.
s.selectQueryBuilder = s.getQueryBuilder().Select(attributesSliceColumns()...)
return s
}
func (s *SqlAttributesStore) RefreshAttributes() error {
// Both per-object-type views (migration 000216) refresh on one cadence, so a
// caller does not have to know which object types it is about to read. They
// refresh in separate statements, though, so this is not an atomic snapshot:
// a read straddling a refresh can see the two views a generation apart. Every
// consumer tolerates that — values are eventually consistent and the
// membership sync re-runs on cadence and converges. Refreshing only the view
// whose attributes actually changed is a scale follow-up. Refresh order is
// unspecified (map iteration) and does not matter: each REFRESH is its own
// independent statement.
for _, view := range attributeViewsByObjectType {
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW " + view); err != nil {
return errors.Wrapf(err, "error refreshing materialized view %s", view)
}
}
return nil
}
func (s *SqlAttributesStore) GetSubject(rctx request.CTX, ID, groupID, objectType string) (*model.Subject, error) {
view, err := attributeViewFor(objectType)
if err != nil {
return nil, err
}
query := s.selectQueryBuilder.From(view).Where(sq.And{sq.Eq{"TargetID": ID}, sq.Eq{"GroupID": groupID}})
q, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to build query for subject")
}
row := s.GetReplica().QueryRowContext(rctx.Context(), q, args...)
var subject model.Subject
var properties []byte
if err := row.Scan(&subject.ID, &subject.Type, &properties); err != nil {
if err == sql.ErrNoRows {
return nil, store.NewErrNotFound("Attributes", ID)
}
return nil, errors.Wrap(err, "failed to scan subject row")
}
if err := json.Unmarshal(properties, &subject.Attributes); err != nil {
return nil, errors.Wrap(err, "failed to unmarshal attributes")
}
return &subject, nil
}
func (s *SqlAttributesStore) SearchUsers(rctx request.CTX, opts model.SubjectSearchOptions) ([]*model.User, int64, error) {
query := s.getQueryBuilder().
Select(getUsersColumns()...).From("Users").LeftJoin("UserAttributeView ON Users.Id = UserAttributeView.TargetID").
OrderBy("Users.Id ASC")
count := s.getQueryBuilder().Select("COUNT(*)").From("Users").LeftJoin("UserAttributeView ON Users.Id = UserAttributeView.TargetID")
if opts.Query != "" {
// Wrap the CEL-derived expression in parentheses so that any top-level
// OR (e.g. produced by "has any of [a, b]") does not bind across the
// AND-joined WHERE clauses appended below (SubjectID, DeleteAt,
// TeamID, ExcludeChannelMembers, Cursor, Term). Without these
// parens, "A OR B AND Users.Id = $X" would be parsed as
// "A OR (B AND Users.Id = $X)" because AND binds tighter than OR.
wrapped := "(" + opts.Query + ")"
query = query.Where(sq.Expr(wrapped, opts.Args...))
count = count.Where(sq.Expr(wrapped, opts.Args...))
}
argCount := len(opts.Args)
if opts.Limit > 0 {
query = query.Limit(uint64(opts.Limit))
} else if opts.Limit > MaxPerPage {
query = query.Limit(uint64(MaxPerPage))
}
if !opts.AllowInactive {
query = query.Where("Users.DeleteAt = 0")
count = count.Where("Users.DeleteAt = 0")
}
if opts.TeamID != "" {
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("Users.Id IN (SELECT UserId FROM TeamMembers WHERE TeamId = $%d AND DeleteAt = 0)", argCount), opts.TeamID))
count = count.Where(sq.Expr(fmt.Sprintf("Users.Id IN (SELECT UserId FROM TeamMembers WHERE TeamId = $%d AND DeleteAt = 0)", argCount), opts.TeamID))
}
if opts.ExcludeChannelMembers != "" {
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("NOT EXISTS (SELECT 1 FROM ChannelMembers WHERE ChannelMembers.UserId = Users.Id AND ChannelMembers.ChannelId = $%d)", argCount), opts.ExcludeChannelMembers))
}
if opts.SubjectID != "" {
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("Users.Id = $%d", argCount), opts.SubjectID))
count = count.Where(sq.Expr(fmt.Sprintf("Users.Id = $%d", argCount), opts.SubjectID))
}
if opts.Cursor.TargetID != "" {
argCount++
// Paginate on Users.Id (the ORDER BY column), not UserAttributeView.TargetID.
// The cursor value is a user id, and TargetID comes from a LEFT JOIN so it
// is NULL for users with no custom-attribute row — comparing against it
// silently drops those users (e.g. matches of a native-only policy).
query = query.Where(sq.Expr(fmt.Sprintf("Users.Id > $%d", argCount), opts.Cursor.TargetID))
}
termFields := UserSearchTypeNames
if opts.ExcludeFullNames {
termFields = UserSearchTypeNamesNoFullName
}
searchFields := make([]string, 0, len(termFields))
for _, field := range termFields {
searchFields = append(searchFields, strings.Join([]string{"Users", field}, "."))
}
if term := opts.Term; strings.TrimSpace(term) != "" {
_, query = generateSearchQueryForExpression(query, strings.Fields(term), searchFields, argCount)
_, count = generateSearchQueryForExpression(count, strings.Fields(term), searchFields, argCount)
}
q, args, err := query.ToSql()
if err != nil {
return nil, 0, errors.Wrap(err, "failed to build query for subjects")
}
users := []*model.User{}
if err = s.GetReplica().Select(&users, q, args...); err != nil {
return nil, 0, errors.Wrapf(err, "failed to find Users with term=%s and searchType=%v", opts.Term, searchFields)
}
for _, u := range users {
u.Sanitize(map[string]bool{})
}
var total int64
if !opts.IgnoreCount {
err = s.GetReplica().GetBuilder(&total, count)
if err != nil {
return nil, 0, errors.Wrapf(err, "failed to count Users with term=%s and searchType=%v", opts.Term, searchFields)
}
}
return users, total, nil
}
func (s *SqlAttributesStore) GetChannelMembersToRemove(rctx request.CTX, channelID string, opts model.SubjectSearchOptions) ([]*model.ChannelMember, error) {
query := s.getQueryBuilder().
Select(qualify("ChannelMembers", channelMemberSliceColumns())...).From("ChannelMembers").
// Join Users so native-attribute expressions (e.g. Users.EmailVerified)
// resolve here, mirroring SearchUsers on the add path.
LeftJoin("Users ON Users.Id = ChannelMembers.UserId").
LeftJoin("UserAttributeView ON ChannelMembers.UserId = UserAttributeView.TargetID").
OrderBy("ChannelMembers.UserId ASC")
if opts.Query != "" {
// A member is removed when they do NOT satisfy the policy; a NULL result
// (e.g. a missing custom attribute) counts as "does not satisfy" via
// COALESCE. We must not additionally remove members just because they
// lack a UserAttributeView row — a native-only policy matches against the
// Users table, so a user with zero custom attributes can still satisfy it.
query = query.Where(sq.Expr(fmt.Sprintf("NOT COALESCE((%s), FALSE)", opts.Query), opts.Args...))
}
argCount := len(opts.Args)
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("ChannelMembers.ChannelId = $%d", argCount), channelID))
if opts.Limit > 0 {
query = query.Limit(uint64(opts.Limit))
} else if opts.Limit > MaxPerPage {
query = query.Limit(uint64(MaxPerPage))
}
if opts.Cursor.TargetID != "" {
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("ChannelMembers.UserId > $%d", argCount), opts.Cursor.TargetID))
}
q, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to build query for subjects")
}
members := []*model.ChannelMember{}
if err := s.GetReplica().Select(&members, q, args...); err != nil {
return nil, errors.Wrapf(err, "failed to find channel members with for channel id=%s", channelID)
}
return members, nil
}
// Count folded in alongside the max to catch soft deletes: dropping a value that isn't the most
// recently updated leaves MAX(UpdateAt) unchanged, and the stale ETag would keep matching.
func (s *SqlAttributesStore) GetUserPropertyValuesEpoch(rctx request.CTX, userID string) (string, error) {
query, args, err := s.getQueryBuilder().
Select("COALESCE(MAX(UpdateAt), 0) AS MaxUpdateAt", "COUNT(*) AS Total").
From("PropertyValues").
Where(sq.Eq{"TargetID": userID}).
Where("DeleteAt = 0").
ToSql()
if err != nil {
return "", errors.Wrap(err, "GetUserPropertyValuesEpoch: failed to build query")
}
var epoch struct {
MaxUpdateAt int64
Total int64
}
// Master, not replica: the caller invalidates this key right after the write commits, so a
// lagging replica would re-cache the pre-write epoch and the client would keep 304ing onto
// content sanitized under the old attributes until the entry expires.
if err := s.GetMaster().Get(&epoch, query, args...); err != nil {
return "", errors.Wrap(err, "GetUserPropertyValuesEpoch: query failed")
}
return fmt.Sprintf("%d-%d", epoch.MaxUpdateAt, epoch.Total), nil
}
// No-op at the SQL layer; the per-user epoch is cached in and invalidated by the local cache layer.
func (s *SqlAttributesStore) InvalidateUserPropertyValuesEpoch(userID string) {}
func (s *SqlAttributesStore) ClearUserPropertyValuesEpochCache() {}
func (s *SqlAttributesStore) GetTeamMembersToRemove(rctx request.CTX, teamID string, opts model.SubjectSearchOptions) ([]*model.TeamMember, error) {
query := s.getQueryBuilder().
Select(qualify("TeamMembers", teamMemberSliceColumns())...).From("TeamMembers").
// Join Users so native-attribute expressions (e.g. Users.EmailVerified)
// resolve here, mirroring SearchUsers on the add path.
LeftJoin("Users ON Users.Id = TeamMembers.UserId").
LeftJoin("UserAttributeView ON TeamMembers.UserId = UserAttributeView.TargetID").
Where("TeamMembers.DeleteAt = 0").
OrderBy("TeamMembers.UserId ASC")
if opts.Query != "" {
// A member is removed when they do NOT satisfy the policy; a NULL result
// (e.g. a missing custom attribute) counts as "does not satisfy" via
// COALESCE. We must not additionally remove members just because they
// lack a UserAttributeView row — a native-only policy matches against the
// Users table, so a user with zero custom attributes can still satisfy it.
query = query.Where(sq.Expr(fmt.Sprintf("NOT COALESCE((%s), FALSE)", opts.Query), opts.Args...))
}
argCount := len(opts.Args)
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("TeamMembers.TeamId = $%d", argCount), teamID))
// An explicit limit is capped at MaxPerPage; an unset limit (0) intentionally
// returns every removal candidate for the team. The membership-sync caller
// consumes the full set in one pass, so capping an unset limit here would
// permanently leave members beyond the cap in a team they no longer qualify
// for. The result is naturally bounded by the team's membership.
if opts.Limit > 0 {
limit := min(opts.Limit, MaxPerPage)
query = query.Limit(uint64(limit))
}
if opts.Cursor.TargetID != "" {
argCount++
query = query.Where(sq.Expr(fmt.Sprintf("TeamMembers.UserId > $%d", argCount), opts.Cursor.TargetID))
}
q, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to build query for subjects")
}
members := []*model.TeamMember{}
if err := s.GetReplica().Select(&members, q, args...); err != nil {
return nil, errors.Wrapf(err, "failed to find team members for team id=%s", teamID)
}
return members, nil
}
func generateSearchQueryForExpression(query sq.SelectBuilder, terms []string, fields []string, prevArgs int) (int, sq.SelectBuilder) {
for _, term := range terms {
searchFields := []string{}
termArgs := []any{}
for _, field := range fields {
prevArgs++
searchFields = append(searchFields, fmt.Sprintf("lower(%s) LIKE lower($%d) escape '*' ", field, prevArgs))
termArgs = append(termArgs, fmt.Sprintf("%%%s%%", strings.TrimLeft(term, "@")))
}
prevArgs++
searchFields = append(searchFields, fmt.Sprintf("lower(%s) LIKE lower($%d) escape '*' ", "Id", prevArgs))
termArgs = append(termArgs, strings.TrimLeft(term, "@"))
query = query.Where(fmt.Sprintf("(%s)", strings.Join(searchFields, " OR ")), termArgs...)
}
return prevArgs, query
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestAttributesStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestAttributesStore)
}

View File

@@ -0,0 +1,84 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SqlAuditStore struct {
*SqlStore
auditQuery sq.SelectBuilder
}
func newSqlAuditStore(sqlStore *SqlStore) store.AuditStore {
s := &SqlAuditStore{
SqlStore: sqlStore,
}
s.auditQuery = s.getQueryBuilder().
Select(
"Id",
"CreateAt",
"UserId",
"Action",
"ExtraInfo",
"IpAddress",
"SessionId",
).
From("Audits")
return s
}
func (s SqlAuditStore) Save(audit *model.Audit) error {
audit.Id = model.NewId()
audit.CreateAt = model.GetMillis()
if _, err := s.GetMaster().NamedExec(`INSERT INTO Audits
(Id, CreateAt, UserId, Action, ExtraInfo, IpAddress, SessionId)
VALUES
(:Id, :CreateAt, :UserId, :Action, :ExtraInfo, :IpAddress, :SessionId)`, audit); err != nil {
return errors.Wrapf(err, "failed to save Audit with userId=%s and action=%s", audit.UserId, audit.Action)
}
return nil
}
func (s SqlAuditStore) Get(userId string, offset int, limit int) (model.Audits, error) {
if limit > 1000 {
return nil, store.NewErrOutOfBounds(limit)
}
query := s.auditQuery.
OrderBy("CreateAt DESC").
Limit(uint64(limit)).
Offset(uint64(offset))
if userId != "" {
query = query.Where(sq.Eq{"UserId": userId})
}
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "audits_tosql")
}
var audits model.Audits
if err := s.GetReplica().Select(&audits, queryString, args...); err != nil {
return nil, errors.Wrapf(err, "failed to get Audit list for userId=%s", userId)
}
return audits, nil
}
func (s SqlAuditStore) PermanentDeleteByUser(userId string) error {
if _, err := s.GetMaster().Exec("DELETE FROM Audits WHERE UserId = ?", userId); err != nil {
return errors.Wrapf(err, "failed to delete Audit with userId=%s", userId)
}
return nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestAuditStore(t *testing.T) {
StoreTest(t, storetest.TestAuditStore)
}

View File

@@ -0,0 +1,497 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"encoding/json"
"maps"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SqlAutoTranslationStore struct {
*SqlStore
}
type TranslationMeta json.RawMessage
type Translation struct {
ObjectType string
ObjectID string
ChannelID *string
DstLang string
ProviderID string
NormHash string
Text string
Confidence *float64
Meta TranslationMeta
State string
UpdateAt int64
}
func (m *TranslationMeta) ToMap() (map[string]any, error) {
if m == nil {
return nil, nil
}
var result map[string]any
if err := json.Unmarshal(*m, &result); err != nil {
return nil, err
}
return result, nil
}
func newSqlAutoTranslationStore(sqlStore *SqlStore) store.AutoTranslationStore {
return &SqlAutoTranslationStore{
SqlStore: sqlStore,
}
}
func (s *SqlAutoTranslationStore) IsUserEnabled(userID, channelID string) (bool, error) {
query := s.getQueryBuilder().
Select("cm.AutoTranslationDisabled").
From("ChannelMembers cm").
Join("Channels c ON cm.ChannelId = c.Id").
Where(sq.Eq{"cm.UserId": userID, "cm.ChannelId": channelID}).
Where("cm.AutoTranslationDisabled != true").
Where("c.AutoTranslation = true")
var disabled bool
if err := s.GetReplica().GetBuilder(&disabled, query); err != nil {
if err == sql.ErrNoRows {
return false, nil
}
return false, errors.Wrapf(err, "failed to get user enabled status for user_id=%s, channel_id=%s", userID, channelID)
}
return !disabled, nil
}
func (s *SqlAutoTranslationStore) GetUserLanguage(userID, channelID string) (string, error) {
query := s.getQueryBuilder().
Select("u.Locale").
From("Users u").
Join("ChannelMembers cm ON u.Id = cm.UserId").
Join("Channels c ON cm.ChannelId = c.Id").
Where(sq.Eq{"u.Id": userID, "c.Id": channelID}).
Where("c.AutoTranslation = true").
Where("cm.AutoTranslationDisabled != true")
var locale string
if err := s.GetReplica().GetBuilder(&locale, query); err != nil {
if err == sql.ErrNoRows {
return "", nil
}
return "", errors.Wrapf(err, "failed to get user language for user_id=%s, channel_id=%s", userID, channelID)
}
return locale, nil
}
func (s *SqlAutoTranslationStore) GetActiveDestinationLanguages(channelID, excludeUserID string, filterUserIDs []string) ([]string, error) {
query := s.getQueryBuilder().
Select("DISTINCT u.Locale").
From("ChannelMembers cm").
Join("Channels c ON c.Id = cm.ChannelId").
Join("Users u ON u.Id = cm.UserId").
Where(sq.Eq{"cm.ChannelId": channelID}).
Where("c.AutoTranslation = true").
Where("cm.AutoTranslationDisabled != true")
// Filter to specific user IDs if provided (e.g., users with active WebSocket connections)
// When filterUserIDs is non-nil and non-empty, squirrel converts it to an IN clause
// Example: WHERE cm.userid IN ('id1', 'id2', 'id3')
if len(filterUserIDs) > 0 {
query = query.Where(sq.Eq{"cm.userid": filterUserIDs})
}
// Exclude specific user if provided (e.g., the message author)
// This works together with the filter above via SQL AND logic
// Example: WHERE cm.userid IN (...) AND cm.userid != 'excludedId'
if excludeUserID != "" {
query = query.Where(sq.NotEq{"cm.userid": excludeUserID})
}
var languages []string
if err := s.GetReplica().SelectBuilder(&languages, query); err != nil {
return nil, errors.Wrapf(err, "failed to get active destination languages for channel_id=%s", channelID)
}
return languages, nil
}
func (s *SqlAutoTranslationStore) Get(objectType, objectID, dstLang string) (*model.Translation, error) {
query := s.getQueryBuilder().
Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt").
From("Translations").
Where(sq.Eq{"ObjectType": objectType, "ObjectId": objectID, "DstLang": dstLang})
var translation Translation
if err := s.GetReplica().GetBuilder(&translation, query); err != nil {
if err == sql.ErrNoRows {
return nil, store.NewErrNotFound("Translation", objectID)
}
return nil, errors.Wrapf(err, "failed to get translation for object_id=%s, dst_lang=%s", objectID, dstLang)
}
meta, err := translation.Meta.ToMap()
var translationTypeStr string
if err != nil {
return nil, errors.Wrapf(err, "failed to parse translation meta for object_id=%s", objectID)
}
if v, ok := meta["type"]; ok {
if s, ok := v.(string); ok {
translationTypeStr = s
}
}
result := &model.Translation{
ObjectID: translation.ObjectID,
ObjectType: objectType,
Lang: translation.DstLang,
Type: model.TranslationType(translationTypeStr),
Confidence: translation.Confidence,
State: model.TranslationState(translation.State),
NormHash: translation.NormHash,
Meta: meta,
}
if result.Type == model.TranslationTypeObject {
result.ObjectJSON = json.RawMessage(translation.Text)
} else {
result.Text = translation.Text
}
return result, nil
}
func (s *SqlAutoTranslationStore) GetBatch(objectType string, objectIDs []string, dstLang string) (map[string]*model.Translation, error) {
if len(objectIDs) == 0 {
return make(map[string]*model.Translation), nil
}
query := s.getQueryBuilder().
Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt").
From("Translations").
Where(sq.Eq{"ObjectType": objectType, "ObjectId": objectIDs, "DstLang": dstLang})
var translations []Translation
if err := s.GetReplica().SelectBuilder(&translations, query); err != nil {
return nil, errors.Wrapf(err, "failed to get batch translations for dst_lang=%s", dstLang)
}
result := make(map[string]*model.Translation, len(translations))
for _, t := range translations {
var translationTypeStr string
meta, err := t.Meta.ToMap()
if err != nil {
// Log error but continue with other translations
continue
}
if v, ok := meta["type"]; ok {
if s, ok := v.(string); ok {
translationTypeStr = s
}
}
modelT := &model.Translation{
ObjectID: t.ObjectID,
ObjectType: objectType,
Lang: t.DstLang,
Type: model.TranslationType(translationTypeStr),
Confidence: t.Confidence,
State: model.TranslationState(t.State),
NormHash: t.NormHash,
Meta: meta,
UpdateAt: t.UpdateAt,
}
if modelT.Type == model.TranslationTypeObject {
modelT.ObjectJSON = json.RawMessage(t.Text)
} else {
modelT.Text = t.Text
}
result[t.ObjectID] = modelT
}
return result, nil
}
func (s *SqlAutoTranslationStore) GetAllForObject(objectType, objectID string) ([]*model.Translation, error) {
query := s.getQueryBuilder().
Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt").
From("Translations").
Where(sq.Eq{"ObjectType": objectType, "ObjectId": objectID})
var translations []Translation
// Use GetMaster to avoid replica lag issues when workers fetch queued items
if err := s.GetMaster().SelectBuilder(&translations, query); err != nil {
return nil, errors.Wrapf(err, "failed to get all translations for object_id=%s", objectID)
}
result := make([]*model.Translation, 0, len(translations))
for _, t := range translations {
var translationTypeStr string
meta, err := t.Meta.ToMap()
if err != nil {
// Log error but continue with other translations
continue
}
if v, ok := meta["type"]; ok {
if s, ok := v.(string); ok {
translationTypeStr = s
}
}
modelT := &model.Translation{
ObjectID: t.ObjectID,
ObjectType: objectType,
Lang: t.DstLang,
Type: model.TranslationType(translationTypeStr),
Confidence: t.Confidence,
State: model.TranslationState(t.State),
NormHash: t.NormHash,
Meta: meta,
UpdateAt: t.UpdateAt,
}
if modelT.Type == model.TranslationTypeObject {
modelT.ObjectJSON = json.RawMessage(t.Text)
} else {
modelT.Text = t.Text
}
result = append(result, modelT)
}
return result, nil
}
func (s *SqlAutoTranslationStore) Save(translation *model.Translation) error {
if err := translation.IsValid(); err != nil {
return err
}
now := model.GetMillis()
var err error
text := translation.Text
if translation.Type == model.TranslationTypeObject && len(translation.ObjectJSON) > 0 {
text = string(translation.ObjectJSON)
}
objectType := translation.ObjectType
if objectType == "" {
objectType = model.TranslationObjectTypePost
}
var channelID *string
if translation.ChannelID != "" {
channelID = &translation.ChannelID
}
objectID := translation.ObjectID
// Preserve existing Meta fields and add/override "type"
metaMap := make(map[string]any)
if translation.Meta != nil {
// Copy existing Meta fields (e.g., "src_lang", "error", etc.)
maps.Copy(metaMap, translation.Meta)
}
// Always set "type" field
metaMap["type"] = string(translation.Type)
metaBytes, err := json.Marshal(metaMap)
if err != nil {
return errors.Wrap(err, "failed to marshal translation meta")
}
// Apply binary flag if enabled (required for PostgreSQL JSONB with binary_parameters=yes)
if s.IsBinaryParamEnabled() {
metaBytes = AppendBinaryFlag(metaBytes)
}
dstLang := translation.Lang
providerID := translation.Provider
confidence := translation.Confidence
query := s.getQueryBuilder().
Insert("Translations").
Columns("ObjectId", "DstLang", "ObjectType", "ChannelId", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt").
Values(objectID, dstLang, objectType, channelID, providerID, translation.NormHash, text, confidence, metaBytes, string(translation.State), now).
Suffix(`ON CONFLICT (ObjectId, ObjectType, dstLang)
DO UPDATE SET
ChannelId = EXCLUDED.ChannelId,
ProviderId = EXCLUDED.ProviderId,
NormHash = EXCLUDED.NormHash,
Text = EXCLUDED.Text,
Confidence = EXCLUDED.Confidence,
Meta = EXCLUDED.Meta,
State = EXCLUDED.State,
UpdateAt = EXCLUDED.UpdateAt
WHERE Translations.NormHash IS DISTINCT FROM EXCLUDED.NormHash
OR Translations.State != EXCLUDED.State`)
if _, err := s.GetMaster().ExecBuilder(query); err != nil {
return errors.Wrapf(err, "failed to save translation for object_id=%s, lang=%s", objectID, dstLang)
}
return nil
}
func (s *SqlAutoTranslationStore) GetByStateOlderThan(state model.TranslationState, olderThanMillis int64, limit int) ([]*model.Translation, error) {
query := s.getQueryBuilder().
Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt").
From("Translations").
Where(sq.Eq{"State": string(state)}).
Where(sq.Lt{"UpdateAt": olderThanMillis}).
OrderBy("UpdateAt ASC").
Limit(uint64(limit))
var translations []Translation
if err := s.GetReplica().SelectBuilder(&translations, query); err != nil {
return nil, errors.Wrapf(err, "failed to get translations by state=%s older than %d", state, olderThanMillis)
}
result := make([]*model.Translation, 0, len(translations))
for _, t := range translations {
var translationTypeStr string
meta, err := t.Meta.ToMap()
if err != nil {
// Log error but continue with other translations
continue
}
if v, ok := meta["type"]; ok {
if s, ok := v.(string); ok {
translationTypeStr = s
}
}
// Default objectType to "post" if not set
objectType := t.ObjectType
if objectType == "" {
objectType = model.TranslationObjectTypePost
}
modelT := &model.Translation{
ObjectID: t.ObjectID,
ObjectType: objectType,
Lang: t.DstLang,
Type: model.TranslationType(translationTypeStr),
Confidence: t.Confidence,
State: model.TranslationState(t.State),
NormHash: t.NormHash,
Meta: meta,
UpdateAt: t.UpdateAt,
}
if modelT.Type == model.TranslationTypeObject {
modelT.ObjectJSON = json.RawMessage(t.Text)
} else {
modelT.Text = t.Text
}
result = append(result, modelT)
}
return result, nil
}
func (s *SqlAutoTranslationStore) ClearCaches() {}
func (s *SqlAutoTranslationStore) InvalidateUserAutoTranslation(userID, channelID string) {}
func (s *SqlAutoTranslationStore) InvalidateUserLocaleCache(userID string) {}
// GetLatestPostUpdateAtForChannel returns the most recent updateAt timestamp for post translations
// in the given channel (across all locales). Uses a direct lookup on the channelid column
// for O(1) performance. Returns 0 if no translations exist.
func (s *SqlAutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) {
query := s.getQueryBuilder().
Select("COALESCE(MAX(updateAt), 0)").
From("translations").
Where(sq.Eq{"channelid": channelID}).
Where(sq.Eq{"objectType": model.TranslationObjectTypePost})
var updateAt int64
if err := s.GetReplica().GetBuilder(&updateAt, query); err != nil {
return 0, errors.Wrapf(err, "failed to get latest translation updateAt for channel_id=%s", channelID)
}
return updateAt, nil
}
func (s *SqlAutoTranslationStore) InvalidatePostTranslationEtag(channelID string) {}
func (s *SqlAutoTranslationStore) GetTranslationsSinceForChannel(channelID, dstLang string, since int64) (map[string]*model.Translation, error) {
query := s.getQueryBuilder().
Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt").
From("Translations").
Where(sq.Eq{"channelid": channelID}).
Where(sq.Eq{"DstLang": dstLang}).
Where(sq.Eq{"ObjectType": model.TranslationObjectTypePost}).
Where(sq.NotEq{"State": string(model.TranslationStateProcessing)}).
Where(sq.Gt{"UpdateAt": since}).
Limit(1000)
var translations []Translation
if err := s.GetReplica().SelectBuilder(&translations, query); err != nil {
return nil, errors.Wrapf(err, "failed to get translations since for channel_id=%s dst_lang=%s", channelID, dstLang)
}
result := make(map[string]*model.Translation, len(translations))
for _, t := range translations {
var translationTypeStr string
meta, err := t.Meta.ToMap()
if err != nil {
continue
}
if v, ok := meta["type"]; ok {
if s, ok := v.(string); ok {
translationTypeStr = s
}
}
objectType := t.ObjectType
if objectType == "" {
objectType = model.TranslationObjectTypePost
}
modelT := &model.Translation{
ObjectID: t.ObjectID,
ObjectType: objectType,
Lang: t.DstLang,
Type: model.TranslationType(translationTypeStr),
Confidence: t.Confidence,
State: model.TranslationState(t.State),
NormHash: t.NormHash,
Meta: meta,
UpdateAt: t.UpdateAt,
}
if modelT.Type == model.TranslationTypeObject {
modelT.ObjectJSON = json.RawMessage(t.Text)
} else {
modelT.Text = t.Text
}
result[t.ObjectID] = modelT
}
return result, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestAutoTranslationStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestAutoTranslationStore)
}

View File

@@ -0,0 +1,270 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/einterfaces"
sq "github.com/mattermost/squirrel"
)
// bot is a subset of the model.Bot type, omitting the model.User fields.
type bot struct {
UserId string `json:"user_id"`
Description string `json:"description"`
OwnerId string `json:"owner_id"`
LastIconUpdate int64 `json:"last_icon_update"`
CreateAt int64 `json:"create_at"`
UpdateAt int64 `json:"update_at"`
DeleteAt int64 `json:"delete_at"`
}
func botFromModel(b *model.Bot) *bot {
return &bot{
UserId: b.UserId,
Description: b.Description,
OwnerId: b.OwnerId,
LastIconUpdate: b.LastIconUpdate,
CreateAt: b.CreateAt,
UpdateAt: b.UpdateAt,
DeleteAt: b.DeleteAt,
}
}
// SqlBotStore is a store for managing bots in the database.
// Bots are otherwise normal users with extra metadata record in the Bots table. The primary key
// for a bot matches the primary key value for corresponding User record.
type SqlBotStore struct {
*SqlStore
metrics einterfaces.MetricsInterface
// botsQuery is a starting point for all queries that return one or more Bots.
botsQuery sq.SelectBuilder
}
// newSqlBotStore creates an instance of SqlBotStore, registering the table schema in question.
func newSqlBotStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) store.BotStore {
bs := &SqlBotStore{
SqlStore: sqlStore,
metrics: metrics,
}
// note: we are providing field names explicitly here to maintain order of columns (needed when using raw queries)
bs.botsQuery = bs.getQueryBuilder().
Select("b.UserId", "u.Username", "u.FirstName AS DisplayName", "b.Description", "b.OwnerId", "COALESCE(b.LastIconUpdate, 0) AS LastIconUpdate", "b.CreateAt", "b.UpdateAt", "b.DeleteAt").
From("Bots b").
Join("Users u ON ( u.Id = b.UserId )")
return bs
}
// Get fetches the given bot in the database.
func (us SqlBotStore) Get(botUserId string, includeDeleted bool) (*model.Bot, error) {
var excludeDeletedSql = "AND b.DeleteAt = 0"
if includeDeleted {
excludeDeletedSql = ""
}
query := `
SELECT
b.UserId,
u.Username,
u.FirstName AS DisplayName,
b.Description,
b.OwnerId,
COALESCE(b.LastIconUpdate, 0) AS LastIconUpdate,
b.CreateAt,
b.UpdateAt,
b.DeleteAt
FROM
Bots b
JOIN
Users u ON (u.Id = b.UserId)
WHERE
b.UserId = ?
` + excludeDeletedSql + `
`
var bot model.Bot
if err := us.GetReplica().Get(&bot, query, botUserId); err == sql.ErrNoRows {
return nil, store.NewErrNotFound("Bot", botUserId)
} else if err != nil {
return nil, errors.Wrapf(err, "selectone: user_id=%s", botUserId)
}
return &bot, nil
}
// GetAll fetches from all bots in the database.
func (us SqlBotStore) GetAll(options *model.BotGetOptions) ([]*model.Bot, error) {
var conditions []string
var conditionsSql string
var additionalJoin string
var args []any
if !options.IncludeDeleted {
conditions = append(conditions, "b.DeleteAt = 0")
}
if options.OwnerId != "" {
conditions = append(conditions, "b.OwnerId = ?")
args = append(args, options.OwnerId)
}
if options.OnlyOrphaned {
additionalJoin = "JOIN Users o ON (o.Id = b.OwnerId)"
conditions = append(conditions, "o.DeleteAt != 0")
}
if len(conditions) > 0 {
conditionsSql = "WHERE " + strings.Join(conditions, " AND ")
}
sql := `
SELECT
b.UserId,
u.Username,
u.FirstName AS DisplayName,
b.Description,
b.OwnerId,
COALESCE(b.LastIconUpdate, 0) AS LastIconUpdate,
b.CreateAt,
b.UpdateAt,
b.DeleteAt
FROM
Bots b
JOIN
Users u ON (u.Id = b.UserId)
` + additionalJoin + `
` + conditionsSql + `
ORDER BY
b.CreateAt ASC,
u.Username ASC
LIMIT
?
OFFSET
?
`
// append limit, offset
args = append(args, options.PerPage, options.Page*options.PerPage)
bots := []*model.Bot{}
if err := us.GetReplica().Select(&bots, sql, args...); err != nil {
return nil, errors.Wrap(err, "error selecting all bots")
}
return bots, nil
}
// Save persists a new bot to the database.
// It assumes the corresponding user was saved via the user store.
func (us SqlBotStore) Save(bot *model.Bot) (*model.Bot, error) {
bot = bot.Clone()
bot.PreSave()
if err := bot.IsValid(); err != nil { // TODO: change to return error in v6.
return nil, err
}
if _, err := us.GetMaster().NamedExec(`INSERT INTO Bots
(UserId, Description, OwnerId, LastIconUpdate, CreateAt, UpdateAt, DeleteAt)
VALUES
(:UserId, :Description, :OwnerId, :LastIconUpdate, :CreateAt, :UpdateAt, :DeleteAt)`, botFromModel(bot)); err != nil {
return nil, errors.Wrapf(err, "insert: user_id=%s", bot.UserId)
}
return bot, nil
}
// Update persists an updated bot to the database.
// It assumes the corresponding user was updated via the user store.
func (us SqlBotStore) Update(bot *model.Bot) (*model.Bot, error) {
bot = bot.Clone()
bot.PreUpdate()
if err := bot.IsValid(); err != nil { // TODO: needs to return error in v6
return nil, err
}
oldBot, err := us.Get(bot.UserId, true)
if err != nil {
return nil, err
}
oldBot.Description = bot.Description
oldBot.OwnerId = bot.OwnerId
oldBot.LastIconUpdate = bot.LastIconUpdate
oldBot.UpdateAt = bot.UpdateAt
oldBot.DeleteAt = bot.DeleteAt
bot = oldBot
res, err := us.GetMaster().NamedExec(`UPDATE Bots
SET Description=:Description, OwnerId=:OwnerId, LastIconUpdate=:LastIconUpdate,
UpdateAt=:UpdateAt, DeleteAt=:DeleteAt
WHERE UserId=:UserId`, botFromModel(bot))
if err != nil {
return nil, errors.Wrapf(err, "update: user_id=%s", bot.UserId)
}
count, err := res.RowsAffected()
if err != nil {
return nil, errors.Wrap(err, "error while getting rows_affected")
}
if count > 1 {
return nil, fmt.Errorf("unexpected count while updating bot: count=%d, userId=%s", count, bot.UserId)
}
return bot, nil
}
// PermanentDelete removes the bot from the database altogether.
// If the corresponding user is to be deleted, it must be done via the user store.
func (us SqlBotStore) PermanentDelete(botUserId string) error {
query := "DELETE FROM Bots WHERE UserId = ?"
if _, err := us.GetMaster().Exec(query, botUserId); err != nil {
return store.NewErrInvalidInput("Bot", "UserId", botUserId).Wrap(err)
}
return nil
}
func (us SqlBotStore) GetAllAfter(limit int, afterId string) ([]*model.Bot, error) {
query := us.botsQuery.Where("b.UserId > ?", afterId).OrderBy("b.UserId ASC").Limit(uint64(limit))
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "get_all_after_tosql")
}
bots := []*model.Bot{}
if err := us.GetReplica().Select(&bots, queryString, args...); err != nil {
return nil, errors.Wrap(err, "failed to find Bots")
}
return bots, nil
}
// Get fetches the given bot in the database.
func (us SqlBotStore) GetByUsername(username string) (*model.Bot, error) {
query := us.botsQuery.Where("u.Username = lower(?)", username)
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "get_by_username_tosql")
}
bot := model.Bot{}
if err := us.GetReplica().Get(&bot, queryString, args...); err != nil {
if err == sql.ErrNoRows {
return nil, errors.Wrap(store.NewErrNotFound("Bot", fmt.Sprintf("username=%s", username)), "failed to find Bot")
}
return nil, errors.Wrapf(err, "failed to find Bot with username=%s", username)
}
return &bot, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestBotStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestBotStore)
}

View File

@@ -0,0 +1,391 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestBulkInsertChunking(t *testing.T) {
StoreTest(t, testBulkInsertChunking)
}
// setMaxInsertParams overrides the max-insert-params threshold on the
// underlying SqlStore and returns a cleanup function that restores the default.
func setMaxInsertParams(t *testing.T, ss store.Store, maxParams int) {
t.Helper()
sqlStore := ss.(*SqlStore)
sqlStore.maxInsertParams = maxParams
t.Cleanup(func() { sqlStore.maxInsertParams = 0 })
}
func testBulkInsertChunking(t *testing.T, rctx request.CTX, ss store.Store) {
t.Run("ChannelMembers across multiple chunks", func(t *testing.T) {
// 15 columns/row. With maxInsertParams=30, chunkSize=2,
// so 5 members split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 30)
team := &model.Team{
DisplayName: "ChunkTest",
Name: storetest.NewTestID(),
Email: storetest.MakeEmail(),
Type: model.TeamOpen,
}
team, err := ss.Team().Save(team)
require.NoError(t, err)
channel := &model.Channel{
DisplayName: "ChunkTest",
Name: "z-z-z" + model.NewId(),
Type: model.ChannelTypeOpen,
TeamId: team.Id,
}
channel, err = ss.Channel().Save(rctx, channel, -1)
require.NoError(t, err)
const n = 5
members := make([]*model.ChannelMember, n)
defaultNotifyProps := model.GetDefaultChannelNotifyProps()
for i := range members {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
members[i] = &model.ChannelMember{
ChannelId: channel.Id,
UserId: u.Id,
NotifyProps: defaultNotifyProps,
}
}
saved, err := ss.Channel().SaveMultipleMembers(members)
require.NoError(t, err)
require.Len(t, saved, n)
for _, m := range saved {
got, gErr := ss.Channel().GetMember(rctx, channel.Id, m.UserId)
require.NoError(t, gErr)
require.Equal(t, channel.Id, got.ChannelId)
}
})
t.Run("ChannelMembers atomicity on conflict", func(t *testing.T) {
// With chunkSize=2, inserting [A, B, B] produces chunks [A,B] and [B].
// Chunk 1 succeeds; chunk 2 hits a unique constraint. The transaction
// must roll back all chunks, leaving zero rows.
setMaxInsertParams(t, ss, 30)
team := &model.Team{
DisplayName: "AtomicTest",
Name: storetest.NewTestID(),
Email: storetest.MakeEmail(),
Type: model.TeamOpen,
}
team, err := ss.Team().Save(team)
require.NoError(t, err)
channel := &model.Channel{
DisplayName: "AtomicTest",
Name: "z-z-z" + model.NewId(),
Type: model.ChannelTypeOpen,
TeamId: team.Id,
}
channel, err = ss.Channel().Save(rctx, channel, -1)
require.NoError(t, err)
users := make([]*model.User, 3)
for i := range users {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
users[i] = u
}
defaultNotifyProps := model.GetDefaultChannelNotifyProps()
members := []*model.ChannelMember{
{ChannelId: channel.Id, UserId: users[0].Id, NotifyProps: defaultNotifyProps},
{ChannelId: channel.Id, UserId: users[1].Id, NotifyProps: defaultNotifyProps},
{ChannelId: channel.Id, UserId: users[1].Id, NotifyProps: defaultNotifyProps}, // duplicate
}
_, err = ss.Channel().SaveMultipleMembers(members)
require.Error(t, err)
// Verify none were persisted (transaction rolled back).
_, gErr := ss.Channel().GetMember(rctx, channel.Id, users[0].Id)
require.Error(t, gErr, "member from chunk 1 should not exist after rollback")
})
t.Run("TeamMembers across multiple chunks", func(t *testing.T) {
// 8 columns/row. With maxInsertParams=16, chunkSize=2,
// so 5 members split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 16)
teamID := model.NewId()
const n = 5
members := make([]*model.TeamMember, n)
for i := range members {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
members[i] = &model.TeamMember{TeamId: teamID, UserId: u.Id}
}
saved, err := ss.Team().SaveMultipleMembers(members, -1)
require.NoError(t, err)
require.Len(t, saved, n)
for _, m := range saved {
got, gErr := ss.Team().GetMember(rctx, teamID, m.UserId)
require.NoError(t, gErr)
require.Equal(t, teamID, got.TeamId)
}
})
t.Run("TeamMembers atomicity on conflict", func(t *testing.T) {
setMaxInsertParams(t, ss, 16)
teamID := model.NewId()
users := make([]*model.User, 3)
for i := range users {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
users[i] = u
}
members := []*model.TeamMember{
{TeamId: teamID, UserId: users[0].Id},
{TeamId: teamID, UserId: users[1].Id},
{TeamId: teamID, UserId: users[1].Id}, // duplicate
}
_, err := ss.Team().SaveMultipleMembers(members, -1)
require.Error(t, err)
_, gErr := ss.Team().GetMember(rctx, teamID, users[0].Id)
require.Error(t, gErr, "member from chunk 1 should not exist after rollback")
})
t.Run("ThreadMemberships across multiple chunks", func(t *testing.T) {
// 6 columns/row. With maxInsertParams=12, chunkSize=2,
// so 5 memberships split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 12)
const n = 5
memberships := make([]*model.ThreadMembership, n)
for i := range memberships {
memberships[i] = &model.ThreadMembership{
PostId: model.NewId(),
UserId: model.NewId(),
Following: true,
}
}
saved, err := ss.Thread().SaveMultipleMemberships(memberships)
require.NoError(t, err)
require.Len(t, saved, n)
for _, m := range saved {
got, gErr := ss.Thread().GetMembershipForUser(m.UserId, m.PostId)
require.NoError(t, gErr)
require.Equal(t, m.PostId, got.PostId)
require.Equal(t, m.UserId, got.UserId)
}
})
t.Run("Posts across multiple chunks", func(t *testing.T) {
// 18 columns/row. With maxInsertParams=36, chunkSize=2,
// so 5 posts split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 36)
channel := &model.Channel{
DisplayName: "PostChunkTest",
Name: "z-z-z" + model.NewId(),
Type: model.ChannelTypeOpen,
TeamId: model.NewId(),
}
channel, err := ss.Channel().Save(rctx, channel, -1)
require.NoError(t, err)
u, err := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, err)
const n = 5
posts := make([]*model.Post, n)
for i := range posts {
posts[i] = &model.Post{
ChannelId: channel.Id,
UserId: u.Id,
Message: "chunk test post",
}
}
saved, _, err := ss.Post().SaveMultiple(rctx, posts)
require.NoError(t, err)
require.Len(t, saved, n)
for _, p := range saved {
got, gErr := ss.Post().GetSingle(rctx, p.Id, false)
require.NoError(t, gErr)
require.Equal(t, p.Id, got.Id)
}
})
t.Run("Posts atomicity on conflict", func(t *testing.T) {
// Use remote posts to allow pre-set IDs. With chunkSize=2,
// posts [A, B, A-dup] produce chunks [A,B] and [A-dup].
// Chunk 2 hits PK violation; transaction must roll back all.
setMaxInsertParams(t, ss, 36)
channel := &model.Channel{
DisplayName: "PostAtomicTest",
Name: "z-z-z" + model.NewId(),
Type: model.ChannelTypeOpen,
TeamId: model.NewId(),
}
channel, err := ss.Channel().Save(rctx, channel, -1)
require.NoError(t, err)
u, err := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, err)
remoteId := new(model.NewId())
duplicateId := model.NewId()
posts := []*model.Post{
{Id: duplicateId, RemoteId: remoteId, ChannelId: channel.Id, UserId: u.Id, Message: "post A"},
{Id: model.NewId(), RemoteId: remoteId, ChannelId: channel.Id, UserId: u.Id, Message: "post B"},
{Id: duplicateId, RemoteId: remoteId, ChannelId: channel.Id, UserId: u.Id, Message: "post A dup"},
}
_, _, err = ss.Post().SaveMultiple(rctx, posts)
require.Error(t, err)
// Post from chunk 1 should not exist (transaction rolled back).
_, gErr := ss.Post().GetSingle(rctx, duplicateId, false)
require.Error(t, gErr, "post from chunk 1 should not exist after rollback")
})
t.Run("GroupMembers via CreateWithUserIds atomicity on conflict", func(t *testing.T) {
// With chunkSize=2, userIds [A, B, B] produce chunks [A,B] and [B].
// Chunk 2 hits unique constraint; transaction must roll back all
// (including the group row itself).
setMaxInsertParams(t, ss, 8)
users := make([]*model.User, 2)
for i := range users {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
users[i] = u
}
group := &model.GroupWithUserIds{
Group: model.Group{
DisplayName: "AtomicGroup",
Name: new("atomic-group-" + model.NewId()),
Source: model.GroupSourceCustom,
},
UserIds: []string{users[0].Id, users[1].Id, users[1].Id}, // duplicate
}
_, err := ss.Group().CreateWithUserIds(group)
require.Error(t, err)
})
t.Run("Status upserts across multiple chunks", func(t *testing.T) {
// 6 columns/row. With maxInsertParams=12, chunkSize=2,
// so 5 statuses split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 12)
const n = 5
statuses := make(map[string]*model.Status, n)
userIDs := make([]string, 0, n)
for range n {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
statuses[u.Id] = &model.Status{
UserId: u.Id,
Status: model.StatusOnline,
Manual: false,
}
userIDs = append(userIDs, u.Id)
}
err := ss.Status().SaveOrUpdateMany(statuses)
require.NoError(t, err)
for _, uid := range userIDs {
got, gErr := ss.Status().Get(uid)
require.NoError(t, gErr)
require.Equal(t, model.StatusOnline, got.Status)
}
})
t.Run("GroupMembers via CreateWithUserIds across chunks", func(t *testing.T) {
// 4 columns/row. With maxInsertParams=8, chunkSize=2,
// so 5 members split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 8)
const n = 5
userIDs := make([]string, n)
for i := range userIDs {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
userIDs[i] = u.Id
}
group := &model.GroupWithUserIds{
Group: model.Group{
DisplayName: "ChunkGroup",
Name: new("chunk-group-" + model.NewId()),
Source: model.GroupSourceCustom,
},
UserIds: userIDs,
}
created, err := ss.Group().CreateWithUserIds(group)
require.NoError(t, err)
require.NotNil(t, created)
members, err := ss.Group().GetMemberUsers(created.Id)
require.NoError(t, err)
require.Len(t, members, n)
})
t.Run("GroupMembers via UpsertMembers across chunks", func(t *testing.T) {
// 4 columns/row. With maxInsertParams=8, chunkSize=2,
// so 5 members split into 3 chunks (2+2+1).
setMaxInsertParams(t, ss, 8)
// Create a group first (with no members).
group := &model.GroupWithUserIds{
Group: model.Group{
DisplayName: "UpsertGroup",
Name: new("upsert-group-" + model.NewId()),
Source: model.GroupSourceCustom,
},
}
created, err := ss.Group().CreateWithUserIds(group)
require.NoError(t, err)
const n = 5
userIDs := make([]string, n)
for i := range userIDs {
u, uErr := ss.User().Save(rctx, &model.User{Username: model.NewUsername(), Email: storetest.MakeEmail()})
require.NoError(t, uErr)
userIDs[i] = u.Id
}
upserted, err := ss.Group().UpsertMembers(created.Id, userIDs)
require.NoError(t, err)
require.Len(t, upserted, n)
members, err := ss.Group().GetMemberUsers(created.Id)
require.NoError(t, err)
require.Len(t, members, n)
})
}

View File

@@ -0,0 +1,394 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"slices"
"strconv"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/channels/utils"
)
type SqlChannelBookmarkStore struct {
*SqlStore
}
func newSqlChannelBookmarkStore(sqlStore *SqlStore) store.ChannelBookmarkStore {
return &SqlChannelBookmarkStore{sqlStore}
}
func bookmarkWithFileInfoSliceColumns() []string {
return []string{
"cb.Id",
"cb.OwnerId",
"cb.ChannelId",
"cb.FileInfoId",
"cb.CreateAt",
"cb.UpdateAt",
"cb.DeleteAt",
"cb.DisplayName",
"cb.SortOrder",
"cb.LinkUrl",
"cb.ImageUrl",
"cb.Emoji",
"cb.Type",
"COALESCE(cb.TargetId, '') as TargetId",
"COALESCE(cb.OriginalId, '') as OriginalId",
"COALESCE(fi.Id, '') as FileId",
"COALESCE(fi.Name, '') as FileName",
"COALESCE(fi.Extension, '') as Extension",
"COALESCE(fi.Size, 0) as Size",
"COALESCE(fi.MimeType, '') as MimeType",
"COALESCE(fi.Width, 0) as Width",
"COALESCE(fi.Height, 0) as Height",
"COALESCE(fi.HasPreviewImage, false) as HasPreviewImage",
"COALESCE(fi.MiniPreview, '') as MiniPreview",
}
}
func (s *SqlChannelBookmarkStore) ErrorIfBookmarkFileInfoAlreadyAttached(fileId string, channelId string) error {
existingQuery := s.getSubQueryBuilder().
Select("FileInfoId").
From("ChannelBookmarks").
Where(sq.And{
sq.Eq{"FileInfoId": fileId},
sq.Eq{"DeleteAt": 0},
})
alreadyAttachedQuery := s.getQueryBuilder().
Select("COUNT(*)").
From("FileInfo").
Where(sq.Or{
sq.Expr("Id IN (?)", existingQuery),
sq.And{
sq.Eq{"Id": fileId},
sq.Or{
sq.NotEq{"PostId": ""},
sq.NotEq{"CreatorId": model.BookmarkFileOwner},
sq.NotEq{"ChannelId": channelId},
sq.NotEq{"DeleteAt": 0},
},
},
})
var attached int64
err := s.GetReplica().GetBuilder(&attached, alreadyAttachedQuery)
if err != nil {
return errors.Wrap(err, "unable_to_save_channel_bookmark")
}
if attached > 0 {
return store.NewErrInvalidInput("ChannelBookmarks", "FileInfoId", fileId)
}
return nil
}
func (s *SqlChannelBookmarkStore) Get(Id string, includeDeleted bool) (*model.ChannelBookmarkWithFileInfo, error) {
query := s.getQueryBuilder().
Select(bookmarkWithFileInfoSliceColumns()...).
From("ChannelBookmarks cb").
LeftJoin("FileInfo fi ON cb.FileInfoId = fi.Id").
Where(sq.Eq{"cb.Id": Id})
if !includeDeleted {
query = query.Where(sq.Eq{"cb.DeleteAt": 0})
}
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_bookmark_getforchanneltsince_tosql")
}
bookmark := model.ChannelBookmarkAndFileInfo{}
if err := s.GetReplica().Get(&bookmark, queryString, args...); err != nil {
return nil, store.NewErrNotFound("ChannelBookmark", Id)
}
return bookmark.ToChannelBookmarkWithFileInfo(), nil
}
func (s *SqlChannelBookmarkStore) Save(bookmark *model.ChannelBookmark, increaseSortOrder bool) (b *model.ChannelBookmarkWithFileInfo, err error) {
bookmark.PreSave()
if err := bookmark.IsValid(); err != nil {
return nil, err
}
transaction, err := s.GetMaster().Begin()
if err != nil {
return nil, err
}
defer finalizeTransactionX(transaction, &err)
var currentBookmarksCount int64
query := s.getQueryBuilder().
Select("COUNT(*) as count").
From("ChannelBookmarks").
Where(sq.Eq{"ChannelId": bookmark.ChannelId, "DeleteAt": 0})
err = transaction.GetBuilder(&currentBookmarksCount, query)
if err != nil {
return nil, errors.Wrap(err, "failed while getting the count of ChannelBookmarks")
}
if currentBookmarksCount >= model.MaxBookmarksPerChannel {
return nil, store.NewErrLimitExceeded("bookmarks_per_channel", int(currentBookmarksCount), "channelId="+bookmark.ChannelId)
}
if bookmark.FileId != "" {
err = s.ErrorIfBookmarkFileInfoAlreadyAttached(bookmark.FileId, bookmark.ChannelId)
if err != nil {
return nil, errors.Wrap(err, "unable_to_save_channel_bookmark")
}
}
if increaseSortOrder {
var sortOrder int64
query := s.getQueryBuilder().
Select("COALESCE(MAX(SortOrder), -1) as SortOrder").
From("ChannelBookmarks").
Where(sq.Eq{"ChannelId": bookmark.ChannelId, "DeleteAt": 0})
err = transaction.GetBuilder(&sortOrder, query)
if err != nil {
return nil, errors.Wrap(err, "failed while getting the sortOrder from ChannelBookmarks")
}
bookmark.SortOrder = sortOrder + 1
}
targetID := any(bookmark.TargetId)
if bookmark.TargetId == "" {
targetID = nil
}
sql, args, sqlErr := s.getQueryBuilder().
Insert("ChannelBookmarks").
Columns("Id", "CreateAt", "UpdateAt", "DeleteAt", "ChannelId", "OwnerId", "FileInfoId", "DisplayName", "SortOrder", "LinkUrl", "ImageUrl", "Emoji", "Type", "TargetId").
Values(bookmark.Id, bookmark.CreateAt, bookmark.UpdateAt, bookmark.DeleteAt, bookmark.ChannelId, bookmark.OwnerId, bookmark.FileId, bookmark.DisplayName, bookmark.SortOrder, bookmark.LinkUrl, bookmark.ImageUrl, bookmark.Emoji, bookmark.Type, targetID).
ToSql()
if sqlErr != nil {
return nil, errors.Wrap(err, "insert_channel_bookmark_to_sql")
}
if _, insertErr := transaction.Exec(sql, args...); insertErr != nil {
return nil, errors.Wrap(insertErr, "unable_to_save_channel_bookmark")
}
var fileInfo model.FileInfo
if bookmark.FileId != "" {
query, args, queryErr := s.getQueryBuilder().
Select("Id, Name, Extension, Size, MimeType, Width, Height, HasPreviewImage, MiniPreview").
From("FileInfo").
Where(sq.Eq{"Id": bookmark.FileId}).
ToSql()
if queryErr != nil {
return nil, errors.Wrap(queryErr, "channel_bookmark_get_file_info_to_sql")
}
if queryErr = transaction.Get(&fileInfo, query, args...); queryErr != nil {
return nil, errors.Wrap(queryErr, "unable_to_get_channel_bookmark_file_info")
}
}
err = transaction.Commit()
return bookmark.ToBookmarkWithFileInfo(&fileInfo), err
}
func (s *SqlChannelBookmarkStore) Update(bookmark *model.ChannelBookmark) error {
bookmark.PreUpdate()
if err := bookmark.IsValid(); err != nil {
return err
}
targetID := any(bookmark.TargetId)
if bookmark.TargetId == "" {
targetID = nil
}
query, args, err := s.getQueryBuilder().
Update("ChannelBookmarks").
Set("DisplayName", bookmark.DisplayName).
Set("SortOrder", bookmark.SortOrder).
Set("LinkUrl", bookmark.LinkUrl).
Set("ImageUrl", bookmark.ImageUrl).
Set("Emoji", bookmark.Emoji).
Set("FileInfoId", bookmark.FileId).
Set("TargetId", targetID).
Set("UpdateAt", bookmark.UpdateAt).
Where(sq.Eq{
"Id": bookmark.Id,
"DeleteAt": 0,
}).
ToSql()
if err != nil {
return errors.Wrap(err, "channel_bookmark_update_tosql")
}
res, err := s.GetMaster().Exec(query, args...)
if err != nil {
return errors.Wrapf(err, "failed to update channel bookmark with id=%s", bookmark.Id)
}
rowsAffected, err := res.RowsAffected()
if err != nil {
return errors.Wrapf(err, "failed to get affected rows after updating bookmark with id=%s", bookmark.Id)
}
if rowsAffected == 0 {
return store.NewErrNotFound("ChannelBookmark", bookmark.Id)
}
return nil
}
func (s *SqlChannelBookmarkStore) UpdateSortOrder(bookmarkId, channelId string, newIndex int64) ([]*model.ChannelBookmarkWithFileInfo, error) {
now := model.GetMillis()
transaction, err := s.GetMaster().Begin()
if err != nil {
return nil, err
}
defer finalizeTransactionX(transaction, &err)
bookmarks, err := s.GetBookmarksForChannelSince(channelId, 0)
if err != nil {
return nil, err
}
if (int(newIndex) > len(bookmarks)-1) || newIndex < 0 {
return nil, store.NewErrInvalidInput("ChannelBookmark", "SortOrder", newIndex)
}
currentIndex := -1
var current *model.ChannelBookmarkWithFileInfo
for index, b := range bookmarks {
if b.Id == bookmarkId {
currentIndex = index
current = b
break
}
}
if currentIndex == -1 {
return nil, store.NewErrNotFound("ChannelBookmark", bookmarkId)
}
bookmarks = utils.RemoveElementFromSliceAtIndex(bookmarks, currentIndex)
bookmarks = slices.Insert(bookmarks, int(newIndex), current)
caseStmt := sq.Case()
query := s.getQueryBuilder().
Update("ChannelBookmarks")
ids := []string{}
for index, b := range bookmarks {
b.SortOrder = int64(index)
b.UpdateAt = now
caseStmt = caseStmt.When(sq.Eq{"Id": b.Id}, strconv.FormatInt(int64(index), 10))
ids = append(ids, b.Id)
}
query = query.Set("SortOrder", caseStmt)
query = query.Set("UpdateAt", now)
query = query.Where(sq.Eq{"Id": ids})
queryStr, args, queryErr := query.ToSql()
if queryErr != nil {
return nil, queryErr
}
if _, updateSortOrderErr := transaction.Exec(queryStr, args...); updateSortOrderErr != nil {
return nil, updateSortOrderErr
}
err = transaction.Commit()
return bookmarks, err
}
func (s *SqlChannelBookmarkStore) Delete(bookmarkId string, deleteFile bool) error {
now := model.GetMillis()
transaction, err := s.GetMaster().Begin()
if err != nil {
return err
}
defer finalizeTransactionX(transaction, &err)
query, args, err := s.getQueryBuilder().
Update("ChannelBookmarks").
Set("DeleteAt", now).
Set("UpdateAt", now).
Where(sq.Eq{"Id": bookmarkId}).
ToSql()
if err != nil {
return errors.Wrap(err, "channel_bookmark_delete_tosql")
}
_, err = transaction.Exec(query, args...)
if err != nil {
return errors.Wrapf(err, "failed to delete channel bookmark with id=%s", bookmarkId)
}
if deleteFile {
fileIdQuery := s.getSubQueryBuilder().
Select("FileInfoId").
From("ChannelBookmarks").
Where(sq.And{
sq.Eq{"Id": bookmarkId},
})
fileQuery, fileArgs, fileErr := s.getQueryBuilder().
Update("FileInfo").
Set("DeleteAt", now).
Set("UpdateAt", now).
Where(sq.Expr("Id IN (?)", fileIdQuery)).
ToSql()
if fileErr != nil {
return errors.Wrap(err, "channel_bookmark_delete_tosql")
}
_, err = transaction.Exec(fileQuery, fileArgs...)
if err != nil {
return errors.Wrapf(err, "failed to delete channel bookmark with id=%s", bookmarkId)
}
}
return transaction.Commit()
}
func (s *SqlChannelBookmarkStore) GetBookmarksForChannelSince(channelId string, since int64) ([]*model.ChannelBookmarkWithFileInfo, error) {
query := s.getQueryBuilder().
Select(bookmarkWithFileInfoSliceColumns()...).
From("ChannelBookmarks cb").
LeftJoin("FileInfo fi ON cb.FileInfoId = fi.Id").
Where(sq.Eq{"cb.ChannelId": channelId})
if since > 0 {
query = query.Where(sq.Or{
sq.GtOrEq{"cb.UpdateAt": since},
sq.GtOrEq{"cb.DeleteAt": since},
})
} else {
query = query.Where(sq.Eq{"cb.DeleteAt": 0})
}
query = query.
OrderBy("cb.SortOrder ASC").
OrderBy("cb.DeleteAt ASC").
Limit(model.MaxBookmarksPerChannel * 2) // limit to the double of the cap as an edge case
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_bookmark_getforchanneltsince_tosql")
}
bookmarkRows := []model.ChannelBookmarkAndFileInfo{}
bookmarks := []*model.ChannelBookmarkWithFileInfo{}
if err := s.GetReplica().Select(&bookmarkRows, queryString, args...); err != nil {
return nil, errors.Wrapf(err, "failed to find bookmarks")
}
for _, bookmark := range bookmarkRows {
bookmarks = append(bookmarks, bookmark.ToChannelBookmarkWithFileInfo())
}
return bookmarks, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestChannelBookmarkStore(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestChannelBookmarkStore)
}

View File

@@ -0,0 +1,83 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SqlChannelGuardStore struct {
*SqlStore
channelGuardSelectQuery sq.SelectBuilder
}
func newSqlChannelGuardStore(sqlStore *SqlStore) store.ChannelGuardStore {
s := &SqlChannelGuardStore{SqlStore: sqlStore}
s.channelGuardSelectQuery = s.getQueryBuilder().
Select("ChannelId", "PluginId", "CreatedAt").
From("ChannelGuards")
return s
}
func (s *SqlChannelGuardStore) Save(rctx request.CTX, guard *store.ChannelGuard) error {
builder := s.getQueryBuilder().
Insert("ChannelGuards").
Columns("ChannelId", "PluginId", "CreatedAt").
Values(guard.ChannelId, guard.PluginId, guard.CreatedAt).
SuffixExpr(sq.Expr("ON CONFLICT (ChannelId, PluginId) DO NOTHING"))
if _, err := s.GetMaster().ExecBuilder(builder); err != nil {
return errors.Wrapf(err, "failed to save channel guard for channel=%s plugin=%s", guard.ChannelId, guard.PluginId)
}
return nil
}
func (s *SqlChannelGuardStore) Delete(rctx request.CTX, channelID, pluginID string) (int64, error) {
builder := s.getQueryBuilder().
Delete("ChannelGuards").
Where(sq.Eq{
"ChannelId": channelID,
"PluginId": pluginID,
})
result, err := s.GetMaster().ExecBuilder(builder)
if err != nil {
return 0, errors.Wrapf(err, "failed to delete channel guard for channel=%s plugin=%s", channelID, pluginID)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return 0, errors.Wrapf(err, "failed to get rows affected for channel guard delete channel=%s plugin=%s", channelID, pluginID)
}
return rowsAffected, nil
}
func (s *SqlChannelGuardStore) GetForChannel(rctx request.CTX, channelID string) ([]*store.ChannelGuard, error) {
query := s.channelGuardSelectQuery.Where(sq.Eq{"ChannelId": channelID})
guards := []*store.ChannelGuard{}
if err := s.DBXFromContext(rctx.Context()).SelectBuilder(&guards, query); err != nil {
return nil, errors.Wrapf(err, "failed to get channel guards for channel=%s", channelID)
}
return guards, nil
}
func (s *SqlChannelGuardStore) GetAll(rctx request.CTX) ([]*store.ChannelGuard, error) {
guards := []*store.ChannelGuard{}
if err := s.DBXFromContext(rctx.Context()).SelectBuilder(&guards, s.channelGuardSelectQuery); err != nil {
return nil, errors.Wrap(err, "failed to get all channel guards")
}
return guards, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestChannelGuardStore(t *testing.T) {
StoreTest(t, storetest.TestChannelGuardStore)
}

View File

@@ -0,0 +1,244 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
)
const channelJoinRequestsTable = "ChannelJoinRequests"
var channelJoinRequestColumns = []string{
"Id",
"ChannelId",
"UserId",
"Message",
"Status",
"DenialReason",
"CreateAt",
"UpdateAt",
"ReviewedBy",
"ReviewedAt",
}
type SqlChannelJoinRequestStore struct {
*SqlStore
selectQuery sq.SelectBuilder
}
func newSqlChannelJoinRequestStore(sqlStore *SqlStore) store.ChannelJoinRequestStore {
s := &SqlChannelJoinRequestStore{SqlStore: sqlStore}
s.selectQuery = s.getQueryBuilder().
Select(channelJoinRequestColumns...).
From(channelJoinRequestsTable)
return s
}
func (s *SqlChannelJoinRequestStore) toMap(r *model.ChannelJoinRequest) map[string]any {
return map[string]any{
"Id": r.Id,
"ChannelId": r.ChannelId,
"UserId": r.UserId,
"Message": r.Message,
"Status": r.Status,
"DenialReason": r.DenialReason,
"CreateAt": r.CreateAt,
"UpdateAt": r.UpdateAt,
"ReviewedBy": r.ReviewedBy,
"ReviewedAt": r.ReviewedAt,
}
}
// Save inserts a new join request. The partial unique index in Postgres
// (channelid, userid) WHERE status = 'pending' enforces at-most-one pending
// row per (channel, user). On conflict we translate the unique-violation into
// a store.ErrConflict so the app layer can return 409.
func (s *SqlChannelJoinRequestStore) Save(req *model.ChannelJoinRequest) (*model.ChannelJoinRequest, error) {
req.PreSave()
if err := req.IsValid(); err != nil {
return nil, err
}
query := s.getQueryBuilder().
Insert(channelJoinRequestsTable).
SetMap(s.toMap(req))
if _, err := s.GetMaster().ExecBuilder(query); err != nil {
if IsUniqueConstraintError(err, []string{"idx_channeljoinrequests_pending_unique"}) {
return nil, store.NewErrConflict("ChannelJoinRequest", err, "channel_id="+req.ChannelId+" user_id="+req.UserId)
}
return nil, errors.Wrap(err, "failed to save ChannelJoinRequest")
}
return req, nil
}
func (s *SqlChannelJoinRequestStore) Get(id string) (*model.ChannelJoinRequest, error) {
var req model.ChannelJoinRequest
query := s.selectQuery.Where(sq.Eq{"Id": id})
if err := s.GetReplica().GetBuilder(&req, query); err != nil {
if err == sql.ErrNoRows {
return nil, store.NewErrNotFound("ChannelJoinRequest", id)
}
return nil, errors.Wrapf(err, "failed to get ChannelJoinRequest with id=%s", id)
}
return &req, nil
}
func (s *SqlChannelJoinRequestStore) GetPendingForChannelAndUser(channelId, userId string) (*model.ChannelJoinRequest, error) {
var req model.ChannelJoinRequest
query := s.selectQuery.Where(sq.Eq{
"ChannelId": channelId,
"UserId": userId,
"Status": model.ChannelJoinRequestStatusPending,
})
if err := s.GetReplica().GetBuilder(&req, query); err != nil {
if err == sql.ErrNoRows {
return nil, store.NewErrNotFound("ChannelJoinRequest", "channel_id="+channelId+" user_id="+userId)
}
return nil, errors.Wrapf(err, "failed to get pending ChannelJoinRequest for channel_id=%s user_id=%s", channelId, userId)
}
return &req, nil
}
// applyStatusFilter applies the opts.Status filter (defaulting to pending if empty)
// to both the select and count queries. Returning the two filtered builders keeps
// list and count perfectly in sync.
func applyJoinRequestStatusFilter(opts model.GetChannelJoinRequestsOpts) sq.Eq {
status := opts.Status
if status == "" {
status = model.ChannelJoinRequestStatusPending
}
return sq.Eq{"Status": status}
}
func paginate(opts model.GetChannelJoinRequestsOpts) (limit, offset uint64) {
perPage := opts.PerPage
if perPage <= 0 {
perPage = 60
}
page := max(opts.Page, 0)
return uint64(perPage), uint64(page) * uint64(perPage)
}
func (s *SqlChannelJoinRequestStore) GetForChannel(channelId string, opts model.GetChannelJoinRequestsOpts) ([]*model.ChannelJoinRequest, int64, error) {
where := sq.And{sq.Eq{"ChannelId": channelId}, applyJoinRequestStatusFilter(opts)}
limit, offset := paginate(opts)
listQuery := s.selectQuery.
Where(where).
OrderBy("CreateAt DESC", "Id DESC").
Limit(limit).
Offset(offset)
var rows []*model.ChannelJoinRequest
if err := s.GetReplica().SelectBuilder(&rows, listQuery); err != nil {
return nil, 0, errors.Wrapf(err, "failed to list ChannelJoinRequests for channel_id=%s", channelId)
}
countQuery := s.getQueryBuilder().
Select("COUNT(*)").
From(channelJoinRequestsTable).
Where(where)
var total int64
if err := s.GetReplica().GetBuilder(&total, countQuery); err != nil {
return nil, 0, errors.Wrapf(err, "failed to count ChannelJoinRequests for channel_id=%s", channelId)
}
return rows, total, nil
}
func (s *SqlChannelJoinRequestStore) GetForUser(userId string, opts model.GetChannelJoinRequestsOpts) ([]*model.ChannelJoinRequest, int64, error) {
where := sq.And{sq.Eq{"UserId": userId}, applyJoinRequestStatusFilter(opts)}
limit, offset := paginate(opts)
listQuery := s.selectQuery.
Where(where).
OrderBy("CreateAt DESC", "Id DESC").
Limit(limit).
Offset(offset)
var rows []*model.ChannelJoinRequest
if err := s.GetReplica().SelectBuilder(&rows, listQuery); err != nil {
return nil, 0, errors.Wrapf(err, "failed to list ChannelJoinRequests for user_id=%s", userId)
}
countQuery := s.getQueryBuilder().
Select("COUNT(*)").
From(channelJoinRequestsTable).
Where(where)
var total int64
if err := s.GetReplica().GetBuilder(&total, countQuery); err != nil {
return nil, 0, errors.Wrapf(err, "failed to count ChannelJoinRequests for user_id=%s", userId)
}
return rows, total, nil
}
// Update writes the mutable fields back. Id/ChannelId/UserId/CreateAt are
// immutable post-create — the partial-unique index relies on (ChannelId, UserId)
// being stable for the lifetime of a row.
func (s *SqlChannelJoinRequestStore) Update(req *model.ChannelJoinRequest) (*model.ChannelJoinRequest, error) {
req.PreUpdate()
if err := req.IsValid(); err != nil {
return nil, err
}
query := s.getQueryBuilder().
Update(channelJoinRequestsTable).
SetMap(map[string]any{
"Status": req.Status,
"Message": req.Message,
"DenialReason": req.DenialReason,
"UpdateAt": req.UpdateAt,
"ReviewedBy": req.ReviewedBy,
"ReviewedAt": req.ReviewedAt,
}).
Where(sq.Eq{"Id": req.Id})
res, err := s.GetMaster().ExecBuilder(query)
if err != nil {
return nil, errors.Wrapf(err, "failed to update ChannelJoinRequest with id=%s", req.Id)
}
n, err := res.RowsAffected()
if err != nil {
return nil, errors.Wrap(err, "failed to read RowsAffected on ChannelJoinRequest update")
}
if n == 0 {
return nil, store.NewErrNotFound("ChannelJoinRequest", req.Id)
}
return req, nil
}
func (s *SqlChannelJoinRequestStore) CountPending(channelId string) (int64, error) {
query := s.getQueryBuilder().
Select("COUNT(*)").
From(channelJoinRequestsTable).
Where(sq.Eq{
"ChannelId": channelId,
"Status": model.ChannelJoinRequestStatusPending,
})
var count int64
if err := s.GetReplica().GetBuilder(&count, query); err != nil {
return 0, errors.Wrapf(err, "failed to count pending ChannelJoinRequests for channel_id=%s", channelId)
}
return count, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestChannelJoinRequestStore(t *testing.T) {
StoreTest(t, storetest.TestChannelJoinRequestStore)
}

View File

@@ -0,0 +1,380 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"fmt"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SqlChannelMemberHistoryStore struct {
*SqlStore
channelMemberHistoryQuery sq.SelectBuilder
}
func newSqlChannelMemberHistoryStore(sqlStore *SqlStore) store.ChannelMemberHistoryStore {
s := &SqlChannelMemberHistoryStore{
SqlStore: sqlStore,
}
s.channelMemberHistoryQuery = s.getQueryBuilder().
Select(
"ChannelMemberHistory.ChannelId",
"ChannelMemberHistory.UserId",
"ChannelMemberHistory.JoinTime",
"ChannelMemberHistory.LeaveTime",
).
From("ChannelMemberHistory")
return s
}
func (s SqlChannelMemberHistoryStore) LogJoinEvent(userId string, channelId string, joinTime int64) error {
channelMemberHistory := &model.ChannelMemberHistory{
UserId: userId,
ChannelId: channelId,
JoinTime: joinTime,
}
if _, err := s.GetMaster().NamedExec(`INSERT INTO ChannelMemberHistory
(UserId, ChannelId, JoinTime)
VALUES
(:UserId, :ChannelId, :JoinTime)`, channelMemberHistory); err != nil {
return errors.Wrapf(err, "LogJoinEvent userId=%s channelId=%s joinTime=%d", userId, channelId, joinTime)
}
return nil
}
func (s SqlChannelMemberHistoryStore) LogLeaveEvent(rctx request.CTX, userId string, channelId string, leaveTime int64) error {
query, params, err := s.getQueryBuilder().
Update("ChannelMemberHistory").
Set("LeaveTime", leaveTime).
Where(sq.And{
sq.Eq{"UserId": userId},
sq.Eq{"ChannelId": channelId},
sq.Eq{"LeaveTime": nil},
}).ToSql()
if err != nil {
return errors.Wrap(err, "channel_member_history_to_sql")
}
sqlResult, err := s.GetMaster().Exec(query, params...)
if err != nil {
return errors.Wrapf(err, "LogLeaveEvent userId=%s channelId=%s leaveTime=%d", userId, channelId, leaveTime)
}
if rows, err := sqlResult.RowsAffected(); err == nil && rows != 1 {
// there was no join event to update - this is best effort, so no need to raise an error
rctx.Logger().Warn("Channel join event for user and channel not found", mlog.String("user", userId), mlog.String("channel", channelId))
}
return nil
}
// GetEverMembersInChannel returns user IDs that have at least one membership history row in the channel.
func (s SqlChannelMemberHistoryStore) GetEverMembersInChannel(channelID string, userIDs []string) ([]string, error) {
if len(userIDs) == 0 {
return []string{}, nil
}
query, args, err := s.getQueryBuilder().
Select("UserId").
Distinct().
From("ChannelMemberHistory").
Where(sq.And{
sq.Eq{"ChannelId": channelID},
sq.Eq{"UserId": userIDs},
}).
OrderBy("UserId ASC").
ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_member_history_to_sql")
}
everMembers := []string{}
if err := s.GetReplica().Select(&everMembers, query, args...); err != nil {
return nil, errors.Wrapf(err, "GetEverMembersInChannel channelId=%s users=%d", channelID, len(userIDs))
}
return everMembers, nil
}
func (s SqlChannelMemberHistoryStore) GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error) {
// ChannelMemberHistory has been in production for long enough that we are assuming the export period
// starts after the ChannelMemberHistory table was first introduced
subqueryPosts := s.getSubQueryBuilder().
Select("p.ChannelId AS ChannelId").
Distinct().
From("Posts AS p").
Where(
sq.And{
sq.GtOrEq{"p.UpdateAt": startTime},
sq.LtOrEq{"p.UpdateAt": endTime},
sq.NotLike{"p.Type": "system_%"},
})
subqueryCMH := s.getSubQueryBuilder().
Select("cmh.ChannelId AS ChannelId").
Distinct().
From("ChannelMemberHistory AS cmh").
Where(
sq.Or{
sq.And{
sq.GtOrEq{"cmh.JoinTime": startTime},
sq.LtOrEq{"cmh.JoinTime": endTime},
},
sq.And{
sq.GtOrEq{"cmh.LeaveTime": startTime},
sq.LtOrEq{"cmh.LeaveTime": endTime},
},
})
unionExpr, args, err := sq.Expr("(? UNION ?) AS cm", subqueryPosts, subqueryCMH).ToSql()
if err != nil {
return nil, errors.Wrap(err, "GetChannelsWithActivityDuring unionExpr to sql")
}
query, _, err := s.getQueryBuilder().
Select("ChannelId").
From(unionExpr).ToSql()
if err != nil {
return nil, errors.Wrap(err, "GetChannelsWithActivityDuring query to sql")
}
channelIds := make([]string, 0)
if err := s.GetReplica().Select(&channelIds, query, args...); err != nil {
return nil, err
}
return channelIds, nil
}
func (s SqlChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelIds []string) ([]*model.ChannelMemberHistoryResult, error) {
useChannelMemberHistory, err := s.hasDataAtOrBefore(startTime)
if err != nil {
return nil, errors.Wrapf(err, "hasDataAtOrBefore startTime=%d endTime=%d channelId=%v", startTime, endTime, channelIds)
}
if useChannelMemberHistory {
// the export period starts after the ChannelMemberHistory table was first introduced, so we can use the
// data from it for our export
channelMemberHistories, err2 := s.getFromChannelMemberHistoryTable(startTime, endTime, channelIds)
if err2 != nil {
return nil, errors.Wrapf(err2, "getFromChannelMemberHistoryTable startTime=%d endTime=%d channelId=%v", startTime, endTime, channelIds)
}
return channelMemberHistories, nil
}
// the export period starts before the ChannelMemberHistory table was introduced, so we need to fake the
// data by assuming that anybody who has ever joined the channel in question was present during the export period.
// this may not always be true, but it's better than saying that somebody wasn't there when they were
channelMemberHistories, err := s.getFromChannelMembersTable(startTime, endTime, channelIds)
if err != nil {
return nil, errors.Wrapf(err, "getFromChannelMembersTable startTime=%d endTime=%d channelId=%v", startTime, endTime, channelIds)
}
return channelMemberHistories, nil
}
func (s SqlChannelMemberHistoryStore) hasDataAtOrBefore(time int64) (bool, error) {
type NullableCountResult struct {
Min sql.NullInt64
}
query, _, err := s.getQueryBuilder().Select("MIN(JoinTime) as Min").From("ChannelMemberHistory").ToSql()
if err != nil {
return false, errors.Wrap(err, "channel_member_history_to_sql")
}
var result NullableCountResult
if err := s.GetReplica().Get(&result, query); err != nil {
return false, err
} else if result.Min.Valid {
return result.Min.Int64 <= time, nil
}
// if the result was null, there are no rows in the table, so there is no data from before
return false, nil
}
func (s SqlChannelMemberHistoryStore) getFromChannelMemberHistoryTable(startTime int64, endTime int64, channelIds []string) ([]*model.ChannelMemberHistoryResult, error) {
query := s.channelMemberHistoryQuery.
Column("u.Email AS \"Email\"").
Column("u.Username").
Column("Bots.UserId IS NOT NULL AS IsBot").
Column("u.DeleteAt AS UserDeleteAt").
Join("Users u ON ChannelMemberHistory.UserId = u.Id").
LeftJoin("Bots ON Bots.UserId = u.Id").
Where(sq.And{
sq.Eq{"ChannelMemberHistory.ChannelId": channelIds},
sq.LtOrEq{"ChannelMemberHistory.JoinTime": endTime},
sq.Or{
sq.Eq{"ChannelMemberHistory.LeaveTime": nil},
sq.GtOrEq{"ChannelMemberHistory.LeaveTime": startTime},
},
}).
OrderBy("ChannelMemberHistory.JoinTime ASC")
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_member_history_to_sql")
}
histories := []*model.ChannelMemberHistoryResult{}
if err := s.GetReplica().Select(&histories, queryString, args...); err != nil {
return nil, err
}
return histories, nil
}
func (s SqlChannelMemberHistoryStore) getFromChannelMembersTable(startTime int64, endTime int64, channelIds []string) ([]*model.ChannelMemberHistoryResult, error) {
query, args, err := s.getQueryBuilder().
Select(`ch.ChannelId, ch.UserId, u.Email AS "Email", u.Username, Bots.UserId IS NOT NULL AS IsBot, u.DeleteAt AS UserDeleteAt`).
Distinct().
From("ChannelMembers ch").
Join("Users u ON ch.UserId = u.id").
LeftJoin("Bots ON Bots.UserId = u.id").
Where(sq.Eq{"ch.ChannelId": channelIds}).ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_member_history_to_sql")
}
histories := []*model.ChannelMemberHistoryResult{}
if err := s.GetReplica().Select(&histories, query, args...); err != nil {
return nil, err
}
// we have to fill in the join/leave times, because that data doesn't exist in the channel members table
for _, channelMemberHistory := range histories {
channelMemberHistory.JoinTime = startTime
channelMemberHistory.LeaveTime = new(endTime)
}
return histories, nil
}
// PermanentDeleteBatchForRetentionPolicies deletes a batch of records which are affected by
// the global or a granular retention policy.
// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
func (s SqlChannelMemberHistoryStore) PermanentDeleteBatchForRetentionPolicies(retentionPolicyBatchConfigs model.RetentionPolicyBatchConfigs, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
builder := s.getQueryBuilder().
Select("ChannelMemberHistory.ChannelId, ChannelMemberHistory.UserId, ChannelMemberHistory.JoinTime").
From("ChannelMemberHistory")
return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
BaseBuilder: builder,
Table: "ChannelMemberHistory",
TimeColumn: "LeaveTime",
PrimaryKeys: []string{"ChannelId", "UserId", "JoinTime"},
ChannelIDTable: "ChannelMemberHistory",
NowMillis: retentionPolicyBatchConfigs.Now,
GlobalPolicyEndTime: retentionPolicyBatchConfigs.GlobalPolicyEndTime,
Limit: retentionPolicyBatchConfigs.Limit,
StoreDeletedIds: false,
}, s.SqlStore, cursor)
}
// DeleteOrphanedRows removes entries from ChannelMemberHistory when a corresponding channel no longer exists.
func (s SqlChannelMemberHistoryStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
const query = `
DELETE FROM ChannelMemberHistory WHERE ctid IN (
SELECT ChannelMemberHistory.ctid FROM ChannelMemberHistory
LEFT JOIN Channels ON ChannelMemberHistory.ChannelId = Channels.Id
WHERE Channels.Id IS NULL
LIMIT $1
)`
result, err := s.GetMaster().Exec(query, limit)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func (s SqlChannelMemberHistoryStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
innerSelect, args, err := s.getQueryBuilder().
Select("ctid").
From("ChannelMemberHistory").
Where(sq.And{
sq.NotEq{"LeaveTime": nil},
sq.LtOrEq{"LeaveTime": endTime},
}).Limit(uint64(limit)).
ToSql()
if err != nil {
return 0, errors.Wrap(err, "channel_member_history_to_sql")
}
query, _, err := s.getQueryBuilder().
Delete("ChannelMemberHistory").
Where(fmt.Sprintf(
"ctid IN (%s)", innerSelect,
)).ToSql()
if err != nil {
return 0, errors.Wrap(err, "channel_member_history_to_sql")
}
sqlResult, err := s.GetMaster().Exec(query, args...)
if err != nil {
return 0, errors.Wrapf(err, "PermanentDeleteBatch endTime=%d limit=%d", endTime, limit)
}
rowsAffected, err := sqlResult.RowsAffected()
if err != nil {
return 0, errors.Wrapf(err, "PermanentDeleteBatch endTime=%d limit=%d", endTime, limit)
}
return rowsAffected, nil
}
// GetMembershipChanges returns all membership events (joins and leaves) for a channel since the given timestamp.
// Uses inclusive comparison (>=) so events at the cursor timestamp are re-fetched rather than lost at batch
// boundaries. This may cause redundant re-sends when consecutive batches share a boundary timestamp, but the
// receiver is idempotent so duplicates are harmless. A composite cursor (timestamp + ID) like posts use would
// eliminate duplicates, but would require a schema change to SharedChannelRemotes; the current trade-off avoids that.
func (s SqlChannelMemberHistoryStore) GetMembershipChanges(channelID string, since int64, limit int) ([]*model.ChannelMemberHistory, error) {
query, args, err := s.getQueryBuilder().
Select("ChannelId", "UserId", "JoinTime", "LeaveTime").
From("ChannelMemberHistory").
Where(sq.And{
sq.Eq{"ChannelId": channelID},
sq.Or{
sq.GtOrEq{"JoinTime": since},
sq.And{
sq.NotEq{"LeaveTime": nil},
sq.GtOrEq{"LeaveTime": since},
},
},
}).
OrderBy("GREATEST(JoinTime, COALESCE(LeaveTime, 0)) ASC", "UserId ASC").
Limit(uint64(limit)).
ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_member_history_to_sql")
}
histories := []*model.ChannelMemberHistory{}
if err := s.GetReplica().Select(&histories, query, args...); err != nil {
return nil, errors.Wrapf(err, "GetMembershipChanges channelId=%s since=%d limit=%d", channelID, since, limit)
}
return histories, nil
}
// GetChannelsLeftSince returns list of channels that the user has left after a given time,
// but has not rejoined again.
func (s SqlChannelMemberHistoryStore) GetChannelsLeftSince(userID string, since int64) ([]string, error) {
query, params, err := s.getQueryBuilder().
Select("ChannelId").
From("ChannelMemberHistory").
GroupBy("ChannelId").
Where(sq.Eq{"UserId": userID}).
Having("MAX(LeaveTime) > MAX(JoinTime) AND MAX(LeaveTime) IS NOT NULL AND MAX(LeaveTime) >= ?", since).ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_member_history_to_sql")
}
channelIds := []string{}
err = s.GetReplica().Select(&channelIds, query, params...)
if err != nil {
return nil, errors.Wrapf(err, "GetChannelsLeftSince userId=%s since=%d", userID, since)
}
return channelIds, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestChannelMemberHistoryStore(t *testing.T) {
StoreTest(t, storetest.TestChannelMemberHistoryStore)
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestChannelStoreCategories(t *testing.T) {
StoreTestWithSqlStore(t, storetest.TestChannelStoreCategories)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,156 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type sqlClusterDiscoveryStore struct {
*SqlStore
clusterDiscoveryQuery sq.SelectBuilder
}
func newSqlClusterDiscoveryStore(sqlStore *SqlStore) store.ClusterDiscoveryStore {
s := &sqlClusterDiscoveryStore{
SqlStore: sqlStore,
}
s.clusterDiscoveryQuery = s.getQueryBuilder().
Select(
"Id",
"Type",
"ClusterName",
"Hostname",
"GossipPort",
"Port",
"CreateAt",
"LastPingAt",
).
From("ClusterDiscovery")
return s
}
func (s sqlClusterDiscoveryStore) Save(ClusterDiscovery *model.ClusterDiscovery) error {
ClusterDiscovery.PreSave()
if err := ClusterDiscovery.IsValid(); err != nil {
return err
}
if _, err := s.GetMaster().NamedExec(`
INSERT INTO
ClusterDiscovery
(Id, Type, ClusterName, Hostname, GossipPort, Port, CreateAt, LastPingAt)
VALUES
(:Id, :Type, :ClusterName, :Hostname, :GossipPort, :Port, :CreateAt, :LastPingAt)
`, ClusterDiscovery); err != nil {
return errors.Wrap(err, "failed to save ClusterDiscovery")
}
return nil
}
func (s sqlClusterDiscoveryStore) Delete(ClusterDiscovery *model.ClusterDiscovery) (bool, error) {
query := s.getQueryBuilder().
Delete("ClusterDiscovery").
Where(sq.Eq{"Type": ClusterDiscovery.Type}).
Where(sq.Eq{"ClusterName": ClusterDiscovery.ClusterName}).
Where(sq.Eq{"Hostname": ClusterDiscovery.Hostname})
queryString, args, err := query.ToSql()
if err != nil {
return false, errors.Wrap(err, "cluster_discovery_tosql")
}
res, err := s.GetMaster().Exec(queryString, args...)
if err != nil {
return false, errors.Wrap(err, "failed to delete ClusterDiscovery")
}
count, err := res.RowsAffected()
if err != nil {
return false, errors.Wrap(err, "failed to count rows affected")
}
return count != 0, nil
}
func (s sqlClusterDiscoveryStore) Exists(ClusterDiscovery *model.ClusterDiscovery) (bool, error) {
query := s.getQueryBuilder().
Select("COUNT(*)").
From("ClusterDiscovery").
Where(sq.Eq{"Type": ClusterDiscovery.Type}).
Where(sq.Eq{"ClusterName": ClusterDiscovery.ClusterName}).
Where(sq.Eq{"Hostname": ClusterDiscovery.Hostname})
queryString, args, err := query.ToSql()
if err != nil {
return false, errors.Wrap(err, "cluster_discovery_tosql")
}
var count int
if err := s.GetMaster().Get(&count, queryString, args...); err != nil {
return false, errors.Wrap(err, "failed to count ClusterDiscovery")
}
return count != 0, nil
}
func (s sqlClusterDiscoveryStore) GetAll(ClusterDiscoveryType, clusterName string) ([]*model.ClusterDiscovery, error) {
query := s.clusterDiscoveryQuery.
Where(sq.Eq{"Type": ClusterDiscoveryType}).
Where(sq.Eq{"ClusterName": clusterName}).
Where(sq.Gt{"LastPingAt": model.GetMillis() - model.CDSOfflineAfterMillis})
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "cluster_discovery_tosql")
}
list := []*model.ClusterDiscovery{}
if err := s.GetMaster().Select(&list, queryString, args...); err != nil {
return nil, errors.Wrap(err, "failed to find ClusterDiscovery")
}
return list, nil
}
func (s sqlClusterDiscoveryStore) SetLastPingAt(ClusterDiscovery *model.ClusterDiscovery) error {
query := s.getQueryBuilder().
Update("ClusterDiscovery").
Set("LastPingAt", model.GetMillis()).
Where(sq.Eq{"Type": ClusterDiscovery.Type}).
Where(sq.Eq{"ClusterName": ClusterDiscovery.ClusterName}).
Where(sq.Eq{"Hostname": ClusterDiscovery.Hostname})
queryString, args, err := query.ToSql()
if err != nil {
return errors.Wrap(err, "cluster_discovery_tosql")
}
if _, err := s.GetMaster().Exec(queryString, args...); err != nil {
return errors.Wrap(err, "failed to update ClusterDiscovery")
}
return nil
}
func (s sqlClusterDiscoveryStore) Cleanup() error {
query := s.getQueryBuilder().
Delete("ClusterDiscovery").
Where(sq.Lt{"LastPingAt": model.GetMillis() - model.CDSOfflineAfterMillis})
queryString, args, err := query.ToSql()
if err != nil {
return errors.Wrap(err, "cluster_discovery_tosql")
}
if _, err := s.GetMaster().Exec(queryString, args...); err != nil {
return errors.Wrap(err, "failed to delete ClusterDiscoveries")
}
return nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestClusterDiscoveryStore(t *testing.T) {
StoreTest(t, storetest.TestClusterDiscoveryStore)
}

View File

@@ -0,0 +1,261 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"fmt"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SqlCommandStore struct {
*SqlStore
commandColumns []string
commandsQuery sq.SelectBuilder
}
func newSqlCommandStore(sqlStore *SqlStore) store.CommandStore {
s := &SqlCommandStore{SqlStore: sqlStore}
s.commandColumns = []string{
"Id",
"Token",
"CreateAt",
"UpdateAt",
"DeleteAt",
"CreatorId",
"TeamId",
s.toReserveCase("trigger"),
"Method",
"Username",
"IconURL",
"AutoComplete",
"AutoCompleteDesc",
"AutoCompleteHint",
"DisplayName",
"Description",
"URL",
"PluginId",
}
s.commandsQuery = s.getQueryBuilder().
Select(s.commandColumns...).
From("Commands")
return s
}
func (s SqlCommandStore) Save(command *model.Command) (*model.Command, error) {
if command.Id != "" {
return nil, store.NewErrInvalidInput("Command", "CommandId", command.Id)
}
command.PreSave()
if err := command.IsValid(); err != nil {
return nil, err
}
insertQuery := s.getQueryBuilder().
Insert("Commands").
Columns(s.commandColumns...).
Values(
command.Id,
command.Token,
command.CreateAt,
command.UpdateAt,
command.DeleteAt,
command.CreatorId,
command.TeamId,
command.Trigger,
command.Method,
command.Username,
command.IconURL,
command.AutoComplete,
command.AutoCompleteDesc,
command.AutoCompleteHint,
command.DisplayName,
command.Description,
command.URL,
command.PluginId,
)
if _, err := s.GetMaster().ExecBuilder(insertQuery); err != nil {
return nil, errors.Wrapf(err, "insert: command_id=%s", command.Id)
}
return command, nil
}
func (s SqlCommandStore) Get(id string) (*model.Command, error) {
var command model.Command
query, args, err := s.commandsQuery.
Where(sq.Eq{"Id": id, "DeleteAt": 0}).ToSql()
if err != nil {
return nil, errors.Wrapf(err, "commands_tosql")
}
if err = s.GetReplica().Get(&command, query, args...); err == sql.ErrNoRows {
return nil, store.NewErrNotFound("Command", id)
} else if err != nil {
return nil, errors.Wrapf(err, "selectone: command_id=%s", id)
}
return &command, nil
}
func (s SqlCommandStore) GetByTeam(teamId string) ([]*model.Command, error) {
commands := []*model.Command{}
sql, args, err := s.commandsQuery.
Where(sq.Eq{"TeamId": teamId, "DeleteAt": 0}).ToSql()
if err != nil {
return nil, errors.Wrapf(err, "commands_tosql")
}
if err := s.GetReplica().Select(&commands, sql, args...); err != nil {
return nil, errors.Wrapf(err, "select: team_id=%s", teamId)
}
return commands, nil
}
func (s SqlCommandStore) GetByTrigger(teamId string, trigger string) (*model.Command, error) {
var command model.Command
query, args, err := s.commandsQuery.
Where(sq.Eq{"TeamId": teamId, "DeleteAt": 0, "\"trigger\"": trigger}).ToSql()
if err != nil {
return nil, errors.Wrapf(err, "commands_tosql")
}
if err := s.GetReplica().Get(&command, query, args...); err == sql.ErrNoRows {
errorId := "teamId=" + teamId + ", trigger=" + trigger
return nil, store.NewErrNotFound("Command", errorId)
} else if err != nil {
return nil, errors.Wrapf(err, "selectone: team_id=%s, trigger=%s", teamId, trigger)
}
return &command, nil
}
func (s SqlCommandStore) Delete(commandId string, time int64) error {
sql, args, err := s.getQueryBuilder().
Update("Commands").
SetMap(sq.Eq{"DeleteAt": time, "UpdateAt": time}).
Where(sq.Eq{"Id": commandId}).ToSql()
if err != nil {
return errors.Wrapf(err, "commands_tosql")
}
_, err = s.GetMaster().Exec(sql, args...)
if err != nil {
errors.Wrapf(err, "delete: command_id=%s", commandId)
}
return nil
}
func (s SqlCommandStore) PermanentDeleteByTeam(teamId string) error {
sql, args, err := s.getQueryBuilder().
Delete("Commands").
Where(sq.Eq{"TeamId": teamId}).ToSql()
if err != nil {
return errors.Wrapf(err, "commands_tosql")
}
_, err = s.GetMaster().Exec(sql, args...)
if err != nil {
return errors.Wrapf(err, "delete: team_id=%s", teamId)
}
return nil
}
func (s SqlCommandStore) PermanentDeleteByUser(userId string) error {
sql, args, err := s.getQueryBuilder().
Delete("Commands").
Where(sq.Eq{"CreatorId": userId}).ToSql()
if err != nil {
return errors.Wrapf(err, "commands_tosql")
}
_, err = s.GetMaster().Exec(sql, args...)
if err != nil {
return errors.Wrapf(err, "delete: user_id=%s", userId)
}
return nil
}
func (s SqlCommandStore) Update(cmd *model.Command) (*model.Command, error) {
cmd.UpdateAt = model.GetMillis()
if err := cmd.IsValid(); err != nil {
return nil, err
}
query := s.getQueryBuilder().
Update("Commands").
Set("Token", cmd.Token).
Set("CreateAt", cmd.CreateAt).
Set("UpdateAt", cmd.UpdateAt).
Set("CreatorId", cmd.CreatorId).
Set("TeamId", cmd.TeamId).
Set("Method", cmd.Method).
Set("Username", cmd.Username).
Set("IconURL", cmd.IconURL).
Set("AutoComplete", cmd.AutoComplete).
Set("AutoCompleteDesc", cmd.AutoCompleteDesc).
Set("AutoCompleteHint", cmd.AutoCompleteHint).
Set("DisplayName", cmd.DisplayName).
Set("Description", cmd.Description).
Set("URL", cmd.URL).
Set("PluginId", cmd.PluginId).
Where(sq.Eq{"Id": cmd.Id})
// Trigger is a keyword
query = query.Set(s.toReserveCase("trigger"), cmd.Trigger)
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "commands_tosql")
}
res, err := s.GetMaster().Exec(queryString, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to update commands")
}
count, err := res.RowsAffected()
if err != nil {
return nil, errors.Wrap(err, "error while getting rows_affected")
}
if count > 1 {
return nil, fmt.Errorf("unexpected count while updating commands: count=%d, Id=%s", count, cmd.Id)
}
return cmd, nil
}
func (s SqlCommandStore) AnalyticsCommandCount(teamId string) (int64, error) {
query := s.getQueryBuilder().
Select("COUNT(*)").
From("Commands").
Where(sq.Eq{"DeleteAt": 0})
if teamId != "" {
query = query.Where(sq.Eq{"TeamId": teamId})
}
sql, args, err := query.ToSql()
if err != nil {
return 0, errors.Wrapf(err, "commands_tosql")
}
var c int64
err = s.GetReplica().Get(&c, sql, args...)
if err != nil {
return 0, errors.Wrapf(err, "unable to count the commands: team_id=%s", teamId)
}
return c, nil
}

View File

@@ -0,0 +1,14 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
)
func TestCommandStore(t *testing.T) {
StoreTest(t, storetest.TestCommandStore)
}

View File

@@ -0,0 +1,137 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
type SqlCommandWebhookStore struct {
*SqlStore
commandWebhookColumns []string
commandWebhookQuery sq.SelectBuilder
}
func newSqlCommandWebhookStore(sqlStore *SqlStore) store.CommandWebhookStore {
s := &SqlCommandWebhookStore{
SqlStore: sqlStore,
}
s.commandWebhookColumns = []string{
"Id",
"CreateAt",
"CommandId",
"UserId",
"ChannelId",
"RootId",
"UseCount",
}
s.commandWebhookQuery = s.getQueryBuilder().
Select(s.commandWebhookColumns...).
From("CommandWebhooks")
return s
}
func (s SqlCommandWebhookStore) Save(webhook *model.CommandWebhook) (*model.CommandWebhook, error) {
if webhook.Id != "" {
return nil, store.NewErrInvalidInput("CommandWebhook", "id", webhook.Id)
}
webhook.PreSave()
if err := webhook.IsValid(); err != nil {
return nil, err
}
insertQuery := s.getQueryBuilder().
Insert("CommandWebhooks").
Columns(s.commandWebhookColumns...).
Values(
webhook.Id,
webhook.CreateAt,
webhook.CommandId,
webhook.UserId,
webhook.ChannelId,
webhook.RootId,
webhook.UseCount,
)
if _, err := s.GetMaster().ExecBuilder(insertQuery); err != nil {
return nil, errors.Wrapf(err, "save: id=%s", webhook.Id)
}
return webhook, nil
}
func (s SqlCommandWebhookStore) Get(id string) (*model.CommandWebhook, error) {
var webhook model.CommandWebhook
exptime := model.GetMillis() - model.CommandWebhookLifetime
query := s.commandWebhookQuery.
Where(sq.Eq{"Id": id}).
Where(sq.Gt{"CreateAt": exptime})
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "get_tosql")
}
if err := s.GetReplica().Get(&webhook, queryString, args...); err != nil {
if err == sql.ErrNoRows {
return nil, store.NewErrNotFound("CommandWebhook", id)
}
return nil, errors.Wrapf(err, "get: id=%s", id)
}
return &webhook, nil
}
func (s SqlCommandWebhookStore) TryUse(id string, limit int) error {
query := s.getQueryBuilder().
Update("CommandWebhooks").
Set("UseCount", sq.Expr("UseCount + 1")).
Where(sq.Eq{"Id": id}).
Where(sq.Lt{"UseCount": limit})
queryString, args, err := query.ToSql()
if err != nil {
return errors.Wrap(err, "tryuse_tosql")
}
if sqlResult, err := s.GetMaster().Exec(queryString, args...); err != nil {
return errors.Wrapf(err, "tryuse: id=%s limit=%d", id, limit)
} else if rows, err := sqlResult.RowsAffected(); rows == 0 {
return store.NewErrInvalidInput("CommandWebhook", "id", id).Wrap(err)
}
return nil
}
func (s SqlCommandWebhookStore) Cleanup() {
exptime := model.GetMillis() - model.CommandWebhookLifetime
query := s.getQueryBuilder().
Delete("CommandWebhooks").
Where(sq.Lt{"CreateAt": exptime})
queryString, args, err := query.ToSql()
if err != nil {
mlog.Error("Failed to build query when trying to perform a cleanup in command webhook store.")
return
}
if _, err := s.GetMaster().Exec(queryString, args...); err != nil {
mlog.Error("Unable to cleanup command webhook store.")
}
}

Some files were not shown because too many files have changed in this diff Show More