snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)

server/public/shared/i18n/i18n.go

@@ -0,0 +1,354 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package i18n
+
+import (
+	"fmt"
+	"html/template"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/mattermost/go-i18n/i18n"
+	"github.com/mattermost/go-i18n/i18n/bundle"
+
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+// mut is used to protect other global variables from concurrent access.
+// This should only be a concern in parallel tests.
+var mut sync.Mutex
+
+const defaultLocale = "en"
+
+// TranslateFunc is the type of the translate functions
+type TranslateFunc func(translationID string, args ...any) string
+
+// TranslationFuncByLocal is the type of function that takes local as a string and returns the translation function
+type TranslationFuncByLocal func(locale string) TranslateFunc
+
+var (
+	t        TranslateFunc
+	tDefault TranslateFunc
+)
+
+// T is the translate function using the default server language as fallback language
+var T TranslateFunc = func(translationID string, args ...any) string {
+	mut.Lock()
+	defer mut.Unlock()
+
+	if t == nil {
+		return translationID
+	}
+
+	return t(translationID, args...)
+}
+
+// TranslationId is a no-op translation implementation for ensuring the string is retained in translation files
+var TranslationId TranslateFunc = func(translationID string, args ...any) string {
+	return translationID
+}
+
+// TDefault is the translate function using english as fallback language
+var TDefault TranslateFunc = func(translationID string, args ...any) string {
+	mut.Lock()
+	defer mut.Unlock()
+
+	if tDefault == nil {
+		return translationID
+	}
+
+	return t(translationID, args...)
+}
+
+var locales = make(map[string]string)
+
+// supportedLocales is a hard-coded list of locales considered ready for production use. It must
+// be kept in sync with ../../../../webapp/channels/src/i18n/i18n.ts.
+var supportedLocales = []string{
+	"de",
+	"en",
+	"en-AU",
+	"es",
+	"fr",
+	"it",
+	"hu",
+	"nl",
+	"pl",
+	"pt-BR",
+	"ro",
+	"sv",
+	"vi",
+	"tr",
+	"bg",
+	"ru",
+	"uk",
+	"fa",
+	"ko",
+	"zh-CN",
+	"zh-TW",
+	"ja",
+}
+
+var (
+	defaultServerLocale string
+	defaultClientLocale string
+)
+
+// TranslationsPreInit loads translations from filesystem if they are not
+// loaded already and assigns english while loading server config
+func TranslationsPreInit(translationsDir string) error {
+	mut.Lock()
+	defer mut.Unlock()
+	if t != nil {
+		return nil
+	}
+
+	// Set T even if we fail to load the translations. Lots of shutdown handling code will
+	// segfault trying to handle the error, and the untranslated IDs are strictly better.
+	t = tfuncWithFallback(defaultLocale)
+	tDefault = tfuncWithFallback(defaultLocale)
+
+	return initTranslationsWithDir(translationsDir)
+}
+
+// TranslationsPreInitFromFileBytes loads translations from a buffer -- useful if
+// we need to initialize i18n from an embedded i18n file (e.g., from a CLI tool)
+func TranslationsPreInitFromFileBytes(filename string, buf []byte) error {
+	mut.Lock()
+	defer mut.Unlock()
+	if t != nil {
+		return nil
+	}
+
+	// Set T even if we fail to load the translations. Lots of shutdown handling code will
+	// segfault trying to handle the error, and the untranslated IDs are strictly better.
+	t = tfuncWithFallback(defaultLocale)
+	tDefault = tfuncWithFallback(defaultLocale)
+
+	locale := strings.Split(filename, ".")[0]
+	if !isSupportedLocale(locale) {
+		return fmt.Errorf("locale not supported: %s", locale)
+	}
+
+	locales[locale] = filename
+
+	return i18n.ParseTranslationFileBytes(filename, buf)
+}
+
+// InitTranslations set the defaults configured in the server and initialize
+// the T function using the server default as fallback language
+func InitTranslations(serverLocale, clientLocale string) error {
+	mut.Lock()
+	defaultServerLocale = serverLocale
+	defaultClientLocale = clientLocale
+	mut.Unlock()
+
+	tfn, err := GetTranslationsBySystemLocale()
+
+	mut.Lock()
+	t = tfn
+	mut.Unlock()
+
+	return err
+}
+
+func initTranslationsWithDir(dir string) error {
+	files, _ := os.ReadDir(dir)
+	for _, f := range files {
+		if filepath.Ext(f.Name()) == ".json" {
+			filename := f.Name()
+
+			locale := strings.Split(filename, ".")[0]
+			if !isSupportedLocale(locale) {
+				continue
+			}
+
+			locales[locale] = filepath.Join(dir, filename)
+
+			if err := i18n.LoadTranslationFile(filepath.Join(dir, filename)); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// GetTranslationFuncForDir loads translations from the filesystem into a new instance of the bundle.
+// It returns a function to access loaded translations.
+func GetTranslationFuncForDir(dir string) (TranslationFuncByLocal, error) {
+	availableLocals := make(map[string]string)
+	bundle := bundle.New()
+	files, _ := os.ReadDir(dir)
+	for _, f := range files {
+		if filepath.Ext(f.Name()) != ".json" {
+			continue
+		}
+
+		locale := strings.Split(f.Name(), ".")[0]
+		if !isSupportedLocale(locale) {
+			continue
+		}
+
+		filename := f.Name()
+		availableLocals[locale] = filepath.Join(dir, filename)
+		if err := bundle.LoadTranslationFile(filepath.Join(dir, filename)); err != nil {
+			return nil, err
+		}
+	}
+
+	return func(locale string) TranslateFunc {
+		if _, ok := availableLocals[locale]; !ok {
+			locale = defaultLocale
+		}
+
+		t, _ := bundle.Tfunc(locale)
+		return func(translationID string, args ...any) string {
+			if translated := t(translationID, args...); translated != translationID {
+				return translated
+			}
+
+			t, _ := bundle.Tfunc(defaultLocale)
+			return t(translationID, args...)
+		}
+	}, nil
+}
+
+func GetTranslationsBySystemLocale() (TranslateFunc, error) {
+	mut.Lock()
+	defer mut.Unlock()
+	locale := defaultServerLocale
+	if _, ok := locales[locale]; !ok {
+		mlog.Warn("Failed to load system translations for selected locale, attempting to fall back to default", mlog.String("locale", locale), mlog.String("default_locale", defaultLocale))
+		locale = defaultLocale
+	}
+
+	if !isSupportedLocale(locale) {
+		mlog.Warn("Selected locale is unsupported, attempting to fall back to default", mlog.String("locale", locale), mlog.String("default_locale", defaultLocale))
+		locale = defaultLocale
+	}
+
+	if locales[locale] == "" {
+		return nil, fmt.Errorf("failed to load system translations for '%v'", defaultLocale)
+	}
+
+	translations := tfuncWithFallback(locale)
+	if translations == nil {
+		return nil, fmt.Errorf("failed to load system translations")
+	}
+
+	mlog.Info("Loaded system translations", mlog.String("for locale", locale), mlog.String("from locale", locales[locale]))
+	return translations, nil
+}
+
+// GetUserTranslations get the translation function for an specific locale
+func GetUserTranslations(locale string) TranslateFunc {
+	mut.Lock()
+	defer mut.Unlock()
+	if _, ok := locales[locale]; !ok {
+		locale = defaultLocale
+	}
+
+	translations := tfuncWithFallback(locale)
+	return translations
+}
+
+// GetTranslationsAndLocaleFromRequest return the translation function and the
+// locale based on a request headers
+func GetTranslationsAndLocaleFromRequest(r *http.Request) (TranslateFunc, string) {
+	mut.Lock()
+	defer mut.Unlock()
+	// This is for checking against locales like pt_BR or zn_CN
+	headerLocaleFull := strings.Split(r.Header.Get("Accept-Language"), ",")[0]
+	// This is for checking against locales like en, es
+	headerLocale := strings.Split(strings.Split(r.Header.Get("Accept-Language"), ",")[0], "-")[0]
+	defaultLocale := defaultClientLocale
+	if locales[headerLocaleFull] != "" {
+		translations := tfuncWithFallback(headerLocaleFull)
+		return translations, headerLocaleFull
+	} else if locales[headerLocale] != "" {
+		translations := tfuncWithFallback(headerLocale)
+		return translations, headerLocale
+	} else if locales[defaultLocale] != "" {
+		translations := tfuncWithFallback(defaultLocale)
+		return translations, headerLocale
+	}
+
+	translations := tfuncWithFallback(defaultLocale)
+	return translations, defaultLocale
+}
+
+// GetSupportedLocales return a map of locale code and the file path with the
+// translations
+func GetSupportedLocales() map[string]string {
+	mut.Lock()
+	defer mut.Unlock()
+	return locales
+}
+
+func tfuncWithFallback(pref string) TranslateFunc {
+	t, _ := i18n.Tfunc(pref)
+	return func(translationID string, args ...any) string {
+		if translated := t(translationID, args...); translated != translationID {
+			return translated
+		}
+
+		t, _ := i18n.Tfunc(defaultLocale)
+		return t(translationID, args...)
+	}
+}
+
+// TranslateAsHTML translates the translationID provided and return a
+// template.HTML object
+func TranslateAsHTML(t TranslateFunc, translationID string, args map[string]any) template.HTML {
+	message := t(translationID, escapeForHTML(args))
+	message = strings.Replace(message, "[[", "<strong>", -1)
+	message = strings.Replace(message, "]]", "</strong>", -1)
+	return template.HTML(message)
+}
+
+func escapeForHTML(arg any) any {
+	switch typedArg := arg.(type) {
+	case string:
+		return template.HTMLEscapeString(typedArg)
+	case *string:
+		return template.HTMLEscapeString(*typedArg)
+	case int:
+		return typedArg
+	case int64:
+		return typedArg
+	case float64:
+		return typedArg
+	case map[string]any:
+		safeArg := make(map[string]any, len(typedArg))
+		for key, value := range typedArg {
+			safeArg[key] = escapeForHTML(value)
+		}
+		return safeArg
+	default:
+		mlog.Warn(
+			"Unable to escape value for HTML template",
+			mlog.Any("html_template", arg),
+			mlog.String("template_type", reflect.ValueOf(arg).Type().String()),
+		)
+		return ""
+	}
+}
+
+// IdentityTfunc returns a translation function that don't translate, only
+// returns the same id
+func IdentityTfunc() TranslateFunc {
+	return func(translationID string, args ...any) string {
+		return translationID
+	}
+}
+
+func isSupportedLocale(locale string) bool {
+	return slices.Contains(supportedLocales, locale)
+}

server/public/shared/i18n/i18n_test.go

@@ -0,0 +1,298 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package i18n
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/mattermost/go-i18n/i18n/bundle"
+	"github.com/mattermost/go-i18n/i18n/language"
+	"github.com/mattermost/go-i18n/i18n/translation"
+	"github.com/mattermost/mattermost/server/public/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var htmlTestTranslationBundle *bundle.Bundle
+
+func init() {
+	htmlTestTranslationBundle = bundle.New()
+	fooBold, _ := translation.NewTranslation(map[string]any{
+		"id":          "foo.bold",
+		"translation": "<p>[[{{ .Foo }}]]</p>",
+	})
+	htmlTestTranslationBundle.AddTranslation(&language.Language{Tag: "en"}, fooBold)
+}
+
+func TestTranslateAsHTML(t *testing.T) {
+	assert.EqualValues(t, "<p><strong>&lt;i&gt;foo&lt;/i&gt;</strong></p>", TranslateAsHTML(TranslateFunc(htmlTestTranslationBundle.MustTfunc("en")), "foo.bold", map[string]any{
+		"Foo": "<i>foo</i>",
+	}))
+}
+
+func TestEscapeForHTML(t *testing.T) {
+	stringForPointer := "<b>abc</b>"
+	for name, tc := range map[string]struct {
+		In       any
+		Expected any
+	}{
+		"NoHTML": {
+			In:       "abc",
+			Expected: "abc",
+		},
+		"String": {
+			In:       "<b>abc</b>",
+			Expected: "&lt;b&gt;abc&lt;/b&gt;",
+		},
+		"StringPointer": {
+			In:       &stringForPointer,
+			Expected: "&lt;b&gt;abc&lt;/b&gt;",
+		},
+		"Map": {
+			In: map[string]any{
+				"abc": "abc",
+				"123": "<b>123</b>",
+			},
+			Expected: map[string]any{
+				"abc": "abc",
+				"123": "&lt;b&gt;123&lt;/b&gt;",
+			},
+		},
+		"Int": {
+			In:       59,
+			Expected: 59,
+		},
+		"Int64": {
+			In:       int64(59),
+			Expected: int64(59),
+		},
+		"Float64": {
+			In:       3.14,
+			Expected: 3.14,
+		},
+		"Unsupported": {
+			In:       struct{ string }{"<b>abc</b>"},
+			Expected: "",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, tc.Expected, escapeForHTML(tc.In))
+		})
+	}
+}
+
+func TestInitTranslationsWithDir(t *testing.T) {
+	i18nDir, found := utils.FindDir("server/i18n")
+	require.True(t, found, "unable to find i18n dir")
+
+	setup := func(t *testing.T, localesToCopy map[string]string) string {
+		tempDir, err := os.MkdirTemp(os.TempDir(), "TestGetTranslationFuncForDir")
+		require.NoError(t, err, "unable to create temporary directory")
+
+		t.Cleanup(func() {
+			err = os.RemoveAll(tempDir)
+			require.NoError(t, err)
+		})
+
+		for locale, fromLocale := range localesToCopy {
+			err = utils.CopyFile(
+				filepath.Join(i18nDir, fmt.Sprintf("%s.json", fromLocale)),
+				filepath.Join(tempDir, fmt.Sprintf("%s.json", locale)),
+			)
+			require.NoError(t, err)
+		}
+
+		return tempDir
+	}
+
+	t.Run("unsupported locale ignored", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		err := initTranslationsWithDir(tempDir)
+		require.NoError(t, err)
+
+		_, found := locales["zz"]
+		require.False(t, found, "should have ignored unsupported locale")
+	})
+
+	t.Run("malformed, unsupported locale ignored", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "xx.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		err = initTranslationsWithDir(tempDir)
+		require.NoError(t, err)
+
+		_, found := locales["xx"]
+		require.False(t, found, "should have ignored malformed, unsupported locale")
+	})
+
+	t.Run("malformed, supported locale causes error", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "en.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		err = initTranslationsWithDir(tempDir)
+		require.Error(t, err, "should have failed to load malformed, supported locale")
+	})
+
+	t.Run("known locales loaded ", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr"})
+
+		err := initTranslationsWithDir(tempDir)
+		require.NoError(t, err)
+
+		_, found := locales["en"]
+		require.True(t, found, "should have found en locale")
+		_, found = locales["fr"]
+		require.True(t, found, "should have found fr locale")
+		_, found = locales["es"]
+		require.False(t, found, "should not have found unloaded es locale")
+	})
+}
+
+func TestGetTranslationFuncForDir(t *testing.T) {
+	i18nDir, found := utils.FindDir("server/i18n")
+	require.True(t, found, "unable to find i18n dir")
+
+	setup := func(t *testing.T, localesToCopy map[string]string) string {
+		tempDir, err := os.MkdirTemp(os.TempDir(), "TestGetTranslationFuncForDir")
+		require.NoError(t, err, "unable to create temporary directory")
+
+		t.Cleanup(func() {
+			err = os.RemoveAll(tempDir)
+			require.NoError(t, err)
+		})
+
+		for locale, fromLocale := range localesToCopy {
+			err = utils.CopyFile(
+				filepath.Join(i18nDir, fmt.Sprintf("%s.json", fromLocale)),
+				filepath.Join(tempDir, fmt.Sprintf("%s.json", locale)),
+			)
+			require.NoError(t, err)
+		}
+
+		return tempDir
+	}
+
+	t.Run("unknown locale falls back to english", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "December", translationFunc("unknown")("December"))
+	})
+
+	t.Run("unsupported locale falls back to english", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "December", translationFunc("zz")("December"))
+	})
+
+	t.Run("malformed, unsupported locale ignored and falls back to english", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "xx.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "December", translationFunc("xx")("December"))
+	})
+
+	t.Run("malformed, supported locale causes error", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "en.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.Error(t, err)
+		require.Nil(t, translationFunc)
+	})
+
+	t.Run("known locale matches", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr"})
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "Décembre", translationFunc("fr")("December"))
+		require.Equal(t, "December", translationFunc("en")("December"))
+	})
+}
+
+// localeFilesIn returns the locale codes of the *.json catalogs in dir.
+func localeFilesIn(t *testing.T, dir string) []string {
+	t.Helper()
+
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+
+	var locales []string
+	for _, entry := range entries {
+		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
+			continue
+		}
+		locales = append(locales, strings.TrimSuffix(entry.Name(), ".json"))
+	}
+
+	return locales
+}
+
+// webappLanguageValues returns the locale codes in the webapp's languages map.
+func webappLanguageValues(t *testing.T, path string) []string {
+	t.Helper()
+
+	contents, err := os.ReadFile(path)
+	require.NoError(t, err)
+
+	block := regexp.MustCompile(`(?s)export const languages = \{.*?\n\};`).Find(contents)
+	require.NotNil(t, block, "could not locate the languages map in %s", path)
+
+	var locales []string
+	for _, match := range regexp.MustCompile(`value: '([^']+)'`).FindAllSubmatch(block, -1) {
+		locales = append(locales, string(match[1]))
+	}
+
+	return locales
+}
+
+// supportedLocales is hand-maintained in three other places: the catalogs
+// shipped by the server, the catalogs shipped by the webapp, and the webapp's
+// own languages map. Nothing but this test keeps the four in step, and when
+// they drift the symptom is a language that is offered but cannot load, or a
+// catalog that is translated but never reachable.
+func TestSupportedLocalesAreInSync(t *testing.T) {
+	t.Run("server translation files", func(t *testing.T) {
+		assert.ElementsMatch(t, supportedLocales, localeFilesIn(t, "../../../i18n"),
+			"supportedLocales and server/i18n/*.json disagree")
+	})
+
+	t.Run("webapp translation files", func(t *testing.T) {
+		assert.ElementsMatch(t, supportedLocales, localeFilesIn(t, "../../../../webapp/channels/src/i18n"),
+			"supportedLocales and webapp/channels/src/i18n/*.json disagree")
+	})
+
+	t.Run("webapp languages map", func(t *testing.T) {
+		assert.ElementsMatch(t, supportedLocales, webappLanguageValues(t, "../../../../webapp/channels/src/i18n/i18n.ts"),
+			"supportedLocales and the webapp languages map disagree")
+	})
+}