// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. package app import ( "cmp" "encoding/csv" "encoding/json" "fmt" "io" "net/http" "slices" "strings" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/public/shared/i18n" "github.com/mattermost/mattermost/server/public/shared/mlog" "github.com/mattermost/mattermost/server/public/shared/request" ) const exposureProfileBatchSize = 1000 var exposurePreambleSanitizer = strings.NewReplacer("\r\n", " ", "\n", " ", "\r", " ", ",", " ", ";", " ", "\t", " ") func (a *App) ComputePostExposure(rctx request.CTX, postID string) (*model.PostExposureReport, *model.AppError) { post, appErr := a.GetSinglePost(rctx, postID, true) if appErr != nil { return nil, appErr } if post.OriginalId != "" { return nil, model.NewAppError("ComputePostExposure", "app.data_spillage.exposure.edit_history_post.app_error", nil, "", http.StatusBadRequest) } channel, appErr := a.GetChannel(rctx, post.ChannelId) if appErr != nil { return nil, appErr } if channel.Type == model.ChannelTypeDirect || channel.Type == model.ChannelTypeGroup { return nil, model.NewAppError("ComputePostExposure", "app.data_spillage.exposure.unsupported_channel_type.app_error", nil, "", http.StatusBadRequest) } windowEnd, appErr := a.getPostFlagTime(rctx, post.Id) if appErr != nil { return nil, appErr } report := &model.PostExposureReport{ Version: model.PostExposureReportVersion, PostID: post.Id, ChannelID: channel.Id, ChannelName: channel.DisplayName, ChannelType: channel.Type, TeamID: channel.TeamId, WindowStart: post.CreateAt, WindowEnd: windowEnd, GeneratedAt: model.GetMillis(), Entries: []*model.PostExposureReportEntry{}, } // Data source 1: who was in the channel while the post was live. histories, err := a.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(post.CreateAt, windowEnd, []string{channel.Id}) if err != nil { return nil, model.NewAppError("ComputePostExposure", "app.data_spillage.exposure.get_channel_members.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } // GetUsersInChannelDuring returns one row per membership interval, so a user who left // and rejoined appears more than once. The report is a list of users, so collapse them. memberUserIDs := make([]string, 0, len(histories)) seen := make(map[string]bool, len(histories)) for _, h := range histories { if h.IsBot || seen[h.UserId] { continue } seen[h.UserId] = true memberUserIDs = append(memberUserIDs, h.UserId) } if len(memberUserIDs) == 0 { return report, nil } // Data source 2: channel read state. lastViewedByUser, appErr := a.getChannelLastViewedAt(rctx, channel.Id) if appErr != nil { return nil, appErr } users, appErr := a.getUsersProfiles(rctx, memberUserIDs) if appErr != nil { return nil, appErr } for _, userID := range memberUserIDs { user, ok := users[userID] if !ok { rctx.Logger().Warn("Skipping exposure report entry for a user that no longer exists", mlog.String("user_id", userID), mlog.String("post_id", post.Id)) continue } entry := &model.PostExposureReportEntry{ UserID: user.Id, Username: user.Username, UserEmail: user.Email, IsGuest: user.IsGuest(), IsRemote: user.IsRemote(), IsDeactivated: user.DeleteAt != 0, WasChannelMember: true, } if lastViewedAt, isMember := lastViewedByUser[userID]; isMember { entry.LastViewedAt = model.NewPointer(lastViewedAt) } if !entry.IsDeactivated { lastActivityAt, appErr := a.getUserLastActivityAt(rctx, userID) if appErr != nil { return nil, appErr } entry.LastActivityAt = lastActivityAt } report.Entries = append(report.Entries, entry) } slices.SortFunc(report.Entries, func(x, y *model.PostExposureReportEntry) int { return cmp.Or(cmp.Compare(x.Username, y.Username), cmp.Compare(x.UserID, y.UserID)) }) return report, nil } func (a *App) getPostFlagTime(rctx request.CTX, postID string) (int64, *model.AppError) { value, appErr := a.GetPostContentFlaggingPropertyValue(rctx, postID, contentFlaggingPropertyNameReportingTime) if appErr != nil { if appErr.StatusCode == http.StatusNotFound { return 0, model.NewAppError("getPostFlagTime", "app.data_spillage.exposure.missing_reporting_time.app_error", nil, "", http.StatusInternalServerError) } return 0, appErr } var reportingTime int64 if err := json.Unmarshal(value.Value, &reportingTime); err != nil { return 0, model.NewAppError("getPostFlagTime", "app.data_spillage.exposure.missing_reporting_time.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } if reportingTime <= 0 { return 0, model.NewAppError("getPostFlagTime", "app.data_spillage.exposure.missing_reporting_time.app_error", nil, "", http.StatusInternalServerError) } return reportingTime, nil } func (a *App) getChannelLastViewedAt(rctx request.CTX, channelID string) (map[string]int64, *model.AppError) { lastViewedByUser := map[string]int64{} afterUserID := "" for { members, err := a.Srv().Store().Channel().GetMembersWithLastViewedAtSince(rctx, channelID, 0, afterUserID, model.ChannelMemberLastViewedMaxPerPage) if err != nil { return nil, model.NewAppError("getChannelLastViewedAt", "app.data_spillage.exposure.get_possible_viewers.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } if len(members) == 0 { break } for _, m := range members { lastViewedByUser[m.UserId] = m.LastViewedAt } afterUserID = members[len(members)-1].UserId if len(members) < model.ChannelMemberLastViewedMaxPerPage { break } } return lastViewedByUser, nil } func (a *App) getUsersProfiles(rctx request.CTX, userIDs []string) (map[string]*model.User, *model.AppError) { profiles := make(map[string]*model.User, len(userIDs)) for batch := range slices.Chunk(userIDs, exposureProfileBatchSize) { users, appErr := a.GetUsers(rctx, batch) if appErr != nil { return nil, appErr } for _, u := range users { profiles[u.Id] = u } } return profiles, nil } func (a *App) getUserLastActivityAt(rctx request.CTX, userID string) (*int64, *model.AppError) { sessions, appErr := a.GetSessions(rctx, userID) if appErr != nil { return nil, model.NewAppError("getUserLastActivityAt", "app.data_spillage.exposure.get_last_activity.app_error", nil, "", http.StatusInternalServerError).Wrap(appErr) } if len(sessions) == 0 { return nil, nil } var lastActivityAt int64 for _, session := range sessions { if session.LastActivityAt > lastActivityAt { lastActivityAt = session.LastActivityAt } } return model.NewPointer(lastActivityAt), nil } func WritePostExposureCSV(w io.Writer, report *model.PostExposureReport, T i18n.TranslateFunc) error { if err := writePostExposurePreamble(w, report, T); err != nil { return err } cw := csv.NewWriter(w) if err := cw.Write(model.PostExposureReportCSVHeader(T)); err != nil { return err } for _, entry := range report.Entries { if err := cw.Write(entry.ToCSVRow(T)); err != nil { return err } } cw.Flush() return cw.Error() } // writePostExposurePreamble writes the report metadata as CSV comment lines above the header // row, so the artifact is self-describing without a sidecar file. Readers skip these with // the standard comment option (Go's csv.Reader Comment: '#', pandas comment='#'). func writePostExposurePreamble(w io.Writer, report *model.PostExposureReport, T i18n.TranslateFunc) error { lines := [][2]string{ {T("app.data_spillage.exposure.meta.report_version"), report.Version}, {T("app.data_spillage.exposure.meta.post_id"), report.PostID}, {T("app.data_spillage.exposure.meta.channel"), fmt.Sprintf("%s (%s)", report.ChannelName, report.ChannelID)}, {T("app.data_spillage.exposure.meta.window_start"), model.FormatExposureTime(report.WindowStart)}, {T("app.data_spillage.exposure.meta.window_end"), model.FormatExposureTime(report.WindowEnd)}, {T("app.data_spillage.exposure.meta.generated_at"), model.FormatExposureTime(report.GeneratedAt)}, {T("app.data_spillage.exposure.meta.total_users"), fmt.Sprintf("%d", len(report.Entries))}, } for _, line := range lines { if _, err := fmt.Fprintf(w, "# %s: %s\n", line[0], exposurePreambleSanitizer.Replace(line[1])); err != nil { return err } } return nil }