Skip to content

Commit a4df01b

Browse files
authored
Optimize total count of feed when loading activities in user dashboard. (go-gitea#33841)
Two SQLs are very slow when `action` table have over 5M records. ``` database duration=1.8881s db.sql="SELECT created_unix DIV 900 * 900 AS timestamp, count(user_id) as contributions FROM `action` WHERE user_id=? AND act_user_id=? AND (created_unix > ?) GROUP BY timestamp ORDER BY timestamp" database duration=1.5408s db.sql="SELECT count(*) FROM `action` WHERE (user_id = ?) AND (is_deleted = ?)" ``` This will cache the count for the first loading or when the activities changed.
1 parent ef09705 commit a4df01b

File tree

4 files changed

+46
-7
lines changed

4 files changed

+46
-7
lines changed

models/activities/action.go

+1
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ type GetFeedsOptions struct {
445445
OnlyPerformedBy bool // only actions performed by requested user
446446
IncludeDeleted bool // include deleted actions
447447
Date string // the day we want activity for: YYYY-MM-DD
448+
DontCount bool // do counting in GetFeeds
448449
}
449450

450451
// ActivityReadable return whether doer can read activities of user

models/activities/action_list.go

+18-6
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,11 @@ func GetFeeds(ctx context.Context, opts GetFeedsOptions) (ActionList, int64, err
243243
sess := db.GetEngine(ctx).Where(cond)
244244
sess = db.SetSessionPagination(sess, &opts)
245245

246-
count, err = sess.Desc("`action`.created_unix").FindAndCount(&actions)
246+
if opts.DontCount {
247+
err = sess.Desc("`action`.created_unix").Find(&actions)
248+
} else {
249+
count, err = sess.Desc("`action`.created_unix").FindAndCount(&actions)
250+
}
247251
if err != nil {
248252
return nil, 0, fmt.Errorf("FindAndCount: %w", err)
249253
}
@@ -257,11 +261,13 @@ func GetFeeds(ctx context.Context, opts GetFeedsOptions) (ActionList, int64, err
257261
return nil, 0, fmt.Errorf("Find(actionsIDs): %w", err)
258262
}
259263

260-
count, err = db.GetEngine(ctx).Where(cond).
261-
Table("action").
262-
Cols("`action`.id").Count()
263-
if err != nil {
264-
return nil, 0, fmt.Errorf("Count: %w", err)
264+
if !opts.DontCount {
265+
count, err = db.GetEngine(ctx).Where(cond).
266+
Table("action").
267+
Cols("`action`.id").Count()
268+
if err != nil {
269+
return nil, 0, fmt.Errorf("Count: %w", err)
270+
}
265271
}
266272

267273
if err := db.GetEngine(ctx).In("`action`.id", actionIDs).Desc("`action`.created_unix").Find(&actions); err != nil {
@@ -275,3 +281,9 @@ func GetFeeds(ctx context.Context, opts GetFeedsOptions) (ActionList, int64, err
275281

276282
return actions, count, nil
277283
}
284+
285+
func CountUserFeeds(ctx context.Context, userID int64) (int64, error) {
286+
return db.GetEngine(ctx).Where("user_id = ?", userID).
287+
And("is_deleted = ?", false).
288+
Count(&Action{})
289+
}

routers/web/user/home.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func Dashboard(ctx *context.Context) {
119119
ctx.Data["HeatmapTotalContributions"] = activities_model.GetTotalContributionsInHeatmap(data)
120120
}
121121

122-
feeds, count, err := feed_service.GetFeeds(ctx, activities_model.GetFeedsOptions{
122+
feeds, count, err := feed_service.GetFeedsForDashboard(ctx, activities_model.GetFeedsOptions{
123123
RequestedUser: ctxUser,
124124
RequestedTeam: ctx.Org.Team,
125125
Actor: ctx.Doer,

services/feed/feed.go

+26
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,28 @@ import (
1313
repo_model "code.gitea.io/gitea/models/repo"
1414
"code.gitea.io/gitea/models/unit"
1515
user_model "code.gitea.io/gitea/models/user"
16+
"code.gitea.io/gitea/modules/cache"
1617
"code.gitea.io/gitea/modules/setting"
1718
)
1819

20+
func userFeedCacheKey(userID int64) string {
21+
return fmt.Sprintf("user_feed_%d", userID)
22+
}
23+
24+
func GetFeedsForDashboard(ctx context.Context, opts activities_model.GetFeedsOptions) (activities_model.ActionList, int64, error) {
25+
opts.DontCount = opts.RequestedTeam == nil && opts.Date == ""
26+
results, cnt, err := activities_model.GetFeeds(ctx, opts)
27+
if err != nil {
28+
return nil, 0, err
29+
}
30+
if opts.DontCount {
31+
cnt, err = cache.GetInt64(userFeedCacheKey(opts.Actor.ID), func() (int64, error) {
32+
return activities_model.CountUserFeeds(ctx, opts.Actor.ID)
33+
})
34+
}
35+
return results, cnt, err
36+
}
37+
1938
// GetFeeds returns actions according to the provided options
2039
func GetFeeds(ctx context.Context, opts activities_model.GetFeedsOptions) (activities_model.ActionList, int64, error) {
2140
return activities_model.GetFeeds(ctx, opts)
@@ -68,6 +87,13 @@ func notifyWatchers(ctx context.Context, act *activities_model.Action, watchers
6887
if err := db.Insert(ctx, act); err != nil {
6988
return fmt.Errorf("insert new action: %w", err)
7089
}
90+
91+
total, err := activities_model.CountUserFeeds(ctx, act.UserID)
92+
if err != nil {
93+
return fmt.Errorf("count user feeds: %w", err)
94+
}
95+
96+
_ = cache.GetCache().Put(userFeedCacheKey(act.UserID), fmt.Sprintf("%d", total), setting.CacheService.TTLSeconds())
7197
}
7298

7399
return nil

0 commit comments

Comments
 (0)