Skip to content

Cache GPG keys, emails and users when list commits #34086

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Apr 9, 2025
7 changes: 7 additions & 0 deletions models/asymkey/gpg_key.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,3 +240,10 @@ func DeleteGPGKey(ctx context.Context, doer *user_model.User, id int64) (err err

return committer.Commit()
}

func FindGPGKeyWithSubKeys(ctx context.Context, keyID string) ([]*GPGKey, error) {
return db.Find[GPGKey](ctx, FindGPGKeyOptions{
KeyID: keyID,
IncludeSubKeys: true,
})
}
27 changes: 13 additions & 14 deletions models/user/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -1187,29 +1187,28 @@ func GetUsersByEmails(ctx context.Context, emails []string) (map[string]*User, e
for _, email := range emailAddresses {
userIDs.Add(email.UID)
}
users, err := GetUsersMapByIDs(ctx, userIDs.Values())
if err != nil {
return nil, err
}

results := make(map[string]*User, len(emails))
for _, email := range emailAddresses {
user := users[email.UID]
if user != nil {
if user.KeepEmailPrivate {
results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user
} else {
results[email.Email] = user

if len(userIDs) > 0 {
users, err := GetUsersMapByIDs(ctx, userIDs.Values())
if err != nil {
return nil, err
}

for _, email := range emailAddresses {
user := users[email.UID]
if user != nil {
results[user.GetEmail()] = user
}
}
}

users = make(map[int64]*User, len(needCheckUserNames))
users := make(map[int64]*User, len(needCheckUserNames))
if err := db.GetEngine(ctx).In("lower_name", needCheckUserNames.Values()).Find(&users); err != nil {
return nil, err
}
for _, user := range users {
results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user
results[user.GetPlaceholderEmail()] = user
}
return results, nil
}
Expand Down
8 changes: 4 additions & 4 deletions modules/cache/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,15 +166,15 @@ func RemoveContextData(ctx context.Context, tp, key any) {
}

// GetWithContextCache returns the cache value of the given key in the given context.
func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) {
v := GetContextData(ctx, cacheGroupKey, cacheTargetID)
func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
v := GetContextData(ctx, groupKey, targetKey)
if vv, ok := v.(T); ok {
return vv, nil
}
t, err := f()
t, err := f(ctx, targetKey)
if err != nil {
return t, err
}
SetContextData(ctx, cacheGroupKey, cacheTargetID, t)
SetContextData(ctx, groupKey, targetKey, t)
return t, nil
}
3 changes: 2 additions & 1 deletion modules/cache/context_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package cache

import (
"context"
"testing"
"time"

Expand All @@ -30,7 +31,7 @@ func TestWithCacheContext(t *testing.T) {
v = GetContextData(ctx, field, "my_config1")
assert.Nil(t, v)

vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) {
vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) {
return 1, nil
})
assert.NoError(t, err)
Expand Down
12 changes: 12 additions & 0 deletions modules/cachegroup/cachegroup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package cachegroup

const (
User = "user"
EmailAvatarLink = "email_avatar_link"
UserEmailAddresses = "user_email_addresses"
GPGKeyWithSubKeys = "gpg_key_with_subkeys"
RepoUserPermission = "repo_user_permission"
)
3 changes: 2 additions & 1 deletion modules/repository/commits.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Expand Down Expand Up @@ -131,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model
func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string {
size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor

v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) {
v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) {
u, err := user_model.GetUserByEmail(ctx, email)
if err != nil {
if !user_model.IsErrUserNotExist(err) {
Expand Down
5 changes: 2 additions & 3 deletions routers/private/hook_post_receive.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
Expand Down Expand Up @@ -326,9 +327,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) {
}

func loadContextCacheUser(ctx context.Context, id int64) (*user_model.User, error) {
return cache.GetWithContextCache(ctx, "hook_post_receive_user", id, func() (*user_model.User, error) {
return user_model.GetUserByID(ctx, id)
})
return cache.GetWithContextCache(ctx, cachegroup.User, id, user_model.GetUserByID)
}

// handlePullRequestMerging handle pull request merging, a pull request action should push at least 1 commit
Expand Down
25 changes: 10 additions & 15 deletions services/asymkey/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Expand Down Expand Up @@ -115,7 +117,7 @@ func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, commi
}
}

committerEmailAddresses, _ := user_model.GetEmailAddresses(ctx, committer.ID)
committerEmailAddresses, _ := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses)
activated := false
for _, e := range committerEmailAddresses {
if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) {
Expand Down Expand Up @@ -209,10 +211,9 @@ func checkKeyEmails(ctx context.Context, email string, keys ...*asymkey_model.GP
}
if key.Verified && key.OwnerID != 0 {
if uid != key.OwnerID {
userEmails, _ = user_model.GetEmailAddresses(ctx, key.OwnerID)
userEmails, _ = cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, key.OwnerID, user_model.GetEmailAddresses)
uid = key.OwnerID
user = &user_model.User{ID: uid}
_, _ = user_model.GetUser(ctx, user)
user, _ = cache.GetWithContextCache(ctx, cachegroup.User, uid, user_model.GetUserByID)
}
for _, e := range userEmails {
if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) {
Expand All @@ -231,10 +232,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
if keyID == "" {
return nil
}
keys, err := db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: keyID,
IncludeSubKeys: true,
})
keys, err := cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, keyID, asymkey_model.FindGPGKeyWithSubKeys)
if err != nil {
log.Error("GetGPGKeysByKeyID: %v", err)
return &asymkey_model.CommitVerification{
Expand All @@ -249,10 +247,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
for _, key := range keys {
var primaryKeys []*asymkey_model.GPGKey
if key.PrimaryKeyID != "" {
primaryKeys, err = db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: key.PrimaryKeyID,
IncludeSubKeys: true,
})
primaryKeys, err = cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, key.PrimaryKeyID, asymkey_model.FindGPGKeyWithSubKeys)
if err != nil {
log.Error("GetGPGKeysByKeyID: %v", err)
return &asymkey_model.CommitVerification{
Expand All @@ -272,8 +267,8 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
Name: name,
Email: email,
}
if key.OwnerID != 0 {
owner, err := user_model.GetUserByID(ctx, key.OwnerID)
if key.OwnerID > 0 {
owner, err := cache.GetWithContextCache(ctx, cachegroup.User, key.OwnerID, user_model.GetUserByID)
if err == nil {
signer = owner
} else if !user_model.IsErrUserNotExist(err) {
Expand Down Expand Up @@ -381,7 +376,7 @@ func ParseCommitWithSSHSignature(ctx context.Context, c *git.Commit, committer *
}
}

committerEmailAddresses, err := user_model.GetEmailAddresses(ctx, committer.ID)
committerEmailAddresses, err := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses)
if err != nil {
log.Error("GetEmailAddresses: %v", err)
}
Expand Down
13 changes: 7 additions & 6 deletions services/convert/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
Expand Down Expand Up @@ -60,14 +61,14 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u
doerID = doer.ID
}

const repoDoerPermCacheKey = "repo_doer_perm_cache"
p, err := cache.GetWithContextCache(ctx, repoDoerPermCacheKey, fmt.Sprintf("%d_%d", pr.BaseRepoID, doerID),
func() (access_model.Permission, error) {
repoUserPerm, err := cache.GetWithContextCache(ctx, cachegroup.RepoUserPermission, fmt.Sprintf("%d-%d", pr.BaseRepoID, doerID),
func(ctx context.Context, _ string) (access_model.Permission, error) {
return access_model.GetUserRepoPermission(ctx, pr.BaseRepo, doer)
})
},
)
if err != nil {
log.Error("GetUserRepoPermission[%d]: %v", pr.BaseRepoID, err)
p.AccessMode = perm.AccessModeNone
repoUserPerm.AccessMode = perm.AccessModeNone
}

apiPullRequest := &api.PullRequest{
Expand Down Expand Up @@ -107,7 +108,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u
Name: pr.BaseBranch,
Ref: pr.BaseBranch,
RepoID: pr.BaseRepoID,
Repository: ToRepo(ctx, pr.BaseRepo, p),
Repository: ToRepo(ctx, pr.BaseRepo, repoUserPerm),
},
Head: &api.PRBranchInfo{
Name: pr.HeadBranch,
Expand Down
10 changes: 6 additions & 4 deletions services/git/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
)

// ParseCommitsWithSignature checks if signaute of commits are corresponding to users gpg keys.
func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType, isOwnerMemberCollaborator func(*user_model.User) (bool, error)) ([]*asymkey_model.SignCommit, error) {
func ParseCommitsWithSignature(ctx context.Context, repo *repo_model.Repository, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType) ([]*asymkey_model.SignCommit, error) {
newCommits := make([]*asymkey_model.SignCommit, 0, len(oldCommits))
keyMap := map[string]bool{}

Expand Down Expand Up @@ -47,6 +47,10 @@ func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.Use
Verification: asymkey_service.ParseCommitWithSignatureCommitter(ctx, c.Commit, committer),
}

isOwnerMemberCollaborator := func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
}

_ = asymkey_model.CalculateTrustStatus(signCommit.Verification, repoTrustModel, isOwnerMemberCollaborator, &keyMap)

newCommits = append(newCommits, signCommit)
Expand All @@ -62,11 +66,9 @@ func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo
}
signedCommits, err := ParseCommitsWithSignature(
ctx,
repo,
validatedCommits,
repo.GetTrustModel(),
func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
},
)
if err != nil {
return nil, err
Expand Down