Adjust gitea doctor --run storages to check all storage types (#21785)

The doctor check `storages` currently only checks the attachment
storage. This PR adds some basic garbage collection functionality for
the other types of storage.

Signed-off-by: Andrew Thornton <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

Author: zeripath

models/git/lfs.go

@@ -235,9 +235,9 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) {
 	return count > 0, err
 }
 
-// LFSObjectIsAssociated checks if a provided Oid is associated
-func LFSObjectIsAssociated(oid string) (bool, error) {
-	return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
+// ExistsLFSObject checks if a provided Oid exists within the DB
+func ExistsLFSObject(ctx context.Context, oid string) (bool, error) {
+	return db.GetEngine(ctx).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
 }
 
 // LFSAutoAssociate auto associates accessible LFSMetaObjects

models/packages/package_blob.go

@@ -62,6 +62,13 @@ func GetBlobByID(ctx context.Context, blobID int64) (*PackageBlob, error) {
 	return pb, nil
 }
 
+// ExistPackageBlobWithSHA returns if a package blob exists with the provided sha
+func ExistPackageBlobWithSHA(ctx context.Context, blobSha256 string) (bool, error) {
+	return db.GetEngine(ctx).Exist(&PackageBlob{
+		HashSHA256: blobSha256,
+	})
+}
+
 // FindExpiredUnreferencedBlobs gets all blobs without associated files older than the specific duration
 func FindExpiredUnreferencedBlobs(ctx context.Context, olderThan time.Duration) ([]*PackageBlob, error) {
 	pbs := make([]*PackageBlob, 0, 10)

models/repo/archiver.go

@@ -7,11 +7,14 @@ package repo
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/builder"
 )
@@ -44,6 +47,28 @@ func (archiver *RepoArchiver) RelativePath() string {
 	return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String())
 }
 
+// repoArchiverForRelativePath takes a relativePath created from (archiver *RepoArchiver) RelativePath() and creates a shell repoArchiver struct representing it
+func repoArchiverForRelativePath(relativePath string) (*RepoArchiver, error) {
+	parts := strings.SplitN(relativePath, "/", 3)
+	if len(parts) != 3 {
+		return nil, util.SilentWrap{Message: fmt.Sprintf("invalid storage path: %s", relativePath), Err: util.ErrInvalidArgument}
+	}
+	repoID, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil {
+		return nil, util.SilentWrap{Message: fmt.Sprintf("invalid storage path: %s", relativePath), Err: util.ErrInvalidArgument}
+	}
+	nameExts := strings.SplitN(parts[2], ".", 2)
+	if len(nameExts) != 2 {
+		return nil, util.SilentWrap{Message: fmt.Sprintf("invalid storage path: %s", relativePath), Err: util.ErrInvalidArgument}
+	}
+
+	return &RepoArchiver{
+		RepoID:   repoID,
+		CommitID: parts[1] + nameExts[0],
+		Type:     git.ToArchiveType(nameExts[1]),
+	}, nil
+}
+
 var delRepoArchiver = new(RepoArchiver)
 
 // DeleteRepoArchiver delete archiver
@@ -65,6 +90,17 @@ func GetRepoArchiver(ctx context.Context, repoID int64, tp git.ArchiveType, comm
 	return nil, nil
 }
 
+// ExistsRepoArchiverWithStoragePath checks if there is a RepoArchiver for a given storage path
+func ExistsRepoArchiverWithStoragePath(ctx context.Context, storagePath string) (bool, error) {
+	// We need to invert the path provided func (archiver *RepoArchiver) RelativePath() above
+	archiver, err := repoArchiverForRelativePath(storagePath)
+	if err != nil {
+		return false, err
+	}
+
+	return db.GetEngine(ctx).Exist(archiver)
+}
+
 // AddRepoArchiver adds an archiver
 func AddRepoArchiver(ctx context.Context, archiver *RepoArchiver) error {
 	_, err := db.GetEngine(ctx).Insert(archiver)

models/repo/attachment.go

@@ -122,9 +122,9 @@ func GetAttachmentsByUUIDs(ctx context.Context, uuids []string) ([]*Attachment,
 	return attachments, db.GetEngine(ctx).In("uuid", uuids).Find(&attachments)
 }
 
-// ExistAttachmentsByUUID returns true if attachment is exist by given UUID
-func ExistAttachmentsByUUID(uuid string) (bool, error) {
-	return db.GetEngine(db.DefaultContext).Where("`uuid`=?", uuid).Exist(new(Attachment))
+// ExistAttachmentsByUUID returns true if attachment exists with the given UUID
+func ExistAttachmentsByUUID(ctx context.Context, uuid string) (bool, error) {
+	return db.GetEngine(ctx).Where("`uuid`=?", uuid).Exist(new(Attachment))
 }
 
 // GetAttachmentsByIssueID returns all attachments of an issue.

models/repo/avatar.go

@@ -24,6 +24,13 @@ func (repo *Repository) CustomAvatarRelativePath() string {
 	return repo.Avatar
 }
 
+// ExistsWithAvatarAtStoragePath returns true if there is a user with this Avatar
+func ExistsWithAvatarAtStoragePath(ctx context.Context, storagePath string) (bool, error) {
+	// See func (repo *Repository) CustomAvatarRelativePath()
+	// repo.Avatar is used directly as the storage path - therefore we can check for existence directly using the path
+	return db.GetEngine(ctx).Where("`avatar`=?", storagePath).Exist(new(Repository))
+}
+
 // RelAvatarLink returns a relative link to the repository's avatar.
 func (repo *Repository) RelAvatarLink() string {
 	return repo.relAvatarLink(db.DefaultContext)

models/user/avatar.go

@@ -111,3 +111,10 @@ func (u *User) IsUploadAvatarChanged(data []byte) bool {
 	avatarID := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", u.ID, md5.Sum(data)))))
 	return u.Avatar != avatarID
 }
+
+// ExistsWithAvatarAtStoragePath returns true if there is a user with this Avatar
+func ExistsWithAvatarAtStoragePath(ctx context.Context, storagePath string) (bool, error) {
+	// See func (u *User) CustomAvatarRelativePath()
+	// u.Avatar is used directly as the storage path - therefore we can check for existence directly using the path
+	return db.GetEngine(ctx).Where("`avatar`=?", storagePath).Exist(new(User))
+}