Skip to content

Do not allow different storage configurations to point to the same directory #30169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/setting/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) {
if !filepath.IsAbs(Indexer.IssuePath) {
Indexer.IssuePath = filepath.ToSlash(filepath.Join(AppWorkPath, Indexer.IssuePath))
}
fatalDuplicatedPath("issue_indexer", Indexer.IssuePath)
checkOverlappedPath("indexer.ISSUE_INDEXER_PATH", Indexer.IssuePath)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's better if we notify the users explicitly what the section is:

Suggested change
checkOverlappedPath("indexer.ISSUE_INDEXER_PATH", Indexer.IssuePath)
checkOverlappedPath("[indexer].ISSUE_INDEXER_PATH", Indexer.IssuePath)

} else {
Indexer.IssueConnStr = sec.Key("ISSUE_INDEXER_CONN_STR").MustString(Indexer.IssueConnStr)
if Indexer.IssueType == "meilisearch" {
Expand Down
4 changes: 0 additions & 4 deletions modules/setting/path.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,8 @@ func init() {
AppWorkPath = filepath.Dir(AppPath)
}

fatalDuplicatedPath("app_work_path", AppWorkPath)

appWorkPathBuiltin = AppWorkPath
customPathBuiltin = CustomPath

fatalDuplicatedPath("custom_path", CustomPath)
customConfBuiltin = CustomConf
}

Expand Down
2 changes: 1 addition & 1 deletion modules/setting/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) {
RepoRootPath = filepath.Clean(RepoRootPath)
}

fatalDuplicatedPath("repository.ROOT", RepoRootPath)
checkOverlappedPath("repository.ROOT", RepoRootPath)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
checkOverlappedPath("repository.ROOT", RepoRootPath)
checkOverlappedPath("[repository].ROOT", RepoRootPath)


defaultDetectedCharsetsOrder := make([]string, 0, len(Repository.DetectedCharsetsOrder))
for _, charset := range Repository.DetectedCharsetsOrder {
Expand Down
3 changes: 1 addition & 2 deletions modules/setting/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -324,15 +324,14 @@ func loadServerFrom(rootCfg ConfigProvider) {
if !filepath.IsAbs(AppDataPath) {
AppDataPath = filepath.ToSlash(filepath.Join(AppWorkPath, AppDataPath))
}
fatalDuplicatedPath("app_data_path", AppDataPath)

EnableGzip = sec.Key("ENABLE_GZIP").MustBool()
EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false)
PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(filepath.Join(AppWorkPath, "data/tmp/pprof"))
if !filepath.IsAbs(PprofDataPath) {
PprofDataPath = filepath.Join(AppWorkPath, PprofDataPath)
}
fatalDuplicatedPath("pprof_data_path", PprofDataPath)
checkOverlappedPath("server.PPROF_DATA_PATH", PprofDataPath)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
checkOverlappedPath("server.PPROF_DATA_PATH", PprofDataPath)
checkOverlappedPath("[server].PPROF_DATA_PATH", PprofDataPath)


landingPage := sec.Key("LANDING_PAGE").MustString("home")
switch landingPage {
Expand Down
2 changes: 1 addition & 1 deletion modules/setting/session.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func loadSessionFrom(rootCfg ConfigProvider) {
SessionConfig.ProviderConfig = strings.Trim(sec.Key("PROVIDER_CONFIG").MustString(filepath.Join(AppDataPath, "sessions")), "\" ")
if SessionConfig.Provider == "file" && !filepath.IsAbs(SessionConfig.ProviderConfig) {
SessionConfig.ProviderConfig = filepath.Join(AppWorkPath, SessionConfig.ProviderConfig)
fatalDuplicatedPath("session", SessionConfig.ProviderConfig)
checkOverlappedPath("session.PROVIDER_CONFIG", SessionConfig.ProviderConfig)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
checkOverlappedPath("session.PROVIDER_CONFIG", SessionConfig.ProviderConfig)
checkOverlappedPath("[session].PROVIDER_CONFIG", SessionConfig.ProviderConfig)

}
SessionConfig.CookieName = sec.Key("COOKIE_NAME").MustString("i_like_gitea")
SessionConfig.CookiePath = AppSubURL
Expand Down
13 changes: 8 additions & 5 deletions modules/setting/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,11 +230,14 @@ func LoadSettingsForInstall() {
loadMailerFrom(CfgProvider)
}

var uniquePaths = make(map[string]string)
var configuredPaths = make(map[string]string)

func fatalDuplicatedPath(name, p string) {
if targetName, ok := uniquePaths[p]; ok && targetName != name {
log.Fatal("storage path %q is being used by %q and %q and all storage paths must be unique to prevent data loss.", p, targetName, name)
func checkOverlappedPath(name, path string) {
// TODO: some paths shouldn't overlap (storage.xxx.path), while some could (data path is the base path for storage path)
if targetName, ok := configuredPaths[path]; ok && targetName != name {
msg := fmt.Sprintf("Configured path %q is used by %q and %q at the same time. The paths must be unique to prevent data loss.", path, targetName, name)
log.Error("%s", msg)
DeprecatedWarnings = append(DeprecatedWarnings, msg)
}
Comment on lines +236 to 241
Copy link
Member

@delvh delvh Mar 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we fix this completely using

Suggested change
// TODO: some paths shouldn't overlap (storage.xxx.path), while some could (data path is the base path for storage path)
if targetName, ok := configuredPaths[path]; ok && targetName != name {
msg := fmt.Sprintf("Configured path %q is used by %q and %q at the same time. The paths must be unique to prevent data loss.", path, targetName, name)
log.Error("%s", msg)
DeprecatedWarnings = append(DeprecatedWarnings, msg)
}
path = gopath.Clean(path)
for existingPath, existingName := range configuredPaths {
if strings.HasPrefix(path, existingPath) || strings.HasPrefix(existingPath, path) {
var msg string
if path == existingPath {
msg = fmt.Sprintf("Path %q is used by %q and %q simultaneously. This may lead to data loss. To prevent this, the paths must be different and they must not overlap.", path, existingName, name)
} else {
msg = fmt.Sprintf("Path %q (used by %q) is contained within %q (used by %q). This may lead to data loss. To prevent this, the paths may not overlap.", existingPath, existingName, path, name)
}
log.Error("%s", msg)
DeprecatedWarnings = append(DeprecatedWarnings, msg)
}

(assuming we have an import gopath path above)
?

Copy link
Contributor Author

@wxiaoguang wxiaoguang Mar 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not right (incomplete). /a/b overlaps /a, meanwhile /a overlaps /a/b

If you would like to "complete" it, it also needs enough tests.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, fixed my comment.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But your "msg" is still not right, and it doesn't have tests.

As I said: "I won't do any further change". So feel free to take it, or improve it, or discard it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And one more thing, path = gopath.Clean(path) is also not right. It should use filepath.

uniquePaths[p] = name
configuredPaths[path] = name
}
2 changes: 1 addition & 1 deletion modules/setting/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ func getStorageForLocal(targetSec, overrideSec ConfigSection, tp targetSecType,
}
}

fatalDuplicatedPath("storage."+name, storage.Path)
checkOverlappedPath("storage."+name+".PATH", storage.Path)
Copy link
Member

@delvh delvh Mar 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
checkOverlappedPath("storage."+name+".PATH", storage.Path)
checkOverlappedPath("[storage."+name+"].PATH", storage.Path)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not right by your approach. [storage.xxx] is the section name. [storage] is not.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, fixed my comment


return &storage, nil
}
Expand Down
4 changes: 3 additions & 1 deletion options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2775,6 +2775,7 @@ teams.invite.by = Invited by %s
teams.invite.description = Please click the button below to join the team.

[admin]
maintenance = Maintenance
dashboard = Dashboard
self_check = Self Check
identity_access = Identity & Access
Expand All @@ -2798,7 +2799,7 @@ settings = Admin Settings

dashboard.new_version_hint = Gitea %s is now available, you are running %s. Check <a target="_blank" rel="noreferrer" href="https://blog.gitea.io">the blog</a> for more details.
dashboard.statistic = Summary
dashboard.operations = Maintenance Operations
dashboard.maintenance_operations = Maintenance Operations
dashboard.system_status = System Status
dashboard.operation_name = Operation Name
dashboard.operation_switch = Switch
Expand Down Expand Up @@ -3305,6 +3306,7 @@ notices.op = Op.
notices.delete_success = The system notices have been deleted.

self_check.no_problem_found = No problem found yet.
self_check.startup_warnings = Startup warnings:
self_check.database_collation_mismatch = Expect database to use collation: %s
self_check.database_collation_case_insensitive = Database is using a collation %s, which is an insensitive collation. Although Gitea could work with it, there might be some rare cases which don't work as expected.
self_check.database_inconsistent_collation_columns = Database is using collation %s, but these columns are using mismatched collations. It might cause some unexpected problems.
Expand Down
8 changes: 8 additions & 0 deletions routers/web/admin/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,14 @@ func DashboardPost(ctx *context.Context) {

func SelfCheck(ctx *context.Context) {
ctx.Data["PageIsAdminSelfCheck"] = true

ctx.Data["DeprecatedWarnings"] = setting.DeprecatedWarnings
if len(setting.DeprecatedWarnings) == 0 && !setting.IsProd {
if time.Now().Unix()%2 == 0 {
ctx.Data["DeprecatedWarnings"] = []string{"This is a test warning message in dev mode"}
}
}

r, err := db.CheckCollationsDefaultEngine()
if err != nil {
ctx.Flash.Error(fmt.Sprintf("CheckCollationsDefaultEngine: %v", err), true)
Expand Down
2 changes: 1 addition & 1 deletion templates/admin/dashboard.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
</div>
{{end}}
<h4 class="ui top attached header">
{{ctx.Locale.Tr "admin.dashboard.operations"}}
{{ctx.Locale.Tr "admin.dashboard.maintenance_operations"}}
</h4>
<div class="ui attached table segment">
<form method="post" action="{{AppSubUrl}}/admin">
Expand Down
18 changes: 12 additions & 6 deletions templates/admin/navbar.tmpl
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
<div class="flex-container-nav">
<div class="ui fluid vertical menu">
<div class="header item">{{ctx.Locale.Tr "admin.settings"}}</div>
<a class="{{if .PageIsAdminDashboard}}active {{end}}item" href="{{AppSubUrl}}/admin">
{{ctx.Locale.Tr "admin.dashboard"}}
</a>
<a class="{{if .PageIsAdminSelfCheck}}active {{end}}item" href="{{AppSubUrl}}/admin/self_check">
{{ctx.Locale.Tr "admin.self_check"}}
</a>

<details class="item toggleable-item" {{if or .PageIsAdminDashboard .PageIsAdminSelfCheck}}open{{end}}>
<summary>{{ctx.Locale.Tr "admin.maintenance"}}</summary>
<div class="menu">
<a class="{{if .PageIsAdminDashboard}}active {{end}}item" href="{{AppSubUrl}}/admin">
{{ctx.Locale.Tr "admin.dashboard"}}
</a>
<a class="{{if .PageIsAdminSelfCheck}}active {{end}}item" href="{{AppSubUrl}}/admin/self_check">
{{ctx.Locale.Tr "admin.self_check"}}
</a>
</div>
</details>
<details class="item toggleable-item" {{if or .PageIsAdminUsers .PageIsAdminEmails .PageIsAdminOrganizations .PageIsAdminAuthentications}}open{{end}}>
<summary>{{ctx.Locale.Tr "admin.identity_access"}}</summary>
<div class="menu">
Expand Down
62 changes: 38 additions & 24 deletions templates/admin/self_check.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,47 @@
<h4 class="ui top attached header">
{{ctx.Locale.Tr "admin.self_check"}}
</h4>

{{if .DeprecatedWarnings}}
<div class="ui attached segment">
{{if .DatabaseCheckHasProblems}}
{{if .DatabaseType.IsMySQL}}
<div class="tw-p-2">{{ctx.Locale.Tr "admin.self_check.database_fix_mysql"}}</div>
{{else if .DatabaseType.IsMSSQL}}
<div class="tw-p-2">{{ctx.Locale.Tr "admin.self_check.database_fix_mssql"}}</div>
{{end}}
{{if .DatabaseCheckCollationMismatch}}
<div class="ui red message">{{ctx.Locale.Tr "admin.self_check.database_collation_mismatch" .DatabaseCheckResult.ExpectedCollation}}</div>
{{end}}
{{if .DatabaseCheckCollationCaseInsensitive}}
<div class="ui warning message">{{ctx.Locale.Tr "admin.self_check.database_collation_case_insensitive" .DatabaseCheckResult.DatabaseCollation}}</div>
{{end}}
{{if .DatabaseCheckInconsistentCollationColumns}}
<div class="ui red message">
{{ctx.Locale.Tr "admin.self_check.database_inconsistent_collation_columns" .DatabaseCheckResult.DatabaseCollation}}
<ul class="tw-w-full">
{{range .DatabaseCheckInconsistentCollationColumns}}
<li>{{.}}</li>
{{end}}
</ul>
</div>
{{end}}
{{else}}
<div class="tw-p-2">{{ctx.Locale.Tr "admin.self_check.no_problem_found"}}</div>
<div class="ui warning message">
<div>{{ctx.Locale.Tr "admin.self_check.startup_warnings"}}</div>
<ul class="tw-w-full">{{range .DeprecatedWarnings}}<li>{{.}}</li>{{end}}</ul>
</div>
</div>
{{end}}

{{if .DatabaseCheckHasProblems}}
<div class="ui attached segment">
{{if .DatabaseType.IsMySQL}}
<div class="tw-p-2">{{ctx.Locale.Tr "admin.self_check.database_fix_mysql"}}</div>
{{else if .DatabaseType.IsMSSQL}}
<div class="tw-p-2">{{ctx.Locale.Tr "admin.self_check.database_fix_mssql"}}</div>
{{end}}
{{if .DatabaseCheckCollationMismatch}}
<div class="ui red message">{{ctx.Locale.Tr "admin.self_check.database_collation_mismatch" .DatabaseCheckResult.ExpectedCollation}}</div>
{{end}}
{{if .DatabaseCheckCollationCaseInsensitive}}
<div class="ui warning message">{{ctx.Locale.Tr "admin.self_check.database_collation_case_insensitive" .DatabaseCheckResult.DatabaseCollation}}</div>
{{end}}
{{if .DatabaseCheckInconsistentCollationColumns}}
<div class="ui red message">
{{ctx.Locale.Tr "admin.self_check.database_inconsistent_collation_columns" .DatabaseCheckResult.DatabaseCollation}}
<ul class="tw-w-full">
{{range .DatabaseCheckInconsistentCollationColumns}}
<li>{{.}}</li>
{{end}}
</ul>
</div>
{{end}}
</div>
{{end}}

{{if and (not .DeprecatedWarnings) (not .DatabaseCheckHasProblems)}}
<div class="ui attached segment">
{{ctx.Locale.Tr "admin.self_check.no_problem_found"}}
</div>
{{end}}
</div>

{{template "admin/layout_footer" .}}