Skip to content

Commit 15e93a7

Browse files
wxiaoguangwxiaoguang
authored
Add a config option to block "expensive" pages for anonymous users (go-gitea#34024) (go-gitea#34071)
Backport go-gitea#34024 since there are too many AI crawlers. The new code is covered by tests and it does nothing if users don't set it.
1 parent 5a9b3bf commit 15e93a7

File tree

21 files changed

+228
-38
lines changed

21 files changed

+228
-38
lines changed

‎custom/conf/app.example.ini

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,6 +774,9 @@ LEVEL = Info
774774
;ALLOW_ONLY_EXTERNAL_REGISTRATION = false
775775
;;
776776
;; User must sign in to view anything.
777+
;; After 1.23.7, it could be set to "expensive" to block anonymous users accessing some pages which consume a lot of resources,
778+
;; for example: block anonymous AI crawlers from accessing repo code pages.
779+
;; The "expensive" mode is experimental and subject to change.
777780
;REQUIRE_SIGNIN_VIEW = false
778781
;;
779782
;; Mail notification

‎modules/setting/config_provider.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type ConfigKey interface {
2626
In(defaultVal string, candidates []string) string
2727
String() string
2828
Strings(delim string) []string
29+
Bool() (bool, error)
2930

3031
MustString(defaultVal string) string
3132
MustBool(defaultVal ...bool) bool

‎modules/setting/service.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ var Service = struct {
4343
ShowRegistrationButton bool
4444
EnablePasswordSignInForm bool
4545
ShowMilestonesDashboardPage bool
46-
RequireSignInView bool
46+
RequireSignInViewStrict bool
47+
BlockAnonymousAccessExpensive bool
4748
EnableNotifyMail bool
4849
EnableBasicAuth bool
4950
EnablePasskeyAuth bool
@@ -159,7 +160,18 @@ func loadServiceFrom(rootCfg ConfigProvider) {
159160
Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST")
160161
Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration))
161162
Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true)
162-
Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool()
163+
164+
// boolean values are considered as "strict"
165+
var err error
166+
Service.RequireSignInViewStrict, err = sec.Key("REQUIRE_SIGNIN_VIEW").Bool()
167+
if s := sec.Key("REQUIRE_SIGNIN_VIEW").String(); err != nil && s != "" {
168+
// non-boolean value only supports "expensive" at the moment
169+
Service.BlockAnonymousAccessExpensive = s == "expensive"
170+
if !Service.BlockAnonymousAccessExpensive {
171+
log.Error("Invalid config option: REQUIRE_SIGNIN_VIEW = %s", s)
172+
}
173+
}
174+
163175
Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true)
164176
Service.EnablePasswordSignInForm = sec.Key("ENABLE_PASSWORD_SIGNIN_FORM").MustBool(true)
165177
Service.EnablePasskeyAuth = sec.Key("ENABLE_PASSKEY_AUTHENTICATION").MustBool(true)

‎modules/setting/service_test.go

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,14 @@ import (
77
"testing"
88

99
"code.gitea.io/gitea/modules/structs"
10+
"code.gitea.io/gitea/modules/test"
1011

1112
"github.com/gobwas/glob"
1213
"github.com/stretchr/testify/assert"
1314
)
1415

1516
func TestLoadServices(t *testing.T) {
16-
oldService := Service
17-
defer func() {
18-
Service = oldService
19-
}()
17+
defer test.MockVariableValue(&Service)()
2018

2119
cfg, err := NewConfigProviderFromData(`
2220
[service]
@@ -48,10 +46,7 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b
4846
}
4947

5048
func TestLoadServiceVisibilityModes(t *testing.T) {
51-
oldService := Service
52-
defer func() {
53-
Service = oldService
54-
}()
49+
defer test.MockVariableValue(&Service)()
5550

5651
kases := map[string]func(){
5752
`
@@ -130,3 +125,33 @@ ALLOWED_USER_VISIBILITY_MODES = public, limit, privated
130125
})
131126
}
132127
}
128+
129+
func TestLoadServiceRequireSignInView(t *testing.T) {
130+
defer test.MockVariableValue(&Service)()
131+
132+
cfg, err := NewConfigProviderFromData(`
133+
[service]
134+
`)
135+
assert.NoError(t, err)
136+
loadServiceFrom(cfg)
137+
assert.False(t, Service.RequireSignInViewStrict)
138+
assert.False(t, Service.BlockAnonymousAccessExpensive)
139+
140+
cfg, err = NewConfigProviderFromData(`
141+
[service]
142+
REQUIRE_SIGNIN_VIEW = true
143+
`)
144+
assert.NoError(t, err)
145+
loadServiceFrom(cfg)
146+
assert.True(t, Service.RequireSignInViewStrict)
147+
assert.False(t, Service.BlockAnonymousAccessExpensive)
148+
149+
cfg, err = NewConfigProviderFromData(`
150+
[service]
151+
REQUIRE_SIGNIN_VIEW = expensive
152+
`)
153+
assert.NoError(t, err)
154+
loadServiceFrom(cfg)
155+
assert.False(t, Service.RequireSignInViewStrict)
156+
assert.True(t, Service.BlockAnonymousAccessExpensive)
157+
}

‎routers/api/packages/cargo/cargo.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func apiError(ctx *context.Context, status int, obj any) {
5151

5252
// https://rust-lang.github.io/rfcs/2789-sparse-index.html
5353
func RepositoryConfig(ctx *context.Context) {
54-
ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner, setting.Service.RequireSignInView || ctx.Package.Owner.Visibility != structs.VisibleTypePublic))
54+
ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner, setting.Service.RequireSignInViewStrict || ctx.Package.Owner.Visibility != structs.VisibleTypePublic))
5555
}
5656

5757
func EnumeratePackageVersions(ctx *context.Context) {

‎routers/api/packages/container/container.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func apiUnauthorizedError(ctx *context.Context) {
126126

127127
// ReqContainerAccess is a middleware which checks the current user valid (real user or ghost if anonymous access is enabled)
128128
func ReqContainerAccess(ctx *context.Context) {
129-
if ctx.Doer == nil || (setting.Service.RequireSignInView && ctx.Doer.IsGhost()) {
129+
if ctx.Doer == nil || (setting.Service.RequireSignInViewStrict && ctx.Doer.IsGhost()) {
130130
apiUnauthorizedError(ctx)
131131
}
132132
}
@@ -152,7 +152,7 @@ func Authenticate(ctx *context.Context) {
152152
u := ctx.Doer
153153
packageScope := auth_service.GetAccessScope(ctx.Data)
154154
if u == nil {
155-
if setting.Service.RequireSignInView {
155+
if setting.Service.RequireSignInViewStrict {
156156
apiUnauthorizedError(ctx)
157157
return
158158
}

‎routers/api/v1/api.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@ func reqToken() func(ctx *context.APIContext) {
356356

357357
func reqExploreSignIn() func(ctx *context.APIContext) {
358358
return func(ctx *context.APIContext) {
359-
if (setting.Service.RequireSignInView || setting.Service.Explore.RequireSigninView) && !ctx.IsSigned {
359+
if (setting.Service.RequireSignInViewStrict || setting.Service.Explore.RequireSigninView) && !ctx.IsSigned {
360360
ctx.Error(http.StatusUnauthorized, "reqExploreSignIn", "you must be signed in to search for users")
361361
}
362362
}
@@ -874,7 +874,7 @@ func Routes() *web.Router {
874874
m.Use(apiAuth(buildAuthGroup()))
875875

876876
m.Use(verifyAuthWithOptions(&common.VerifyOptions{
877-
SignInRequired: setting.Service.RequireSignInView,
877+
SignInRequired: setting.Service.RequireSignInViewStrict,
878878
}))
879879

880880
addActionsRoutes := func(

‎routers/common/blockexpensive.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright 2025 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package common
5+
6+
import (
7+
"context"
8+
"net/http"
9+
"strings"
10+
11+
user_model "code.gitea.io/gitea/models/user"
12+
"code.gitea.io/gitea/modules/setting"
13+
"code.gitea.io/gitea/modules/web/middleware"
14+
15+
"github.com/go-chi/chi/v5"
16+
)
17+
18+
func BlockExpensive() func(next http.Handler) http.Handler {
19+
if !setting.Service.BlockAnonymousAccessExpensive {
20+
return nil
21+
}
22+
return func(next http.Handler) http.Handler {
23+
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
24+
ret := determineRequestPriority(req.Context())
25+
if !ret.SignedIn {
26+
if ret.Expensive || ret.LongPolling {
27+
http.Redirect(w, req, setting.AppSubURL+"/user/login", http.StatusSeeOther)
28+
return
29+
}
30+
}
31+
next.ServeHTTP(w, req)
32+
})
33+
}
34+
}
35+
36+
func isRoutePathExpensive(routePattern string) bool {
37+
if strings.HasPrefix(routePattern, "/user/") || strings.HasPrefix(routePattern, "/login/") {
38+
return false
39+
}
40+
41+
expensivePaths := []string{
42+
// code related
43+
"/{username}/{reponame}/archive/",
44+
"/{username}/{reponame}/blame/",
45+
"/{username}/{reponame}/commit/",
46+
"/{username}/{reponame}/commits/",
47+
"/{username}/{reponame}/graph",
48+
"/{username}/{reponame}/media/",
49+
"/{username}/{reponame}/raw/",
50+
"/{username}/{reponame}/src/",
51+
52+
// issue & PR related (no trailing slash)
53+
"/{username}/{reponame}/issues",
54+
"/{username}/{reponame}/{type:issues}",
55+
"/{username}/{reponame}/pulls",
56+
"/{username}/{reponame}/{type:pulls}",
57+
58+
// wiki
59+
"/{username}/{reponame}/wiki/",
60+
61+
// activity
62+
"/{username}/{reponame}/activity/",
63+
}
64+
for _, path := range expensivePaths {
65+
if strings.HasPrefix(routePattern, path) {
66+
return true
67+
}
68+
}
69+
return false
70+
}
71+
72+
func isRoutePathForLongPolling(routePattern string) bool {
73+
return routePattern == "/user/events"
74+
}
75+
76+
func determineRequestPriority(ctx context.Context) (ret struct {
77+
SignedIn bool
78+
Expensive bool
79+
LongPolling bool
80+
},
81+
) {
82+
dataStore := middleware.GetContextData(ctx)
83+
chiRoutePath := chi.RouteContext(ctx).RoutePattern()
84+
if _, ok := dataStore[middleware.ContextDataKeySignedUser].(*user_model.User); ok {
85+
ret.SignedIn = true
86+
} else {
87+
ret.Expensive = isRoutePathExpensive(chiRoutePath)
88+
ret.LongPolling = isRoutePathForLongPolling(chiRoutePath)
89+
}
90+
return ret
91+
}

‎routers/common/blockexpensive_test.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2025 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package common
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
func TestBlockExpensive(t *testing.T) {
13+
cases := []struct {
14+
expensive bool
15+
routePath string
16+
}{
17+
{false, "/user/xxx"},
18+
{false, "/login/xxx"},
19+
{true, "/{username}/{reponame}/archive/xxx"},
20+
{true, "/{username}/{reponame}/graph"},
21+
{true, "/{username}/{reponame}/src/xxx"},
22+
{true, "/{username}/{reponame}/wiki/xxx"},
23+
{true, "/{username}/{reponame}/activity/xxx"},
24+
}
25+
for _, c := range cases {
26+
assert.Equal(t, c.expensive, isRoutePathExpensive(c.routePath), "routePath: %s", c.routePath)
27+
}
28+
29+
assert.True(t, isRoutePathForLongPolling("/user/events"))
30+
}

‎routers/install/install.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func Install(ctx *context.Context) {
156156
form.DisableRegistration = setting.Service.DisableRegistration
157157
form.AllowOnlyExternalRegistration = setting.Service.AllowOnlyExternalRegistration
158158
form.EnableCaptcha = setting.Service.EnableCaptcha
159-
form.RequireSignInView = setting.Service.RequireSignInView
159+
form.RequireSignInView = setting.Service.RequireSignInViewStrict
160160
form.DefaultKeepEmailPrivate = setting.Service.DefaultKeepEmailPrivate
161161
form.DefaultAllowCreateOrganization = setting.Service.DefaultAllowCreateOrganization
162162
form.DefaultEnableTimetracking = setting.Service.DefaultEnableTimetracking

‎routers/private/serv.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ func ServCommand(ctx *context.PrivateContext) {
286286
repo.IsPrivate ||
287287
owner.Visibility.IsPrivate() ||
288288
(user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey
289-
setting.Service.RequireSignInView) {
289+
setting.Service.RequireSignInViewStrict) {
290290
if key.Type == asymkey_model.KeyTypeDeploy {
291291
if deployKey.Mode < mode {
292292
ctx.JSON(http.StatusUnauthorized, private.Response{

‎routers/web/repo/githttp.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func httpBase(ctx *context.Context) *serviceHandler {
127127
// Only public pull don't need auth.
128128
isPublicPull := repoExist && !repo.IsPrivate && isPull
129129
var (
130-
askAuth = !isPublicPull || setting.Service.RequireSignInView
130+
askAuth = !isPublicPull || setting.Service.RequireSignInViewStrict
131131
environ []string
132132
)
133133

‎routers/web/web.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -285,23 +285,23 @@ func Routes() *web.Router {
285285
mid = append(mid, repo.GetActiveStopwatch)
286286
mid = append(mid, goGet)
287287

288-
others := web.NewRouter()
289-
others.Use(mid...)
290-
registerRoutes(others)
291-
routes.Mount("", others)
288+
webRoutes := web.NewRouter()
289+
webRoutes.Use(mid...)
290+
webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive())
291+
routes.Mount("", webRoutes)
292292
return routes
293293
}
294294

295295
var optSignInIgnoreCsrf = verifyAuthWithOptions(&common.VerifyOptions{DisableCSRF: true})
296296

297-
// registerRoutes register routes
298-
func registerRoutes(m *web.Router) {
297+
// registerWebRoutes register routes
298+
func registerWebRoutes(m *web.Router) {
299299
// required to be signed in or signed out
300300
reqSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: true})
301301
reqSignOut := verifyAuthWithOptions(&common.VerifyOptions{SignOutRequired: true})
302302
// optional sign in (if signed in, use the user as doer, if not, no doer)
303-
optSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInView})
304-
optExploreSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInView || setting.Service.Explore.RequireSigninView})
303+
optSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInViewStrict})
304+
optExploreSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInViewStrict || setting.Service.Explore.RequireSigninView})
305305

306306
validation.AddBindingRules()
307307

‎services/context/package.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any))
9393
}
9494

9595
func determineAccessMode(ctx *Base, pkg *Package, doer *user_model.User) (perm.AccessMode, error) {
96-
if setting.Service.RequireSignInView && (doer == nil || doer.IsGhost()) {
96+
if setting.Service.RequireSignInViewStrict && (doer == nil || doer.IsGhost()) {
9797
return perm.AccessModeNone, nil
9898
}
9999

‎services/packages/cargo/index.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository,
248248
"Initialize Cargo Config",
249249
func(t *files_service.TemporaryUploadRepository) error {
250250
var b bytes.Buffer
251-
err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInView || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate))
251+
err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInViewStrict || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate))
252252
if err != nil {
253253
return err
254254
}

‎templates/admin/config.tmpl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@
148148
<dt>{{ctx.Locale.Tr "admin.config.enable_openid_signin"}}</dt>
149149
<dd>{{svg (Iif .Service.EnableOpenIDSignIn "octicon-check" "octicon-x")}}</dd>
150150
<dt>{{ctx.Locale.Tr "admin.config.require_sign_in_view"}}</dt>
151-
<dd>{{svg (Iif .Service.RequireSignInView "octicon-check" "octicon-x")}}</dd>
151+
<dd>{{svg (Iif .Service.RequireSignInViewStrict "octicon-check" "octicon-x")}}</dd>
152152
<dt>{{ctx.Locale.Tr "admin.config.mail_notify"}}</dt>
153153
<dd>{{svg (Iif .Service.EnableNotifyMail "octicon-check" "octicon-x")}}</dd>
154154
<dt>{{ctx.Locale.Tr "admin.config.enable_captcha"}}</dt>

‎tests/integration/api_org_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,9 +148,9 @@ func TestAPIOrgEditBadVisibility(t *testing.T) {
148148

149149
func TestAPIOrgDeny(t *testing.T) {
150150
onGiteaRun(t, func(*testing.T, *url.URL) {
151-
setting.Service.RequireSignInView = true
151+
setting.Service.RequireSignInViewStrict = true
152152
defer func() {
153-
setting.Service.RequireSignInView = false
153+
setting.Service.RequireSignInViewStrict = false
154154
}()
155155

156156
orgName := "user1_org"

‎tests/integration/api_packages_container_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func TestPackageContainer(t *testing.T) {
111111
AddTokenAuth(anonymousToken)
112112
MakeRequest(t, req, http.StatusOK)
113113

114-
defer test.MockVariableValue(&setting.Service.RequireSignInView, true)()
114+
defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
115115

116116
req = NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL))
117117
MakeRequest(t, req, http.StatusUnauthorized)

0 commit comments

Comments
 (0)