Merge branch 'main' into incr-issue-paging-num

Author: noerw

Diff for: custom/conf/app.example.ini

@@ -313,6 +313,7 @@ USER = root
 ;DB_TYPE = sqlite3
 ;PATH= ; defaults to data/gitea.db
 ;SQLITE_TIMEOUT = ; Query timeout defaults to: 500
+;SQLITE_JOURNAL_MODE = ; defaults to sqlite database default (often DELETE), can be used to enable WAL mode. https://www.sqlite.org/pragma.html#pragma_journal_mode
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
@@ -878,6 +879,9 @@ ROUTER = console
 ;; Allow deletion of unadopted repositories
 ;ALLOW_DELETION_OF_UNADOPTED_REPOSITORIES = false
 
+;; Don't allow download source archive files from UI
+;DISABLE_DOWNLOAD_SOURCE_ARCHIVES = false
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;[repository.editor]

Diff for: docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -78,6 +78,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `DEFAULT_BRANCH`: **main**: Default branch name of all repositories.
 - `ALLOW_ADOPTION_OF_UNADOPTED_REPOSITORIES`: **false**: Allow non-admin users to adopt unadopted repositories
 - `ALLOW_DELETION_OF_UNADOPTED_REPOSITORIES`: **false**: Allow non-admin users to delete unadopted repositories
+- `DISABLE_DOWNLOAD_SOURCE_ARCHIVES`: **false**: Don't allow download source archive files from UI
 
 ### Repository - Editor (`repository.editor`)
 
@@ -382,6 +383,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
     - `verify-ca`: Enable TLS with verification of the database server certificate against its root certificate.
     - `verify-full`: Enable TLS and verify the database server name matches the given certificate in either the `Common Name` or `Subject Alternative Name` fields.
 - `SQLITE_TIMEOUT`: **500**: Query timeout for SQLite3 only.
+- `SQLITE_JOURNAL_MODE`: **""**: Change journal mode for SQlite3. Can be used to enable [WAL mode](https://www.sqlite.org/wal.html) when high load causes write congestion. See [SQlite3 docs](https://www.sqlite.org/pragma.html#pragma_journal_mode) for possible values. Defaults to the default for the database file, often DELETE.
 - `ITERATE_BUFFER_SIZE`: **50**: Internal buffer size for iterating.
 - `CHARSET`: **utf8mb4**: For MySQL only, either "utf8" or "utf8mb4". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.

Diff for: docs/content/doc/packages/composer.en-us.md

@@ -60,6 +60,8 @@ curl --user your_username:your_password_or_token \
      https://gitea.example.com/api/packages/testuser/composer?version=1.0.3
 ```
 
+If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password.
+
 The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |

Diff for: docs/content/doc/packages/conan.en-us.md

@@ -37,7 +37,7 @@ conan user --remote {remote} --password {password} {username}
 | -----------| ----------- |
 | `remote`   | The remote name. |
 | `username` | Your Gitea username. |
-| `password` | Your Gitea password or a personal access token. |
+| `password` | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. |
 | `owner`    | The owner of the package. |
 
 For example:

Diff for: docs/content/doc/packages/container.en-us.md

@@ -34,6 +34,8 @@ To push an image or if the image is in a private registry, you have to authentic
 docker login gitea.example.com
 ```
 
+If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password.
+
 ## Image naming convention
 
 Images must follow this naming convention:

Diff for: docs/content/doc/packages/generic.en-us.md

@@ -48,6 +48,8 @@ curl --user your_username:your_password_or_token \
      https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin
 ```
 
+If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password.
+
 The server reponds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |

Diff for: docs/content/doc/packages/helm.en-us.md

@@ -42,7 +42,7 @@ helm cm-push ./{chart_file}.tgz {repo}
 | Parameter    | Description |
 | ------------ | ----------- |
 | `username`   | Your Gitea username. |
-| `password`   | Your Gitea password or a personal access token. |
+| `password`   | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. |
 | `repo`       | The name for the repository. |
 | `chart_file` | The Helm Chart archive. |
 | `owner`      | The owner of the package. |

Diff for: docs/content/doc/packages/nuget.en-us.md

@@ -38,7 +38,7 @@ dotnet nuget add source --name {source_name} --username {username} --password {p
 | ------------- | ----------- |
 | `source_name` | The desired source name. |
 | `username`    | Your Gitea username. |
-| `password`    | Your Gitea password or a personal access token. |
+| `password`    | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. |
 | `owner`       | The owner of the package. |
 
 For example:

Diff for: docs/content/doc/packages/pypi.en-us.md

@@ -42,7 +42,7 @@ password = {password}
 | ------------ | ----------- |
 | `owner`      | The owner of the package. |
 | `username`   | Your Gitea username. |
-| `password`   | Your Gitea password or a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). |
+| `password`   | Your Gitea password. If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) instead of the password. |
 
 ## Publish a package
 

Diff for: docs/content/doc/packages/rubygems.en-us.md

@@ -36,7 +36,7 @@ https://gitea.example.com/api/packages/{owner}/rubygems: Bearer {token}
 | Parameter     | Description |
 | ------------- | ----------- |
 | `owner`       | The owner of the package. |
-| `token`       | Your personal access token. |
+| `token`       | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). |
 
 For example:
 

Diff for: integrations/mirror_push_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
@@ -47,7 +48,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {
 
 	doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)
 
-	mirrors, err := repo_model.GetPushMirrorsByRepoID(srcRepo.ID)
+	mirrors, _, err := repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
 	assert.NoError(t, err)
 	assert.Len(t, mirrors, 1)
 
@@ -72,7 +73,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {
 
 	// Cleanup
 	doRemovePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword, int(mirrors[0].ID))(t)
-	mirrors, err = repo_model.GetPushMirrorsByRepoID(srcRepo.ID)
+	mirrors, _, err = repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
 	assert.NoError(t, err)
 	assert.Len(t, mirrors, 0)
 }

Diff for: models/auth/webauthn.go

@@ -6,7 +6,6 @@ package auth
 
 import (
 	"context"
-	"encoding/base32"
 	"fmt"
 	"strings"
 
@@ -20,14 +19,14 @@ import (
 // ErrWebAuthnCredentialNotExist represents a "ErrWebAuthnCRedentialNotExist" kind of error.
 type ErrWebAuthnCredentialNotExist struct {
 	ID           int64
-	CredentialID string
+	CredentialID []byte
 }
 
 func (err ErrWebAuthnCredentialNotExist) Error() string {
-	if err.CredentialID == "" {
+	if len(err.CredentialID) == 0 {
 		return fmt.Sprintf("WebAuthn credential does not exist [id: %d]", err.ID)
 	}
-	return fmt.Sprintf("WebAuthn credential does not exist [credential_id: %s]", err.CredentialID)
+	return fmt.Sprintf("WebAuthn credential does not exist [credential_id: %x]", err.CredentialID)
 }
 
 // IsErrWebAuthnCredentialNotExist checks if an error is a ErrWebAuthnCredentialNotExist.
@@ -43,7 +42,7 @@ type WebAuthnCredential struct {
 	Name            string
 	LowerName       string `xorm:"unique(s)"`
 	UserID          int64  `xorm:"INDEX unique(s)"`
-	CredentialID    string `xorm:"INDEX VARCHAR(410)"`
+	CredentialID    []byte `xorm:"INDEX VARBINARY(1024)"`
 	PublicKey       []byte
 	AttestationType string
 	AAGUID          []byte
@@ -94,9 +93,8 @@ type WebAuthnCredentialList []*WebAuthnCredential
 func (list WebAuthnCredentialList) ToCredentials() []webauthn.Credential {
 	creds := make([]webauthn.Credential, 0, len(list))
 	for _, cred := range list {
-		credID, _ := base32.HexEncoding.DecodeString(cred.CredentialID)
 		creds = append(creds, webauthn.Credential{
-			ID:              credID,
+			ID:              cred.CredentialID,
 			PublicKey:       cred.PublicKey,
 			AttestationType: cred.AttestationType,
 			Authenticator: webauthn.Authenticator{
@@ -164,11 +162,11 @@ func HasWebAuthnRegistrationsByUID(uid int64) (bool, error) {
 }
 
 // GetWebAuthnCredentialByCredID returns WebAuthn credential by credential ID
-func GetWebAuthnCredentialByCredID(userID int64, credID string) (*WebAuthnCredential, error) {
+func GetWebAuthnCredentialByCredID(userID int64, credID []byte) (*WebAuthnCredential, error) {
 	return getWebAuthnCredentialByCredID(db.DefaultContext, userID, credID)
 }
 
-func getWebAuthnCredentialByCredID(ctx context.Context, userID int64, credID string) (*WebAuthnCredential, error) {
+func getWebAuthnCredentialByCredID(ctx context.Context, userID int64, credID []byte) (*WebAuthnCredential, error) {
 	cred := new(WebAuthnCredential)
 	if found, err := db.GetEngine(ctx).Where("user_id = ? AND credential_id = ?", userID, credID).Get(cred); err != nil {
 		return nil, err
@@ -187,7 +185,7 @@ func createCredential(ctx context.Context, userID int64, name string, cred *weba
 	c := &WebAuthnCredential{
 		UserID:          userID,
 		Name:            name,
-		CredentialID:    base32.HexEncoding.EncodeToString(cred.ID),
+		CredentialID:    cred.ID,
 		PublicKey:       cred.PublicKey,
 		AttestationType: cred.AttestationType,
 		AAGUID:          cred.Authenticator.AAGUID,

Diff for: models/auth/webauthn_test.go

@@ -5,7 +5,6 @@
 package auth
 
 import (
-	"encoding/base32"
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
@@ -61,9 +60,7 @@ func TestCreateCredential(t *testing.T) {
 	res, err := CreateCredential(1, "WebAuthn Created Credential", &webauthn.Credential{ID: []byte("Test")})
 	assert.NoError(t, err)
 	assert.Equal(t, "WebAuthn Created Credential", res.Name)
-	bs, err := base32.HexEncoding.DecodeString(res.CredentialID)
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("Test"), bs)
+	assert.Equal(t, []byte("Test"), res.CredentialID)
 
 	unittest.AssertExistsIf(t, true, &WebAuthnCredential{Name: "WebAuthn Created Credential", UserID: 1})
 }

Diff for: models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/expected_webauthn_credential.yml

@@ -0,0 +1,9 @@
+-
+  id: 1
+  credential_id: "TVHE44TOH7DF7V48SEAIT3EMMJ7TGBOQ289E5AQB34S98LFCUFJ7U2NAVI8RJG6K2F4TC8AQ8KBNO7AGEOQOL9NE43GR63HTEHJSLOG="
+-
+  id: 2
+  credential_id: "051CLMMKB62S6M9M2A4H54K7MMCQALFJ36G4TGB2S9A47APLTILU6C6744CEBG4EKCGV357N21BSLH8JD33GQMFAR6DQ70S76P34J6FR="
+-
+  id: 4
+  credential_id: "APU4B1NDTEVTEM60V4T0FRL7SRJMO9KIE2AKFQ8JDGTQ7VHFI41FDEFTDLBVQEAE4ER49QV2GTGVFDNBO31BPOA3OQN6879OT6MTU3G="

Diff for: models/migrations/fixtures/Test_storeWebauthnCredentialIDAsBytes/webauthn_credential.yml

@@ -0,0 +1,31 @@
+-
+  id: 1
+  lower_name: "u2fkey-correctly-migrated"
+  name: "u2fkey-correctly-migrated"
+  user_id: 1
+  credential_id: "TVHE44TOH7DF7V48SEAIT3EMMJ7TGBOQ289E5AQB34S98LFCUFJ7U2NAVI8RJG6K2F4TC8AQ8KBNO7AGEOQOL9NE43GR63HTEHJSLOG="
+  public_key: 0x040d0967a2cad045011631187576492a0beb5b377954b4f694c5afc8bdf25270f87f09a9ab6ce9c282f447ba71b2f2bae2105b32b847e0704f310f48644e3eddf2
+  attestation_type: 'fido-u2f'
+  sign_count: 1
+  clone_warning: false
+-
+  id: 2
+  lower_name: "non-u2f-key"
+  name: "non-u2f-key"
+  user_id: 1
+  credential_id: "051CLMMKB62S6M9M2A4H54K7MMCQALFJ36G4TGB2S9A47APLTILU6C6744CEBG4EKCGV357N21BSLH8JD33GQMFAR6DQ70S76P34J6FR"
+  public_key: 0x040d0967a2cad045011631187576492a0beb5b377954b4f694c5afc8bdf25270f87f09a9ab6ce9c282f447ba71b2f2bae2105b32b847e0704f310f48644e3eddf2
+  attestation_type: 'none'
+  sign_count: 1
+  clone_warning: false
+-
+  id: 4
+  lower_name: "packed-key"
+  name: "packed-key"
+  user_id: 1
+  credential_id: "APU4B1NDTEVTEM60V4T0FRL7SRJMO9KIE2AKFQ8JDGTQ7VHFI41FDEFTDLBVQEAE4ER49QV2GTGVFDNBO31BPOA3OQN6879OT6MTU3G="
+  public_key: 0x040d0967a2cad045011631187576492a0beb5b377954b4f694c5afc8bdf25270f87f09a9ab6ce9c282f447ba71b2f2bae2105b32b847e0704f310f48644e3eddf2
+  attestation_type: 'fido-u2f'
+  sign_count: 1
+  clone_warning: false
+

Diff for: models/migrations/migrations.go

@@ -400,6 +400,12 @@ var migrations = []Migration{
 	NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror),
 	// v220 -> v221
 	NewMigration("Add container repository property", addContainerRepositoryProperty),
+	// v221 -> v222
+	NewMigration("Store WebAuthentication CredentialID as bytes and increase size to at least 1024", storeWebauthnCredentialIDAsBytes),
+	// v222 -> v223
+	NewMigration("Drop old CredentialID column", dropOldCredentialIDColumn),
+	// v223 -> v224
+	NewMigration("Rename CredentialIDBytes column to CredentialID", renameCredentialIDBytes),
 }
 
 // GetCurrentDBVersion returns the current db version

Diff for: models/migrations/v221.go

@@ -0,0 +1,75 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"encoding/base32"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func storeWebauthnCredentialIDAsBytes(x *xorm.Engine) error {
+	// Create webauthnCredential table
+	type webauthnCredential struct {
+		ID           int64 `xorm:"pk autoincr"`
+		Name         string
+		LowerName    string `xorm:"unique(s)"`
+		UserID       int64  `xorm:"INDEX unique(s)"`
+		CredentialID string `xorm:"INDEX VARCHAR(410)"`
+		// Note the lack of INDEX here - these will be created once the column is renamed in v223.go
+		CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022
+		PublicKey         []byte
+		AttestationType   string
+		AAGUID            []byte
+		SignCount         uint32 `xorm:"BIGINT"`
+		CloneWarning      bool
+		CreatedUnix       timeutil.TimeStamp `xorm:"INDEX created"`
+		UpdatedUnix       timeutil.TimeStamp `xorm:"INDEX updated"`
+	}
+	if err := x.Sync2(&webauthnCredential{}); err != nil {
+		return err
+	}
+
+	var start int
+	creds := make([]*webauthnCredential, 0, 50)
+	for {
+		err := x.Select("id, credential_id").OrderBy("id").Limit(50, start).Find(&creds)
+		if err != nil {
+			return err
+		}
+
+		err = func() error {
+			sess := x.NewSession()
+			defer sess.Close()
+			if err := sess.Begin(); err != nil {
+				return fmt.Errorf("unable to allow start session. Error: %w", err)
+			}
+			for _, cred := range creds {
+				cred.CredentialIDBytes, err = base32.HexEncoding.DecodeString(cred.CredentialID)
+				if err != nil {
+					return fmt.Errorf("unable to parse credential id %s for credential[%d]: %w", cred.CredentialID, cred.ID, err)
+				}
+				count, err := sess.ID(cred.ID).Cols("credential_id_bytes").Update(cred)
+				if count != 1 || err != nil {
+					return fmt.Errorf("unable to update credential id bytes for credential[%d]: %d,%w", cred.ID, count, err)
+				}
+			}
+			return sess.Commit()
+		}()
+		if err != nil {
+			return err
+		}
+
+		if len(creds) < 50 {
+			break
+		}
+		start += 50
+		creds = creds[:0]
+	}
+	return nil
+}