Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)

* add migrations

* fix package dependency

* fix lints

* implements migrations except pull requests

* add releases

* migrating releases

* fix bug

* fix lint

* fix migrate releases

* fix tests

* add rollback

* pull request migtations

* fix import

* fix go module vendor

* add tests for upload to gitea

* more migrate options

* fix swagger-check

* fix misspell

* add options on migration UI

* fix log error

* improve UI options on migrating

* add support for username password when migrating from github

* fix tests

* remove comments and fix migrate limitation

* improve error handles

* migrate API will also support migrate milestones/labels/issues/pulls/releases

* fix tests and remove unused codes

* add DownloaderFactory and docs about how to create a new Downloader

* fix misspell

* fix migration docs

* Add hints about migrate options on migration page

* fix tests

Author: lunny

.gitignore

@@ -66,7 +66,7 @@ coverage.all
 /integrations/mssql.ini
 /node_modules
 /modules/indexer/issues/indexers
-
+routers/repo/authorized_keys
 
 # Snapcraft
 snap/.snapcraft/

docs/content/doc/advanced/migrations.en-us.md

@@ -0,0 +1,72 @@
+---
+date: "2019-04-15T17:29:00+08:00"
+title: "Advanced: Migrations Interfaces"
+slug: "migrations-interfaces"
+weight: 30
+toc: true
+draft: false
+menu:
+  sidebar:
+    parent: "advanced"
+    name: "Migrations Interfaces"
+    weight: 55
+    identifier: "migrations-interfaces"
+---
+
+# Migration Features
+
+The new migration features were introduced in Gitea 1.9.0. It defines two interfaces to support migrating 
+repositories data from other git host platforms to gitea or, in the future migrating gitea data to other 
+git host platforms. Currently, only the migrations from github via APIv3 to Gitea is implemented.
+
+First of all, Gitea defines some standard objects in packages `modules/migrations/base`. They are
+ `Repository`, `Milestone`, `Release`, `Label`, `Issue`, `Comment`, `PullRequest`.
+
+## Downloader Interfaces
+
+To migrate from a new git host platform, there are two steps to be updated.
+
+- You should implement a `Downloader` which will get all kinds of repository informations.
+- You should implement a `DownloaderFactory` which is used to detect if the URL matches and 
+create a Downloader.
+- You'll need to register the `DownloaderFactory` via `RegisterDownloaderFactory` on init.
+
+```Go
+type Downloader interface {
+	GetRepoInfo() (*Repository, error)
+	GetMilestones() ([]*Milestone, error)
+	GetReleases() ([]*Release, error)
+	GetLabels() ([]*Label, error)
+	GetIssues(start, limit int) ([]*Issue, error)
+	GetComments(issueNumber int64) ([]*Comment, error)
+	GetPullRequests(start, limit int) ([]*PullRequest, error)
+}
+```
+
+```Go
+type DownloaderFactory interface {
+	Match(opts MigrateOptions) (bool, error)
+	New(opts MigrateOptions) (Downloader, error)
+}
+```
+
+## Uploader Interface
+
+Currently, only a `GiteaLocalUploader` is implemented, so we only save downloaded 
+data via this `Uploader` on the local Gitea instance. Other uploaders are not supported
+and will be implemented in future.
+
+```Go
+// Uploader uploads all the informations
+type Uploader interface {
+	CreateRepo(repo *Repository, includeWiki bool) error
+	CreateMilestone(milestone *Milestone) error
+	CreateRelease(release *Release) error
+	CreateLabel(label *Label) error
+	CreateIssue(issue *Issue) error
+	CreateComment(issueNumber int64, comment *Comment) error
+	CreatePullRequest(pr *PullRequest) error
+	Rollback() error
+}
+
+```

go.mod

@@ -62,6 +62,7 @@ require (
 	github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561
 	github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183
 	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/google/go-github/v24 v24.0.1
 	github.com/gorilla/context v1.1.1
 	github.com/issue9/assert v1.3.2 // indirect
 	github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c
@@ -115,7 +116,7 @@ require (
 	go.etcd.io/bbolt v1.3.2 // indirect
 	golang.org/x/crypto v0.0.0-20180904163835-0709b304e793
 	golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519
-	golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 // indirect
+	golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759
 	golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
 	golang.org/x/text v0.3.0
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

go.sum

@@ -142,6 +142,12 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pO
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-github/v24 v24.0.1 h1:KCt1LjMJEey1qvPXxa9SjaWxwTsCWSq6p2Ju57UR4Q4=
+github.com/google/go-github/v24 v24.0.1/go.mod h1:CRqaW1Uns1TCkP0wqTpxYyRxRjxwvKU/XSS44u6X74M=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
@@ -309,24 +315,29 @@ github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759 h1:TMrx+Qdx7uJAeUbv15N72h5Hmyb5+VDjEiMufAEAM04=
 golang.org/x/oauth2 v0.0.0-20181101160152-c453e0c75759/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=

models/migrate.go

@@ -0,0 +1,145 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import "github.com/go-xorm/xorm"
+
+// InsertIssue insert one issue to database
+func InsertIssue(issue *Issue, labelIDs []int64) error {
+	sess := x.NewSession()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := insertIssue(sess, issue, labelIDs); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error {
+	if issue.MilestoneID > 0 {
+		sess.Incr("num_issues")
+		if issue.IsClosed {
+			sess.Incr("num_closed_issues")
+		}
+		if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil {
+			return err
+		}
+	}
+	if _, err := sess.NoAutoTime().Insert(issue); err != nil {
+		return err
+	}
+	var issueLabels = make([]IssueLabel, 0, len(labelIDs))
+	for _, labelID := range labelIDs {
+		issueLabels = append(issueLabels, IssueLabel{
+			IssueID: issue.ID,
+			LabelID: labelID,
+		})
+	}
+	if _, err := sess.Insert(issueLabels); err != nil {
+		return err
+	}
+	if !issue.IsPull {
+		sess.ID(issue.RepoID).Incr("num_issues")
+		if issue.IsClosed {
+			sess.Incr("num_closed_issues")
+		}
+	} else {
+		sess.ID(issue.RepoID).Incr("num_pulls")
+		if issue.IsClosed {
+			sess.Incr("num_closed_pulls")
+		}
+	}
+	if _, err := sess.NoAutoTime().Update(issue.Repo); err != nil {
+		return err
+	}
+
+	sess.Incr("num_issues")
+	if issue.IsClosed {
+		sess.Incr("num_closed_issues")
+	}
+	if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil {
+		return err
+	}
+
+	if issue.MilestoneID > 0 {
+		if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// InsertComment inserted a comment
+func InsertComment(comment *Comment) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if _, err := sess.NoAutoTime().Insert(comment); err != nil {
+		return err
+	}
+	if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+// InsertPullRequest inserted a pull request
+func InsertPullRequest(pr *PullRequest, labelIDs []int64) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if err := insertIssue(sess, pr.Issue, labelIDs); err != nil {
+		return err
+	}
+	pr.IssueID = pr.Issue.ID
+	if _, err := sess.NoAutoTime().Insert(pr); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+// MigrateRelease migrates release
+func MigrateRelease(rel *Release) error {
+	sess := x.NewSession()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	var oriRel = Release{
+		RepoID:  rel.RepoID,
+		TagName: rel.TagName,
+	}
+	exist, err := sess.Get(&oriRel)
+	if err != nil {
+		return err
+	}
+	if !exist {
+		if _, err := sess.NoAutoTime().Insert(rel); err != nil {
+			return err
+		}
+	} else {
+		rel.ID = oriRel.ID
+		if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil {
+			return err
+		}
+	}
+
+	for i := 0; i < len(rel.Attachments); i++ {
+		rel.Attachments[i].ReleaseID = rel.ID
+	}
+
+	if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/release_test.go

@@ -107,6 +107,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 		IsPrivate:   false,
 		IsMirror:    true,
 		RemoteAddr:  repoPath,
+		Wiki:        true,
 	}
 	mirror, err := MigrateRepository(user, user, migrationOptions)
 	assert.NoError(t, err)

models/repo.go

@@ -896,6 +896,7 @@ type MigrateRepoOptions struct {
 	IsPrivate   bool
 	IsMirror    bool
 	RemoteAddr  string
+	Wiki        bool // include wiki repository
 }
 
 /*
@@ -917,7 +918,7 @@ func wikiRemoteURL(remote string) string {
 	return ""
 }
 
-// MigrateRepository migrates a existing repository from other project hosting.
+// MigrateRepository migrates an existing repository from other project hosting.
 func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
 	repo, err := CreateRepository(doer, u, CreateRepoOptions{
 		Name:        opts.Name,
@@ -930,7 +931,6 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
 	}
 
 	repoPath := RepoPath(u.Name, opts.Name)
-	wikiPath := WikiPath(u.Name, opts.Name)
 
 	if u.IsOrganization() {
 		t, err := u.GetOwnerTeam()
@@ -956,22 +956,25 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
 		return repo, fmt.Errorf("Clone: %v", err)
 	}
 
-	wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
-	if len(wikiRemotePath) > 0 {
-		if err := os.RemoveAll(wikiPath); err != nil {
-			return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
-		}
-
-		if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
-			Mirror:  true,
-			Quiet:   true,
-			Timeout: migrateTimeout,
-			Branch:  "master",
-		}); err != nil {
-			log.Warn("Clone wiki: %v", err)
+	if opts.Wiki {
+		wikiPath := WikiPath(u.Name, opts.Name)
+		wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
+		if len(wikiRemotePath) > 0 {
 			if err := os.RemoveAll(wikiPath); err != nil {
 				return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
 			}
+
+			if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
+				Mirror:  true,
+				Quiet:   true,
+				Timeout: migrateTimeout,
+				Branch:  "master",
+			}); err != nil {
+				log.Warn("Clone wiki: %v", err)
+				if err := os.RemoveAll(wikiPath); err != nil {
+					return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+				}
+			}
 		}
 	}
 

modules/auth/repo_form.go

@@ -51,10 +51,16 @@ type MigrateRepoForm struct {
 	// required: true
 	UID int64 `json:"uid" binding:"Required"`
 	// required: true
-	RepoName    string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
-	Mirror      bool   `json:"mirror"`
-	Private     bool   `json:"private"`
-	Description string `json:"description" binding:"MaxSize(255)"`
+	RepoName     string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
+	Mirror       bool   `json:"mirror"`
+	Private      bool   `json:"private"`
+	Description  string `json:"description" binding:"MaxSize(255)"`
+	Wiki         bool   `json:"wiki"`
+	Milestones   bool   `json:"milestones"`
+	Labels       bool   `json:"labels"`
+	Issues       bool   `json:"issues"`
+	PullRequests bool   `json:"pull_requests"`
+	Releases     bool   `json:"releases"`
 }
 
 // Validate validates the fields

modules/migrations/base/comment.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Comment is a standard comment information
+type Comment struct {
+	PosterName  string
+	PosterEmail string
+	Created     time.Time
+	Content     string
+	Reactions   *Reactions
+}