Update migrated repositories' issues/comments/prs poster id if user has a github external user saved (#7751)

* update migrated issues/comments when login as github

* add get userid when migrating or login with github oauth2

* fix lint

* add migrations for repository service type

* fix build

* remove unnecessary dependencies on migrations

* add cron task to update migrations poster ids and fix posterid when migrating

* fix lint

* fix lint

* improve code

* fix lint

* improve code

* replace releases publish id to actual author id

* fix import

* fix bug

* fix lint

* fix rawdata definition

* fix some bugs

* fix error message

modules/cron/cron.go

@@ -10,6 +10,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/sync"
 	mirror_service "code.gitea.io/gitea/services/mirror"
@@ -18,12 +19,13 @@ import (
 )
 
 const (
-	mirrorUpdate           = "mirror_update"
-	gitFsck                = "git_fsck"
-	checkRepos             = "check_repos"
-	archiveCleanup         = "archive_cleanup"
-	syncExternalUsers      = "sync_external_users"
-	deletedBranchesCleanup = "deleted_branches_cleanup"
+	mirrorUpdate            = "mirror_update"
+	gitFsck                 = "git_fsck"
+	checkRepos              = "check_repos"
+	archiveCleanup          = "archive_cleanup"
+	syncExternalUsers       = "sync_external_users"
+	deletedBranchesCleanup  = "deleted_branches_cleanup"
+	updateMigrationPosterID = "update_migration_post_id"
 )
 
 var c = cron.New()
@@ -117,6 +119,15 @@ func NewContext() {
 			go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)()
 		}
 	}
+
+	entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID))
+	if err != nil {
+		log.Fatal("Cron[Update migrated repositories]: %v", err)
+	}
+	entry.Prev = time.Now()
+	entry.ExecTimes++
+	go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)()
+
 	c.Start()
 }
 

modules/migrations/base/downloader.go

@@ -5,6 +5,8 @@
 
 package base
 
+import "code.gitea.io/gitea/modules/structs"
+
 // Downloader downloads the site repo informations
 type Downloader interface {
 	GetRepoInfo() (*Repository, error)
@@ -21,4 +23,5 @@ type Downloader interface {
 type DownloaderFactory interface {
 	Match(opts MigrateOptions) (bool, error)
 	New(opts MigrateOptions) (Downloader, error)
+	GitServiceType() structs.GitServiceType
 }

modules/migrations/gitea.go

@@ -34,15 +34,17 @@ var (
 
 // GiteaLocalUploader implements an Uploader to gitea sites
 type GiteaLocalUploader struct {
-	doer        *models.User
-	repoOwner   string
-	repoName    string
-	repo        *models.Repository
-	labels      sync.Map
-	milestones  sync.Map
-	issues      sync.Map
-	gitRepo     *git.Repository
-	prHeadCache map[string]struct{}
+	doer           *models.User
+	repoOwner      string
+	repoName       string
+	repo           *models.Repository
+	labels         sync.Map
+	milestones     sync.Map
+	issues         sync.Map
+	gitRepo        *git.Repository
+	prHeadCache    map[string]struct{}
+	userMap        map[int64]int64 // external user id mapping to user id
+	gitServiceType structs.GitServiceType
 }
 
 // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
@@ -52,6 +54,7 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea
 		repoOwner:   repoOwner,
 		repoName:    repoName,
 		prHeadCache: make(map[string]struct{}),
+		userMap:     make(map[int64]int64),
 	}
 }
 
@@ -109,13 +112,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 	}
 
 	r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
-		RepoName:    g.repoName,
-		Description: repo.Description,
-		Mirror:      repo.IsMirror,
-		CloneAddr:   remoteAddr,
-		Private:     repo.IsPrivate,
-		Wiki:        opts.Wiki,
-		Releases:    opts.Releases, // if didn't get releases, then sync them from tags
+		RepoName:       g.repoName,
+		Description:    repo.Description,
+		OriginalURL:    repo.OriginalURL,
+		GitServiceType: opts.GitServiceType,
+		Mirror:         repo.IsMirror,
+		CloneAddr:      remoteAddr,
+		Private:        repo.IsPrivate,
+		Wiki:           opts.Wiki,
+		Releases:       opts.Releases, // if didn't get releases, then sync them from tags
 	})
 
 	g.repo = r
@@ -193,20 +198,38 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 	var rels = make([]*models.Release, 0, len(releases))
 	for _, release := range releases {
 		var rel = models.Release{
-			RepoID:           g.repo.ID,
-			PublisherID:      g.doer.ID,
-			TagName:          release.TagName,
-			LowerTagName:     strings.ToLower(release.TagName),
-			Target:           release.TargetCommitish,
-			Title:            release.Name,
-			Sha1:             release.TargetCommitish,
-			Note:             release.Body,
-			IsDraft:          release.Draft,
-			IsPrerelease:     release.Prerelease,
-			IsTag:            false,
-			CreatedUnix:      timeutil.TimeStamp(release.Created.Unix()),
-			OriginalAuthor:   release.PublisherName,
-			OriginalAuthorID: release.PublisherID,
+			RepoID:       g.repo.ID,
+			TagName:      release.TagName,
+			LowerTagName: strings.ToLower(release.TagName),
+			Target:       release.TargetCommitish,
+			Title:        release.Name,
+			Sha1:         release.TargetCommitish,
+			Note:         release.Body,
+			IsDraft:      release.Draft,
+			IsPrerelease: release.Prerelease,
+			IsTag:        false,
+			CreatedUnix:  timeutil.TimeStamp(release.Created.Unix()),
+		}
+
+		userid, ok := g.userMap[release.PublisherID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[release.PublisherID] = userid
+			}
+		}
+
+		if userid > 0 {
+			rel.PublisherID = userid
+		} else {
+			rel.PublisherID = g.doer.ID
+			rel.OriginalAuthor = release.PublisherName
+			rel.OriginalAuthorID = release.PublisherID
 		}
 
 		// calc NumCommits
@@ -284,20 +307,39 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 		}
 
 		var is = models.Issue{
-			RepoID:           g.repo.ID,
-			Repo:             g.repo,
-			Index:            issue.Number,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   issue.PosterName,
-			OriginalAuthorID: issue.PosterID,
-			Title:            issue.Title,
-			Content:          issue.Content,
-			IsClosed:         issue.State == "closed",
-			IsLocked:         issue.IsLocked,
-			MilestoneID:      milestoneID,
-			Labels:           labels,
-			CreatedUnix:      timeutil.TimeStamp(issue.Created.Unix()),
+			RepoID:      g.repo.ID,
+			Repo:        g.repo,
+			Index:       issue.Number,
+			Title:       issue.Title,
+			Content:     issue.Content,
+			IsClosed:    issue.State == "closed",
+			IsLocked:    issue.IsLocked,
+			MilestoneID: milestoneID,
+			Labels:      labels,
+			CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
 		}
+
+		userid, ok := g.userMap[issue.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[issue.PosterID] = userid
+			}
+		}
+
+		if userid > 0 {
+			is.PosterID = userid
+		} else {
+			is.PosterID = g.doer.ID
+			is.OriginalAuthor = issue.PosterName
+			is.OriginalAuthorID = issue.PosterID
+		}
+
 		if issue.Closed != nil {
 			is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
 		}
@@ -331,15 +373,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 			issueID = issueIDStr.(int64)
 		}
 
-		cms = append(cms, &models.Comment{
-			IssueID:          issueID,
-			Type:             models.CommentTypeComment,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   comment.PosterName,
-			OriginalAuthorID: comment.PosterID,
-			Content:          comment.Content,
-			CreatedUnix:      timeutil.TimeStamp(comment.Created.Unix()),
-		})
+		userid, ok := g.userMap[comment.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[comment.PosterID] = userid
+			}
+		}
+
+		cm := models.Comment{
+			IssueID:     issueID,
+			Type:        models.CommentTypeComment,
+			Content:     comment.Content,
+			CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
+		}
+
+		if userid > 0 {
+			cm.PosterID = userid
+		} else {
+			cm.PosterID = g.doer.ID
+			cm.OriginalAuthor = comment.PosterName
+			cm.OriginalAuthorID = comment.PosterID
+		}
+
+		cms = append(cms, &cm)
 
 		// TODO: Reactions
 	}
@@ -355,6 +417,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 		if err != nil {
 			return err
 		}
+
+		userid, ok := g.userMap[pr.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[pr.PosterID] = userid
+			}
+		}
+
+		if userid > 0 {
+			gpr.Issue.PosterID = userid
+		} else {
+			gpr.Issue.PosterID = g.doer.ID
+			gpr.Issue.OriginalAuthor = pr.PosterName
+			gpr.Issue.OriginalAuthorID = pr.PosterID
+		}
+
 		gprs = append(gprs, gpr)
 	}
 	if err := models.InsertPullRequests(gprs...); err != nil {
@@ -460,6 +544,40 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		head = pr.Head.Ref
 	}
 
+	var issue = models.Issue{
+		RepoID:      g.repo.ID,
+		Repo:        g.repo,
+		Title:       pr.Title,
+		Index:       pr.Number,
+		Content:     pr.Content,
+		MilestoneID: milestoneID,
+		IsPull:      true,
+		IsClosed:    pr.State == "closed",
+		IsLocked:    pr.IsLocked,
+		Labels:      labels,
+		CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
+	}
+
+	userid, ok := g.userMap[pr.PosterID]
+	if !ok {
+		var err error
+		userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID))
+		if err != nil {
+			log.Error("GetUserIDByExternalUserID: %v", err)
+		}
+		if userid > 0 {
+			g.userMap[pr.PosterID] = userid
+		}
+	}
+
+	if userid > 0 {
+		issue.PosterID = userid
+	} else {
+		issue.PosterID = g.doer.ID
+		issue.OriginalAuthor = pr.PosterName
+		issue.OriginalAuthorID = pr.PosterID
+	}
+
 	var pullRequest = models.PullRequest{
 		HeadRepoID:   g.repo.ID,
 		HeadBranch:   head,
@@ -470,22 +588,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		Index:        pr.Number,
 		HasMerged:    pr.Merged,
 
-		Issue: &models.Issue{
-			RepoID:           g.repo.ID,
-			Repo:             g.repo,
-			Title:            pr.Title,
-			Index:            pr.Number,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   pr.PosterName,
-			OriginalAuthorID: pr.PosterID,
-			Content:          pr.Content,
-			MilestoneID:      milestoneID,
-			IsPull:           true,
-			IsClosed:         pr.State == "closed",
-			IsLocked:         pr.IsLocked,
-			Labels:           labels,
-			CreatedUnix:      timeutil.TimeStamp(pr.Created.Unix()),
-		},
+		Issue: &issue,
 	}
 
 	if pullRequest.Issue.IsClosed && pr.Closed != nil {

modules/migrations/github.go

@@ -14,6 +14,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/google/go-github/v24/github"
 	"golang.org/x/oauth2"
@@ -39,7 +40,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error
 		return false, err
 	}
 
-	return u.Host == "github.com" && opts.AuthUsername != "", nil
+	return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil
 }
 
 // New returns a Downloader related to this factory according MigrateOptions
@@ -58,6 +59,11 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
 	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
 }
 
+// GitServiceType returns the type of git service
+func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
+	return structs.GithubService
+}
+
 // GithubDownloaderV3 implements a Downloader interface to get repository informations
 // from github via APIv3
 type GithubDownloaderV3 struct {

modules/migrations/migrate.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
 )
 
 // MigrateOptions is equal to base.MigrateOptions
@@ -30,6 +31,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 	var (
 		downloader base.Downloader
 		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.RepoName)
+		theFactory base.DownloaderFactory
 	)
 
 	for _, factory := range factories {
@@ -40,6 +42,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 			if err != nil {
 				return nil, err
 			}
+			theFactory = factory
 			break
 		}
 	}
@@ -52,10 +55,14 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 		opts.Comments = false
 		opts.Issues = false
 		opts.PullRequests = false
+		opts.GitServiceType = structs.PlainGitService
 		downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
 		log.Trace("Will migrate from git: %s", opts.CloneAddr)
+	} else if opts.GitServiceType == structs.NotMigrated {
+		opts.GitServiceType = theFactory.GitServiceType()
 	}
 
+	uploader.gitServiceType = opts.GitServiceType
 	if err := migrateRepository(downloader, uploader, opts); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)

modules/migrations/update.go

@@ -0,0 +1,59 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"strconv"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/structs"
+)
+
+// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
+func UpdateMigrationPosterID() {
+	for _, gitService := range structs.SupportedFullGitService {
+		if err := updateMigrationPosterIDByGitService(gitService); err != nil {
+			log.Error("updateMigrationPosterIDByGitService failed: %v", err)
+		}
+	}
+}
+
+func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error {
+	provider := tp.Name()
+	if len(provider) == 0 {
+		return nil
+	}
+
+	const batchSize = 100
+	var start int
+	for {
+		users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
+			Provider: provider,
+			Start:    start,
+			Limit:    batchSize,
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, user := range users {
+			externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64)
+			if err != nil {
+				log.Warn("Parse externalUser %#v 's userID failed: %v", user, err)
+				continue
+			}
+			if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
+				log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
+			}
+		}
+
+		if len(users) < batchSize {
+			break
+		}
+		start += len(users)
+	}
+	return nil
+}

modules/setting/cron.go

@@ -49,6 +49,9 @@ var (
 			Schedule   string
 			OlderThan  time.Duration
 		} `ini:"cron.deleted_branches_cleanup"`
+		UpdateMigrationPosterID struct {
+			Schedule string
+		} `ini:"cron.update_migration_poster_id"`
 	}{
 		UpdateMirror: struct {
 			Enabled    bool
@@ -114,6 +117,11 @@ var (
 			Schedule:   "@every 24h",
 			OlderThan:  24 * time.Hour,
 		},
+		UpdateMigrationPosterID: struct {
+			Schedule string
+		}{
+			Schedule: "@every 24h",
+		},
 	}
 )
 

modules/structs/repo.go

@@ -153,6 +153,43 @@ type EditRepoOption struct {
 	Archived *bool `json:"archived,omitempty"`
 }
 
+// GitServiceType represents a git service
+type GitServiceType int
+
+// enumerate all GitServiceType
+const (
+	NotMigrated     GitServiceType = iota // 0 not migrated from external sites
+	PlainGitService                       // 1 plain git service
+	GithubService                         // 2 github.com
+	GiteaService                          // 3 gitea service
+	GitlabService                         // 4 gitlab service
+	GogsService                           // 5 gogs service
+)
+
+// Name represents the service type's name
+// WARNNING: the name have to be equal to that on goth's library
+func (gt GitServiceType) Name() string {
+	switch gt {
+	case GithubService:
+		return "github"
+	case GiteaService:
+		return "gitea"
+	case GitlabService:
+		return "gitlab"
+	case GogsService:
+		return "gogs"
+	}
+	return ""
+}
+
+var (
+	// SupportedFullGitService represents all git services supported to migrate issues/labels/prs and etc.
+	// TODO: add to this list after new git service added
+	SupportedFullGitService = []GitServiceType{
+		GithubService,
+	}
+)
+
 // MigrateRepoOption options for migrating a repository from an external service
 type MigrateRepoOption struct {
 	// required: true
@@ -166,6 +203,8 @@ type MigrateRepoOption struct {
 	Mirror          bool   `json:"mirror"`
 	Private         bool   `json:"private"`
 	Description     string `json:"description"`
+	OriginalURL     string
+	GitServiceType  GitServiceType
 	Wiki            bool
 	Issues          bool
 	Milestones      bool