// Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package migrations import ( "context" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/structs" "github.com/hashicorp/go-version" ) const OneDevRequiredVersion = "12.0.1" var ( _ base.Downloader = &OneDevDownloader{} _ base.DownloaderFactory = &OneDevDownloaderFactory{} ) func init() { RegisterDownloaderFactory(&OneDevDownloaderFactory{}) } // OneDevDownloaderFactory defines a downloader factory type OneDevDownloaderFactory struct{} // New returns a downloader related to this factory according MigrateOptions func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { u, err := url.Parse(opts.CloneAddr) if err != nil { return nil, err } repoPath := strings.Trim(u.Path, "/") u.Path = "" u.Fragment = "" log.Trace("Create onedev downloader. BaseURL: %v RepoPath: %s", u, repoPath) return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoPath), nil } // GitServiceType returns the type of git service func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType { return structs.OneDevService } type onedevUser struct { ID int64 Name string Email string } // OneDevDownloader implements a Downloader interface to get repository information // from OneDev type OneDevDownloader struct { base.NullDownloader client *http.Client baseURL *url.URL repoPath string repoID int64 maxIssueIndex int64 userMap map[int64]*onedevUser milestoneMap map[int64]string } // NewOneDevDownloader creates a new downloader func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader { downloader := &OneDevDownloader{ baseURL: baseURL, repoPath: repoPath, client: &http.Client{ Transport: &http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { if len(username) > 0 && len(password) > 0 { req.SetBasicAuth(username, password) } return nil, nil }, }, }, userMap: make(map[int64]*onedevUser), milestoneMap: make(map[int64]string), } return downloader } // String implements Stringer func (d *OneDevDownloader) String() string { return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoPath) } func (d *OneDevDownloader) LogString() string { if d == nil { return "" } return fmt.Sprintf("", d.baseURL, d.repoID, d.repoPath) } func (d *OneDevDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error { u, err := d.baseURL.Parse(endpoint) if err != nil { return err } if parameter != nil { query := u.Query() for k, v := range parameter { query.Set(k, v) } u.RawQuery = query.Encode() } req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } resp, err := d.client.Do(req) if err != nil { return err } defer resp.Body.Close() // special case to read OneDev server version, which is not valid JSON if presult, ok := result.(**version.Version); ok { bytes, err := io.ReadAll(resp.Body) if err != nil { return err } vers, err := version.NewVersion(string(bytes)) if err != nil { return err } *presult = vers return nil } decoder := json.NewDecoder(resp.Body) return decoder.Decode(&result) } // GetRepoInfo returns repository information func (d *OneDevDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { // check OneDev server version var serverVersion *version.Version err := d.callAPI( ctx, "/~api/version/server", nil, &serverVersion, ) if err != nil { return nil, fmt.Errorf("failed to get OneDev server version; OneDev %s or newer required", OneDevRequiredVersion) } requiredVersion, _ := version.NewVersion(OneDevRequiredVersion) if serverVersion.LessThan(requiredVersion) { return nil, fmt.Errorf("OneDev %s or newer required; currently running OneDev %s", OneDevRequiredVersion, serverVersion) } info := make([]struct { ID int64 `json:"id"` Name string `json:"name"` Path string `json:"path"` Description string `json:"description"` }, 0, 1) err = d.callAPI( ctx, "/~api/projects", map[string]string{ "query": `"Path" is "` + d.repoPath + `"`, "offset": "0", "count": "1", }, &info, ) if err != nil { return nil, err } if len(info) != 1 { return nil, fmt.Errorf("Project %s not found", d.repoPath) } d.repoID = info[0].ID cloneURL, err := d.baseURL.Parse(info[0].Path) if err != nil { return nil, err } return &base.Repository{ Name: info[0].Name, Description: info[0].Description, CloneURL: cloneURL.String(), OriginalURL: cloneURL.String(), }, nil } // GetMilestones returns milestones func (d *OneDevDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { endpoint := fmt.Sprintf("/~api/projects/%d/iterations", d.repoID) milestones := make([]*base.Milestone, 0, 100) offset := 0 for { rawMilestones := make([]struct { ID int64 `json:"id"` Name string `json:"name"` Description string `json:"description"` DueDay int64 `json:"dueDay"` Closed bool `json:"closed"` }, 0, 100) err := d.callAPI( ctx, endpoint, map[string]string{ "offset": strconv.Itoa(offset), "count": "100", }, &rawMilestones, ) if err != nil { return nil, err } if len(rawMilestones) == 0 { break } offset += 100 for _, milestone := range rawMilestones { d.milestoneMap[milestone.ID] = milestone.Name var dueDate *time.Time if milestone.DueDay != 0 { d := time.Unix(milestone.DueDay*24*60*60, 0) dueDate = &d } var closedDate *time.Time state := "open" if milestone.Closed { closedDate = dueDate state = "closed" } milestones = append(milestones, &base.Milestone{ Title: milestone.Name, Description: milestone.Description, Deadline: dueDate, Closed: closedDate, State: state, }) } } return milestones, nil } // GetLabels returns labels func (d *OneDevDownloader) GetLabels(_ context.Context) ([]*base.Label, error) { return []*base.Label{ { Name: "Bug", Color: "f64e60", }, { Name: "Build Failure", Color: "f64e60", }, { Name: "Discussion", Color: "8950fc", }, { Name: "Improvement", Color: "1bc5bd", }, { Name: "New Feature", Color: "1bc5bd", }, { Name: "Support Request", Color: "8950fc", }, }, nil } type onedevIssueContext struct { IsPullRequest bool } // GetIssues returns issues func (d *OneDevDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { type Field struct { Name string `json:"name"` Value string `json:"value"` } rawIssues := make([]struct { ID int64 `json:"id"` Number int64 `json:"number"` State string `json:"state"` Title string `json:"title"` Description string `json:"description"` SubmitterID int64 `json:"submitterId"` SubmitDate time.Time `json:"submitDate"` Fields []Field `json:"fields"` }, 0, perPage) err := d.callAPI( ctx, "/~api/issues", map[string]string{ "query": `"Project" is "` + d.repoPath + `"`, "offset": strconv.Itoa((page - 1) * perPage), "count": strconv.Itoa(perPage), "withFields": "true", }, &rawIssues, ) if err != nil { return nil, false, err } issues := make([]*base.Issue, 0, len(rawIssues)) for _, issue := range rawIssues { var label *base.Label for _, field := range issue.Fields { if field.Name == "Type" { label = &base.Label{Name: field.Value} break } } milestones := make([]struct { ID int64 `json:"id"` Name string `json:"name"` }, 0, 10) err = d.callAPI( ctx, fmt.Sprintf("/~api/issues/%d/iterations", issue.ID), nil, &milestones, ) if err != nil { return nil, false, err } milestoneID := int64(0) if len(milestones) > 0 { milestoneID = milestones[0].ID } state := strings.ToLower(issue.State) if state == "released" { state = "closed" } poster := d.tryGetUser(ctx, issue.SubmitterID) issues = append(issues, &base.Issue{ Title: issue.Title, Number: issue.Number, PosterName: poster.Name, PosterEmail: poster.Email, Content: issue.Description, Milestone: d.milestoneMap[milestoneID], State: state, Created: issue.SubmitDate, Updated: issue.SubmitDate, Labels: []*base.Label{label}, ForeignIndex: issue.ID, Context: onedevIssueContext{IsPullRequest: false}, }) if d.maxIssueIndex < issue.Number { d.maxIssueIndex = issue.Number } } return issues, len(issues) == 0, nil } // GetComments returns comments func (d *OneDevDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(onedevIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) } rawComments := make([]struct { ID int64 `json:"id"` Date time.Time `json:"date"` UserID int64 `json:"userId"` Content string `json:"content"` }, 0, 100) var endpoint string if context.IsPullRequest { endpoint = fmt.Sprintf("/~api/pulls/%d/comments", commentable.GetForeignIndex()) } else { endpoint = fmt.Sprintf("/~api/issues/%d/comments", commentable.GetForeignIndex()) } err := d.callAPI( ctx, endpoint, nil, &rawComments, ) if err != nil { return nil, false, err } rawChanges := make([]struct { Date time.Time `json:"date"` UserID int64 `json:"userId"` Data map[string]any `json:"data"` }, 0, 100) if context.IsPullRequest { endpoint = fmt.Sprintf("/~api/pulls/%d/changes", commentable.GetForeignIndex()) } else { endpoint = fmt.Sprintf("/~api/issues/%d/changes", commentable.GetForeignIndex()) } err = d.callAPI( ctx, endpoint, nil, &rawChanges, ) if err != nil { return nil, false, err } comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges)) for _, comment := range rawComments { if len(comment.Content) == 0 { continue } poster := d.tryGetUser(ctx, comment.UserID) comments = append(comments, &base.Comment{ IssueIndex: commentable.GetLocalIndex(), Index: comment.ID, PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, Content: comment.Content, Created: comment.Date, Updated: comment.Date, }) } for _, change := range rawChanges { contentV, ok := change.Data["content"] if !ok { contentV, ok = change.Data["comment"] if !ok { continue } } content, ok := contentV.(string) if !ok || len(content) == 0 { continue } poster := d.tryGetUser(ctx, change.UserID) comments = append(comments, &base.Comment{ IssueIndex: commentable.GetLocalIndex(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, Content: content, Created: change.Date, Updated: change.Date, }) } return comments, true, nil } // GetPullRequests returns pull requests func (d *OneDevDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { rawPullRequests := make([]struct { ID int64 `json:"id"` Number int64 `json:"number"` Title string `json:"title"` SubmitterID int64 `json:"submitterId"` SubmitDate time.Time `json:"submitDate"` Description string `json:"description"` TargetBranch string `json:"targetBranch"` SourceBranch string `json:"sourceBranch"` BaseCommitHash string `json:"baseCommitHash"` CloseDate *time.Time `json:"closeDate"` Status string `json:"status"` // Possible values: OPEN, MERGED, DISCARDED }, 0, perPage) err := d.callAPI( ctx, "/~api/pulls", map[string]string{ "query": `"Target Project" is "` + d.repoPath + `"`, "offset": strconv.Itoa((page - 1) * perPage), "count": strconv.Itoa(perPage), }, &rawPullRequests, ) if err != nil { return nil, false, err } pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests)) for _, pr := range rawPullRequests { var mergePreview struct { TargetHeadCommitHash string `json:"targetHeadCommitHash"` HeadCommitHash string `json:"headCommitHash"` MergeStrategy string `json:"mergeStrategy"` MergeCommitHash string `json:"mergeCommitHash"` } err := d.callAPI( ctx, fmt.Sprintf("/~api/pulls/%d/merge-preview", pr.ID), nil, &mergePreview, ) if err != nil { return nil, false, err } state := "open" merged := false var closeTime *time.Time var mergedTime *time.Time if pr.Status != "OPEN" { state = "closed" closeTime = pr.CloseDate if pr.Status == "MERGED" { // "DISCARDED" merged = true mergedTime = pr.CloseDate } } poster := d.tryGetUser(ctx, pr.SubmitterID) number := pr.Number + d.maxIssueIndex pullRequests = append(pullRequests, &base.PullRequest{ Title: pr.Title, Number: number, PosterName: poster.Name, PosterID: poster.ID, Content: pr.Description, State: state, Created: pr.SubmitDate, Updated: pr.SubmitDate, Closed: closeTime, Merged: merged, MergedTime: mergedTime, Head: base.PullRequestBranch{ Ref: pr.SourceBranch, SHA: mergePreview.HeadCommitHash, RepoName: d.repoPath, }, Base: base.PullRequestBranch{ Ref: pr.TargetBranch, SHA: mergePreview.TargetHeadCommitHash, RepoName: d.repoPath, }, ForeignIndex: pr.ID, Context: onedevIssueContext{IsPullRequest: true}, }) // SECURITY: Ensure that the PR is safe _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d) } return pullRequests, len(pullRequests) == 0, nil } // GetReviews returns pull requests reviews func (d *OneDevDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` Status string `json:"status"` // Possible values: PENDING, APPROVED, REQUESTED_FOR_CHANGES, EXCLUDED }, 0, 100) err := d.callAPI( ctx, fmt.Sprintf("/~api/pulls/%d/reviews", reviewable.GetForeignIndex()), nil, &rawReviews, ) if err != nil { return nil, err } reviews := make([]*base.Review, 0, len(rawReviews)) for _, review := range rawReviews { state := base.ReviewStatePending content := "" switch review.Status { case "APPROVED": state = base.ReviewStateApproved case "REQUESTED_FOR_CHANGES": state = base.ReviewStateChangesRequested } poster := d.tryGetUser(ctx, review.UserID) reviews = append(reviews, &base.Review{ IssueIndex: reviewable.GetLocalIndex(), ReviewerID: poster.ID, ReviewerName: poster.Name, Content: content, State: state, }) } return reviews, nil } // GetTopics return repository topics func (d *OneDevDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } func (d *OneDevDownloader) tryGetUser(ctx context.Context, userID int64) *onedevUser { user, ok := d.userMap[userID] if !ok { // get user name type RawUser struct { Name string `json:"name"` } var rawUser RawUser err := d.callAPI( ctx, fmt.Sprintf("/~api/users/%d", userID), nil, &rawUser, ) var userName string if err == nil { userName = rawUser.Name } else { userName = fmt.Sprintf("User %d", userID) } // get (primary) user Email address rawEmailAddresses := make([]struct { Value string `json:"value"` Primary bool `json:"primary"` }, 0, 10) err = d.callAPI( ctx, fmt.Sprintf("/~api/users/%d/email-addresses", userID), nil, &rawEmailAddresses, ) var userEmail string if err == nil { for _, email := range rawEmailAddresses { if userEmail == "" || email.Primary { userEmail = email.Value } if email.Primary { break } } } user = &onedevUser{ ID: userID, Name: userName, Email: userEmail, } d.userMap[userID] = user } return user }