完成任务组装和分发

This commit is contained in:
巴拉迪维 2022-04-24 19:32:13 +08:00
parent 7b43576e0e
commit 3ca7d88759
9 changed files with 303 additions and 18 deletions

1
go.mod
View File

@ -14,6 +14,7 @@ require (
github.com/jmoiron/sqlx v1.3.4
github.com/lib/pq v1.10.5
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/remeh/sizedwaitgroup v1.0.0 // indirect
gopkg.in/guregu/null.v4 v4.0.0 // indirect
gopkg.in/ini.v1 v1.66.4
)

2
go.sum
View File

@ -53,6 +53,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

View File

@ -18,11 +18,13 @@ import (
"time"
)
const COMMITS_SINCE = "2021-01-01"
// 获取 Gitee 仓库信息
//
func GetGiteeRepo(owner, repo string) (gitee_model.Repository, error) {
var foundRepo = gitee_model.Repository{}
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return foundRepo, err
}
@ -45,7 +47,7 @@ func GetGiteeRepo(owner, repo string) (gitee_model.Repository, error) {
// login 是 Gitee 登录帐号
func GetGiteeUserInfo(login string) (gitee_model.User, error) {
var foundUser gitee_model.User
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return foundUser, err
}
@ -70,7 +72,7 @@ func GetGiteeUserInfo(login string) (gitee_model.User, error) {
// login 是 Gitee 登录帐号
func GetGiteeOrgInfo(login string) (gitee_model.User, error) {
var foundUser gitee_model.User
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return foundUser, err
}
@ -95,7 +97,7 @@ func GetGiteeOrgInfo(login string) (gitee_model.User, error) {
//
func GetGiteeStargazers(owner, repo string) ([]gitee_model.Stargazer, error) {
var allStargazers = []gitee_model.Stargazer{}
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return allStargazers, err
}
@ -136,7 +138,7 @@ func GetGiteeStargazers(owner, repo string) ([]gitee_model.Stargazer, error) {
// 获取仓库的 Collaborators
func GetGiteeCollaborators(owner, repo string) ([]gitee_model.Collaborator, error) {
var foundUser = []gitee_model.Collaborator{}
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return foundUser, err
}
@ -177,7 +179,7 @@ func GetGiteeCollaborators(owner, repo string) ([]gitee_model.Collaborator, erro
//
//
func GetGiteePullRequests(owner, repo string) ([]gitee_model.PullRequest, error) {
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return nil, err
}
@ -220,7 +222,7 @@ func GetGiteePullRequests(owner, repo string) ([]gitee_model.PullRequest, error)
//
// 调用此方法之前,务必确保是组织帐号
func GetGiteeOrgRepos(org string) ([]gitee_model.Repository, error) {
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return nil, err
}
@ -264,7 +266,7 @@ func GetGiteeOrgRepos(org string) ([]gitee_model.Repository, error) {
//
// 调用此方法之前,务必确保是个人帐号
func GetGiteeUserRepos(name string) ([]gitee_model.Repository, error) {
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return nil, err
}
@ -310,7 +312,7 @@ func GetGiteeUserRepos(name string) ([]gitee_model.Repository, error) {
//
func GetGiteeIssues(owner, repo string) ([]gitee_model.Issue, error) {
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return nil, err
}
@ -353,7 +355,7 @@ func GetGiteeIssues(owner, repo string) ([]gitee_model.Issue, error) {
//
// 从制定的 owner 和 repo 中获取全部提交
func GetGiteeCommits(owner, repo string) ([]gitee_model.Commit, error) {
token, err := validGiteeToken()
token, err := ValidGiteeToken()
if err != nil {
return nil, err
}
@ -365,6 +367,7 @@ func GetGiteeCommits(owner, repo string) ([]gitee_model.Commit, error) {
code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE),
"since": COMMITS_SINCE,
})
if err != nil {
@ -393,7 +396,7 @@ func GetGiteeCommits(owner, repo string) ([]gitee_model.Commit, error) {
// 获取一个可用、有效的 token
//
// 先从本地配置文件中获取 access_token ,如果该 access_token 已失效,则调用 refreshGiteeToken() 更新
func validGiteeToken() (OauthToken, error) {
func ValidGiteeToken() (OauthToken, error) {
var token OauthToken
token, err := retrieveGiteeToken()
if err != nil {

174
schedule/gitee_schedule.go Normal file
View File

@ -0,0 +1,174 @@
package schedule
import (
"fmt"
"log"
gitee_model "repostats/model/gitee"
"repostats/network"
gitee_storage "repostats/storage/gitee"
"strings"
"time"
"github.com/remeh/sizedwaitgroup"
)
const (
GITEE_SCHEDULER_INTERVAL = 4 * time.Hour
MAX_ROUTINE_NUMBER = 20
)
// 启动 Gitee 定时器
//
func StartGiteeSchedule() error {
ticker := time.NewTicker(GITEE_SCHEDULER_INTERVAL)
for range ticker.C {
log.Println("[RepoStats] Gitee Schedule Start.")
if err := StarGiteeJobs(); err != nil {
log.Printf("[RepoStats] error while doing schedule jobs. %s", err)
}
log.Println("[RepoStats] Gitee Schedule Finish.")
}
return nil
}
// 启动 Gitee 任务
//
func StarGiteeJobs() error {
//检查 Grafana Token, Datasource
grafanaToken, err := network.RetrieveGrafanaToken()
if err != nil {
return fmt.Errorf("无法获取 Grafana Token. \n %s", err)
}
datasource, err := network.RetrieveGrafanaDatasource()
if err != nil {
return fmt.Errorf("无法获取 Grafana 必须的数据源配置. \n %s", err)
}
folder, err := network.RetrieveGiteeRepostatsFolder()
if err != nil {
return fmt.Errorf("无法获取 Grafana 必须的 Folder . \n %s", err)
}
//检查 Gitee Token
giteeToken, err := network.ValidGiteeToken()
if err != nil {
return fmt.Errorf("无法获取 Gitee Token. \n %s", err)
}
//获取所有需要爬取的仓库信息
repos, err := gitee_storage.FindRepos()
if err != nil {
return fmt.Errorf("无法获取需要爬取的仓库. \n %s", err)
}
//抓取 Gitee 信息并存储到数据库
wg := sizedwaitgroup.New(MAX_ROUTINE_NUMBER)
for _, repo := range repos {
if !repo.EnableCrawl {
continue
}
wg.Add()
go GrabRepo(&wg, repo, giteeToken, grafanaToken, datasource, folder)
}
wg.Wait()
return nil
}
func GrabRepo(wg *sizedwaitgroup.SizedWaitGroup, repo gitee_model.Repository,
giteeToken network.OauthToken, grafanaToken network.GrafanaToken, grafanaDatasource network.GrafanaDatasource, grafanaFolder network.GrafanaFolder) error {
defer wg.Done()
log.Printf("[RepoStats] start to grab [%s]", repo.HTMLURL)
str := strings.Split(repo.FullName, "/")
repoInfo, err := network.GetGiteeRepo(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeRepo %s", repo.HTMLURL)
return err
}
repoInfo.EnableCrawl = true
err = gitee_storage.BulkSaveRepos([]gitee_model.Repository{repoInfo}) //update newest repo info
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveRepos %s,%s", repo.HTMLURL, err)
return err
}
var users []gitee_model.User
commits, err := network.GetGiteeCommits(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeCommits %s,%s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(commits); i++ {
commits[i].RepoID = repo.ID
users = append(users, commits[i].Author)
users = append(users, commits[i].Committer)
}
err = gitee_storage.BulkSaveCommits(commits)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveCommits %s, %s", repo.HTMLURL, err)
// return err
}
issues, err := network.GetGiteeIssues(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeIssues %s, %s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(issues); i++ {
issues[i].RepoID = int64(repo.ID)
users = append(users, issues[i].User)
}
err = gitee_storage.BulkSaveIssues(issues)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveIssues %s, %s", repo.HTMLURL, err)
// return err
}
prs, err := network.GetGiteePullRequests(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteePullRequests %s, %s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(prs); i++ {
prs[i].RepoID = int64(repo.ID)
users = append(users, prs[i].User)
}
usersNeededToSave := gitee_model.RemoveDuplicateUsers(users)
err = gitee_storage.BulkSaveUsers(usersNeededToSave)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveUsers %s, %s", repo.HTMLURL, err)
// return err
}
err = gitee_storage.BulkSavePullRequests(prs)
if err != nil {
log.Printf("[RepoStats] failed during BulkSavePullRequests %s,%s", repo.HTMLURL, err)
// return err
}
stargazers, err := network.GetGiteeStargazers(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeStargazers %s, %s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(stargazers); i++ {
stargazers[i].RepoID = int64(repo.ID)
}
err = gitee_storage.BulkSaveStargazers(stargazers)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveStargazers %s, %s", repo.HTMLURL, err)
// return err
}
err = network.CreateGiteeRepoDashboard(grafanaToken, grafanaFolder, grafanaDatasource, repo)
if err != nil {
log.Printf("[RepoStats] failed during CreateGiteeRepoDashboard %s, %s", repo.HTMLURL, err)
// return err
}
log.Printf("[RepoStats] finish to grab [%s]", repo.HTMLURL)
return nil
}

View File

@ -0,0 +1,38 @@
package schedule
import (
"log"
"repostats/storage"
"repostats/utils"
"testing"
)
func TestStarGiteeJobs(t *testing.T) {
testSetup(t)
defer testTeardown(t)
tests := []struct {
name string
wantErr bool
}{
{name: "TestCase after OH", wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := StarGiteeJobs(); (err != nil) != tt.wantErr {
t.Errorf("StarGiteeJobs() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func testSetup(t *testing.T) {
utils.InitConfig("../repostats.ini")
storage.InitDatabaseService()
log.Println("test start -->")
}
func testTeardown(t *testing.T) {
log.Println("test done <--")
}

View File

@ -243,4 +243,71 @@ SELECT
NOW() AS time
FROM gitee.commits c
GROUP BY c.repo_id, c.committer_email
ORDER BY COUNT(c.sha) DESC;
ORDER BY COUNT(c.sha) DESC;
-- pr merge time hours diff
CREATE OR REPLACE VIEW gitee_state.pr_merge_hours_diff AS
SELECT
pr.id AS pr_id,
pr.repo_id AS repo_id,
pr.created_at AS created_at,
pr.merged_at AS merged_at,
EXTRACT(EPOCH FROM (pr.merged_at - pr.created_at))/3600 AS hours_diff
FROM gitee.pull_requests pr WHERE pr.mergeable = TRUE AND pr.state = 'merged';
-- issue close time hours diff
CREATE OR REPLACE VIEW gitee_state.issue_close_hours_diff AS
SELECT
iss.repo_id AS repo_id,
iss.id AS issue_id,
iss.created_at AS created_at,
iss.finished_at AS finished_at,
EXTRACT(EPOCH FROM (iss.finished_at - iss.created_at))/3600 AS hours_diff
FROM gitee.issues iss WHERE iss.state = 'closed' OR iss.state = 'rejected';
-- commit list
CREATE OR REPLACE VIEW gitee_state.commits_list AS
SELECT
'Gitee' AS platform,
r.id AS repo_id,
r.full_name AS repo_name,
c.sha AS sha,
c.detail_message AS message,
c.author_name AS author_name,
c.author_email AS author_email,
c.author_date AS author_date,
c.committer_name AS committer_name,
c.committer_email AS committer_email,
c.committer_date AS committer_date
FROM gitee.commits c , gitee.repos r
WHERE c.repo_id = r.id;
-- issue list
CREATE OR REPLACE VIEW gitee_state.issues_list AS
SELECT
'Gitee' AS platform,
r.id AS repo_id,
r.full_name AS repo_name,
iss.state AS issue_state,
u."name" AS user_name,
iss.title AS title,
iss.created_at AS created_at
FROM gitee.issues iss , gitee.repos r , gitee.users u
WHERE iss.repo_id = r.id AND iss.user_id = u.id;
-- pull request list
CREATE OR REPLACE VIEW gitee_state.prs_list AS
SELECT
'Gitee' AS platform,
pr.id AS pr_id,
pr.repo_id AS repo_id,
pr."number" AS pr_number,
pr.created_at AS created_at,
pr.title AS title,
pr.mergeable AS mergeable,
r.full_name AS repo_name,
u."name" AS user_name
FROM gitee.pull_requests pr , gitee.repos r, gitee.users u
WHERE pr.repo_id = r.id AND pr.user_id = u.id;

View File

@ -59,7 +59,7 @@ func TestBulkSaveRepos(t *testing.T) {
args args
wantErr bool
}{
{name: "TestCase barat/all", args: args{found1}, wantErr: false},
// {name: "TestCase barat/all", args: args{found1}, wantErr: false},
{name: "TestCase openharmony/all", args: args{found2}, wantErr: false},
}
for _, tt := range tests {

View File

@ -38,8 +38,8 @@
<td class="center aligned collapsing">{{$iss.IssueState}}</td>
<td><a href="{{$iss.HTMLURL}}" target="_blank">{{$iss.Title}}</a></td>
<td class="center aligned collapsing">{{$iss.CreatedAt | date "2006-01-02 15:04:05"}}</td>
<td class="center aligned collapsing">{{if not $iss.UpdatedAt.IsZero}}{{$iss.UpdatedAt | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{if not $iss.FinishedAt.IsZero}}{{$iss.FinishedAt | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{if not $iss.UpdatedAt.Time.IsZero}}{{$iss.UpdatedAt.Time | date "2006-01-02 15:04:05"}}{{else}}--{{end}}</td>
<td class="center aligned collapsing">{{if not $iss.FinishedAt.IsZero}}{{$iss.FinishedAt.Time | date "2006-01-02 15:04:05"}}{{else}}--{{end}}</td>
<td class="center aligned collapsing">
--
</td>

View File

@ -38,10 +38,10 @@
<td><a href="{{$pr.HTMLURL}}" target="_blank">{{$pr.Title}}</a></td>
<td class="center aligned collapsing">{{$pr.State}}</td>
<td class="center aligned collapsing">{{$pr.CreatedAt | date "2006-01-02 15:04:05"}}</td>
<td class="center aligned collapsing">{{if not $pr.ClosedAt.IsZero}}{{$pr.UpdatedAt | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{if not $pr.ClosedAt.IsZero}}{{$pr.ClosedAt | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{if not $pr.ClosedAt.IsZero}}{{$pr.UpdatedAt.Time | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{if not $pr.ClosedAt.IsZero}}{{$pr.ClosedAt.Time | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{$pr.Mergeable}}</td>
<td class="center aligned collapsing">{{if not $pr.MergedAt.IsZero}}{{$pr.MergedAt | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">{{if not $pr.MergedAt.IsZero}}{{$pr.MergedAt.Time | date "2006-01-02 15:04:05"}}{{end}}</td>
<td class="center aligned collapsing">
<a href="javascript:deletePR('{{$pr.ID}}')">删除</a>
</td>