diff --git a/assets/admin.js b/assets/admin.js index 22e37b8..cd38a0c 100644 --- a/assets/admin.js +++ b/assets/admin.js @@ -342,4 +342,40 @@ function searchRepos() { } searchForm.submit(); +} + +function CreateOrUpdateGrafana(repoID) { + $.ajax({ + type:"PUT", + url: "/admin/grafana/"+repoID+"/update", + data: { + "type": "gitee" + }, + success: function() { + successToast('操作成功') + }, + error: function(e) { + errorToast($.parseJSON(e.responseText).message) + } + }); +} + +function startToGrabRepos() { + $('body').modal('confirm','开始爬取','确认现在就开始爬取数据吗？
如果需要爬取的仓库较多，过程可能比较久。', function(choice){ + if(choice) { + $.ajax({ + type:"POST", + url: "/admin/repos/grab", + data: { + "type": "gitee" + }, + success: function() { + successToast('操作成功') + }, + error: function(e) { + errorToast($.parseJSON(e.responseText).message) + } + }); + }//end of if + }); } \ No newline at end of file diff --git a/controller/grafana_c.go b/controller/grafana_c.go index 1daa17e..5d0cf0f 100644 --- a/controller/grafana_c.go +++ b/controller/grafana_c.go @@ -10,9 +10,13 @@ package controller import ( "encoding/json" + "log" "net/http" "repostats/network" + "repostats/schedule" + gitee_storage "repostats/storage/gitee" "repostats/utils" + "strconv" "github.com/gin-gonic/gin" ) @@ -64,6 +68,98 @@ func GrafanaToken(ctx *gin.Context) { return } + err = network.CreateDatasource(token, utils.DatabaseConifg) + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": err.Error(), + }) + return + } + + datasource, err := network.RetrieveGrafanaDatasource() + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": err.Error(), + }) + return + } + + err = network.CreateGiteeRepostatsFolder(token) + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": err.Error(), + }) + return + } + + folder, err := network.RetrieveGiteeRepostatsFolder() + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": err.Error(), + }) + return + } + + err = network.CreateGiteeHomeDashboard(token, folder, datasource) + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": err.Error(), + }) + return + } + rs, _ := json.Marshal(token) ctx.JSON(http.StatusOK, rs) } + +//创建或更新制定仓库的 Grafana 视图面板 +// +func CreateOrUpdateGrafanaDashboard(ctx *gin.Context) { + strRepoID := ctx.Param("repoID") + + repoID, err := strconv.Atoi(strRepoID) + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": "repoID 参数非法", + }) + return + } + + if repoID <= 0 { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": "repoID 参数非法", + }) + return + } + + repo, err := gitee_storage.FindRepoByID(repoID) + if err != nil { + log.Println(err) + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": "内部错误，请联系管理员！ ", + }) + return + } + + if repo.IsNilOrEmpty() { + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": "找不到指定的仓库，repoID =" + strRepoID, + }) + return + } + + token, _ := network.RetrieveGrafanaToken() + datasource, _ := network.RetrieveGrafanaDatasource() + folder, _ := network.RetrieveGiteeRepostatsFolder() + + err = schedule.CreateOrUpdateGrafanaRepo(repo, token, folder, datasource) + if err != nil { + log.Println(err) + ctx.JSON(http.StatusBadRequest, gin.H{ + "message": "内部错误，请联系管理员！ " + err.Error(), + }) + return + } + + ctx.JSON(http.StatusOK, nil) +} diff --git a/controller/repos_c.go b/controller/repos_c.go index f6f9fc4..f6b2992 100644 --- a/controller/repos_c.go +++ b/controller/repos_c.go @@ -13,6 +13,7 @@ import ( "net/http" gitee_model "repostats/model/gitee" "repostats/network" + "repostats/schedule" "repostats/storage" gitee_storage "repostats/storage/gitee" "repostats/utils" @@ -290,3 +291,8 @@ func RepoDelete(ctx *gin.Context) { ctx.JSON(http.StatusOK, nil) } + +func StartToGrab(ctx *gin.Context) { + go schedule.StarGiteeJobs(false) + ctx.JSON(http.StatusOK, nil) +} diff --git a/docker/vars.env b/docker/vars.env index 269c5c0..d6c2625 100644 --- a/docker/vars.env +++ b/docker/vars.env @@ -8,6 +8,6 @@ PG_SUPER_USER=postgres PG_SUPER_PWD=DePmoG_123 # Grafana Vars -GR_VERSION=8.3.4 +GR_VERSION=8.5.0 GR_CONTAINER_NAME=repostats_gr GR_LOCAL_PORT=13000 \ No newline at end of file diff --git a/network/templates/gitee-home-dashboard.tmpl b/network/templates/gitee-home-dashboard.tmpl index 478e624..7428b37 100644 --- a/network/templates/gitee-home-dashboard.tmpl +++ b/network/templates/gitee-home-dashboard.tmpl @@ -10,6 +10,7 @@ "from": "now-30d", "to": "now" }, + "editable": false, "panels": [ { "collapsed": false, diff --git a/network/templates/gitee-repo-dashboard.tmpl b/network/templates/gitee-repo-dashboard.tmpl index ad4866d..73f6e2d 100644 --- a/network/templates/gitee-repo-dashboard.tmpl +++ b/network/templates/gitee-repo-dashboard.tmpl @@ -10,6 +10,7 @@ "from": "now-30d", "to": "now" }, + "editable": false, "panels": [ { "collapsed": false, @@ -68,7 +69,6 @@ }, "textMode": "auto" }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -187,7 +187,6 @@ }, "textMode": "auto" }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -306,7 +305,6 @@ "showThresholdLabels": false, "showThresholdMarkers": true }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -460,7 +458,6 @@ }, "textMode": "auto" }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -754,7 +751,6 @@ "showThresholdLabels": false, "showThresholdMarkers": true }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -1030,7 +1026,6 @@ }, "textMode": "auto" }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -1630,7 +1625,7 @@ ], "metricColumn": "none", "rawQuery": false, - "rawSql": "SELECT\n $__timeGroupAlias(created_at,$__interval),\n count(id) AS \"Pull Request 数\"\nFROM gitee.pull_requests\nWHERE\n $__timeFilter(created_at)\nGROUP BY 1\nORDER BY 1", + "rawSql": "SELECT\n $__timeGroupAlias(created_at,$__interval),\n count(id) AS \"Pull Request 数\"\nFROM gitee.pull_requests\nWHERE\n $__timeFilter(created_at) and repo_id = {{.repo_id}}\nGROUP BY 1\nORDER BY 1", "refId": "A", "select": [ [ @@ -1662,7 +1657,17 @@ "name": "$__timeFilter", "params": [], "type": "macro" - } + }, + { + "datatype": "int8", + "name": "", + "params": [ + "repo_id", + "=", + "{{.repo_id}}" + ], + "type": "expression" + } ] } ], @@ -1714,7 +1719,6 @@ "showThresholdLabels": false, "showThresholdMarkers": true }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -1870,7 +1874,6 @@ "showThresholdLabels": false, "showThresholdMarkers": true }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -2043,7 +2046,6 @@ } ] }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -2161,7 +2163,6 @@ } ] }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -2399,7 +2400,6 @@ } ] }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -2711,7 +2711,6 @@ } ] }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { @@ -2988,7 +2987,6 @@ } ] }, - "pluginVersion": "8.3.4", "targets": [ { "datasource": { diff --git a/repostats.go b/repostats.go index e37ea2e..fbd98f3 100644 --- a/repostats.go +++ b/repostats.go @@ -46,6 +46,8 @@ func main() { _, err := utils.InitConfig(cmdConfig) utils.ExitOnError(err) + utils.InitWaitingGruop() + _, err = storage.InitDatabaseService() utils.ExitOnError(err) @@ -79,6 +81,7 @@ func initRouter(router *gin.Engine) { admin.PUT("/repos/:repoID/change_state", controller.RepoStateChange) admin.POST("/repos/:repoID/delete", controller.RepoDelete) admin.POST("/repos", controller.AddRepo) + admin.POST("/repos/grab", controller.StartToGrab) admin.GET("/commits", controller.CommitsPage) admin.POST("/commits/:sha/delete", controller.CommitDelete) @@ -88,6 +91,7 @@ func initRouter(router *gin.Engine) { admin.GET("/grafana", controller.GrafanaPage) admin.POST("/grafana/token", controller.GrafanaToken) + admin.PUT("/grafana/:repoID/update", controller.CreateOrUpdateGrafanaDashboard) admin.GET("/issues", controller.IssuesPage) diff --git a/schedule/gitee_schedule.go b/schedule/gitee_schedule.go index 3e50887..d87e210 100644 --- a/schedule/gitee_schedule.go +++ b/schedule/gitee_schedule.go @@ -6,24 +6,20 @@ import ( gitee_model "repostats/model/gitee" "repostats/network" gitee_storage "repostats/storage/gitee" + "repostats/utils" "strings" "time" "github.com/remeh/sizedwaitgroup" ) -const ( - GITEE_SCHEDULER_INTERVAL = 4 * time.Hour - MAX_ROUTINE_NUMBER = 20 -) - // 启动 Gitee 定时器 // func StartGiteeSchedule() error { - ticker := time.NewTicker(GITEE_SCHEDULER_INTERVAL) + ticker := time.NewTicker(utils.GITEE_SCHEDULER_INTERVAL) for range ticker.C { log.Println("[RepoStats] Gitee Schedule Start.") - if err := StarGiteeJobs(); err != nil { + if err := StarGiteeJobs(true); err != nil { log.Printf("[RepoStats] error while doing schedule jobs. %s", err) } log.Println("[RepoStats] Gitee Schedule Finish.") @@ -33,7 +29,7 @@ func StartGiteeSchedule() error { // 启动 Gitee 任务 // -func StarGiteeJobs() error { +func StarGiteeJobs(wait bool) error { //检查 Grafana Token, Datasource grafanaToken, err := network.RetrieveGrafanaToken() @@ -64,7 +60,7 @@ func StarGiteeJobs() error { } //抓取 Gitee 信息并存储到数据库 - wg := sizedwaitgroup.New(MAX_ROUTINE_NUMBER) + wg := utils.WaitingGroup for _, repo := range repos { if !repo.EnableCrawl { continue @@ -72,13 +68,21 @@ func StarGiteeJobs() error { wg.Add() go GrabRepo(&wg, repo, giteeToken, grafanaToken, datasource, folder) } - wg.Wait() + + if wait { + wg.Wait() + } return nil } +//抓取 Gitee 仓库信息并更新 Grafana 面板 +// func GrabRepo(wg *sizedwaitgroup.SizedWaitGroup, repo gitee_model.Repository, giteeToken network.OauthToken, grafanaToken network.GrafanaToken, grafanaDatasource network.GrafanaDatasource, grafanaFolder network.GrafanaFolder) error { - defer wg.Done() + + if wg != nil { + defer wg.Done() + } log.Printf("[RepoStats] start to grab [%s]", repo.HTMLURL) str := strings.Split(repo.FullName, "/") @@ -96,79 +100,85 @@ func GrabRepo(wg *sizedwaitgroup.SizedWaitGroup, repo gitee_model.Repository, } var users []gitee_model.User - commits, err := network.GetGiteeCommits(str[0], str[1]) if err != nil { log.Printf("[RepoStats] failed during GetGiteeCommits %s,%s", repo.HTMLURL, err) - // return err } for i := 0; i < len(commits); i++ { commits[i].RepoID = repo.ID users = append(users, commits[i].Author) users = append(users, commits[i].Committer) } - err = gitee_storage.BulkSaveCommits(commits) - if err != nil { - log.Printf("[RepoStats] failed during BulkSaveCommits %s, %s", repo.HTMLURL, err) - // return err + if len(commits) > 0 { + err = gitee_storage.BulkSaveCommits(commits) + if err != nil { + log.Printf("[RepoStats] failed during BulkSaveCommits %s, %s", repo.HTMLURL, err) + } } issues, err := network.GetGiteeIssues(str[0], str[1]) if err != nil { log.Printf("[RepoStats] failed during GetGiteeIssues %s, %s", repo.HTMLURL, err) - // return err } for i := 0; i < len(issues); i++ { issues[i].RepoID = int64(repo.ID) users = append(users, issues[i].User) } - err = gitee_storage.BulkSaveIssues(issues) - if err != nil { - log.Printf("[RepoStats] failed during BulkSaveIssues %s, %s", repo.HTMLURL, err) - // return err + if len(issues) > 0 { + err = gitee_storage.BulkSaveIssues(issues) + if err != nil { + log.Printf("[RepoStats] failed during BulkSaveIssues %s, %s", repo.HTMLURL, err) + } } prs, err := network.GetGiteePullRequests(str[0], str[1]) if err != nil { log.Printf("[RepoStats] failed during GetGiteePullRequests %s, %s", repo.HTMLURL, err) - // return err } for i := 0; i < len(prs); i++ { prs[i].RepoID = int64(repo.ID) users = append(users, prs[i].User) } usersNeededToSave := gitee_model.RemoveDuplicateUsers(users) - err = gitee_storage.BulkSaveUsers(usersNeededToSave) - if err != nil { - log.Printf("[RepoStats] failed during BulkSaveUsers %s, %s", repo.HTMLURL, err) - // return err + if len(usersNeededToSave) > 0 { + err = gitee_storage.BulkSaveUsers(usersNeededToSave) + if err != nil { + log.Printf("[RepoStats] failed during BulkSaveUsers %s, %s", repo.HTMLURL, err) + } } - err = gitee_storage.BulkSavePullRequests(prs) - if err != nil { - log.Printf("[RepoStats] failed during BulkSavePullRequests %s,%s", repo.HTMLURL, err) - // return err + if len(prs) > 0 { + err = gitee_storage.BulkSavePullRequests(prs) + if err != nil { + log.Printf("[RepoStats] failed during BulkSavePullRequests %s,%s", repo.HTMLURL, err) + } } stargazers, err := network.GetGiteeStargazers(str[0], str[1]) if err != nil { log.Printf("[RepoStats] failed during GetGiteeStargazers %s, %s", repo.HTMLURL, err) - // return err } for i := 0; i < len(stargazers); i++ { stargazers[i].RepoID = int64(repo.ID) } - err = gitee_storage.BulkSaveStargazers(stargazers) - if err != nil { - log.Printf("[RepoStats] failed during BulkSaveStargazers %s, %s", repo.HTMLURL, err) - // return err + if len(stargazers) > 0 { + err = gitee_storage.BulkSaveStargazers(stargazers) + if err != nil { + log.Printf("[RepoStats] failed during BulkSaveStargazers %s, %s", repo.HTMLURL, err) + } } - err = network.CreateGiteeRepoDashboard(grafanaToken, grafanaFolder, grafanaDatasource, repo) + err = CreateOrUpdateGrafanaRepo(repo, grafanaToken, grafanaFolder, grafanaDatasource) if err != nil { log.Printf("[RepoStats] failed during CreateGiteeRepoDashboard %s, %s", repo.HTMLURL, err) - // return err } log.Printf("[RepoStats] finish to grab [%s]", repo.HTMLURL) return nil } + +//创建或更新 Grafana 的项目视图面板 +// +func CreateOrUpdateGrafanaRepo(repo gitee_model.Repository, token network.GrafanaToken, + folder network.GrafanaFolder, datasource network.GrafanaDatasource) error { + return network.CreateGiteeRepoDashboard(token, folder, datasource, repo) +} diff --git a/schedule/gitee_schedule_test.go b/schedule/gitee_schedule_test.go index c0c06f9..71c3aa1 100644 --- a/schedule/gitee_schedule_test.go +++ b/schedule/gitee_schedule_test.go @@ -20,7 +20,7 @@ func TestStarGiteeJobs(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := StarGiteeJobs(); (err != nil) != tt.wantErr { + if err := StarGiteeJobs(true); (err != nil) != tt.wantErr { t.Errorf("StarGiteeJobs() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/storage/gitee/commit_s.go b/storage/gitee/commit_s.go index ed4aaca..f3dd2b5 100644 --- a/storage/gitee/commit_s.go +++ b/storage/gitee/commit_s.go @@ -13,6 +13,8 @@ import ( "repostats/storage" ) +const max_array_size = 60000 + var commitQueryPrefix = `SELECT c.author_name AS "author.name",c.author_email AS "author.email", c.author_name AS "commit.author.name", c.author_email AS "commit.author.email", c.author_date AS "commit.author.date", c.committer_name AS "committer.name",c.committer_email AS "committer.email", @@ -27,6 +29,13 @@ func BulkSaveCommits(commits []gitee_model.Commit) error { ON CONFLICT (repo_id,sha) DO UPDATE SET repo_id=EXCLUDED.repo_id,html_url=EXCLUDED.html_url, author_name=EXCLUDED.author_name,author_email=EXCLUDED.author_email,author_date=EXCLUDED.author_date,committer_name=EXCLUDED.committer_name, committer_email=EXCLUDED.committer_email,committer_date=EXCLUDED.committer_date,detail_message=EXCLUDED.detail_message,tree=EXCLUDED.tree` + if len(commits) > max_array_size { + nc := splitCommitsArray(commits) + for i := 0; i < len(nc); i++ { + storage.DbNamedExec(query, nc[i]) + } + return nil + } return storage.DbNamedExec(query, commits) } @@ -103,3 +112,18 @@ func DeleteCommitBySha(sha string) error { query := `DELETE FROM gitee.commits WHERE sha = $1` return storage.DbExec(query, sha) } + +func splitCommitsArray(slice []gitee_model.Commit) [][]gitee_model.Commit { + var chunks [][]gitee_model.Commit + for i := 0; i < len(slice); i += max_array_size { + end := i + max_array_size + + if end > len(slice) { + end = len(slice) + } + + chunks = append(chunks, slice[i:end]) + } + + return chunks +} diff --git a/templates/issues.html b/templates/issues.html index eb722c9..26e074b 100644 --- a/templates/issues.html +++ b/templates/issues.html @@ -29,7 +29,7 @@ {{range $i,$iss := .issues}} - {{if eq "closed" $iss.State}}{{else if eq "progressing" $iss.State}}{{else}}{{end}} + {{if eq "closed" $iss.State}}{{else if eq "rejected" $iss.State}}{{else}}{{end}} {{add $i 1}} Gitee {{$iss.Number}} diff --git a/templates/repos.html b/templates/repos.html index 07f123e..470a0d4 100644 --- a/templates/repos.html +++ b/templates/repos.html @@ -28,7 +28,10 @@

新增、删除或禁止爬取仓库数据，不会删除 Gitee 数据，但会影响 RepoStats 统计结果

-

+

+ + +

@@ -69,7 +72,9 @@ 禁止爬取 {{else}} 开启爬取 - {{end}} | 删除 + {{end}} + | 更新 Grafana 面板 + | 删除 {{end}} diff --git a/utils/utils.go b/utils/utils.go index 66ef3a8..1ac4819 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -16,13 +16,26 @@ import ( "path/filepath" "reflect" "strings" + "time" + + "github.com/remeh/sizedwaitgroup" +) + +const ( + MAX_ROUTINE_NUMBER = 25 + GITEE_SCHEDULER_INTERVAL = 4 * time.Hour ) var ( - Version = "1.0" - Build = "2204111911" + Version = "1.0" + Build = "2204111911" + WaitingGroup = sizedwaitgroup.SizedWaitGroup{} ) +func InitWaitingGruop() { + WaitingGroup = sizedwaitgroup.New(MAX_ROUTINE_NUMBER) +} + func ExitOnError(err error) { if err != nil { log.Fatalln(err)