diff --git a/assets/admin.css b/assets/admin.css index f34f1a3..a0de282 100644 --- a/assets/admin.css +++ b/assets/admin.css @@ -1,3 +1,11 @@ +/* Copyright (c) [2022] [巴拉迪维 BaratSemet] +[ohUrlShortener] is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + #admin-left-menu { max-width: 200px; } diff --git a/assets/admin.js b/assets/admin.js index eede20d..3972e8e 100644 --- a/assets/admin.js +++ b/assets/admin.js @@ -1,5 +1,21 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + $(document).ready(function() { + $('.message .close') + .on('click', function() { + $(this) + .closest('.message') + .transition('fade') + ; + }); + $('#form-gitee-authorize').form({ fields: { client_id: { @@ -74,4 +90,4 @@ function GetGiteeToken(form) { $('#gitee-token-message').removeClass('positive').addClass('error').addClass('visible'); } }); -} \ No newline at end of file +}//end of function GetGiteeToken \ No newline at end of file diff --git a/model/gitee/commit_m.go b/model/gitee/commit_m.go new file mode 100644 index 0000000..9bc98b7 --- /dev/null +++ b/model/gitee/commit_m.go @@ -0,0 +1,44 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + "reflect" + "time" +) + +type Commit struct { + RepoID int `db:"repo_id"` + RepoURL string `db:"repo_url"` + + Sha string `db:"sha" json:"sha"` + CommitURL string `db:"commit_url" json:"html_url"` + + Committer User `json:"committer"` + Author User `json:"author"` + + Detail struct { + Author struct { + User + Date time.Time `json:"date" db:"date"` + } `json:"author" db:"author"` + Committer struct { + User + Date time.Time `json:"date" db:"date"` + } `json:"committer" db:"committer"` + Message string `json:"message" db:"message"` + Tree struct { + Sha string `json:"sha"` + } `json:"tree"` + } `json:"commit" db:"commit"` +} + +func (u Commit) isNilOrEmpty() bool { + return reflect.DeepEqual(u, Commit{}) +} diff --git a/model/gitee/common.go b/model/gitee/common.go deleted file mode 100644 index 7fdc86f..0000000 --- a/model/gitee/common.go +++ /dev/null @@ -1,9 +0,0 @@ -package gitee - -const ( - GITEE_TOKEN_FILE = "gitee_oauth_token.json" - GITEE_OAUTH_V5PREFIX = "https://gitee.com/api/v5" - GITEE_OAUTH_TOKEN_URL = "https://gitee.com/oauth/token" - GITEE_API_START_PAGE = 0 - GITEE_API_PAGE_SIZE = 30 -) diff --git a/model/gitee/const_m.go b/model/gitee/const_m.go new file mode 100644 index 0000000..862c261 --- /dev/null +++ b/model/gitee/const_m.go @@ -0,0 +1,17 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +const ( + GITEE_TOKEN_FILE = "gitee_oauth_token.json" + GITEE_OAUTH_V5PREFIX = "https://gitee.com/api/v5" + GITEE_OAUTH_TOKEN_URL = "https://gitee.com/oauth/token" + GITEE_API_START_PAGE = 0 + GITEE_API_PAGE_SIZE = 55 +) diff --git a/model/gitee/issue_m.go b/model/gitee/issue_m.go new file mode 100644 index 0000000..6c579b9 --- /dev/null +++ b/model/gitee/issue_m.go @@ -0,0 +1,33 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + "time" +) + +type Issue struct { + ID int `json:"id" db:"id"` + RepositoryURL string `json:"repository_url" db:"repository_url"` + HTMLURL string `json:"html_url" db:"html_url"` + Number string `json:"number" db:"number"` + State string `json:"state" db:"state"` + Title string `json:"title" db:"title"` + User User `json:"user" db:"user"` + Repository Repository `json:"repository" db:"repository"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + FinishedAt time.Time `json:"finished_at" db:"finished_at"` + PlanStarted_at time.Time `json:"plan_started_at" db:"plan_started_at"` + Comments int `json:"comments" db:"comments"` + Priority int `json:"priority" db:"priority"` + IssueType string `json:"issue_type" db:"issue_type"` + SecurityHole bool `json:"security_hole" db:"security_hole"` + IssueState string `json:"issue_state" db:"issue_state"` +} diff --git a/model/gitee/repository_m.go b/model/gitee/repository_m.go new file mode 100644 index 0000000..fcf8595 --- /dev/null +++ b/model/gitee/repository_m.go @@ -0,0 +1,41 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + "reflect" + "time" +) + +type Repository struct { + ID int `json:"id" db:"id"` + FullName string `json:"full_name" db:"full_name"` + HumanName string `json:"human_name" db:"human_name"` + Path string `json:"path" db:"path"` + Name string `json:"name" db:"name"` + URL string `json:"url" db:"url"` + Owner User `json:"owner" db:"owner"` + Assigner User `json:"assigner" db:"assigner"` + Description string `json:"description" db:"description"` + HTMLURL string `json:"html_url" db:"html_url"` + SSHURL string `json:"ssh_url" db:"ssh_url"` + Fork bool `json:"fork" db:"forked_repo"` + DefaultBranch string `json:"default_branch" db:"default_branch"` + ForksCount int `json:"forks_count" db:"forks_count"` + StargazersCount int `json:"stargazers_count" db:"stargazers_count"` + WatchersCount int `json:"watchers_count" db:"watchers_count"` + License string `json:"license" db:"license"` + PushedAt time.Time `json:"pushed_at" db:"pushed_at"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +func (r Repository) isNilOrEmpty() bool { + return reflect.DeepEqual(r, Repository{}) +} diff --git a/model/gitee/user_m.go b/model/gitee/user_m.go new file mode 100644 index 0000000..7146955 --- /dev/null +++ b/model/gitee/user_m.go @@ -0,0 +1,40 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + "reflect" + "time" +) + +type User struct { + ID int `json:"id"` + Login string `json:"login" db:"login"` + Name string `json:"name" db:"name"` + AvatarURL string `json:"avatar_url"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + Remark string `json:"remark"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + Date time.Time `json:"date"` + Email string `json:"email" db:"email"` +} + +func (u User) isNilOrEmpty() bool { + return reflect.DeepEqual(u, User{}) +} diff --git a/network/base_n.go b/network/base_n.go index 75d61b0..024f68a 100644 --- a/network/base_n.go +++ b/network/base_n.go @@ -17,7 +17,7 @@ import ( const ( CONTENT_TYPE = "application/json" - USER_AGENT = "RepoStats https://gitee.com/barat" + USER_AGENT = "RepoStats https://gitee.com/barat | https://github.com/barats" ) type OauthToken struct { diff --git a/network/gitee_n.go b/network/gitee_n.go new file mode 100644 index 0000000..a82591e --- /dev/null +++ b/network/gitee_n.go @@ -0,0 +1,245 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package network + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + gitee_model "repostats/model/gitee" + "repostats/utils" + "strconv" + "time" +) + +// 获取组织下的所有公开仓库 +// +// 调用此方法之前,务必确保是组织帐号 +func GetOrgRepos(org string) ([]gitee_model.Repository, error) { + token, err := validGiteeToken() + if err != nil { + return nil, err + } + + url := fmt.Sprintf("%s/orgs/%s/repos", gitee_model.GITEE_OAUTH_V5PREFIX, org) + page := gitee_model.GITEE_API_START_PAGE + allRepos := []gitee_model.Repository{} + for { + page += 1 + code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{ + "type": "public", + "page": strconv.Itoa(page), + "per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE), + }) + + if err != nil { + return allRepos, err + } + + if code != http.StatusOK { + return allRepos, errors.New("unexpected StatusCode") + } + + var foundRepos = []gitee_model.Repository{} + err = json.Unmarshal([]byte(rs), &foundRepos) + if err != nil { + return allRepos, err + } + + if len(foundRepos) > 0 { + allRepos = append(allRepos, foundRepos...) + continue + } + + break + } //end of for + return allRepos, nil +} + +// 获取个人用户名下的所有公开仓库 +// +// 调用此方法之前,务必确保是个人帐号 +func GetUserRepos(name string) ([]gitee_model.Repository, error) { + token, err := validGiteeToken() + if err != nil { + return nil, err + } + + url := fmt.Sprintf("%s/users/%s/repos", gitee_model.GITEE_OAUTH_V5PREFIX, name) + page := gitee_model.GITEE_API_START_PAGE + allRepos := []gitee_model.Repository{} + for { + page += 1 + code, res, err := HttpGet(token.AccessToken, url, nil, map[string]string{ + "page": strconv.Itoa(page), + "per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE), + "type": "all", + }) + + if err != nil { + return allRepos, err + } + + if code != http.StatusOK { + return allRepos, errors.New("unexpected StatusCode") + } + + var foundRepos = []gitee_model.Repository{} + err = json.Unmarshal([]byte(res), &foundRepos) + if err != nil { + return allRepos, err + } + + if len(foundRepos) > 0 { + allRepos = append(allRepos, foundRepos...) + continue + } + + break + } //end of for + + return allRepos, nil +} + +//获取指定仓库的 issue +// +// +func GetIssues(owner string, repo string) ([]gitee_model.Issue, error) { + + token, err := validGiteeToken() + if err != nil { + return nil, err + } + + var foundIssues = []gitee_model.Issue{} + page := gitee_model.GITEE_API_START_PAGE + for { + page += 1 + url := fmt.Sprintf("%s/repos/%s/%s/issues", gitee_model.GITEE_OAUTH_V5PREFIX, owner, repo) + code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{ + "page": strconv.Itoa(page), + "per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE), + "state": "all", + }) + + if err != nil { + return foundIssues, err + } + + if code != http.StatusOK { + return foundIssues, fmt.Errorf("GrabIssue failed during network. Status Code: %d", code) + } + + var issues = []gitee_model.Issue{} + e := json.Unmarshal([]byte(rs), &issues) + if e != nil { + return foundIssues, err + } + + if len(issues) > 0 { + foundIssues = append(foundIssues, issues...) + continue + } + break + } //end of for + return foundIssues, nil +} + +// 从仓库中获取提交记录 +// +// 从制定的 owner 和 repo 中获取全部提交 +func GetCommits(owner string, repo string) ([]gitee_model.Commit, error) { + token, err := validGiteeToken() + if err != nil { + return nil, err + } + var allCommits = []gitee_model.Commit{} + page := gitee_model.GITEE_API_START_PAGE + for { + page += 1 + url := fmt.Sprintf("%s/repos/%s/%s/commits", gitee_model.GITEE_OAUTH_V5PREFIX, owner, repo) + code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{ + "page": strconv.Itoa(page), + "per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE), + }) + + if err != nil { + return allCommits, err + } + + if code != http.StatusOK { + return allCommits, fmt.Errorf("GrabCommit failed during network. Status Code: %d", code) + } + + var commits = []gitee_model.Commit{} + e := json.Unmarshal([]byte(rs), &commits) + if e != nil { + log.Printf("GrabCommit Failed during json parse. %s", e) + return allCommits, e + } + + if len(commits) > 0 { + allCommits = append(allCommits, commits...) + continue + } + break + } //end of for + return allCommits, nil +} + +// 获取一个可用、有效的 token +// +// 先从本地配置文件中获取 access_token ,如果该 access_token 已失效,则调用 refreshGiteeToken() 更新 +func validGiteeToken() (OauthToken, error) { + var token OauthToken + token, err := retrieveGiteeToken() + if err != nil { + return token, err + } + + if time.Now().Unix() >= (token.CreatedAt + token.ExpiresIn) { + err := refreshGiteeToken(&token) + if err != nil { + return token, err + } + } + + return token, nil +} + +// 从本地配置文件中获取 access_token +// +// 从 ~/.repostats/{gitee_token_file}.json 中获取本地配置文件中的 access_token +func retrieveGiteeToken() (OauthToken, error) { + var giteeOauth OauthToken + if data, err := utils.ReadRepoStatsFile(gitee_model.GITEE_TOKEN_FILE); err != nil { + return giteeOauth, err + } else { + return giteeOauth, json.Unmarshal(data, &giteeOauth) + } +} + +// 更新 access_token +// +// 使用已存在的 refresh_token 更新 access_token +func refreshGiteeToken(token *OauthToken) error { + tokenUrl := fmt.Sprintf("%s?grant_type=refresh_token&refresh_token=%s", gitee_model.GITEE_OAUTH_TOKEN_URL, token.RefreshToken) + rc, rs, err := HttpPost(token.AccessToken, tokenUrl, nil, nil) + if err != nil { + return err + } + + if rc == http.StatusOK { + return utils.WriteRepoStatsFile(gitee_model.GITEE_TOKEN_FILE, []byte(rs)) + } + + return json.Unmarshal([]byte(rs), &token) +} diff --git a/network/gitee_n_test.go b/network/gitee_n_test.go new file mode 100644 index 0000000..f9e6572 --- /dev/null +++ b/network/gitee_n_test.go @@ -0,0 +1,149 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package network + +import ( + "log" + "reflect" + "testing" +) + +func testSetup(t *testing.T) { + log.Println("test start -->") +} + +func testTeardown(t *testing.T) { + log.Println("test done <--") +} + +func TestGetRepoCommits(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + //https://gitee.com/barat/ohurlshortener 51 commits so far + + type args struct { + owner string + repo string + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + {name: "TestCase1", args: args{owner: "barat", repo: "ohurlshortener"}, want: 51, wantErr: false}, + {name: "TestCase1", args: args{owner: "barat111", repo: "ohurlshortener"}, want: 0, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetCommits(tt.args.owner, tt.args.repo) + if (err != nil) != tt.wantErr { + t.Errorf("GetCommits() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(len(got), tt.want) { + t.Errorf("GetCommits() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGrabOrgRepos(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + type args struct { + org string + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + {name: "TestOpenharmony", args: args{org: "openharmony"}, want: 394, wantErr: false}, //currently has 394 repos + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetOrgRepos(tt.args.org) + if (err != nil) != tt.wantErr { + t.Errorf("GetOrgRepos() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(len(got), tt.want) { + t.Errorf("GetOrgRepos() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGrabUserRepos(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + type args struct { + name string + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + {name: "TestCase barat", args: args{name: "barat"}, want: 6, wantErr: false}, // I have 6 public repos + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetUserRepos(tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("GetUserRepos() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(len(got), tt.want) { + t.Errorf("GetUserRepos() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGrabIssues(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + type args struct { + owner string + repo string + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + {name: "TestCase barat/ohurlshortener", args: args{owner: "barat", repo: "ohurlshortener"}, want: 3, wantErr: false}, //should be 3 at the moment + // {name: "TestCase openharmony/community", args: args{owner: "openharmony", repo: "community"}, want: 107, wantErr: false}, //should be 107 at the moment + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetIssues(tt.args.owner, tt.args.repo) + if (err != nil) != tt.wantErr { + t.Errorf("GetIssues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(len(got), tt.want) { + t.Errorf("GetIssues() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/sql/gitee.sql b/sql/gitee.sql index bd2ecb5..038cd9e 100644 --- a/sql/gitee.sql +++ b/sql/gitee.sql @@ -6,68 +6,70 @@ CREATE SCHEMA gitee; -- Repos CREATE TABLE gitee.repos ( - id BIGINT NOT NULL, - full_name VARCHAR(255) NOT NULL, - human_name VARCHAR(255) NOT NULL, - owner_id BIGINT NOT NULL, - html_url VARCHAR(500) NOT NULL, - ssh_url VARCHAR(500) NOT NULL, - recommend BOOLEAN NOT NULL DEFAULT false, - gvp BOOLEAN NOT NULL DEFAULT false, - homepage VARCHAR(500), - language VARCHAR(500), - forks_count BIGINT NOT NULL DEFAULT 0, - stargazers_count BIGINT NOT NULL DEFAULT 0, - watchers_count BIGINT NOT NULL DEFAULT 0, - open_issues_count BIGINT NOT NULL DEFAULT 0, - license VARCHAR(500), - project_creator VARCHAR(500), + id int8 NOT NULL, + full_name VARCHAR(1000), + human_name VARCHAR(1000), + path VARCHAR(1000), + name VARCHAR(1000), + url VARCHAR(1000), + owner_id int8, + assigner_id int8, + description VARCHAR(1000), + html_url VARCHAR(2000), + ssh_url VARCHAR(2000), + forked_repo BOOLEAN, + default_branch VARCHAR(1000), + forks_count INT , + stargazers_count INT, + watchers_count INT, + license VARCHAR(1000), pushed_at TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL, - updated_at TIMESTAMP WITH TIME ZONE + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT uni_gitee_repo_id UNIQUE (id) ); -ALTER TABLE gitee.repos ADD CONSTRAINT uni_gitee_repos_id UNIQUE (id); --- Users -CREATE TABLE gitee.users ( - id BIGINT NOT NULL, - login VARCHAR(100) NOT NULL, - "name" VARCHAR(255) NOT NULL, - html_url VARCHAR(255) NOT NULL, - "type" VARCHAR(50) NULL +CREATE TABLE gitee.commits ( + repo_id int8, + repo_url VARCHAR(2000), + sha VARCHAR(80) NOT NULL, + commit_url VARCHAR(2000) NOT NULL, + author_name VARCHAR(500) NULL, + author_email VARCHAR(500) NULL, + author_date TIMESTAMP WITH TIME ZONE, + committer_name VARCHAR(200) , + committer_email VARCHAR(200) , + committer_date TIMESTAMP WITH TIME ZONE , + detail_message TEXT , + tree VARCHAR(80), + CONSTRAINT uni_gitee_commits UNIQUE (sha,repo_id) ); -ALTER TABLE gitee.users ADD CONSTRAINT uni_gitee_users_id UNIQUE (id); - --- Collaborators -CREATE TABLE gitee.collaborators ( - "user_id" BIGINT NOT NULL, - repo_id BIGINT NOT NULL -); -ALTER TABLE gitee.collaborators ADD CONSTRAINT uni_gitee_rcs UNIQUE (user_id,repo_id); -- Issues CREATE TABLE gitee.issues ( - id BIGINT NOT NULL, - repo_id BIGINT NOT NULL, - "user_id" BIGINT NOT NULL, - html_url VARCHAR(500) NOT NULL, - "number" VARCHAR(40) NOT NULL, - "state" VARCHAR(40) NOT NULL, - scheduled_time INT, + id int8 , + html_url VARCHAR(2000), + "number" VARCHAR(100), + "state" VARCHAR(100), + title VARCHAR(1000), + "user_id" int8, + repo_id int8 , + finished_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE, + plan_started_at TIMESTAMP WITH TIME ZONE, comments INT, priority INT, - issue_type VARCHAR(40), - issue_state VARCHAR(40), - finished_at TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL, - updated_at TIMESTAMP WITH TIME ZONE + issue_type VARCHAR(100), + issue_state VARCHAR(100), + security_hole BOOLEAN, + CONSTRAINT uni_gitee_issue_id UNIQUE (id) ); -ALTER TABLE gitee.issues ADD CONSTRAINT uni_gitee_issue UNIQUE (id); -- Pull requests -CREATE TABLE gitee.pullrequests ( - id BIGINT NOT NULL, - repo_id BIGINT NOT NULL, +CREATE TABLE gitee.pull_requests ( + id int8 NOT NULL, + repo_id int8 NOT NULL, "user_id" BIGINT NOT NULL, html_url VARCHAR(500) NOT NULL, "number" VARCHAR(40) NOT NULL, @@ -78,26 +80,31 @@ CREATE TABLE gitee.pullrequests ( closed_at TIMESTAMP WITH TIME ZONE, merged_at TIMESTAMP WITH TIME ZONE, mergeable BOOLEAN, - can_merge_check BOOLEAN + can_merge_check BOOLEAN, + CONSTRAINT uni_gitee_prs UNIQUE (id) ); -ALTER TABLE gitee.pullrequests ADD CONSTRAINT uni_gitee_prs UNIQUE (id); - --- Commits -CREATE TABLE gitee.commits ( - sha VARCHAR(100) NOT NULL, - repo_id BIGINT NOT NULL, - author BIGINT NOT NULL, - html_url VARCHAR(500) NOT NULL, - commiter BIGINT NOT NULL, - commit_at TIMESTAMP WITH TIME ZONE -); -ALTER TABLE gitee.commits ADD CONSTRAINT uni_gitee_commits UNIQUE(sha,repo_id); -- Stargazers CREATE TABLE gitee.stargazers ( - user_id BIGINT NOT NULL, - repo_id BIGINT NOT NULL, - star_at TIMESTAMP WITH TIME ZONE NOT NULL + user_id int8 NOT NULL, + repo_id int8 NOT NULL, + star_at TIMESTAMP WITH TIME ZONE NOT NULL, + CONSTRAINT uni_gitee_stargazers UNIQUE(user_id,repo_id) ); -ALTER TABLE gitee.stargazers ADD CONSTRAINT uni_gitee_stargazers UNIQUE(user_id,repo_id); +-- Collaborators +CREATE TABLE gitee.collaborators ( + "user_id" int8 NOT NULL, + repo_id int8 NOT NULL, + CONSTRAINT uni_gitee_rcs UNIQUE (user_id,repo_id) +); + +-- Users +CREATE TABLE gitee.users ( + id int8 NOT NULL, + login VARCHAR(100) NOT NULL, + "name" VARCHAR(255) NOT NULL, + html_url VARCHAR(255) NOT NULL, + "type" VARCHAR(50) NULL, + CONSTRAINT uni_gitee_users_id UNIQUE (id) +); diff --git a/storage/gitee/commit_s.go b/storage/gitee/commit_s.go new file mode 100644 index 0000000..069e9d2 --- /dev/null +++ b/storage/gitee/commit_s.go @@ -0,0 +1,25 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + gitee_model "repostats/model/gitee" + "repostats/storage" +) + +func BulkSaveCommits(commits []gitee_model.Commit) error { + query := `INSERT INTO gitee.commits (repo_id, repo_url, sha, commit_url, author_name, author_email, author_date, committer_name, + committer_email, committer_date, detail_message,tree) + VALUES(:repo_id, :repo_url,:sha,:commit_url,:commit.author.name, :commit.author.email,:commit.author.date, + :commit.committer.name,:commit.committer.email,:commit.committer.date,:commit.message,:commit.tree.sha) + ON CONFLICT (repo_id,sha) DO UPDATE SET repo_id=EXCLUDED.repo_id,repo_url=EXCLUDED.repo_url, commit_url=EXCLUDED.commit_url, + author_name=EXCLUDED.author_name,author_email=EXCLUDED.author_email,author_date=EXCLUDED.author_date,committer_name=EXCLUDED.committer_name, + committer_email=EXCLUDED.committer_email,committer_date=EXCLUDED.committer_date,detail_message=EXCLUDED.detail_message,tree=EXCLUDED.tree` + return storage.DbNamedExec(query, commits) +} diff --git a/storage/gitee/commit_s_test.go b/storage/gitee/commit_s_test.go new file mode 100644 index 0000000..950799d --- /dev/null +++ b/storage/gitee/commit_s_test.go @@ -0,0 +1,63 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + gitee_model "repostats/model/gitee" + "repostats/network" + "repostats/storage" + "repostats/utils" + "testing" +) + +func testSetup(t *testing.T) { + _, err := utils.InitConfig("../../repostats.ini") + if err != nil { + t.Error(err) + utils.ExitOnError(err) + } + + _, err = storage.InitDatabaseService() + if err != nil { + t.Error(err) + utils.ExitOnError(err) + } +} + +func testTeardown(t *testing.T) { + storage.DbClose() +} + +func TestBulkSaveCommits(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + found, err := network.GetCommits("openharmony", "community") + utils.ExitOnError(err) + + type args struct { + commits []gitee_model.Commit + } + tests := []struct { + name string + args args + wantErr bool + }{ + + {name: "TestCase1", args: args{commits: found}, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := BulkSaveCommits(tt.args.commits); (err != nil) != tt.wantErr { + t.Errorf("BulkSaveCommits() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/storage/gitee/issue_s.go b/storage/gitee/issue_s.go new file mode 100644 index 0000000..b402178 --- /dev/null +++ b/storage/gitee/issue_s.go @@ -0,0 +1,27 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + gitee_model "repostats/model/gitee" + "repostats/storage" +) + +func BulkSaveIssues(iss []gitee_model.Issue) error { + query := `INSERT INTO gitee.issues (id, html_url, "number", state, title, user_id, repo_id, finished_at, created_at, + updated_at, plan_started_at, "comments", priority, issue_type, issue_state, security_hole) + VALUES(:id,:html_url,:number,:state,:title,:user.id,:repository.id,:finished_at,:created_at, + :updated_at,:plan_started_at,:comments,:priority,:issue_type, :issue_state, :security_hole) + ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id,html_url=EXCLUDED.html_url,number=EXCLUDED.number, + state=EXCLUDED.state,title=EXCLUDED.title,user_id=EXCLUDED.user_id,repo_id=EXCLUDED.repo_id, + finished_at=EXCLUDED.finished_at,created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, + plan_started_at=EXCLUDED.plan_started_at,comments=EXCLUDED.comments,priority=EXCLUDED.priority, + issue_type=EXCLUDED.issue_type,issue_state=EXCLUDED.issue_state,security_hole=EXCLUDED.security_hole` + return storage.DbNamedExec(query, iss) +} diff --git a/storage/gitee/issue_s_test.go b/storage/gitee/issue_s_test.go new file mode 100644 index 0000000..63f505f --- /dev/null +++ b/storage/gitee/issue_s_test.go @@ -0,0 +1,48 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + gitee_model "repostats/model/gitee" + "repostats/network" + "repostats/utils" + "testing" +) + +func TestBulkSaveIssues(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + found, err := network.GetIssues("barat", "ohurlshortener") + utils.ExitOnError(err) + + found2, err := network.GetIssues("openharmony", "community") + utils.ExitOnError(err) + + type args struct { + iss []gitee_model.Issue + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "Testcase barat/ohurlshortener", args: args{iss: found}, wantErr: false}, + {name: "Testcase barat/ohurlshortener again", args: args{iss: found}, wantErr: false}, + {name: "Testcase openharmony/community", args: args{iss: found2}, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := BulkSaveIssues(tt.args.iss); (err != nil) != tt.wantErr { + t.Errorf("BulkSaveIssues() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/storage/gitee/repository_s.go b/storage/gitee/repository_s.go new file mode 100644 index 0000000..ac729ad --- /dev/null +++ b/storage/gitee/repository_s.go @@ -0,0 +1,27 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + gitee_model "repostats/model/gitee" + "repostats/storage" +) + +func BulkSaveRepos(repos []gitee_model.Repository) error { + query := `INSERT INTO gitee.repos (id, full_name, human_name, path,name, url, owner_id,assigner_id, description, + html_url, ssh_url,forked_repo,default_branch, forks_count, stargazers_count, watchers_count,license, pushed_at, created_at, updated_at) + VALUES(:id, :full_name, :human_name, :path, :name, :url, :owner.id, :assigner.id, :description, :html_url, :ssh_url, :forked_repo, + :default_branch, :forks_count, :stargazers_count, :watchers_count, :license, :pushed_at, :created_at, :updated_at) + ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id,full_name=EXCLUDED.full_name,human_name=EXCLUDED.human_name,path=EXCLUDED.path, + url=EXCLUDED.url,owner_id=EXCLUDED.owner_id,assigner_id=EXCLUDED.assigner_id, description=EXCLUDED.description, + html_url=EXCLUDED.html_url,ssh_url=EXCLUDED.ssh_url,forked_repo=EXCLUDED.forked_repo, forks_count=EXCLUDED.forks_count, + stargazers_count=EXCLUDED.stargazers_count, watchers_count=EXCLUDED.watchers_count, + license=EXCLUDED.license,pushed_at=EXCLUDED.pushed_at,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at` + return storage.DbNamedExec(query, repos) +} diff --git a/storage/gitee/repository_s_test.go b/storage/gitee/repository_s_test.go new file mode 100644 index 0000000..ef2d0cb --- /dev/null +++ b/storage/gitee/repository_s_test.go @@ -0,0 +1,47 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee + +import ( + gitee_model "repostats/model/gitee" + "repostats/network" + "repostats/utils" + "testing" +) + +func TestBulkSaveRepos(t *testing.T) { + + testSetup(t) + defer testTeardown(t) + + found1, err := network.GetUserRepos("barat") + utils.ExitOnError(err) + + found2, err := network.GetOrgRepos("openharmony") + utils.ExitOnError(err) + + type args struct { + repos []gitee_model.Repository + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "TestCase barat", args: args{found1}, wantErr: false}, + {name: "TestCase openharmony", args: args{found2}, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := BulkSaveRepos(tt.args.repos); (err != nil) != tt.wantErr { + t.Errorf("BulkSaveRepos() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/storage/gitee/user_s.go b/storage/gitee/user_s.go new file mode 100644 index 0000000..1f83729 --- /dev/null +++ b/storage/gitee/user_s.go @@ -0,0 +1,9 @@ +// Copyright (c) [2022] [巴拉迪维 BaratSemet] +// [ohUrlShortener] is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. + +package gitee diff --git a/templates/gitee.html b/templates/gitee.html index 05a5676..059996b 100644 --- a/templates/gitee.html +++ b/templates/gitee.html @@ -5,10 +5,16 @@

Gitee 配置

-

RepoStats 使用 Oauth 授权码模式访问 Gitee 授权许可的接口并获取相关数据。

-

1. 请根据 《Gitee Oauth 文档》 指南新建第三方应用。

-

2. 创建第三方应用成功后,请将对应的信息填入下列表格中。

-

3. 首先点击「应用授权」完成操作。再获取 code 之后,再点击「获取 Token」Oauth 过程。

+
+
须知事项
+

RepoStats 使用 Oauth 授权码模式访问 Gitee 授权许可的接口并获取相关数据。

+
    +
  • 请根据 《Gitee Oauth 文档》 指南在 Gitee 平台新建属于您的第三方应用
  • +
  • 将您的应用信息填入下列表格中并点击「应用授权」以获取 Gitee 平台授权 Code,此过程将打开新窗口页面
  • +
  • 将授权成功后转向页面 url 中的 code 信息填入下列表格,再点击「获取 Token」完成全过程
  • +
  • 在此过程中,RepoStats 不会保存任何第三方应用信息,只记录最终的 AccessToken 以备网络请求使用
  • +
+

应用信息

@@ -36,9 +42,10 @@
{{if .oauth_info}}
+
Gitee AccessToken

当前 token 信息已存在,若无必要不必重新获取。

-

{{.oauth_info}}

+ {{.oauth_info}}
{{end}}