完成 gitee commit、repo、issue 获取与存储

This commit is contained in:
巴拉迪维 2022-04-15 18:45:36 +08:00
parent 58ca2c4c39
commit 1a1d91352b
20 changed files with 927 additions and 83 deletions

View File

@ -1,3 +1,11 @@
/* Copyright (c) [2022] [巴拉迪维 BaratSemet]
[ohUrlShortener] is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:
http://license.coscl.org.cn/MulanPSL2
THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details. */
#admin-left-menu { #admin-left-menu {
max-width: 200px; max-width: 200px;
} }

View File

@ -1,5 +1,21 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
$(document).ready(function() { $(document).ready(function() {
$('.message .close')
.on('click', function() {
$(this)
.closest('.message')
.transition('fade')
;
});
$('#form-gitee-authorize').form({ $('#form-gitee-authorize').form({
fields: { fields: {
client_id: { client_id: {
@ -74,4 +90,4 @@ function GetGiteeToken(form) {
$('#gitee-token-message').removeClass('positive').addClass('error').addClass('visible'); $('#gitee-token-message').removeClass('positive').addClass('error').addClass('visible');
} }
}); });
} }//end of function GetGiteeToken

44
model/gitee/commit_m.go Normal file
View File

@ -0,0 +1,44 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
"reflect"
"time"
)
type Commit struct {
RepoID int `db:"repo_id"`
RepoURL string `db:"repo_url"`
Sha string `db:"sha" json:"sha"`
CommitURL string `db:"commit_url" json:"html_url"`
Committer User `json:"committer"`
Author User `json:"author"`
Detail struct {
Author struct {
User
Date time.Time `json:"date" db:"date"`
} `json:"author" db:"author"`
Committer struct {
User
Date time.Time `json:"date" db:"date"`
} `json:"committer" db:"committer"`
Message string `json:"message" db:"message"`
Tree struct {
Sha string `json:"sha"`
} `json:"tree"`
} `json:"commit" db:"commit"`
}
func (u Commit) isNilOrEmpty() bool {
return reflect.DeepEqual(u, Commit{})
}

View File

@ -1,9 +0,0 @@
package gitee
const (
GITEE_TOKEN_FILE = "gitee_oauth_token.json"
GITEE_OAUTH_V5PREFIX = "https://gitee.com/api/v5"
GITEE_OAUTH_TOKEN_URL = "https://gitee.com/oauth/token"
GITEE_API_START_PAGE = 0
GITEE_API_PAGE_SIZE = 30
)

17
model/gitee/const_m.go Normal file
View File

@ -0,0 +1,17 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
const (
GITEE_TOKEN_FILE = "gitee_oauth_token.json"
GITEE_OAUTH_V5PREFIX = "https://gitee.com/api/v5"
GITEE_OAUTH_TOKEN_URL = "https://gitee.com/oauth/token"
GITEE_API_START_PAGE = 0
GITEE_API_PAGE_SIZE = 55
)

33
model/gitee/issue_m.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
"time"
)
type Issue struct {
ID int `json:"id" db:"id"`
RepositoryURL string `json:"repository_url" db:"repository_url"`
HTMLURL string `json:"html_url" db:"html_url"`
Number string `json:"number" db:"number"`
State string `json:"state" db:"state"`
Title string `json:"title" db:"title"`
User User `json:"user" db:"user"`
Repository Repository `json:"repository" db:"repository"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
FinishedAt time.Time `json:"finished_at" db:"finished_at"`
PlanStarted_at time.Time `json:"plan_started_at" db:"plan_started_at"`
Comments int `json:"comments" db:"comments"`
Priority int `json:"priority" db:"priority"`
IssueType string `json:"issue_type" db:"issue_type"`
SecurityHole bool `json:"security_hole" db:"security_hole"`
IssueState string `json:"issue_state" db:"issue_state"`
}

View File

@ -0,0 +1,41 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
"reflect"
"time"
)
type Repository struct {
ID int `json:"id" db:"id"`
FullName string `json:"full_name" db:"full_name"`
HumanName string `json:"human_name" db:"human_name"`
Path string `json:"path" db:"path"`
Name string `json:"name" db:"name"`
URL string `json:"url" db:"url"`
Owner User `json:"owner" db:"owner"`
Assigner User `json:"assigner" db:"assigner"`
Description string `json:"description" db:"description"`
HTMLURL string `json:"html_url" db:"html_url"`
SSHURL string `json:"ssh_url" db:"ssh_url"`
Fork bool `json:"fork" db:"forked_repo"`
DefaultBranch string `json:"default_branch" db:"default_branch"`
ForksCount int `json:"forks_count" db:"forks_count"`
StargazersCount int `json:"stargazers_count" db:"stargazers_count"`
WatchersCount int `json:"watchers_count" db:"watchers_count"`
License string `json:"license" db:"license"`
PushedAt time.Time `json:"pushed_at" db:"pushed_at"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
}
func (r Repository) isNilOrEmpty() bool {
return reflect.DeepEqual(r, Repository{})
}

40
model/gitee/user_m.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
"reflect"
"time"
)
type User struct {
ID int `json:"id"`
Login string `json:"login" db:"login"`
Name string `json:"name" db:"name"`
AvatarURL string `json:"avatar_url"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
Remark string `json:"remark"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
Date time.Time `json:"date"`
Email string `json:"email" db:"email"`
}
func (u User) isNilOrEmpty() bool {
return reflect.DeepEqual(u, User{})
}

View File

@ -17,7 +17,7 @@ import (
const ( const (
CONTENT_TYPE = "application/json" CONTENT_TYPE = "application/json"
USER_AGENT = "RepoStats https://gitee.com/barat" USER_AGENT = "RepoStats https://gitee.com/barat | https://github.com/barats"
) )
type OauthToken struct { type OauthToken struct {

245
network/gitee_n.go Normal file
View File

@ -0,0 +1,245 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package network
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
gitee_model "repostats/model/gitee"
"repostats/utils"
"strconv"
"time"
)
// 获取组织下的所有公开仓库
//
// 调用此方法之前,务必确保是组织帐号
func GetOrgRepos(org string) ([]gitee_model.Repository, error) {
token, err := validGiteeToken()
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/orgs/%s/repos", gitee_model.GITEE_OAUTH_V5PREFIX, org)
page := gitee_model.GITEE_API_START_PAGE
allRepos := []gitee_model.Repository{}
for {
page += 1
code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{
"type": "public",
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE),
})
if err != nil {
return allRepos, err
}
if code != http.StatusOK {
return allRepos, errors.New("unexpected StatusCode")
}
var foundRepos = []gitee_model.Repository{}
err = json.Unmarshal([]byte(rs), &foundRepos)
if err != nil {
return allRepos, err
}
if len(foundRepos) > 0 {
allRepos = append(allRepos, foundRepos...)
continue
}
break
} //end of for
return allRepos, nil
}
// 获取个人用户名下的所有公开仓库
//
// 调用此方法之前,务必确保是个人帐号
func GetUserRepos(name string) ([]gitee_model.Repository, error) {
token, err := validGiteeToken()
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/users/%s/repos", gitee_model.GITEE_OAUTH_V5PREFIX, name)
page := gitee_model.GITEE_API_START_PAGE
allRepos := []gitee_model.Repository{}
for {
page += 1
code, res, err := HttpGet(token.AccessToken, url, nil, map[string]string{
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE),
"type": "all",
})
if err != nil {
return allRepos, err
}
if code != http.StatusOK {
return allRepos, errors.New("unexpected StatusCode")
}
var foundRepos = []gitee_model.Repository{}
err = json.Unmarshal([]byte(res), &foundRepos)
if err != nil {
return allRepos, err
}
if len(foundRepos) > 0 {
allRepos = append(allRepos, foundRepos...)
continue
}
break
} //end of for
return allRepos, nil
}
//获取指定仓库的 issue
//
//
func GetIssues(owner string, repo string) ([]gitee_model.Issue, error) {
token, err := validGiteeToken()
if err != nil {
return nil, err
}
var foundIssues = []gitee_model.Issue{}
page := gitee_model.GITEE_API_START_PAGE
for {
page += 1
url := fmt.Sprintf("%s/repos/%s/%s/issues", gitee_model.GITEE_OAUTH_V5PREFIX, owner, repo)
code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE),
"state": "all",
})
if err != nil {
return foundIssues, err
}
if code != http.StatusOK {
return foundIssues, fmt.Errorf("GrabIssue failed during network. Status Code: %d", code)
}
var issues = []gitee_model.Issue{}
e := json.Unmarshal([]byte(rs), &issues)
if e != nil {
return foundIssues, err
}
if len(issues) > 0 {
foundIssues = append(foundIssues, issues...)
continue
}
break
} //end of for
return foundIssues, nil
}
// 从仓库中获取提交记录
//
// 从制定的 owner 和 repo 中获取全部提交
func GetCommits(owner string, repo string) ([]gitee_model.Commit, error) {
token, err := validGiteeToken()
if err != nil {
return nil, err
}
var allCommits = []gitee_model.Commit{}
page := gitee_model.GITEE_API_START_PAGE
for {
page += 1
url := fmt.Sprintf("%s/repos/%s/%s/commits", gitee_model.GITEE_OAUTH_V5PREFIX, owner, repo)
code, rs, err := HttpGet(token.AccessToken, url, nil, map[string]string{
"page": strconv.Itoa(page),
"per_page": strconv.Itoa(gitee_model.GITEE_API_PAGE_SIZE),
})
if err != nil {
return allCommits, err
}
if code != http.StatusOK {
return allCommits, fmt.Errorf("GrabCommit failed during network. Status Code: %d", code)
}
var commits = []gitee_model.Commit{}
e := json.Unmarshal([]byte(rs), &commits)
if e != nil {
log.Printf("GrabCommit Failed during json parse. %s", e)
return allCommits, e
}
if len(commits) > 0 {
allCommits = append(allCommits, commits...)
continue
}
break
} //end of for
return allCommits, nil
}
// 获取一个可用、有效的 token
//
// 先从本地配置文件中获取 access_token ,如果该 access_token 已失效,则调用 refreshGiteeToken() 更新
func validGiteeToken() (OauthToken, error) {
var token OauthToken
token, err := retrieveGiteeToken()
if err != nil {
return token, err
}
if time.Now().Unix() >= (token.CreatedAt + token.ExpiresIn) {
err := refreshGiteeToken(&token)
if err != nil {
return token, err
}
}
return token, nil
}
// 从本地配置文件中获取 access_token
//
// 从 ~/.repostats/{gitee_token_file}.json 中获取本地配置文件中的 access_token
func retrieveGiteeToken() (OauthToken, error) {
var giteeOauth OauthToken
if data, err := utils.ReadRepoStatsFile(gitee_model.GITEE_TOKEN_FILE); err != nil {
return giteeOauth, err
} else {
return giteeOauth, json.Unmarshal(data, &giteeOauth)
}
}
// 更新 access_token
//
// 使用已存在的 refresh_token 更新 access_token
func refreshGiteeToken(token *OauthToken) error {
tokenUrl := fmt.Sprintf("%s?grant_type=refresh_token&refresh_token=%s", gitee_model.GITEE_OAUTH_TOKEN_URL, token.RefreshToken)
rc, rs, err := HttpPost(token.AccessToken, tokenUrl, nil, nil)
if err != nil {
return err
}
if rc == http.StatusOK {
return utils.WriteRepoStatsFile(gitee_model.GITEE_TOKEN_FILE, []byte(rs))
}
return json.Unmarshal([]byte(rs), &token)
}

149
network/gitee_n_test.go Normal file
View File

@ -0,0 +1,149 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package network
import (
"log"
"reflect"
"testing"
)
func testSetup(t *testing.T) {
log.Println("test start -->")
}
func testTeardown(t *testing.T) {
log.Println("test done <--")
}
func TestGetRepoCommits(t *testing.T) {
testSetup(t)
defer testTeardown(t)
//https://gitee.com/barat/ohurlshortener 51 commits so far
type args struct {
owner string
repo string
}
tests := []struct {
name string
args args
want int
wantErr bool
}{
{name: "TestCase1", args: args{owner: "barat", repo: "ohurlshortener"}, want: 51, wantErr: false},
{name: "TestCase1", args: args{owner: "barat111", repo: "ohurlshortener"}, want: 0, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetCommits(tt.args.owner, tt.args.repo)
if (err != nil) != tt.wantErr {
t.Errorf("GetCommits() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(len(got), tt.want) {
t.Errorf("GetCommits() = %v, want %v", got, tt.want)
}
})
}
}
func TestGrabOrgRepos(t *testing.T) {
testSetup(t)
defer testTeardown(t)
type args struct {
org string
}
tests := []struct {
name string
args args
want int
wantErr bool
}{
{name: "TestOpenharmony", args: args{org: "openharmony"}, want: 394, wantErr: false}, //currently has 394 repos
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetOrgRepos(tt.args.org)
if (err != nil) != tt.wantErr {
t.Errorf("GetOrgRepos() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(len(got), tt.want) {
t.Errorf("GetOrgRepos() = %v, want %v", got, tt.want)
}
})
}
}
func TestGrabUserRepos(t *testing.T) {
testSetup(t)
defer testTeardown(t)
type args struct {
name string
}
tests := []struct {
name string
args args
want int
wantErr bool
}{
{name: "TestCase barat", args: args{name: "barat"}, want: 6, wantErr: false}, // I have 6 public repos
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetUserRepos(tt.args.name)
if (err != nil) != tt.wantErr {
t.Errorf("GetUserRepos() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(len(got), tt.want) {
t.Errorf("GetUserRepos() = %v, want %v", got, tt.want)
}
})
}
}
func TestGrabIssues(t *testing.T) {
testSetup(t)
defer testTeardown(t)
type args struct {
owner string
repo string
}
tests := []struct {
name string
args args
want int
wantErr bool
}{
{name: "TestCase barat/ohurlshortener", args: args{owner: "barat", repo: "ohurlshortener"}, want: 3, wantErr: false}, //should be 3 at the moment
// {name: "TestCase openharmony/community", args: args{owner: "openharmony", repo: "community"}, want: 107, wantErr: false}, //should be 107 at the moment
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetIssues(tt.args.owner, tt.args.repo)
if (err != nil) != tt.wantErr {
t.Errorf("GetIssues() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(len(got), tt.want) {
t.Errorf("GetIssues() = %v, want %v", got, tt.want)
}
})
}
}

View File

@ -6,68 +6,70 @@ CREATE SCHEMA gitee;
-- Repos -- Repos
CREATE TABLE gitee.repos ( CREATE TABLE gitee.repos (
id BIGINT NOT NULL, id int8 NOT NULL,
full_name VARCHAR(255) NOT NULL, full_name VARCHAR(1000),
human_name VARCHAR(255) NOT NULL, human_name VARCHAR(1000),
owner_id BIGINT NOT NULL, path VARCHAR(1000),
html_url VARCHAR(500) NOT NULL, name VARCHAR(1000),
ssh_url VARCHAR(500) NOT NULL, url VARCHAR(1000),
recommend BOOLEAN NOT NULL DEFAULT false, owner_id int8,
gvp BOOLEAN NOT NULL DEFAULT false, assigner_id int8,
homepage VARCHAR(500), description VARCHAR(1000),
language VARCHAR(500), html_url VARCHAR(2000),
forks_count BIGINT NOT NULL DEFAULT 0, ssh_url VARCHAR(2000),
stargazers_count BIGINT NOT NULL DEFAULT 0, forked_repo BOOLEAN,
watchers_count BIGINT NOT NULL DEFAULT 0, default_branch VARCHAR(1000),
open_issues_count BIGINT NOT NULL DEFAULT 0, forks_count INT ,
license VARCHAR(500), stargazers_count INT,
project_creator VARCHAR(500), watchers_count INT,
license VARCHAR(1000),
pushed_at TIMESTAMP WITH TIME ZONE, pushed_at TIMESTAMP WITH TIME ZONE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL, created_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE updated_at TIMESTAMP WITH TIME ZONE,
CONSTRAINT uni_gitee_repo_id UNIQUE (id)
); );
ALTER TABLE gitee.repos ADD CONSTRAINT uni_gitee_repos_id UNIQUE (id);
-- Users CREATE TABLE gitee.commits (
CREATE TABLE gitee.users ( repo_id int8,
id BIGINT NOT NULL, repo_url VARCHAR(2000),
login VARCHAR(100) NOT NULL, sha VARCHAR(80) NOT NULL,
"name" VARCHAR(255) NOT NULL, commit_url VARCHAR(2000) NOT NULL,
html_url VARCHAR(255) NOT NULL, author_name VARCHAR(500) NULL,
"type" VARCHAR(50) NULL author_email VARCHAR(500) NULL,
author_date TIMESTAMP WITH TIME ZONE,
committer_name VARCHAR(200) ,
committer_email VARCHAR(200) ,
committer_date TIMESTAMP WITH TIME ZONE ,
detail_message TEXT ,
tree VARCHAR(80),
CONSTRAINT uni_gitee_commits UNIQUE (sha,repo_id)
); );
ALTER TABLE gitee.users ADD CONSTRAINT uni_gitee_users_id UNIQUE (id);
-- Collaborators
CREATE TABLE gitee.collaborators (
"user_id" BIGINT NOT NULL,
repo_id BIGINT NOT NULL
);
ALTER TABLE gitee.collaborators ADD CONSTRAINT uni_gitee_rcs UNIQUE (user_id,repo_id);
-- Issues -- Issues
CREATE TABLE gitee.issues ( CREATE TABLE gitee.issues (
id BIGINT NOT NULL, id int8 ,
repo_id BIGINT NOT NULL, html_url VARCHAR(2000),
"user_id" BIGINT NOT NULL, "number" VARCHAR(100),
html_url VARCHAR(500) NOT NULL, "state" VARCHAR(100),
"number" VARCHAR(40) NOT NULL, title VARCHAR(1000),
"state" VARCHAR(40) NOT NULL, "user_id" int8,
scheduled_time INT, repo_id int8 ,
finished_at TIMESTAMP WITH TIME ZONE,
created_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE,
plan_started_at TIMESTAMP WITH TIME ZONE,
comments INT, comments INT,
priority INT, priority INT,
issue_type VARCHAR(40), issue_type VARCHAR(100),
issue_state VARCHAR(40), issue_state VARCHAR(100),
finished_at TIMESTAMP WITH TIME ZONE, security_hole BOOLEAN,
created_at TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT uni_gitee_issue_id UNIQUE (id)
updated_at TIMESTAMP WITH TIME ZONE
); );
ALTER TABLE gitee.issues ADD CONSTRAINT uni_gitee_issue UNIQUE (id);
-- Pull requests -- Pull requests
CREATE TABLE gitee.pullrequests ( CREATE TABLE gitee.pull_requests (
id BIGINT NOT NULL, id int8 NOT NULL,
repo_id BIGINT NOT NULL, repo_id int8 NOT NULL,
"user_id" BIGINT NOT NULL, "user_id" BIGINT NOT NULL,
html_url VARCHAR(500) NOT NULL, html_url VARCHAR(500) NOT NULL,
"number" VARCHAR(40) NOT NULL, "number" VARCHAR(40) NOT NULL,
@ -78,26 +80,31 @@ CREATE TABLE gitee.pullrequests (
closed_at TIMESTAMP WITH TIME ZONE, closed_at TIMESTAMP WITH TIME ZONE,
merged_at TIMESTAMP WITH TIME ZONE, merged_at TIMESTAMP WITH TIME ZONE,
mergeable BOOLEAN, mergeable BOOLEAN,
can_merge_check BOOLEAN can_merge_check BOOLEAN,
CONSTRAINT uni_gitee_prs UNIQUE (id)
); );
ALTER TABLE gitee.pullrequests ADD CONSTRAINT uni_gitee_prs UNIQUE (id);
-- Commits
CREATE TABLE gitee.commits (
sha VARCHAR(100) NOT NULL,
repo_id BIGINT NOT NULL,
author BIGINT NOT NULL,
html_url VARCHAR(500) NOT NULL,
commiter BIGINT NOT NULL,
commit_at TIMESTAMP WITH TIME ZONE
);
ALTER TABLE gitee.commits ADD CONSTRAINT uni_gitee_commits UNIQUE(sha,repo_id);
-- Stargazers -- Stargazers
CREATE TABLE gitee.stargazers ( CREATE TABLE gitee.stargazers (
user_id BIGINT NOT NULL, user_id int8 NOT NULL,
repo_id BIGINT NOT NULL, repo_id int8 NOT NULL,
star_at TIMESTAMP WITH TIME ZONE NOT NULL star_at TIMESTAMP WITH TIME ZONE NOT NULL,
CONSTRAINT uni_gitee_stargazers UNIQUE(user_id,repo_id)
); );
ALTER TABLE gitee.stargazers ADD CONSTRAINT uni_gitee_stargazers UNIQUE(user_id,repo_id);
-- Collaborators
CREATE TABLE gitee.collaborators (
"user_id" int8 NOT NULL,
repo_id int8 NOT NULL,
CONSTRAINT uni_gitee_rcs UNIQUE (user_id,repo_id)
);
-- Users
CREATE TABLE gitee.users (
id int8 NOT NULL,
login VARCHAR(100) NOT NULL,
"name" VARCHAR(255) NOT NULL,
html_url VARCHAR(255) NOT NULL,
"type" VARCHAR(50) NULL,
CONSTRAINT uni_gitee_users_id UNIQUE (id)
);

25
storage/gitee/commit_s.go Normal file
View File

@ -0,0 +1,25 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
gitee_model "repostats/model/gitee"
"repostats/storage"
)
func BulkSaveCommits(commits []gitee_model.Commit) error {
query := `INSERT INTO gitee.commits (repo_id, repo_url, sha, commit_url, author_name, author_email, author_date, committer_name,
committer_email, committer_date, detail_message,tree)
VALUES(:repo_id, :repo_url,:sha,:commit_url,:commit.author.name, :commit.author.email,:commit.author.date,
:commit.committer.name,:commit.committer.email,:commit.committer.date,:commit.message,:commit.tree.sha)
ON CONFLICT (repo_id,sha) DO UPDATE SET repo_id=EXCLUDED.repo_id,repo_url=EXCLUDED.repo_url, commit_url=EXCLUDED.commit_url,
author_name=EXCLUDED.author_name,author_email=EXCLUDED.author_email,author_date=EXCLUDED.author_date,committer_name=EXCLUDED.committer_name,
committer_email=EXCLUDED.committer_email,committer_date=EXCLUDED.committer_date,detail_message=EXCLUDED.detail_message,tree=EXCLUDED.tree`
return storage.DbNamedExec(query, commits)
}

View File

@ -0,0 +1,63 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
gitee_model "repostats/model/gitee"
"repostats/network"
"repostats/storage"
"repostats/utils"
"testing"
)
func testSetup(t *testing.T) {
_, err := utils.InitConfig("../../repostats.ini")
if err != nil {
t.Error(err)
utils.ExitOnError(err)
}
_, err = storage.InitDatabaseService()
if err != nil {
t.Error(err)
utils.ExitOnError(err)
}
}
func testTeardown(t *testing.T) {
storage.DbClose()
}
func TestBulkSaveCommits(t *testing.T) {
testSetup(t)
defer testTeardown(t)
found, err := network.GetCommits("openharmony", "community")
utils.ExitOnError(err)
type args struct {
commits []gitee_model.Commit
}
tests := []struct {
name string
args args
wantErr bool
}{
{name: "TestCase1", args: args{commits: found}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := BulkSaveCommits(tt.args.commits); (err != nil) != tt.wantErr {
t.Errorf("BulkSaveCommits() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

27
storage/gitee/issue_s.go Normal file
View File

@ -0,0 +1,27 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
gitee_model "repostats/model/gitee"
"repostats/storage"
)
func BulkSaveIssues(iss []gitee_model.Issue) error {
query := `INSERT INTO gitee.issues (id, html_url, "number", state, title, user_id, repo_id, finished_at, created_at,
updated_at, plan_started_at, "comments", priority, issue_type, issue_state, security_hole)
VALUES(:id,:html_url,:number,:state,:title,:user.id,:repository.id,:finished_at,:created_at,
:updated_at,:plan_started_at,:comments,:priority,:issue_type, :issue_state, :security_hole)
ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id,html_url=EXCLUDED.html_url,number=EXCLUDED.number,
state=EXCLUDED.state,title=EXCLUDED.title,user_id=EXCLUDED.user_id,repo_id=EXCLUDED.repo_id,
finished_at=EXCLUDED.finished_at,created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at,
plan_started_at=EXCLUDED.plan_started_at,comments=EXCLUDED.comments,priority=EXCLUDED.priority,
issue_type=EXCLUDED.issue_type,issue_state=EXCLUDED.issue_state,security_hole=EXCLUDED.security_hole`
return storage.DbNamedExec(query, iss)
}

View File

@ -0,0 +1,48 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
gitee_model "repostats/model/gitee"
"repostats/network"
"repostats/utils"
"testing"
)
func TestBulkSaveIssues(t *testing.T) {
testSetup(t)
defer testTeardown(t)
found, err := network.GetIssues("barat", "ohurlshortener")
utils.ExitOnError(err)
found2, err := network.GetIssues("openharmony", "community")
utils.ExitOnError(err)
type args struct {
iss []gitee_model.Issue
}
tests := []struct {
name string
args args
wantErr bool
}{
{name: "Testcase barat/ohurlshortener", args: args{iss: found}, wantErr: false},
{name: "Testcase barat/ohurlshortener again", args: args{iss: found}, wantErr: false},
{name: "Testcase openharmony/community", args: args{iss: found2}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := BulkSaveIssues(tt.args.iss); (err != nil) != tt.wantErr {
t.Errorf("BulkSaveIssues() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@ -0,0 +1,27 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
gitee_model "repostats/model/gitee"
"repostats/storage"
)
func BulkSaveRepos(repos []gitee_model.Repository) error {
query := `INSERT INTO gitee.repos (id, full_name, human_name, path,name, url, owner_id,assigner_id, description,
html_url, ssh_url,forked_repo,default_branch, forks_count, stargazers_count, watchers_count,license, pushed_at, created_at, updated_at)
VALUES(:id, :full_name, :human_name, :path, :name, :url, :owner.id, :assigner.id, :description, :html_url, :ssh_url, :forked_repo,
:default_branch, :forks_count, :stargazers_count, :watchers_count, :license, :pushed_at, :created_at, :updated_at)
ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id,full_name=EXCLUDED.full_name,human_name=EXCLUDED.human_name,path=EXCLUDED.path,
url=EXCLUDED.url,owner_id=EXCLUDED.owner_id,assigner_id=EXCLUDED.assigner_id, description=EXCLUDED.description,
html_url=EXCLUDED.html_url,ssh_url=EXCLUDED.ssh_url,forked_repo=EXCLUDED.forked_repo, forks_count=EXCLUDED.forks_count,
stargazers_count=EXCLUDED.stargazers_count, watchers_count=EXCLUDED.watchers_count,
license=EXCLUDED.license,pushed_at=EXCLUDED.pushed_at,created_at=EXCLUDED.created_at,updated_at=EXCLUDED.updated_at`
return storage.DbNamedExec(query, repos)
}

View File

@ -0,0 +1,47 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee
import (
gitee_model "repostats/model/gitee"
"repostats/network"
"repostats/utils"
"testing"
)
func TestBulkSaveRepos(t *testing.T) {
testSetup(t)
defer testTeardown(t)
found1, err := network.GetUserRepos("barat")
utils.ExitOnError(err)
found2, err := network.GetOrgRepos("openharmony")
utils.ExitOnError(err)
type args struct {
repos []gitee_model.Repository
}
tests := []struct {
name string
args args
wantErr bool
}{
{name: "TestCase barat", args: args{found1}, wantErr: false},
{name: "TestCase openharmony", args: args{found2}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := BulkSaveRepos(tt.args.repos); (err != nil) != tt.wantErr {
t.Errorf("BulkSaveRepos() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

9
storage/gitee/user_s.go Normal file
View File

@ -0,0 +1,9 @@
// Copyright (c) [2022] [巴拉迪维 BaratSemet]
// [ohUrlShortener] is licensed under Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PSL v2 for more details.
package gitee

View File

@ -5,10 +5,16 @@
<div class="ui basic segment"> <div class="ui basic segment">
<h3 class="ui header">Gitee 配置</h3> <h3 class="ui header">Gitee 配置</h3>
<div class="ui content"> <div class="ui content">
<div class="ui compact info message">
<div class="header">须知事项</div>
<p>RepoStats 使用 Oauth 授权码模式访问 Gitee 授权许可的接口并获取相关数据。</p> <p>RepoStats 使用 Oauth 授权码模式访问 Gitee 授权许可的接口并获取相关数据。</p>
<p>1. 请根据 <a target="_blank" href="https://gitee.com/api/v5/oauth_doc#/">《Gitee Oauth 文档》</a> 指南新建第三方应用。 </p> <ul class="list">
<p>2. 创建第三方应用成功后,请将对应的信息填入下列表格中。</p> <li>请根据 <a target="_blank" href="https://gitee.com/api/v5/oauth_doc#/">《Gitee Oauth 文档》</a> 指南在 Gitee 平台新建属于您的第三方应用 </li>
<p>3. 首先点击「应用授权」完成操作。再获取 code 之后,再点击「获取 Token」Oauth 过程。</p> <li>将您的应用信息填入下列表格中并点击「应用授权」以获取 Gitee 平台授权 Code此过程将打开新窗口页面</li>
<li>将授权成功后转向页面 url 中的 code 信息填入下列表格,再点击「获取 Token」完成全过程</li>
<li>在此过程中RepoStats 不会保存任何第三方应用信息,只记录最终的 AccessToken 以备网络请求使用</li>
</ul>
</div>
<div class="ui two column stackable grid"> <div class="ui two column stackable grid">
<div class="eight wide column"> <div class="eight wide column">
<h3>应用信息</h3> <h3>应用信息</h3>
@ -36,9 +42,10 @@
<div id="gitee-token-message" class="ui error message"></div> <div id="gitee-token-message" class="ui error message"></div>
{{if .oauth_info}} {{if .oauth_info}}
<div class="ui bottom attached compact positive message"> <div class="ui bottom attached compact positive message">
<i class="close icon"></i>
<div class="header">Gitee AccessToken</div> <div class="header">Gitee AccessToken</div>
<p>当前 token 信息已存在,若无必要不必重新获取。</p> <p>当前 token 信息已存在,若无必要不必重新获取。</p>
<p style="word-break: break-word;">{{.oauth_info}}</p> <code style="word-break: break-word;">{{.oauth_info}}</code>
</div> </div>
{{end}} {{end}}
<button id="btn-gitee-authorize" class="ui button" type="button">应用授权</button> <button id="btn-gitee-authorize" class="ui button" type="button">应用授权</button>