6.27 获取仓库、issue评论
Signed-off-by: Agazelle <764467302@qq.com>
This commit is contained in:
commit
8c23c6572e
|
|
@ -0,0 +1,152 @@
|
|||
import requests
|
||||
import pymysql
|
||||
import datetime
|
||||
|
||||
def get_comment(page):
|
||||
url = 'https://api.github.com/repos/rails/rails/issues/comments'
|
||||
print(url)
|
||||
headers = {'User-Agent': 'Mozilla/5.0',
|
||||
'Authorization': 'token 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',
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
params = {#'sort': 'created',
|
||||
'page': page,
|
||||
'per_page': 100,
|
||||
#'q': "comments:>10 created:>2015-01-01"
|
||||
'since': '2001-06-01'
|
||||
}
|
||||
s = requests.session()
|
||||
s.keep_alive = False
|
||||
response = requests.get(url, headers=headers,params=params)
|
||||
if response.status_code != 200:
|
||||
print(response.json())
|
||||
print('get_commit error: fail to request')
|
||||
j = response.json()
|
||||
for i in range(len(j)):
|
||||
value = [j[i]['issue_url'].split('https://api.github.com/repos/')[1].split('/issues/')[0],
|
||||
j[i]['issue_url'].split('/issues/')[1], j[i]['user']['login'], j[i]['user']['type'],
|
||||
str_to_datetime(str(j[i]['created_at'])), str_to_datetime(str(j[i]['updated_at'])),j[i]['body']]
|
||||
#print(value)
|
||||
save_comments(value)
|
||||
|
||||
def str_to_datetime(str):
|
||||
#return str.split('T')[0] + ' ' + str.split('T')[1].split('Z')[0]
|
||||
return datetime.datetime.strptime(str.split('T')[0] + ' ' + str.split('T')[1].split('Z')[0], "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def save_comments(value):
|
||||
# 打开数据库连接
|
||||
db = pymysql.connect(host='localhost',
|
||||
user='root',
|
||||
password='123456',
|
||||
database='rails',
|
||||
use_unicode=True,
|
||||
charset='utf8')
|
||||
# 使用 cursor() 方法创建一个游标对象 cursor
|
||||
cursor = db.cursor()
|
||||
# SQL 插入语句
|
||||
sql = """INSERT INTO rails_comments(repo,issue_id,user_login,user_type,created_time,updated_time,comment)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)"""
|
||||
|
||||
# 使用 execute() 方法执行 SQL 查询
|
||||
try:
|
||||
# 执行sql语句
|
||||
cursor.execute(sql, value)
|
||||
# 提交到数据库执行
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
# 如果发生错误则回滚
|
||||
db.rollback()
|
||||
value[6]="-1-1-1"
|
||||
cursor.execute(sql, value)
|
||||
db.commit()
|
||||
print("ERR0R:")
|
||||
print(e)
|
||||
# 关闭数据库连接
|
||||
db.close()
|
||||
|
||||
def get_repo(page):
|
||||
url = 'https://api.github.com/search/repositories'
|
||||
print(url)
|
||||
# 'Accept': 'application/vnd.github.cloak-preview'
|
||||
headers = {'User-Agent': 'Mozilla/5.0',
|
||||
'Authorization': 'token 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',
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
params = {'sort': 'stars',
|
||||
'page': page,
|
||||
'per_page': 100,
|
||||
'order': 'asc',
|
||||
'q': "stars:>3000"
|
||||
}
|
||||
response = requests.get(url, headers=headers, params=params)
|
||||
if response.status_code != 200:
|
||||
print(response.json())
|
||||
print('get_commit error: fail to request')
|
||||
j = response.json()
|
||||
update_date = str_to_datetime('2022-01-01T00:00:00Z')
|
||||
for repo in j['items']:
|
||||
if repo['fork'] == False and str_to_datetime(repo['updated_at']) > update_date:
|
||||
get_issue()
|
||||
|
||||
def save_repos(value):
|
||||
# 打开数据库连接
|
||||
db = pymysql.connect(host='localhost',
|
||||
user='root',
|
||||
password='123456',
|
||||
database='rails',
|
||||
use_unicode=True,
|
||||
charset='utf8')
|
||||
# 使用 cursor() 方法创建一个游标对象 cursor
|
||||
cursor = db.cursor()
|
||||
# SQL 插入语句
|
||||
sql = """INSERT INTO rails_comments(repo,issue_id,user_login,user_type,created_time,updated_time,comment)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)"""
|
||||
|
||||
# 使用 execute() 方法执行 SQL 查询
|
||||
try:
|
||||
# 执行sql语句
|
||||
cursor.execute(sql, value)
|
||||
# 提交到数据库执行
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
# 如果发生错误则回滚
|
||||
db.rollback()
|
||||
value[6]="-1-1-1"
|
||||
cursor.execute(sql, value)
|
||||
db.commit()
|
||||
print("ERR0R:")
|
||||
print(e)
|
||||
# 关闭数据库连接
|
||||
db.close()
|
||||
|
||||
|
||||
def get_issue(page):
|
||||
url = 'https://api.github.com/search/issues'
|
||||
print(url)
|
||||
# 'Accept': 'application/vnd.github.cloak-preview'
|
||||
headers = {'User-Agent': 'Mozilla/5.0',
|
||||
'Authorization': 'token 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',
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
params = {#'sort': 'stars',
|
||||
'page': page,
|
||||
'per_page': 100,
|
||||
'order': 'asc',
|
||||
'q': "created_at:>2010-01-01 "
|
||||
}
|
||||
response = requests.get(url, headers=headers, params=params)
|
||||
if response.status_code != 200:
|
||||
print(response.json())
|
||||
print('get_commit error: fail to request')
|
||||
j = response.json()
|
||||
#print(j)
|
||||
print(j['total_count'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
get_repo(1)
|
||||
#for i in range(0,10000):
|
||||
# get_comment(i)
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
from senta import Senta
|
||||
import requests
|
||||
import pymysql
|
||||
from util import config
|
||||
|
||||
my_senta = Senta()
|
||||
|
||||
# 获取目前支持的情感预训练模型, 我们开放了以ERNIE 1.0 large(中文)、ERNIE 2.0 large(英文)和RoBERTa large(英文)作为初始化的SKEP模型
|
||||
print(my_senta.get_support_model()) # ["ernie_1.0_skep_large_ch", "ernie_2.0_skep_large_en", "roberta_skep_large_en"]
|
||||
|
||||
# 获取目前支持的预测任务
|
||||
print(my_senta.get_support_task()) # ["sentiment_classify", "aspect_sentiment_classify", "extraction"]
|
||||
|
||||
# 选择是否使用gpu
|
||||
use_cuda = False # 设置True or False
|
||||
|
||||
# 预测英文句子级情感分类任务（基于SKEP-ERNIE2.0模型）
|
||||
my_senta.init_model(model_class="roberta_skep_large_en", task="sentiment_classify", use_cuda=use_cuda)
|
||||
|
||||
def find_issue():
|
||||
# 打开数据库连接
|
||||
db = pymysql.connect(host='localhost',
|
||||
user='root',
|
||||
password='123456',
|
||||
database='repo_res',
|
||||
use_unicode=True,
|
||||
charset='utf8')
|
||||
# 使用 cursor() 方法创建一个游标对象 cursor
|
||||
cursor = db.cursor()
|
||||
# SQL查询语句
|
||||
#sql_query = "SELECT id,body FROM issuesbot WHERE id < '220000'"
|
||||
sql_query = "SELECT id,body FROM issuesbot WHERE id > '230000'"
|
||||
s = requests.session()
|
||||
s.keep_alive = False
|
||||
try:
|
||||
# 执行SQL语句
|
||||
cursor.execute(sql_query)
|
||||
# 获取所有记录列表
|
||||
results = cursor.fetchall()
|
||||
print('length:')
|
||||
print(len(results))
|
||||
#text = list(results['body'])
|
||||
text = [r[1] for r in results]
|
||||
#input_dict = {"text":text}
|
||||
#sa_res = Senta.sentiment_classify(data = input_dict)
|
||||
said = 230001
|
||||
for t in text:
|
||||
if len(t) > 300 or t is None:
|
||||
said += 1
|
||||
continue
|
||||
res_tmp = my_senta.predict(t)
|
||||
print(t)
|
||||
print(res_tmp[0][1])
|
||||
# SQL更新语句
|
||||
sql = "UPDATE issuesbot SET sares = '%s' WHERE id = '%d'" % (res_tmp[0][1], said)
|
||||
#sql_update = "UPDATE repos SET is_refine2 = 1,refine2 = '%d' WHERE id = '%d'" % (flag, results[i][0])
|
||||
try:
|
||||
# 执行SQL语句
|
||||
cursor.execute(sql)
|
||||
# 提交到数据库执行
|
||||
db.commit()
|
||||
said += 1
|
||||
except Exception as e:
|
||||
# 如果发生错误则回滚
|
||||
db.rollback()
|
||||
print("ERR0R1:")
|
||||
print(e)
|
||||
except Exception as e:
|
||||
# 如果发生错误则回滚
|
||||
db.rollback()
|
||||
print("ERR0R2:")
|
||||
print(e)
|
||||
finally:
|
||||
# 关闭数据库连接
|
||||
db.close()
|
||||
|
||||
|
||||
find_issue()
|
||||
#texts = ["my name is Gao"]
|
||||
#result = my_senta.predict(texts)
|
||||
#print(result)
|
||||
Loading…
Reference in New Issue