From 8c23c6572e21feb10ead5ef5ea3d164f1dec73c4 Mon Sep 17 00:00:00 2001 From: Agazelle <764467302@qq.com> Date: Mon, 27 Jun 2022 11:07:04 +0800 Subject: [PATCH] =?UTF-8?q?6.27=20=E8=8E=B7=E5=8F=96=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E3=80=81issue=E8=AF=84=E8=AE=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Agazelle <764467302@qq.com> --- getRepoComment.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++ issues_SA.py | 81 ++++++++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 getRepoComment.py create mode 100644 issues_SA.py diff --git a/getRepoComment.py b/getRepoComment.py new file mode 100644 index 0000000..d4d9826 --- /dev/null +++ b/getRepoComment.py @@ -0,0 +1,152 @@ +import requests +import pymysql +import datetime + +def get_comment(page): + url = 'https://api.github.com/repos/rails/rails/issues/comments' + print(url) + headers = {'User-Agent': 'Mozilla/5.0', + 'Authorization': 'token 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', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + params = {#'sort': 'created', + 'page': page, + 'per_page': 100, + #'q': "comments:>10 created:>2015-01-01" + 'since': '2001-06-01' + } + s = requests.session() + s.keep_alive = False + response = requests.get(url, headers=headers,params=params) + if response.status_code != 200: + print(response.json()) + print('get_commit error: fail to request') + j = response.json() + for i in range(len(j)): + value = [j[i]['issue_url'].split('https://api.github.com/repos/')[1].split('/issues/')[0], + j[i]['issue_url'].split('/issues/')[1], j[i]['user']['login'], j[i]['user']['type'], + str_to_datetime(str(j[i]['created_at'])), str_to_datetime(str(j[i]['updated_at'])),j[i]['body']] + #print(value) + save_comments(value) + +def str_to_datetime(str): + #return str.split('T')[0] + ' ' + str.split('T')[1].split('Z')[0] + return datetime.datetime.strptime(str.split('T')[0] + ' ' + str.split('T')[1].split('Z')[0], "%Y-%m-%d %H:%M:%S") + +def save_comments(value): + # 打开数据库连接 + db = pymysql.connect(host='localhost', + user='root', + password='123456', + database='rails', + use_unicode=True, + charset='utf8') + # 使用 cursor() 方法创建一个游标对象 cursor + cursor = db.cursor() + # SQL 插入语句 + sql = """INSERT INTO rails_comments(repo,issue_id,user_login,user_type,created_time,updated_time,comment) + VALUES (%s, %s, %s, %s, %s, %s, %s)""" + + # 使用 execute() 方法执行 SQL 查询 + try: + # 执行sql语句 + cursor.execute(sql, value) + # 提交到数据库执行 + db.commit() + except Exception as e: + # 如果发生错误则回滚 + db.rollback() + value[6]="-1-1-1" + cursor.execute(sql, value) + db.commit() + print("ERR0R:") + print(e) + # 关闭数据库连接 + db.close() + +def get_repo(page): + url = 'https://api.github.com/search/repositories' + print(url) + # 'Accept': 'application/vnd.github.cloak-preview' + headers = {'User-Agent': 'Mozilla/5.0', + 'Authorization': 'token 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', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + params = {'sort': 'stars', + 'page': page, + 'per_page': 100, + 'order': 'asc', + 'q': "stars:>3000" + } + response = requests.get(url, headers=headers, params=params) + if response.status_code != 200: + print(response.json()) + print('get_commit error: fail to request') + j = response.json() + update_date = str_to_datetime('2022-01-01T00:00:00Z') + for repo in j['items']: + if repo['fork'] == False and str_to_datetime(repo['updated_at']) > update_date: + get_issue() + +def save_repos(value): + # 打开数据库连接 + db = pymysql.connect(host='localhost', + user='root', + password='123456', + database='rails', + use_unicode=True, + charset='utf8') + # 使用 cursor() 方法创建一个游标对象 cursor + cursor = db.cursor() + # SQL 插入语句 + sql = """INSERT INTO rails_comments(repo,issue_id,user_login,user_type,created_time,updated_time,comment) + VALUES (%s, %s, %s, %s, %s, %s, %s)""" + + # 使用 execute() 方法执行 SQL 查询 + try: + # 执行sql语句 + cursor.execute(sql, value) + # 提交到数据库执行 + db.commit() + except Exception as e: + # 如果发生错误则回滚 + db.rollback() + value[6]="-1-1-1" + cursor.execute(sql, value) + db.commit() + print("ERR0R:") + print(e) + # 关闭数据库连接 + db.close() + + +def get_issue(page): + url = 'https://api.github.com/search/issues' + print(url) + # 'Accept': 'application/vnd.github.cloak-preview' + headers = {'User-Agent': 'Mozilla/5.0', + 'Authorization': 'token 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', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + params = {#'sort': 'stars', + 'page': page, + 'per_page': 100, + 'order': 'asc', + 'q': "created_at:>2010-01-01 " + } + response = requests.get(url, headers=headers, params=params) + if response.status_code != 200: + print(response.json()) + print('get_commit error: fail to request') + j = response.json() + #print(j) + print(j['total_count']) + + +if __name__ == '__main__': + get_repo(1) + #for i in range(0,10000): + # get_comment(i) \ No newline at end of file diff --git a/issues_SA.py b/issues_SA.py new file mode 100644 index 0000000..23cb062 --- /dev/null +++ b/issues_SA.py @@ -0,0 +1,81 @@ +from senta import Senta +import requests +import pymysql +from util import config + +my_senta = Senta() + +# 获取目前支持的情感预训练模型, 我们开放了以ERNIE 1.0 large(中文)、ERNIE 2.0 large(英文)和RoBERTa large(英文)作为初始化的SKEP模型 +print(my_senta.get_support_model()) # ["ernie_1.0_skep_large_ch", "ernie_2.0_skep_large_en", "roberta_skep_large_en"] + +# 获取目前支持的预测任务 +print(my_senta.get_support_task()) # ["sentiment_classify", "aspect_sentiment_classify", "extraction"] + +# 选择是否使用gpu +use_cuda = False # 设置True or False + +# 预测英文句子级情感分类任务（基于SKEP-ERNIE2.0模型） +my_senta.init_model(model_class="roberta_skep_large_en", task="sentiment_classify", use_cuda=use_cuda) + +def find_issue(): + # 打开数据库连接 + db = pymysql.connect(host='localhost', + user='root', + password='123456', + database='repo_res', + use_unicode=True, + charset='utf8') + # 使用 cursor() 方法创建一个游标对象 cursor + cursor = db.cursor() + # SQL查询语句 + #sql_query = "SELECT id,body FROM issuesbot WHERE id < '220000'" + sql_query = "SELECT id,body FROM issuesbot WHERE id > '230000'" + s = requests.session() + s.keep_alive = False + try: + # 执行SQL语句 + cursor.execute(sql_query) + # 获取所有记录列表 + results = cursor.fetchall() + print('length:') + print(len(results)) + #text = list(results['body']) + text = [r[1] for r in results] + #input_dict = {"text":text} + #sa_res = Senta.sentiment_classify(data = input_dict) + said = 230001 + for t in text: + if len(t) > 300 or t is None: + said += 1 + continue + res_tmp = my_senta.predict(t) + print(t) + print(res_tmp[0][1]) + # SQL更新语句 + sql = "UPDATE issuesbot SET sares = '%s' WHERE id = '%d'" % (res_tmp[0][1], said) + #sql_update = "UPDATE repos SET is_refine2 = 1,refine2 = '%d' WHERE id = '%d'" % (flag, results[i][0]) + try: + # 执行SQL语句 + cursor.execute(sql) + # 提交到数据库执行 + db.commit() + said += 1 + except Exception as e: + # 如果发生错误则回滚 + db.rollback() + print("ERR0R1:") + print(e) + except Exception as e: + # 如果发生错误则回滚 + db.rollback() + print("ERR0R2:") + print(e) + finally: + # 关闭数据库连接 + db.close() + + +find_issue() +#texts = ["my name is Gao"] +#result = my_senta.predict(texts) +#print(result) \ No newline at end of file