468 lines
15 KiB
Python
468 lines
15 KiB
Python
#coding:utf-8
|
||
import pandas as pd
|
||
import sys
|
||
import time
|
||
import MySQLdb
|
||
import json
|
||
with open("_config.json") as fp:
|
||
config = json.load(fp)
|
||
local_db_config = config["local_db"]
|
||
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
||
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
||
cursor = conn.cursor()
|
||
|
||
# github id 到 ghtorrent的 id 转换
|
||
gh_gt = {}
|
||
gt_gh = {}
|
||
with open("prj_par.txt","r")as fp:
|
||
for line in fp.readlines():
|
||
ps = line.split("\t")
|
||
gt_gh[int(ps[0])] = int(ps[1])
|
||
gh_gt[int(ps[1])] = int(ps[0])
|
||
|
||
|
||
def _author_core():
|
||
cursor.execute("select repo_id,name from user_role ")
|
||
cores = set(["%d-%s"%(gh_gt[item[0]],item[1]) for item in cursor.fetchall()])
|
||
|
||
cursor.execute("select prj_id,mst_pr,dup_pr from duplicate")
|
||
dups = cursor.fetchall()
|
||
for dup in dups:
|
||
repo_id, dup_pr, mst_pr = dup
|
||
core_dup,core_mst = 0,0
|
||
|
||
cursor.execute("select author from `pull-request` where prj_id=%s and pr_num=%s",
|
||
(repo_id,dup_pr))
|
||
pr_author = cursor.fetchone()[0]
|
||
if "%d-%s"%(repo_id, pr_author) in cores:
|
||
core_dup = 1
|
||
|
||
cursor.execute("select author from `pull-request` where prj_id=%s and pr_num=%s",
|
||
(repo_id,mst_pr))
|
||
pr_author = cursor.fetchone()[0]
|
||
if "%d-%s"%(repo_id, pr_author) in cores:
|
||
core_mst = 1
|
||
|
||
cursor.execute("update compt_metrics set author_core =%s where repo_id=%s and pr_num=%s",
|
||
(core_dup,repo_id,dup_pr))
|
||
cursor.execute("update compt_metrics set author_core =%s where repo_id=%s and pr_num=%s",
|
||
(core_dup,repo_id,mst_pr))
|
||
conn.commit()
|
||
|
||
|
||
def _pr_order():
|
||
cursor.execute("select prj_id,author,pr_num from `pull-request`")
|
||
pr_history = {}
|
||
for item in cursor.fetchall():
|
||
if item[0] not in pr_history:
|
||
pr_history[item[0]] = {}
|
||
if item[1] not in pr_history[item[0]]:
|
||
pr_history[item[0]][item[1]] = []
|
||
pr_history[item[0]][item[1]].append(item[2])
|
||
for repo_id, a_p in pr_history.items():
|
||
for author, prs in a_p.items():
|
||
pr_history[repo_id][author] = sorted(prs)
|
||
|
||
|
||
cursor.execute("select prj_id,mst_pr,dup_pr from duplicate")
|
||
dups = cursor.fetchall()
|
||
for dup in dups:
|
||
repo_id, dup_pr, mst_pr = dup
|
||
|
||
cursor.execute("select author from `pull-request` where prj_id=%s and pr_num=%s",
|
||
(repo_id,dup_pr))
|
||
pr_author = cursor.fetchone()[0]
|
||
prs = pr_history[repo_id][pr_author]
|
||
pr_pos = prs.index(dup_pr)
|
||
if pr_pos == 0:
|
||
pr_order_dup = [1]
|
||
else:
|
||
pr_order_dup = [0]
|
||
|
||
cursor.execute("select author from `pull-request` where prj_id=%s and pr_num=%s",
|
||
(repo_id,mst_pr))
|
||
pr_author = cursor.fetchone()[0]
|
||
prs = pr_history[repo_id][pr_author]
|
||
pr_pos = prs.index(mst_pr)
|
||
if pr_pos == 0:
|
||
pr_order_mst = [1]
|
||
else:
|
||
pr_order_mst = [0]
|
||
|
||
pr_order_dup.extend([repo_id,dup_pr])
|
||
pr_order_mst.extend([repo_id,mst_pr])
|
||
cursor.execute("update compt_metrics set first_pr =%s where repo_id=%s and pr_num=%s",
|
||
pr_order_dup)
|
||
cursor.execute("update compt_metrics set first_pr =%s where repo_id=%s and pr_num=%s",
|
||
pr_order_mst)
|
||
conn.commit()
|
||
|
||
|
||
from unidiff import PatchSet
|
||
from io import StringIO
|
||
def _pr_work(repo_id, pr_num):
|
||
cursor.execute("select diff from pr_diff where project_id=%s and pr_id=%s",
|
||
(repo_id, pr_num))
|
||
pr_diff = cursor.fetchone()
|
||
if pr_diff is None or len(pr_diff[0].strip()) == 0:
|
||
return (0,0,repo_id, pr_num)
|
||
diff_files = PatchSet(StringIO(pr_diff[0]))
|
||
tmp_files = 0
|
||
tmp_lines = 0
|
||
for df in diff_files:
|
||
tmp_files += 1
|
||
tmp_lines += (df.added + df.removed)
|
||
return (tmp_files,tmp_lines,repo_id, pr_num)
|
||
|
||
def _work():
|
||
cursor.execute("select prj_id,mst_pr,dup_pr from duplicate")
|
||
dups = cursor.fetchall()
|
||
for dup in dups:
|
||
repo_id, dup_pr, mst_pr = dup
|
||
pr_work = _pr_work(repo_id,dup_pr)
|
||
cursor.execute("update compt_metrics set files=%s,churn =%s where repo_id=%s and pr_num=%s",
|
||
pr_work )
|
||
pr_work = _pr_work(repo_id,mst_pr)
|
||
cursor.execute("update compt_metrics set files=%s,churn =%s where repo_id=%s and pr_num=%s",
|
||
pr_work )
|
||
conn.commit()
|
||
|
||
def _strtime2int(strtime):
|
||
return time.mktime(time.strptime(strtime, '%Y-%m-%dT%H:%M:%SZ'))
|
||
def _rw_dur(repo_id, pr_num):
|
||
cursor.execute("select pr_created_at, pr_closed_at from pr_time_info where base_repo=%s and pr_num=%s",
|
||
(repo_id, pr_num))
|
||
result = cursor.fetchone()
|
||
pr_ct = _strtime2int(result[0])
|
||
if result[1] is None or len(result[1]) == 0:
|
||
return (pr_ct, sys.maxint, None)
|
||
pr_dt = _strtime2int(result[1][2:])
|
||
return (pr_ct, pr_dt, result[1][0])
|
||
|
||
|
||
|
||
def _cmts(repo_id, pr_num, ind_t):
|
||
cursor.execute("select id from `pull-request` where prj_id=%s and pr_num=%s",
|
||
(repo_id, pr_num))
|
||
pr_id = cursor.fetchone()[0]
|
||
cursor.execute("select comment_type,created_at from comments where pr_id=%s",(pr_id,))
|
||
cmts = cursor.fetchall()
|
||
cmt_count = 0
|
||
inline_cmt_count = 0
|
||
for cmt in cmts:
|
||
if cmt[1] < ind_t:
|
||
cmt_count += 1
|
||
if cmt[0] == 0:
|
||
inline_cmt_count += 1
|
||
return cmt_count, inline_cmt_count
|
||
|
||
|
||
def _time_info():
|
||
cursor.execute("select prj_id,mst_pr,dup_pr,idn_cmt from duplicate")
|
||
dups = cursor.fetchall()
|
||
for dup in dups:
|
||
repo_id, dup_pr, mst_pr, idn_cmt = dup
|
||
cursor.execute("update compt_metrics set early_arrival=1 where repo_id=%s and pr_num=%s",
|
||
(repo_id, mst_pr))
|
||
cursor.execute("update compt_metrics set early_arrival=0 where repo_id=%s and pr_num=%s",
|
||
(repo_id, dup_pr))
|
||
|
||
dup_ct, dup_dt, dup_d = _rw_dur(gt_gh[repo_id],dup_pr)
|
||
mst_ct, mst_dt, mst_d = _rw_dur(gt_gh[repo_id],mst_pr)
|
||
cursor.execute("select created_at from comment where id=%s",(idn_cmt,))
|
||
idn_t = cursor.fetchone()[0]
|
||
|
||
dup_cmts, dup_inl_cmts = _cmts(repo_id,dup_pr,idn_t)
|
||
mst_cmts, mst_inl_cmts = _cmts(repo_id,mst_pr,idn_t)
|
||
|
||
together = 1 # 是否mst 都关闭(merge 或者 close)了，dup才又提交的
|
||
if mst_dt is not None and dup_ct > mst_dt:
|
||
together = 0
|
||
|
||
idn_t = _strtime2int(idn_t)
|
||
dup_prefer = 1 # 到底选择了哪一个
|
||
# 被merge的表示被倾向了；如果都是close那就看谁的时间晚，谁就被倾向了；都是merge的不考虑
|
||
if mst_d == "M":
|
||
dup_prefer = 0 # mst 被merge了
|
||
else:
|
||
if dup_d == "M": # mst 被close了，dup被merge了
|
||
dup_prefer = 1
|
||
else:
|
||
# mst:C、O, dup: C、O
|
||
if mst_d is None and dup_d is None:
|
||
dup_prefer = None
|
||
else:
|
||
if dup_dt < mst_dt: # mst被close了，dup被close了，但是dup被close的更早（！是否需要设置两个close间隔长一些，因为有可能是人忘了关了）
|
||
dup_prefer = 0
|
||
|
||
cursor.execute("update compt_metrics set review_duration=%s, together=%s, prefer=%s, comments=%s, inline_cmts=%s where repo_id=%s and pr_num=%s",
|
||
(idn_t - dup_ct, together, dup_prefer,dup_cmts,dup_inl_cmts, repo_id, dup_pr))
|
||
cursor.execute("update compt_metrics set review_duration=%s, together=%s, prefer=%s, comments=%s, inline_cmts=%s where repo_id=%s and pr_num=%s",
|
||
(idn_t - mst_ct, together, (dup_prefer+1)%2, mst_cmts,mst_inl_cmts, repo_id, mst_pr))
|
||
|
||
|
||
conn.commit()
|
||
|
||
|
||
def init():
|
||
cursor.execute("select prj_id,mst_pr,dup_pr from duplicate")
|
||
dups = cursor.fetchall()
|
||
for dup in dups:
|
||
repo_id, dup_pr, mst_pr = dup
|
||
cursor.execute("insert into compt_metrics(repo_id, pr_num) values(%s,%s)", (repo_id, dup_pr))
|
||
cursor.execute("insert into compt_metrics(repo_id, pr_num) values(%s,%s)", (repo_id, mst_pr))
|
||
conn.commit()
|
||
|
||
def find_new_c():
|
||
cursor.execute("select prj_id,mst_pr,dup_pr from duplicate")
|
||
dups = cursor.fetchall()
|
||
for dup in dups:
|
||
repo_id, dup_pr, mst_pr = dup
|
||
cursor.execute("select author_core, prefer from compt_metrics where repo_id=%s and pr_num=%s",
|
||
(repo_id, mst_pr))
|
||
mst_info = cursor.fetchone()
|
||
cursor.execute("select author_core, prefer from compt_metrics where repo_id=%s and pr_num=%s",
|
||
(repo_id, dup_pr))
|
||
dup_info = cursor.fetchone()
|
||
if mst_info[0]==1 and mst_info[1]==0 and dup_info[0] == 0 and dup_info[1]==1:
|
||
print repo_id, dup_pr, mst_pr
|
||
if mst_info[0]==0 and mst_info[1]==1 and dup_info[0] == 1 and dup_info[1]==0:
|
||
print repo_id, dup_pr, mst_pr
|
||
|
||
|
||
def prior_exp():
|
||
# 加载贡献经验数据
|
||
sub_exp = {}
|
||
with open("exp_data/prj_all_pr_sub.txt") as fp:
|
||
for item in fp.readlines():
|
||
ids, value = item.strip().split("\t")
|
||
if ids not in sub_exp:
|
||
sub_exp[ids] = int(value)
|
||
|
||
with open("exp_data/prj_all_issue_sub.txt") as fp:
|
||
for item in fp.readlines():
|
||
ids, value = item.strip().split("\t")
|
||
if ids not in sub_exp:
|
||
sub_exp[ids] = int(value)
|
||
else:
|
||
sub_exp[ids] += int(value)
|
||
for key, value in sub_exp.items():
|
||
repo_id, pr_num = key.split("-")
|
||
cursor.execute("update compt_metrics set sub_exp=%s where repo_id=%s and pr_num=%s",
|
||
(value,repo_id,pr_num))
|
||
conn.commit()
|
||
|
||
|
||
# 加载评论经验数据
|
||
cmt_sub = {}
|
||
with open("exp_data/prj_all_pr_cmt.txt") as fp:
|
||
for item in fp.readlines():
|
||
ids, value = item.strip().split("\t")
|
||
if ids not in cmt_sub:
|
||
cmt_sub[ids] = int(value)
|
||
|
||
with open("exp_data/prj_all_issue_cmt.txt") as fp:
|
||
for item in fp.readlines():
|
||
ids, value = item.strip().split("\t")
|
||
if ids not in cmt_sub:
|
||
cmt_sub[ids] = int(value)
|
||
else:
|
||
cmt_sub[ids] += int(value)
|
||
for key, value in cmt_sub.items():
|
||
repo_id, pr_num = key.split("-")
|
||
cursor.execute("update compt_metrics set cmt_exp=%s where repo_id=%s and pr_num=%s",
|
||
(value,repo_id,pr_num))
|
||
conn.commit()
|
||
|
||
|
||
def metric_dist():
|
||
fields = "author_core, sub_exp, cmt_exp, first_pr, early_arrival, churn, files, review_duration, comments, inline_cmts"
|
||
cursor.execute("select " + fields + " from compt_metrics where together=1")
|
||
metrics = cursor.fetchall()
|
||
mlst = []
|
||
fields = fields.split(", ")
|
||
for item in fields:
|
||
mlst.append([])
|
||
for metric in metrics:
|
||
for i in range(0, len(metric)):
|
||
mlst[i].append(metric[i])
|
||
for i in range(0, len(fields)):
|
||
print fields[i], ": "
|
||
ipd = pd.Series(mlst[i])
|
||
print ipd.describe()
|
||
print ">>>>>>>" * 10
|
||
|
||
|
||
def veriry_metrics():
|
||
import random
|
||
cursor.execute("select * from compt_metrics")
|
||
metrics = [item for item in cursor.fetchall()]
|
||
random.shuffle(metrics)
|
||
for metric in metrics[:10]:
|
||
repo_id,pr_num = metric[1], metric[2]
|
||
cursor.execute("select user_name, repo_name from project where id=%s",(repo_id,))
|
||
u_n, r_n = cursor.fetchone()
|
||
print repo_id,pr_num
|
||
print "https://github.com/%s/%s/pull/%d"%(u_n, r_n, pr_num)
|
||
|
||
|
||
# 2019-02-27以后的代码
|
||
# 计算每一个pr所有评论的情感极性
|
||
# 只看issue级别的；如果有@人的话，说明是在对话，可以先着重看@作者的；应该说越往后的评论越重要
|
||
|
||
|
||
|
||
|
||
def get_deci_time(repo_id, dup_pr, mst_pr):
|
||
'''获取一对重复pr的决策时间 '''
|
||
# 哪一个被先关闭了或者被先合并了
|
||
|
||
# 现获取各自的时间点
|
||
repo_id = gt_gh[repo_id]
|
||
dup_ct, dup_dt, dup_D = _rw_dur(repo_id, dup_pr)
|
||
mst_ct, mst_dt, mst_D = _rw_dur(repo_id, mst_pr)
|
||
|
||
# 判断是否有overlap #其实只看dup_ct与mst_dt的关系就可以了，因为dup是晚于mst提交的
|
||
if dup_ct > mst_dt or mst_ct > dup_dt:
|
||
return None
|
||
|
||
# 判断决策时间
|
||
if dup_dt < mst_dt:
|
||
return dup_dt
|
||
else:
|
||
return mst_dt
|
||
|
||
import re
|
||
import subprocess
|
||
import shlex
|
||
def _txt_clean(text):
|
||
'''清洗文本'''
|
||
# 把评论中的一些干扰项去除掉
|
||
# 代码去掉
|
||
text = re.sub('(```[\s\S]*?```|``[\s\S]*?``|`[\s\S]*?`)',"@@cmmcode@@",text)
|
||
# 他人的引用去掉
|
||
text = re.sub('(>.*\n|on.*\nwrote:)', "@@cmmref@@", text.lower())#去掉引用块
|
||
# 超链接去掉
|
||
text = re.sub("((http|ftp|https)://)(([a-zA-Z0-9\._-]+\.[a-zA-Z]{2,6})|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}))(:[0-9]{1,4})*(/[a-zA-Z0-9\&%_\./-~-]*)?","@@cmmhttp@@",text)#去掉http
|
||
|
||
# 把换行符去掉
|
||
text = re.sub(r"\n"," ", text)
|
||
# 把多个空格替换成一个空格
|
||
text = re.sub(r"\s+"," ", text)
|
||
|
||
return text
|
||
|
||
|
||
def _txt_sent(txt):
|
||
'''一段文本的情感'''
|
||
# print "\tbc: ", txt.strip()
|
||
txt = _txt_clean(txt)
|
||
# print "\tac: ", txt
|
||
|
||
ss_data_path = "/Users/lizhixing/Documents/project"
|
||
cmd_str = "java -jar SentiStrength.jar stdin sentidata SentiStrength_Data/ explain additionalFile ReviewCommentAddition.txt"
|
||
p = subprocess.Popen(shlex.split(cmd_str), cwd=ss_data_path,
|
||
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
||
|
||
stdout_text, stderr_text = p.communicate(txt)
|
||
sentis = [item.strip() for item in stdout_text.split("\t")]
|
||
# extract the positive value, negative value, and explanation.
|
||
# print "\t", sentis
|
||
return sentis[:2]
|
||
|
||
|
||
def _cmt_senti(repo_id, pr_num, decide_time):
|
||
'''在decide_time前提交的几个comment的情感分析'''
|
||
cursor.execute("select id,author from `pull-request` where prj_id=%s and pr_num=%s",
|
||
(repo_id, pr_num))
|
||
pr_id, pr_author = cursor.fetchone()
|
||
print pr_id, repo_id, pr_num, pr_author
|
||
|
||
# 取出pr的所有comment
|
||
cursor.execute("select author_name, comment_body, comment_type, created_at,id from comments where pr_id=%s",(pr_id,))
|
||
cmts = cursor.fetchall()
|
||
print"\ttotal:",len(cmts)
|
||
|
||
# 挑选出非作者本人的评论（非决策comment到底还留不留呢？有些是夸奖两句直接接受了）
|
||
cmts = [item for item in cmts if item[0] != pr_author]
|
||
print"\tno pr_author:",len(cmts)
|
||
# print"\n"
|
||
|
||
# 按时间排序，从决策时间往前看n个
|
||
decide_time = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime(decide_time))
|
||
cmts = [item for item in sorted(cmts, key=lambda x:x[3], reverse=True) if item[3]<=decide_time]
|
||
if len(cmts) == 0:
|
||
return
|
||
senti_num = [0,0] #记录各种极性情感的评论的个数
|
||
for cmt in cmts:
|
||
# print "\t", cmt[0], cmt[3], cmt[2]
|
||
cmt_senti = _txt_sent(cmt[1])
|
||
if int(cmt_senti[0]) > 1:
|
||
senti_num[0] += 1
|
||
if int(cmt_senti[1]) < -1:
|
||
senti_num[1] += 1
|
||
# print "\n"
|
||
|
||
print "\t SENTI_P: ", senti_num, [1.0*item/len(cmts) for item in senti_num]
|
||
|
||
def load_cust_cfg():
|
||
# 把项目中的定制化配置文件更新到 sentistrength相应目录下
|
||
ss_dir = ss_data_path = "/Users/lizhixing/Documents/project/SentiStrength_Data"
|
||
|
||
with open("%s/ReviewCommentAddition.txt"%(ss_dir,),"w+") as t_fp:
|
||
t_fp.truncate()
|
||
with open("custom-config/ReviewCommentAddition.txt","r+") as s_fp:
|
||
for line in s_fp:
|
||
t_fp.write(line)
|
||
with open("%s/EmoticonLookupTable.txt"%(ss_dir,),"w+") as t_fp:
|
||
t_fp.truncate()
|
||
with open("custom-config/EmoticonLookupTable.txt","r+") as s_fp:
|
||
for line in s_fp:
|
||
t_fp.write(line)
|
||
# 目前配置的有
|
||
# lgtm: 3.
|
||
|
||
def cmt_sentiment():
|
||
# 可以这样：每一个评论有这样的极性得分[p,n]，然后看他所有的评论中，p>1的比例有多少，n<-1的比例有多少
|
||
# 也就说有两个指标，包含积极情感的审阅意见占比； 包含消极情感的审阅意见占比
|
||
load_cust_cfg()
|
||
cursor.execute("select prj_id,mst_pr,dup_pr,idn_cmt from duplicate")
|
||
import random
|
||
dups = [list(item) for item in cursor.fetchall()] #cvt 2 list to shuffle
|
||
random.shuffle(dups)
|
||
for dup in dups[:5]:
|
||
# 取出pr的相关信息
|
||
repo_id, dup_pr, mst_pr, idn_cmt = dup
|
||
print dup
|
||
|
||
# 判断两个pr的决策时间
|
||
decide_time = get_deci_time(repo_id,mst_pr,dup_pr)
|
||
if decide_time is None:
|
||
print "no overlapped"
|
||
print "*"*20
|
||
continue
|
||
|
||
# 挨个判断
|
||
_cmt_senti(repo_id, dup_pr, decide_time)
|
||
_cmt_senti(repo_id, mst_pr, decide_time)
|
||
print "*"*20
|
||
|
||
if __name__ == '__main__':
|
||
# init()
|
||
# _author_core()
|
||
# _pr_order()
|
||
# _work()
|
||
# _time_info()
|
||
# find_new_c()
|
||
# prior_exp()
|
||
|
||
# metric_dist()
|
||
|
||
# veriry_metrics()
|
||
|
||
# 测试情感分析效果
|
||
# load_cust_cfg()
|
||
# cmt_str = "lgtm, we can merge this pr. +1"
|
||
# _txt_sent(cmt_str)
|
||
cmt_sentiment()
|