313 lines
9.3 KiB
Python
313 lines
9.3 KiB
Python
|
|
# coding: utf-8
|
|
import MySQLdb
|
|
import json
|
|
import time
|
|
import random
|
|
import urllib2
|
|
import pandas as pd
|
|
import matplotlib.mlab as mlab
|
|
import matplotlib.pyplot as plt
|
|
|
|
|
|
with open("../.config.json") as fp:
|
|
config = json.load(fp)
|
|
local_db_config = config["local_db"]
|
|
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
|
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
|
cursor = conn.cursor()
|
|
|
|
def profile_type(doc):
|
|
doc_id,repo_id,doc_location,doc_name,doc_type = doc
|
|
if doc_location.lower() in ["root",".github","docs"] and doc_type=="blob":
|
|
doc_name = doc_name.lower()
|
|
for fi in ["readme"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "readme"
|
|
for fi in ["license","licence","copyright"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "license"
|
|
for fi in ["issue_template","pull_request_template"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "template"
|
|
for fi in ["code_of_conduct"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "COC"
|
|
for fi in ["contributing"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "contributing"
|
|
return None
|
|
elif doc_location.lower() in ["issue_template", "pull_request_template"] and doc_type=="blob":
|
|
return "template"
|
|
else:
|
|
return None
|
|
return None
|
|
|
|
|
|
def overview():
|
|
cursor.execute("select id from random_repos")
|
|
repos = set([item[0] for item in cursor.fetchall()])
|
|
prf_files = []
|
|
cursor.execute("select * from files")
|
|
files = cursor.fetchall()
|
|
for f in files:
|
|
if f[1] in repos:
|
|
prf_files.append([f[0],f[1],profile_type(f)])
|
|
# print f, profile_type(f)
|
|
|
|
repo_counts = len(repos)
|
|
dist = {"readme":set(),"license":set(),"template":set(),
|
|
"COC":set(),"contributing":set()}
|
|
|
|
for _f in prf_files:
|
|
doc_id,repo_id,doc_type = _f
|
|
if doc_type is not None:
|
|
dist[doc_type].add(repo_id)
|
|
|
|
for key, value in dist.items():
|
|
dist[key] = len(value)
|
|
|
|
cursor.execute("select count(id) from random_repos where description is not null")
|
|
dist["description"] = cursor.fetchone()[0]
|
|
|
|
for key, value in dist.items():
|
|
print key, "%.2f%%"%(value*100.0/repo_counts)
|
|
|
|
|
|
|
|
def repo_profile(repo_id):
|
|
file_types = set()
|
|
cursor.execute("select * from files where repo_id=%s",(repo_id,))
|
|
files = cursor.fetchall()
|
|
for f in files:
|
|
f_t = profile_type(f)
|
|
if f_t is not None:
|
|
file_types.add(f_t)
|
|
return file_types
|
|
|
|
def by_language():
|
|
cursor.execute("select id, language from random_repos")
|
|
repos = cursor.fetchall()
|
|
language_dist = {}
|
|
|
|
file_dist = {}
|
|
|
|
for item in repos:
|
|
repo_id, repo_lan = item
|
|
if repo_lan not in language_dist:
|
|
language_dist[repo_lan] = 0
|
|
language_dist[repo_lan] += 1
|
|
if repo_lan not in file_dist:
|
|
file_dist[repo_lan] = {}
|
|
repo_files = repo_profile(repo_id)
|
|
for rf in repo_files:
|
|
if rf not in file_dist[repo_lan]:
|
|
file_dist[repo_lan][rf] = 0
|
|
file_dist[repo_lan][rf] += 1
|
|
lans = sorted(language_dist.items(), key=lambda x: x[1], reverse=True)
|
|
print lans
|
|
for lan, lan_count in lans[0:10]:
|
|
tmp_lan_dist = file_dist[lan]
|
|
percentages = []
|
|
for _k in [ "readme","license","contributing","COC","template"]:
|
|
if _k not in tmp_lan_dist:
|
|
percentages.append(0)
|
|
else:
|
|
percentages.append(tmp_lan_dist[_k])
|
|
p_str = "&\t".join(["%.2f\%%"%(item*100.0/language_dist[lan]) for item in percentages])
|
|
print "%s\t&%d\t&%s\\\\"%(lan,language_dist[lan],p_str)
|
|
|
|
def by_role():
|
|
cursor.execute("select id, owner_type from random_repos")
|
|
repos = cursor.fetchall()
|
|
role_type = {}
|
|
|
|
file_dist = {}
|
|
|
|
for item in repos:
|
|
repo_id, repo_lan = item
|
|
if repo_lan not in role_type:
|
|
role_type[repo_lan] = 0
|
|
role_type[repo_lan] += 1
|
|
if repo_lan not in file_dist:
|
|
file_dist[repo_lan] = {}
|
|
repo_files = repo_profile(repo_id)
|
|
for rf in repo_files:
|
|
if rf not in file_dist[repo_lan]:
|
|
file_dist[repo_lan][rf] = 0
|
|
file_dist[repo_lan][rf] += 1
|
|
lans = sorted(role_type.items(), key=lambda x: x[1], reverse=True)
|
|
print lans
|
|
for lan, lan_count in lans:
|
|
tmp_lan_dist = file_dist[lan]
|
|
percentages = []
|
|
for _k in [ "readme","license","contributing","COC","template"]:
|
|
if _k not in tmp_lan_dist:
|
|
percentages.append(0)
|
|
else:
|
|
percentages.append(tmp_lan_dist[_k])
|
|
p_str = "&\t".join(["%.2f\%%"%(item*100.0/role_type[lan]) for item in percentages])
|
|
print "%s&%d&%s\\\\"%(lan,role_type[lan],p_str)
|
|
|
|
|
|
import time
|
|
def _strtime2int(strtime):
|
|
return time.mktime(time.strptime(strtime, '%Y-%m-%dT%H:%M:%SZ'))
|
|
|
|
|
|
import seaborn as sns
|
|
import math
|
|
from scipy.stats import mannwhitneyu
|
|
def by_age():
|
|
cursor.execute("select id, created_at from random_repos")
|
|
repos = cursor.fetchall()
|
|
# 第一个是包含的，第二个是不包含的
|
|
file_dist = { "readme":[[],[]],"license":[[],[]],"contributing":[[],[]],"COC":[[],[]],"template":[[],[]]}
|
|
|
|
for item in repos:
|
|
repo_id, repo_create = item
|
|
repo_files = repo_profile(repo_id)
|
|
repo_age = (_strtime2int("2020-07-18T00:00:00Z") - _strtime2int(repo_create))*1.0/60/60/24/30
|
|
if repo_age <= 0:
|
|
print repo_create
|
|
for key in file_dist.keys():
|
|
if key in repo_files:
|
|
file_dist[key][0].append(repo_age)
|
|
else:
|
|
file_dist[key][1].append(repo_age)
|
|
|
|
x,y,c = [],[],[]
|
|
for key, value in file_dist.items():
|
|
for vl in value[0]:
|
|
x.append(key)
|
|
y.append(vl)
|
|
c.append("Included")
|
|
for vl in value[1]:
|
|
x.append(key)
|
|
y.append(vl)
|
|
c.append("Not-included")
|
|
|
|
stats1 = pd.Series(value[0])
|
|
stats2 = pd.Series(value[1])
|
|
print key, "Yes", stats1.min(), stats1.mean(),stats1.median(),stats1.std(),stats1.max()
|
|
print key, "No", stats2.min(), stats2.mean(),stats2.median(),stats2.std(),stats2.max()
|
|
print "%.2f (median: %.2f)"%(stats1.mean(),stats1.median())
|
|
print "%.2f (median: %.2f)"%(stats2.mean(),stats2.median())
|
|
|
|
mwws = mannwhitneyu(value[0],value[1],alternative='two-sided')
|
|
print mwws
|
|
|
|
print "----"*10
|
|
data = {
|
|
'Docmentation Type':x,
|
|
'Repository age':y,
|
|
'Group':c}
|
|
df = pd.DataFrame(data, columns=["Docmentation Type", "Repository age", "Group"])
|
|
|
|
plt.figure(figsize=(10,5))
|
|
|
|
sns.set(style="ticks", palette="muted")
|
|
sns_plot = sns.violinplot(x= "Docmentation Type", y="Repository age",data=df,
|
|
hue="Group",
|
|
split=True,
|
|
linewidth = 2, #线宽
|
|
width = 0.8, #箱之间的间隔比例
|
|
palette= {"Included":"#C0C0C0","Not-included":"#DCDCDC"}, #设置调色板
|
|
order = ["readme","license","contributing","COC","template"],
|
|
# scale = 'count', #测度小提琴图的宽度： area-面积相同,count-按照样本数量决定宽度,width-宽度一样
|
|
gridsize = 50, #设置小提琴图的平滑度，越高越平滑
|
|
cut = 0,
|
|
inner='quartiles'
|
|
)
|
|
|
|
|
|
plt.legend(ncol=2)
|
|
plt.xticks(range(0,5),["README", "LICENSE", "CONTRIBUTING", "CONDUCT", "TEMPLATE"])
|
|
plt.ylim(None,170)
|
|
plt.ylabel("Repository age (in month)")
|
|
plt.show()
|
|
fig = sns_plot.get_figure()
|
|
fig.savefig("../../resources/repo_age.pdf",dpi=200)
|
|
plt.close()
|
|
|
|
def by_forks():
|
|
cursor.execute("select id, forks from random_repos")
|
|
repos = cursor.fetchall()
|
|
# 第一个是包含的，第二个是不包含的
|
|
file_dist = { "readme":[[],[]],"license":[[],[]],"contributing":[[],[]],"COC":[[],[]],"template":[[],[]]}
|
|
|
|
for item in repos:
|
|
repo_id, repo_forks = item
|
|
repo_forks = math.log(repo_forks)
|
|
repo_files = repo_profile(repo_id)
|
|
for key in file_dist.keys():
|
|
if key in repo_files:
|
|
file_dist[key][0].append(repo_forks)
|
|
else:
|
|
file_dist[key][1].append(repo_forks)
|
|
|
|
x,y,c = [],[],[]
|
|
for key, value in file_dist.items():
|
|
for vl in value[0]:
|
|
x.append(key)
|
|
y.append(vl)
|
|
c.append("Included")
|
|
for vl in value[1]:
|
|
x.append(key)
|
|
y.append(vl)
|
|
c.append("Not-included")
|
|
|
|
stats1 = pd.Series(value[0])
|
|
stats2 = pd.Series(value[1])
|
|
print key, "Yes", stats1.min(), stats1.mean(),stats1.median(),stats1.std(),stats1.max()
|
|
print key, "No", stats2.min(), stats2.mean(),stats2.median(),stats2.std(),stats2.max()
|
|
|
|
mwws = mannwhitneyu(value[0],value[1],alternative='two-sided')
|
|
print mwws
|
|
|
|
print "%.2f (median: %d)"%(stats1.mean(),stats1.median())
|
|
print "%.2f (median: %d)"%(stats2.mean(),stats2.median())
|
|
|
|
|
|
print "----"*10
|
|
data = {
|
|
'Docmentation Type':x,
|
|
'Repository forks':y,
|
|
'Group':c}
|
|
df = pd.DataFrame(data, columns=["Docmentation Type", "Repository forks", "Group"])
|
|
plt.figure(figsize=(10,5))
|
|
sns.set(style="ticks", palette="muted")
|
|
sns_plot = sns.violinplot(x= "Docmentation Type", y="Repository forks",data=df,
|
|
hue="Group",
|
|
split=True,
|
|
linewidth = 2, #线宽
|
|
width = 0.8, #箱之间的间隔比例
|
|
palette= {"Included":"#C0C0C0","Not-included":"#DCDCDC"}, #设置调色板
|
|
order = ["readme","license","contributing","COC","template"],
|
|
# scale = 'count', #测度小提琴图的宽度： area-面积相同,count-按照样本数量决定宽度,width-宽度一样
|
|
gridsize = 50, #设置小提琴图的平滑度，越高越平滑
|
|
cut = 0,
|
|
inner='quartiles'
|
|
)
|
|
plt.legend(ncol=2)
|
|
plt.ylim(None,12)
|
|
plt.xticks(range(0,5),["README", "LICENSE", "CONTRIBUTING", "CONDUCT", "TEMPLATE"])
|
|
plt.ylabel("Repository forks (log)")
|
|
plt.show()
|
|
fig = sns_plot.get_figure()
|
|
fig.savefig("../../resources/repo_fork.pdf",dpi=200)
|
|
plt.close()
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
overview()
|
|
# by_language()
|
|
# by_role()
|
|
# by_age()
|
|
# by_forks()
|
|
|
|
|
|
|