get file content
This commit is contained in:
parent
01522c3efe
commit
e27d4f891f
|
|
@ -0,0 +1,91 @@
|
|||
|
||||
# coding: utf-8
|
||||
import MySQLdb
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import urllib2
|
||||
import pandas as pd
|
||||
import matplotlib.mlab as mlab
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
with open("../.config.json") as fp:
|
||||
config = json.load(fp)
|
||||
local_db_config = config["local_db"]
|
||||
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
||||
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
||||
cursor = conn.cursor()
|
||||
|
||||
import collections
|
||||
creation = collections.OrderedDict()
|
||||
creation["RDM"]=[]
|
||||
creation["COD"]=[]
|
||||
creation["CON"]=[]
|
||||
creation["LIC"]=[]
|
||||
|
||||
def _dur(t1, t2):
|
||||
t1 = time.mktime(time.strptime(t1, '%Y-%m-%dT%H:%M:%SZ'))
|
||||
t2 = time.mktime(time.strptime(t2, '%Y-%m-%dT%H:%M:%SZ'))
|
||||
# return math.log((t2-t1)*1.0/60/60+1)
|
||||
return (t2-t1)*1.0/60/60/24
|
||||
|
||||
|
||||
early_c = collections.OrderedDict()
|
||||
early_c["RDM"]=0
|
||||
early_c["COD"]=0
|
||||
early_c["CON"]=0
|
||||
early_c["LIC"]=0
|
||||
|
||||
cursor.execute("select id from repos")
|
||||
repos = cursor.fetchall()
|
||||
for repo in repos:
|
||||
cursor.execute("select created_at from repos where id=%s",repo)
|
||||
repo_time = cursor.fetchone()[0]
|
||||
|
||||
ts = {}
|
||||
cursor.execute("select id, doc_type, doc_name from docs where repo_id=%s and doc_type is not null",repo)
|
||||
docs = cursor.fetchall()
|
||||
for doc in docs:
|
||||
doc_id, doc_type, doc_name = doc
|
||||
if doc_type in ts: # 多个同类型的文件，取名字较短的那个
|
||||
if len(doc_name) < ts[doc_type][1]:
|
||||
ts[doc_type] = (doc_id, doc_name)
|
||||
else:
|
||||
ts[doc_type] = (doc_id, doc_name)
|
||||
|
||||
for doc_type, doc_info in ts.items():
|
||||
doc_id, doc_name = doc_info
|
||||
cursor.execute("select act_at from doc_history where doc_id=%s order by act_at asc limit 1",
|
||||
(doc_id,))
|
||||
doc_time = cursor.fetchone()
|
||||
if doc_time is None or doc_time[0] is None:
|
||||
continue
|
||||
doc_time = doc_time[0]
|
||||
if repo_time > doc_time: # repo创建前已就已经有了文档
|
||||
# print repo[0], doc_id,doc_type,repo_time,doc_time
|
||||
early_c[doc_type] += 1
|
||||
creation[doc_type].append(0)
|
||||
else:
|
||||
delay = _dur(repo_time,doc_time)
|
||||
creation[doc_type].append(delay)
|
||||
if delay > 1500:
|
||||
print repo[0], doc_id,doc_type,repo_time,doc_time
|
||||
|
||||
plt.figure(figsize=(6,4),num="correlation")
|
||||
|
||||
data = []
|
||||
xtcks = []
|
||||
for key, value in creation.items():
|
||||
print key
|
||||
print pd.Series(value).describe()
|
||||
data.append(value)
|
||||
xtcks.append(key)
|
||||
print early_c
|
||||
plt.boxplot(x=data,positions=range(1,len(data)+1))
|
||||
plt.xticks(range(1,len(data)+1),xtcks)
|
||||
plt.ylabel("Creation latency (in days)")
|
||||
plt.savefig("../../resources/creation.png",dpi=200)
|
||||
plt.show()
|
||||
plt.close()
|
||||
|
||||
|
|
@ -1,18 +1,19 @@
|
|||
# coding:utf-8
|
||||
# by Star Lee
|
||||
# @2018/12/02
|
||||
# 获取文件的原始数据
|
||||
import urllib2
|
||||
# coding: utf-8
|
||||
import MySQLdb
|
||||
local_db_config = {
|
||||
"db_host" : "localhost",
|
||||
"db_name" : "repo_doc",
|
||||
"db_user" : "root",
|
||||
"db_passwd" : "pdl123456"
|
||||
}
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import urllib2
|
||||
|
||||
|
||||
with open("../.config.json") as fp:
|
||||
config = json.load(fp)
|
||||
local_db_config = config["local_db"]
|
||||
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
||||
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
||||
cursor = conn.cursor()
|
||||
send_headers = {"Content-Type":"application/json","Authorization":None}
|
||||
|
||||
|
||||
|
||||
def _get_url(url,retry_times=3):
|
||||
|
|
@ -29,42 +30,75 @@ def _get_url(url,retry_times=3):
|
|||
|
||||
if error_msg != None:
|
||||
print error_msg
|
||||
time.sleep(3*(4-retry_times))
|
||||
if retry_times > 0:
|
||||
return _get_url(url,retry_times-1)
|
||||
else:
|
||||
return None
|
||||
return ini_html
|
||||
|
||||
|
||||
|
||||
def profile_type(doc):
|
||||
doc_id,repo_id,doc_location,doc_name,doc_type = doc
|
||||
if doc_location.lower() in ["root",".github","docs"] and doc_type=="blob":
|
||||
doc_name = doc_name.lower()
|
||||
for fi in ["readme"]:
|
||||
if doc_name.lower().find(fi) != -1:
|
||||
return "readme"
|
||||
for fi in ["license"]:
|
||||
if doc_name.lower().find(fi) != -1:
|
||||
return "license"
|
||||
for fi in ["issue_template","pull_request_template"]:
|
||||
if doc_name.lower().find(fi) != -1:
|
||||
return "template"
|
||||
for fi in ["code_of_conduct"]:
|
||||
if doc_name.lower().find(fi) != -1:
|
||||
return "COC"
|
||||
for fi in ["contributing"]:
|
||||
if doc_name.lower().find(fi) != -1:
|
||||
return "contributing"
|
||||
return None
|
||||
elif doc_location.lower() in ["issue_template", "pull_request_template"] and doc_type=="blob":
|
||||
return "template"
|
||||
else:
|
||||
return None
|
||||
return None
|
||||
|
||||
import codecs
|
||||
if __name__ == "__main__":
|
||||
# 取出需要下载的项目
|
||||
cursor.execute("select distinct repo_id from rootdir_files")
|
||||
allrepos = set([item[0] for item in cursor.fetchall()])
|
||||
cursor.execute("select distinct repo_id from docs")
|
||||
donerepos = set([item[0] for item in cursor.fetchall()])
|
||||
doingrepos = allrepos.difference(donerepos)
|
||||
|
||||
# 都用大写表示，虽然有些项目用的是小写的。对比时先把文件名都转换成大写
|
||||
doc_types = ["CODE_OF_CONDUCT","README","CONTRIBUTING","LICENSE"]
|
||||
cursor.execute("select id from random_repos")
|
||||
repos = set([item[0] for item in cursor.fetchall()])
|
||||
prf_files = []
|
||||
cursor.execute("select * from files")
|
||||
files = cursor.fetchall()
|
||||
for f in files:
|
||||
doc_id,repo_id,doc_location,doc_name,doc_type = f
|
||||
if repo_id in repos:
|
||||
# 已经采集过了
|
||||
cursor.execute("select file_id from file_content where file_id=%s",(doc_id,))
|
||||
result = cursor.fetchone()
|
||||
if not(result is None or result[0] is None):
|
||||
continue
|
||||
|
||||
for drepo in doingrepos:
|
||||
cursor.execute("select user,repo,branch,id from repos where id=%s",(drepo,))
|
||||
un, rn, bc, repo_id = cursor.fetchone()
|
||||
print repo_id,"https://github.com/%s/%s"%(un,rn)
|
||||
|
||||
cursor.execute("select file from rootdir_files where repo_id=%s",(drepo,))
|
||||
for rootdir_file in cursor.fetchall():
|
||||
rootdir_file = rootdir_file[0]
|
||||
for doc_type in doc_types:
|
||||
if rootdir_file.upper().find(doc_type) != -1:
|
||||
print "\tcrawl",rootdir_file
|
||||
url ='https://raw.githubusercontent.com/%s/%s/%s/%s'%(un,rn,bc,rootdir_file)
|
||||
# 是profile doc
|
||||
if profile_type(f) is not None:
|
||||
cursor.execute("select full_name, default_branch from random_repos where id=%s",(repo_id,))
|
||||
result = cursor.fetchone()
|
||||
repo_name, repo_branch = result
|
||||
print "\tcrawl",repo_name, repo_branch, doc_name,doc_type
|
||||
if doc_location != "root":
|
||||
doc_name = "%s/%s"%(doc_location, doc_name)
|
||||
url ='https://raw.githubusercontent.com/%s/%s/%s'%(repo_name,repo_branch,doc_name)
|
||||
while True:
|
||||
print url
|
||||
file_content = _get_url(url)
|
||||
if file_content is not None:
|
||||
cursor.execute("insert into docs(repo_id,doc_name,doc_content) values(%s,%s,%s)",
|
||||
(drepo,rootdir_file, file_content))
|
||||
cursor.execute("insert into file_content(file_id, content) values(%s,%s)",
|
||||
(doc_id, file_content))
|
||||
conn.commit()
|
||||
break
|
||||
else:
|
||||
cursor.execute("insert into docs(repo_id,doc_name) values(%s,%s)",
|
||||
(drepo,rootdir_file))
|
||||
break
|
||||
conn.commit()
|
||||
pass
|
||||
|
||||
|
|
|
|||
2
idea.md
2
idea.md
|
|
@ -61,7 +61,7 @@ GitHub official suggestions:
|
|||
|
||||
- https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/licensing-a-repository
|
||||
- Most people place their license text in a file named LICENSE.txt (or LICENSE.md) in the root of the repository. Some projects include information about their license in their README. For example, a project's README may include a note saying "This project is licensed under the terms of the MIT license."
|
||||
- 现实情况是，即使使用copying、copyright等关键词GitHub也能关联起来
|
||||
- 现实情况是，即使使用copying、copyright等关键词GitHub也能关联起来; 名字是MIT-licence也能识别出来
|
||||
|
||||
#### *Summary*
|
||||
- location:
|
||||
|
|
|
|||
Loading…
Reference in New Issue