From e27d4f891f9e0d8aec5bacd6620ad4b8c91829f2 Mon Sep 17 00:00:00 2001 From: whystar Date: Mon, 27 Jul 2020 11:28:58 +0800 Subject: [PATCH] get file content --- experiment_code/2-maintenance/creation.py | 91 +++++++++++++++++ experiment_code/dataset/get_file_content.py | 108 +++++++++++++------- idea.md | 2 +- 3 files changed, 163 insertions(+), 38 deletions(-) diff --git a/experiment_code/2-maintenance/creation.py b/experiment_code/2-maintenance/creation.py index e69de29..3aec2ec 100644 --- a/experiment_code/2-maintenance/creation.py +++ b/experiment_code/2-maintenance/creation.py @@ -0,0 +1,91 @@ + +# coding: utf-8 +import MySQLdb +import json +import time +import random +import urllib2 +import pandas as pd +import matplotlib.mlab as mlab +import matplotlib.pyplot as plt + + +with open("../.config.json") as fp: + config = json.load(fp) +local_db_config = config["local_db"] +conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"], + passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4') +cursor = conn.cursor() + +import collections +creation = collections.OrderedDict() +creation["RDM"]=[] +creation["COD"]=[] +creation["CON"]=[] +creation["LIC"]=[] + +def _dur(t1, t2): + t1 = time.mktime(time.strptime(t1, '%Y-%m-%dT%H:%M:%SZ')) + t2 = time.mktime(time.strptime(t2, '%Y-%m-%dT%H:%M:%SZ')) + # return math.log((t2-t1)*1.0/60/60+1) + return (t2-t1)*1.0/60/60/24 + + +early_c = collections.OrderedDict() +early_c["RDM"]=0 +early_c["COD"]=0 +early_c["CON"]=0 +early_c["LIC"]=0 + +cursor.execute("select id from repos") +repos = cursor.fetchall() +for repo in repos: + cursor.execute("select created_at from repos where id=%s",repo) + repo_time = cursor.fetchone()[0] + + ts = {} + cursor.execute("select id, doc_type, doc_name from docs where repo_id=%s and doc_type is not null",repo) + docs = cursor.fetchall() + for doc in docs: + doc_id, doc_type, doc_name = doc + if doc_type in ts: # 多个同类型的文件，取名字较短的那个 + if len(doc_name) < ts[doc_type][1]: + ts[doc_type] = (doc_id, doc_name) + else: + ts[doc_type] = (doc_id, doc_name) + + for doc_type, doc_info in ts.items(): + doc_id, doc_name = doc_info + cursor.execute("select act_at from doc_history where doc_id=%s order by act_at asc limit 1", + (doc_id,)) + doc_time = cursor.fetchone() + if doc_time is None or doc_time[0] is None: + continue + doc_time = doc_time[0] + if repo_time > doc_time: # repo创建前已就已经有了文档 + # print repo[0], doc_id,doc_type,repo_time,doc_time + early_c[doc_type] += 1 + creation[doc_type].append(0) + else: + delay = _dur(repo_time,doc_time) + creation[doc_type].append(delay) + if delay > 1500: + print repo[0], doc_id,doc_type,repo_time,doc_time + +plt.figure(figsize=(6,4),num="correlation") + +data = [] +xtcks = [] +for key, value in creation.items(): + print key + print pd.Series(value).describe() + data.append(value) + xtcks.append(key) +print early_c +plt.boxplot(x=data,positions=range(1,len(data)+1)) +plt.xticks(range(1,len(data)+1),xtcks) +plt.ylabel("Creation latency (in days)") +plt.savefig("../../resources/creation.png",dpi=200) +plt.show() +plt.close() + \ No newline at end of file diff --git a/experiment_code/dataset/get_file_content.py b/experiment_code/dataset/get_file_content.py index 74d79bf..7d2d433 100644 --- a/experiment_code/dataset/get_file_content.py +++ b/experiment_code/dataset/get_file_content.py @@ -1,18 +1,19 @@ -# coding:utf-8 -# by Star Lee -# @2018/12/02 -# 获取文件的原始数据 -import urllib2 +# coding: utf-8 import MySQLdb -local_db_config = { - "db_host" : "localhost", - "db_name" : "repo_doc", - "db_user" : "root", - "db_passwd" : "pdl123456" - } +import json +import time +import random +import urllib2 + + +with open("../.config.json") as fp: + config = json.load(fp) +local_db_config = config["local_db"] conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"], passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4') cursor = conn.cursor() +send_headers = {"Content-Type":"application/json","Authorization":None} + def _get_url(url,retry_times=3): @@ -29,42 +30,75 @@ def _get_url(url,retry_times=3): if error_msg != None: print error_msg + time.sleep(3*(4-retry_times)) if retry_times > 0: return _get_url(url,retry_times-1) else: return None return ini_html + + +def profile_type(doc): + doc_id,repo_id,doc_location,doc_name,doc_type = doc + if doc_location.lower() in ["root",".github","docs"] and doc_type=="blob": + doc_name = doc_name.lower() + for fi in ["readme"]: + if doc_name.lower().find(fi) != -1: + return "readme" + for fi in ["license"]: + if doc_name.lower().find(fi) != -1: + return "license" + for fi in ["issue_template","pull_request_template"]: + if doc_name.lower().find(fi) != -1: + return "template" + for fi in ["code_of_conduct"]: + if doc_name.lower().find(fi) != -1: + return "COC" + for fi in ["contributing"]: + if doc_name.lower().find(fi) != -1: + return "contributing" + return None + elif doc_location.lower() in ["issue_template", "pull_request_template"] and doc_type=="blob": + return "template" + else: + return None + return None + import codecs if __name__ == "__main__": - # 取出需要下载的项目 - cursor.execute("select distinct repo_id from rootdir_files") - allrepos = set([item[0] for item in cursor.fetchall()]) - cursor.execute("select distinct repo_id from docs") - donerepos = set([item[0] for item in cursor.fetchall()]) - doingrepos = allrepos.difference(donerepos) - # 都用大写表示，虽然有些项目用的是小写的。对比时先把文件名都转换成大写 - doc_types = ["CODE_OF_CONDUCT","README","CONTRIBUTING","LICENSE"] + cursor.execute("select id from random_repos") + repos = set([item[0] for item in cursor.fetchall()]) + prf_files = [] + cursor.execute("select * from files") + files = cursor.fetchall() + for f in files: + doc_id,repo_id,doc_location,doc_name,doc_type = f + if repo_id in repos: + # 已经采集过了 + cursor.execute("select file_id from file_content where file_id=%s",(doc_id,)) + result = cursor.fetchone() + if not(result is None or result[0] is None): + continue - for drepo in doingrepos: - cursor.execute("select user,repo,branch,id from repos where id=%s",(drepo,)) - un, rn, bc, repo_id = cursor.fetchone() - print repo_id,"https://github.com/%s/%s"%(un,rn) - - cursor.execute("select file from rootdir_files where repo_id=%s",(drepo,)) - for rootdir_file in cursor.fetchall(): - rootdir_file = rootdir_file[0] - for doc_type in doc_types: - if rootdir_file.upper().find(doc_type) != -1: - print "\tcrawl",rootdir_file - url ='https://raw.githubusercontent.com/%s/%s/%s/%s'%(un,rn,bc,rootdir_file) + # 是profile doc + if profile_type(f) is not None: + cursor.execute("select full_name, default_branch from random_repos where id=%s",(repo_id,)) + result = cursor.fetchone() + repo_name, repo_branch = result + print "\tcrawl",repo_name, repo_branch, doc_name,doc_type + if doc_location != "root": + doc_name = "%s/%s"%(doc_location, doc_name) + url ='https://raw.githubusercontent.com/%s/%s/%s'%(repo_name,repo_branch,doc_name) + while True: + print url file_content = _get_url(url) if file_content is not None: - cursor.execute("insert into docs(repo_id,doc_name,doc_content) values(%s,%s,%s)", - (drepo,rootdir_file, file_content)) + cursor.execute("insert into file_content(file_id, content) values(%s,%s)", + (doc_id, file_content)) + conn.commit() + break else: - cursor.execute("insert into docs(repo_id,doc_name) values(%s,%s)", - (drepo,rootdir_file)) - break - conn.commit() + pass + diff --git a/idea.md b/idea.md index 6468906..e681e86 100644 --- a/idea.md +++ b/idea.md @@ -61,7 +61,7 @@ GitHub official suggestions: - https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/licensing-a-repository - Most people place their license text in a file named LICENSE.txt (or LICENSE.md) in the root of the repository. Some projects include information about their license in their README. For example, a project's README may include a note saying "This project is licensed under the terms of the MIT license." -- 现实情况是，即使使用copying、copyright等关键词GitHub也能关联起来 +- 现实情况是，即使使用copying、copyright等关键词GitHub也能关联起来; 名字是MIT-licence也能识别出来 #### *Summary* - location: