116 lines
3.2 KiB
Python
116 lines
3.2 KiB
Python
# coding: utf-8
|
|
import MySQLdb
|
|
import json
|
|
import time
|
|
import random
|
|
import urllib2
|
|
|
|
|
|
with open("../.config.json") as fp:
|
|
config = json.load(fp)
|
|
local_db_config = config["local_db"]
|
|
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
|
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
|
cursor = conn.cursor()
|
|
send_headers = {"Content-Type":"application/json","Authorization":None}
|
|
|
|
|
|
|
|
def _get_url(url,retry_times=3):
|
|
req = urllib2.Request(url)
|
|
try:
|
|
error_msg = None
|
|
ini_html = urllib2.urlopen(req,timeout=30).read().decode('utf-8')
|
|
except urllib2.HTTPError, e:
|
|
error_msg = e.code
|
|
except urllib2.URLError, e:
|
|
error_msg = e.reason
|
|
except Exception,e:
|
|
error_msg = e.message
|
|
|
|
if error_msg != None:
|
|
print error_msg
|
|
time.sleep(3*(4-retry_times))
|
|
if retry_times > 0:
|
|
return _get_url(url,retry_times-1)
|
|
else:
|
|
return None
|
|
return ini_html
|
|
|
|
|
|
|
|
def profile_type(doc):
|
|
doc_id,repo_id,doc_location,doc_name,doc_type = doc
|
|
if doc_location.lower() in ["root",".github","docs"] and doc_type=="blob":
|
|
doc_name = doc_name.lower()
|
|
for fi in ["readme"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "readme"
|
|
for fi in ["license"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "license"
|
|
for fi in ["issue_template","pull_request_template"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "template"
|
|
for fi in ["code_of_conduct"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "COC"
|
|
for fi in ["contributing"]:
|
|
if doc_name.lower().find(fi) != -1:
|
|
return "contributing"
|
|
return None
|
|
elif doc_location.lower() in ["issue_template", "pull_request_template"] and doc_type=="blob":
|
|
return "template"
|
|
else:
|
|
return None
|
|
return None
|
|
|
|
import codecs
|
|
if __name__ == "__main__":
|
|
|
|
cursor.execute("select id from random_repos")
|
|
repos = set([item[0] for item in cursor.fetchall()])
|
|
prf_files = []
|
|
cursor.execute("select * from files")
|
|
files = cursor.fetchall()
|
|
for f in files:
|
|
doc_id,repo_id,doc_location,doc_name,doc_type = f
|
|
if repo_id in repos:
|
|
|
|
# 已经采集过了
|
|
cursor.execute("select file_id from file_content where file_id=%s",(doc_id,))
|
|
result = cursor.fetchone()
|
|
if not(result is None or result[0] is None):
|
|
continue
|
|
|
|
# 是profile doc
|
|
if profile_type(f) is not None:
|
|
print f
|
|
# 如果是特殊文件的话
|
|
if doc_name.endswith(".pdf"):
|
|
print "pdf file"
|
|
continue
|
|
|
|
cursor.execute("select full_name, default_branch from random_repos where id=%s",(repo_id,))
|
|
result = cursor.fetchone()
|
|
repo_name, repo_branch = result
|
|
print "\tcrawl",repo_name, repo_branch, doc_name,doc_type
|
|
if doc_location != "root":
|
|
if doc_location.lower().find("template") != -1:
|
|
doc_name = ".github/%s/%s"%(doc_location, doc_name)
|
|
else:
|
|
doc_name = "%s/%s"%(doc_location, doc_name)
|
|
|
|
url ='https://raw.githubusercontent.com/%s/%s/%s'%(repo_name,repo_branch,doc_name)
|
|
while True:
|
|
print url
|
|
file_content = _get_url(url)
|
|
if file_content is not None:
|
|
cursor.execute("insert into file_content(file_id, content) values(%s,%s)",
|
|
(doc_id, file_content))
|
|
conn.commit()
|
|
break
|
|
else:
|
|
pass
|
|
|