init
This commit is contained in:
commit
526de98e44
|
|
@ -0,0 +1 @@
|
|||
experiment_code/rawFiles/*
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# coding:utf-8
|
||||
# by Star Lee
|
||||
# @2018/12/02
|
||||
# 获取文件的原始数据
|
||||
import urllib2
|
||||
import MySQLdb
|
||||
local_db_config = {
|
||||
"db_host" : "localhost",
|
||||
"db_name" : "duppr",
|
||||
"db_user" : "root",
|
||||
"db_passwd" : "pdl123456"
|
||||
}
|
||||
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
||||
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
||||
cursor = conn.cursor()
|
||||
|
||||
import codecs
|
||||
if __name__ == "__main__":
|
||||
cursor.execute("select user_name, repo_name from project")
|
||||
prjs = cursor.fetchall()
|
||||
file_names = ["CODE_OF_CONDUCT","README","CONTRIBUTING"]
|
||||
for prj in prjs[1:]:
|
||||
un, rn = prj
|
||||
print prj
|
||||
for file_name in file_names:
|
||||
url ='https://raw.githubusercontent.com/%s/%s/master/%s.md'%(un,rn,file_name)
|
||||
try:
|
||||
req = urllib2.Request(url)
|
||||
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
||||
with codecs.open("rawFiles/%s-%s-%s"%(un,rn,file_name),"w","utf-8") as fp:
|
||||
fp.write(ini_html)
|
||||
except Exception,e:
|
||||
print e
|
||||
print "pass",url
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
# coding:utf-8
|
||||
# by Star Lee
|
||||
# @2018/12/02
|
||||
# 获取github平台上forks靠前的项目的基本数据信息
|
||||
import urllib2
|
||||
import MySQLdb
|
||||
import time
|
||||
from lxml import etree
|
||||
|
||||
local_db_config = {
|
||||
"db_host" : "localhost",
|
||||
"db_name" : "repo_doc",
|
||||
"db_user" : "root",
|
||||
"db_passwd" : "pdl123456"
|
||||
}
|
||||
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
||||
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
||||
cursor = conn.cursor()
|
||||
|
||||
def get_top_repo_list():
|
||||
fork_limit = 2500
|
||||
# 选出fork数大于某值的所有repo，并按照fork数排序
|
||||
search_url = "https://github.com/search?o=desc&p=%d&q=forks%%3A%%3E%d&s=forks&type=Repositories"
|
||||
|
||||
page_i = 1
|
||||
while True:
|
||||
tmp_url = search_url%(page_i,fork_limit)
|
||||
print tmp_url
|
||||
req = urllib2.Request(tmp_url)
|
||||
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
||||
lis=etree.HTML(ini_html).xpath('//*/ul[@class="repo-list"]/li/div[1]/h3/a/text()')
|
||||
if len(lis) == 0:
|
||||
print "no more repos"
|
||||
break
|
||||
for li in lis:
|
||||
if li is not None:
|
||||
un, rn = li.split("/")
|
||||
cursor.execute("select id from repos where user=%s and repo=%s",
|
||||
(un,rn))
|
||||
if cursor.fetchone() is None:
|
||||
cursor.execute("insert into repos(user,repo) values(%s,%s)",
|
||||
(un,rn))
|
||||
conn.commit()
|
||||
time.sleep(10)
|
||||
page_i += 1
|
||||
|
||||
|
||||
|
||||
def _uni_field(field):
|
||||
# contributor,release,commit,branch 存在单复数区别
|
||||
if not field.endswith("s"):
|
||||
if field.endswith("h"):
|
||||
return field + "es"
|
||||
else:
|
||||
return field + "s"
|
||||
else:
|
||||
return field
|
||||
def _extract_html(ini_html):
|
||||
nums = {}
|
||||
lis = etree.HTML(ini_html).xpath('//*/ul[@class="pagehead-actions"]/li')
|
||||
for li in lis:
|
||||
try:
|
||||
tmp_text = li.xpath("./a[1]/text()")[-1].strip()
|
||||
tmp_num = li.xpath("./a[2]/text()")[0].strip()
|
||||
nums[tmp_text] = tmp_num.replace(",", "")
|
||||
except Exception,e:
|
||||
pass
|
||||
|
||||
#contributor,release,commit,branch
|
||||
lis = etree.HTML(ini_html).xpath('//*/ul[@class="numbers-summary"]/li/a')
|
||||
for lia in lis:
|
||||
try:
|
||||
tmp_txt = _uni_field(lia.xpath("./text()")[-1].strip())
|
||||
if tmp_txt not in ["commit","commits","branch","branches","release","releases",
|
||||
"contributor","contributors"]:
|
||||
continue
|
||||
tmp_num = lia.xpath("./span")[0].text.strip()
|
||||
nums[tmp_txt] = tmp_num.replace(",","")
|
||||
try: #有些不是数字，用无穷大表示
|
||||
tmp_int = int(nums[tmp_txt])
|
||||
except Exception,e:
|
||||
del nums[tmp_txt]
|
||||
|
||||
except Exception,e:
|
||||
print e.message
|
||||
print e
|
||||
|
||||
return nums
|
||||
|
||||
def _dir_files(ini_html):
|
||||
files = []
|
||||
lis = etree.HTML(ini_html).xpath('//*/table[@class="files js-navigation-container js-active-navigation-container"]/tbody/tr')
|
||||
if len(lis) < 1:
|
||||
print "no tr"
|
||||
return []
|
||||
|
||||
for li in lis[1:]:
|
||||
try:
|
||||
tmp_text = li.xpath("./td[2]/span/a/text()")[0].strip()
|
||||
files.append(tmp_text)
|
||||
except Exception,e:
|
||||
print e.message
|
||||
print e
|
||||
pass
|
||||
return files
|
||||
|
||||
def fetchHtmlInfo(prj):
|
||||
url = "https://github.com/%s"%prj
|
||||
req = urllib2.Request(url)
|
||||
try:
|
||||
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
||||
except Exception,e:
|
||||
print "download exception"
|
||||
print e.message
|
||||
print e
|
||||
return {},{}
|
||||
nums= _extract_html(ini_html)
|
||||
files = _dir_files(ini_html)
|
||||
return nums,files
|
||||
|
||||
|
||||
def get_repo_info():
|
||||
cursor.execute("select id,user,repo from repos where watch is null")
|
||||
repos = cursor.fetchall()
|
||||
for repo in repos:
|
||||
print repo
|
||||
nums,files = fetchHtmlInfo("%s/%s"%repo[1:])
|
||||
if len(nums) != 0:
|
||||
update_sql = ""
|
||||
for item in nums.items():
|
||||
update_sql += "%s=%s,"%item
|
||||
update_sql = update_sql[:-1]
|
||||
print update_sql
|
||||
cursor.execute("update repos set " + update_sql + " where id=%s"%repo[0])
|
||||
if len(files) != 0:
|
||||
for file in files:
|
||||
cursor.execute("insert into rootdir_files(repo_id,file) values(%s,%s)",
|
||||
(repo[0],file))
|
||||
conn.commit()
|
||||
time.sleep(8)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# get_top_repo_list()
|
||||
get_repo_info()
|
||||
Loading…
Reference in New Issue