63 lines
1.8 KiB
Python
63 lines
1.8 KiB
Python
#coding:utf-8
|
||
'''
|
||
Created on 2017年10月24日
|
||
下载pr的home html，从中抽取出pr之间的链接，尤其是别的项目的链接
|
||
@author: StarLee
|
||
'''
|
||
import MySQLdb
|
||
conn = MySQLdb.connect(host="localhost",user="starlee",passwd="1234",db="duppr",charset='utf8' )
|
||
import urllib2
|
||
from lxml import etree
|
||
|
||
|
||
def main(prj_id):
|
||
cursor = conn.cursor()
|
||
|
||
# 确认上次爬到哪里了
|
||
# cursor.execute("select max(pr_number) from pr_home_page_ref where prj_id= %s",(prj_id,))
|
||
# pr_pos = cursor.fetchone()
|
||
|
||
cursor.execute("select pos from record where prj_id= %s",(prj_id,))
|
||
pr_pos = cursor.fetchone()
|
||
|
||
if pr_pos[0] is None:
|
||
pr_pos = 0
|
||
cursor.execute("insert into record(prj_id,pos) values(%s,%s)",(prj_id,0))
|
||
conn.commit()
|
||
else:
|
||
pr_pos = pr_pos[0]
|
||
|
||
|
||
#确定要爬取的pr列表
|
||
cursor.execute("select pr_number from prs where prj_id = %s and pr_number > %s",(prj_id,pr_pos))
|
||
prs = cursor.fetchall()
|
||
|
||
# 开始爬
|
||
for pr in prs:
|
||
print pr
|
||
pr_url = "https://github.com/%s/pull/%d"%(prj_name[prj_id],pr[0])
|
||
req = urllib2.Request(pr_url)
|
||
try:
|
||
ini_html = urllib2.urlopen(req,timeout=40).read().lower().decode("utf8")
|
||
ch_txt = ini_html.strip()
|
||
except Exception,e:
|
||
|
||
continue
|
||
|
||
|
||
# 抽取外链接
|
||
# //*h4[@class="discussion-item-ref-title"]/a
|
||
out_refs = etree.HTML(ini_html).xpath('//*/h4[@class="discussion-item-ref-title"]/a')
|
||
for out_ref in out_refs:
|
||
out_ref = out_ref.get("href")
|
||
print out_ref
|
||
if not out_ref.startswith("/%s"%prj_name[prj_id]):
|
||
cursor.execute("insert into pr_home_page_ref(prj_id,pr_number,out_ref) values(%s,%s,%s)",
|
||
(prj_id,pr[0], out_ref))
|
||
cursor.execute("update record set pos =%s where prj_id =%s",(pr[0],prj_id))
|
||
conn.commit()
|
||
|
||
if __name__ == '__main__':
|
||
prj_name = {1334:"rails/rails"}
|
||
main(1334)
|