97 lines
2.8 KiB
Python
97 lines
2.8 KiB
Python
#coding:utf-8
|
|
'''
|
|
Created on 2017年06月07日
|
|
从redis队列中领取 pr的url进行下载并存储
|
|
应注意判断下载的url是pr的还是issue的
|
|
@author: StarLee
|
|
'''
|
|
|
|
import sys
|
|
import logging
|
|
import time
|
|
import re
|
|
logger = logging.getLogger()
|
|
hdlr = logging.FileHandler("consumer-%s.log"%sys.argv[1])
|
|
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
|
|
hdlr.setFormatter(formatter)
|
|
logger.addHandler(hdlr)
|
|
logger.setLevel(logging.NOTSET)
|
|
old
|
|
import urllib2
|
|
import redis
|
|
from lxml import etree
|
|
r = redis.Redis(host="127.0.0.1",port=6379,db=0)
|
|
import MySQLdb
|
|
conn = MySQLdb.connect(host="localhost",user="root",passwd="influx1234",db="github_zlb",charset='utf8' )
|
|
cursor = conn.cursor()
|
|
|
|
while True:
|
|
url = r.lpop("pr_diff")
|
|
logger.info( url)
|
|
if url is not None:
|
|
items = url.split("&&")
|
|
project_id = items[0]
|
|
user_name = items[1]
|
|
repo_name = items[2]
|
|
page_url = items[3]
|
|
try:
|
|
# 先下载页面
|
|
req = urllib2.Request(page_url)
|
|
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
|
|
|
pr_lis=etree.HTML(ini_html).xpath('//*/ul[@class="js-navigation-container js-active-navigation-container"]/li')
|
|
logger.info( "len prl: %d"%(len(pr_lis),))
|
|
for pr_li in pr_lis:
|
|
pr_number = pr_li.xpath('div/div[2]/div/span[1]')
|
|
pr_number = pr_number[0].text
|
|
logger.info("pr text : %s"%(pr_number,))
|
|
pr_number = re.findall(r"#(\d+)",pr_number)[0]
|
|
try:
|
|
diff_url = "https://patch-diff.githubusercontent.com/raw/%s/%s/pull/%s.diff"%(user_name,repo_name,pr_number)
|
|
req = urllib2.Request(diff_url)
|
|
logger.info(url)
|
|
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
|
logger.info("done downloading")
|
|
try:
|
|
conn.ping(True)
|
|
except Exception,e:
|
|
logger.error("ping error")
|
|
conn = MySQLdb.connect(host="localhost",user="root",passwd="influx1234",db="github_zlb",charset='utf8' )
|
|
cursor = conn.cursor()
|
|
logger.info("done ping")
|
|
cursor.execute("insert into pr_diff(project_id,pr_id,diff) values(%s,%s,%s)",(project_id,pr_number,ini_html))
|
|
logger.info("done execue")
|
|
conn.commit()
|
|
logger.info("done commit")
|
|
except Exception,e:
|
|
logger.error("pr error")
|
|
logger.error(e)
|
|
logger.error(diff_url)
|
|
r.rpush("diff_url",diff_url)
|
|
try:
|
|
conn.ping(True)
|
|
except Exception,e:
|
|
logger.error("ping error")
|
|
conn = MySQLdb.connect(host="localhost",user="root",passwd="influx1234",db="github_zlb",charset='utf8' )
|
|
cursor = conn.cursor()
|
|
cursor.execute("insert into pr_diff(project_id,pr_id,diff) values(%s,%s,%s)",(project_id,pr_number,""))
|
|
conn.commit()
|
|
|
|
except Exception,e:
|
|
logger.error("page error")
|
|
logger.error(e)
|
|
logger.error(page_url)
|
|
r.rpush("pr_diff",url)
|
|
time.sleep(20)
|
|
cusror = conn.cursor()
|
|
else:
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|