57 lines
1.3 KiB
Python
57 lines
1.3 KiB
Python
#coding:utf-8
|
|
'''
|
|
Created on 2017年06月07日
|
|
从redis队列中领取 pr的url进行下载并存储
|
|
应注意判断下载的url是pr的还是issue的
|
|
@author: StarLee
|
|
'''
|
|
|
|
import sys
|
|
import logging
|
|
import time
|
|
logger = logging.getLogger()
|
|
hdlr = logging.FileHandler("consumer-%s.log"%sys.argv[1])
|
|
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
|
|
hdlr.setFormatter(formatter)
|
|
logger.addHandler(hdlr)
|
|
logger.setLevel(logging.NOTSET)
|
|
|
|
import urllib2
|
|
import redis
|
|
r = redis.Redis(host="127.0.0.1",port=6379,db=0)
|
|
import MySQLdb
|
|
conn = MySQLdb.connect(host="localhost",user="root",passwd="influx1234",db="github_zlb",charset='utf8' )
|
|
cursor = conn.cursor()
|
|
|
|
while True:
|
|
url = r.lpop("pr_diff")
|
|
logger.info( url)
|
|
if url is not None:
|
|
items = url.split("&&")
|
|
project_id = int(items[0])
|
|
pr = int(items[1])
|
|
diff_url = items[2]
|
|
try:
|
|
req = urllib2.Request(diff_url)
|
|
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
|
#判断 是 pr 还是 issue
|
|
if ini_html.startswith("diff"):
|
|
conn.ping(True)
|
|
cursor.execute("insert into pr_diff(project_id,pr_id,diff) values(%s,%s,%s)",(project_id,pr,ini_html))
|
|
conn.commit()
|
|
except Exception,e:
|
|
logger.info(url)
|
|
r.rpush("pr_diff",url)
|
|
time.sleep(20)
|
|
cusror = conn.cursor()
|
|
else:
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|