80 lines
2.9 KiB
Python
80 lines
2.9 KiB
Python
import urllib2
|
|
import json
|
|
import MySQLdb
|
|
import time
|
|
import logging
|
|
|
|
logger = logging.getLogger()
|
|
hdlr = logging.FileHandler("log")
|
|
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
|
|
hdlr.setFormatter(formatter)
|
|
logger.addHandler(hdlr)
|
|
logger.setLevel(logging.NOTSET)
|
|
|
|
conn = MySQLdb.connect(host="192.168.80.104",user="influx",passwd="influx1234",db="pages",charset='utf8' )
|
|
|
|
cursor = conn.cursor()
|
|
sql_insert = "insert into repo_urls(link,url,full_name,state) values(%s,%s,%s,%s)"
|
|
sql_error = "insert into errors(url,message,time) values(%s,%s,%s)"
|
|
|
|
send_headers = {"Content-Type":"application/json","Authorization":"token e41a69a15db215bd7959320360e77a988c4e19a3"}
|
|
url = "https://api.github.com/repositories?since=444679"
|
|
|
|
while True:
|
|
try:
|
|
req = urllib2.Request(url,headers = send_headers)
|
|
logger.info("process link :%s"%url)
|
|
result = urllib2.urlopen(req,timeout=10)
|
|
prjs = json.loads(result.read())
|
|
logger.info("done downloading and parsing json")
|
|
for prj in prjs:
|
|
try:
|
|
cursor.execute(sql_insert,(url,prj.get("url"),prj.get("full_name"),0))
|
|
except Exception,e:
|
|
logger.error(e.message)
|
|
logger.info("done storing to db")
|
|
conn.commit()
|
|
links = result.info().get("Link")
|
|
if "next" in links:
|
|
url = links[1:links.index(">")]
|
|
logger.info("next url:%s"%url)
|
|
else:
|
|
logger.info("no next link any more")
|
|
num_repos = len(prjs)
|
|
while True:
|
|
time.sleep(10*60)
|
|
logger.info("check it again")
|
|
result = urllib2.urlopen(req)
|
|
prjs = json.loads(result.read())
|
|
new_num_repos = len(prjs)
|
|
if new_num_repos > num_repos:
|
|
for prj in prjs[num_repos:new_num_repos]:
|
|
cursor.execute(sql_insert,(url,prj.get("url"),prj.get("full_name"),0))
|
|
conn.commit()
|
|
num_repos = new_num_repos
|
|
links = result.info().get("Link")
|
|
if "next" in links:
|
|
url = links[0:links.index(">")]
|
|
logger.info("yeah, next link is here!!")
|
|
break
|
|
|
|
except urllib2.URLError,e:
|
|
if hasattr(e,"reason"):
|
|
logger.error("Failed to reach the server")
|
|
logger.error("The reason: "%e.reason)
|
|
cursor.execute(sql_error,(url,e.reason,time.strftime('%Y-%m-%d %H:%M:%S')))
|
|
elif hasattr(e,"code"):
|
|
logger.error("The server couldn't fulfill the request")
|
|
logger.error("Error code: %s"%str(e.code))
|
|
logger.error("Return content: %s"%e.read())
|
|
cursor.execute(sql_error,(url,e.read(),time.strftime('%Y-%m-%d %H:%M:%S')))
|
|
else:
|
|
pass
|
|
conn.commit()
|
|
except Exception,e:
|
|
logger.error(e.message)
|
|
pass
|
|
|
|
cursor.close()
|
|
conn.close()
|