76 lines
2.2 KiB
Python
76 lines
2.2 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import urllib2
|
|
import sys
|
|
import codecs
|
|
import json
|
|
import MySQLdb
|
|
import socket
|
|
socket.setdefaulttimeout(10)
|
|
reload(sys)
|
|
sys.setdefaultencoding('utf-8')
|
|
|
|
#crawl the builds#
|
|
def CrawlBuilds(outfile_path,repo,repo_id,builds_id,cur_delete,cur_miss):
|
|
try:
|
|
url = "https://api.travis-ci.org/repos/%s/builds/%d"%(repo,builds_id)
|
|
builds_content = urllib2.Request(url)
|
|
builds_content.add_header('Accept', "application/vnd.travis-ci.2+json")
|
|
build_content = urllib2.urlopen(builds_content).read()
|
|
output_file = open("%s\\%d\\%d.txt"%(outfile_path,repo_id,builds_id),'wb')
|
|
output_file.write(build_content)
|
|
output_file.close()
|
|
query_delete = "delete from ci_builds_missing where builds_id=%d"%builds_id
|
|
cur_delete.execute(query_delete)
|
|
except Exception as e:
|
|
#insert into the missing table#
|
|
try:
|
|
query_miss = "insert into ci_builds_missing (repo_id,builds_id) values(%s,%s)"%(repo_id,builds_id)
|
|
#print query
|
|
cur_miss.execute(query_miss)
|
|
except MySQLdb.Error, e:
|
|
print "Mysql Error!", e;
|
|
print e
|
|
|
|
|
|
|
|
#main#
|
|
conn = MySQLdb.connect(host='localhost',user='root',passwd='',db='ci')
|
|
cur = conn.cursor()
|
|
conn_delete = MySQLdb.connect(host='localhost',user='root',passwd='',db='ci')
|
|
cur_delete= conn_delete.cursor()
|
|
conn_miss = MySQLdb.connect(host='localhost',user='root',passwd='',db='ci')
|
|
cur_miss= conn_miss.cursor()
|
|
repo_id = 1334
|
|
repo = "rails/rails"
|
|
outfile_path = "E:\\Young\\CI\\result\\builds_yu"
|
|
try:
|
|
cur.execute('select builds_id from ci_builds_jobs_yu where repo_id=%d;'%repo_id)
|
|
data = cur.fetchone()
|
|
while data != None:
|
|
print data[0]
|
|
CrawlBuilds(outfile_path,repo,repo_id,data[0],cur_delete,cur_miss)
|
|
data = cur.fetchone()
|
|
except MySQLdb.Error, e:
|
|
print "Mysql Error!", e;
|
|
#crawl missing builds
|
|
while(1):
|
|
try:
|
|
cur_miss.execute('select builds_id from ci_builds_missing where repo_id=%d;'%repo_id)
|
|
data = cur_miss.fetchone()
|
|
if data is None:
|
|
break
|
|
while data != None:
|
|
print data[0]
|
|
CrawlBuilds(outfile_path,repo,repo_id,data[0],cur_delete,cur_miss)
|
|
data = cur_miss.fetchone()
|
|
except MySQLdb.Error, e:
|
|
print "Mysql Error!", e;
|
|
cur_miss.close()
|
|
conn_miss.close()
|
|
cur_delete.close()
|
|
conn_delete.close()
|
|
cur.close()
|
|
conn.close()
|