35 lines
1.0 KiB
Python
35 lines
1.0 KiB
Python
# coding:utf-8
|
|
# by Star Lee
|
|
# @2018/12/02
|
|
# 获取文件的原始数据
|
|
import urllib2
|
|
import MySQLdb
|
|
local_db_config = {
|
|
"db_host" : "localhost",
|
|
"db_name" : "duppr",
|
|
"db_user" : "root",
|
|
"db_passwd" : "pdl123456"
|
|
}
|
|
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
|
|
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
|
|
cursor = conn.cursor()
|
|
|
|
import codecs
|
|
if __name__ == "__main__":
|
|
cursor.execute("select user_name, repo_name from project")
|
|
prjs = cursor.fetchall()
|
|
file_names = ["CODE_OF_CONDUCT","README","CONTRIBUTING"]
|
|
for prj in prjs[1:]:
|
|
un, rn = prj
|
|
print prj
|
|
for file_name in file_names:
|
|
url ='https://raw.githubusercontent.com/%s/%s/master/%s.md'%(un,rn,file_name)
|
|
try:
|
|
req = urllib2.Request(url)
|
|
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
|
with codecs.open("rawFiles/%s-%s-%s"%(un,rn,file_name),"w","utf-8") as fp:
|
|
fp.write(ini_html)
|
|
except Exception,e:
|
|
print e
|
|
print "pass",url
|