add collection for questionnaire
This commit is contained in:
parent
80313f546c
commit
77b032190b
|
|
@ -0,0 +1,125 @@
|
|||
# aim: find maintainers for questionnaire
|
||||
# author: zxh
|
||||
# date: 2021-06-02
|
||||
|
||||
import pymysql, yaml, math, datetime, json
|
||||
from utils import *
|
||||
import seaborn as sns
|
||||
import pandas as pd
|
||||
from matplotlib import pyplot as plt
|
||||
import numpy as np
|
||||
import threading, queue # 多线程搜集数据
|
||||
from ghapi.all import GhApi
|
||||
from pymongo import MongoClient
|
||||
|
||||
f = open('config.yaml', 'r')
|
||||
config = yaml.load(f.read(), Loader=yaml.BaseLoader)
|
||||
conn = connectMysqlDB(config, autocommit = True)
|
||||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
# conn_delft = connectMysqlDBSpecifyDB(config, "ghtorrent0619_tudelft", autocommit = True)
|
||||
# cur_delft = conn_delft.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
# Client = MongoClient("mongodb://localhost:27017/")
|
||||
# mongodb = Client['ghtorrent0619']
|
||||
|
||||
# # find all the handled users
|
||||
# cur.execute("select login from github_user_email")
|
||||
# handled_users = cur.fetchall()
|
||||
# handled_users = [user['login'] for user in handled_users]
|
||||
|
||||
# # find all the maintainers
|
||||
# cur.execute("select login from github_sponsor_listing")
|
||||
# maintainers = cur.fetchall()
|
||||
# for maintainer in maintainers:
|
||||
# login = maintainer['login']
|
||||
# if login in handled_users:
|
||||
# continue
|
||||
# print("handle user: %s" % (login))
|
||||
# cur.execute("select name, email from github_user where login=%s", (login,))
|
||||
# info = cur.fetchone()
|
||||
# if info is None or info['email'] is None:
|
||||
# # 缺少当前用户信息
|
||||
# # 1. 查看ghtorrent0619_tudelft数据库中user_private表中是否有对应的信息
|
||||
# cur_delft.execute("select name, email from users_private where login=%s", (login,))
|
||||
# private_info = cur_delft.fetchone()
|
||||
# if private_info is None or private_info['email'] is None or len(private_info['email'].strip()) == 0:
|
||||
# # 表示private表中也没有相关信息
|
||||
# # 查看commits表中是否有相关commit
|
||||
# cur.execute("insert into github_user_email (login, name, email, type) values (%s, %s, %s, %s)", (login, private_info['name'], private_info['email'], "transfer"))
|
||||
# else:
|
||||
# cur.execute("insert into github_user_email (login, name, email, type) values (%s, %s, %s, %s)", (login, private_info['name'], private_info['email'], "transfer"))
|
||||
# else:
|
||||
# cur.execute("insert into github_user_email (login, name, email, type) values (%s, %s, %s, %s)", (login, info['name'], info['email'], "transfer"))
|
||||
|
||||
|
||||
# print("finish transfer email and name")
|
||||
|
||||
|
||||
# f = open('github_tokens.txt', 'r')
|
||||
# github_tokens = f.read().strip().split("\n")
|
||||
|
||||
# def findUserByCommit_online(ownername, reponame, sha):
|
||||
# # Github Enterprise with custom hostname
|
||||
# count = 0
|
||||
# while(True):
|
||||
# try:
|
||||
# count += 1
|
||||
# c = api.git.get_commit(ownername, reponame, sha, token="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")
|
||||
# author_name = c.author.name
|
||||
# author_email = c.author.email
|
||||
# return (author_name, author_email)
|
||||
# except Exception as e:
|
||||
# if e.code == 404:
|
||||
# return False
|
||||
# elif e.code == 403:
|
||||
# if count > 100:
|
||||
# return False
|
||||
# continue
|
||||
# else:
|
||||
# print("pause")
|
||||
# return False
|
||||
|
||||
|
||||
# # 0. update email and name for each user
|
||||
# cur.execute("select id, login, name from github_user_email where email is null or email='' and type !='mongodb'")
|
||||
# maintainers = cur.fetchall()
|
||||
# api = GhApi()
|
||||
# for maintainer in maintainers:
|
||||
# cur_delft.execute("select c.sha, u2.login as ownername, p.name as reponame from commits c, projects p, users u, users u2 where u.id=c.author_id and p.id=c.project_id and p.owner_id=u2.id and u2.login=%s and p.deleted=0 and p.forked_from is null limit 10", (maintainer['login'],))
|
||||
# commits = cur_delft.fetchall()
|
||||
# for commit in commits:
|
||||
# sha = commit['sha']
|
||||
# ownername = commit['ownername']
|
||||
# reponame = commit['reponame']
|
||||
|
||||
# cur.execute("select name, email from github_user_email where login=%s and type='mongodb'", (maintainer['login'],))
|
||||
# mongodb_r = cur.fetchone()
|
||||
# if mongodb_r is not None:
|
||||
# if maintainer['name'] is None or maintainer['name'].strip() == "":
|
||||
# cur.execute("update github_user_email set name=%s, email=%s, type='crawl' where id=%s", (mongodb_r['name'], mongodb_r['email'], maintainer['id']))
|
||||
# else:
|
||||
# cur.execute("update github_user_email set email=%s, type='crawl' where id=%s", (mongodb_r['email'], maintainer['id']))
|
||||
# break
|
||||
# else:
|
||||
# r = findUserByCommit_online(ownername, reponame, sha)
|
||||
# if r == False:
|
||||
# break
|
||||
# else:
|
||||
# if maintainer['name'] is None or maintainer['name'].strip() == "":
|
||||
# cur.execute("update github_user_email set name=%s, email=%s, type='crawl' where id=%s", (r[0], r[1], maintainer['id']))
|
||||
# else:
|
||||
# cur.execute("update github_user_email set email=%s, type='crawl' where id=%s", (r[1], maintainer['id']))
|
||||
# break
|
||||
|
||||
|
||||
cur.execute("select login from github_sponsor_listing where deleted=0")
|
||||
all_sponsor_account_users = cur.fetchall()
|
||||
all_sponsor_account_users = [user["login"] for user in all_sponsor_account_users]
|
||||
|
||||
import random
|
||||
# 1. random 20% for first round
|
||||
random.seed( 10 )
|
||||
first_round = random.sample(all_sponsor_account_users, int(len(all_sponsor_account_users) * 0.1))
|
||||
# 2. the rest 80% for second round
|
||||
second_round = list(set(all_sponsor_account_users) - set(first_round))
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
# aim: find users(maintainer, sponsor, others)
|
||||
# author: zxh
|
||||
# date: 2021-06-02
|
||||
|
||||
import pymysql, yaml, math, datetime, json
|
||||
from utils import *
|
||||
import seaborn as sns
|
||||
import pandas as pd
|
||||
from matplotlib import pyplot as plt
|
||||
import numpy as np
|
||||
import threading, queue # 多线程搜集数据
|
||||
from ghapi.all import GhApi
|
||||
from pymongo import MongoClient
|
||||
|
||||
f = open('config.yaml', 'r')
|
||||
config = yaml.load(f.read(), Loader=yaml.BaseLoader)
|
||||
conn = connectMysqlDB(config, autocommit = False)
|
||||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
import random
|
||||
random.seed( 10 )
|
||||
|
||||
# find maintainers
|
||||
# 要先删除组织用户
|
||||
cur.execute("select gsl.login, gu.name, gu.email from github_sponsor_listing gsl, github_user gu where gsl.deleted=0 and gsl.login=gu.login and gu.flag=0")
|
||||
items = cur.fetchall()
|
||||
query_dict = {}
|
||||
all_sponsor_account_users = []
|
||||
for item in items:
|
||||
query_dict[item['login']] = {
|
||||
"name": item['name'],
|
||||
"email": item['email']
|
||||
}
|
||||
all_sponsor_account_users.append(item['login'])
|
||||
|
||||
# 1. random 20% for first round
|
||||
first_round = random.sample(all_sponsor_account_users, int(len(all_sponsor_account_users) * 0.2))
|
||||
# 2. the rest 80% for second round
|
||||
second_round = list(set(all_sponsor_account_users) - set(first_round))
|
||||
|
||||
for user in first_round:
|
||||
if user in query_dict:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "maintainer", 1, query_dict[user]['name'], query_dict[user]['email']))
|
||||
else:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "maintainer", 1))
|
||||
|
||||
for user in second_round:
|
||||
if user in query_dict:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "maintainer", 2, query_dict[user]['name'], query_dict[user]['email']))
|
||||
else:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "maintainer", 2))
|
||||
conn.commit()
|
||||
|
||||
# find sponsors
|
||||
cur.execute("select gsam.sponsor_login, gu.name, gu.email from github_sponsorships_as_maintainer gsam, github_user gu where gsam.flag=0 and gsam.sponsor_login=gu.login and gu.flag=0")
|
||||
items = cur.fetchall()
|
||||
query_dict = {}
|
||||
all_sponsors = []
|
||||
for item in items:
|
||||
query_dict[item['sponsor_login']] = {
|
||||
"name": item['name'],
|
||||
"email": item['email']
|
||||
}
|
||||
all_sponsors.append(item['sponsor_login'])
|
||||
all_sponsors = list(set(all_sponsors) - set(all_sponsor_account_users))
|
||||
|
||||
# 1. random 20% for first round
|
||||
first_round = random.sample(all_sponsors, int(len(all_sponsors) * 0.2))
|
||||
# 2. the rest 80% for second round
|
||||
second_round = list(set(all_sponsors) - set(first_round))
|
||||
|
||||
for user in first_round:
|
||||
if user in query_dict:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "sponsors", 1, query_dict[user]['name'], query_dict[user]['email']))
|
||||
else:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "sponsors", 1))
|
||||
|
||||
for user in second_round:
|
||||
if user in query_dict:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "sponsors", 2, query_dict[user]['name'], query_dict[user]['email']))
|
||||
else:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "sponsors", 2))
|
||||
conn.commit()
|
||||
|
||||
|
||||
# find others
|
||||
cur.execute("select md.login, up.name, up.email from middle_data_no_sponsor_account_users md, users_private up where md.login=up.login")
|
||||
items = cur.fetchall()
|
||||
query_dict = {}
|
||||
all_others = []
|
||||
for item in items:
|
||||
query_dict[item['login']] = {
|
||||
"name": item['name'],
|
||||
"email": item['email']
|
||||
}
|
||||
all_others.append(item['login'])
|
||||
all_others = list(set(all_others) - set(all_sponsor_account_users) - set(all_sponsors))
|
||||
|
||||
# first random 10k users
|
||||
all_others = random.sample(all_others, 10000)
|
||||
|
||||
# 1. random 20% for first round
|
||||
first_round = random.sample(all_others, int(len(all_others) * 0.2))
|
||||
# 2. the rest 80% for second round
|
||||
second_round = list(set(all_others) - set(first_round))
|
||||
|
||||
for user in first_round:
|
||||
if user in query_dict:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "others", 1, query_dict[user]['name'], query_dict[user]['email']))
|
||||
else:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "others", 1))
|
||||
|
||||
for user in second_round:
|
||||
if user in query_dict:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "others", 2, query_dict[user]['name'], query_dict[user]['email']))
|
||||
else:
|
||||
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "others", 2))
|
||||
conn.commit()
|
||||
|
|
@ -13,150 +13,212 @@ import threading, queue # 多线程搜集数据
|
|||
|
||||
f = open('config.yaml', 'r')
|
||||
config = yaml.load(f.read(), Loader=yaml.BaseLoader)
|
||||
conn = connectMysqlDB(config, autocommit = True)
|
||||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||||
conn_delft = connectMysqlDBSpecifyDB(config, "ghtorrent0619_tudelft", autocommit = True)
|
||||
cur_delft = conn_delft.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
conn_sponsor = connectMysqlDBSpecifyDB(config, "github-sponsors", autocommit = True)
|
||||
cur_sponsor = conn_sponsor.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
allowed_countries = [
|
||||
"Australia",
|
||||
"Austria",
|
||||
"Belgium",
|
||||
"Bulgaria",
|
||||
"Canada",
|
||||
"Cyprus",
|
||||
"Czech Republic",
|
||||
"Denmark",
|
||||
"Estonia",
|
||||
"Finland",
|
||||
"France",
|
||||
"Germany",
|
||||
"Greece",
|
||||
"Hong Kong",
|
||||
"Ireland",
|
||||
"Italy",
|
||||
"Denmark",
|
||||
"Japan",
|
||||
"Latvia",
|
||||
"Lithuania",
|
||||
"Luxembourg",
|
||||
"Malta",
|
||||
"Mexico",
|
||||
"Netherlands",
|
||||
"New Zealand",
|
||||
"Norway",
|
||||
"Poland",
|
||||
"Portugal",
|
||||
"Romania",
|
||||
"Singapore",
|
||||
"Slovakia",
|
||||
"Slovenia",
|
||||
"Spain",
|
||||
"Sweden",
|
||||
"Switzerland",
|
||||
"United Kingdom",
|
||||
"United States of America"
|
||||
"au",
|
||||
"at",
|
||||
"be",
|
||||
"bg",
|
||||
"ca",
|
||||
"cy",
|
||||
"cz",
|
||||
"dk",
|
||||
"ee",
|
||||
"fi",
|
||||
"fr",
|
||||
"de",
|
||||
"gr",
|
||||
"hk",
|
||||
"ie",
|
||||
"it",
|
||||
"jp",
|
||||
"lv",
|
||||
"lt",
|
||||
"lu",
|
||||
"mt",
|
||||
"mx",
|
||||
"nl",
|
||||
"nz",
|
||||
"no",
|
||||
"pl",
|
||||
"pt",
|
||||
"ro",
|
||||
"sg",
|
||||
"sk",
|
||||
"si",
|
||||
"es",
|
||||
"se",
|
||||
"ch",
|
||||
"uk",
|
||||
"us"
|
||||
]
|
||||
allowed_countries = [c.lower() for c in allowed_countries]
|
||||
|
||||
result = []
|
||||
# find User(not organization) and have owned repositories with more than 1k stars and the location (country) information is well defined
|
||||
cur.execute("select u.id, u.login, u.country, sum(w.user_id) as star_num from users u, watchers w, projects p where u.type='User' and u.location is not null and u.id=p.user_id and w.repo_id=p.id and w.user_id!=u.id group by u.id having sum(w.user_id)>1000")
|
||||
users = cur.fetchall()
|
||||
for user in users:
|
||||
if user['country'].lower() not in allowed_countries:
|
||||
continue
|
||||
else:
|
||||
result.append({
|
||||
"id": user['id'],
|
||||
"login": user['login'],
|
||||
"country": user['country'],
|
||||
"star_num": user['star_num']
|
||||
})
|
||||
|
||||
# find whether users have participants in sponsor mechanism
|
||||
for user in result:
|
||||
# 1. whether exists in github_sponsor_listing table
|
||||
cur.execute("select * from github_sponsor_listing where login=%s", (user['login'],))
|
||||
if cur.fetchone() is not None:
|
||||
continue # already had sponsor account
|
||||
else:
|
||||
cur_delft.execute("select u.id, u.login, u.country_code from users u where u.type='USR' and u.fake=0 and u.deleted=0 and u.country_code is not null")
|
||||
users = cur_delft.fetchall()
|
||||
print("finish reading all users")
|
||||
|
||||
cur_sponsor.execute("select user_id from middle_data_no_sponsor_account_users")
|
||||
handled_users = cur_sponsor.fetchall()
|
||||
handled_users = [user['user_id'] for user in handled_users]
|
||||
print("finish reading handled users")
|
||||
|
||||
# find User(not organization) and have owned repositories with more than 1k stars and the location (country) information is well defined
|
||||
class FindUser(threading.Thread):
|
||||
def __init__(self, q):
|
||||
threading.Thread.__init__(self)
|
||||
self.q = q
|
||||
self.conn_delft = connectMysqlDBSpecifyDB(config, "ghtorrent0619_tudelft", autocommit = True)
|
||||
self.cur_delft = self.conn_delft.cursor(pymysql.cursors.DictCursor)
|
||||
self.conn_sponsor = connectMysqlDBSpecifyDB(config, "github-sponsors", autocommit = True)
|
||||
self.cur_sponsor = self.conn_sponsor.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
|
||||
def run(self):
|
||||
while(True):
|
||||
try:
|
||||
task = self.q.get(timeout=0)
|
||||
print("loop how many threads left: %d" % (self.q.qsize()))
|
||||
|
||||
user_id = task['id']
|
||||
login = task['login']
|
||||
country_code = task['country_code']
|
||||
print("handle user: %s" % (login))
|
||||
|
||||
if country_code in allowed_countries:
|
||||
sum_star_num = 0
|
||||
self.cur_delft.execute("select id from projects where owner_id=%s and deleted=0", (user_id,))
|
||||
projects = self.cur_delft.fetchall()
|
||||
if projects is not None:
|
||||
for project in projects:
|
||||
self.cur_delft.execute("select count(*) as star_num from watchers where repo_id=%s and user_id!=%s", (project['id'], user_id))
|
||||
sum_star_num += self.cur_delft.fetchone()['star_num']
|
||||
|
||||
self.cur_sponsor.execute("insert into middle_data_no_sponsor_account_users (user_id, login, country_code, star_num) values (%s, %s, %s, %s)", (user_id, login, country_code, sum_star_num))
|
||||
except queue.Empty:
|
||||
sys.exit(-1)
|
||||
return
|
||||
self.q.task_done()
|
||||
|
||||
THREADNUM = 50
|
||||
tasks = queue.Queue()
|
||||
for user in users:
|
||||
if user['id'] in handled_users:
|
||||
continue # already handled
|
||||
tasks.put(user)
|
||||
for _ in range(THREADNUM):
|
||||
t = FindUser(tasks)
|
||||
t.start()
|
||||
tasks.join()
|
||||
|
||||
print("finish")
|
||||
|
||||
|
||||
|
||||
# for user in users:
|
||||
# if user['country_code'].lower() not in allowed_countries:
|
||||
# continue
|
||||
# else:
|
||||
# result.append({
|
||||
# "id": user['id'],
|
||||
# "login": user['login'],
|
||||
# "country_code": user['country_code'],
|
||||
# "star_num": user['star_num']
|
||||
# })
|
||||
|
||||
# # find whether users have participants in sponsor mechanism
|
||||
# for user in result:
|
||||
# # 1. whether exists in github_sponsor_listing table
|
||||
# cur_sponsor.execute("select * from github_sponsor_listing where login=%s", (user['login'],))
|
||||
# if cur_sponsor.fetchone() is not None:
|
||||
# continue # already had sponsor account
|
||||
# else:
|
||||
# pass
|
||||
|
||||
|
||||
|
||||
|
||||
class whetherUserSponsorMechanism(threading.Thread):
|
||||
def __init__(self, q, threadNo):
|
||||
threading.Thread.__init__(self)
|
||||
self.q = q
|
||||
self.threadNo = threadNo
|
||||
self.thread_interval_count = 0
|
||||
# class whetherUserSponsorMechanism(threading.Thread):
|
||||
# def __init__(self, q, threadNo):
|
||||
# threading.Thread.__init__(self)
|
||||
# self.q = q
|
||||
# self.threadNo = threadNo
|
||||
# self.thread_interval_count = 0
|
||||
|
||||
def run(self):
|
||||
global mutex, github_tokens, sleep_gap_token, sleep_time_tokens, list_users_tasks_table_name
|
||||
while not self.q.empty():
|
||||
work = self.q.get(timeout=0)
|
||||
logging.info("Thread No: %s (%s), the number of work in queue: %s" % (self.threadNo, self.thread_interval_count, str(self.q.qsize())))
|
||||
ownername = work[0]
|
||||
reponame = work[1]
|
||||
# def run(self):
|
||||
# global mutex, github_tokens, sleep_gap_token, sleep_time_tokens, list_users_tasks_table_name
|
||||
# while not self.q.empty():
|
||||
# work = self.q.get(timeout=0)
|
||||
# logging.info("Thread No: %s (%s), the number of work in queue: %s" % (self.threadNo, self.thread_interval_count, str(self.q.qsize())))
|
||||
# ownername = work[0]
|
||||
# reponame = work[1]
|
||||
|
||||
# get a suitable token and combine header
|
||||
github_token = base.get_token(github_tokens, sleep_time_tokens, sleep_gap_token)
|
||||
headers = {
|
||||
'Authorization': 'Bearer ' + github_token,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
# # get a suitable token and combine header
|
||||
# github_token = base.get_token(github_tokens, sleep_time_tokens, sleep_gap_token)
|
||||
# headers = {
|
||||
# 'Authorization': 'Bearer ' + github_token,
|
||||
# 'Content-Type': 'application/json'
|
||||
# }
|
||||
|
||||
results = []
|
||||
has_next = True
|
||||
end_cursor = ""
|
||||
# results = []
|
||||
# has_next = True
|
||||
# end_cursor = ""
|
||||
|
||||
while has_next:
|
||||
values = {"query": query % (ownername, reponame, end_cursor), "variables": {}}
|
||||
try:
|
||||
# request data and parse response
|
||||
response = requests.post(url=url, headers = headers, json=values, timeout = 40)
|
||||
response.encoding = 'utf-8'
|
||||
if response.status_code != 200:
|
||||
logging.error('threadNo %s: status code %s, url: %s' % (self.threadNo, response.status_code, url))
|
||||
mutex.acquire()
|
||||
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
|
||||
mutex.release()
|
||||
continue
|
||||
response_json = response.json()
|
||||
if "errors" in response_json:
|
||||
logging.error('threadNo %s: status code %s, url: %s, errors: %s' % (self.threadNo, response.status_code, url, json.dumps(response_json)))
|
||||
if response_json["errors"][0]["type"] == "RATE_LIMITED":
|
||||
mutex.acquire()
|
||||
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
|
||||
mutex.release()
|
||||
continue
|
||||
elif response_json["errors"][0]["type"] == "NOT_FOUND":
|
||||
with open('sponsor_create_repo_star_results/' + ownername + "_-_" + reponame, "w") as f:
|
||||
f.write("not found")
|
||||
else:
|
||||
logging.error("unknown error, don't handle it!!!")
|
||||
mutex.acquire()
|
||||
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
|
||||
mutex.release()
|
||||
continue
|
||||
else:
|
||||
results.append(response_json)
|
||||
# whether there exists next page
|
||||
if response_json["data"]["repository"]["stargazers"]["pageInfo"]["hasNextPage"] == False:
|
||||
with open('sponsor_create_repo_star_results/' + ownername + "_-_" + reponame, "w") as f:
|
||||
json.dump(results, f)
|
||||
else:
|
||||
end_cursor = ', after:"' + response_json["data"]["repository"]["stargazers"]["pageInfo"]["endCursor"] + '"'
|
||||
continue
|
||||
# update task queue
|
||||
self.q.task_done()
|
||||
self.thread_interval_count += 1
|
||||
break
|
||||
except Exception as e:
|
||||
logging.error('threadNo %s: error - %s' % (self.threadNo, str(e)))
|
||||
traceback.print_exc()
|
||||
mutex.acquire()
|
||||
sleep_time_tokens[github_token] = time.time()
|
||||
mutex.release()
|
||||
time.sleep(3)
|
||||
# while has_next:
|
||||
# values = {"query": query % (ownername, reponame, end_cursor), "variables": {}}
|
||||
# try:
|
||||
# # request data and parse response
|
||||
# response = requests.post(url=url, headers = headers, json=values, timeout = 40)
|
||||
# response.encoding = 'utf-8'
|
||||
# if response.status_code != 200:
|
||||
# logging.error('threadNo %s: status code %s, url: %s' % (self.threadNo, response.status_code, url))
|
||||
# mutex.acquire()
|
||||
# sleep_time_tokens[github_token] = time.time() # set sleep time for that token
|
||||
# mutex.release()
|
||||
# continue
|
||||
# response_json = response.json()
|
||||
# if "errors" in response_json:
|
||||
# logging.error('threadNo %s: status code %s, url: %s, errors: %s' % (self.threadNo, response.status_code, url, json.dumps(response_json)))
|
||||
# if response_json["errors"][0]["type"] == "RATE_LIMITED":
|
||||
# mutex.acquire()
|
||||
# sleep_time_tokens[github_token] = time.time() # set sleep time for that token
|
||||
# mutex.release()
|
||||
# continue
|
||||
# elif response_json["errors"][0]["type"] == "NOT_FOUND":
|
||||
# with open('sponsor_create_repo_star_results/' + ownername + "_-_" + reponame, "w") as f:
|
||||
# f.write("not found")
|
||||
# else:
|
||||
# logging.error("unknown error, don't handle it!!!")
|
||||
# mutex.acquire()
|
||||
# sleep_time_tokens[github_token] = time.time() # set sleep time for that token
|
||||
# mutex.release()
|
||||
# continue
|
||||
# else:
|
||||
# results.append(response_json)
|
||||
# # whether there exists next page
|
||||
# if response_json["data"]["repository"]["stargazers"]["pageInfo"]["hasNextPage"] == False:
|
||||
# with open('sponsor_create_repo_star_results/' + ownername + "_-_" + reponame, "w") as f:
|
||||
# json.dump(results, f)
|
||||
# else:
|
||||
# end_cursor = ', after:"' + response_json["data"]["repository"]["stargazers"]["pageInfo"]["endCursor"] + '"'
|
||||
# continue
|
||||
# # update task queue
|
||||
# self.q.task_done()
|
||||
# self.thread_interval_count += 1
|
||||
# break
|
||||
# except Exception as e:
|
||||
# logging.error('threadNo %s: error - %s' % (self.threadNo, str(e)))
|
||||
# traceback.print_exc()
|
||||
# mutex.acquire()
|
||||
# sleep_time_tokens[github_token] = time.time()
|
||||
# mutex.release()
|
||||
# time.sleep(3)
|
||||
|
||||
# input results into database
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
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
|
||||
14
utils.py
14
utils.py
|
|
@ -43,5 +43,19 @@ def connectMysqlDB(config, autocommit = True):
|
|||
use_unicode=True,
|
||||
charset='utf8mb4',
|
||||
|
||||
autocommit=autocommit)
|
||||
return db
|
||||
|
||||
def connectMysqlDBSpecifyDB(config, db, autocommit = True):
|
||||
|
||||
db = pymysql.connect(host='localhost',
|
||||
user=config['mysql']['user'],
|
||||
passwd=config['mysql']['passwd'],
|
||||
db=db,
|
||||
|
||||
local_infile=1,
|
||||
use_unicode=True,
|
||||
charset='utf8mb4',
|
||||
|
||||
autocommit=autocommit)
|
||||
return db
|
||||
Loading…
Reference in New Issue