Nigel
/
cscw_2021_sponsor
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

modify user collector for second round and crawler of stars

This commit is contained in:
zhangxunhui 2021-06-29 02:25:32 -04:00
parent b15da08480
commit c15d65893e
3 changed files with 165 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
collect_data_questionnaire_star.py
View File

@ -33,7 +33,6 @@ max_commit_time_pr = {} # record the max created_at time for each pr
mutex = threading.Lock() # used for lock the queue
url = "https://api.github.com/graphql"
since_str = "2021-05-03T00:00:00Z"
class findUserStars(threading.Thread):
@ -46,18 +45,23 @@ class findUserStars(threading.Thread):
self.thread_interval_count = 0
def run(self):
while(True):
while not self.q.empty():
try:
login = self.q.get(timeout=0)
print("loop how many threads left: %d" % (self.q.qsize()))
query_user_activity = """
query_user_stars = """
query {
user(login:"%s") {
login
updatedAt
topRepositories(since: "%s", orderBy: {field: STARGAZERS, direction: DESC}) {
totalCount
repositories(first:100%s){
pageInfo {
endCursor
hasNextPage
}
nodes{
nameWithOwner
stargazerCount
}
}
}
}
@ -66,47 +70,59 @@ class findUserStars(threading.Thread):
# request data and parse response
github_token = base.get_token(github_tokens, sleep_time_tokens, sleep_gap_token)
headers = {
'Authorization': 'Bearer ' + github_token,
'Content-Type': 'application/json'
}
values = {"query": query_user_activity % (login, since_str), "variables": {}}
response = requests.post(url=url, headers = headers, json=values, timeout = 40)
response.encoding = 'utf-8'
if response.status_code != 200:
logging.error('login %s: status code %s, url: %s' % (login, response.status_code, url))
mutex.acquire()
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
mutex.release()
continue
response_json = response.json()
if "errors" in response_json:
logging.error('login %s: status code %s, url: %s, errors: %s' % (login, response.status_code, url, json.dumps(response_json)))
if response_json["errors"][0]["type"] == "RATE_LIMITED":
'Authorization': 'Bearer ' + github_token,
'Content-Type': 'application/json'
}
results = []
has_next = True
end_cursor = ""
while has_next:
values = {"query": query_user_stars % (login, end_cursor), "variables": {}}
response = requests.post(url=url, headers = headers, json=values, timeout = 40)
response.encoding = 'utf-8'
if response.status_code != 200:
logging.error('login %s: status code %s, url: %s' % (login, response.status_code, url))
mutex.acquire()
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
mutex.release()
continue
elif response_json["errors"][0]["type"] == "NOT_FOUND":
# user not found
self.cur.execute("insert into middle_data_recent_events (login, deleted) values (%s, %s)", (login, 1))
response_json = response.json()
if "errors" in response_json:
logging.error('login %s: status code %s, url: %s, errors: %s' % (login, response.status_code, url, json.dumps(response_json)))
if response_json["errors"][0]["type"] == "RATE_LIMITED":
mutex.acquire()
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
mutex.release()
continue
elif response_json["errors"][0]["type"] == "NOT_FOUND":
# user not found
self.cur.execute("insert into middle_data_recent_stars (login, deleted) values (%s, %s)", (login, 1))
else:
logging.error("unknown error, don't handle it!!!")
mutex.acquire()
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
mutex.release()
continue
else:
logging.error("unknown error, don't handle it!!!")
mutex.acquire()
sleep_time_tokens[github_token] = time.time() # set sleep time for that token
mutex.release()
continue
else:
# find the related repositories
user = response_json['data']['user']
updated_at = datetime.datetime.strptime(user['updatedAt'], "%Y-%m-%dT%H:%M:%S%z")
top_repo_count = int(user['topRepositories']['totalCount'])
since_at = datetime.datetime.strptime(since_str, "%Y-%m-%dT%H:%M:%S%z")
self.cur.execute("insert into middle_data_recent_events (login, since, top_repo_count, updated_at, deleted) values (%s, %s, %s, %s, %s)", (login, since_at, top_repo_count, updated_at, 0))
# update task queue
self.q.task_done()
self.thread_interval_count += 1
except queue.Empty:
return
results.append(response_json)
# whether there exists next page
if response_json["data"]["user"]["repositories"]["pageInfo"]["hasNextPage"] == False:
total_own_repo_stars = 0
for r in results:
nodes = r["data"]["user"]["repositories"]["nodes"]
for node in nodes:
total_own_repo_stars += node['stargazerCount']
self.cur.execute("insert into middle_data_recent_stars (login, total_own_repo_stars, created_at, deleted) values (%s, %s, %s, %s)", (login, total_own_repo_stars, datetime.datetime.now(), 0))
else:
end_cursor = ', after:"' + response_json["data"]["user"]["repositories"]["pageInfo"]["endCursor"] + '"'
continue
# update task queue
self.q.task_done()
self.thread_interval_count += 1
break
except Exception as e:
logging.error('error - %s' % (str(e)))
traceback.print_exc()
@ -116,23 +132,19 @@ class findUserStars(threading.Thread):
time.sleep(3)
cur.execute('select login from github_user where flag=0')
all_users = cur.fetchall()
all_users = [user['login'] for user in all_users]
cur.execute("select md.login, mdre.name, mdre.email from middle_data_no_sponsor_account_users md, middle_data_recent_emails mdre where md.login=mdre.login and mdre.deleted=0 and mdre.name is not null and mdre.name != '' and mdre.email not like '%noreply%'")
items = cur.fetchall()
all_users = []
for item in items:
all_users.append(item['login'])
cur.execute("select mdn.login, up.name, up.email from middle_data_no_sponsor_account_users mdn, users_private up where mdn.login = up.login and up.email is not null and up.email != '' and up.name is not null and up.name != ''")
users = cur.fetchall()
users = [user['login'] for user in users]
all_users = list(set(all_users) | set(users))
cur.execute("select login from middle_data_recent_events")
cur.execute("select login from middle_data_recent_stars")
handled_users = cur.fetchall()
handled_users = [user['login'] for user in handled_users]
remained_users = list(set(all_users) - set(handled_users))
THREADNUM = 1
THREADNUM = 50
tasks = queue.Queue()
for user in remained_users:
# for user in ['Bradsif']:

38
collect_data_questionnaire_user_second_round.py
View File

@ -33,9 +33,10 @@ for item in items:
all_sponsor_account_users.append(item['login'])
all_sponsor_account_users = list(set(all_sponsor_account_users))
all_sponsor_account_users_random = random.sample(all_sponsor_account_users, 2000) # firstly random 2000
# 1. random 20% for first round
first_round = random.sample(all_sponsor_account_users_random, int(len(all_sponsor_account_users_random) * 0.2))
# read first round user logins
cur.execute("select login from middle_data_questionnaire_users_firstround where type='maintainer' and which_round=1")
first_round = cur.fetchall()
first_round = [u['login'] for u in first_round]
# 2. the rest 80% for second round
second_round = list(set(all_sponsor_account_users) - set(first_round))
@ -65,9 +66,10 @@ for item in items:
all_sponsors.append(item['sponsor_login'])
all_sponsors = list(set(all_sponsors) - set(all_sponsor_account_users))
# 1. random 20% for first round
all_sponsors_random = random.sample(all_sponsors, 2000)
first_round = random.sample(all_sponsors_random, int(len(all_sponsors_random) * 0.2))
# read first round user logins
cur.execute("select login from middle_data_questionnaire_users_firstround where type='sponsors' and which_round=1")
first_round = cur.fetchall()
first_round = [u['login'] for u in first_round]
# 2. the rest
second_round = list(set(all_sponsors) - set(first_round))
@ -86,7 +88,7 @@ conn.commit()
# find others
cur.execute("select md.login, mdre.name, mdre.email from middle_data_no_sponsor_account_users md, middle_data_recent_emails mdre where md.login=mdre.login and mdre.deleted=0 and mdre.name is not null and mdre.name != '' and mdre.email not like '%noreply%'")
cur.execute("select md.login, mdre.name, mdre.email from middle_data_no_sponsor_account_users md, middle_data_recent_emails mdre, middle_data_recent_stars mdrs where md.login=mdre.login and mdre.login=mdrs.login and mdre.deleted=0 and mdre.name is not null and mdre.name != '' and mdre.email not like '%noreply%' and mdrs.total_own_repo_stars>=10 and mdrs.deleted=0")
items = cur.fetchall()
query_dict = {}
all_others = []
@ -98,13 +100,14 @@ for item in items:
all_others.append(item['login'])
all_others = list(set(all_others) - set(all_sponsor_account_users) - set(all_sponsors))
# first random 2000 users
all_others_random = random.sample(all_others, 2000)
# 1. random 20% for first round
first_round = random.sample(all_others_random, int(len(all_others_random) * 0.2))
# 2. the rest 80% for second round
second_round = list(set(all_others) - set(first_round))
# read first round user logins
cur.execute("select login from middle_data_questionnaire_users_firstround where type='others' and which_round=1")
first_round = cur.fetchall()
first_round = [u['login'] for u in first_round]
# 2. random 7500
second_round = random.sample(list(set(all_others) - set(first_round)), 7500)
# 3. random 7500 for the third time
third_round = random.sample(list((set(all_others) - set(first_round)) - set(second_round)), 7500)
for user in first_round:
if user in query_dict:
@ -117,4 +120,11 @@ for user in second_round:
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "others", 2, query_dict[user]['name'], query_dict[user]['email']))
else:
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "others", 2))
for user in third_round:
if user in query_dict:
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round, name, email) values (%s, %s, %s, %s, %s)", (user, "others", 3, query_dict[user]['name'], query_dict[user]['email']))
else:
cur.execute("insert into middle_data_questionnaire_users (login, type, which_round) values (%s, %s, %s)", (user, "others", 3))
conn.commit()

75
collect_data_surveymonkey_user_second_round.py Normal file
View File

@ -0,0 +1,75 @@
# aim: collect user information to csv file for survey monkey
# author: zhangxunhui
# date: 2021-06-07
import pymysql, yaml, math, datetime, json
from utils import *
import seaborn as sns
import pandas as pd
from matplotlib import pyplot as plt
import numpy as np
import threading, queue # 多线程搜集数据
from ghapi.all import GhApi
from pymongo import MongoClient
f = open('config.yaml', 'r')
config = yaml.load(f.read(), Loader=yaml.BaseLoader)
conn = connectMysqlDB(config, autocommit = False)
cur = conn.cursor(pymysql.cursors.DictCursor)
# first round (read)
# for maintainers
first_maintainer_df = pd.read_csv('survey_monkey/maintainer_1.csv', index_col=False)
first_maintainers = first_maintainer_df['Email'].values.tolist()
# for sponsors
first_sponsor_df = pd.read_csv('survey_monkey/sponsor_1.csv', index_col=False)
first_sponsors = first_sponsor_df['Email'].values.tolist()
# for others
first_other_df = pd.read_csv('survey_monkey/other_1.csv', index_col=False)
first_others = first_other_df['Email'].values.tolist()
# second round
# for maintainers
cur.execute("select email from middle_data_questionnaire_users where which_round=2 and type='maintainer'")
items = cur.fetchall()
df = pd.DataFrame(items)
# 判断第一轮是否出现了某些用户
print(set(df['email'].values.tolist()) & set(first_maintainers))
df.to_csv('survey_monkey/maintainer_2.csv', encoding='utf-8')
# for sponsors
cur.execute("select email from middle_data_questionnaire_users where which_round=2 and type='sponsors'")
items = cur.fetchall()
df = pd.DataFrame(items)
# 判断第一轮是否出现了某些用户
print(set(df['email'].values.tolist()) & set(first_sponsors))
df.to_csv('survey_monkey/sponsor_2.csv', encoding='utf-8')
# for others
cur.execute("select email from middle_data_questionnaire_users where which_round=2 and type='others'")
items = cur.fetchall()
df = pd.DataFrame(items)
# 判断第一轮是否出现了某些用户
print(set(df['email'].values.tolist()) & set(first_others))
df.to_csv('survey_monkey/other_2.csv', encoding='utf-8')
# for others (third round)
cur.execute("select email from middle_data_questionnaire_users where which_round=3 and type='others'")
items = cur.fetchall()
df = pd.DataFrame(items)
# 判断第一轮是否出现了某些用户
print(set(df['email'].values.tolist()) & set(first_others))
df.to_csv('survey_monkey/other_3.csv', encoding='utf-8')