change time gap and change data collection file
This commit is contained in:
parent
1c750fd3b4
commit
c3ef772b66
|
|
@ -22,4 +22,7 @@ m = lmer(
|
|||
data=data
|
||||
)
|
||||
|
||||
car::Anova(m, type=II)
|
||||
car::vif(m)
|
||||
anova(m)
|
||||
|
||||
car::Anova(m, type=II)
|
||||
|
|
|
|||
489
collect_data.py
489
collect_data.py
|
|
@ -6,19 +6,22 @@ import seaborn as sns
|
|||
import pandas as pd
|
||||
from matplotlib import pyplot as plt
|
||||
import numpy as np
|
||||
import threading, queue # 多线程搜集数据
|
||||
|
||||
f = open('config.yaml', 'r')
|
||||
config = yaml.load(f.read(), Loader=yaml.BaseLoader)
|
||||
conn = connectMysqlDB(config, autocommit = True)
|
||||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
n = 6 * 30 # 多少天
|
||||
unstable_n = 15 # 前后多少天是不稳定时间
|
||||
step = 30 # 一个gap多少天
|
||||
interval_n = 6 # 前后区间数量
|
||||
|
||||
# read logins and sponsor created at
|
||||
selected_users_df = pd.read_csv("selected_sponsor_create_logins.csv",header=0)
|
||||
|
||||
n = 12*14 # 多少天
|
||||
unstable_n = 7 # 前后多少天是不稳定时间
|
||||
|
||||
# 形成所有开发者的前后12个gap，以及中间的一个gap
|
||||
# 形成所有开发者的前后6个gap，以及中间的一个gap
|
||||
gap_indexes = []
|
||||
gap_starts = []
|
||||
gap_ends = []
|
||||
|
|
@ -28,14 +31,14 @@ for row in selected_users_df.itertuples():
|
|||
sponsor_created_at = getattr(row, 'sponsor_created_at')
|
||||
sponsor_created_at = datetime.datetime.fromisoformat(sponsor_created_at)
|
||||
|
||||
for gap_index in range(-12, 13, 1):
|
||||
for gap_index in range(-6, 7, 1):
|
||||
if gap_index < 0:
|
||||
start_at = sponsor_created_at - datetime.timedelta(days=unstable_n) + gap_index * datetime.timedelta(days=n/12)
|
||||
end_at = start_at + datetime.timedelta(days=n/12)
|
||||
start_at = sponsor_created_at - datetime.timedelta(days=unstable_n) + gap_index * datetime.timedelta(days=n/interval_n)
|
||||
end_at = start_at + datetime.timedelta(days=n/interval_n)
|
||||
|
||||
elif gap_index > 0:
|
||||
start_at = sponsor_created_at + datetime.timedelta(days=unstable_n) + gap_index * datetime.timedelta(days=n/12)
|
||||
end_at = start_at + datetime.timedelta(days=n/12)
|
||||
start_at = sponsor_created_at + datetime.timedelta(days=unstable_n) + gap_index * datetime.timedelta(days=n/interval_n)
|
||||
end_at = start_at + datetime.timedelta(days=n/interval_n)
|
||||
|
||||
else:
|
||||
start_at = sponsor_created_at - datetime.timedelta(days=unstable_n)
|
||||
|
|
@ -50,7 +53,8 @@ gap_df = pd.DataFrame.from_dict({"index": gap_indexes, "start_at": gap_starts, "
|
|||
# 读取用户不同index下的所有属性
|
||||
# 1. user age in months; 2. history commits; 3. history prs; 4. history issues; 5. history issue_comments; 6. history pr_reviews
|
||||
# 7. time in days from the start of observation period; 8. interventionTRUE; 9. time_after_intervention
|
||||
# 10. commit_num in this interval; 11. pr_num in this interval; 12. issue_num in this interval; 13. issue_comment_num in this interval; 14. pr_review_num in this interval
|
||||
# 10. commit_num in this interval; 11. pr_num in this interval; 12. issue_num in this interval; 13. issue_comment_num in this interval; 14. pr_review_num in this interval;
|
||||
# 15. history sponsored users; 16. average tier amount
|
||||
|
||||
# store results
|
||||
gap_df['user_age'] = -1
|
||||
|
|
@ -67,115 +71,368 @@ gap_df['pr_num'] = -1
|
|||
gap_df['issue_num'] = -1
|
||||
gap_df['issue_comment_num'] = -1
|
||||
gap_df['pr_review_num'] = -1
|
||||
gap_df['history_sponsored_num'] = -1
|
||||
gap_df['avg_tier'] = -1
|
||||
|
||||
|
||||
min_start_at = min(gap_df['start_at'])
|
||||
|
||||
# 形成task, 多线程运行
|
||||
class myThread_sponsor_create(threading.Thread):
|
||||
def __init__(self, q):
|
||||
threading.Thread.__init__(self)
|
||||
self.q = q
|
||||
self.conn = connectMysqlDB(config, autocommit = True)
|
||||
self.cur = self.conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
def run(self):
|
||||
while(True):
|
||||
try:
|
||||
row = self.q.get(timeout=0)
|
||||
print("loop how many threads left: %d" % (self.q.qsize()))
|
||||
login = getattr(row, 'login')
|
||||
# 1.
|
||||
self.cur.execute("select created_at from github_user where login=%s", (login,))
|
||||
created_at = self.cur.fetchone()['created_at']
|
||||
start_at = getattr(row, 'start_at')
|
||||
user_age = int((start_at - created_at).days / 30)
|
||||
|
||||
# 2.
|
||||
self.cur.execute("select sum(contribution_count) as history_commit_num from github_user_commits_per_day where date<%s and login=%s", (start_at,login))
|
||||
history_commit_num = self.cur.fetchone()['history_commit_num']
|
||||
if history_commit_num is None:
|
||||
history_commit_num = 0
|
||||
else:
|
||||
history_commit_num = int(history_commit_num)
|
||||
|
||||
# 3.
|
||||
self.cur.execute("select count(*) as history_pr_num from github_user_pr where created_at<%s and login=%s", (start_at, login))
|
||||
history_pr_num = self.cur.fetchone()['history_pr_num']
|
||||
|
||||
# 4.
|
||||
self.cur.execute("select count(*) as history_issue_num from github_user_issue where created_at<%s and login=%s", (start_at, login))
|
||||
history_issue_num = self.cur.fetchone()['history_issue_num']
|
||||
|
||||
# 5.
|
||||
self.cur.execute("select count(*) as history_issue_comment_num from github_issue_comment where created_at<%s and login=%s", (start_at, login))
|
||||
history_issue_comment_num = self.cur.fetchone()['history_issue_comment_num']
|
||||
|
||||
# 6.
|
||||
self.cur.execute("select count(*) as count_1 from github_pr_comment where created_at<%s and login=%s", (start_at, login))
|
||||
count_1 = self.cur.fetchone()['count_1']
|
||||
self.cur.execute("select count(*) as count_2 from github_commit_comment where created_at<%s and login=%s", (start_at, login))
|
||||
count_2 = self.cur.fetchone()['count_2']
|
||||
self.cur.execute("select count(*) as count_3 from github_user_pr_review where created_at<%s and login=%s", (start_at, login))
|
||||
count_3 = self.cur.fetchone()['count_3']
|
||||
history_pr_review_num = count_1 + count_2 + count_3
|
||||
|
||||
# 7.
|
||||
index = getattr(row, 'index')
|
||||
time = int(index + n/step)
|
||||
|
||||
# 8.
|
||||
if index > 0:
|
||||
intervention = 1
|
||||
elif index < 0:
|
||||
intervention = 0
|
||||
else:
|
||||
intervention = -1
|
||||
|
||||
# 9.
|
||||
if index > 0:
|
||||
time_after_intervention = time - int(n/step)
|
||||
elif index < 0:
|
||||
time_after_intervention = 0
|
||||
else:
|
||||
time_after_intervention = -1
|
||||
|
||||
# 10.
|
||||
end_at = getattr(row, 'end_at')
|
||||
self.cur.execute("select sum(contribution_count) as commit_num from github_user_commits_per_day where date>%s and date<=%s and login=%s", (start_at, end_at, login))
|
||||
commit_num = self.cur.fetchone()['commit_num']
|
||||
if commit_num is None:
|
||||
commit_num = 0
|
||||
else:
|
||||
commit_num = int(commit_num)
|
||||
|
||||
# 11.
|
||||
self.cur.execute("select count(*) as pr_num from github_user_pr where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
pr_num = self.cur.fetchone()['pr_num']
|
||||
|
||||
# 12.
|
||||
self.cur.execute("select count(*) as issue_num from github_user_issue where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
issue_num = self.cur.fetchone()['issue_num']
|
||||
|
||||
# 13.
|
||||
self.cur.execute("select count(*) as issue_comment_num from github_issue_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
issue_comment_num = self.cur.fetchone()['issue_comment_num']
|
||||
|
||||
# 14.
|
||||
self.cur.execute("select count(*) as num from github_pr_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_1 = self.cur.fetchone()['num']
|
||||
self.cur.execute("select count(*) as num from github_commit_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_2 = self.cur.fetchone()['num']
|
||||
self.cur.execute("select count(*) as num from github_user_pr_review where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_3 = self.cur.fetchone()['num']
|
||||
pr_review_num = count_1 + count_2 + count_3
|
||||
|
||||
|
||||
# 15.
|
||||
self.cur.execute("select count(*) as num from github_sponsorships_as_maintainer where login=%s and created_at<%s", (login, start_at))
|
||||
history_sponsored_num = self.cur.fetchone()['num']
|
||||
|
||||
# 16.
|
||||
self.cur.execute("select avg(monthly_price_in_dollars) as avg_tier from github_sponsor_listing_tiers where login=%s", (login,))
|
||||
avg_tier = self.cur.fetchone()['avg_tier']
|
||||
if avg_tier is not None:
|
||||
avg_tier = float(avg_tier)
|
||||
else:
|
||||
avg_tier = float(-2)
|
||||
|
||||
|
||||
# 赋值
|
||||
gap_df.loc[row.Index, 'user_age'] = user_age
|
||||
gap_df.loc[row.Index, 'history_commit_num'] = history_commit_num
|
||||
gap_df.loc[row.Index, 'history_pr_num'] = history_pr_num
|
||||
gap_df.loc[row.Index, 'history_issue_num'] = history_issue_num
|
||||
gap_df.loc[row.Index, 'history_issue_comment_num'] = history_issue_comment_num
|
||||
gap_df.loc[row.Index, 'history_pr_review_num'] = history_pr_review_num
|
||||
gap_df.loc[row.Index, 'time'] = time
|
||||
gap_df.loc[row.Index, 'intervention'] = intervention
|
||||
gap_df.loc[row.Index, 'time_after_intervention'] = time_after_intervention
|
||||
gap_df.loc[row.Index, 'commit_num'] = commit_num
|
||||
gap_df.loc[row.Index, 'pr_num'] = pr_num
|
||||
gap_df.loc[row.Index, 'issue_num'] = issue_num
|
||||
gap_df.loc[row.Index, 'issue_comment_num'] = issue_comment_num
|
||||
gap_df.loc[row.Index, 'pr_review_num'] = pr_review_num
|
||||
gap_df.loc[row.Index, 'history_sponsored_num'] = history_sponsored_num
|
||||
gap_df.loc[row.Index, 'avg_tier'] = avg_tier
|
||||
|
||||
except queue.Empty:
|
||||
return
|
||||
self.q.task_done()
|
||||
|
||||
THREADNUM = 8
|
||||
tasks = queue.Queue()
|
||||
for row in gap_df.itertuples():
|
||||
|
||||
login = getattr(row, 'login')
|
||||
# 1.
|
||||
cur.execute("select created_at from github_user where login=%s", (login,))
|
||||
created_at = cur.fetchone()['created_at']
|
||||
start_at = getattr(row, 'start_at')
|
||||
user_age = int((start_at - created_at).days / 30)
|
||||
|
||||
# 2.
|
||||
cur.execute("select sum(contribution_count) as history_commit_num from github_user_commits_per_day where date<%s and login=%s", (start_at,login))
|
||||
history_commit_num = cur.fetchone()['history_commit_num']
|
||||
if history_commit_num is None:
|
||||
history_commit_num = 0
|
||||
else:
|
||||
history_commit_num = int(history_commit_num)
|
||||
|
||||
# 3.
|
||||
cur.execute("select count(*) as history_pr_num from github_user_pr where created_at<%s and login=%s", (start_at, login))
|
||||
history_pr_num = cur.fetchone()['history_pr_num']
|
||||
|
||||
# 4.
|
||||
cur.execute("select count(*) as history_issue_num from github_user_issue where created_at<%s and login=%s", (start_at, login))
|
||||
history_issue_num = cur.fetchone()['history_issue_num']
|
||||
|
||||
# 5.
|
||||
cur.execute("select count(*) as history_issue_comment_num from github_issue_comment where created_at<%s and login=%s", (start_at, login))
|
||||
history_issue_comment_num = cur.fetchone()['history_issue_comment_num']
|
||||
|
||||
# 6.
|
||||
cur.execute("select count(*) as count_1 from github_pr_comment where created_at<%s and login=%s", (start_at, login))
|
||||
count_1 = cur.fetchone()['count_1']
|
||||
cur.execute("select count(*) as count_2 from github_commit_comment where created_at<%s and login=%s", (start_at, login))
|
||||
count_2 = cur.fetchone()['count_2']
|
||||
cur.execute("select count(*) as count_3 from github_user_pr_review where created_at<%s and login=%s", (start_at, login))
|
||||
count_3 = cur.fetchone()['count_3']
|
||||
history_pr_review_num = count_1 + count_2 + count_3
|
||||
|
||||
# 7.
|
||||
time = (start_at - min_start_at).days
|
||||
|
||||
# 8.
|
||||
index = getattr(row, 'index')
|
||||
if index > 0:
|
||||
intervention = 1
|
||||
elif index < 0:
|
||||
intervention = 0
|
||||
else:
|
||||
intervention = -1
|
||||
|
||||
# 9.
|
||||
if index > 0:
|
||||
time_after_intervention = (index - 1) * n / 12 + unstable_n
|
||||
elif index < 0:
|
||||
time_after_intervention = 0
|
||||
else:
|
||||
time_after_intervention = -1
|
||||
|
||||
# 10.
|
||||
end_at = getattr(row, 'end_at')
|
||||
cur.execute("select sum(contribution_count) as commit_num from github_user_commits_per_day where date>%s and date<=%s and login=%s", (start_at, end_at, login))
|
||||
commit_num = cur.fetchone()['commit_num']
|
||||
if commit_num is None:
|
||||
commit_num = 0
|
||||
else:
|
||||
commit_num = int(commit_num)
|
||||
|
||||
# 11.
|
||||
cur.execute("select count(*) as pr_num from github_user_pr where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
pr_num = cur.fetchone()['pr_num']
|
||||
|
||||
# 12.
|
||||
cur.execute("select count(*) as issue_num from github_user_issue where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
issue_num = cur.fetchone()['issue_num']
|
||||
|
||||
# 13.
|
||||
cur.execute("select count(*) as issue_comment_num from github_issue_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
issue_comment_num = cur.fetchone()['issue_comment_num']
|
||||
|
||||
# 14.
|
||||
cur.execute("select count(*) as num from github_pr_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_1 = cur.fetchone()['num']
|
||||
cur.execute("select count(*) as num from github_commit_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_2 = cur.fetchone()['num']
|
||||
cur.execute("select count(*) as num from github_user_pr_review where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_3 = cur.fetchone()['num']
|
||||
pr_review_num = count_1 + count_2 + count_3
|
||||
|
||||
# 赋值
|
||||
gap_df.loc[row.Index, 'user_age'] = user_age
|
||||
gap_df.loc[row.Index, 'history_commit_num'] = history_commit_num
|
||||
gap_df.loc[row.Index, 'history_pr_num'] = history_pr_num
|
||||
gap_df.loc[row.Index, 'history_issue_num'] = history_issue_num
|
||||
gap_df.loc[row.Index, 'history_issue_comment_num'] = history_issue_comment_num
|
||||
gap_df.loc[row.Index, 'history_pr_review_num'] = history_pr_review_num
|
||||
gap_df.loc[row.Index, 'time'] = time
|
||||
gap_df.loc[row.Index, 'intervention'] = intervention
|
||||
gap_df.loc[row.Index, 'time_after_intervention'] = time_after_intervention
|
||||
gap_df.loc[row.Index, 'commit_num'] = commit_num
|
||||
gap_df.loc[row.Index, 'pr_num'] = pr_num
|
||||
gap_df.loc[row.Index, 'issue_num'] = issue_num
|
||||
gap_df.loc[row.Index, 'issue_comment_num'] = issue_comment_num
|
||||
gap_df.loc[row.Index, 'pr_review_num'] = pr_review_num
|
||||
|
||||
print(row.Index)
|
||||
|
||||
tasks.put(row)
|
||||
break
|
||||
for _ in range(THREADNUM):
|
||||
t = myThread_sponsor_create(tasks)
|
||||
t.start()
|
||||
tasks.join()
|
||||
# write to file
|
||||
gap_df.to_csv("data_4_filtered_users.csv", index=None)
|
||||
gap_df.to_csv("data_4_filtered_users_sponsor_create.csv", index=None)
|
||||
print("finish sponsor create")
|
||||
|
||||
|
||||
|
||||
#======================= 处理first sponsored time
|
||||
|
||||
# read logins and sponsor created at
|
||||
selected_users_df = pd.read_csv("selected_sponsor_firstSponsored_logins.csv",header=0)
|
||||
|
||||
# 形成所有开发者的前后6个gap，以及中间的一个gap
|
||||
gap_indexes = []
|
||||
gap_starts = []
|
||||
gap_ends = []
|
||||
gap_logins = []
|
||||
for row in selected_users_df.itertuples():
|
||||
login = getattr(row, 'login')
|
||||
first_sponsored_at = getattr(row, 'first_sponsored_at')
|
||||
first_sponsored_at = datetime.datetime.fromisoformat(first_sponsored_at)
|
||||
|
||||
for gap_index in range(-6, 7, 1):
|
||||
if gap_index < 0:
|
||||
start_at = first_sponsored_at - datetime.timedelta(days=unstable_n) + gap_index * datetime.timedelta(days=n/interval_n)
|
||||
end_at = start_at + datetime.timedelta(days=n/interval_n)
|
||||
|
||||
elif gap_index > 0:
|
||||
start_at = first_sponsored_at + datetime.timedelta(days=unstable_n) + gap_index * datetime.timedelta(days=n/interval_n)
|
||||
end_at = start_at + datetime.timedelta(days=n/interval_n)
|
||||
|
||||
else:
|
||||
start_at = first_sponsored_at - datetime.timedelta(days=unstable_n)
|
||||
end_at = first_sponsored_at + datetime.timedelta(days=unstable_n)
|
||||
|
||||
gap_indexes.append(gap_index)
|
||||
gap_starts.append(start_at)
|
||||
gap_ends.append(end_at)
|
||||
gap_logins.append(login)
|
||||
gap_df = pd.DataFrame.from_dict({"index": gap_indexes, "start_at": gap_starts, "end_at": gap_ends, "login": gap_logins})
|
||||
|
||||
# 读取用户不同index下的所有属性
|
||||
# 1. user age in months; 2. history commits; 3. history prs; 4. history issues; 5. history issue_comments; 6. history pr_reviews
|
||||
# 7. time in days from the start of observation period; 8. interventionTRUE; 9. time_after_intervention
|
||||
# 10. commit_num in this interval; 11. pr_num in this interval; 12. issue_num in this interval; 13. issue_comment_num in this interval; 14. pr_review_num in this interval;
|
||||
# 15. history sponsored users; 16. average tier amount
|
||||
|
||||
# store results
|
||||
gap_df['user_age'] = -1
|
||||
gap_df['history_commit_num'] = -1
|
||||
gap_df['history_pr_num'] = -1
|
||||
gap_df['history_issue_num'] = -1
|
||||
gap_df['history_issue_comment_num'] = -1
|
||||
gap_df['history_pr_review_num'] = -1
|
||||
gap_df['time'] = -1
|
||||
gap_df['intervention'] = -1
|
||||
gap_df['time_after_intervention'] = -1
|
||||
gap_df['commit_num'] = -1
|
||||
gap_df['pr_num'] = -1
|
||||
gap_df['issue_num'] = -1
|
||||
gap_df['issue_comment_num'] = -1
|
||||
gap_df['pr_review_num'] = -1
|
||||
gap_df['history_sponsored_num'] = -1
|
||||
gap_df['avg_tier'] = -1
|
||||
|
||||
|
||||
min_start_at = min(gap_df['start_at'])
|
||||
|
||||
# 形成task, 多线程运行
|
||||
class myThread_first_sponsored(threading.Thread):
|
||||
def __init__(self, q):
|
||||
threading.Thread.__init__(self)
|
||||
self.q = q
|
||||
self.conn = connectMysqlDB(config, autocommit = True)
|
||||
self.cur = self.conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
def run(self):
|
||||
while(True):
|
||||
try:
|
||||
row = self.q.get(timeout=0)
|
||||
print("loop how many threads left: %d" % (self.q.qsize()))
|
||||
login = getattr(row, 'login')
|
||||
# 1.
|
||||
self.cur.execute("select created_at from github_user where login=%s", (login,))
|
||||
created_at = self.cur.fetchone()['created_at']
|
||||
start_at = getattr(row, 'start_at')
|
||||
user_age = int((start_at - created_at).days / 30)
|
||||
|
||||
# 2.
|
||||
self.cur.execute("select sum(contribution_count) as history_commit_num from github_user_commits_per_day where date<%s and login=%s", (start_at,login))
|
||||
history_commit_num = self.cur.fetchone()['history_commit_num']
|
||||
if history_commit_num is None:
|
||||
history_commit_num = 0
|
||||
else:
|
||||
history_commit_num = int(history_commit_num)
|
||||
|
||||
# 3.
|
||||
self.cur.execute("select count(*) as history_pr_num from github_user_pr where created_at<%s and login=%s", (start_at, login))
|
||||
history_pr_num = self.cur.fetchone()['history_pr_num']
|
||||
|
||||
# 4.
|
||||
self.cur.execute("select count(*) as history_issue_num from github_user_issue where created_at<%s and login=%s", (start_at, login))
|
||||
history_issue_num = self.cur.fetchone()['history_issue_num']
|
||||
|
||||
# 5.
|
||||
self.cur.execute("select count(*) as history_issue_comment_num from github_issue_comment where created_at<%s and login=%s", (start_at, login))
|
||||
history_issue_comment_num = self.cur.fetchone()['history_issue_comment_num']
|
||||
|
||||
# 6.
|
||||
self.cur.execute("select count(*) as count_1 from github_pr_comment where created_at<%s and login=%s", (start_at, login))
|
||||
count_1 = self.cur.fetchone()['count_1']
|
||||
self.cur.execute("select count(*) as count_2 from github_commit_comment where created_at<%s and login=%s", (start_at, login))
|
||||
count_2 = self.cur.fetchone()['count_2']
|
||||
self.cur.execute("select count(*) as count_3 from github_user_pr_review where created_at<%s and login=%s", (start_at, login))
|
||||
count_3 = self.cur.fetchone()['count_3']
|
||||
history_pr_review_num = count_1 + count_2 + count_3
|
||||
|
||||
# 7.
|
||||
index = getattr(row, 'index')
|
||||
time = int(index + n/step)
|
||||
|
||||
# 8.
|
||||
if index > 0:
|
||||
intervention = 1
|
||||
elif index < 0:
|
||||
intervention = 0
|
||||
else:
|
||||
intervention = -1
|
||||
|
||||
# 9.
|
||||
if index > 0:
|
||||
time_after_intervention = time - int(n/step)
|
||||
elif index < 0:
|
||||
time_after_intervention = 0
|
||||
else:
|
||||
time_after_intervention = -1
|
||||
|
||||
# 10.
|
||||
end_at = getattr(row, 'end_at')
|
||||
self.cur.execute("select sum(contribution_count) as commit_num from github_user_commits_per_day where date>%s and date<=%s and login=%s", (start_at, end_at, login))
|
||||
commit_num = self.cur.fetchone()['commit_num']
|
||||
if commit_num is None:
|
||||
commit_num = 0
|
||||
else:
|
||||
commit_num = int(commit_num)
|
||||
|
||||
# 11.
|
||||
self.cur.execute("select count(*) as pr_num from github_user_pr where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
pr_num = self.cur.fetchone()['pr_num']
|
||||
|
||||
# 12.
|
||||
self.cur.execute("select count(*) as issue_num from github_user_issue where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
issue_num = self.cur.fetchone()['issue_num']
|
||||
|
||||
# 13.
|
||||
self.cur.execute("select count(*) as issue_comment_num from github_issue_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
issue_comment_num = self.cur.fetchone()['issue_comment_num']
|
||||
|
||||
# 14.
|
||||
self.cur.execute("select count(*) as num from github_pr_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_1 = self.cur.fetchone()['num']
|
||||
self.cur.execute("select count(*) as num from github_commit_comment where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_2 = self.cur.fetchone()['num']
|
||||
self.cur.execute("select count(*) as num from github_user_pr_review where created_at>%s and created_at<=%s and login=%s", (start_at, end_at, login))
|
||||
count_3 = self.cur.fetchone()['num']
|
||||
pr_review_num = count_1 + count_2 + count_3
|
||||
|
||||
|
||||
# 15.
|
||||
self.cur.execute("select count(*) as num from github_sponsorships_as_maintainer where login=%s and created_at<%s", (login, start_at))
|
||||
history_sponsored_num = self.cur.fetchone()['num']
|
||||
|
||||
# 16.
|
||||
self.cur.execute("select avg(monthly_price_in_dollars) as avg_tier from github_sponsor_listing_tiers where login=%s", (login,))
|
||||
avg_tier = self.cur.fetchone()['avg_tier']
|
||||
if avg_tier is not None:
|
||||
avg_tier = float(avg_tier)
|
||||
else:
|
||||
avg_tier = float(-2)
|
||||
|
||||
|
||||
# 赋值
|
||||
gap_df.loc[row.Index, 'user_age'] = user_age
|
||||
gap_df.loc[row.Index, 'history_commit_num'] = history_commit_num
|
||||
gap_df.loc[row.Index, 'history_pr_num'] = history_pr_num
|
||||
gap_df.loc[row.Index, 'history_issue_num'] = history_issue_num
|
||||
gap_df.loc[row.Index, 'history_issue_comment_num'] = history_issue_comment_num
|
||||
gap_df.loc[row.Index, 'history_pr_review_num'] = history_pr_review_num
|
||||
gap_df.loc[row.Index, 'time'] = time
|
||||
gap_df.loc[row.Index, 'intervention'] = intervention
|
||||
gap_df.loc[row.Index, 'time_after_intervention'] = time_after_intervention
|
||||
gap_df.loc[row.Index, 'commit_num'] = commit_num
|
||||
gap_df.loc[row.Index, 'pr_num'] = pr_num
|
||||
gap_df.loc[row.Index, 'issue_num'] = issue_num
|
||||
gap_df.loc[row.Index, 'issue_comment_num'] = issue_comment_num
|
||||
gap_df.loc[row.Index, 'pr_review_num'] = pr_review_num
|
||||
gap_df.loc[row.Index, 'history_sponsored_num'] = history_sponsored_num
|
||||
gap_df.loc[row.Index, 'avg_tier'] = avg_tier
|
||||
|
||||
except queue.Empty:
|
||||
return
|
||||
self.q.task_done()
|
||||
|
||||
tasks = queue.Queue()
|
||||
for row in gap_df.itertuples():
|
||||
tasks.put(row)
|
||||
break
|
||||
for _ in range(THREADNUM):
|
||||
t = myThread_first_sponsored(tasks)
|
||||
t.start()
|
||||
tasks.join()
|
||||
# write to file
|
||||
gap_df.to_csv("data_4_filtered_users_first_sponsored.csv", index=None)
|
||||
print("finish sponsor create")
|
||||
|
||||
print("finish")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -13,8 +13,8 @@ conn = connectMysqlDB(config, autocommit = True)
|
|||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
|
||||
n = 12*14 # 多少天
|
||||
unstable_n = 7 # 前后多少天是不稳定时间
|
||||
n = 6 * 30 # 前后6个月
|
||||
unstable_n = 15 # 前后多少天是不稳定时间
|
||||
|
||||
collect_date = datetime.datetime.fromisoformat("2021-01-20 00:00:00").date() # 设定数据搜集时间
|
||||
|
||||
|
|
@ -45,4 +45,48 @@ selected_users_pd = pd.DataFrame.from_dict(selected_users)
|
|||
|
||||
print("被选中的开发者（以开通sponsor为时间点的开发者）共有: %s" % (len(selected_users_login)))
|
||||
# 存储到csv文件中
|
||||
selected_users_pd.to_csv("selected_sponsor_create_logins.csv", index=None)
|
||||
selected_users_pd.to_csv("selected_sponsor_create_logins.csv", index=None)
|
||||
|
||||
|
||||
|
||||
|
||||
##### 找到所有被打赏的用户
|
||||
# find all the users being sponsored by others
|
||||
cur.execute("select gsam.login, gsam.created_at as first_sponsored_at, gu.created_at as account_created_at from github_sponsorships_as_maintainer gsam, github_user gu where gu.login=gsam.login")
|
||||
users_db = cur.fetchall()
|
||||
selected_users_login = []
|
||||
selected_users_first_sponsored_at = []
|
||||
users_dict = {}
|
||||
users = [] # 最终存储信息的list
|
||||
for user in users_db:
|
||||
login = user['login']
|
||||
first_sponsored_at = user['first_sponsored_at'].date()
|
||||
account_created_at = user['account_created_at'].date()
|
||||
if ((login in users_dict and first_sponsored_at < users_dict[login][0]) or (login not in users_dict)):
|
||||
users_dict[login] = (first_sponsored_at, account_created_at)
|
||||
|
||||
for (login, t) in users_dict.items():
|
||||
users.append({"login": login, "first_sponsored_at": t[0], "account_created_at": t[1]})
|
||||
|
||||
for user in users:
|
||||
# 判断sponsor创建时间之间是不是有n+unstable_n天
|
||||
login = user['login']
|
||||
first_sponsored_at = user['first_sponsored_at']
|
||||
account_created_at = user['account_created_at']
|
||||
gap_1 = first_sponsored_at - account_created_at
|
||||
if gap_1.days < n+unstable_n:
|
||||
continue
|
||||
|
||||
# 判断数据搜集时间与sponsor创建时间之间是不是有n+unstable_n天
|
||||
gap_2 = collect_date - first_sponsored_at
|
||||
if gap_2.days < n+unstable_n:
|
||||
continue
|
||||
|
||||
selected_users_login.append(login)
|
||||
selected_users_first_sponsored_at.append(first_sponsored_at)
|
||||
selected_users = {"login": selected_users_login, "first_sponsored_at": selected_users_first_sponsored_at}
|
||||
selected_users_pd = pd.DataFrame.from_dict(selected_users)
|
||||
|
||||
print("被选中的开发者（以第一次被赞助为时间点的开发者）共有: %s" % (len(selected_users_login)))
|
||||
# 存储到csv文件中
|
||||
selected_users_pd.to_csv("selected_sponsor_firstSponsored_logins.csv", index=None)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue