add sponsor_rate analysis
This commit is contained in:
parent
d6a8bd9a81
commit
89c3150561
|
|
@ -0,0 +1,85 @@
|
|||
# aim: analysis what and how factors influence the sponsored rate
|
||||
# date: 2021-05-17
|
||||
# author: zxh
|
||||
|
||||
library(RMySQL)
|
||||
library(yaml)
|
||||
library(plyr)
|
||||
library(pROC)
|
||||
library("rjson")
|
||||
library("stringr")
|
||||
require(lmerTest)
|
||||
library(dplyr)
|
||||
|
||||
dbConfig <- yaml.load_file('R/config.yaml')
|
||||
db_user <- dbConfig$mysql$user
|
||||
db_password <- dbConfig$mysql$passwd
|
||||
db_name <- dbConfig$mysql$db
|
||||
db_host <- dbConfig$mysql$host # for local access
|
||||
db_port <- as.numeric(dbConfig$mysql$port)
|
||||
conn <- dbConnect(MySQL(), user = db_user, password = db_password,
|
||||
dbname = db_name, host = db_host, port = db_port)
|
||||
q = paste("select * from middle_data_sponsored_rate")
|
||||
rs <- dbSendQuery(conn, q)
|
||||
df <- fetch(rs, n = -1)
|
||||
|
||||
df = na.omit(df)
|
||||
|
||||
categorical_factors = c('has_goal', 'freelancer')
|
||||
|
||||
preprocessing<-function(v) {
|
||||
a <- log(v + 0.5) # log the value
|
||||
a <- scale(a) # centering mean=0, variance=1
|
||||
a
|
||||
}
|
||||
convertData_preprocess <- function(df, exclude_cols = c()) {
|
||||
for (coln in colnames(df)) {
|
||||
if ((!coln %in% categorical_factors) & (!coln %in% exclude_cols)) {
|
||||
df[[coln]] = preprocessing(df[[coln]])
|
||||
}
|
||||
}
|
||||
df
|
||||
}
|
||||
|
||||
convertData_type <- function(df) {
|
||||
for (coln in colnames(df)) {
|
||||
if (coln %in% categorical_factors) {
|
||||
df[[coln]] = as.factor(df[[coln]]) # do not convert to logical factor
|
||||
}
|
||||
}
|
||||
df
|
||||
}
|
||||
|
||||
# df = convertData_preprocess(df, exclude_cols = c('sponsored_rate'))
|
||||
df = convertData_type(df)
|
||||
df$has_goal = relevel(df$has_goal, ref = 0)
|
||||
df$freelancer = relevel(df$freelancer, ref = 0)
|
||||
|
||||
m = lmer(
|
||||
formula=log(sponsored_rate+0.5)~
|
||||
log(min_tier+0.5) +
|
||||
log(max_tier+0.5) +
|
||||
has_goal +
|
||||
log(intro_complexity+0.5) +
|
||||
log(followers+0.5) +
|
||||
log(followings+0.5) +
|
||||
log(all_commit_num+0.5) +
|
||||
log(all_discussion_num+0.5) +
|
||||
log(user_age+0.5) +
|
||||
freelancer +
|
||||
log(pinned_proj_num+0.5) +
|
||||
log(max_pinned_proj_star+0.5) +
|
||||
log(avg_pinned_proj_star+0.5) +
|
||||
(1|login),
|
||||
verbose=TRUE,
|
||||
data=df
|
||||
)
|
||||
|
||||
car::vif(m)
|
||||
summary(m)
|
||||
anova(m)
|
||||
|
||||
library(MuMIn)
|
||||
r.squaredGLMM(m)
|
||||
|
||||
car::Anova(m, type=II)
|
||||
|
|
@ -19,11 +19,11 @@ cur.execute("select login from github_sponsor_listing where deleted=0")
|
|||
all_users = cur.fetchall()
|
||||
all_users = [user["login"] for user in all_users]
|
||||
|
||||
cur.execute("select login from github_sponsorships_as_maintainer")
|
||||
maintainers = cur.fetchall()
|
||||
maintainers = [user["login"] for user in maintainers]
|
||||
# cur.execute("select login from github_sponsorships_as_maintainer")
|
||||
# maintainers = cur.fetchall()
|
||||
# maintainers = [user["login"] for user in maintainers]
|
||||
|
||||
all_users = list(set(all_users) & set(maintainers))
|
||||
# all_users = list(set(all_users) & set(maintainers))
|
||||
|
||||
result = {}
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ for item in items:
|
|||
|
||||
if company is not None and ("freelance" in company.lower() or "independent" in company.lower() or "individual" in company.lower()):
|
||||
result[login] = 1
|
||||
elif company is not None and ("@" in company.lower() or "company" in company.lower() or "university" in company.lower() or "college" in company.lower() or "institute" in company.lower() or "school" in company.lower() or "gmbh" in company.lower() or "corporation" in company.lower() or "corp" in company.lower() or " tech" in company.lower()):
|
||||
elif company is not None and ("@" in company.lower() or "company" in company.lower() or "university" in company.lower() or "college" in company.lower() or "institute" in company.lower() or "school" in company.lower() or "gmbh" in company.lower() or "corporation" in company.lower() or "corp" in company.lower() or " tech" in company.lower() or " inc" in company.lower() or "lab" in company.lower()):
|
||||
result[login] = 0
|
||||
if login not in result and company is not None and (" llc" in company.lower() or " inc" in company.lower() or " ltd" in company.lower() or "limited" in company.lower() or is_company(company, company_list)):
|
||||
result[login] = 0
|
||||
|
|
@ -194,6 +194,8 @@ for login in remained_users:
|
|||
if full_description is None:
|
||||
full_description="null"
|
||||
|
||||
if company == "null":
|
||||
continue
|
||||
# if "full-time" not in full_description.lower(): # full-time这个词可以体现开发者的愿望
|
||||
# continue # 优先处理part time关键词的用户
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,181 @@
|
|||
# sponsor_rate = number of newly added sponsors per month from the creation of sponsor account to the data collection time
|
||||
# aim: find what and how factors influence sponsor_rate
|
||||
# date: 2021-05-17
|
||||
# author: zxh
|
||||
|
||||
import pymysql, yaml, math, datetime, json
|
||||
from utils import *
|
||||
import seaborn as sns
|
||||
import pandas as pd
|
||||
from matplotlib import pyplot as plt
|
||||
import numpy as np
|
||||
import threading, queue # 多线程搜集数据
|
||||
|
||||
f = open('config.yaml', 'r')
|
||||
config = yaml.load(f.read(), Loader=yaml.BaseLoader)
|
||||
conn = connectMysqlDB(config, autocommit = True)
|
||||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
# read all the sponsors from database (0 sponsor, >=1 sponsor all considered)
|
||||
cur.execute("select login from github_sponsor_listing where deleted=0")
|
||||
users = cur.fetchall()
|
||||
users = [u["login"] for u in users]
|
||||
|
||||
cur.execute("select login from middle_data_sponsored_rate")
|
||||
handled_users = cur.fetchall()
|
||||
handled_users = [u["login"] for u in handled_users]
|
||||
|
||||
users = list(set(users) - set(handled_users))
|
||||
|
||||
|
||||
data = {} # key: login, value: {factor name: factor value}
|
||||
|
||||
|
||||
class myThread(threading.Thread):
|
||||
def __init__(self, q):
|
||||
threading.Thread.__init__(self)
|
||||
self.q = q
|
||||
self.conn = connectMysqlDB(config, autocommit = True)
|
||||
self.cur = self.conn.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
def run(self):
|
||||
while(True):
|
||||
try:
|
||||
login = self.q.get(timeout=0)
|
||||
print("loop how many threads left: %d" % (self.q.qsize()))
|
||||
|
||||
# 0.1 the data collection time
|
||||
collection_time = "2021-01-23 11:24:27"
|
||||
collection_date = "2021-01-24"
|
||||
# 0.2 sponsor account creation time
|
||||
self.cur.execute("select created_at from github_sponsor_listing where login=%s", (login,))
|
||||
sponsor_account_createtime = self.cur.fetchone()['created_at']
|
||||
# 0.3 user account creation time
|
||||
self.cur.execute("select created_at from github_user where login=%s", (login,))
|
||||
user_account_createtime = self.cur.fetchone()['created_at']
|
||||
|
||||
# 1. tiers
|
||||
self.cur.execute("select avg(monthly_price_in_dollars) as avg_tier, min(monthly_price_in_dollars) as min_tier, max(monthly_price_in_dollars) as max_tier from github_sponsor_listing_tiers where login=%s", (login,))
|
||||
tier = self.cur.fetchone()
|
||||
avg_tier = tier['avg_tier']
|
||||
min_tier = tier['min_tier']
|
||||
max_tier = tier['max_tier']
|
||||
|
||||
if avg_tier is not None:
|
||||
avg_tier = int(avg_tier)
|
||||
else:
|
||||
avg_tier = None
|
||||
if min_tier is not None:
|
||||
min_tier = int(min_tier)
|
||||
else:
|
||||
min_tier = None
|
||||
if max_tier is not None:
|
||||
max_tier = int(max_tier)
|
||||
else:
|
||||
max_tier = None
|
||||
|
||||
# 2. has goal
|
||||
self.cur.execute("select has_goal from github_sponsor_listing_desc_goal where login=%s", (login,))
|
||||
has_goal = self.cur.fetchone()['has_goal']
|
||||
|
||||
|
||||
# 3. intro_complexity
|
||||
self.cur.execute("select full_description from github_sponsor_listing_desc_goal where login=%s", (login,))
|
||||
intro_complexity = self.cur.fetchone()['full_description']
|
||||
if intro_complexity is not None:
|
||||
intro_complexity = len(intro_complexity.split())
|
||||
|
||||
# 4. followers, followings
|
||||
self.cur.execute("select following_num, follower_num from github_user_follow where login=%s", (login,))
|
||||
item = self.cur.fetchone()
|
||||
followers = item["follower_num"]
|
||||
followings = item["following_num"]
|
||||
|
||||
# 5. all commit num
|
||||
self.cur.execute("select sum(contribution_count) as all_commit_num from github_user_commits_per_day where date<%s and login=%s", (collection_date, login))
|
||||
all_commit_num = self.cur.fetchone()['all_commit_num']
|
||||
if all_commit_num is None:
|
||||
all_commit_num = 0
|
||||
else:
|
||||
all_commit_num = int(all_commit_num)
|
||||
|
||||
# 6. all discussion num
|
||||
self.cur.execute("select count(*) as issue_comment_num from github_issue_comment where created_at<%s and login=%s", (collection_date, login))
|
||||
issue_comment_num = self.cur.fetchone()['issue_comment_num']
|
||||
self.cur.execute("select count(*) as num from github_pr_comment where created_at<%s and login=%s", (collection_date, login))
|
||||
pr_comment_num = self.cur.fetchone()['num']
|
||||
self.cur.execute("select count(*) as num from github_commit_comment where created_at<%s and login=%s", (collection_date, login))
|
||||
commit_comment = self.cur.fetchone()['num']
|
||||
all_discussion_num = issue_comment_num + pr_comment_num + commit_comment
|
||||
|
||||
# 7. user age collection
|
||||
user_age = int((datetime.datetime.strptime(collection_time, "%Y-%m-%d %H:%M:%S") - user_account_createtime).days / 30)
|
||||
|
||||
# 8. freelancer
|
||||
self.cur.execute("select freelancer_or_not from github_sponsor_freelancer_or_not where login=%s", (login,))
|
||||
freelancer = self.cur.fetchone()
|
||||
if freelancer is not None:
|
||||
freelancer = freelancer['freelancer_or_not']
|
||||
|
||||
# 9. max/avg pinned proj star
|
||||
self.cur.execute("select pinned_repo_num from github_user_follow where login=%s", (login,))
|
||||
pinned_proj_num = self.cur.fetchone()
|
||||
if pinned_proj_num is None or pinned_proj_num['pinned_repo_num'] is None or int(pinned_proj_num['pinned_repo_num']) == 0:
|
||||
max_pinned_proj_star = 0
|
||||
avg_pinned_proj_star = 0
|
||||
pinned_proj_num = 0
|
||||
else:
|
||||
pinned_proj_num = int(pinned_proj_num['pinned_repo_num'])
|
||||
self.cur.execute("select pinned_repo_info from github_user_follow where login=%s", (login,))
|
||||
pinned_proj_info = json.loads(self.cur.fetchone()['pinned_repo_info'])
|
||||
star_counts = [p['stargazerCount'] for p in pinned_proj_info]
|
||||
max_pinned_proj_star = max(star_counts)
|
||||
avg_pinned_proj_star = np.mean(star_counts)
|
||||
|
||||
# 10. sponsored rate
|
||||
t_month = float((datetime.datetime.strptime(collection_time, "%Y-%m-%d %H:%M:%S") - sponsor_account_createtime).days / 30)
|
||||
if t_month == 0:
|
||||
t_month = 1 / 30
|
||||
self.cur.execute("select count(*) as num from github_sponsorships_as_maintainer where login=%s and created_at<%s", (login, collection_time))
|
||||
num = self.cur.fetchone()['num']
|
||||
sponsored_rate = num / t_month
|
||||
|
||||
item_dict = {
|
||||
"avg_tier": avg_tier,
|
||||
"min_tier": min_tier,
|
||||
"max_tier": max_tier,
|
||||
"has_goal": has_goal,
|
||||
"intro_complexity": intro_complexity,
|
||||
"followers": followers,
|
||||
"followings": followings,
|
||||
"all_commit_num": all_commit_num,
|
||||
"all_discussion_num": all_discussion_num,
|
||||
"user_age": user_age,
|
||||
"freelancer": freelancer,
|
||||
"pinned_proj_num": pinned_proj_num,
|
||||
"max_pinned_proj_star": max_pinned_proj_star,
|
||||
"avg_pinned_proj_star": avg_pinned_proj_star,
|
||||
"sponsored_rate": sponsored_rate
|
||||
}
|
||||
|
||||
self.cur.execute("insert ignore into middle_data_sponsored_rate (login, avg_tier, min_tier, max_tier, has_goal, intro_complexity, followers, followings, all_commit_num, all_discussion_num, user_age, freelancer, pinned_proj_num, max_pinned_proj_star, avg_pinned_proj_star, sponsored_rate) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (login, item_dict["avg_tier"], item_dict["min_tier"], item_dict["max_tier"], item_dict["has_goal"], item_dict["intro_complexity"], item_dict["followers"], item_dict["followings"], item_dict["all_commit_num"], item_dict["all_discussion_num"], item_dict["user_age"], item_dict["freelancer"], item_dict["pinned_proj_num"], item_dict["max_pinned_proj_star"], item_dict["avg_pinned_proj_star"], item_dict["sponsored_rate"]))
|
||||
|
||||
# data[login] = item_dict
|
||||
|
||||
except queue.Empty:
|
||||
return
|
||||
self.q.task_done()
|
||||
|
||||
|
||||
|
||||
# gather all the infos for each user
|
||||
THREADNUM = 10
|
||||
tasks = queue.Queue()
|
||||
for user in users:
|
||||
tasks.put(user)
|
||||
for _ in range(THREADNUM):
|
||||
t = myThread(tasks)
|
||||
t.start()
|
||||
tasks.join()
|
||||
|
||||
print("finish")
|
||||
Loading…
Reference in New Issue