modify get freelancer or not

check_freelancer_or_not.py

@@ -0,0 +1,89 @@
+# aim: check whether all the related users are freelancer or not
+# author: zxh
+# date: 2020-04-25
+
+# steps:
+# 1. read github-sponsors/github_user_follow table's company/bio column to see whether they belong to a company/university/freelancer or not.
+# 2. read github-sponsors/github_sponsor_listing_desc_goal table's full_description column to see whether they are freelancer or not (whether freelance keyword exists or not)
+
+import pymysql, yaml, math, datetime
+from utils import *
+
+f = open('config.yaml', 'r')
+config = yaml.load(f.read(), Loader=yaml.BaseLoader)
+conn = connectMysqlDB(config, autocommit = True)
+cur = conn.cursor(pymysql.cursors.DictCursor)
+
+# read all the users
+cur.execute("select login from github_sponsor_listing where deleted=0")
+all_users = cur.fetchall()
+all_users = [user["login"] for user in all_users]
+
+cur.execute("select login from github_sponsorships_as_maintainer")
+maintainers = cur.fetchall()
+maintainers = [user["login"] for user in maintainers]
+
+all_users = list(set(all_users) & set(maintainers))
+
+result = {}
+
+cur.execute("select login, bio, company from github_user_follow")
+items = cur.fetchall()
+for item in items:
+    login = item["login"]
+    bio = item["bio"]
+    company = item['company']
+
+    if company is not None and "freelance" in company:
+        result[login] = 1
+    elif company is not None and ("@" in company or "company" in company or "university" in company):
+        result[login] = 0
+    if login not in result and bio is not None and "freelance" in bio:
+        result[login] = 1
+    elif login not in result and bio is not None and ("@" in bio or "company" in bio or "university" in bio):
+        result[login] = 0
+
+cur.execute("select login, full_description from github_sponsor_listing_desc_goal")
+items = cur.fetchall()
+for item in items:
+    login = item["login"]
+    full_description = item['full_description']
+    if login in result:
+        continue # already handled
+    if full_description is not None and "freelance" in full_description:
+        result[login] = 1
+
+remained_users = list(set(all_users) - set(result.keys()))
+print(len(remained_users))
+
+# 存储result
+for login, freelancer_or_not in result.items():
+    cur.execute("select login from github_sponsor_freelancer_or_not where login=%s", (login,))
+    item = cur.fetchone()
+    if item is not None:
+        cur.execute("update github_sponsor_freelancer_or_not set freelancer_or_not=%s where login=%s", (freelancer_or_not, login))
+    else:
+        cur.execute('insert into github_sponsor_freelancer_or_not (login, freelancer_or_not) values (%s, %s)', (login, freelancer_or_not))
+
+# 人工判断
+for login in remained_users:
+    cur.execute("select bio, company from github_user_follow where login=%s", (login,))
+    item = cur.fetchone()
+    bio = item['bio']
+    if bio is None: 
+        bio="null"
+    company = item['company']
+    if company is None: 
+        company="null"
+
+    cur.execute("select full_description from github_sponsor_listing_desc_goal where login=%s", (login,))
+    item = cur.fetchone()
+    full_description = item["full_description"]
+    if full_description is None: 
+        full_description="null"
+
+    # 输出结果
+    print("login: %s\n\nbio: %s\n\ncompany: %s\n\nfull_description: %s\n\nhomepage url: %s\n\nsponsorpage url: %s\n\n" % (login, bio, company, full_description, "https://github.com/" + login, "https://github.com/sponsors/" + login))
+    freelancer_or_not = input("freelancer or not (1: yes; 0: no; 2: not sure)")
+
+    cur.execute('insert into github_sponsor_freelancer_or_not (login, freelancer_or_not) values (%s, %s)', (login, freelancer_or_not))

crawler_sponsor_page.py

@@ -0,0 +1,25 @@
+# aim: crawl the sponsor page
+# date: 2021-03-31
+# author: zhangxunhui
+
+import pymysql, yaml, math, datetime
+from utils import *
+import seaborn as sns
+import pandas as pd
+from matplotlib import pyplot as plt
+import numpy as np
+
+f = open('config.yaml', 'r')
+config = yaml.load(f.read(), Loader=yaml.BaseLoader)
+conn = connectMysqlDB(config, autocommit = True)
+cur = conn.cursor(pymysql.cursors.DictCursor)
+
+time_per_hour = 50
+
+cur.execute("select login from github_sponsor_listing where deleted=0")
+items = cur.fetchall()
+for item in items:
+    login = item['login']
+    url = "https://github.com/sponsors/" + login
+
+