starlee
/
prj_doc
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

popularity by category

This commit is contained in:
whystar 2020-07-18 21:20:40 +08:00
parent f9f5b1ca06
commit 610fa1bddf
4 changed files with 252 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -15,4 +15,5 @@ experiment_code/lans/.Rhistory
experiment_code/rq1/correlations/*
.RData
.Rhistory
.config.json
.config.json
*pyc

181
experiment_code/1-popularity/overvie.py Normal file
View File

@ -0,0 +1,181 @@
# coding: utf-8
import MySQLdb
import json
import time
import random
import urllib2
import pandas as pd
with open("../.config.json") as fp:
config = json.load(fp)
local_db_config = config["local_db"]
conn = MySQLdb.connect(host=local_db_config["db_host"],user=local_db_config["db_user"],
passwd=local_db_config["db_passwd"],db=local_db_config["db_name"],port=3306,charset='utf8mb4')
cursor = conn.cursor()
def profile_type(doc):
doc_id,repo_id,doc_location,doc_name,doc_type = doc
if doc_location.lower() in ["root",".github","docs"] and doc_type=="blob":
doc_name = doc_name.lower()
for fi in ["readme"]:
if doc_name.lower().find(fi) != -1:
return "readme"
for fi in ["license"]:
if doc_name.lower().find(fi) != -1:
return "license"
for fi in ["issue_template","pull_request_template"]:
if doc_name.lower().find(fi) != -1:
return "template"
for fi in ["code_of_conduct"]:
if doc_name.lower().find(fi) != -1:
return "COC"
for fi in ["contributing"]:
if doc_name.lower().find(fi) != -1:
return "contributing"
return None
elif doc_location.lower() in ["issue_template", "pull_request_template"] and doc_type=="blob":
return "template"
else:
return None
return None
def overview():
cursor.execute("select id from random_repos")
repos = set([item[0] for item in cursor.fetchall()])
prf_files = []
cursor.execute("select * from files")
files = cursor.fetchall()
for f in files:
if f[1] in repos:
prf_files.append([f[0],f[1],profile_type(f)])
# print f, profile_type(f)
repo_counts = len(repos)
dist = {"readme":set(),"license":set(),"template":set(),
"COC":set(),"contributing":set()}
for _f in prf_files:
doc_id,repo_id,doc_type = _f
if doc_type is not None:
dist[doc_type].add(repo_id)
for key, value in dist.items():
print key, "%.2f%%"%(len(value)*100.0/repo_counts)
cursor.execute("select count(id) from random_repos where description is not null")
print "desc", "%.2f%%"%(cursor.fetchone()[0]*100.0/repo_counts)
def repo_profile(repo_id):
file_types = set()
cursor.execute("select * from files where repo_id=%s",(repo_id,))
files = cursor.fetchall()
for f in files:
f_t = profile_type(f)
if f_t is not None:
file_types.add(f_t)
return file_types
def by_language():
cursor.execute("select id, language from random_repos")
repos = cursor.fetchall()
language_dist = {}
file_dist = {}
for item in repos:
repo_id, repo_lan = item
if repo_lan not in language_dist:
language_dist[repo_lan] = 0
language_dist[repo_lan] += 1
if repo_lan not in file_dist:
file_dist[repo_lan] = {}
repo_files = repo_profile(repo_id)
for rf in repo_files:
if rf not in file_dist[repo_lan]:
file_dist[repo_lan][rf] = 0
file_dist[repo_lan][rf] += 1
lans = sorted(language_dist.items(), key=lambda x: x[1], reverse=True)
print lans
for lan, lan_count in lans:
tmp_lan_dist = file_dist[lan]
percentages = []
for _k in [ "readme","license","contributing","COC","template"]:
if _k not in tmp_lan_dist:
percentages.append(0)
else:
percentages.append(tmp_lan_dist[_k])
p_str = "|".join(["%.2f%%"%(item*100.0/language_dist[lan]) for item in percentages])
print "|%s|%d|%s|"%(lan,language_dist[lan],p_str)
def by_role():
cursor.execute("select id, owner_type from random_repos")
repos = cursor.fetchall()
role_type = {}
file_dist = {}
for item in repos:
repo_id, repo_lan = item
if repo_lan not in role_type:
role_type[repo_lan] = 0
role_type[repo_lan] += 1
if repo_lan not in file_dist:
file_dist[repo_lan] = {}
repo_files = repo_profile(repo_id)
for rf in repo_files:
if rf not in file_dist[repo_lan]:
file_dist[repo_lan][rf] = 0
file_dist[repo_lan][rf] += 1
lans = sorted(role_type.items(), key=lambda x: x[1], reverse=True)
print lans
for lan, lan_count in lans:
tmp_lan_dist = file_dist[lan]
percentages = []
for _k in [ "readme","license","contributing","COC","template"]:
if _k not in tmp_lan_dist:
percentages.append(0)
else:
percentages.append(tmp_lan_dist[_k])
p_str = "|".join(["%.2f%%"%(item*100.0/role_type[lan]) for item in percentages])
print "|%s|%d|%s|"%(lan,role_type[lan],p_str)
import time
def _strtime2int(strtime):
return time.mktime(time.strptime(strtime, '%Y-%m-%dT%H:%M:%SZ'))
def by_age():
cursor.execute("select id, created_at from random_repos")
repos = cursor.fetchall()
# 第一个是包含的，第二个是不包含的
file_dist = { "readme":[[],[]],"license":[[],[]],"contributing":[[],[]],"COC":[[],[]],"template":[[],[]]}
for item in repos:
repo_id, repo_create = item
repo_files = repo_profile(repo_id)
repo_age = (_strtime2int("2020-07-18T00:00:00Z") - _strtime2int(repo_create))/(60*60*24)
for key in file_dist.keys():
if key in repo_files:
file_dist[key][0].append(repo_age)
else:
file_dist[key][1].append(repo_age)
for key, value in file_dist.items():
stats1 = pd.Series(value[0])
stats2 = pd.Series(value[1])
print key, "Yes", stats1.min(), stats1.mean(),stats1.median(),stats1.std(),stats1.max()
print key, "No", stats2.min(), stats2.mean(),stats2.median(),stats2.std(),stats2.max()
print "----"*10
if __name__ == "__main__":
# overview()
# by_language()
# by_role()
by_age()

9
experiment_code/dataset/get_repo_files.py
View File

@ -103,7 +103,7 @@ def get_files(repo):
break
def get_dir_files(repo_id):
cursor.execute("select full_name from random_repos where id=%s",(repo_id,))
cursor.execute("select full_name from random_repos_before_fitler where id=%s",(repo_id,))
repo_slug = cursor.fetchone()[0]
file_url = "https://api.github.com/repos/%s/git/trees/%s"
@ -147,8 +147,13 @@ def get_dir_files(repo_id):
print files
print "***"*10
for f in files:
cursor.execute("insert into files(repo_id, location, name, type) values(%s,%s,%s,%s)",
cursor.execute("select id from files where repo_id=%s and location=%s and name=%s",(repo_id, f[0], f[1]))
tmp_r = cursor.fetchone()
if tmp_r is None:
cursor.execute("insert into files(repo_id, location, name, type) values(%s,%s,%s,%s)",
(repo_id, f[0], f[1], f[2]))
else:
print "already"
conn.commit()
break

62
idea.md
View File

@ -30,6 +30,40 @@ min、25%、mean、median、75%、max
- age
### 2.2 Identification of documentation
GitHub official suggestions:
#### code-fo-coduct
- https://docs.github.com/en/github/building-a-strong-community/adding-a-code-of-conduct-to-your-project
- root directory， docs directory，.github directory
- In the file name field, type CODE_OF_CONDUCT.md.
#### contributing
- https://docs.github.com/en/github/building-a-strong-community/setting-guidelines-for-repository-contributors
- root, docs, .github
- Contributing guidelines filenames are not case sensitive, and can have an extension such as .md or .txt.
#### template
- https://docs.github.com/en/github/building-a-strong-community/about-issue-and-pull-request-templates
- You must create templates on the repository's default branch. Templates created in other branches are not available for collaborators to use.
- root directory, the docs folder, or the hidden .github directory
- https://docs.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository
- https://docs.github.com/en/github/building-a-strong-community/creating-a-pull-request-template-for-your-repository
- - root, docs, .github
- 可以创建template 模板，然后再在配置文件里设置使用哪个template
#### readme
- https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-readmes#relative-links-and-image-paths-in-readme-files
- root, docs, .github
#### license
- https://docs.github.com/en/github/building-a-strong-community/adding-a-license-to-a-repository
- LICENSE or LICENSE.md (with all caps)
- https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/licensing-a-repository
- Most people place their license text in a file named LICENSE.txt (or LICENSE.md) in the root of the repository. Some projects include information about their license in their README. For example, a project's README may include a note saying "This project is licensed under the terms of the MIT license."
- 现实情况是，即使使用copying、copyright等关键词GitHub也能关联起来
#### *Summary*
- location:
According to GitHub's conventions, community profile documentations can be placed in three directories. i.e.,
- root
@ -45,9 +79,37 @@ According to GitHub's conventions, community profile documentations can be place
#### RQ1. What is the frequency/popularity of community profile documentation?
- Overview
| Type | desc | readme | license| contributing | COC|template |
|:-|:-|:-|:-|:-|:-|:-|
|Percentage|100.00%|99.25%|82.05%|28.70%|11.15%|24.05%
- by programing language
| Language | Count | readme | license| contributing | COC|template |
|:-|:-|:-|:-|:-|:-|:-|:-|
JavaScript|386|99.74%|86.53%|30.31%|13.73%|22.28%|
|Python|285|100.00%|84.56%|26.67%|8.07%|26.32%|
|Java|179|99.44%|79.89%|25.14%|14.53%|27.93%|
|C++|116|100.00%|76.72%|23.28%|6.90%|25.86%|
|Go|116|100.00%|95.69%|43.10%|12.07%|32.76%|
|PHP|111|97.30%|82.88%|36.94%|6.31%|28.83%|
|TypeScript|88|100.00%|89.77%|**56.82%**|**18.18%**|**45.45%**|
|C|83|100.00%|62.65%|24.10%|9.64%|20.48%|
|Ruby|79|100.00%|83.54%|26.58%|8.86%|17.72%|
|C#|71|100.00%|**98.59%**|28.17%|12.68%|25.35%|
- by user role
| OwerType | Count | readme | license| contributing | COC|template |
|:-|:-|:-|:-|:-|:-|:-|:-|
|User|1033|98.94%|76.86%|13.36%|5.61%|13.75%|
|Organization|967|99.59%|87.59%|45.09%|17.06%|36.40%|
- by age
- by forks (comparison of boxplot)
#### RQ2. How is community profile documentation maintained?