forked from googol/benchmark
479 lines
15 KiB
Python
479 lines
15 KiB
Python
import requests
|
||
import os
|
||
import time
|
||
import datetime
|
||
import base64
|
||
import json
|
||
import xml.etree.ElementTree as ET
|
||
import subprocess
|
||
import re
|
||
import yaml
|
||
|
||
# 路径配置
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
|
||
DATA_DIR = os.path.join(SCRIPT_DIR, "data")
|
||
REPO_DIR = os.path.join(DATA_DIR, "repos1")
|
||
LOG_DIR = os.path.join(SCRIPT_DIR, "logs")
|
||
|
||
OUTPUT_JSON = os.path.join(DATA_DIR, "collected_repos.json")
|
||
CLONE_FAIL_FILE = os.path.join(LOG_DIR, "clone_fail.json")
|
||
RATE_LIMIT_LOG = os.path.join(LOG_DIR, "rate_limit.log")
|
||
MISSING_REPO_LOG = os.path.join(LOG_DIR, "missing_repo.log")
|
||
SKIPPED_REPOS_FILE = os.path.join(LOG_DIR, "skipped_repos.json")
|
||
COLLECTED_REPOS_FILE = os.path.join(LOG_DIR, "collected_repos.json")
|
||
|
||
for d in [DATA_DIR, REPO_DIR, LOG_DIR]:
|
||
os.makedirs(d, exist_ok=True)
|
||
|
||
# 读取配置文件
|
||
def load_config():
|
||
config_path = os.path.join(SCRIPT_DIR, ".env.yml")
|
||
if not os.path.exists(config_path):
|
||
print(f"Config file not found: {config_path}")
|
||
print(f"Please create it based on .env.example.yml")
|
||
exit(1)
|
||
|
||
with open(config_path, "r") as f:
|
||
return yaml.safe_load(f)
|
||
|
||
# 加载配置
|
||
config = load_config()
|
||
|
||
# GitHub 配置
|
||
GITHUB_TOKENS = config.get("github_tokens", [])
|
||
# 兼容旧版配置
|
||
if not GITHUB_TOKENS and config.get("github_token"):
|
||
GITHUB_TOKENS = [config.get("github_token")]
|
||
|
||
# Token轮训索引
|
||
current_token_index = 0
|
||
|
||
# 获取当前Token并轮换
|
||
def get_next_token():
|
||
global current_token_index
|
||
if not GITHUB_TOKENS:
|
||
return None
|
||
token = GITHUB_TOKENS[current_token_index]
|
||
# 轮换到下一个Token
|
||
current_token_index = (current_token_index + 1) % len(GITHUB_TOKENS)
|
||
return token
|
||
|
||
BASE_URL = config.get("base_url", "https://api.github.com")
|
||
MIN_STARS = config.get("min_stars", 20)
|
||
REQUEST_INTERVAL = config.get("request_interval", 0.8)
|
||
MAX_RETRY_5XX = config.get("max_retry_5xx", 3)
|
||
|
||
# clone 控制参数
|
||
CLONE_RETRY = config.get("clone_retry", 2)
|
||
CLONE_TIMEOUT = config.get("clone_timeout", 180)
|
||
|
||
# 日志工具
|
||
def log_to_file(path, msg):
|
||
with open(path, "a", encoding="utf-8") as f:
|
||
f.write(msg + "\n")
|
||
|
||
|
||
# 加载已处理的仓库列表（包括已收集和已跳过的）
|
||
def load_processed_repos():
|
||
"""加载所有已处理的仓库列表，包括已收集和已跳过的，用于后续运行时跳过这些仓库"""
|
||
processed = set()
|
||
|
||
# 加载已跳过的仓库
|
||
if os.path.exists(SKIPPED_REPOS_FILE):
|
||
try:
|
||
with open(SKIPPED_REPOS_FILE, "r", encoding="utf-8") as f:
|
||
skipped = json.load(f)
|
||
processed.update(skipped)
|
||
except Exception as e:
|
||
print(f"Failed to load skipped repos: {e}")
|
||
|
||
# 加载已收集的仓库
|
||
if os.path.exists(OUTPUT_JSON):
|
||
try:
|
||
with open(OUTPUT_JSON, "r", encoding="utf-8") as f:
|
||
collected = json.load(f)
|
||
processed.update(repo["repo"] for repo in collected)
|
||
except Exception as e:
|
||
print(f"Failed to load collected repos: {e}")
|
||
|
||
return processed
|
||
|
||
|
||
# 保存已跳过的仓库列表
|
||
def save_skipped_repos(skipped_repos):
|
||
"""保存已跳过的仓库列表，以便后续运行时跳过这些仓库"""
|
||
try:
|
||
with open(SKIPPED_REPOS_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(list(skipped_repos), f, indent=2, ensure_ascii=False)
|
||
except Exception as e:
|
||
print(f"Failed to save skipped repos: {e}")
|
||
|
||
|
||
# 保存已收集的仓库列表
|
||
def save_collected_repos(collected_repos):
|
||
"""保存已收集的仓库列表，以便后续运行时跳过这些仓库"""
|
||
try:
|
||
with open(COLLECTED_REPOS_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(list(collected_repos), f, indent=2, ensure_ascii=False)
|
||
except Exception as e:
|
||
print(f"Failed to save collected repos: {e}")
|
||
|
||
|
||
# 加载克隆失败的仓库列表
|
||
def load_clone_failures():
|
||
"""加载克隆失败的仓库列表"""
|
||
if not os.path.exists(CLONE_FAIL_FILE):
|
||
return []
|
||
|
||
try:
|
||
with open(CLONE_FAIL_FILE, "r", encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception as e:
|
||
print(f"Failed to load clone failures: {e}")
|
||
return []
|
||
|
||
|
||
# 保存克隆失败的仓库列表
|
||
def save_clone_failures(clone_failures):
|
||
"""保存克隆失败的仓库列表"""
|
||
try:
|
||
with open(CLONE_FAIL_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(clone_failures, f, indent=2, ensure_ascii=False)
|
||
except Exception as e:
|
||
print(f"Failed to save clone failures: {e}")
|
||
|
||
# GitHub GET
|
||
def github_get(url, params=None):
|
||
retry_5xx = 0
|
||
while True:
|
||
# 获取当前请求的Token和Headers
|
||
token = get_next_token()
|
||
headers = {"Accept": "application/vnd.github+json"}
|
||
if token:
|
||
headers["Authorization"] = f"token {token}"
|
||
|
||
r = requests.get(url, headers=headers, params=params)
|
||
if r.status_code == 200:
|
||
time.sleep(REQUEST_INTERVAL)
|
||
return r
|
||
elif r.status_code == 403:
|
||
reset = r.headers.get("X-RateLimit-Reset")
|
||
if reset:
|
||
sleep_time = int(reset) - int(time.time()) + 1
|
||
log_to_file(RATE_LIMIT_LOG, f"403 rate limit → sleep {sleep_time}s, url={url}")
|
||
time.sleep(max(sleep_time, 1))
|
||
continue
|
||
return None
|
||
elif r.status_code == 404:
|
||
log_to_file(MISSING_REPO_LOG, f"404 missing {url}")
|
||
return None
|
||
elif 500 <= r.status_code < 600:
|
||
retry_5xx += 1
|
||
if retry_5xx <= MAX_RETRY_5XX:
|
||
time.sleep(2)
|
||
continue
|
||
return None
|
||
else:
|
||
return None
|
||
|
||
# 搜索仓库（按日期）
|
||
def search_repositories_by_stars(start_date, end_date):
|
||
query = f"language:Java stars:>={MIN_STARS} created:{start_date}..{end_date}"
|
||
repos = []
|
||
|
||
for page in range(1, 11):
|
||
r = github_get(
|
||
f"{BASE_URL}/search/repositories",
|
||
params={
|
||
"q": query,
|
||
"sort": "stars",
|
||
"order": "desc",
|
||
"per_page": 100,
|
||
"page": page
|
||
},
|
||
)
|
||
if not r:
|
||
break
|
||
items = r.json().get("items", [])
|
||
if not items:
|
||
break
|
||
repos.extend(items)
|
||
return repos
|
||
|
||
# 检查文件是否存在于仓库中并返回相对路径（支持任意路径）
|
||
def check_file_exists(repo, filename):
|
||
"""
|
||
使用 GitHub 搜索 API 检查文件是否存在于仓库的任何位置
|
||
使用模式：repo:{repo} path:{filename}
|
||
返回：如果存在返回文件的相对路径，否则返回 None
|
||
"""
|
||
search_url = f"{BASE_URL}/search/code"
|
||
params = {
|
||
"q": f"repo:{repo} path:{filename}",
|
||
"per_page": 1 # 只需要知道第一个匹配结果
|
||
}
|
||
r = github_get(search_url, params=params)
|
||
if not r:
|
||
return None
|
||
try:
|
||
result = r.json()
|
||
if result.get("total_count", 0) > 0:
|
||
# 提取第一个匹配结果的相对路径
|
||
item = result.get("items", [])[0]
|
||
return item.get("path", None)
|
||
return None
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
# 检查仓库中是否包含@Test注解（支持任意路径）
|
||
def check_has_test_annotations(repo):
|
||
"""
|
||
使用 GitHub 搜索 API 检查仓库中是否包含@Test注解
|
||
使用模式：repo:{repo} @Test
|
||
"""
|
||
search_url = f"{BASE_URL}/search/code"
|
||
params = {
|
||
"q": f"repo:{repo} @Test",
|
||
"per_page": 1 # 只需要知道是否存在，不需要所有结果
|
||
}
|
||
r = github_get(search_url, params=params)
|
||
if not r:
|
||
return False
|
||
try:
|
||
result = r.json()
|
||
return result.get("total_count", 0) > 0
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
# 检查仓库中是否包含 Spring Framework 导入语句（支持任意路径）
|
||
def check_has_spring_imports(repo):
|
||
"""
|
||
使用 GitHub 搜索 API 检查仓库中是否包含 Spring Framework 导入语句
|
||
使用模式：repo:{repo} import org.springframework
|
||
"""
|
||
search_url = f"{BASE_URL}/search/code"
|
||
params = {
|
||
"q": f'repo:{repo} "import org.springframework"',
|
||
"per_page": 1 # 只需要知道是否存在，不需要所有结果
|
||
}
|
||
r = github_get(search_url, params=params)
|
||
if not r:
|
||
return False
|
||
try:
|
||
result = r.json()
|
||
return result.get("total_count", 0) > 0
|
||
except Exception:
|
||
return False
|
||
|
||
# 获取文件内容（用于获取构建文件内容进行分析，支持任意路径）
|
||
def get_file_content(repo, path):
|
||
r = github_get(f"{BASE_URL}/repos/{repo}/contents/{path}")
|
||
if not r:
|
||
return None
|
||
c = r.json().get("content")
|
||
if not c:
|
||
return None
|
||
return base64.b64decode(c).decode("utf-8", errors="ignore")
|
||
|
||
# Maven 解析
|
||
def analyze_pom(pom_text):
|
||
uses_test = False
|
||
uses_spring = False
|
||
|
||
try:
|
||
root = ET.fromstring(pom_text)
|
||
except ET.ParseError:
|
||
return False, False
|
||
|
||
for dep in root.findall(".//dependencies/dependency"):
|
||
gid = dep.findtext("groupId")
|
||
if not gid:
|
||
continue
|
||
gid = gid.strip()
|
||
|
||
# 测试框架
|
||
if gid in {"junit", "org.junit.jupiter", "org.testng"}:
|
||
uses_test = True
|
||
|
||
# Spring Framework
|
||
if gid == "org.springframework":
|
||
uses_spring = True
|
||
|
||
return uses_test, uses_spring
|
||
|
||
# Gradle 解析（正则）
|
||
SPRING_RE = re.compile(r'["\']org\.springframework:([^"\']+)["\']')
|
||
JUNIT_RE = re.compile(r'["\'](junit:junit|org\.junit\.jupiter:[^"\']+)["\']')
|
||
|
||
def analyze_gradle(gradle_text):
|
||
uses_spring = bool(SPRING_RE.search(gradle_text))
|
||
uses_test = bool(JUNIT_RE.search(gradle_text))
|
||
return uses_test, uses_spring
|
||
|
||
# 克隆仓库
|
||
def clone_repo(repo_full_name, clone_failures):
|
||
local_name = repo_full_name.replace("/", "_")
|
||
target = os.path.join(REPO_DIR, local_name)
|
||
|
||
if os.path.exists(target):
|
||
print(f" [SKIP] {repo_full_name} already cloned")
|
||
return target
|
||
|
||
for attempt in range(1, CLONE_RETRY + 1):
|
||
print(f" [CLONE] {repo_full_name} (attempt {attempt}/{CLONE_RETRY})")
|
||
|
||
try:
|
||
result = subprocess.run(
|
||
["git", "clone", "--depth", "1",
|
||
f"git@github.com:{repo_full_name}.git", target],
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
timeout=CLONE_TIMEOUT
|
||
)
|
||
except subprocess.TimeoutExpired:
|
||
msg = f"timeout after {CLONE_TIMEOUT}s"
|
||
print(f" ↳ {msg}")
|
||
if attempt == CLONE_RETRY:
|
||
# 添加到克隆失败列表
|
||
clone_failures.append({
|
||
"repo": repo_full_name,
|
||
"error": msg,
|
||
"timestamp": datetime.datetime.now().isoformat()
|
||
})
|
||
# 立即保存
|
||
save_clone_failures(clone_failures)
|
||
continue
|
||
|
||
if result.returncode == 0:
|
||
return target
|
||
|
||
error_msg = result.stderr.strip() or "unknown error"
|
||
print(f" ↳ git error: {error_msg}")
|
||
|
||
if attempt == CLONE_RETRY:
|
||
# 添加到克隆失败列表
|
||
clone_failures.append({
|
||
"repo": repo_full_name,
|
||
"error": error_msg,
|
||
"timestamp": datetime.datetime.now().isoformat()
|
||
})
|
||
# 立即保存
|
||
save_clone_failures(clone_failures)
|
||
|
||
time.sleep(2)
|
||
|
||
return None
|
||
|
||
# 分析仓库
|
||
def analyze_repository(repo):
|
||
name = repo["full_name"]
|
||
stars = repo.get("stargazers_count", 0)
|
||
|
||
# 使用 GitHub 搜索 API 检查是否包含 Spring Framework 导入语句
|
||
uses_spring = check_has_spring_imports(name)
|
||
if not uses_spring:
|
||
print(f" [SKIP] {name} no Spring Framework imports detected")
|
||
return None
|
||
|
||
# 使用 GitHub 搜索 API 检查是否包含 @Test 注解
|
||
uses_test = check_has_test_annotations(name)
|
||
|
||
if not uses_test:
|
||
print(f" [SKIP] {name} no junit/testng detected")
|
||
return None
|
||
|
||
print(f" [OK] {name} passed all filters")
|
||
return {
|
||
"repo": name,
|
||
"stars": stars
|
||
}
|
||
|
||
def main():
|
||
results = []
|
||
|
||
# 加载已跳过的仓库列表
|
||
skipped_repos = set()
|
||
if os.path.exists(SKIPPED_REPOS_FILE):
|
||
try:
|
||
with open(SKIPPED_REPOS_FILE, "r", encoding="utf-8") as f:
|
||
skipped_repos = set(json.load(f))
|
||
except Exception as e:
|
||
print(f"Failed to load skipped repos: {e}")
|
||
|
||
# 加载已收集的仓库列表
|
||
collected_repos = set()
|
||
if os.path.exists(OUTPUT_JSON):
|
||
try:
|
||
with open(OUTPUT_JSON, "r", encoding="utf-8") as f:
|
||
collected = json.load(f)
|
||
collected_repos = set(repo["repo"] for repo in collected)
|
||
except Exception as e:
|
||
print(f"Failed to load collected repos: {e}")
|
||
|
||
# 加载克隆失败的仓库列表
|
||
clone_failures = load_clone_failures()
|
||
# 提取克隆失败的仓库名集合，用于跳过已失败的仓库
|
||
failed_repos = set(failure["repo"] for failure in clone_failures)
|
||
|
||
# 合并为所有已处理的仓库（包括克隆失败的）
|
||
processed_repos = skipped_repos.union(collected_repos).union(failed_repos)
|
||
print(f"Loaded {len(processed_repos)} processed repositories")
|
||
print(f"Loaded {len(clone_failures)} clone failures")
|
||
|
||
start_date = datetime.date(2025, 2, 8)
|
||
end_date = datetime.date(2025, 6, 30)
|
||
|
||
delta = datetime.timedelta(days=1)
|
||
current_date = start_date
|
||
|
||
while current_date <= end_date:
|
||
next_date = current_date + delta
|
||
print(f"\n Searching date: {current_date}")
|
||
|
||
repos = search_repositories_by_stars(
|
||
current_date.isoformat(),
|
||
(next_date - datetime.timedelta(seconds=1)).isoformat()
|
||
)
|
||
print(f" found {len(repos)} repos")
|
||
|
||
for repo in repos:
|
||
repo_full_name = repo["full_name"]
|
||
|
||
# 检查是否已处理此仓库（无论是收集还是跳过）
|
||
if repo_full_name in processed_repos:
|
||
print(f" [SKIP] {repo_full_name} already processed in previous run")
|
||
continue
|
||
|
||
info = analyze_repository(repo)
|
||
if info:
|
||
results.append(info)
|
||
# 每次成功添加结果后立即保存
|
||
with open(OUTPUT_JSON, "w", encoding="utf-8") as f:
|
||
json.dump(results, f, indent=2)
|
||
# 更新已处理和已收集的仓库列表
|
||
processed_repos.add(repo_full_name)
|
||
collected_repos.add(repo_full_name)
|
||
# 立即保存已收集的仓库列表，确保数据不丢失
|
||
save_collected_repos(collected_repos)
|
||
else:
|
||
# 更新已处理和已跳过的仓库列表
|
||
processed_repos.add(repo_full_name)
|
||
skipped_repos.add(repo_full_name)
|
||
# 立即保存跳过的仓库列表，确保数据不丢失
|
||
save_skipped_repos(skipped_repos)
|
||
|
||
current_date = next_date
|
||
|
||
print(f"\n Total collected: {len(results)} repositories")
|
||
print(f" Saved to {OUTPUT_JSON}")
|
||
print(f" Clone failed repos saved to {CLONE_FAIL_FILE}")
|
||
print(f" Skipped {len(skipped_repos)} repositories")
|
||
print(f" Skipped repos saved to {SKIPPED_REPOS_FILE}")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|