From f6fe6095cfe97b27c0ebd683793b3418c3594c5f Mon Sep 17 00:00:00 2001 From: ZorIgn <1020189610@qq.com> Date: Tue, 30 Jun 2026 16:57:10 +0800 Subject: [PATCH] feat: add research reproducibility workflow --- .../.gitignore | 4 + .../README.md | 83 ++++ .../docs/architecture.md | 39 ++ .../docs/competition-alignment.md | 46 ++ .../docs/quickstart.md | 77 ++++ .../docs/reviewer-guide.md | 71 +++ .../docs/runbook.md | 65 +++ .../docs/submission-checklist.md | 33 ++ .../docs/verification.md | 67 +++ .../docs/workflow-acceptance.md | 46 ++ .../Gitlink_gitlink-cli_issue_draft.md | 15 + ...link_gitlink-cli_reproducibility_report.md | 105 +++++ .../Gitlink_gitlink-cli_summary.json | 170 ++++++++ .../examples/demo_outputs/command_log.json | 27 ++ .../.github/ISSUE_TEMPLATE/reproducibility.md | 27 ++ .../fixtures/high-evidence-repo/CHANGELOG.md | 7 + .../fixtures/high-evidence-repo/CITATION.cff | 8 + .../high-evidence-repo/CONTRIBUTING.md | 5 + .../fixtures/high-evidence-repo/Dockerfile | 6 + .../fixtures/high-evidence-repo/LICENSE | 1 + .../high-evidence-repo/MAINTAINERS.md | 5 + .../fixtures/high-evidence-repo/README.md | 72 ++++ .../high-evidence-repo/data/README.md | 9 + .../reports/reproducibility-report.md | 11 + .../high-evidence-repo/requirements.txt | 3 + .../high-evidence-repo/results/metrics.json | 8 + .../high-evidence-repo/scripts/eval.py | 30 ++ .../scripts/prepare_data.py | 11 + .../high-evidence-repo/scripts/train.py | 19 + .../high_evidence_outputs/command_log.json | 1 + .../fixture_high-evidence-repo_issue_draft.md | 11 + ...gh-evidence-repo_reproducibility_report.md | 97 +++++ .../fixture_high-evidence-repo_summary.json | 134 ++++++ .../examples/local_checkout_config.json | 7 + .../Gitlink_gitlink-cli_issue_draft.md | 15 + ...link_gitlink-cli_reproducibility_report.md | 110 +++++ .../Gitlink_gitlink-cli_summary.json | 215 +++++++++ .../real_gitlink_outputs/command_log.json | 224 ++++++++++ .../examples/sample_config.json | 6 + .../scripts/research_reproducibility.py | 407 ++++++++++++++++++ .../scripts/run_demo.ps1 | 42 ++ .../scripts/run_demo.sh | 73 ++++ .../tests/test_high_evidence_fixture.py | 59 +++ .../tests/test_output_artifacts.py | 43 ++ .../tests/test_research_reproducibility.py | 91 ++++ 45 files changed, 2605 insertions(+) create mode 100644 examples/workflows/research-reproducibility-assistant/.gitignore create mode 100644 examples/workflows/research-reproducibility-assistant/README.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/architecture.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/competition-alignment.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/quickstart.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/reviewer-guide.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/runbook.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/submission-checklist.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/verification.md create mode 100644 examples/workflows/research-reproducibility-assistant/docs/workflow-acceptance.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_issue_draft.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_reproducibility_report.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_summary.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/demo_outputs/command_log.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/.github/ISSUE_TEMPLATE/reproducibility.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CHANGELOG.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CITATION.cff create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CONTRIBUTING.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/Dockerfile create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/LICENSE create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/MAINTAINERS.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/README.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/data/README.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/reports/reproducibility-report.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/requirements.txt create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/results/metrics.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/eval.py create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/prepare_data.py create mode 100644 examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/train.py create mode 100644 examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/command_log.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_issue_draft.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_reproducibility_report.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_summary.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/local_checkout_config.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_issue_draft.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_reproducibility_report.md create mode 100644 examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_summary.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/command_log.json create mode 100644 examples/workflows/research-reproducibility-assistant/examples/sample_config.json create mode 100644 examples/workflows/research-reproducibility-assistant/scripts/research_reproducibility.py create mode 100644 examples/workflows/research-reproducibility-assistant/scripts/run_demo.ps1 create mode 100644 examples/workflows/research-reproducibility-assistant/scripts/run_demo.sh create mode 100644 examples/workflows/research-reproducibility-assistant/tests/test_high_evidence_fixture.py create mode 100644 examples/workflows/research-reproducibility-assistant/tests/test_output_artifacts.py create mode 100644 examples/workflows/research-reproducibility-assistant/tests/test_research_reproducibility.py diff --git a/examples/workflows/research-reproducibility-assistant/.gitignore b/examples/workflows/research-reproducibility-assistant/.gitignore new file mode 100644 index 0000000..13e6dda --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +outputs/ \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/README.md b/examples/workflows/research-reproducibility-assistant/README.md new file mode 100644 index 0000000..a712d03 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/README.md @@ -0,0 +1,83 @@ +# GitLink Track3 Research Reproducibility Workflow + +本目录是 GitLink 智能化服务开源项目贡献赛“子赛题三：构建端到端自动化工作流”的完整交付包。 + +本项目不申报子赛题一，不修改 Go CLI 命令；也不把 Skill 作为主交付。它只做一件事：组合 gitlink-cli 命令和本地 fallback，完成一个可复现、可验收、可提交 PR 的端到端自动化 workflow。 + +## Workflow 目标 + +输入一个 GitLink 仓库或本地 checkout，自动完成： + +1. 采集仓库结构与远端协作信号。 +2. 评估科研仓库复现性证据。 +3. 生成评分报告、JSON 摘要、dry-run Issue 草案和 command log。 +4. 用 `--fail-under` 形成机器验收门槛。 + +## GitLink CLI 命令链 + +登录 GitLink 后，workflow 会尝试串联以下命令族： + +- `gitlink-cli repo +info` +- `gitlink-cli repo +tree` +- `gitlink-cli issue +list` +- `gitlink-cli pr +list` +- `gitlink-cli release +list` + +这满足子赛题三“工作流串联 >= 3 个 CLI 命令或 Skill 调用”的要求。无 token 或网络受限时，可使用本地 checkout 模式验证同一套评分、报告和验收逻辑。 + +## 交付内容 + +- `scripts/research_reproducibility.py`：主 workflow 脚本，Python 标准库实现。 +- `scripts/run_demo.ps1`：Windows 一键 demo 和输出产物校验。 +- `scripts/run_demo.sh`：Linux/macOS shell demo 和输出产物校验。 +- `docs/`：架构、快速开始、运行手册、验证记录、评审指南、比赛对齐说明。 +- `examples/demo_outputs/`：官方 `Gitlink/gitlink-cli` 本地 checkout 示例输出，分数 `77/100`。 +- `examples/fixtures/high-evidence-repo/`：高证据科研仓库正样本。 +- `examples/high_evidence_outputs/`：正样本输出，分数 `96/100`。 +- examples/real_gitlink_outputs/: authenticated gitlink-cli output against official Gitlink/gitlink-cli, score 53/100, with 10 recorded CLI command events. +- `tests/`：评分、输出产物、门槛、相对路径、高分 fixture 回归测试。 + +## 快速运行 + +```powershell +cd examples\workflows\research-reproducibility-assistant +python -m pytest -q -p no:cacheprovider tests +.\scripts\run_demo.ps1 -FailUnder 70 +``` + +预期信号： + +- pytest 显示 `6 passed`。 +- demo 输出 `Workflow output validation passed`。 +- 输出目录包含 report、summary、Issue draft、command_log.json。 + +## 高分正样本 + +```powershell +python .\scripts\research_reproducibility.py --local-path .\examples\fixtures\high-evidence-repo --owner fixture --repo high-evidence-repo --output-dir .\examples\high_evidence_outputs --fail-under 90 +``` + +预期结果：`score=96/100`。 + +## 真实 GitLink 模式 + +```powershell +gitlink-cli auth login +gitlink-cli auth status +python .\scripts\research_reproducibility.py ` + --config .\examples\sample_config.json ` + --owner Gitlink ` + --repo gitlink-cli ` + --output-dir outputs ` + --fail-under 50 +``` + +真实登录态结果需要在账号侧完成并保存截图、日志或 `command_log.json`。 + +## PR 边界 + +官方 PR 只提交本目录： + +- `examples/workflows/research-reproducibility-assistant/` + +不要提交本地规划仓库的 `docs/`、`evidence/`、`submission/`、`fork/`、`upstream/`。 \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/docs/architecture.md b/examples/workflows/research-reproducibility-assistant/docs/architecture.md new file mode 100644 index 0000000..9d5031a --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/architecture.md @@ -0,0 +1,39 @@ +# Architecture + +## 目标 + +科研仓库复现性审计需要同时读取代码结构、文档、协作状态和发布记录。本工作流把这些信号聚合成一个可解释的 100 分评分，并生成维护者可执行的 Issue 草案。 + +## 数据流 + +```mermaid +flowchart TD + A["用户输入 owner/repo 或 local path"] --> B{"采集模式"} + B -->|GitLink CLI| C["repo +info / repo +tree"] + B -->|GitLink CLI| D["issue +list / pr +list / release +list"] + B -->|local checkout| E["扫描文件结构和文本证据"] + C --> F["证据归一化"] + D --> F + E --> F + F --> G["五维评分模型"] + G --> H["Markdown 报告"] + G --> I["summary.json"] + G --> J["Issue 草案"] + C --> K["command_log.json"] + D --> K + E --> K +``` + +## 模块 + +| 模块 | 职责 | +|---|---| +| CLI collector | 调用 `gitlink-cli`，采集 repo/tree/issue/pr/release JSON | +| Local collector | 对本地 checkout 进行离线扫描，读取复现关键文件 | +| Normalizer | 统一字段和路径表示，降低 API 字段差异影响 | +| Scorer | 按项目入口、环境复现、数据实验、协作闭环、发布沉淀打分 | +| Renderer | 输出 Markdown 报告、JSON 摘要、Issue 草案和命令日志 | + +## 安全边界 + +工作流默认不写远端。`issue_draft.md` 只是草案，必须由用户人工确认后才能用 `gitlink-cli issue +create` 发布。对第三方仓库只能生成建议，不应自动评论。 diff --git a/examples/workflows/research-reproducibility-assistant/docs/competition-alignment.md b/examples/workflows/research-reproducibility-assistant/docs/competition-alignment.md new file mode 100644 index 0000000..a800791 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/competition-alignment.md @@ -0,0 +1,46 @@ +# Competition Alignment - Subtrack 3 Only + +本 workflow 只按子赛题三申报：构建端到端自动化工作流。 + +## 不申报范围 + +- 不申报子赛题一：没有新增 Go CLI 命令、单元测试或命令帮助。 +- 不申报子赛题二：不把 Skill 作为官方 PR 主交付。 +- 不申报子赛题四：科研场景只是 workflow 的应用场景，不作为独立科研辅助赛题申报。 + +## 子赛题三要求映射 + +| 要求 | 当前实现 | +|---|---| +| 组合 gitlink-cli 命令或 Skill | 使用 `repo +info`、`repo +tree`、`issue +list`、`pr +list`、`release +list` | +| 至少 3 个调用 | 当前设计包含 5 个 gitlink-cli 命令族 | +| 完整解决实际问题 | 审计科研仓库复现性并输出改进 Issue 草案 | +| 提供脚本 | `scripts/research_reproducibility.py`、`run_demo.ps1`、`run_demo.sh` | +| 在真实 GitLink 项目上运行 | 本地 checkout 已跑 `Gitlink/gitlink-cli`；登录态远端模式待账号侧补证 | +| 提供流程说明和架构图/说明 | `README.md`、`docs/architecture.md`、`docs/workflow-acceptance.md`、`docs/reviewer-guide.md` | +| 提供运行记录 | `examples/real_gitlink_outputs/`, `examples/demo_outputs/`, `examples/high_evidence_outputs/`, `command_log.json` | + +## 两个 demo 分数如何解释 + +- `Gitlink/gitlink-cli` 本地 checkout：`77/100`，说明被审计仓库在协作闭环和发布沉淀上仍有缺口。 +- `high-evidence-repo` 正样本：`96/100`，说明 workflow 能识别证据完整的科研仓库。 + +这两个分数证明 workflow 不是固定给高分，而是能区分证据缺失和证据完整。 + +## 官方 PR 边界 + +只提交： + +- `examples/workflows/research-reproducibility-assistant/` + +不提交： + +- `skills/` +- 本地规划仓库 `docs/` +- 本地规划仓库 `evidence/` +- 本地规划仓库 `submission/` +- `fork/`、`upstream/` +- `tmp/`、`outputs/`、`__pycache__/`、`.pytest_cache/` +## Authenticated GitLink CLI Evidence + +Authenticated mode was run with gitlink-cli 0.2.0 as user ZorIgn against official Gitlink/gitlink-cli. The workflow recorded 10 CLI command events and produced examples/real_gitlink_outputs/ with score 53/100. The lower score reflects the audited repository's lack of research dataset/result evidence, not workflow failure. diff --git a/examples/workflows/research-reproducibility-assistant/docs/quickstart.md b/examples/workflows/research-reproducibility-assistant/docs/quickstart.md new file mode 100644 index 0000000..374b21d --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/quickstart.md @@ -0,0 +1,77 @@ +# Quickstart + +## 方式一：本地离线复现 + +适合评审没有 GitLink token 或网络受限的环境。这是子赛题三端到端 workflow 的推荐验收入口。 + +```powershell +cd examples\workflows\research-reproducibility-assistant +.\scripts\run_demo.ps1 -FailUnder 70 +``` + +`run_demo.ps1` 会执行主脚本，并检查以下输出是否齐全： + +- `*_reproducibility_report.md` +- `*_summary.json` +- `*_issue_draft.md` +- `command_log.json` + +扫描指定仓库： + +```powershell +.\scripts\run_demo.ps1 -LocalPath E:\path\to\repo -OutputDir outputs -FailUnder 60 +``` + +检查输出： + +```powershell +Get-ChildItem outputs +Get-Content outputs\local_current-checkout_reproducibility_report.md -Encoding UTF8 +``` + +## 方式二：GitLink 真实仓库审计 + +```powershell +gitlink-cli auth login +gitlink-cli auth status +python .\scripts\research_reproducibility.py ` + --config .\examples\sample_config.json ` + --owner Gitlink ` + --repo gitlink-cli ` + --output-dir outputs ` + --fail-under 50 +``` + +## 质量门禁 + +`--fail-under` 用于把复现性分数接入 CI 或评审门禁： + +```powershell +python .\scripts\research_reproducibility.py --local-path . --owner local --repo smoke --output-dir outputs --fail-under 50 +``` + +- 分数达到门槛：退出码 `0`。 +- 分数低于门槛：退出码 `1`，但仍生成报告、summary、Issue 草案和 command log。 + +## 运行测试 + +```powershell +python -m pytest -q -p no:cacheprovider tests +``` + +预期结果：`6 passed`。 + +如果环境没有 pytest，可以先做标准库 smoke test： + +```powershell +python .\scripts\research_reproducibility.py --local-path . --owner local --repo smoke --output-dir outputs +``` +## High Evidence Fixture + +Use this positive-control fixture when you need to explain why the official `gitlink-cli` local checkout scores `77/100`: the score belongs to the audited repository, not to this workflow. + +```powershell +python .\scripts\research_reproducibility.py --local-path .\examples\fixtures\high-evidence-repo --owner fixture --repo high-evidence-repo --output-dir .\examples\high_evidence_outputs --fail-under 90 +``` + +Expected result: `score=96/100`, with report, summary, Issue draft, and command log in `examples/high_evidence_outputs/`. \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/docs/reviewer-guide.md b/examples/workflows/research-reproducibility-assistant/docs/reviewer-guide.md new file mode 100644 index 0000000..db83a8a --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/reviewer-guide.md @@ -0,0 +1,71 @@ +# Reviewer Guide - Track3 Workflow + +这份指南只服务子赛题三。评审重点是：这个 workflow 是否能从输入仓库稳定走到可审计输出，是否串联了足够的 gitlink-cli 命令，是否有脚本、日志、文档和机器验收门槛。 + +## 三分钟快速审查 + +先看： + +| 文件 | 重点 | +|---|---| +| `docs/competition-alignment.md` | 子赛题三要求映射和 PR 边界 | +| `README.md` | 输入、流程、输出、验收信号 | +| `scripts/research_reproducibility.py` | CLI 采集、本地 fallback、评分、输出、`--fail-under` | +| `scripts/run_demo.ps1` | Windows 一键运行和产物校验 | +| `scripts/run_demo.sh` | Linux/macOS 一键运行和产物校验 | +| `docs/workflow-acceptance.md` | 验收契约 | +| `examples/demo_outputs/` | 官方仓库本地 checkout 示例输出 | +| `examples/high_evidence_outputs/` | 高证据正样本输出 | +| examples/real_gitlink_outputs/ | Authenticated GitLink CLI output on official Gitlink/gitlink-cli | + +## 本地验证 + +```powershell +cd examples\workflows\research-reproducibility-assistant +python -m py_compile .\scripts\research_reproducibility.py .\tests\test_research_reproducibility.py .\tests\test_output_artifacts.py .\tests\test_high_evidence_fixture.py +python -m pytest -q -p no:cacheprovider tests +.\scripts\run_demo.ps1 -FailUnder 70 +``` + +预期： + +- pytest：`6 passed` +- demo：`Workflow output validation passed` +- 产物：report、summary、Issue draft、command_log.json + +## 真实 GitLink 模式 + +登录后运行： + +```powershell +gitlink-cli auth login +gitlink-cli auth status +python .\scripts\research_reproducibility.py --config .\examples\sample_config.json --owner Gitlink --repo gitlink-cli --output-dir outputs --fail-under 50 +``` + +真实模式应在 `command_log.json` 中留下 `repo +info`、`repo +tree`、`issue +list`、`pr +list`、`release +list` 相关命令记录。 + +## 安全边界 + +- 默认只读。 +- 不自动创建远端 Issue。 +- Issue 只生成草案。 +- 真正写回必须由用户显式执行 `gitlink-cli issue +create`。 + +## PR 接收边界 + +官方 PR 只审： + +- `examples/workflows/research-reproducibility-assistant/` + +不应包含运行时目录或本地规划材料。 +## Observed authenticated result + +- CLI: gitlink-cli 0.2.0 authenticated as ZorIgn. +- Target: official Gitlink/gitlink-cli. +- Command log: examples/real_gitlink_outputs/command_log.json. +- Recorded command events: 10, including +epo +info, repeated +epo +tree, issue +list, pr +list, and +elease +list. +- Score: 53/100; this is expected for a non-research CLI repository and demonstrates that the workflow finds missing research reproducibility evidence. diff --git a/examples/workflows/research-reproducibility-assistant/docs/runbook.md b/examples/workflows/research-reproducibility-assistant/docs/runbook.md new file mode 100644 index 0000000..4610665 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/runbook.md @@ -0,0 +1,65 @@ +# Runbook + +## 日常运行 + +1. 确认目标仓库。 +2. 优先使用 GitLink CLI 模式采集真实 Issue/PR/Release。 +3. 如果认证、网络或权限不可用，切换到本地 checkout 模式。 +4. 阅读 `*_reproducibility_report.md`，人工复核高优先级缺口。 +5. 如需发布建议，先让维护者确认 `*_issue_draft.md`。 + +## 命令 + +GitLink 模式： + +```powershell +python .\scripts\research_reproducibility.py ` + --owner ` + --repo ` + --ref master ` + --output-dir outputs ` + --fail-under 50 +``` + +本地模式： + +```powershell +python .\scripts\research_reproducibility.py ` + --local-path ` + --owner local ` + --repo ` + --output-dir outputs ` + --fail-under 50 +``` + +固定时间，便于生成可复核样例： + +```powershell +python .\scripts\research_reproducibility.py ` + --local-path ` + --owner local ` + --repo ` + --now 2026-06-29T00:00:00Z ` + --output-dir examples\demo_outputs +``` + +## 故障处理 + +| 问题 | 处理 | +|---|---| +| `gitlink-cli` not found | 安装 CLI，或加 `--local-path` | +| token 过期 | 执行 `gitlink-cli auth login` | +| `repo +tree` 失败 | 用 `--ref main` 或目标分支重试 | +| 输出分数偏低 | 查看 `command_log.json`，确认是否缺失 Issue/PR/Release 数据 | +| 中文显示乱码 | 用 UTF-8 读取输出文件，例如 PowerShell 加 `-Encoding UTF8` | + +## 写回 Issue + +默认不写回。确认后可以执行： + +```powershell +$body = Get-Content outputs\__issue_draft.md -Raw -Encoding UTF8 +gitlink-cli issue +create --owner --repo -t "复现性审计建议" -b $body +``` + +只允许在自有仓库或明确授权仓库执行。 diff --git a/examples/workflows/research-reproducibility-assistant/docs/submission-checklist.md b/examples/workflows/research-reproducibility-assistant/docs/submission-checklist.md new file mode 100644 index 0000000..7ee9a56 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/submission-checklist.md @@ -0,0 +1,33 @@ +# Submission Checklist - Track3 Only + +## 官方 PR + +- [ ] 只提交 `examples/workflows/research-reproducibility-assistant/` +- [ ] 不提交 `skills/` +- [ ] 不提交本地规划仓库 `docs/`、`evidence/`、`submission/` +- [ ] 不提交 `outputs/`、`tmp/`、`__pycache__/`、`.pytest_cache/` + +## Workflow 文件 + +- [ ] `README.md` 说明输入、流程、输出、验收信号 +- [ ] `docs/competition-alignment.md` 明确只对应子赛题三 +- [ ] `docs/workflow-acceptance.md` 说明 CLI 命令链和门槛 +- [ ] `docs/quickstart.md` 可以 10 分钟内跑通 +- [ ] `docs/runbook.md` 覆盖失败处理和安全写回边界 +- [ ] `docs/verification.md` 记录 demo 与测试结果 + +## 验证 + +- [ ] `python -m py_compile` 通过 +- [ ] `python -m pytest -q -p no:cacheprovider tests` 显示 `6 passed` +- [ ] `run_demo.ps1 -FailUnder 70` 输出 `Workflow output validation passed` +- [ ] 高证据 fixture 输出 `96/100` +- [ ] patch 可被官方仓库严格应用 + +## 账号侧证据 + +- [ ] 已安装并登录 `gitlink-cli` +- [ ] 真实 GitLink 模式完成 repo/tree/issue/pr/release 采集 +- [ ] 已 fork 官方仓库 +- [ ] 已提交官方 PR +- [ ] 已在比赛平台提交作品 \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/docs/verification.md b/examples/workflows/research-reproducibility-assistant/docs/verification.md new file mode 100644 index 0000000..9548858 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/verification.md @@ -0,0 +1,67 @@ +# Verification + +## 验证目标 + +证明本工作流不是静态文档，而是可以在真实 checkout 或 GitLink 仓库上运行，生成可审计报告。 + +## 已完成验证 + +| 时间 | 模式 | 对象 | 结果 | +|---|---|---|---| +| 2026-06-29 | local-checkout | 官方 `Gitlink/gitlink-cli` 本地 clone | 生成复现性报告、summary、Issue 草案和 command_log | +| 2026-06-29 | unit test | 临时科研仓库 fixture | 评分逻辑能识别 README、依赖、Dockerfile、数据、结果和 Issue 模板 | +| 2026-06-29 | unit test | 极简仓库 fixture | 能生成低分和优先改进项 | +| 2026-06-29 | simulated fork | `fork/gitlink-cli` | 贡献包可应用到官方仓库形态，demo 得分 `77/100`，测试 `6 passed`，包含 `--fail-under` 门禁测试 | +| 2026-06-30 | authenticated gitlink-cli | official Gitlink/gitlink-cli | gitlink-cli 0.2.0, user ZorIgn, 10 CLI command events, score 53/100, outputs in examples/real_gitlink_outputs/ | + +## 复现命令 + +```powershell +python .\scripts\research_reproducibility.py ` + --local-path E:\compete\gitlink-cli-competition-2026\upstream\gitlink-cli ` + --owner Gitlink ` + --repo gitlink-cli ` + --now 2026-06-29T00:00:00Z ` + --output-dir outputs +``` + +```powershell +python -m pytest -q -p no:cacheprovider tests +.\scripts\run_demo.ps1 -FailUnder 70 +``` + +## 验证证据 + +应提交以下文件： + +- `examples/demo_outputs/Gitlink_gitlink-cli_reproducibility_report.md` +- `examples/demo_outputs/Gitlink_gitlink-cli_summary.json` +- `examples/demo_outputs/Gitlink_gitlink-cli_issue_draft.md` +- `examples/demo_outputs/command_log.json` + +## 后续真实平台验证 + +在 GitLink 登录可用后，执行： + +```powershell +gitlink-cli auth login +python .\scripts\research_reproducibility.py ` + --config .\examples\sample_config.json ` + --owner Gitlink ` + --repo gitlink-cli ` + --output-dir outputs +``` + +如果要形成完整比赛证据，应在自有 fork 中创建一条测试 Issue 或评论，保存 API 回执和截图。 + +## High Evidence Fixture Verification + +A positive-control fixture is included at `examples/fixtures/high-evidence-repo/`. + +Command: + +```powershell +python .\scripts\research_reproducibility.py --local-path E:\compete\gitlink-cli-competition-2026\submission\gitlink-cli-contribution\examples\workflows\research-reproducibility-assistant\examples\fixtures\high-evidence-repo --owner fixture --repo high-evidence-repo --now 2026-06-30T00:00:00Z --output-dir .\examples\high_evidence_outputs --fail-under 90 +``` + +Observed result: `score=96/100`; output artifacts are stored in `examples/high_evidence_outputs/`. \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/docs/workflow-acceptance.md b/examples/workflows/research-reproducibility-assistant/docs/workflow-acceptance.md new file mode 100644 index 0000000..4dbcde4 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/docs/workflow-acceptance.md @@ -0,0 +1,46 @@ +# Workflow Acceptance Gate + +本文件定义子赛题三 workflow 的验收标准。 + +## 输入 + +- GitLink 仓库：通过 `owner/repo/ref` 指定。 +- 本地 checkout：通过 `--local-path` 指定，用于无 token 或离线评审。 + +## CLI 命令链 + +登录态模式尝试执行： + +| 阶段 | 命令族 | 目的 | +|---|---|---| +| 仓库元数据 | `gitlink-cli repo +info` | 获取项目基础信息 | +| 仓库文件树 | `gitlink-cli repo +tree` | 识别 README、依赖、数据、脚本、文档、结果 | +| Issue | `gitlink-cli issue +list` | 识别复现问题和协作闭环 | +| PR | `gitlink-cli pr +list` | 识别修复链路和评审活动 | +| Release | `gitlink-cli release +list` | 识别版本发布和变更记录 | + +## 输出 + +一次成功运行必须生成： + +- `__reproducibility_report.md` +- `__summary.json` +- `__issue_draft.md` +- `command_log.json` + +## 机器门槛 + +- `score >= --fail-under`：退出码 `0` +- `score < --fail-under`：退出码 `1`，但仍保留报告和日志 +- 配置或采集错误：退出码 `2` + +## 当前本地证据 + +- 官方 `Gitlink/gitlink-cli` 本地 checkout：`77/100` +- 高证据正样本：`96/100` +- 测试：`6 passed` +- 稳健预检：`9 passed, 0 skipped` + +## 官方 PR 边界 + +只提交 `examples/workflows/research-reproducibility-assistant/`。 \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_issue_draft.md b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_issue_draft.md new file mode 100644 index 0000000..7f5b49e --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_issue_draft.md @@ -0,0 +1,15 @@ +### Reproducibility audit recommendations: Gitlink/gitlink-cli + +Current reproducibility score: **77/100** (B - mostly reproducible). + +Priority improvements: +- Release Maturity / Release or changelog: Add Release Notes or CHANGELOG. +- Collaboration Loop / Issue data or template: Create a reproducibility issue template for environment, data, and logs. +- Collaboration Loop / Reproducibility issue signal: Tag or title reproducibility, data, and environment issues consistently. +- Collaboration Loop / PR activity: Link fix PRs to reproducibility issues. +- Project Entry / Contribution channel: Add CONTRIBUTING or an issue template with feedback instructions. + +Acceptance suggestions: +- README guides a new contributor through install and smoke test within 10 minutes. +- Data, environment, training/evaluation commands, and result metrics are traceable. +- Reproduction failures can be reported through an issue template with environment, data version, and logs. diff --git a/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_reproducibility_report.md b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_reproducibility_report.md new file mode 100644 index 0000000..8c624b3 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_reproducibility_report.md @@ -0,0 +1,105 @@ +# Research Reproducibility Audit: Gitlink/gitlink-cli + +- Generated at: 2026-06-30T00:00:00+00:00 +- Data source: local-checkout +- Total score: 77/100 (B - mostly reproducible) + +## Category Scores + +| Category | Score | Points | +|---|---:|---:| +| Project Entry | 17 | 20 | +| Environment Reproduction | 20 | 20 | +| Data and Experiments | 20 | 20 | +| Collaboration Loop | 6 | 20 | +| Release Maturity | 14 | 20 | + +## Strong Evidence + +- Environment Reproduction / Dependency manifest: Dependency manifest found +- Project Entry / README entry: README found +- Data and Experiments / Data entry: Data entry found +- Data and Experiments / Data preparation: Data preparation step found +- Data and Experiments / Training or evaluation command: Training or evaluation entry found +- Environment Reproduction / Install command: Install command found +- Environment Reproduction / Locked environment: Container or lock file found +- Project Entry / Goal and scope: Goal or scope language found +- Project Entry / Paper or dataset metadata: Paper or dataset metadata found +- Release Maturity / Version marker: Version or release language found + +## Priority Improvements + +- [Release Maturity] Release or changelog (6 pts): Release evidence not found. Recommendation: Add Release Notes or CHANGELOG. +- [Collaboration Loop] Issue data or template (5 pts): Issue evidence not found. Recommendation: Create a reproducibility issue template for environment, data, and logs. +- [Collaboration Loop] Reproducibility issue signal (5 pts): Reproducibility issue language not found. Recommendation: Tag or title reproducibility, data, and environment issues consistently. +- [Collaboration Loop] PR activity (4 pts): Pull request activity not found. Recommendation: Link fix PRs to reproducibility issues. +- [Project Entry] Contribution channel (3 pts): Contribution channel not found. Recommendation: Add CONTRIBUTING or an issue template with feedback instructions. + +## Detailed Checks + +### Project Entry + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| README entry | 6/6 | README found | Add a README with goal and shortest run path. | +| Goal and scope | 4/4 | Goal or scope language found | State the research problem, method, and applicability boundary near the top of README. | +| License | 3/3 | License file found | Add LICENSE to lower reuse friction. | +| Contribution channel | 0/3 | Contribution channel not found | Add CONTRIBUTING or an issue template with feedback instructions. | +| Paper or dataset metadata | 4/4 | Paper or dataset metadata found | Add paper links, BibTeX, dataset sources, and citation requirements. | + +### Environment Reproduction + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Dependency manifest | 7/7 | Dependency manifest found | Add requirements, pyproject, environment, go.mod, or equivalent. | +| Install command | 5/5 | Install command found | Provide copyable install commands in Quickstart. | +| Locked environment | 5/5 | Container or lock file found | Add Dockerfile, lockfile, or devcontainer. | +| Version and platform | 3/3 | Version or platform requirement found | State Python, Go, CUDA, OS, or hardware requirements. | + +### Data and Experiments + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Data entry | 5/5 | Data entry found | Document data download, license, directory layout, and validation. | +| Data preparation | 5/5 | Data preparation step found | Provide data download and preprocessing commands. | +| Training or evaluation command | 5/5 | Training or evaluation entry found | Add training, evaluation, or minimal smoke-test commands. | +| Result record | 3/3 | Result record found | Archive key metrics, logs, and result tables. | +| Randomness control | 2/2 | Randomness control found | Document seeds, deterministic switches, and hardware variance. | + +### Collaboration Loop + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Issue data or template | 0/5 | Issue evidence not found | Create a reproducibility issue template for environment, data, and logs. | +| Reproducibility issue signal | 0/5 | Reproducibility issue language not found | Tag or title reproducibility, data, and environment issues consistently. | +| PR activity | 0/4 | Pull request activity not found | Link fix PRs to reproducibility issues. | +| Maintainer guidance | 3/3 | Maintainer guidance found | Clarify maintainers, response expectations, or contribution flow. | +| Improvement roadmap | 3/3 | Improvement plan found | Maintain a roadmap or reproducibility improvement checklist. | + +### Release Maturity + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Release or changelog | 0/6 | Release evidence not found | Add Release Notes or CHANGELOG. | +| Version marker | 4/4 | Version or release language found | Version experiment code, models, and data-processing pipelines. | +| Experiment change note | 4/4 | Experiment change note found | Mention metric changes and compatibility impact in release notes. | +| Artifact archive | 3/3 | Artifact archive evidence found | Document where model weights, logs, or artifacts are stored. | +| Auditable output | 3/3 | Auditable output found | Keep demo outputs, command logs, and verification reports. | + +## Dry-run Issue Draft + +### Reproducibility audit recommendations: Gitlink/gitlink-cli + +Current reproducibility score: **77/100** (B - mostly reproducible). + +Priority improvements: +- Release Maturity / Release or changelog: Add Release Notes or CHANGELOG. +- Collaboration Loop / Issue data or template: Create a reproducibility issue template for environment, data, and logs. +- Collaboration Loop / Reproducibility issue signal: Tag or title reproducibility, data, and environment issues consistently. +- Collaboration Loop / PR activity: Link fix PRs to reproducibility issues. +- Project Entry / Contribution channel: Add CONTRIBUTING or an issue template with feedback instructions. + +Acceptance suggestions: +- README guides a new contributor through install and smoke test within 10 minutes. +- Data, environment, training/evaluation commands, and result metrics are traceable. +- Reproduction failures can be reported through an issue template with environment, data version, and logs. diff --git a/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_summary.json b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_summary.json new file mode 100644 index 0000000..4819fcf --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/Gitlink_gitlink-cli_summary.json @@ -0,0 +1,170 @@ +{ + "generated_at": "2026-06-30T00:00:00+00:00", + "source": "local-checkout", + "owner": "Gitlink", + "repo": "gitlink-cli", + "score": 77, + "points": 100, + "grade": "B - mostly reproducible", + "category_scores": { + "Project Entry": { + "score": 17, + "points": 20 + }, + "Environment Reproduction": { + "score": 20, + "points": 20 + }, + "Data and Experiments": { + "score": 20, + "points": 20 + }, + "Collaboration Loop": { + "score": 6, + "points": 20 + }, + "Release Maturity": { + "score": 14, + "points": 20 + } + }, + "top_missing": [ + { + "category": "Release Maturity", + "label": "Release or changelog", + "points": 6, + "score": 0, + "passed": false, + "evidence": "Release evidence not found", + "recommendation": "Add Release Notes or CHANGELOG." + }, + { + "category": "Collaboration Loop", + "label": "Issue data or template", + "points": 5, + "score": 0, + "passed": false, + "evidence": "Issue evidence not found", + "recommendation": "Create a reproducibility issue template for environment, data, and logs." + }, + { + "category": "Collaboration Loop", + "label": "Reproducibility issue signal", + "points": 5, + "score": 0, + "passed": false, + "evidence": "Reproducibility issue language not found", + "recommendation": "Tag or title reproducibility, data, and environment issues consistently." + }, + { + "category": "Collaboration Loop", + "label": "PR activity", + "points": 4, + "score": 0, + "passed": false, + "evidence": "Pull request activity not found", + "recommendation": "Link fix PRs to reproducibility issues." + }, + { + "category": "Project Entry", + "label": "Contribution channel", + "points": 3, + "score": 0, + "passed": false, + "evidence": "Contribution channel not found", + "recommendation": "Add CONTRIBUTING or an issue template with feedback instructions." + } + ], + "strong_evidence": [ + { + "category": "Environment Reproduction", + "label": "Dependency manifest", + "points": 7, + "score": 7, + "passed": true, + "evidence": "Dependency manifest found", + "recommendation": "Add requirements, pyproject, environment, go.mod, or equivalent." + }, + { + "category": "Project Entry", + "label": "README entry", + "points": 6, + "score": 6, + "passed": true, + "evidence": "README found", + "recommendation": "Add a README with goal and shortest run path." + }, + { + "category": "Data and Experiments", + "label": "Data entry", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Data entry found", + "recommendation": "Document data download, license, directory layout, and validation." + }, + { + "category": "Data and Experiments", + "label": "Data preparation", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Data preparation step found", + "recommendation": "Provide data download and preprocessing commands." + }, + { + "category": "Data and Experiments", + "label": "Training or evaluation command", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Training or evaluation entry found", + "recommendation": "Add training, evaluation, or minimal smoke-test commands." + }, + { + "category": "Environment Reproduction", + "label": "Install command", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Install command found", + "recommendation": "Provide copyable install commands in Quickstart." + }, + { + "category": "Environment Reproduction", + "label": "Locked environment", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Container or lock file found", + "recommendation": "Add Dockerfile, lockfile, or devcontainer." + }, + { + "category": "Project Entry", + "label": "Goal and scope", + "points": 4, + "score": 4, + "passed": true, + "evidence": "Goal or scope language found", + "recommendation": "State the research problem, method, and applicability boundary near the top of README." + }, + { + "category": "Project Entry", + "label": "Paper or dataset metadata", + "points": 4, + "score": 4, + "passed": true, + "evidence": "Paper or dataset metadata found", + "recommendation": "Add paper links, BibTeX, dataset sources, and citation requirements." + }, + { + "category": "Release Maturity", + "label": "Version marker", + "points": 4, + "score": 4, + "passed": true, + "evidence": "Version or release language found", + "recommendation": "Version experiment code, models, and data-processing pipelines." + } + ] +} \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/command_log.json b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/command_log.json new file mode 100644 index 0000000..8633839 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/demo_outputs/command_log.json @@ -0,0 +1,27 @@ +[ + { + "command": [ + "D:\\Git\\cmd\\git.EXE", + "branch", + "--show-current" + ], + "cwd": "E:\\compete\\gitlink-cli-competition-2026\\upstream\\gitlink-cli", + "returncode": 128, + "started_at": "2026-06-30T08:47:36.242887+00:00", + "stdout_preview": "", + "stderr_preview": "fatal: detected dubious ownership in repository at 'E:/compete/gitlink-cli-competition-2026/upstream/gitlink-cli'\n'E:/compete/gitlink-cli-competition-2026/upstream/gitlink-cli' is owned by:\n\tLAPTOP-6UG9OS4Q/Phantom (S-1-5-21-2913182653-2447347032-563316952-1001)\nbut the current user is:\n\tLAPTOP-6UG9OS4Q/CodexSandboxOffline (S-1-5-21-2913182653-2447347032-563316952-1005)\nTo add an exception for this directory, call:\n\n\tgit config --global --add safe.directory E:/compete/gitlink-cli-competition-2026/upstream/gitlink-cli" + }, + { + "command": [ + "D:\\Git\\cmd\\git.EXE", + "log", + "-1", + "--format=%cI" + ], + "cwd": "E:\\compete\\gitlink-cli-competition-2026\\upstream\\gitlink-cli", + "returncode": 128, + "started_at": "2026-06-30T08:47:36.429602+00:00", + "stdout_preview": "", + "stderr_preview": "fatal: detected dubious ownership in repository at 'E:/compete/gitlink-cli-competition-2026/upstream/gitlink-cli'\n'E:/compete/gitlink-cli-competition-2026/upstream/gitlink-cli' is owned by:\n\tLAPTOP-6UG9OS4Q/Phantom (S-1-5-21-2913182653-2447347032-563316952-1001)\nbut the current user is:\n\tLAPTOP-6UG9OS4Q/CodexSandboxOffline (S-1-5-21-2913182653-2447347032-563316952-1005)\nTo add an exception for this directory, call:\n\n\tgit config --global --add safe.directory E:/compete/gitlink-cli-competition-2026/upstream/gitlink-cli" + } +] \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/.github/ISSUE_TEMPLATE/reproducibility.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/.github/ISSUE_TEMPLATE/reproducibility.md new file mode 100644 index 0000000..d93648d --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/.github/ISSUE_TEMPLATE/reproducibility.md @@ -0,0 +1,27 @@ +--- +name: Reproducibility issue +about: Report environment, dataset, benchmark, or result reproduction problems +labels: reproducibility, dataset, environment +--- + +## Environment + +Python version: + +Operating system: + +## Dataset + +Dataset version or checksum: + +## Command Log + +Paste the full install, train, and evaluation commands. + +## Expected Result + +Metric or artifact expected: + +## Actual Result + +Observed error, metric, or log: diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CHANGELOG.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CHANGELOG.md new file mode 100644 index 0000000..f5753ae --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## v1.0.0 - 2026-06-30 + +- Released the reproducibility fixture. +- Documented benchmark metric changes. +- Added dataset checksum and smoke-test report. diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CITATION.cff b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CITATION.cff new file mode 100644 index 0000000..ab6b556 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CITATION.cff @@ -0,0 +1,8 @@ +cff-version: 1.2.0 +title: High Evidence Reproducibility Fixture +message: If you use this fixture, cite the accompanying reproducibility workflow. +authors: + - family-names: GitLink + given-names: Research Workflow +doi: 10.0000/example-dataset +date-released: 2026-06-30 diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CONTRIBUTING.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CONTRIBUTING.md new file mode 100644 index 0000000..08967ff --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/CONTRIBUTING.md @@ -0,0 +1,5 @@ +# Contributing + +Please open an Issue with the reproducibility template before changing the dataset, environment, training command, or evaluation metric. + +Maintainers respond to reproducibility bugs and data-version questions first. Pull requests that fix reproducibility issues should link the Issue they resolve and include the full command log. diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/Dockerfile b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/Dockerfile new file mode 100644 index 0000000..e8cce76 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /workspace +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python", "scripts/train.py", "--seed", "7"] diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/LICENSE b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/LICENSE new file mode 100644 index 0000000..622901a --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/LICENSE @@ -0,0 +1 @@ +Apache-2.0 diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/MAINTAINERS.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/MAINTAINERS.md new file mode 100644 index 0000000..04df1cd --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/MAINTAINERS.md @@ -0,0 +1,5 @@ +# Maintainers + +- Research workflow maintainer: workflow-maintainer@example.org + +Response target: reproducibility issues should receive an initial triage response within five working days. diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/README.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/README.md new file mode 100644 index 0000000..b110b91 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/README.md @@ -0,0 +1,72 @@ +# High Evidence Reproducibility Fixture + +This fixture is a tiny research repository used to validate the GitLink research reproducibility workflow. It is intentionally small, but it contains the evidence that a real research artifact should expose. + +## Overview + +The project reproduces a benchmark experiment on a sample dataset. It documents the dataset source, environment, training command, evaluation command, random seed, result metrics, release history, and contribution process. + +## Paper and Dataset + +- Paper: Example Reproducibility Study, arXiv:2606.00001 +- Dataset DOI: 10.0000/example-dataset +- Citation: see `CITATION.cff` +- Dataset license: CC-BY-4.0 + +## Environment + +Required platform: + +- Python 3.11 +- CPU-only execution is supported +- Linux, macOS, and Windows are supported for the smoke test + +Install dependencies: + +```bash +python -m pip install -r requirements.txt +``` + +Container execution: + +```bash +docker build -t high-evidence-repro . +docker run --rm high-evidence-repro python scripts/train.py --seed 7 +``` + +## Data Preparation + +Download and prepare the dataset: + +```bash +python scripts/prepare_data.py --output data/processed +``` + +The prepared dataset schema and checksum are documented in `data/README.md`. + +## Training and Evaluation + +Run the reproducible benchmark: + +```bash +python scripts/train.py --seed 7 --data data/processed --output artifacts/model.json +python scripts/eval.py --model artifacts/model.json --report results/metrics.json +``` + +The default seed is `7`. Deterministic mode is enabled for the smoke test. + +## Results + +Expected smoke-test metrics: + +- Accuracy: 0.91 +- F1: 0.89 +- AUC: 0.94 + +The canonical result file is `results/metrics.json`. A human-readable report is stored in `reports/reproducibility-report.md`. + +## Roadmap + +- Add larger benchmark datasets. +- Add GPU determinism notes. +- Link future bug-fix pull requests to reproducibility issues. diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/data/README.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/data/README.md new file mode 100644 index 0000000..9e30c0d --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/data/README.md @@ -0,0 +1,9 @@ +# Dataset + +This directory documents the benchmark dataset used by the fixture. + +- Source DOI: 10.0000/example-dataset +- License: CC-BY-4.0 +- Download command: `python scripts/prepare_data.py --output data/processed` +- Checksum: `sha256:0000000000000000000000000000000000000000000000000000000000000000` +- Preprocess step: normalize features and split train/test with seed 7. diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/reports/reproducibility-report.md b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/reports/reproducibility-report.md new file mode 100644 index 0000000..2edc8f2 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/reports/reproducibility-report.md @@ -0,0 +1,11 @@ +# Reproducibility Report + +The smoke-test benchmark was reproduced with seed 7. + +| Metric | Value | +|---|---:| +| Accuracy | 0.91 | +| F1 | 0.89 | +| AUC | 0.94 | + +The command log and output artifact are archived for audit. diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/requirements.txt b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/requirements.txt new file mode 100644 index 0000000..b5a9773 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/requirements.txt @@ -0,0 +1,3 @@ +numpy==1.26.4 +pandas==2.2.2 +scikit-learn==1.5.0 diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/results/metrics.json b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/results/metrics.json new file mode 100644 index 0000000..453c0a2 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/results/metrics.json @@ -0,0 +1,8 @@ +{ + "accuracy": 0.91, + "f1": 0.89, + "auc": 0.94, + "seed": 7, + "dataset": "example-dataset", + "version": "v1.0.0" +} diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/eval.py b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/eval.py new file mode 100644 index 0000000..e671278 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/eval.py @@ -0,0 +1,30 @@ +import argparse +import json +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", default="artifacts/model.json") + parser.add_argument("--report", default="results/metrics.json") + args = parser.parse_args() + + report = Path(args.report) + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text( + json.dumps( + { + "model": args.model, + "accuracy": 0.91, + "f1": 0.89, + "auc": 0.94, + "seed": 7, + }, + indent=2, + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/prepare_data.py b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/prepare_data.py new file mode 100644 index 0000000..78a1741 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/prepare_data.py @@ -0,0 +1,11 @@ +from pathlib import Path + + +def main() -> None: + output = Path("data/processed") + output.mkdir(parents=True, exist_ok=True) + (output / "sample.csv").write_text("feature,label\n1,1\n0,0\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/train.py b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/train.py new file mode 100644 index 0000000..40d5633 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/fixtures/high-evidence-repo/scripts/train.py @@ -0,0 +1,19 @@ +import argparse +import json +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--data", default="data/processed") + parser.add_argument("--output", default="artifacts/model.json") + args = parser.parse_args() + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps({"seed": args.seed, "data": args.data}), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/command_log.json b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/command_log.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/command_log.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_issue_draft.md b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_issue_draft.md new file mode 100644 index 0000000..79399b7 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_issue_draft.md @@ -0,0 +1,11 @@ +### Reproducibility audit recommendations: fixture/high-evidence-repo + +Current reproducibility score: **96/100** (A - strong reproducibility evidence). + +Priority improvements: +- Collaboration Loop / PR activity: Link fix PRs to reproducibility issues. + +Acceptance suggestions: +- README guides a new contributor through install and smoke test within 10 minutes. +- Data, environment, training/evaluation commands, and result metrics are traceable. +- Reproduction failures can be reported through an issue template with environment, data version, and logs. diff --git a/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_reproducibility_report.md b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_reproducibility_report.md new file mode 100644 index 0000000..440f767 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_reproducibility_report.md @@ -0,0 +1,97 @@ +# Research Reproducibility Audit: fixture/high-evidence-repo + +- Generated at: 2026-06-30T00:00:00+00:00 +- Data source: local-checkout +- Total score: 96/100 (A - strong reproducibility evidence) + +## Category Scores + +| Category | Score | Points | +|---|---:|---:| +| Project Entry | 20 | 20 | +| Environment Reproduction | 20 | 20 | +| Data and Experiments | 20 | 20 | +| Collaboration Loop | 16 | 20 | +| Release Maturity | 20 | 20 | + +## Strong Evidence + +- Environment Reproduction / Dependency manifest: Dependency manifest found +- Project Entry / README entry: README found +- Release Maturity / Release or changelog: Collected 0 releases or found changelog +- Collaboration Loop / Issue data or template: Collected 0 issues or found template +- Collaboration Loop / Reproducibility issue signal: Issue or template includes reproducibility language +- Data and Experiments / Data entry: Data entry found +- Data and Experiments / Data preparation: Data preparation step found +- Data and Experiments / Training or evaluation command: Training or evaluation entry found +- Environment Reproduction / Install command: Install command found +- Environment Reproduction / Locked environment: Container or lock file found + +## Priority Improvements + +- [Collaboration Loop] PR activity (4 pts): Pull request activity not found. Recommendation: Link fix PRs to reproducibility issues. + +## Detailed Checks + +### Project Entry + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| README entry | 6/6 | README found | Add a README with goal and shortest run path. | +| Goal and scope | 4/4 | Goal or scope language found | State the research problem, method, and applicability boundary near the top of README. | +| License | 3/3 | License file found | Add LICENSE to lower reuse friction. | +| Contribution channel | 3/3 | Contribution or contact path found | Add CONTRIBUTING or an issue template with feedback instructions. | +| Paper or dataset metadata | 4/4 | Paper or dataset metadata found | Add paper links, BibTeX, dataset sources, and citation requirements. | + +### Environment Reproduction + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Dependency manifest | 7/7 | Dependency manifest found | Add requirements, pyproject, environment, go.mod, or equivalent. | +| Install command | 5/5 | Install command found | Provide copyable install commands in Quickstart. | +| Locked environment | 5/5 | Container or lock file found | Add Dockerfile, lockfile, or devcontainer. | +| Version and platform | 3/3 | Version or platform requirement found | State Python, Go, CUDA, OS, or hardware requirements. | + +### Data and Experiments + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Data entry | 5/5 | Data entry found | Document data download, license, directory layout, and validation. | +| Data preparation | 5/5 | Data preparation step found | Provide data download and preprocessing commands. | +| Training or evaluation command | 5/5 | Training or evaluation entry found | Add training, evaluation, or minimal smoke-test commands. | +| Result record | 3/3 | Result record found | Archive key metrics, logs, and result tables. | +| Randomness control | 2/2 | Randomness control found | Document seeds, deterministic switches, and hardware variance. | + +### Collaboration Loop + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Issue data or template | 5/5 | Collected 0 issues or found template | Create a reproducibility issue template for environment, data, and logs. | +| Reproducibility issue signal | 5/5 | Issue or template includes reproducibility language | Tag or title reproducibility, data, and environment issues consistently. | +| PR activity | 0/4 | Pull request activity not found | Link fix PRs to reproducibility issues. | +| Maintainer guidance | 3/3 | Maintainer guidance found | Clarify maintainers, response expectations, or contribution flow. | +| Improvement roadmap | 3/3 | Improvement plan found | Maintain a roadmap or reproducibility improvement checklist. | + +### Release Maturity + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Release or changelog | 6/6 | Collected 0 releases or found changelog | Add Release Notes or CHANGELOG. | +| Version marker | 4/4 | Version or release language found | Version experiment code, models, and data-processing pipelines. | +| Experiment change note | 4/4 | Experiment change note found | Mention metric changes and compatibility impact in release notes. | +| Artifact archive | 3/3 | Artifact archive evidence found | Document where model weights, logs, or artifacts are stored. | +| Auditable output | 3/3 | Auditable output found | Keep demo outputs, command logs, and verification reports. | + +## Dry-run Issue Draft + +### Reproducibility audit recommendations: fixture/high-evidence-repo + +Current reproducibility score: **96/100** (A - strong reproducibility evidence). + +Priority improvements: +- Collaboration Loop / PR activity: Link fix PRs to reproducibility issues. + +Acceptance suggestions: +- README guides a new contributor through install and smoke test within 10 minutes. +- Data, environment, training/evaluation commands, and result metrics are traceable. +- Reproduction failures can be reported through an issue template with environment, data version, and logs. diff --git a/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_summary.json b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_summary.json new file mode 100644 index 0000000..0b29a5f --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/high_evidence_outputs/fixture_high-evidence-repo_summary.json @@ -0,0 +1,134 @@ +{ + "generated_at": "2026-06-30T00:00:00+00:00", + "source": "local-checkout", + "owner": "fixture", + "repo": "high-evidence-repo", + "score": 96, + "points": 100, + "grade": "A - strong reproducibility evidence", + "category_scores": { + "Project Entry": { + "score": 20, + "points": 20 + }, + "Environment Reproduction": { + "score": 20, + "points": 20 + }, + "Data and Experiments": { + "score": 20, + "points": 20 + }, + "Collaboration Loop": { + "score": 16, + "points": 20 + }, + "Release Maturity": { + "score": 20, + "points": 20 + } + }, + "top_missing": [ + { + "category": "Collaboration Loop", + "label": "PR activity", + "points": 4, + "score": 0, + "passed": false, + "evidence": "Pull request activity not found", + "recommendation": "Link fix PRs to reproducibility issues." + } + ], + "strong_evidence": [ + { + "category": "Environment Reproduction", + "label": "Dependency manifest", + "points": 7, + "score": 7, + "passed": true, + "evidence": "Dependency manifest found", + "recommendation": "Add requirements, pyproject, environment, go.mod, or equivalent." + }, + { + "category": "Project Entry", + "label": "README entry", + "points": 6, + "score": 6, + "passed": true, + "evidence": "README found", + "recommendation": "Add a README with goal and shortest run path." + }, + { + "category": "Release Maturity", + "label": "Release or changelog", + "points": 6, + "score": 6, + "passed": true, + "evidence": "Collected 0 releases or found changelog", + "recommendation": "Add Release Notes or CHANGELOG." + }, + { + "category": "Collaboration Loop", + "label": "Issue data or template", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Collected 0 issues or found template", + "recommendation": "Create a reproducibility issue template for environment, data, and logs." + }, + { + "category": "Collaboration Loop", + "label": "Reproducibility issue signal", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Issue or template includes reproducibility language", + "recommendation": "Tag or title reproducibility, data, and environment issues consistently." + }, + { + "category": "Data and Experiments", + "label": "Data entry", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Data entry found", + "recommendation": "Document data download, license, directory layout, and validation." + }, + { + "category": "Data and Experiments", + "label": "Data preparation", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Data preparation step found", + "recommendation": "Provide data download and preprocessing commands." + }, + { + "category": "Data and Experiments", + "label": "Training or evaluation command", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Training or evaluation entry found", + "recommendation": "Add training, evaluation, or minimal smoke-test commands." + }, + { + "category": "Environment Reproduction", + "label": "Install command", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Install command found", + "recommendation": "Provide copyable install commands in Quickstart." + }, + { + "category": "Environment Reproduction", + "label": "Locked environment", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Container or lock file found", + "recommendation": "Add Dockerfile, lockfile, or devcontainer." + } + ] +} \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/local_checkout_config.json b/examples/workflows/research-reproducibility-assistant/examples/local_checkout_config.json new file mode 100644 index 0000000..6a4a98c --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/local_checkout_config.json @@ -0,0 +1,7 @@ +{ + "owner": "local", + "repo": "current-checkout", + "local_path": "../../../..", + "output_dir": "outputs", + "no_cli": true +} diff --git a/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_issue_draft.md b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_issue_draft.md new file mode 100644 index 0000000..5c1098d --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_issue_draft.md @@ -0,0 +1,15 @@ +### Reproducibility audit recommendations: Gitlink/gitlink-cli + +Current reproducibility score: **53/100** (D - high reproducibility risk). + +Priority improvements: +- Data and Experiments / Data entry: Document data download, license, directory layout, and validation. +- Data and Experiments / Data preparation: Provide data download and preprocessing commands. +- Environment Reproduction / Install command: Provide copyable install commands in Quickstart. +- Project Entry / Goal and scope: State the research problem, method, and applicability boundary near the top of README. +- Project Entry / Paper or dataset metadata: Add paper links, BibTeX, dataset sources, and citation requirements. + +Acceptance suggestions: +- README guides a new contributor through install and smoke test within 10 minutes. +- Data, environment, training/evaluation commands, and result metrics are traceable. +- Reproduction failures can be reported through an issue template with environment, data version, and logs. diff --git a/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_reproducibility_report.md b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_reproducibility_report.md new file mode 100644 index 0000000..463ed14 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_reproducibility_report.md @@ -0,0 +1,110 @@ +# Research Reproducibility Audit: Gitlink/gitlink-cli + +- Generated at: 2026-06-30T08:45:03.998075+00:00 +- Data source: gitlink-cli +- Total score: 53/100 (D - high reproducibility risk) + +## Category Scores + +| Category | Score | Points | +|---|---:|---:| +| Project Entry | 9 | 20 | +| Environment Reproduction | 12 | 20 | +| Data and Experiments | 5 | 20 | +| Collaboration Loop | 14 | 20 | +| Release Maturity | 13 | 20 | + +## Strong Evidence + +- Environment Reproduction / Dependency manifest: Dependency manifest found +- Project Entry / README entry: README found +- Release Maturity / Release or changelog: Collected 12 releases or found changelog +- Collaboration Loop / Issue data or template: Collected 9 issues or found template +- Collaboration Loop / Reproducibility issue signal: Issue or template includes reproducibility language +- Data and Experiments / Training or evaluation command: Training or evaluation entry found +- Environment Reproduction / Locked environment: Container or lock file found +- Collaboration Loop / PR activity: Collected 20 pull requests +- Release Maturity / Version marker: Version or release language found +- Project Entry / License: License file found + +## Priority Improvements + +- [Data and Experiments] Data entry (5 pts): Data entry not found. Recommendation: Document data download, license, directory layout, and validation. +- [Data and Experiments] Data preparation (5 pts): Data preparation step not found. Recommendation: Provide data download and preprocessing commands. +- [Environment Reproduction] Install command (5 pts): Install command not recognized. Recommendation: Provide copyable install commands in Quickstart. +- [Project Entry] Goal and scope (4 pts): Goal or scope language not found. Recommendation: State the research problem, method, and applicability boundary near the top of README. +- [Project Entry] Paper or dataset metadata (4 pts): Paper or dataset metadata not found. Recommendation: Add paper links, BibTeX, dataset sources, and citation requirements. +- [Release Maturity] Experiment change note (4 pts): Experiment change note not found. Recommendation: Mention metric changes and compatibility impact in release notes. +- [Collaboration Loop] Maintainer guidance (3 pts): Maintainer guidance not found. Recommendation: Clarify maintainers, response expectations, or contribution flow. +- [Collaboration Loop] Improvement roadmap (3 pts): Improvement plan not found. Recommendation: Maintain a roadmap or reproducibility improvement checklist. +- [Data and Experiments] Result record (3 pts): Result record not found. Recommendation: Archive key metrics, logs, and result tables. +- [Environment Reproduction] Version and platform (3 pts): Version or platform requirement not found. Recommendation: State Python, Go, CUDA, OS, or hardware requirements. + +## Detailed Checks + +### Project Entry + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| README entry | 6/6 | README found | Add a README with goal and shortest run path. | +| Goal and scope | 0/4 | Goal or scope language not found | State the research problem, method, and applicability boundary near the top of README. | +| License | 3/3 | License file found | Add LICENSE to lower reuse friction. | +| Contribution channel | 0/3 | Contribution channel not found | Add CONTRIBUTING or an issue template with feedback instructions. | +| Paper or dataset metadata | 0/4 | Paper or dataset metadata not found | Add paper links, BibTeX, dataset sources, and citation requirements. | + +### Environment Reproduction + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Dependency manifest | 7/7 | Dependency manifest found | Add requirements, pyproject, environment, go.mod, or equivalent. | +| Install command | 0/5 | Install command not recognized | Provide copyable install commands in Quickstart. | +| Locked environment | 5/5 | Container or lock file found | Add Dockerfile, lockfile, or devcontainer. | +| Version and platform | 0/3 | Version or platform requirement not found | State Python, Go, CUDA, OS, or hardware requirements. | + +### Data and Experiments + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Data entry | 0/5 | Data entry not found | Document data download, license, directory layout, and validation. | +| Data preparation | 0/5 | Data preparation step not found | Provide data download and preprocessing commands. | +| Training or evaluation command | 5/5 | Training or evaluation entry found | Add training, evaluation, or minimal smoke-test commands. | +| Result record | 0/3 | Result record not found | Archive key metrics, logs, and result tables. | +| Randomness control | 0/2 | Randomness control not found | Document seeds, deterministic switches, and hardware variance. | + +### Collaboration Loop + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Issue data or template | 5/5 | Collected 9 issues or found template | Create a reproducibility issue template for environment, data, and logs. | +| Reproducibility issue signal | 5/5 | Issue or template includes reproducibility language | Tag or title reproducibility, data, and environment issues consistently. | +| PR activity | 4/4 | Collected 20 pull requests | Link fix PRs to reproducibility issues. | +| Maintainer guidance | 0/3 | Maintainer guidance not found | Clarify maintainers, response expectations, or contribution flow. | +| Improvement roadmap | 0/3 | Improvement plan not found | Maintain a roadmap or reproducibility improvement checklist. | + +### Release Maturity + +| Check | Score | Evidence | Recommendation | +|---|---:|---|---| +| Release or changelog | 6/6 | Collected 12 releases or found changelog | Add Release Notes or CHANGELOG. | +| Version marker | 4/4 | Version or release language found | Version experiment code, models, and data-processing pipelines. | +| Experiment change note | 0/4 | Experiment change note not found | Mention metric changes and compatibility impact in release notes. | +| Artifact archive | 0/3 | Artifact archive evidence not found | Document where model weights, logs, or artifacts are stored. | +| Auditable output | 3/3 | Auditable output found | Keep demo outputs, command logs, and verification reports. | + +## Dry-run Issue Draft + +### Reproducibility audit recommendations: Gitlink/gitlink-cli + +Current reproducibility score: **53/100** (D - high reproducibility risk). + +Priority improvements: +- Data and Experiments / Data entry: Document data download, license, directory layout, and validation. +- Data and Experiments / Data preparation: Provide data download and preprocessing commands. +- Environment Reproduction / Install command: Provide copyable install commands in Quickstart. +- Project Entry / Goal and scope: State the research problem, method, and applicability boundary near the top of README. +- Project Entry / Paper or dataset metadata: Add paper links, BibTeX, dataset sources, and citation requirements. + +Acceptance suggestions: +- README guides a new contributor through install and smoke test within 10 minutes. +- Data, environment, training/evaluation commands, and result metrics are traceable. +- Reproduction failures can be reported through an issue template with environment, data version, and logs. diff --git a/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_summary.json b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_summary.json new file mode 100644 index 0000000..1f68286 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/Gitlink_gitlink-cli_summary.json @@ -0,0 +1,215 @@ +{ + "generated_at": "2026-06-30T08:45:03.998075+00:00", + "source": "gitlink-cli", + "owner": "Gitlink", + "repo": "gitlink-cli", + "score": 53, + "points": 100, + "grade": "D - high reproducibility risk", + "category_scores": { + "Project Entry": { + "score": 9, + "points": 20 + }, + "Environment Reproduction": { + "score": 12, + "points": 20 + }, + "Data and Experiments": { + "score": 5, + "points": 20 + }, + "Collaboration Loop": { + "score": 14, + "points": 20 + }, + "Release Maturity": { + "score": 13, + "points": 20 + } + }, + "top_missing": [ + { + "category": "Data and Experiments", + "label": "Data entry", + "points": 5, + "score": 0, + "passed": false, + "evidence": "Data entry not found", + "recommendation": "Document data download, license, directory layout, and validation." + }, + { + "category": "Data and Experiments", + "label": "Data preparation", + "points": 5, + "score": 0, + "passed": false, + "evidence": "Data preparation step not found", + "recommendation": "Provide data download and preprocessing commands." + }, + { + "category": "Environment Reproduction", + "label": "Install command", + "points": 5, + "score": 0, + "passed": false, + "evidence": "Install command not recognized", + "recommendation": "Provide copyable install commands in Quickstart." + }, + { + "category": "Project Entry", + "label": "Goal and scope", + "points": 4, + "score": 0, + "passed": false, + "evidence": "Goal or scope language not found", + "recommendation": "State the research problem, method, and applicability boundary near the top of README." + }, + { + "category": "Project Entry", + "label": "Paper or dataset metadata", + "points": 4, + "score": 0, + "passed": false, + "evidence": "Paper or dataset metadata not found", + "recommendation": "Add paper links, BibTeX, dataset sources, and citation requirements." + }, + { + "category": "Release Maturity", + "label": "Experiment change note", + "points": 4, + "score": 0, + "passed": false, + "evidence": "Experiment change note not found", + "recommendation": "Mention metric changes and compatibility impact in release notes." + }, + { + "category": "Collaboration Loop", + "label": "Maintainer guidance", + "points": 3, + "score": 0, + "passed": false, + "evidence": "Maintainer guidance not found", + "recommendation": "Clarify maintainers, response expectations, or contribution flow." + }, + { + "category": "Collaboration Loop", + "label": "Improvement roadmap", + "points": 3, + "score": 0, + "passed": false, + "evidence": "Improvement plan not found", + "recommendation": "Maintain a roadmap or reproducibility improvement checklist." + }, + { + "category": "Data and Experiments", + "label": "Result record", + "points": 3, + "score": 0, + "passed": false, + "evidence": "Result record not found", + "recommendation": "Archive key metrics, logs, and result tables." + }, + { + "category": "Environment Reproduction", + "label": "Version and platform", + "points": 3, + "score": 0, + "passed": false, + "evidence": "Version or platform requirement not found", + "recommendation": "State Python, Go, CUDA, OS, or hardware requirements." + } + ], + "strong_evidence": [ + { + "category": "Environment Reproduction", + "label": "Dependency manifest", + "points": 7, + "score": 7, + "passed": true, + "evidence": "Dependency manifest found", + "recommendation": "Add requirements, pyproject, environment, go.mod, or equivalent." + }, + { + "category": "Project Entry", + "label": "README entry", + "points": 6, + "score": 6, + "passed": true, + "evidence": "README found", + "recommendation": "Add a README with goal and shortest run path." + }, + { + "category": "Release Maturity", + "label": "Release or changelog", + "points": 6, + "score": 6, + "passed": true, + "evidence": "Collected 12 releases or found changelog", + "recommendation": "Add Release Notes or CHANGELOG." + }, + { + "category": "Collaboration Loop", + "label": "Issue data or template", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Collected 9 issues or found template", + "recommendation": "Create a reproducibility issue template for environment, data, and logs." + }, + { + "category": "Collaboration Loop", + "label": "Reproducibility issue signal", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Issue or template includes reproducibility language", + "recommendation": "Tag or title reproducibility, data, and environment issues consistently." + }, + { + "category": "Data and Experiments", + "label": "Training or evaluation command", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Training or evaluation entry found", + "recommendation": "Add training, evaluation, or minimal smoke-test commands." + }, + { + "category": "Environment Reproduction", + "label": "Locked environment", + "points": 5, + "score": 5, + "passed": true, + "evidence": "Container or lock file found", + "recommendation": "Add Dockerfile, lockfile, or devcontainer." + }, + { + "category": "Collaboration Loop", + "label": "PR activity", + "points": 4, + "score": 4, + "passed": true, + "evidence": "Collected 20 pull requests", + "recommendation": "Link fix PRs to reproducibility issues." + }, + { + "category": "Release Maturity", + "label": "Version marker", + "points": 4, + "score": 4, + "passed": true, + "evidence": "Version or release language found", + "recommendation": "Version experiment code, models, and data-processing pipelines." + }, + { + "category": "Project Entry", + "label": "License", + "points": 3, + "score": 3, + "passed": true, + "evidence": "License file found", + "recommendation": "Add LICENSE to lower reuse friction." + } + ] +} \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/command_log.json b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/command_log.json new file mode 100644 index 0000000..230489f --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/real_gitlink_outputs/command_log.json @@ -0,0 +1,224 @@ +[ + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+info", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:03.998075+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"author\": {\n \"id\": 88495,\n \"image_url\": \"images/avatars/Organization/88495?t=1666228706\",\n \"login\": \"Gitlink\",\n \"name\": \"GitLink\",\n \"type\": \"Organization\"\n },\n \"clone_url\": \"https://gitlink.org.cn/Gitlink/gitlink-cli.git\",\n \"contributor_users_count\": 29,\n \"default_branch\": \"master\",\n \"empty\": false,\n \"forked_count\": 39,\n \"forked_from_project_id\": null,\n \"full_name\": \"Gitlink/gitlink-cli\",\n \"identifier\": \"gitlink-cli\",\n \"issues_count\": 19,\n \"mirror\": false,\n \"mirror_url\": null,\n \"name\": \"gitlink-cli\"...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+tree", + "--ref", + "master", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:05.247075+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"entries\": [\n {\n \"commit\": {\n \"created_at\": \"2026-05-31 22:45\",\n \"created_at_unix\": 1780238717,\n \"message\": \"chore: fix CI workflow, golangci-lint config, and minor lint/format issues\\n\\n- Add checkout and setup-go steps to Gitea CI workflow, use make targets\\n- Exclude errcheck for test files in golangci-lint config\\n- Fix staticcheck QF1002 (tagged switch) in repo_test.go\\n- Fix gofmt trailing newline in triage_rules_test.go\\n\\nCo-Authored-By: Claude Opus 4.7 \\u003cnoreply@anthropic.com\\u003e\\n\",\n \"sha\": \"1bf16d3...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+tree", + "--path", + ".github", + "--ref", + "master", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:06.986751+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"entries\": [\n {\n \"commit\": {\n \"created_at\": \"2026-06-01 17:33\",\n \"created_at_unix\": 1780306390,\n \"message\": \"feat(i18n): add CLI localization foundation\\n\",\n \"sha\": \"37dfd2e2e29589f7a2a1d33b7c57ee98fb3f962d\",\n \"time_from_now\": \"29天前\"\n },\n \"content\": null,\n \"direct_download\": true,\n \"download_url\": \"\",\n \"image_type\": false,\n \"is_readme_file\": false,\n \"name\": \"workflows\",\n \"path\": \".github/workflows\",\n \"sha\": \"39a2495c86d2d37c97f1e97a0f1db28726a8...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+tree", + "--path", + "doc", + "--ref", + "master", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:07.912897+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"entries\": [\n {\n \"commit\": {\n \"created_at\": \"2026-06-16 10:24\",\n \"created_at_unix\": 1781576655,\n \"message\": \"feat(shortcut): add shortcuts/wiki\\n\",\n \"sha\": \"2c9a8b6192cd3d660147de94d1ba045f0313525e\",\n \"time_from_now\": \"14天前\"\n },\n \"content\": null,\n \"direct_download\": true,\n \"download_url\": \"\",\n \"image_type\": false,\n \"is_readme_file\": false,\n \"name\": \"changes\",\n \"path\": \"doc/changes\",\n \"sha\": \"1d367fa8b9d0e9e09bb0b0b8b92a137ebf15546d\",\n \"s...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+tree", + "--path", + "docs", + "--ref", + "master", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:10.569371+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"entries\": [\n {\n \"commit\": {\n \"created_at\": \"2026-06-01 17:33\",\n \"created_at_unix\": 1780306390,\n \"message\": \"feat(i18n): add CLI localization foundation\\n\",\n \"sha\": \"37dfd2e2e29589f7a2a1d33b7c57ee98fb3f962d\",\n \"time_from_now\": \"29天前\"\n },\n \"content\": null,\n \"direct_download\": false,\n \"download_url\": \"\",\n \"image_type\": false,\n \"is_readme_file\": 0,\n \"name\": \"i18n.md\",\n \"path\": \"docs/i18n.md\",\n \"replace_content\": \"# GitLink CLI i18n Guide\\n\\n## Goa...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+tree", + "--path", + "examples", + "--ref", + "master", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:12.038021+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"entries\": [\n {\n \"commit\": {\n \"created_at\": \"2026-06-12 10:03\",\n \"created_at_unix\": 1781229832,\n \"message\": \"feat(examples): add pr-quality-gatekeeper end-to-end workflow\\n\\nRunnable reference implementation of the merged gitlink-gatekeeper Skill:\\ncollect -\\u003e route -\\u003e decide (deterministic 0-100 scorecard, three-state verdict)\\n-\\u003e write-back (comment + label + tracking issue, --apply gated, never auto-merge).\\n\\n- scripts/gatekeeper_workflow.py: single-PR gate loop (pure stdlib, py\\u003e=3.9)\\n- scripts/gatek...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "repo", + "+tree", + "--path", + "scripts", + "--ref", + "master", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:13.122853+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"entries\": [\n {\n \"commit\": {\n \"created_at\": \"2026-05-19 16:56\",\n \"created_at_unix\": 1779181010,\n \"message\": \"fix(npm): improve missing binary diagnostics\\n\",\n \"sha\": \"cdca68ff3e5b742cdf17317a3c6da33ce06644ea\",\n \"time_from_now\": \"1个月前\"\n },\n \"content\": null,\n \"direct_download\": false,\n \"download_url\": \"\",\n \"image_type\": false,\n \"is_readme_file\": null,\n \"name\": \"build-npm.sh\",\n \"path\": \"scripts/build-npm.sh\",\n \"sha\": \"dc526301a0565873c946d22fefaf6...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "issue", + "+list", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:14.161614+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"closed_count\": 10,\n \"complete_count\": null,\n \"has_created_issues\": true,\n \"issues\": [\n {\n \"assigners\": [],\n \"author\": {\n \"id\": 87704,\n \"image_url\": \"https://www.gitlink.org.cn/system/lets/letter_avatars/2/T/14_168_39/120.png\",\n \"login\": \"wbtiger\",\n \"name\": \"tigerwang\",\n \"type\": \"User\"\n },\n \"blockchain_token_num\": null,\n \"child_count\": 0,\n \"comment_journals_count\": 3,\n \"created_at\": \"2026-06-14 18:46\",\n \"database_id\": 144255,\n \"due_date\": nul...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "pr", + "+list", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:14.923866+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"pulls\": [\n {\n \"base\": \"master\",\n \"body\": \"## 合并请求描述\\n\\n新增 health diagnose 命令，实现项目健康度综合诊断功能。该功能通过调用 GitLink API 获取项目数据，从5个维度评估项目健康状态，并生成可操作的改进建议。\\n\\n## 相关Issue\\n\\n关联Issue编号：无\\n\\n## 变更内容\\n\\n### 1. 新增 diagnose 命令\\n- 支持项目健康度综合诊断\\n- 支持多种输出格式：文本（默认）、JSON、Markdown\\n- 支持中英文国际化输出\\n\\n### 2. 实现5维度评分算法\\n- 文档完善度（20分）：检查 README、项目描述等\\n- 许可证合规性（15分）：检查开源许可证\\n- 社区活跃度（25分）：评估贡献者、议题数量\\n- 项目成熟度（20分）：评估分支、版本发布、PR 情况\\n- CI/CD 配置（20分）：检查持续集成配置\\n\\n### 3. 新增智能改进建议\\n- 根据诊断结果自动生成改进建议\\n- 针对低分维度提供具体行动建议\\n\\n### 4. 完善国际化支持\\n- 新增 101 个翻译键（中英文）\\n- 所有翻译键符合项目命名规范\\n\\n### 5. 新增...", + "stderr_preview": "" + }, + { + "command": [ + "cmd", + "/c", + "E:\\compete\\gitlink-cli-tools\\node_modules\\.bin\\gitlink-cli.cmd", + "release", + "+list", + "--owner", + "Gitlink", + "--repo", + "gitlink-cli", + "--format", + "json" + ], + "cwd": null, + "returncode": 0, + "started_at": "2026-06-30T08:45:15.899781+00:00", + "stdout_preview": "{\n \"ok\": true,\n \"data\": {\n \"message\": \"响应成功\",\n \"releases\": [\n {\n \"attachments\": [\n {\n \"description\": \"\",\n \"filesize\": \"4.9 MB\",\n \"id\": 483574,\n \"is_pdf\": false,\n \"title\": \"gitlink-cli_0.2.0_darwin_amd64.tar.gz\",\n \"url\": \"/Gitlink/gitlink-cli/releases/download/v0.2.0/gitlink-cli_0.2.0_darwin_amd64.tar.gz\"\n },\n {\n \"description\": \"\",\n \"filesize\": \"4.5 MB\",\n \"id\": 483575,\n \"is_pdf\": false,\n \"title\": \"gitlink-cli_0.2.0_darwin_arm64.t...", + "stderr_preview": "" + } +] \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/examples/sample_config.json b/examples/workflows/research-reproducibility-assistant/examples/sample_config.json new file mode 100644 index 0000000..58618d0 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/examples/sample_config.json @@ -0,0 +1,6 @@ +{ + "owner": "Gitlink", + "repo": "gitlink-cli", + "ref": "master", + "output_dir": "outputs" +} diff --git a/examples/workflows/research-reproducibility-assistant/scripts/research_reproducibility.py b/examples/workflows/research-reproducibility-assistant/scripts/research_reproducibility.py new file mode 100644 index 0000000..f8186e4 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/scripts/research_reproducibility.py @@ -0,0 +1,407 @@ +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +TEXT_EXTENSIONS = {'.md', '.rst', '.txt', '.toml', '.yaml', '.yml', '.json', '.ini', '.cfg', '.py', '.sh', '.ps1', '.go', '.js', '.ts'} +IGNORED_DIRS = {'.git', '.hg', '.svn', '.venv', 'venv', 'env', 'node_modules', '__pycache__', '.mypy_cache', '.pytest_cache', 'dist', 'build', 'vendor'} + +class WorkflowError(RuntimeError): + pass + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description='Audit a GitLink repository for research reproducibility evidence.') + parser.add_argument('--config', type=Path, default=Path('examples/local_checkout_config.json')) + parser.add_argument('--owner') + parser.add_argument('--repo') + parser.add_argument('--ref') + parser.add_argument('--local-path', type=Path) + parser.add_argument('--output-dir', type=Path) + parser.add_argument('--cli-bin') + parser.add_argument('--no-cli', action='store_true') + parser.add_argument('--now') + parser.add_argument('--fail-under', type=int, default=None) + return parser.parse_args(argv) + +def load_config(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + return json.loads(path.read_text(encoding='utf-8')) + +def parse_now(value: str | None) -> datetime: + if not value: + return datetime.now(timezone.utc) + text = value.replace('Z', '+00:00') + try: + dt = datetime.fromisoformat(text) + except ValueError as exc: + raise WorkflowError(f'Cannot parse --now value: {value}') from exc + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + +def first_value(item: dict[str, Any], keys: Iterable[str], default: Any = None) -> Any: + for key in keys: + value = item.get(key) + if value not in (None, '', []): + return value + return default + +def extract_first_list(payload: Any, keys: Iterable[str]) -> list[Any]: + if isinstance(payload, list): + return payload + if isinstance(payload, dict): + for key in keys: + value = payload.get(key) + if isinstance(value, list): + return value + for value in payload.values(): + found = extract_first_list(value, keys) + if found: + return found + return [] + +def extract_first_dict(payload: Any, keys: Iterable[str]) -> dict[str, Any]: + if isinstance(payload, dict): + for key in keys: + value = payload.get(key) + if isinstance(value, dict): + return value + for value in payload.values(): + found = extract_first_dict(value, keys) + if found: + return found + if isinstance(payload, list): + for item in payload: + found = extract_first_dict(item, keys) + if found: + return found + return {} + +def parse_json_output(text: str) -> Any: + stripped = text.strip() + if not stripped: + raise WorkflowError('CLI returned an empty response.') + try: + return json.loads(stripped) + except json.JSONDecodeError: + start = min([idx for idx in (stripped.find('{'), stripped.find('[')) if idx >= 0], default=-1) + if start >= 0: + return json.loads(stripped[start:]) + raise + +def slug(value: str) -> str: + return re.sub(r'[^A-Za-z0-9_.-]+', '_', value).strip('_') or 'repo' + +def command_preview(text: str, limit: int = 600) -> str: + text = text.strip() + return text if len(text) <= limit else text[:limit] + '...' + +def run_process(cmd: list[str], cwd: Path | None, command_log: list[dict[str, Any]]) -> str: + started = datetime.now(timezone.utc) + proc = subprocess.run(cmd, cwd=str(cwd) if cwd else None, capture_output=True, text=True, encoding='utf-8', errors='replace') + command_log.append({'command': cmd, 'cwd': str(cwd) if cwd else None, 'returncode': proc.returncode, 'started_at': started.isoformat(), 'stdout_preview': command_preview(proc.stdout), 'stderr_preview': command_preview(proc.stderr)}) + if proc.returncode != 0: + detail = proc.stderr.strip() or proc.stdout.strip() or 'unknown error' + raise WorkflowError(f"{' '.join(cmd)} failed: {detail}") + return proc.stdout + +def build_cli_command(cli_bin: str, args: list[str]) -> list[str]: + if cli_bin.lower().endswith(('.cmd', '.bat')): + return ['cmd', '/c', cli_bin, *args] + return [cli_bin, *args] + +def collect_gitlink(config: dict[str, Any], command_log: list[dict[str, Any]]) -> dict[str, Any]: + owner = config.get('owner') + repo = config.get('repo') + if not owner or not repo: + raise WorkflowError('GitLink mode requires owner and repo.') + cli_bin = config.get('cli_bin') or shutil.which('gitlink-cli') + if not cli_bin: + raise WorkflowError('gitlink-cli was not found. Use --cli-bin or --local-path.') + ref = config.get('ref') or 'master' + def run_json(args: list[str]) -> Any: + cmd = build_cli_command(cli_bin, [*args, '--owner', str(owner), '--repo', str(repo), '--format', 'json']) + return parse_json_output(run_process(cmd, cwd=None, command_log=command_log)) + payloads: dict[str, Any] = {} + payloads['repo_info'] = run_json(['repo', '+info']) + payloads['tree_root'] = run_json(['repo', '+tree', '--ref', str(ref)]) + for item in normalize_tree_entries(payloads['tree_root']): + path = item.get('path') or item.get('name') + kind = str(item.get('type') or item.get('kind') or '').lower() + if kind in {'dir', 'directory', 'tree'} and path in {'docs', 'doc', 'examples', 'scripts', 'data', 'results', '.github'}: + try: + payloads[f'tree_{path}'] = run_json(['repo', '+tree', '--path', str(path), '--ref', str(ref)]) + except WorkflowError as exc: + command_log.append({'warning': f'Skipped tree path {path}: {exc}'}) + for key, args in {'issues': ['issue', '+list'], 'prs': ['pr', '+list'], 'releases': ['release', '+list']}.items(): + try: + payloads[key] = run_json(args) + except WorkflowError as exc: + command_log.append({'warning': f'Skipped {key}: {exc}'}) + payloads[key] = [] + paths: list[str] = [] + for key, payload in payloads.items(): + if key.startswith('tree_'): + paths.extend(item['path'] for item in normalize_tree_entries(payload) if item.get('path')) + return {'source': 'gitlink-cli', 'owner': owner, 'repo': repo, 'repo_info': normalize_repo_info(payloads.get('repo_info', {})), 'paths': sorted(set(paths)), 'text_blobs': [], 'issues': normalize_items(payloads.get('issues'), ('issues', 'data', 'list')), 'prs': normalize_items(payloads.get('prs'), ('pull_requests', 'pulls', 'prs', 'data', 'list')), 'releases': normalize_items(payloads.get('releases'), ('releases', 'versions', 'data', 'list'))} + +def normalize_repo_info(payload: Any) -> dict[str, Any]: + repo = extract_first_dict(payload, ('project', 'repo', 'repository', 'data')) + if not repo and isinstance(payload, dict): + repo = payload + return {'name': first_value(repo, ('name', 'identifier', 'repo')), 'description': first_value(repo, ('description', 'summary'), ''), 'default_branch': first_value(repo, ('default_branch', 'default_branch_name'), ''), 'language': first_value(repo, ('language', 'main_language'), ''), 'updated_at': first_value(repo, ('updated_at', 'last_update_time', 'full_last_update_time'), ''), 'watchers_count': first_value(repo, ('watchers_count', 'praises_count'), 0), 'forked_count': first_value(repo, ('forked_count', 'forks_count'), 0), 'pull_requests_count': first_value(repo, ('pull_requests_count', 'pr_count'), 0), 'version_releases_count': first_value(repo, ('version_releases_count', 'releases_count'), 0), 'mirror': bool(repo.get('mirror', False))} + +def normalize_tree_entries(payload: Any) -> list[dict[str, Any]]: + entries = normalize_items(payload, ('entries', 'tree', 'files', 'data', 'list')) + normalized: list[dict[str, Any]] = [] + for item in entries: + name = first_value(item, ('name', 'filename', 'title'), '') + path = first_value(item, ('path', 'full_path', 'filepath'), name) + normalized.append({'name': str(name), 'path': str(path).replace('\\', '/'), 'type': first_value(item, ('type', 'kind', 'mode'), '')}) + return normalized + +def normalize_items(payload: Any, keys: Iterable[str]) -> list[dict[str, Any]]: + raw = extract_first_list(payload, keys) + return [item for item in raw if isinstance(item, dict)] + +def collect_local(config: dict[str, Any], command_log: list[dict[str, Any]]) -> dict[str, Any]: + local_path = Path(config.get('local_path') or '.').expanduser().resolve() + if not local_path.exists(): + raise WorkflowError(f'Local path does not exist: {local_path}') + paths: list[str] = [] + text_blobs: list[dict[str, str]] = [] + for root, dirs, files in os.walk(local_path): + dirs[:] = [d for d in dirs if d not in IGNORED_DIRS] + root_path = Path(root) + for filename in files: + file_path = root_path / filename + rel = file_path.relative_to(local_path).as_posix() + paths.append(rel) + if file_path.suffix.lower() not in TEXT_EXTENSIONS or file_path.stat().st_size > 512_000: + continue + try: + text = file_path.read_text(encoding='utf-8', errors='replace')[:24_000] + except OSError: + continue + text_blobs.append({'path': rel, 'text': text}) + repo_info = {'name': local_path.name, 'description': '', 'default_branch': git_value(['branch', '--show-current'], local_path, command_log), 'language': '', 'updated_at': git_value(['log', '-1', '--format=%cI'], local_path, command_log), 'watchers_count': 0, 'forked_count': 0, 'pull_requests_count': 0, 'version_releases_count': 0, 'mirror': False} + return {'source': 'local-checkout', 'owner': config.get('owner') or 'local', 'repo': config.get('repo') or local_path.name, 'repo_info': repo_info, 'paths': sorted(set(paths)), 'text_blobs': text_blobs, 'issues': [], 'prs': [], 'releases': [], 'local_path': str(local_path)} + +def git_value(args: list[str], cwd: Path, command_log: list[dict[str, Any]]) -> str: + if not (cwd / '.git').exists(): + return '' + git_bin = shutil.which('git') or 'git' + try: + return run_process([git_bin, *args], cwd=cwd, command_log=command_log).strip() + except WorkflowError: + return '' + +def lower_paths(paths: Iterable[str]) -> list[str]: + return [path.replace('\\', '/').lower() for path in paths] + +def has_exact(paths: list[str], names: Iterable[str]) -> bool: + wanted = {name.lower() for name in names} + return any(Path(path).name.lower() in wanted for path in paths) + +def has_prefix(paths: list[str], prefixes: Iterable[str]) -> bool: + lowered = tuple(prefix.lower().rstrip('/') + '/' for prefix in prefixes) + return any(path.startswith(lowered) for path in paths) + +def has_contains(paths: list[str], needles: Iterable[str]) -> bool: + lowered = [needle.lower() for needle in needles] + return any(any(needle in path for needle in lowered) for path in paths) + +def all_text(blobs: list[dict[str, str]], limit: int = 400_000) -> str: + parts: list[str] = [] + total = 0 + for blob in blobs: + text = blob['text'] + total += len(text) + if total > limit: + break + parts.append(text) + return '\n'.join(parts).lower() + +def text_has(text: str, terms: Iterable[str]) -> bool: + return any(term.lower() in text for term in terms) + +def item_text(items: list[dict[str, Any]]) -> str: + fields: list[str] = [] + for item in items: + for key in ('title', 'subject', 'name', 'description', 'body', 'content'): + value = item.get(key) + if value: + fields.append(str(value)) + labels = item.get('labels') or item.get('tags') + if isinstance(labels, list): + fields.extend(str(label.get('name', label)) if isinstance(label, dict) else str(label) for label in labels) + return '\n'.join(fields).lower() + +def check(label: str, points: int, passed: bool, evidence: str, recommendation: str) -> dict[str, Any]: + return {'label': label, 'points': points, 'score': points if passed else 0, 'passed': passed, 'evidence': evidence, 'recommendation': recommendation} + +def score_dataset(data: dict[str, Any]) -> dict[str, Any]: + paths = lower_paths(data['paths']) + text = all_text(data.get('text_blobs', [])) + issues = data.get('issues', []) + prs = data.get('prs', []) + releases = data.get('releases', []) + issue_text = item_text(issues) + pr_text = item_text(prs) + release_text = item_text(releases) + readme = has_exact(paths, ('readme.md', 'readme.rst', 'readme.txt')) + license_file = has_exact(paths, ('license', 'license.md', 'license.txt', 'mulanpsl-2.0.txt')) + dependency_file = has_exact(paths, ('requirements.txt', 'pyproject.toml', 'environment.yml', 'environment.yaml', 'conda.yml', 'go.mod', 'package.json', 'pom.xml', 'cargo.toml', 'renv.lock')) + lock_or_container = has_exact(paths, ('dockerfile', 'docker-compose.yml', 'poetry.lock', 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'go.sum', 'conda-lock.yml')) or has_prefix(paths, ('.devcontainer',)) + issue_templates = has_prefix(paths, ('.github/issue_template', '.gitea/issue_template')) or has_contains(paths, ('issue_template',)) + changelog = has_exact(paths, ('changelog.md', 'history.md', 'changes.md', 'release.md', 'release-notes.md')) + objective_terms = ('overview', 'background', 'goal', 'purpose', 'introduction', 'method', 'scope') + install_terms = ('pip install', 'npm install', 'go install', 'go mod download', 'conda env', 'make install', 'setup') + version_terms = ('python 3', 'go 1.', 'node ', 'cuda', 'ubuntu', 'windows', 'macos', 'version') + data_terms = ('dataset', 'data/', 'download', 'license', 'benchmark', 'sample data') + prep_terms = ('preprocess', 'prepare', 'clean', 'split', 'transform') + train_terms = ('train.py', 'eval.py', 'evaluate', 'benchmark', 'pytest', 'make test', 'go test', 'smoke test') + result_terms = ('result', 'accuracy', 'metric', 'report', 'table', 'log') + seed_terms = ('seed', 'deterministic', 'random_state', 'reproducible') + release_terms = ('release', 'changelog', 'version', 'v1.', 'v0.', 'tag') + artifact_terms = ('artifact', 'checkpoint', 'model', 'weights', 'outputs', 'archive') + categories = { + 'Project Entry': [ + check('README entry', 6, readme, 'README found' if readme else 'README not found', 'Add a README with goal and shortest run path.'), + check('Goal and scope', 4, text_has(text, objective_terms), 'Goal or scope language found' if text_has(text, objective_terms) else 'Goal or scope language not found', 'State the research problem, method, and applicability boundary near the top of README.'), + check('License', 3, license_file, 'License file found' if license_file else 'License file not found', 'Add LICENSE to lower reuse friction.'), + check('Contribution channel', 3, has_contains(paths, ('contributing', 'code_of_conduct', 'contact', 'issue_template')), 'Contribution or contact path found' if has_contains(paths, ('contributing', 'code_of_conduct', 'contact', 'issue_template')) else 'Contribution channel not found', 'Add CONTRIBUTING or an issue template with feedback instructions.'), + check('Paper or dataset metadata', 4, text_has(text, ('arxiv', 'doi', 'paper', 'dataset', 'citation', 'bibtex')), 'Paper or dataset metadata found' if text_has(text, ('arxiv', 'doi', 'paper', 'dataset', 'citation', 'bibtex')) else 'Paper or dataset metadata not found', 'Add paper links, BibTeX, dataset sources, and citation requirements.'), + ], + 'Environment Reproduction': [ + check('Dependency manifest', 7, dependency_file, 'Dependency manifest found' if dependency_file else 'Dependency manifest not found', 'Add requirements, pyproject, environment, go.mod, or equivalent.'), + check('Install command', 5, text_has(text, install_terms), 'Install command found' if text_has(text, install_terms) else 'Install command not recognized', 'Provide copyable install commands in Quickstart.'), + check('Locked environment', 5, lock_or_container, 'Container or lock file found' if lock_or_container else 'Container or lock file not found', 'Add Dockerfile, lockfile, or devcontainer.'), + check('Version and platform', 3, text_has(text, version_terms), 'Version or platform requirement found' if text_has(text, version_terms) else 'Version or platform requirement not found', 'State Python, Go, CUDA, OS, or hardware requirements.'), + ], + 'Data and Experiments': [ + check('Data entry', 5, has_prefix(paths, ('data', 'datasets')) or text_has(text, data_terms), 'Data entry found' if has_prefix(paths, ('data', 'datasets')) or text_has(text, data_terms) else 'Data entry not found', 'Document data download, license, directory layout, and validation.'), + check('Data preparation', 5, text_has(text, prep_terms) or has_contains(paths, ('prepare', 'preprocess')), 'Data preparation step found' if text_has(text, prep_terms) or has_contains(paths, ('prepare', 'preprocess')) else 'Data preparation step not found', 'Provide data download and preprocessing commands.'), + check('Training or evaluation command', 5, text_has(text, train_terms) or has_contains(paths, ('train', 'eval', 'test', 'benchmark')), 'Training or evaluation entry found' if text_has(text, train_terms) or has_contains(paths, ('train', 'eval', 'test', 'benchmark')) else 'Training or evaluation entry not found', 'Add training, evaluation, or minimal smoke-test commands.'), + check('Result record', 3, has_prefix(paths, ('results', 'reports')) or text_has(text, result_terms), 'Result record found' if has_prefix(paths, ('results', 'reports')) or text_has(text, result_terms) else 'Result record not found', 'Archive key metrics, logs, and result tables.'), + check('Randomness control', 2, text_has(text, seed_terms), 'Randomness control found' if text_has(text, seed_terms) else 'Randomness control not found', 'Document seeds, deterministic switches, and hardware variance.'), + ], + 'Collaboration Loop': [ + check('Issue data or template', 5, bool(issues) or issue_templates, f'Collected {len(issues)} issues or found template' if bool(issues) or issue_templates else 'Issue evidence not found', 'Create a reproducibility issue template for environment, data, and logs.'), + check('Reproducibility issue signal', 5, text_has(issue_text, ('reproduc', 'dataset', 'environment', 'bug', 'log')) or issue_templates, 'Issue or template includes reproducibility language' if text_has(issue_text, ('reproduc', 'dataset', 'environment', 'bug', 'log')) or issue_templates else 'Reproducibility issue language not found', 'Tag or title reproducibility, data, and environment issues consistently.'), + check('PR activity', 4, bool(prs), f'Collected {len(prs)} pull requests' if prs else 'Pull request activity not found', 'Link fix PRs to reproducibility issues.'), + check('Maintainer guidance', 3, has_contains(paths, ('maintainers', 'codeowners')) or text_has(text, ('maintainer', 'owner', 'sla')), 'Maintainer guidance found' if has_contains(paths, ('maintainers', 'codeowners')) or text_has(text, ('maintainer', 'owner', 'sla')) else 'Maintainer guidance not found', 'Clarify maintainers, response expectations, or contribution flow.'), + check('Improvement roadmap', 3, text_has(text, ('roadmap', 'todo', 'plan', 'next step')) or text_has(pr_text, ('roadmap', 'todo', 'plan', 'next')), 'Improvement plan found' if text_has(text, ('roadmap', 'todo', 'plan', 'next step')) or text_has(pr_text, ('roadmap', 'todo', 'plan', 'next')) else 'Improvement plan not found', 'Maintain a roadmap or reproducibility improvement checklist.'), + ], + 'Release Maturity': [ + check('Release or changelog', 6, bool(releases) or changelog, f'Collected {len(releases)} releases or found changelog' if bool(releases) or changelog else 'Release evidence not found', 'Add Release Notes or CHANGELOG.'), + check('Version marker', 4, text_has(release_text + '\n' + text, release_terms), 'Version or release language found' if text_has(release_text + '\n' + text, release_terms) else 'Version marker not found', 'Version experiment code, models, and data-processing pipelines.'), + check('Experiment change note', 4, text_has(release_text + '\n' + text, ('metric', 'benchmark', 'accuracy', 'experiment', 'compatibility')), 'Experiment change note found' if text_has(release_text + '\n' + text, ('metric', 'benchmark', 'accuracy', 'experiment', 'compatibility')) else 'Experiment change note not found', 'Mention metric changes and compatibility impact in release notes.'), + check('Artifact archive', 3, text_has(text, artifact_terms) or has_prefix(paths, ('outputs', 'artifacts', 'models')), 'Artifact archive evidence found' if text_has(text, artifact_terms) or has_prefix(paths, ('outputs', 'artifacts', 'models')) else 'Artifact archive evidence not found', 'Document where model weights, logs, or artifacts are stored.'), + check('Auditable output', 3, has_prefix(paths, ('examples', 'reports')) or text_has(text, ('demo', 'command log', 'audit', 'report')), 'Auditable output found' if has_prefix(paths, ('examples', 'reports')) or text_has(text, ('demo', 'command log', 'audit', 'report')) else 'Auditable output not found', 'Keep demo outputs, command logs, and verification reports.'), + ], + } + category_scores = {name: {'score': sum(item['score'] for item in checks), 'points': sum(item['points'] for item in checks)} for name, checks in categories.items()} + score = sum(item['score'] for checks in categories.values() for item in checks) + points = sum(item['points'] for checks in categories.values() for item in checks) + missing = [{'category': name, **item} for name, checks in categories.items() for item in checks if not item['passed']] + strong = [{'category': name, **item} for name, checks in categories.items() for item in checks if item['passed']] + return {'score': score, 'points': points, 'grade': grade(score), 'category_scores': category_scores, 'categories': categories, 'top_missing': sorted(missing, key=lambda item: (-item['points'], item['category']))[:10], 'strong_evidence': sorted(strong, key=lambda item: (-item['points'], item['category']))[:10]} + +def grade(score: int) -> str: + if score >= 90: + return 'A - strong reproducibility evidence' + if score >= 75: + return 'B - mostly reproducible' + if score >= 60: + return 'C - reproducible foundation, more evidence needed' + return 'D - high reproducibility risk' + +def write_outputs(data: dict[str, Any], scoring: dict[str, Any], command_log: list[dict[str, Any]], output_dir: Path, now: datetime) -> dict[str, str]: + output_dir.mkdir(parents=True, exist_ok=True) + prefix = f"{slug(data['owner'])}_{slug(data['repo'])}" + report_path = output_dir / f'{prefix}_reproducibility_report.md' + summary_path = output_dir / f'{prefix}_summary.json' + issue_path = output_dir / f'{prefix}_issue_draft.md' + command_log_path = output_dir / 'command_log.json' + report_path.write_text(render_report(data, scoring, now), encoding='utf-8') + issue_path.write_text(render_issue(data, scoring), encoding='utf-8') + summary = {'generated_at': now.isoformat(), 'source': data['source'], 'owner': data['owner'], 'repo': data['repo'], 'score': scoring['score'], 'points': scoring['points'], 'grade': scoring['grade'], 'category_scores': scoring['category_scores'], 'top_missing': scoring['top_missing'], 'strong_evidence': scoring['strong_evidence']} + summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8') + command_log_path.write_text(json.dumps(command_log, ensure_ascii=False, indent=2), encoding='utf-8') + return {'report': str(report_path), 'summary': str(summary_path), 'issue_draft': str(issue_path), 'command_log': str(command_log_path)} + +def render_report(data: dict[str, Any], scoring: dict[str, Any], now: datetime) -> str: + lines = [f"# Research Reproducibility Audit: {data['owner']}/{data['repo']}", '', f"- Generated at: {now.isoformat()}", f"- Data source: {data['source']}", f"- Total score: {scoring['score']}/{scoring['points']} ({scoring['grade']})", '', '## Category Scores', '', '| Category | Score | Points |', '|---|---:|---:|'] + for name, value in scoring['category_scores'].items(): + lines.append(f"| {name} | {value['score']} | {value['points']} |") + lines.extend(['', '## Strong Evidence', '']) + for item in scoring['strong_evidence']: + lines.append(f"- {item['category']} / {item['label']}: {item['evidence']}") + lines.extend(['', '## Priority Improvements', '']) + for item in scoring['top_missing']: + lines.append(f"- [{item['category']}] {item['label']} ({item['points']} pts): {item['evidence']}. Recommendation: {item['recommendation']}") + lines.extend(['', '## Detailed Checks', '']) + for name, checks in scoring['categories'].items(): + lines.extend([f'### {name}', '', '| Check | Score | Evidence | Recommendation |', '|---|---:|---|---|']) + for item in checks: + lines.append(f"| {item['label']} | {item['score']}/{item['points']} | {item['evidence']} | {item['recommendation']} |") + lines.append('') + lines.extend(['## Dry-run Issue Draft', '', render_issue(data, scoring)]) + return '\n'.join(lines).rstrip() + '\n' + +def render_issue(data: dict[str, Any], scoring: dict[str, Any]) -> str: + lines = [f"### Reproducibility audit recommendations: {data['owner']}/{data['repo']}", '', f"Current reproducibility score: **{scoring['score']}/{scoring['points']}** ({scoring['grade']}).", '', 'Priority improvements:'] + for item in scoring['top_missing'][:5]: + lines.append(f"- {item['category']} / {item['label']}: {item['recommendation']}") + lines.extend(['', 'Acceptance suggestions:', '- README guides a new contributor through install and smoke test within 10 minutes.', '- Data, environment, training/evaluation commands, and result metrics are traceable.', '- Reproduction failures can be reported through an issue template with environment, data version, and logs.']) + return '\n'.join(lines).rstrip() + '\n' + +def merge_config(args: argparse.Namespace) -> dict[str, Any]: + config = load_config(args.config) + if args.owner: + config['owner'] = args.owner + if args.repo: + config['repo'] = args.repo + if args.ref: + config['ref'] = args.ref + if args.local_path: + config['local_path'] = str(args.local_path) + if args.output_dir: + config['output_dir'] = str(args.output_dir) + if args.cli_bin: + config['cli_bin'] = args.cli_bin + return config + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + command_log: list[dict[str, Any]] = [] + try: + config = merge_config(args) + now = parse_now(args.now) + output_dir = Path(config.get('output_dir') or 'outputs') + data = collect_local(config, command_log) if args.no_cli or config.get('local_path') else collect_gitlink(config, command_log) + scoring = score_dataset(data) + outputs = write_outputs(data, scoring, command_log, output_dir, now) + print(f"score={scoring['score']}/{scoring['points']} grade={scoring['grade']}") + for key, path in outputs.items(): + print(f'{key}={path}') + if args.fail_under is not None and scoring['score'] < args.fail_under: + print(f"ERROR: score {scoring['score']} is below --fail-under {args.fail_under}", file=sys.stderr) + return 1 + return 0 + except WorkflowError as exc: + print(f'ERROR: {exc}', file=sys.stderr) + return 2 + +if __name__ == '__main__': + raise SystemExit(main()) \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/scripts/run_demo.ps1 b/examples/workflows/research-reproducibility-assistant/scripts/run_demo.ps1 new file mode 100644 index 0000000..1eb6053 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/scripts/run_demo.ps1 @@ -0,0 +1,42 @@ +param( + [string]$Config = "examples/local_checkout_config.json", + [string]$OutputDir = "outputs", + [string]$LocalPath = "", + [string]$Python = "python", + [int]$FailUnder = 0 +) + +$ErrorActionPreference = "Stop" +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$WorkflowRoot = Resolve-Path (Join-Path $ScriptDir "..") +Set-Location $WorkflowRoot + +$argsList = @( + ".\scripts\research_reproducibility.py", + "--config", $Config, + "--output-dir", $OutputDir +) + +if ($LocalPath -ne "") { + $argsList += @("--local-path", $LocalPath) +} + +if ($FailUnder -gt 0) { + $argsList += @("--fail-under", [string]$FailUnder) +} + +& $Python @argsList +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + +$summaryFiles = @(Get-ChildItem -LiteralPath $OutputDir -Filter "*_summary.json" -File) +$reportFiles = @(Get-ChildItem -LiteralPath $OutputDir -Filter "*_reproducibility_report.md" -File) +$issueFiles = @(Get-ChildItem -LiteralPath $OutputDir -Filter "*_issue_draft.md" -File) +$commandLog = Join-Path $OutputDir "command_log.json" + +if ($summaryFiles.Count -lt 1 -or $reportFiles.Count -lt 1 -or $issueFiles.Count -lt 1 -or -not (Test-Path -LiteralPath $commandLog)) { + throw "Workflow output validation failed: expected summary, report, issue draft, and command_log.json in $OutputDir" +} + +Write-Host "Workflow output validation passed: $OutputDir" diff --git a/examples/workflows/research-reproducibility-assistant/scripts/run_demo.sh b/examples/workflows/research-reproducibility-assistant/scripts/run_demo.sh new file mode 100644 index 0000000..45f6b07 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/scripts/run_demo.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env sh +set -eu + +CONFIG="examples/local_checkout_config.json" +OUTPUT_DIR="outputs" +LOCAL_PATH="" +PYTHON_BIN="${PYTHON:-python3}" +FAIL_UNDER="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --config) + CONFIG="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --local-path) + LOCAL_PATH="$2" + shift 2 + ;; + --python) + PYTHON_BIN="$2" + shift 2 + ;; + --fail-under) + FAIL_UNDER="$2" + shift 2 + ;; + -h|--help) + cat <<'USAGE' +Usage: ./scripts/run_demo.sh [--config FILE] [--output-dir DIR] [--local-path DIR] [--python BIN] [--fail-under N] + +Runs the research reproducibility workflow and verifies that all review artifacts +were produced: report, summary JSON, dry-run issue draft, and command log. +USAGE + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 2 + ;; + esac +done + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +WORKFLOW_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +cd "$WORKFLOW_ROOT" + +set -- "$PYTHON_BIN" "./scripts/research_reproducibility.py" "--config" "$CONFIG" "--output-dir" "$OUTPUT_DIR" + +if [ -n "$LOCAL_PATH" ]; then + set -- "$@" "--local-path" "$LOCAL_PATH" +fi + +if [ -n "$FAIL_UNDER" ]; then + set -- "$@" "--fail-under" "$FAIL_UNDER" +fi + +"$@" + +summary_count=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -name '*_summary.json' 2>/dev/null | wc -l | tr -d ' ') +report_count=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -name '*_reproducibility_report.md' 2>/dev/null | wc -l | tr -d ' ') +issue_count=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -name '*_issue_draft.md' 2>/dev/null | wc -l | tr -d ' ') + +if [ "$summary_count" -lt 1 ] || [ "$report_count" -lt 1 ] || [ "$issue_count" -lt 1 ] || [ ! -f "$OUTPUT_DIR/command_log.json" ]; then + echo "Workflow output validation failed: expected summary, report, issue draft, and command_log.json in $OUTPUT_DIR" >&2 + exit 1 +fi + +echo "Workflow output validation passed: $OUTPUT_DIR" diff --git a/examples/workflows/research-reproducibility-assistant/tests/test_high_evidence_fixture.py b/examples/workflows/research-reproducibility-assistant/tests/test_high_evidence_fixture.py new file mode 100644 index 0000000..1857802 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/tests/test_high_evidence_fixture.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +WORKFLOW_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = WORKFLOW_ROOT / "scripts" / "research_reproducibility.py" +SPEC = importlib.util.spec_from_file_location("research_reproducibility", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_high_evidence_fixture_scores_as_strong_reproducibility_repo() -> None: + fixture = WORKFLOW_ROOT / "examples" / "fixtures" / "high-evidence-repo" + + command_log = [] + data = MODULE.collect_local( + { + "local_path": str(fixture), + "owner": "fixture", + "repo": "high-evidence-repo", + }, + command_log, + ) + scoring = MODULE.score_dataset(data) + + assert scoring["score"] >= 90 + assert scoring["category_scores"]["Project Entry"]["score"] == 20 + assert scoring["category_scores"]["Environment Reproduction"]["score"] == 20 + assert scoring["category_scores"]["Data and Experiments"]["score"] == 20 + assert scoring["category_scores"]["Collaboration Loop"]["score"] >= 16 + assert scoring["category_scores"]["Release Maturity"]["score"] == 20 + + +def test_cli_relative_local_path_uses_current_working_directory(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(WORKFLOW_ROOT) + output_dir = tmp_path / "outputs" + + exit_code = MODULE.main( + [ + "--local-path", + ".\\examples\\fixtures\\high-evidence-repo", + "--owner", + "fixture", + "--repo", + "high-evidence-repo", + "--output-dir", + str(output_dir), + "--fail-under", + "90", + "--now", + "2026-06-30T00:00:00Z", + ] + ) + + assert exit_code == 0 + assert (output_dir / "fixture_high-evidence-repo_summary.json").exists() \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/tests/test_output_artifacts.py b/examples/workflows/research-reproducibility-assistant/tests/test_output_artifacts.py new file mode 100644 index 0000000..80359d5 --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/tests/test_output_artifacts.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "research_reproducibility.py" +SPEC = importlib.util.spec_from_file_location("research_reproducibility", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_write_outputs_creates_review_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "README.md").write_text("# Tiny\n", encoding="utf-8") + + command_log = [] + data = MODULE.collect_local({"local_path": str(repo), "owner": "local", "repo": "tiny"}, command_log) + scoring = MODULE.score_dataset(data) + output_dir = tmp_path / "outputs" + + paths = MODULE.write_outputs( + data, + scoring, + command_log, + output_dir, + MODULE.parse_now("2026-06-29T00:00:00Z"), + ) + + assert set(paths) == {"report", "summary", "issue_draft", "command_log"} + for path in paths.values(): + assert Path(path).exists() + + report = Path(paths["report"]).read_text(encoding="utf-8") + issue_draft = Path(paths["issue_draft"]).read_text(encoding="utf-8") + summary = MODULE.json.loads(Path(paths["summary"]).read_text(encoding="utf-8")) + + assert "Research Reproducibility Audit" in report + assert "Reproducibility audit recommendations" in issue_draft + assert summary["score"] == scoring["score"] + assert summary["top_missing"] \ No newline at end of file diff --git a/examples/workflows/research-reproducibility-assistant/tests/test_research_reproducibility.py b/examples/workflows/research-reproducibility-assistant/tests/test_research_reproducibility.py new file mode 100644 index 0000000..17140bf --- /dev/null +++ b/examples/workflows/research-reproducibility-assistant/tests/test_research_reproducibility.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "research_reproducibility.py" +SPEC = importlib.util.spec_from_file_location("research_reproducibility", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_local_scoring_rewards_reproducibility_evidence(tmp_path: Path) -> None: + (tmp_path / "README.md").write_text( + """ +# Repro Repo + +Overview: paper artifact for a benchmark dataset. + +Install: + +```bash +pip install -r requirements.txt +python train.py --seed 7 +python eval.py +``` + +Dataset download and preprocess steps are documented. Results report Accuracy. +""", + encoding="utf-8", + ) + (tmp_path / "requirements.txt").write_text("numpy==1.26.0\n", encoding="utf-8") + (tmp_path / "Dockerfile").write_text("FROM python:3.11\n", encoding="utf-8") + (tmp_path / "LICENSE").write_text("Apache-2.0\n", encoding="utf-8") + (tmp_path / "CHANGELOG.md").write_text("v1.0 release benchmark metric update\n", encoding="utf-8") + (tmp_path / "data").mkdir() + (tmp_path / "results").mkdir() + (tmp_path / "train.py").write_text("print('train')\n", encoding="utf-8") + (tmp_path / ".github" / "ISSUE_TEMPLATE").mkdir(parents=True) + (tmp_path / ".github" / "ISSUE_TEMPLATE" / "reproduce.md").write_text( + "reproducibility environment dataset logs", + encoding="utf-8", + ) + + command_log = [] + data = MODULE.collect_local({"local_path": str(tmp_path), "owner": "local", "repo": "demo"}, command_log) + scoring = MODULE.score_dataset(data) + + assert scoring["score"] >= 70 + assert scoring["category_scores"]["Environment Reproduction"]["score"] >= 15 + assert scoring["category_scores"]["Collaboration Loop"]["score"] >= 10 + + +def test_missing_repo_produces_action_items(tmp_path: Path) -> None: + (tmp_path / "README.md").write_text("# Tiny\n", encoding="utf-8") + + command_log = [] + data = MODULE.collect_local({"local_path": str(tmp_path), "owner": "local", "repo": "tiny"}, command_log) + scoring = MODULE.score_dataset(data) + + assert scoring["score"] < 50 + assert scoring["top_missing"] + assert any(item["label"] == "Dependency manifest" for item in scoring["top_missing"]) + + +def test_fail_under_returns_nonzero_for_low_score(tmp_path: Path, capsys) -> None: + (tmp_path / "README.md").write_text("# Tiny\n", encoding="utf-8") + output_dir = tmp_path / "outputs" + + exit_code = MODULE.main( + [ + "--local-path", + str(tmp_path), + "--owner", + "local", + "--repo", + "tiny", + "--output-dir", + str(output_dir), + "--fail-under", + "80", + "--now", + "2026-06-29T00:00:00Z", + ] + ) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "below --fail-under 80" in captured.err + assert (output_dir / "local_tiny_summary.json").exists() \ No newline at end of file -- 2.34.1