106 lines
3.3 KiB
Python
106 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""生成 Benchmark 报告。
|
|
|
|
用法:
|
|
python benchmark_report.py --iteration 1
|
|
python benchmark_report.py --skill product-design-module
|
|
python benchmark_report.py --all
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import click
|
|
|
|
from eval_framework.config import PathConfig, get_path_config
|
|
from eval_framework.benchmark import aggregate_benchmarks, generate_benchmark_md, compute_aggregate_stats
|
|
|
|
|
|
@click.command()
|
|
@click.option("--skill", "skill_name", default=None, help="指定 skill 名称")
|
|
@click.option("--all", "report_all", is_flag=True, default=False, help="生成所有 skill 的报告")
|
|
@click.option("--iteration", default=None, type=int, help="指定迭代编号")
|
|
@click.option("--output-dir", default=None, help="评估结果输出目录")
|
|
def main(
|
|
skill_name: str | None,
|
|
report_all: bool,
|
|
iteration: int | None,
|
|
output_dir: str | None,
|
|
):
|
|
"""生成 Benchmark 报告。"""
|
|
if output_dir:
|
|
path_config = PathConfig(output_dir=Path(output_dir))
|
|
else:
|
|
path_config = get_path_config()
|
|
|
|
output_base = path_config.output_dir
|
|
|
|
if not output_base.exists():
|
|
click.echo(f"输出目录不存在: {output_base}")
|
|
click.echo("请先运行评估: python run_eval.py --skill <name>")
|
|
return
|
|
|
|
# 确定要报告的 skill
|
|
if report_all:
|
|
skill_dirs = [d for d in output_base.iterdir() if d.is_dir()]
|
|
elif skill_name:
|
|
skill_dirs = [output_base / skill_name]
|
|
if not skill_dirs[0].exists():
|
|
click.echo(f"Skill '{skill_name}' 的输出目录不存在: {skill_dirs[0]}")
|
|
return
|
|
else:
|
|
click.echo("请指定 --skill <名称> 或 --all")
|
|
return
|
|
|
|
for skill_dir in skill_dirs:
|
|
skill = skill_dir.name
|
|
click.echo(f"\n生成报告: {skill}")
|
|
|
|
# 聚合 benchmark 数据
|
|
benchmarks = aggregate_benchmarks(
|
|
output_dir=output_base,
|
|
skill_name=skill,
|
|
)
|
|
|
|
if not benchmarks:
|
|
click.echo(f" 未找到 benchmark 数据，跳过")
|
|
continue
|
|
|
|
# 按迭代过滤
|
|
if iteration is not None:
|
|
benchmarks = [b for b in benchmarks if b.iteration == iteration]
|
|
|
|
if not benchmarks:
|
|
click.echo(f" 未找到迭代 {iteration} 的数据，跳过")
|
|
continue
|
|
|
|
# 生成 Markdown 报告
|
|
latest_iter = benchmarks[-1].iteration
|
|
md_path = skill_dir / f"iteration-{latest_iter}" / "benchmark.md"
|
|
report = generate_benchmark_md(benchmarks, md_path)
|
|
|
|
# 也生成聚合报告
|
|
stats = compute_aggregate_stats(benchmarks)
|
|
agg_path = skill_dir / "benchmark_aggregate.json"
|
|
import json
|
|
agg_path.write_text(
|
|
json.dumps(stats, indent=2, ensure_ascii=False, default=str),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
click.echo(f" 报告已保存: {md_path}")
|
|
click.echo(f" 聚合数据: {agg_path}")
|
|
|
|
# 打印摘要
|
|
pr = stats.get("pass_rate", {})
|
|
click.echo(f" 平均通过率: {pr.get('mean', 0):.1%} ± {pr.get('stddev', 0):.1%}")
|
|
|
|
delta = stats.get("baseline_delta", {})
|
|
if delta:
|
|
click.echo(f" Baseline 提升: {delta.get('mean_pass_rate_delta', 0):+.1%}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|