diff --git a/doc/changes/export-shortcut.md b/doc/changes/export-shortcut.md new file mode 100644 index 0000000..8d3616f --- /dev/null +++ b/doc/changes/export-shortcut.md @@ -0,0 +1,15 @@ +# Export shortcut + +新增 `export` Shortcut 组,将仓库数据导出为 CSV / JSON 文件,支撑离线分析、科研数据抽取与外部报表: + +- `export +issues` — 导出 Issue 列表(GET `/v1/:owner/:repo/issues`) +- `export +prs` — 导出 PR 列表(GET `/v1/:owner/:repo/pulls`) +- `export +contributors` — 导出贡献者统计(GET `/:owner/:repo/contributors`) + +实现要点: + +- 统一 `--format csv|json`(默认 csv)与 `--output` 输出路径;`issues`/`prs` 额外支持 `--state open|closed|all` 过滤与 `--page/--limit` 分页。 +- CSV 表头固定(`id,title,state,created_at` 等),便于直接导入 Excel / pandas;JSON 保留原始字段,供 `workflow` 模块与科研 Skill 二次处理。 +- 导出过程只读、分页拉取全量,避免一次性请求超限。 + +背景:此前要做仓库数据导出只能手工拼 Raw API 并自行解析分页。`export` 组将其提升为一等命令,是子任务四科研场景(贡献排行、Issue 趋势、PR 效率)的数据入口,并与 `gitlink-contributor-insight`、`gitlink-research-tracker` 等 Skill 衔接。关联 PR #15。 diff --git a/doc/commands/export.md b/doc/commands/export.md new file mode 100644 index 0000000..64a10b8 --- /dev/null +++ b/doc/commands/export.md @@ -0,0 +1,68 @@ +# export — 数据导出命令 + +> 关联 Issue: #15 | PR: #12 + +## 概述 + +export 模块提供将仓库数据(Issue、PR、贡献者)导出为 CSV 或 JSON 文件的能力,支持离线分析和科研用途。 + +## 命令列表 + +### export +issues +- **用途**: 导出仓库 Issue 列表为 CSV 或 JSON 文件 +- **API**: GET /v1/:owner/:repo/issues +- **参数**: + - --format, -f (可选) 输出格式: csv / json,默认 csv + - --output, -o (可选) 输出文件路径,默认 issues.csv + - --state, -s (可选) 状态过滤: open / closed / all,默认 all + - --page, -p (可选) 起始页,默认 1 + - --limit, -l (可选) 每页数量,默认 50 +- **示例**: + - `gitlink-cli export +issues --format csv --output my_issues.csv` + - `gitlink-cli export +issues --format json --state open` + +### export +prs +- **用途**: 导出仓库 PR 列表为 CSV 或 JSON 文件 +- **API**: GET /v1/:owner/:repo/pulls +- **参数**: + - --format, -f (可选) 输出格式: csv / json,默认 csv + - --output, -o (可选) 输出文件路径,默认 prs.csv + - --state, -s (可选) 状态过滤,默认 all + - --page, -p (可选) 起始页,默认 1 + - --limit, -l (可选) 每页数量,默认 50 +- **示例**: + - `gitlink-cli export +prs --format json` + - `gitlink-cli export +prs --state closed --output closed_prs.csv` + +### export +contributors +- **用途**: 导出贡献者统计为 CSV 或 JSON 文件 +- **API**: GET /:owner/:repo/contributors +- **参数**: + - --format, -f (可选) 输出格式: csv / json,默认 csv + - --output, -o (可选) 输出文件路径,默认 contributors.csv +- **示例**: + - `gitlink-cli export +contributors --format json` + +## CSV 输出格式 + +### issues.csv +```csv +id,title,state,created_at +1,Bug fix,1,2026-01-01 +``` + +### prs.csv +```csv +id,title,state,created_at +2,Feature PR,0,2026-02-01 +``` + +### contributors.csv +```csv +id,login,contributions +1,dev1,42 +``` + +## 向后兼容性 + +无破坏性变更。所有命令通过 export 域组 + 前缀添加。 diff --git a/shortcuts/export/export.go b/shortcuts/export/export.go new file mode 100644 index 0000000..9fe8bdd --- /dev/null +++ b/shortcuts/export/export.go @@ -0,0 +1,179 @@ +package export + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "net/url" + "os" + + "github.com/gitlink-org/gitlink-cli/shortcuts/common" +) + +// Shortcuts returns data export shortcuts for GitLink. +// +// The export domain provides commands for exporting repository data +// (issues, pull requests, contributors) to CSV or JSON files for +// offline analysis and reporting. +func Shortcuts() []*common.Shortcut { + return []*common.Shortcut{ + { + Name: "issues", + Description: "导出仓库 Issue 列表为 CSV 或 JSON 文件", + Flags: []common.Flag{ + {Name: "format", Short: "f", Usage: "输出格式: csv / json", Default: "csv"}, + {Name: "output", Short: "o", Usage: "输出文件路径(默认: issues.csv)", Default: "issues.csv"}, + {Name: "state", Short: "s", Usage: "状态过滤: open / closed / all", Default: "all"}, + {Name: "page", Short: "p", Usage: "起始页", Default: "1"}, + {Name: "limit", Short: "l", Usage: "每页数量", Default: "50"}, + }, + Run: func(ctx *common.RuntimeContext) error { + if err := ctx.ResolveOwnerRepo(); err != nil { + return err + } + q := url.Values{} + q.Set("state", ctx.Arg("state")) + q.Set("page", ctx.Arg("page")) + q.Set("limit", ctx.Arg("limit")) + apiPath := fmt.Sprintf("/v1/%s/%s/issues", ctx.Owner, ctx.Repo) + items, err := ctx.PaginateAll(apiPath, q) + if err != nil { + return err + } + return writeExport(ctx.Arg("format"), ctx.Arg("output"), items, issueCSVHeader, issueCSVRow) + }, + }, + { + Name: "prs", + Description: "导出仓库 PR 列表为 CSV 或 JSON 文件", + Flags: []common.Flag{ + {Name: "format", Short: "f", Usage: "输出格式: csv / json", Default: "csv"}, + {Name: "output", Short: "o", Usage: "输出文件路径", Default: "prs.csv"}, + {Name: "state", Short: "s", Usage: "状态过滤", Default: "all"}, + {Name: "page", Short: "p", Usage: "起始页", Default: "1"}, + {Name: "limit", Short: "l", Usage: "每页数量", Default: "50"}, + }, + Run: func(ctx *common.RuntimeContext) error { + if err := ctx.ResolveOwnerRepo(); err != nil { + return err + } + q := url.Values{} + q.Set("state", ctx.Arg("state")) + q.Set("page", ctx.Arg("page")) + q.Set("limit", ctx.Arg("limit")) + apiPath := fmt.Sprintf("/v1/%s/%s/pulls", ctx.Owner, ctx.Repo) + items, err := ctx.PaginateAll(apiPath, q) + if err != nil { + return err + } + return writeExport(ctx.Arg("format"), ctx.Arg("output"), items, prCSVHeader, prCSVRow) + }, + }, + { + Name: "contributors", + Description: "导出贡献者统计为 CSV 或 JSON 文件", + Flags: []common.Flag{ + {Name: "format", Short: "f", Usage: "输出格式: csv / json", Default: "csv"}, + {Name: "output", Short: "o", Usage: "输出文件路径", Default: "contributors.csv"}, + }, + Run: func(ctx *common.RuntimeContext) error { + if err := ctx.ResolveOwnerRepo(); err != nil { + return err + } + apiPath := fmt.Sprintf("/%s/%s/contributors", ctx.Owner, ctx.Repo) + items, err := ctx.PaginateAll(apiPath, url.Values{}) + if err != nil { + return err + } + return writeExport(ctx.Arg("format"), ctx.Arg("output"), items, contributorCSVHeader, contributorCSVRow) + }, + }, + } +} + +// --- CSV headers --- + +var issueCSVHeader = []string{"id", "title", "state", "created_at"} +var prCSVHeader = []string{"id", "title", "state", "created_at"} +var contributorCSVHeader = []string{"id", "login", "contributions"} + +// --- CSV row extractors --- + +func issueCSVRow(m map[string]interface{}) []string { + return []string{ + fmt.Sprint(m["id"]), + fmt.Sprint(m["subject"]), + fmt.Sprint(m["status"]), + fmt.Sprint(m["created_at"]), + } +} + +func prCSVRow(m map[string]interface{}) []string { + return []string{ + fmt.Sprint(m["id"]), + fmt.Sprint(m["title"]), + fmt.Sprint(m["status"]), + fmt.Sprint(m["created_at"]), + } +} + +func contributorCSVRow(m map[string]interface{}) []string { + return []string{ + fmt.Sprint(m["id"]), + fmt.Sprint(m["login"]), + fmt.Sprint(m["contributions"]), + } +} + +// --- Export writers --- + +type csvRowFunc func(map[string]interface{}) []string + +func writeExport(format, path string, items []json.RawMessage, header []string, rowFn csvRowFunc) error { + switch format { + case "csv": + return writeCSV(path, items, header, rowFn) + case "json": + return writeJSON(path, items) + default: + return fmt.Errorf("不支持的格式: %s(可选: csv, json)", format) + } +} + +func writeCSV(path string, items []json.RawMessage, header []string, rowFn csvRowFunc) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := csv.NewWriter(f) + defer w.Flush() + + if err := w.Write(header); err != nil { + return err + } + for _, item := range items { + var m map[string]interface{} + if err := json.Unmarshal(item, &m); err != nil { + continue + } + if err := w.Write(rowFn(m)); err != nil { + return err + } + } + fmt.Printf("已导出 %d 条记录到 %s\n", len(items), path) + return nil +} + +func writeJSON(path string, items []json.RawMessage) error { + data, err := json.MarshalIndent(items, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(path, data, 0644); err != nil { + return err + } + fmt.Printf("已导出 %d 条记录到 %s\n", len(items), path) + return nil +} diff --git a/shortcuts/export/export_test.go b/shortcuts/export/export_test.go new file mode 100644 index 0000000..cda99f0 --- /dev/null +++ b/shortcuts/export/export_test.go @@ -0,0 +1,158 @@ +package export + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/gitlink-org/gitlink-cli/internal/client" + "github.com/gitlink-org/gitlink-cli/shortcuts/common" +) + +func TestExportIssues(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Errorf("expected GET, got %s", r.Method) + } + w.WriteHeader(200) + w.Write([]byte(`[{"id":1,"subject":"Bug fix","status":1,"created_at":"2026-01-01"}]`)) + })) + defer server.Close() + + shortcut := findExportShortcut(t, "issues") + ctx := &common.RuntimeContext{ + Client: &client.Client{HTTP: server.Client(), BaseURL: server.URL}, + Owner: "test", Repo: "test", Format: "json", + Args: map[string]string{ + "format": "json", + "output": os.TempDir() + "/test_issues_export.json", + "state": "all", + "page": "1", + "limit": "50", + }, + } + if err := shortcut.Run(ctx); err != nil { + t.Fatalf("export issues failed: %v", err) + } +} + +func TestExportPrs(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Errorf("expected GET, got %s", r.Method) + } + w.WriteHeader(200) + w.Write([]byte(`[{"id":2,"title":"Feature PR","status":0,"created_at":"2026-02-01"}]`)) + })) + defer server.Close() + + shortcut := findExportShortcut(t, "prs") + ctx := &common.RuntimeContext{ + Client: &client.Client{HTTP: server.Client(), BaseURL: server.URL}, + Owner: "test", Repo: "test", Format: "json", + Args: map[string]string{ + "format": "json", + "output": os.TempDir() + "/test_prs_export.json", + "state": "all", + "page": "1", + "limit": "50", + }, + } + if err := shortcut.Run(ctx); err != nil { + t.Fatalf("export prs failed: %v", err) + } +} + +func TestExportContributors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Errorf("expected GET, got %s", r.Method) + } + w.WriteHeader(200) + w.Write([]byte(`[{"id":1,"login":"dev1","contributions":42}]`)) + })) + defer server.Close() + + shortcut := findExportShortcut(t, "contributors") + ctx := &common.RuntimeContext{ + Client: &client.Client{HTTP: server.Client(), BaseURL: server.URL}, + Owner: "test", Repo: "test", Format: "json", + Args: map[string]string{ + "format": "json", + "output": os.TempDir() + "/test_contributors_export.json", + }, + } + if err := shortcut.Run(ctx); err != nil { + t.Fatalf("export contributors failed: %v", err) + } +} + +func TestExportUnsupportedFormat(t *testing.T) { + items := []json.RawMessage{[]byte(`{"id":1}`)} + err := writeExport("xml", "/dev/null", items, issueCSVHeader, issueCSVRow) + if err == nil { + t.Fatal("expected error for unsupported format") + } + if !strings.Contains(err.Error(), "不支持的格式") { + t.Errorf("error should mention unsupported format: %v", err) + } +} + +func TestWriteCSV(t *testing.T) { + tmpFile := os.TempDir() + "/test_export_write.csv" + defer os.Remove(tmpFile) + + items := []json.RawMessage{ + []byte(`{"id":1,"subject":"First","status":1,"created_at":"2026-01-01"}`), + []byte(`{"id":2,"subject":"Second","status":0,"created_at":"2026-01-02"}`), + } + if err := writeCSV(tmpFile, items, issueCSVHeader, issueCSVRow); err != nil { + t.Fatalf("writeCSV failed: %v", err) + } + + data, err := os.ReadFile(tmpFile) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + content := string(data) + if !strings.Contains(content, "id,title,state,created_at") { + t.Errorf("CSV header missing in output: %s", content) + } + if !strings.Contains(content, "First") { + t.Errorf("expected 'First' in CSV output: %s", content) + } +} + +func TestWriteJSON(t *testing.T) { + tmpFile := os.TempDir() + "/test_export_write.json" + defer os.Remove(tmpFile) + + items := []json.RawMessage{ + []byte(`{"id":1,"name":"test"}`), + } + if err := writeJSON(tmpFile, items); err != nil { + t.Fatalf("writeJSON failed: %v", err) + } + + data, err := os.ReadFile(tmpFile) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if !strings.Contains(string(data), `"id": 1`) && !strings.Contains(string(data), `"id":1`) { + t.Errorf("JSON content unexpected: %s", string(data)) + } +} + +func findExportShortcut(t *testing.T, name string) *common.Shortcut { + t.Helper() + for _, shortcut := range Shortcuts() { + if shortcut.Name == name { + return shortcut + } + } + t.Fatalf("shortcut %q not found", name) + return nil +} diff --git a/shortcuts/register.go b/shortcuts/register.go index f081a12..df59bc6 100644 --- a/shortcuts/register.go +++ b/shortcuts/register.go @@ -10,6 +10,7 @@ import ( "github.com/gitlink-org/gitlink-cli/shortcuts/common" "github.com/gitlink-org/gitlink-cli/shortcuts/compare" "github.com/gitlink-org/gitlink-cli/shortcuts/dataset" + "github.com/gitlink-org/gitlink-cli/shortcuts/export" "github.com/gitlink-org/gitlink-cli/shortcuts/health" "github.com/gitlink-org/gitlink-cli/shortcuts/ignore" "github.com/gitlink-org/gitlink-cli/shortcuts/issue" @@ -58,6 +59,7 @@ func RegisterAll(root *cobra.Command, translators ...*i18n.Translator) { "ci": ci.Shortcuts(tr), "compare": compare.Shortcuts(), "dataset": dataset.Shortcuts(tr), + "export": export.Shortcuts(), "webhook": webhook.Shortcuts(tr), "wiki": wiki.Shortcuts(), "health": health.Shortcuts(tr), @@ -86,6 +88,7 @@ func RegisterAll(root *cobra.Command, translators ...*i18n.Translator) { "ci": tr.T("cmd.ci.short"), "compare": "Compare branches, tags, or commits", "dataset": tr.T("cmd.dataset.short"), + "export": "Data export to CSV/JSON", "webhook": tr.T("cmd.webhook.short"), "wiki": "Wiki page management", "health": "Project health data collection",