From b82378bb4e55f84e718b32ea41815eea0342aa7b Mon Sep 17 00:00:00 2001 From: laurencewannamaker Date: Sun, 5 Jul 2026 22:05:58 +0000 Subject: [PATCH] =?UTF-8?q?feat(list):=20--all=20=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E7=BF=BB=E9=A1=B5=EF=BC=8C=E7=BF=BB=E9=A1=B5=E5=8A=A9=E6=89=8B?= =?UTF-8?q?=E5=AF=B9=E9=BD=90=E7=94=9F=E4=BA=A7=20API=20=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E9=94=AE=E5=8C=85=E8=A3=B9=E5=BD=A2=E7=8A=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- doc/changes/list-all-pagination.md | 43 ++++++++ internal/client/pagination.go | 103 ++++++++++++++----- internal/client/pagination_test.go | 154 +++++++++++++++++++++++++++++ internal/i18n/locales/en-US.json | 1 + internal/i18n/locales/zh-CN.json | 1 + shortcuts/branch/branch.go | 8 ++ shortcuts/common/types.go | 23 +++++ shortcuts/issue/issue.go | 10 ++ shortcuts/issue/issue_test.go | 36 +++++++ shortcuts/pr/pr.go | 8 ++ shortcuts/release/release.go | 8 ++ 11 files changed, 368 insertions(+), 27 deletions(-) create mode 100644 doc/changes/list-all-pagination.md create mode 100644 internal/client/pagination_test.go diff --git a/doc/changes/list-all-pagination.md b/doc/changes/list-all-pagination.md new file mode 100644 index 0000000..00ab336 --- /dev/null +++ b/doc/changes/list-all-pagination.md @@ -0,0 +1,43 @@ +# list 命令 --all 自动翻页 + +## 背景 + +`issue +list`、`pr +list`、`branch +list`、`release +list` 此前一次只能取一页， +用户或 AI Agent 想拿到全量列表必须手动循环 `--page`。代码中虽有 `PaginateAll` +翻页助手，但它只识别 `data` 包裹键；而 GitLink 生产 API 的列表响应实际用 +资源名包裹数组（如 `{"total_count":N,"issues":[...]}`、`"pulls"`、`"branches"`、 +`"releases"`），导致该助手在真实端点上退化为「单对象」返回，从未被任何命令使用。 + +## 变更内容 + +- `internal/client`：翻页助手对齐生产响应形状 + - 新增 `PaginateAllKey(path, params, listKey)`：按指定资源键提取数组； + `listKey` 为空时自动探测（顶层数组 / `data` 包裹 / 唯一数组字段）。 + - 遵循 `total_count`：达到总数即停止；另设最大页数护栏，防止 + 忽略 `page` 参数的端点造成死循环。 + - `PaginateAll` 保持原签名，委托给 `PaginateAllKey`。 +- 四个 list 命令新增 `--all` 布尔参数（默认 false）： + - `issue +list --all`（合并结果同样应用 number/database_id 规范化） + - `pr +list --all`、`branch +list --all`、`release +list --all` + - 输出与单页响应同构：`{"total_count": N, "<资源名>": [...]}`。 +- 中英文 i18n 新增 `flag.all` 文案。 + +## 命令示例 + +```bash +# 拉取仓库全部 open issue（自动翻页合并） +gitlink-cli issue +list --state open --all --format json + +# 全部分支 / 全部 PR / 全部 release +gitlink-cli branch +list --all +gitlink-cli pr +list --state all --all +gitlink-cli release +list --all +``` + +## 测试 + +- `internal/client/pagination_test.go`：资源键包裹多页合并、`total_count` + 截断（模拟忽略 page 的异常端点）、`data` 包裹、唯一数组字段自动探测、 + 单对象回退、指定键缺失回退，共 6 个用例。 +- `shortcuts/issue`：`--all` 端到端用例验证按页请求序列与合并。 +- `go test ./...`、`go vet`、`gofmt` 全部通过。 diff --git a/internal/client/pagination.go b/internal/client/pagination.go index 6eafc99..02d3bd1 100644 --- a/internal/client/pagination.go +++ b/internal/client/pagination.go @@ -7,8 +7,23 @@ import ( "strconv" ) +// maxPaginationPages caps auto-pagination as a safety guard against +// endpoints that ignore the page parameter and keep returning data. +const maxPaginationPages = 1000 + // PaginateAll fetches all pages and returns combined results. +// The list array is auto-detected inside the response body. func (c *Client) PaginateAll(path string, params url.Values) ([]json.RawMessage, error) { + return c.PaginateAllKey(path, params, "") +} + +// PaginateAllKey fetches all pages, extracting the list array from the +// response field named listKey (e.g. "issues", "pulls", "branches"). +// When listKey is empty the array is auto-detected: top-level arrays, +// the conventional "data" wrapper, or a unique array-valued field. +// Pagination stops when a page returns fewer items than the limit, when +// total_count (if reported) is reached, or at the safety page cap. +func (c *Client) PaginateAllKey(path string, params url.Values, listKey string) ([]json.RawMessage, error) { if params == nil { params = url.Values{} } @@ -17,57 +32,91 @@ func (c *Client) PaginateAll(path string, params url.Values) ([]json.RawMessage, } var all []json.RawMessage - page := 1 + totalCount := -1 - for { + for page := 1; page <= maxPaginationPages; page++ { params.Set("page", strconv.Itoa(page)) env, err := c.Get(path, params) if err != nil { return nil, err } - if !env.OK { return nil, fmt.Errorf("API error on page %d", page) } - // Try to extract array from data - var items []json.RawMessage - switch data := env.Data.(type) { - case []interface{}: - for _, item := range data { - raw, _ := json.Marshal(item) - items = append(items, raw) - } - case map[string]interface{}: - // Some endpoints wrap in {"data": [...], "total_count": N} - if arr, ok := data["data"]; ok { - if slice, ok := arr.([]interface{}); ok { - for _, item := range slice { - raw, _ := json.Marshal(item) - items = append(items, raw) - } - } - } else { - // Single object, not paginated - raw, _ := json.Marshal(data) + items, pageTotal, isList := extractListItems(env.Data, listKey) + if !isList { + if page == 1 { + raw, _ := json.Marshal(env.Data) return []json.RawMessage{raw}, nil } + break + } + if pageTotal >= 0 { + totalCount = pageTotal } if len(items) == 0 { break } - all = append(all, items...) - // Check if we got fewer items than limit + if totalCount >= 0 && len(all) >= totalCount { + break + } limit, _ := strconv.Atoi(params.Get("limit")) if len(items) < limit { break } - - page++ } return all, nil } + +// extractListItems locates the list array inside a decoded response body. +// It returns the items, the reported total_count (-1 when absent) and +// whether a list array was found at all. +func extractListItems(data interface{}, listKey string) ([]json.RawMessage, int, bool) { + switch v := data.(type) { + case []interface{}: + return marshalItems(v), -1, true + case map[string]interface{}: + total := -1 + if tc, ok := v["total_count"].(float64); ok { + total = int(tc) + } + if listKey != "" { + if slice, ok := v[listKey].([]interface{}); ok { + return marshalItems(slice), total, true + } + return nil, total, false + } + if slice, ok := v["data"].([]interface{}); ok { + return marshalItems(slice), total, true + } + // Auto-detect: GitLink v1 list endpoints wrap the array in a + // resource-named field ({"total_count":N,"issues":[...]}). + var found []interface{} + arrays := 0 + for _, val := range v { + if slice, ok := val.([]interface{}); ok { + arrays++ + found = slice + } + } + if arrays == 1 { + return marshalItems(found), total, true + } + return nil, total, false + } + return nil, -1, false +} + +func marshalItems(items []interface{}) []json.RawMessage { + out := make([]json.RawMessage, 0, len(items)) + for _, item := range items { + raw, _ := json.Marshal(item) + out = append(out, raw) + } + return out +} diff --git a/internal/client/pagination_test.go b/internal/client/pagination_test.go new file mode 100644 index 0000000..2948e5f --- /dev/null +++ b/internal/client/pagination_test.go @@ -0,0 +1,154 @@ +package client + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" +) + +func TestPaginateAllKeyResourceWrappedPages(t *testing.T) { + // GitLink v1 list shape: {"total_count":N, "issues":[...]} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + if limit != 2 { + t.Fatalf("limit = %d, want 2", limit) + } + var items []map[string]interface{} + switch page { + case 1: + items = []map[string]interface{}{{"id": 1}, {"id": 2}} + case 2: + items = []map[string]interface{}{{"id": 3}} + default: + t.Fatalf("unexpected page %d", page) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "total_count": 3, + "issues": items, + }) + })) + defer server.Close() + + c := &Client{HTTP: server.Client(), BaseURL: server.URL} + params := url.Values{} + params.Set("limit", "2") + items, err := c.PaginateAllKey("/owner/repo/issues", params, "issues") + if err != nil { + t.Fatalf("PaginateAllKey: %v", err) + } + if len(items) != 3 { + t.Fatalf("len(items) = %d, want 3", len(items)) + } +} + +func TestPaginateAllKeyStopsAtTotalCount(t *testing.T) { + // A broken endpoint that keeps returning full pages must stop at total_count. + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "total_count": 4, + "pulls": []map[string]interface{}{{"id": 1}, {"id": 2}}, + }) + })) + defer server.Close() + + c := &Client{HTTP: server.Client(), BaseURL: server.URL} + params := url.Values{} + params.Set("limit", "2") + items, err := c.PaginateAllKey("/owner/repo/pulls", params, "pulls") + if err != nil { + t.Fatalf("PaginateAllKey: %v", err) + } + if len(items) != 4 { + t.Fatalf("len(items) = %d, want 4", len(items)) + } + if calls != 2 { + t.Fatalf("calls = %d, want 2", calls) + } +} + +func TestPaginateAllAutoDetectsUniqueArrayField(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "total_count": 1, + "branches": []map[string]interface{}{{"name": "master"}}, + }) + })) + defer server.Close() + + c := &Client{HTTP: server.Client(), BaseURL: server.URL} + items, err := c.PaginateAll("/owner/repo/branches", nil) + if err != nil { + t.Fatalf("PaginateAll: %v", err) + } + if len(items) != 1 { + t.Fatalf("len(items) = %d, want 1", len(items)) + } +} + +func TestPaginateAllDataWrapper(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"id":1}],"total_count":1}`) + })) + defer server.Close() + + c := &Client{HTTP: server.Client(), BaseURL: server.URL} + items, err := c.PaginateAll("/things", nil) + if err != nil { + t.Fatalf("PaginateAll: %v", err) + } + if len(items) != 1 { + t.Fatalf("len(items) = %d, want 1", len(items)) + } +} + +func TestPaginateAllSingleObjectFallback(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":42,"name":"solo"}`) + })) + defer server.Close() + + c := &Client{HTTP: server.Client(), BaseURL: server.URL} + items, err := c.PaginateAll("/thing", nil) + if err != nil { + t.Fatalf("PaginateAll: %v", err) + } + if len(items) != 1 { + t.Fatalf("len(items) = %d, want 1", len(items)) + } + var obj map[string]interface{} + if err := json.Unmarshal(items[0], &obj); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if obj["name"] != "solo" { + t.Fatalf("name = %v, want solo", obj["name"]) + } +} + +func TestPaginateAllKeyMissingKeyNotList(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":42}`) + })) + defer server.Close() + + c := &Client{HTTP: server.Client(), BaseURL: server.URL} + items, err := c.PaginateAllKey("/thing", nil, "issues") + if err != nil { + t.Fatalf("PaginateAllKey: %v", err) + } + if len(items) != 1 { + t.Fatalf("len(items) = %d, want 1 (single-object fallback)", len(items)) + } +} diff --git a/internal/i18n/locales/en-US.json b/internal/i18n/locales/en-US.json index 0739395..59fdf8e 100644 --- a/internal/i18n/locales/en-US.json +++ b/internal/i18n/locales/en-US.json @@ -114,6 +114,7 @@ "error.missing_required_flag": "required flag --{name} is missing", "error.profile.user_required": "could not determine target user; pass --user or run gitlink-cli auth login", "error.unsupported_language": "unsupported language: {lang}", + "flag.all": "Fetch all pages automatically (ignores --page)", "flag.api.batch_continue_on_error": "Continue running remaining batch requests after a failure", "flag.api.batch_dry_run": "Preview batch requests without sending remote requests", "flag.api.batch_file": "Read an API batch plan from a JSON file", diff --git a/internal/i18n/locales/zh-CN.json b/internal/i18n/locales/zh-CN.json index 2e6fc4d..2f8c7f7 100644 --- a/internal/i18n/locales/zh-CN.json +++ b/internal/i18n/locales/zh-CN.json @@ -114,6 +114,7 @@ "error.missing_required_flag": "缺少必需参数 --{name}", "error.profile.user_required": "无法确定目标用户；请通过 --user 指定，或先运行 gitlink-cli auth login 登录", "error.unsupported_language": "不支持的语言：{lang}", + "flag.all": "自动获取全部分页（忽略 --page）", "flag.api.batch_continue_on_error": "批处理请求失败后继续执行后续请求", "flag.api.batch_dry_run": "预览批处理请求，不发送远端请求", "flag.api.batch_file": "从 JSON 文件读取 API 批处理计划", diff --git a/shortcuts/branch/branch.go b/shortcuts/branch/branch.go index 0393ad5..a5c1e62 100644 --- a/shortcuts/branch/branch.go +++ b/shortcuts/branch/branch.go @@ -17,6 +17,7 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { Flags: []common.Flag{ {Name: "page", Short: "p", Usage: tr.T("flag.page"), Default: "1"}, {Name: "limit", Short: "l", Usage: tr.T("flag.limit"), Default: "20"}, + {Name: "all", Usage: tr.T("flag.all"), Bool: true, Default: "false"}, }, Run: func(ctx *common.RuntimeContext) error { if err := ctx.ResolveOwnerRepo(); err != nil { @@ -25,6 +26,13 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { q := url.Values{} q.Set("page", ctx.Arg("page")) q.Set("limit", ctx.Arg("limit")) + if ctx.Arg("all") == "true" { + items, err := ctx.PaginateAllKey("/v1"+ctx.RepoPath()+"/branches", q, "branches") + if err != nil { + return err + } + return ctx.Output(common.NewListEnvelope("branches", items)) + } env, err := ctx.CallAPIWithQuery("GET", "/v1"+ctx.RepoPath()+"/branches", q) if err != nil { return err diff --git a/shortcuts/common/types.go b/shortcuts/common/types.go index 87a052a..cb6ad3d 100644 --- a/shortcuts/common/types.go +++ b/shortcuts/common/types.go @@ -95,6 +95,29 @@ func (ctx *RuntimeContext) PaginateAll(path string, params url.Values) ([]json.R return ctx.Client.PaginateAll(path, params) } +// PaginateAllKey fetches all pages of a list endpoint whose response wraps +// the array in the field named listKey (e.g. "issues", "pulls"). +func (ctx *RuntimeContext) PaginateAllKey(path string, params url.Values, listKey string) ([]json.RawMessage, error) { + return ctx.Client.PaginateAllKey(path, params, listKey) +} + +// NewListEnvelope wraps combined pages in the same shape as a single-page +// response: {"total_count": N, "": [...]}. +func NewListEnvelope(listKey string, items []json.RawMessage) *output.Envelope { + decoded := make([]interface{}, 0, len(items)) + for _, item := range items { + var v interface{} + if err := json.Unmarshal(item, &v); err == nil { + decoded = append(decoded, v) + } + } + data := map[string]interface{}{ + "total_count": len(decoded), + listKey: decoded, + } + return output.SuccessEnvelope(data, &output.Meta{TotalCount: len(decoded)}) +} + // Output prints the envelope in the configured format. func (ctx *RuntimeContext) Output(env *output.Envelope) error { return output.Print(env, ctx.Format) diff --git a/shortcuts/issue/issue.go b/shortcuts/issue/issue.go index b19027e..83c06fb 100644 --- a/shortcuts/issue/issue.go +++ b/shortcuts/issue/issue.go @@ -63,6 +63,7 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { {Name: "sort-direction", Usage: tr.T("flag.sort_direction")}, {Name: "page", Short: "p", Usage: tr.T("flag.page"), Default: "1"}, {Name: "limit", Short: "l", Usage: tr.T("flag.limit"), Default: "20"}, + {Name: "all", Usage: tr.T("flag.all"), Bool: true, Default: "false"}, }, Run: func(ctx *common.RuntimeContext) error { if err := ctx.ResolveOwnerRepo(); err != nil { @@ -101,6 +102,15 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { if sortDirection := ctx.Arg("sort-direction"); sortDirection != "" { q.Set("sort_direction", sortDirection) } + if ctx.Arg("all") == "true" { + items, err := ctx.PaginateAllKey(v1RepoPath(ctx)+"/issues", q, "issues") + if err != nil { + return err + } + env := common.NewListEnvelope("issues", items) + normalizeIssueListIDs(env) + return ctx.Output(env) + } env, err := ctx.CallAPIWithQuery("GET", v1RepoPath(ctx)+"/issues", q) if err != nil { return err diff --git a/shortcuts/issue/issue_test.go b/shortcuts/issue/issue_test.go index 48be057..ca57026 100644 --- a/shortcuts/issue/issue_test.go +++ b/shortcuts/issue/issue_test.go @@ -171,6 +171,42 @@ func TestIssueListStateAll(t *testing.T) { } } +func TestIssueListAllPaginates(t *testing.T) { + var pages []string + server := newIssueTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/owner/repo/issues.json" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + page := r.URL.Query().Get("page") + pages = append(pages, page) + assertEqual(t, r.URL.Query().Get("limit"), "2") + var issues []interface{} + if page == "1" { + issues = []interface{}{ + map[string]interface{}{"id": float64(1), "project_issues_index": float64(11)}, + map[string]interface{}{"id": float64(2), "project_issues_index": float64(12)}, + } + } else { + issues = []interface{}{ + map[string]interface{}{"id": float64(3), "project_issues_index": float64(13)}, + } + } + writeJSON(t, w, map[string]interface{}{ + "total_count": float64(3), + "issues": issues, + }) + }) + defer server.Close() + + err := runShortcut(t, server, "list", map[string]string{"all": "true", "limit": "2"}) + if err != nil { + t.Fatalf("list --all failed: %v", err) + } + if len(pages) != 2 || pages[0] != "1" || pages[1] != "2" { + t.Fatalf("pages requested = %v, want [1 2]", pages) + } +} + // --- create --- func TestIssueCreate(t *testing.T) { diff --git a/shortcuts/pr/pr.go b/shortcuts/pr/pr.go index 03f537f..334b8ca 100644 --- a/shortcuts/pr/pr.go +++ b/shortcuts/pr/pr.go @@ -47,6 +47,7 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { {Name: "sort-direction", Usage: tr.T("flag.sort_direction")}, {Name: "page", Short: "p", Usage: tr.T("flag.page"), Default: "1"}, {Name: "limit", Short: "l", Usage: tr.T("flag.limit"), Default: "20"}, + {Name: "all", Usage: tr.T("flag.all"), Bool: true, Default: "false"}, }, Run: func(ctx *common.RuntimeContext) error { if err := ctx.ResolveOwnerRepo(); err != nil { @@ -82,6 +83,13 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { if sortDirection := ctx.Arg("sort-direction"); sortDirection != "" { q.Set("sort_direction", sortDirection) } + if ctx.Arg("all") == "true" { + items, err := ctx.PaginateAllKey(v1RepoPath(ctx)+"/pulls", q, "pulls") + if err != nil { + return err + } + return ctx.Output(common.NewListEnvelope("pulls", items)) + } env, err := ctx.CallAPIWithQuery("GET", v1RepoPath(ctx)+"/pulls", q) if err != nil { return err diff --git a/shortcuts/release/release.go b/shortcuts/release/release.go index 21d7ec4..10deb5a 100644 --- a/shortcuts/release/release.go +++ b/shortcuts/release/release.go @@ -20,6 +20,7 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { Flags: []common.Flag{ {Name: "page", Short: "p", Usage: tr.T("flag.page"), Default: "1"}, {Name: "limit", Short: "l", Usage: tr.T("flag.limit"), Default: "20"}, + {Name: "all", Usage: tr.T("flag.all"), Bool: true, Default: "false"}, }, Run: func(ctx *common.RuntimeContext) error { if err := ctx.ResolveOwnerRepo(); err != nil { @@ -28,6 +29,13 @@ func Shortcuts(translators ...*i18n.Translator) []*common.Shortcut { q := url.Values{} q.Set("page", ctx.Arg("page")) q.Set("limit", ctx.Arg("limit")) + if ctx.Arg("all") == "true" { + items, err := ctx.PaginateAllKey(ctx.RepoPath()+"/releases", q, "releases") + if err != nil { + return err + } + return ctx.Output(common.NewListEnvelope("releases", items)) + } env, err := ctx.CallAPIWithQuery("GET", ctx.RepoPath()+"/releases", q) if err != nil { return err