From 0147dd7fe22053e41955f27f92c8cba76c5af32e Mon Sep 17 00:00:00 2001 From: xxq250 Date: Mon, 15 Jun 2026 16:01:40 +0800 Subject: [PATCH] feat: support owner-specific backend routing --- README.md | 181 ++++++++++++++++++++++- config/owner-map.example | 9 ++ openresty/nginx.conf | 3 +- src/git_ssh_gateway.c | 300 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 480 insertions(+), 13 deletions(-) create mode 100644 config/owner-map.example diff --git a/README.md b/README.md index 3692b94..55d2dc2 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,11 @@ openresty/nginx.conf scripts/gen-host-key.sh ``` -demo 当前实现的是： +demo 支持两种后端模式： ```text +本地模式： + git client | | SSH @@ -27,9 +29,25 @@ libssh gateway | fork/exec v local git-upload-pack / git-receive-pack + +分布式 SSH 后端模式： + +git client + | + | SSH + v +OpenResty stream + | + | TCP + v +libssh gateway + | + | owner hash route + v +server1:2221 / server2:2222 / server3:2223 ``` -也就是说，libssh gateway 会终止 SSH、解析 Git `exec` 命令，然后在 `--repo-root` 目录下执行本机的 `git-upload-pack`、`git-receive-pack` 或 `git-upload-archive`。 +也就是说，libssh gateway 会终止 SSH、解析 Git `exec` 命令，然后按仓库路径里的 owner 做 hash。如果没有配置 `--backend`，就在 `--repo-root` 目录下执行本机的 `git-upload-pack`、`git-receive-pack` 或 `git-upload-archive`；如果配置了 `--backend`，就把请求转发到选中的 Git SSH 后端。 ## 依赖 @@ -68,6 +86,8 @@ git init --bare repositories/owner/repo.git ## 启动 gateway +本地模式： + ```bash ./build/git-ssh-gateway \ --bind 127.0.0.1 \ @@ -76,10 +96,108 @@ git init --bare repositories/owner/repo.git --host-key ./ssh_host_ed25519_key ``` +分布式 SSH 后端模式，按 owner hash 到 3 台 Git server： + +```bash +./build/git-ssh-gateway \ + --bind 127.0.0.1 \ + --port 2200 \ + --host-key ./ssh_host_ed25519_key \ + --backend-user git \ + --backend server1:2221 \ + --backend server2:2222 \ + --backend server3:2223 +``` + +可以为特定 owner 配置固定后端。固定配置优先于 hash： + +```bash +./build/git-ssh-gateway \ + --bind 127.0.0.1 \ + --port 2200 \ + --host-key ./ssh_host_ed25519_key \ + --backend-user git \ + --backend server1:2221 \ + --backend server2:2222 \ + --backend server3:2223 \ + --owner-backend alice=server1:2221 \ + --owner-backend bob=server2:2222 +``` + +也可以放到配置文件里： + +```bash +./build/git-ssh-gateway \ + --bind 127.0.0.1 \ + --port 2200 \ + --host-key ./ssh_host_ed25519_key \ + --backend-user git \ + --backend server1:2221 \ + --backend server2:2222 \ + --backend server3:2223 \ + --owner-map ./config/owner-map.example +``` + +owner map 文件格式： + +```text +# owner=host:port +alice=server1:2221 +bob=server2:2222 +carol=server3:2223 +``` + +如果同一个 owner 配置多次，后面的配置覆盖前面的配置。这样可以先加载文件，再用 `--owner-backend` 临时覆盖： + +```bash +./build/git-ssh-gateway \ + --bind 127.0.0.1 \ + --port 2200 \ + --host-key ./ssh_host_ed25519_key \ + --backend-user git \ + --backend server1:2221 \ + --backend server2:2222 \ + --backend server3:2223 \ + --owner-map ./config/owner-map.example \ + --owner-backend alice=server3:2223 +``` + +路由规则： + +```text +if owner in owner map: + backend = configured owner backend +else: + hash = FNV-1a(owner) + backend = backends[hash % backend_count] +``` + +例如： + +```text +git clone git@git.example.com:alice/repo.git +owner = alice +alice 固定落到同一个 server +``` + +如果后端 Git SSH 需要指定私钥： + +```bash +./build/git-ssh-gateway \ + --bind 127.0.0.1 \ + --port 2200 \ + --host-key ./ssh_host_ed25519_key \ + --backend-user git \ + --backend-identity /etc/git-gateway/backend_id_ed25519 \ + --backend server1:2221 \ + --backend server2:2222 \ + --backend server3:2223 +``` + 直接连 gateway 测试： ```bash -GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2222" \ +GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2200" \ git clone git@127.0.0.1:owner/repo.git ``` @@ -92,7 +210,62 @@ GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null git clone git@127.0.0.1:owner/repo.git ``` -本 demo 为方便本地验证，认证层接受任意 publickey/password。生产环境必须改成真实的 key 校验、用户映射、仓库 ACL 和审计。 +本 demo 为方便本地验证，客户端到 gateway 的认证层接受任意 publickey/password。生产环境必须改成真实的 key 校验、用户映射、仓库 ACL 和审计。gateway 到后端 Git SSH 的认证依赖系统 `ssh`，建议使用专用部署密钥和受控的 `known_hosts`。 + +## 分布式存储设计 + +你的场景中有 3 个 Git server： + +```text +server1:2221 +server2:2222 +server3:2223 +``` + +仓库按 owner 拆分： + +```text +owner/repo.git +``` + +gateway 解析 Git SSH command： + +```text +git-upload-pack 'owner/repo.git' +git-receive-pack 'owner/repo.git' +git-upload-archive 'owner/repo.git' +``` + +然后只对 `owner` 做 hash，而不是对完整 repo 做 hash。这样同一个 owner 下的所有仓库都会在同一台 Git server 上，便于权限、配额、备份和迁移。 + +对于需要迁移、灰度或手工固定位置的 owner，可以使用自定义 owner 配置： + +```text +alice=server1:2221 +bob=server2:2222 +``` + +命中自定义配置时不会再走 hash；未命中时继续走 owner hash。 + +OpenResty 不能直接完成这个 hash 路由，因为 owner 在 SSH 加密通道内部。OpenResty 的职责是： + +```text +监听 22 / 2220 +连接限流 +TCP keepalive +日志 +proxy_pass 到 libssh gateway +``` + +libssh gateway 的职责是： + +```text +终止 SSH +解析 git exec command +提取 owner +owner hash 选后端 +通过 SSH 转发到 server1/server2/server3 +``` ## 结论 diff --git a/config/owner-map.example b/config/owner-map.example new file mode 100644 index 0000000..f950cc8 --- /dev/null +++ b/config/owner-map.example @@ -0,0 +1,9 @@ +# Explicit owner routes. Later duplicates override earlier entries. +# +# Format: +# owner=host:port + +alice=server1:2221 +bob=server2:2222 +carol=server3:2223 + diff --git a/openresty/nginx.conf b/openresty/nginx.conf index 5c1fc97..40cdfea 100644 --- a/openresty/nginx.conf +++ b/openresty/nginx.conf @@ -15,7 +15,7 @@ stream { limit_conn_zone $binary_remote_addr zone=ssh_ip:10m; upstream libssh_git_gateway { - server 127.0.0.1:2222 max_fails=3 fail_timeout=10s; + server 127.0.0.1:2200 max_fails=3 fail_timeout=10s; } server { @@ -31,4 +31,3 @@ stream { proxy_pass libssh_git_gateway; } } - diff --git a/src/git_ssh_gateway.c b/src/git_ssh_gateway.c index 7cf7ccc..02dffe9 100644 --- a/src/git_ssh_gateway.c +++ b/src/git_ssh_gateway.c @@ -18,13 +18,32 @@ #define DEFAULT_BIND_PORT "2222" #define DEFAULT_REPO_ROOT "./repositories" #define DEFAULT_HOST_KEY "./ssh_host_ed25519_key" +#define DEFAULT_BACKEND_USER "git" #define IO_BUF_SIZE 32768 +#define MAX_BACKENDS 16 +#define MAX_OWNER_ROUTES 128 + +struct backend { + char host[256]; + char port[16]; +}; + +struct owner_route { + char owner[256]; + struct backend backend; +}; struct config { const char *bind_addr; const char *bind_port; const char *repo_root; const char *host_key; + const char *backend_user; + const char *backend_identity; + struct backend backends[MAX_BACKENDS]; + size_t backend_count; + struct owner_route owner_routes[MAX_OWNER_ROUTES]; + size_t owner_route_count; }; struct git_request { @@ -51,12 +70,148 @@ static void usage(const char *argv0) { fprintf(stderr, "Usage: %s [--bind ADDR] [--port PORT] [--repo-root DIR] [--host-key FILE]\n" + " [--backend HOST:PORT ...] [--backend-user USER] [--backend-identity FILE]\n" + " [--owner-backend OWNER=HOST:PORT ...] [--owner-map FILE]\n" "\n" - "Example:\n" + "Local backend example:\n" " %s --bind 127.0.0.1 --port 2222 --repo-root ./repositories --host-key ./ssh_host_ed25519_key\n", argv0, argv0); } +static void parse_backend_endpoint(const char *endpoint, struct backend *backend) +{ + const char *colon = strrchr(endpoint, ':'); + size_t host_len; + size_t port_len; + + if (colon == NULL || colon == endpoint || colon[1] == '\0') { + fprintf(stderr, "invalid backend: %s\n", endpoint); + exit(2); + } + + host_len = (size_t)(colon - endpoint); + port_len = strlen(colon + 1); + if (host_len >= sizeof(backend->host) || port_len >= sizeof(backend->port)) { + fprintf(stderr, "backend is too long: %s\n", endpoint); + exit(2); + } + + memcpy(backend->host, endpoint, host_len); + backend->host[host_len] = '\0'; + strcpy(backend->port, colon + 1); +} + +static void add_backend(struct config *cfg, const char *endpoint) +{ + if (cfg->backend_count >= MAX_BACKENDS) { + fprintf(stderr, "too many backends, max=%d\n", MAX_BACKENDS); + exit(2); + } + + parse_backend_endpoint(endpoint, &cfg->backends[cfg->backend_count]); + cfg->backend_count++; +} + +static bool is_safe_owner(const char *owner) +{ + if (owner[0] == '\0') { + return false; + } + + for (const unsigned char *p = (const unsigned char *)owner; *p; p++) { + if ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || + (*p >= '0' && *p <= '9') || *p == '_' || *p == '-' || *p == '.') { + continue; + } + return false; + } + + return true; +} + +static void add_owner_backend(struct config *cfg, const char *rule) +{ + const char *eq = strchr(rule, '='); + size_t owner_len; + + if (cfg->owner_route_count >= MAX_OWNER_ROUTES) { + fprintf(stderr, "too many owner routes, max=%d\n", MAX_OWNER_ROUTES); + exit(2); + } + + if (eq == NULL || eq == rule || eq[1] == '\0') { + fprintf(stderr, "invalid owner backend rule: %s\n", rule); + exit(2); + } + + owner_len = (size_t)(eq - rule); + if (owner_len >= sizeof(cfg->owner_routes[0].owner)) { + fprintf(stderr, "owner is too long: %s\n", rule); + exit(2); + } + + memcpy(cfg->owner_routes[cfg->owner_route_count].owner, rule, owner_len); + cfg->owner_routes[cfg->owner_route_count].owner[owner_len] = '\0'; + if (!is_safe_owner(cfg->owner_routes[cfg->owner_route_count].owner)) { + fprintf(stderr, "invalid owner name: %s\n", cfg->owner_routes[cfg->owner_route_count].owner); + exit(2); + } + + parse_backend_endpoint(eq + 1, &cfg->owner_routes[cfg->owner_route_count].backend); + cfg->owner_route_count++; +} + +static char *trim_line(char *s) +{ + char *end; + + while (*s == ' ' || *s == '\t' || *s == '\r' || *s == '\n') { + s++; + } + + end = s + strlen(s); + while (end > s && (end[-1] == ' ' || end[-1] == '\t' || + end[-1] == '\r' || end[-1] == '\n')) { + *--end = '\0'; + } + + return s; +} + +static void load_owner_map(struct config *cfg, const char *path) +{ + FILE *fp = fopen(path, "r"); + char line[1024]; + unsigned long lineno = 0; + + if (fp == NULL) { + perror(path); + exit(2); + } + + while (fgets(line, sizeof(line), fp) != NULL) { + char *rule; + + lineno++; + rule = trim_line(line); + if (rule[0] == '\0' || rule[0] == '#') { + continue; + } + + add_owner_backend(cfg, rule); + } + + if (ferror(fp)) { + fprintf(stderr, "failed reading owner map: %s\n", path); + fclose(fp); + exit(2); + } + + fclose(fp); + fprintf(stderr, "loaded owner map: %s routes=%zu\n", path, cfg->owner_route_count); + (void)lineno; +} + static struct config parse_args(int argc, char **argv) { struct config cfg = { @@ -64,6 +219,10 @@ static struct config parse_args(int argc, char **argv) .bind_port = DEFAULT_BIND_PORT, .repo_root = DEFAULT_REPO_ROOT, .host_key = DEFAULT_HOST_KEY, + .backend_user = DEFAULT_BACKEND_USER, + .backend_identity = NULL, + .backend_count = 0, + .owner_route_count = 0, }; for (int i = 1; i < argc; i++) { @@ -75,6 +234,16 @@ static struct config parse_args(int argc, char **argv) cfg.repo_root = argv[++i]; } else if (strcmp(argv[i], "--host-key") == 0 && i + 1 < argc) { cfg.host_key = argv[++i]; + } else if (strcmp(argv[i], "--backend") == 0 && i + 1 < argc) { + add_backend(&cfg, argv[++i]); + } else if (strcmp(argv[i], "--backend-user") == 0 && i + 1 < argc) { + cfg.backend_user = argv[++i]; + } else if (strcmp(argv[i], "--backend-identity") == 0 && i + 1 < argc) { + cfg.backend_identity = argv[++i]; + } else if (strcmp(argv[i], "--owner-backend") == 0 && i + 1 < argc) { + add_owner_backend(&cfg, argv[++i]); + } else if (strcmp(argv[i], "--owner-map") == 0 && i + 1 < argc) { + load_owner_map(&cfg, argv[++i]); } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { usage(argv[0]); exit(0); @@ -119,6 +288,8 @@ static bool has_bad_repo_chars(const char *s) case '<': case '>': case '\\': + case '\'': + case '"': return true; default: break; @@ -197,6 +368,84 @@ static int parse_git_exec(const char *command, struct git_request *req) return 0; } +static uint32_t fnv1a_hash(const char *s, size_t len) +{ + uint32_t h = 2166136261u; + + for (size_t i = 0; i < len; i++) { + h ^= (unsigned char)s[i]; + h *= 16777619u; + } + + return h; +} + +static int get_owner(const char *repo, char *owner, size_t owner_len) +{ + const char *slash = strchr(repo, '/'); + size_t n; + + if (slash == NULL || slash == repo) { + return -1; + } + + n = (size_t)(slash - repo); + if (n + 1 > owner_len) { + return -1; + } + + memcpy(owner, repo, n); + owner[n] = '\0'; + return 0; +} + +static const struct backend *select_backend(const struct config *cfg, const char *repo) +{ + char owner[256]; + uint32_t hash; + size_t index; + + if (cfg->backend_count == 0 && cfg->owner_route_count == 0) { + return NULL; + } + + if (get_owner(repo, owner, sizeof(owner)) != 0) { + return NULL; + } + + for (size_t i = cfg->owner_route_count; i > 0; i--) { + const struct owner_route *route = &cfg->owner_routes[i - 1]; + if (strcmp(owner, route->owner) == 0) { + fprintf(stderr, "route: owner=%s override=1 backend=%s:%s\n", + owner, + route->backend.host, + route->backend.port); + return &route->backend; + } + } + + if (cfg->backend_count == 0) { + return NULL; + } + + hash = fnv1a_hash(owner, strlen(owner)); + index = hash % cfg->backend_count; + + fprintf(stderr, "route: owner=%s hash=%u backend_index=%zu backend=%s:%s\n", + owner, + hash, + index, + cfg->backends[index].host, + cfg->backends[index].port); + + return &cfg->backends[index]; +} + +static void build_git_command(const struct git_request *req, char *buf, size_t len) +{ + snprintf(buf, len, "%s '%s'", req->program, req->repo); +} + static int authenticate_client(ssh_session session) { ssh_message msg; @@ -287,12 +536,20 @@ static int spawn_git_child(const struct config *cfg, const struct git_request *r int in_pipe[2] = {-1, -1}; int out_pipe[2] = {-1, -1}; int err_pipe[2] = {-1, -1}; + const struct backend *backend = select_backend(cfg, req->repo); + char remote_target[320]; + char remote_command[1400]; child->pid = -1; child->stdin_fd = -1; child->stdout_fd = -1; child->stderr_fd = -1; + if ((cfg->backend_count > 0 || cfg->owner_route_count > 0) && backend == NULL) { + fprintf(stderr, "distributed backend mode has no route for repo path: %s\n", req->repo); + return -1; + } + if (pipe(in_pipe) != 0 || pipe(out_pipe) != 0 || pipe(err_pipe) != 0) { perror("pipe"); goto fail; @@ -316,14 +573,43 @@ static int spawn_git_child(const struct config *cfg, const struct git_request *r close(err_pipe[0]); close(err_pipe[1]); - if (chdir(cfg->repo_root) != 0) { - perror("chdir repo root"); + if (backend != NULL) { + const char *argv[16]; + int argc = 0; + + snprintf(remote_target, sizeof(remote_target), "%s@%s", cfg->backend_user, backend->host); + build_git_command(req, remote_command, sizeof(remote_command)); + + argv[argc++] = "ssh"; + argv[argc++] = "-o"; + argv[argc++] = "BatchMode=yes"; + argv[argc++] = "-o"; + argv[argc++] = "StrictHostKeyChecking=no"; + argv[argc++] = "-o"; + argv[argc++] = "UserKnownHostsFile=/dev/null"; + if (cfg->backend_identity != NULL) { + argv[argc++] = "-i"; + argv[argc++] = cfg->backend_identity; + } + argv[argc++] = "-p"; + argv[argc++] = backend->port; + argv[argc++] = remote_target; + argv[argc++] = remote_command; + argv[argc] = NULL; + + execvp("ssh", (char *const *)argv); + perror("execvp ssh backend"); + _exit(127); + } else { + if (chdir(cfg->repo_root) != 0) { + perror("chdir repo root"); + _exit(127); + } + + execlp(req->program, req->program, req->repo, (char *)NULL); + perror("execlp git program"); _exit(127); } - - execlp(req->program, req->program, req->repo, (char *)NULL); - perror("execlp git program"); - _exit(127); } close(in_pipe[0]);