feat: support owner-specific backend routing

This commit is contained in:
xxq250 2026-06-15 16:01:40 +08:00
parent e014dca3ca
commit 0147dd7fe2
4 changed files with 480 additions and 13 deletions

181
README.md
View File

@ -11,9 +11,11 @@ openresty/nginx.conf
scripts/gen-host-key.sh
```
demo 当前实现的是
demo 支持两种后端模式
```text
本地模式:
git client
|
| SSH
@ -27,9 +29,25 @@ libssh gateway
| fork/exec
v
local git-upload-pack / git-receive-pack
分布式 SSH 后端模式:
git client
|
| SSH
v
OpenResty stream
|
| TCP
v
libssh gateway
|
| owner hash route
v
server1:2221 / server2:2222 / server3:2223
```
也就是说libssh gateway 会终止 SSH、解析 Git `exec` 命令,然后在 `--repo-root` 目录下执行本机的 `git-upload-pack`、`git-receive-pack` 或 `git-upload-archive`
也就是说libssh gateway 会终止 SSH、解析 Git `exec` 命令,然后按仓库路径里的 owner 做 hash。如果没有配置 `--backend`,就`--repo-root` 目录下执行本机的 `git-upload-pack`、`git-receive-pack` 或 `git-upload-archive`;如果配置了 `--backend`,就把请求转发到选中的 Git SSH 后端
## 依赖
@ -68,6 +86,8 @@ git init --bare repositories/owner/repo.git
## 启动 gateway
本地模式:
```bash
./build/git-ssh-gateway \
--bind 127.0.0.1 \
@ -76,10 +96,108 @@ git init --bare repositories/owner/repo.git
--host-key ./ssh_host_ed25519_key
```
分布式 SSH 后端模式,按 owner hash 到 3 台 Git server
```bash
./build/git-ssh-gateway \
--bind 127.0.0.1 \
--port 2200 \
--host-key ./ssh_host_ed25519_key \
--backend-user git \
--backend server1:2221 \
--backend server2:2222 \
--backend server3:2223
```
可以为特定 owner 配置固定后端。固定配置优先于 hash
```bash
./build/git-ssh-gateway \
--bind 127.0.0.1 \
--port 2200 \
--host-key ./ssh_host_ed25519_key \
--backend-user git \
--backend server1:2221 \
--backend server2:2222 \
--backend server3:2223 \
--owner-backend alice=server1:2221 \
--owner-backend bob=server2:2222
```
也可以放到配置文件里:
```bash
./build/git-ssh-gateway \
--bind 127.0.0.1 \
--port 2200 \
--host-key ./ssh_host_ed25519_key \
--backend-user git \
--backend server1:2221 \
--backend server2:2222 \
--backend server3:2223 \
--owner-map ./config/owner-map.example
```
owner map 文件格式:
```text
# owner=host:port
alice=server1:2221
bob=server2:2222
carol=server3:2223
```
如果同一个 owner 配置多次,后面的配置覆盖前面的配置。这样可以先加载文件,再用 `--owner-backend` 临时覆盖:
```bash
./build/git-ssh-gateway \
--bind 127.0.0.1 \
--port 2200 \
--host-key ./ssh_host_ed25519_key \
--backend-user git \
--backend server1:2221 \
--backend server2:2222 \
--backend server3:2223 \
--owner-map ./config/owner-map.example \
--owner-backend alice=server3:2223
```
路由规则:
```text
if owner in owner map:
backend = configured owner backend
else:
hash = FNV-1a(owner)
backend = backends[hash % backend_count]
```
例如:
```text
git clone git@git.example.com:alice/repo.git
owner = alice
alice 固定落到同一个 server
```
如果后端 Git SSH 需要指定私钥:
```bash
./build/git-ssh-gateway \
--bind 127.0.0.1 \
--port 2200 \
--host-key ./ssh_host_ed25519_key \
--backend-user git \
--backend-identity /etc/git-gateway/backend_id_ed25519 \
--backend server1:2221 \
--backend server2:2222 \
--backend server3:2223
```
直接连 gateway 测试:
```bash
GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2222" \
GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2200" \
git clone git@127.0.0.1:owner/repo.git
```
@ -92,7 +210,62 @@ GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
git clone git@127.0.0.1:owner/repo.git
```
本 demo 为方便本地验证,认证层接受任意 publickey/password。生产环境必须改成真实的 key 校验、用户映射、仓库 ACL 和审计。
本 demo 为方便本地验证,客户端到 gateway 的认证层接受任意 publickey/password。生产环境必须改成真实的 key 校验、用户映射、仓库 ACL 和审计。gateway 到后端 Git SSH 的认证依赖系统 `ssh`,建议使用专用部署密钥和受控的 `known_hosts`
## 分布式存储设计
你的场景中有 3 个 Git server
```text
server1:2221
server2:2222
server3:2223
```
仓库按 owner 拆分:
```text
owner/repo.git
```
gateway 解析 Git SSH command
```text
git-upload-pack 'owner/repo.git'
git-receive-pack 'owner/repo.git'
git-upload-archive 'owner/repo.git'
```
然后只对 `owner` 做 hash而不是对完整 repo 做 hash。这样同一个 owner 下的所有仓库都会在同一台 Git server 上,便于权限、配额、备份和迁移。
对于需要迁移、灰度或手工固定位置的 owner可以使用自定义 owner 配置:
```text
alice=server1:2221
bob=server2:2222
```
命中自定义配置时不会再走 hash未命中时继续走 owner hash。
OpenResty 不能直接完成这个 hash 路由,因为 owner 在 SSH 加密通道内部。OpenResty 的职责是:
```text
监听 22 / 2220
连接限流
TCP keepalive
日志
proxy_pass 到 libssh gateway
```
libssh gateway 的职责是:
```text
终止 SSH
解析 git exec command
提取 owner
owner hash 选后端
通过 SSH 转发到 server1/server2/server3
```
## 结论

9
config/owner-map.example Normal file
View File

@ -0,0 +1,9 @@
# Explicit owner routes. Later duplicates override earlier entries.
#
# Format:
# owner=host:port
alice=server1:2221
bob=server2:2222
carol=server3:2223

View File

@ -15,7 +15,7 @@ stream {
limit_conn_zone $binary_remote_addr zone=ssh_ip:10m;
upstream libssh_git_gateway {
server 127.0.0.1:2222 max_fails=3 fail_timeout=10s;
server 127.0.0.1:2200 max_fails=3 fail_timeout=10s;
}
server {
@ -31,4 +31,3 @@ stream {
proxy_pass libssh_git_gateway;
}
}

View File

@ -18,13 +18,32 @@
#define DEFAULT_BIND_PORT "2222"
#define DEFAULT_REPO_ROOT "./repositories"
#define DEFAULT_HOST_KEY "./ssh_host_ed25519_key"
#define DEFAULT_BACKEND_USER "git"
#define IO_BUF_SIZE 32768
#define MAX_BACKENDS 16
#define MAX_OWNER_ROUTES 128
struct backend {
char host[256];
char port[16];
};
struct owner_route {
char owner[256];
struct backend backend;
};
struct config {
const char *bind_addr;
const char *bind_port;
const char *repo_root;
const char *host_key;
const char *backend_user;
const char *backend_identity;
struct backend backends[MAX_BACKENDS];
size_t backend_count;
struct owner_route owner_routes[MAX_OWNER_ROUTES];
size_t owner_route_count;
};
struct git_request {
@ -51,12 +70,148 @@ static void usage(const char *argv0)
{
fprintf(stderr,
"Usage: %s [--bind ADDR] [--port PORT] [--repo-root DIR] [--host-key FILE]\n"
" [--backend HOST:PORT ...] [--backend-user USER] [--backend-identity FILE]\n"
" [--owner-backend OWNER=HOST:PORT ...] [--owner-map FILE]\n"
"\n"
"Example:\n"
"Local backend example:\n"
" %s --bind 127.0.0.1 --port 2222 --repo-root ./repositories --host-key ./ssh_host_ed25519_key\n",
argv0, argv0);
}
static void parse_backend_endpoint(const char *endpoint, struct backend *backend)
{
const char *colon = strrchr(endpoint, ':');
size_t host_len;
size_t port_len;
if (colon == NULL || colon == endpoint || colon[1] == '\0') {
fprintf(stderr, "invalid backend: %s\n", endpoint);
exit(2);
}
host_len = (size_t)(colon - endpoint);
port_len = strlen(colon + 1);
if (host_len >= sizeof(backend->host) || port_len >= sizeof(backend->port)) {
fprintf(stderr, "backend is too long: %s\n", endpoint);
exit(2);
}
memcpy(backend->host, endpoint, host_len);
backend->host[host_len] = '\0';
strcpy(backend->port, colon + 1);
}
static void add_backend(struct config *cfg, const char *endpoint)
{
if (cfg->backend_count >= MAX_BACKENDS) {
fprintf(stderr, "too many backends, max=%d\n", MAX_BACKENDS);
exit(2);
}
parse_backend_endpoint(endpoint, &cfg->backends[cfg->backend_count]);
cfg->backend_count++;
}
static bool is_safe_owner(const char *owner)
{
if (owner[0] == '\0') {
return false;
}
for (const unsigned char *p = (const unsigned char *)owner; *p; p++) {
if ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') ||
(*p >= '0' && *p <= '9') || *p == '_' || *p == '-' || *p == '.') {
continue;
}
return false;
}
return true;
}
static void add_owner_backend(struct config *cfg, const char *rule)
{
const char *eq = strchr(rule, '=');
size_t owner_len;
if (cfg->owner_route_count >= MAX_OWNER_ROUTES) {
fprintf(stderr, "too many owner routes, max=%d\n", MAX_OWNER_ROUTES);
exit(2);
}
if (eq == NULL || eq == rule || eq[1] == '\0') {
fprintf(stderr, "invalid owner backend rule: %s\n", rule);
exit(2);
}
owner_len = (size_t)(eq - rule);
if (owner_len >= sizeof(cfg->owner_routes[0].owner)) {
fprintf(stderr, "owner is too long: %s\n", rule);
exit(2);
}
memcpy(cfg->owner_routes[cfg->owner_route_count].owner, rule, owner_len);
cfg->owner_routes[cfg->owner_route_count].owner[owner_len] = '\0';
if (!is_safe_owner(cfg->owner_routes[cfg->owner_route_count].owner)) {
fprintf(stderr, "invalid owner name: %s\n", cfg->owner_routes[cfg->owner_route_count].owner);
exit(2);
}
parse_backend_endpoint(eq + 1, &cfg->owner_routes[cfg->owner_route_count].backend);
cfg->owner_route_count++;
}
static char *trim_line(char *s)
{
char *end;
while (*s == ' ' || *s == '\t' || *s == '\r' || *s == '\n') {
s++;
}
end = s + strlen(s);
while (end > s && (end[-1] == ' ' || end[-1] == '\t' ||
end[-1] == '\r' || end[-1] == '\n')) {
*--end = '\0';
}
return s;
}
static void load_owner_map(struct config *cfg, const char *path)
{
FILE *fp = fopen(path, "r");
char line[1024];
unsigned long lineno = 0;
if (fp == NULL) {
perror(path);
exit(2);
}
while (fgets(line, sizeof(line), fp) != NULL) {
char *rule;
lineno++;
rule = trim_line(line);
if (rule[0] == '\0' || rule[0] == '#') {
continue;
}
add_owner_backend(cfg, rule);
}
if (ferror(fp)) {
fprintf(stderr, "failed reading owner map: %s\n", path);
fclose(fp);
exit(2);
}
fclose(fp);
fprintf(stderr, "loaded owner map: %s routes=%zu\n", path, cfg->owner_route_count);
(void)lineno;
}
static struct config parse_args(int argc, char **argv)
{
struct config cfg = {
@ -64,6 +219,10 @@ static struct config parse_args(int argc, char **argv)
.bind_port = DEFAULT_BIND_PORT,
.repo_root = DEFAULT_REPO_ROOT,
.host_key = DEFAULT_HOST_KEY,
.backend_user = DEFAULT_BACKEND_USER,
.backend_identity = NULL,
.backend_count = 0,
.owner_route_count = 0,
};
for (int i = 1; i < argc; i++) {
@ -75,6 +234,16 @@ static struct config parse_args(int argc, char **argv)
cfg.repo_root = argv[++i];
} else if (strcmp(argv[i], "--host-key") == 0 && i + 1 < argc) {
cfg.host_key = argv[++i];
} else if (strcmp(argv[i], "--backend") == 0 && i + 1 < argc) {
add_backend(&cfg, argv[++i]);
} else if (strcmp(argv[i], "--backend-user") == 0 && i + 1 < argc) {
cfg.backend_user = argv[++i];
} else if (strcmp(argv[i], "--backend-identity") == 0 && i + 1 < argc) {
cfg.backend_identity = argv[++i];
} else if (strcmp(argv[i], "--owner-backend") == 0 && i + 1 < argc) {
add_owner_backend(&cfg, argv[++i]);
} else if (strcmp(argv[i], "--owner-map") == 0 && i + 1 < argc) {
load_owner_map(&cfg, argv[++i]);
} else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) {
usage(argv[0]);
exit(0);
@ -119,6 +288,8 @@ static bool has_bad_repo_chars(const char *s)
case '<':
case '>':
case '\\':
case '\'':
case '"':
return true;
default:
break;
@ -197,6 +368,84 @@ static int parse_git_exec(const char *command, struct git_request *req)
return 0;
}
static uint32_t fnv1a_hash(const char *s, size_t len)
{
uint32_t h = 2166136261u;
for (size_t i = 0; i < len; i++) {
h ^= (unsigned char)s[i];
h *= 16777619u;
}
return h;
}
static int get_owner(const char *repo, char *owner, size_t owner_len)
{
const char *slash = strchr(repo, '/');
size_t n;
if (slash == NULL || slash == repo) {
return -1;
}
n = (size_t)(slash - repo);
if (n + 1 > owner_len) {
return -1;
}
memcpy(owner, repo, n);
owner[n] = '\0';
return 0;
}
static const struct backend *select_backend(const struct config *cfg, const char *repo)
{
char owner[256];
uint32_t hash;
size_t index;
if (cfg->backend_count == 0 && cfg->owner_route_count == 0) {
return NULL;
}
if (get_owner(repo, owner, sizeof(owner)) != 0) {
return NULL;
}
for (size_t i = cfg->owner_route_count; i > 0; i--) {
const struct owner_route *route = &cfg->owner_routes[i - 1];
if (strcmp(owner, route->owner) == 0) {
fprintf(stderr, "route: owner=%s override=1 backend=%s:%s\n",
owner,
route->backend.host,
route->backend.port);
return &route->backend;
}
}
if (cfg->backend_count == 0) {
return NULL;
}
hash = fnv1a_hash(owner, strlen(owner));
index = hash % cfg->backend_count;
fprintf(stderr, "route: owner=%s hash=%u backend_index=%zu backend=%s:%s\n",
owner,
hash,
index,
cfg->backends[index].host,
cfg->backends[index].port);
return &cfg->backends[index];
}
static void build_git_command(const struct git_request *req, char *buf, size_t len)
{
snprintf(buf, len, "%s '%s'", req->program, req->repo);
}
static int authenticate_client(ssh_session session)
{
ssh_message msg;
@ -287,12 +536,20 @@ static int spawn_git_child(const struct config *cfg, const struct git_request *r
int in_pipe[2] = {-1, -1};
int out_pipe[2] = {-1, -1};
int err_pipe[2] = {-1, -1};
const struct backend *backend = select_backend(cfg, req->repo);
char remote_target[320];
char remote_command[1400];
child->pid = -1;
child->stdin_fd = -1;
child->stdout_fd = -1;
child->stderr_fd = -1;
if ((cfg->backend_count > 0 || cfg->owner_route_count > 0) && backend == NULL) {
fprintf(stderr, "distributed backend mode has no route for repo path: %s\n", req->repo);
return -1;
}
if (pipe(in_pipe) != 0 || pipe(out_pipe) != 0 || pipe(err_pipe) != 0) {
perror("pipe");
goto fail;
@ -316,14 +573,43 @@ static int spawn_git_child(const struct config *cfg, const struct git_request *r
close(err_pipe[0]);
close(err_pipe[1]);
if (chdir(cfg->repo_root) != 0) {
perror("chdir repo root");
if (backend != NULL) {
const char *argv[16];
int argc = 0;
snprintf(remote_target, sizeof(remote_target), "%s@%s", cfg->backend_user, backend->host);
build_git_command(req, remote_command, sizeof(remote_command));
argv[argc++] = "ssh";
argv[argc++] = "-o";
argv[argc++] = "BatchMode=yes";
argv[argc++] = "-o";
argv[argc++] = "StrictHostKeyChecking=no";
argv[argc++] = "-o";
argv[argc++] = "UserKnownHostsFile=/dev/null";
if (cfg->backend_identity != NULL) {
argv[argc++] = "-i";
argv[argc++] = cfg->backend_identity;
}
argv[argc++] = "-p";
argv[argc++] = backend->port;
argv[argc++] = remote_target;
argv[argc++] = remote_command;
argv[argc] = NULL;
execvp("ssh", (char *const *)argv);
perror("execvp ssh backend");
_exit(127);
} else {
if (chdir(cfg->repo_root) != 0) {
perror("chdir repo root");
_exit(127);
}
execlp(req->program, req->program, req->repo, (char *)NULL);
perror("execlp git program");
_exit(127);
}
execlp(req->program, req->program, req->repo, (char *)NULL);
perror("execlp git program");
_exit(127);
}
close(in_pipe[0]);