forked from Gitlink/gitlink-cli
522 lines
19 KiB
Bash
Executable File
522 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ============================================================================
|
|
# import-test-data.sh
|
|
# Import 15 issues + 6 PRs from upstream (gitlink/gitlink-cli) into the
|
|
# test repo (baoerjun/gitlink-cli) for repeatable testing.
|
|
#
|
|
# Usage:
|
|
# ./scripts/import-test-data.sh [ISSUE_COUNT] [PR_COUNT]
|
|
#
|
|
# ISSUE_COUNT Number of issues to import (default: 15)
|
|
# PR_COUNT Number of PRs to import (default: 6)
|
|
#
|
|
# --yes Skip confirmation prompts
|
|
# ============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
DATA_DIR="$PROJECT_DIR/data"
|
|
CLI="${PROJECT_DIR}/gitlink-cli"
|
|
MANIFEST="$DATA_DIR/test-data-manifest.json"
|
|
|
|
UPSTREAM_OWNER="gitlink"
|
|
TEST_OWNER="baoerjun"
|
|
REPO="gitlink-cli"
|
|
FORK_REMOTE="fork"
|
|
|
|
ISSUE_COUNT="${1:-15}"
|
|
PR_COUNT="${2:-6}"
|
|
|
|
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $*"; }
|
|
log_step() { echo -e "${CYAN}[STEP]${NC} $*"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Check prerequisites
|
|
# ---------------------------------------------------------------------------
|
|
check_prereqs() {
|
|
if [ ! -f "$CLI" ]; then
|
|
log_error "gitlink-cli binary not found at $CLI"
|
|
exit 1
|
|
fi
|
|
if ! git remote get-url "$FORK_REMOTE" &>/dev/null; then
|
|
log_error "Git remote '$FORK_REMOTE' not found. Required for pushing PR branches."
|
|
exit 1
|
|
fi
|
|
mkdir -p "$DATA_DIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Idempotency guard: check for existing manifest
|
|
# ---------------------------------------------------------------------------
|
|
check_existing_manifest() {
|
|
if [ -f "$MANIFEST" ]; then
|
|
log_warn "Existing manifest found at $MANIFEST"
|
|
log_warn "Test data may already exist in the test repo."
|
|
echo ""
|
|
read -r -p "Run restore first, then re-import? [y/N] " answer
|
|
if [[ "$answer" =~ ^[Yy]$ ]]; then
|
|
log_info "Please run: scripts/restore-test-repo.sh first, then re-run this script."
|
|
exit 0
|
|
fi
|
|
log_info "Continuing will overwrite the existing manifest..."
|
|
fi
|
|
|
|
# Also warn if test repo already has issues beyond baseline
|
|
local existing
|
|
existing=$("$CLI" --owner "$TEST_OWNER" --repo "$REPO" \
|
|
issue +list --format json --limit 1 2>/dev/null \
|
|
| python3 -c "import sys,json; d=json.load(sys.stdin); print(d['data'].get('total_count',0))" 2>/dev/null || echo "0")
|
|
if [ "$existing" -gt 0 ] 2>/dev/null; then
|
|
log_warn "Test repo already has $existing issue(s). The restore script will delete ALL of them."
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Git state save/restore
|
|
# ---------------------------------------------------------------------------
|
|
save_git_state() {
|
|
ORIG_BRANCH="$(git branch --show-current)"
|
|
if ! git diff-index --quiet HEAD -- 2>/dev/null; then
|
|
log_info "Stashing uncommitted changes before git operations..."
|
|
git stash push -m "import-test-data: auto-stash $(date +%s)"
|
|
STASHED=1
|
|
else
|
|
STASHED=0
|
|
fi
|
|
}
|
|
|
|
restore_git_state() {
|
|
log_info "Restoring git state..."
|
|
git checkout "$ORIG_BRANCH" 2>/dev/null || true
|
|
if [ "${STASHED:-0}" -eq 1 ]; then
|
|
git stash pop 2>/dev/null || log_warn "Could not pop stash"
|
|
fi
|
|
}
|
|
|
|
# ===========================================================================
|
|
# Python helper: handles ALL CLI interactions involving JSON
|
|
# ===========================================================================
|
|
run_python_phase() {
|
|
local phase="$1" # "issues" or "prs_fetch" or "pr_create"
|
|
shift
|
|
|
|
python3 << 'PYEOF'
|
|
import json, subprocess, sys, os, textwrap
|
|
|
|
CLI = os.environ['CLI_PATH']
|
|
UPSTREAM_OWNER = os.environ['UPSTREAM_OWNER']
|
|
TEST_OWNER = os.environ['TEST_OWNER']
|
|
REPO = os.environ['REPO']
|
|
ISSUE_COUNT = int(os.environ['ISSUE_COUNT'])
|
|
PR_COUNT = int(os.environ['PR_COUNT'])
|
|
DATA_DIR = os.environ['DATA_DIR']
|
|
PHASE = os.environ['PY_PHASE']
|
|
|
|
def run_cli(*args, timeout=90):
|
|
"""Run gitlink-cli, return parsed JSON. Exits on failure."""
|
|
cmd = [CLI] + list(args)
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
|
if result.returncode != 0:
|
|
print(f"CLI error (exit={result.returncode}): {' '.join(cmd)}", file=sys.stderr)
|
|
if result.stderr:
|
|
print(f"stderr: {result.stderr.strip()[:500]}", file=sys.stderr)
|
|
if result.stdout:
|
|
print(f"stdout: {result.stdout.strip()[:500]}", file=sys.stderr)
|
|
sys.exit(1)
|
|
try:
|
|
return json.loads(result.stdout)
|
|
except json.JSONDecodeError as e:
|
|
print(f"JSON parse error: {e}", file=sys.stderr)
|
|
print(f"Command: {' '.join(cmd)}", file=sys.stderr)
|
|
print(f"stdout: {result.stdout[:800]}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
# =========================================================================
|
|
# PHASE: issues
|
|
# =========================================================================
|
|
if PHASE == 'issues':
|
|
print("[INFO] Fetching upstream issues...")
|
|
result = run_cli('--owner', UPSTREAM_OWNER, '--repo', REPO,
|
|
'issue', '+list', '--format', 'json', '--limit', str(ISSUE_COUNT))
|
|
issues = result['data']['issues']
|
|
print(f"[INFO] Got {len(issues)} issues from upstream")
|
|
|
|
# Save raw upstream data
|
|
with open(os.path.join(DATA_DIR, 'upstream-issues.json'), 'w') as f:
|
|
json.dump(issues, f, ensure_ascii=False, indent=2)
|
|
|
|
created = []
|
|
for i, issue in enumerate(issues, 1):
|
|
number = issue['number']
|
|
subject = issue.get('subject', '(no title)')
|
|
subject_short = subject[:80] + ('...' if len(subject) > 80 else '')
|
|
print(f"[INFO] [{i}/{len(issues)}] Fetching issue #{number}: {subject_short}")
|
|
|
|
# Get full details (includes body/description)
|
|
try:
|
|
detail = run_cli('--owner', UPSTREAM_OWNER, '--repo', REPO,
|
|
'issue', '+view', '--number', str(number), '--format', 'json')
|
|
issue_data = detail.get('data', detail)
|
|
body = (issue_data.get('description') or '').strip()
|
|
priority_id = str(issue_data.get('priority_id') or '2')
|
|
except Exception as e:
|
|
print(f"[WARN] Could not fetch details for #{number}: {e}, using list data only")
|
|
body = ''
|
|
priority_id = '2'
|
|
|
|
# Create in test repo via python subprocess (safe body handling)
|
|
print(f"[INFO] [{i}/{len(issues)}] Creating in test repo...")
|
|
create_args = [
|
|
'--owner', TEST_OWNER, '--repo', REPO,
|
|
'issue', '+create', '--format', 'json',
|
|
'--title', subject,
|
|
'--priority-id', priority_id,
|
|
]
|
|
if body:
|
|
create_args.extend(['--body', body])
|
|
|
|
try:
|
|
create_result = run_cli(*create_args, timeout=30)
|
|
new_data = create_result.get('data', create_result)
|
|
new_id = new_data.get('id')
|
|
new_number = new_data.get('project_issues_index')
|
|
print(f"[OK] Created issue #{new_number} (db_id={new_id})")
|
|
created.append({
|
|
'upstream_number': number,
|
|
'upstream_db_id': issue['database_id'],
|
|
'test_db_id': new_id,
|
|
'test_number': new_number,
|
|
'subject': subject,
|
|
})
|
|
except Exception as e:
|
|
print(f"[ERROR] Failed to create issue #{number}: {e}")
|
|
created.append({
|
|
'upstream_number': number,
|
|
'upstream_db_id': issue['database_id'],
|
|
'error': str(e),
|
|
})
|
|
|
|
# Output marker + JSON for bash to capture
|
|
print("__PYTHON_RESULT__")
|
|
print(json.dumps(created, ensure_ascii=False))
|
|
|
|
# =========================================================================
|
|
# PHASE: prs_fetch — fetch upstream PR data
|
|
# =========================================================================
|
|
elif PHASE == 'prs_fetch':
|
|
print("[INFO] Fetching upstream PRs...")
|
|
result = run_cli('--owner', UPSTREAM_OWNER, '--repo', REPO,
|
|
'pr', '+list', '--format', 'json', '--limit', str(PR_COUNT))
|
|
prs = result['data']['pulls']
|
|
print(f"[INFO] Got {len(prs)} PRs from upstream")
|
|
|
|
with open(os.path.join(DATA_DIR, 'upstream-prs.json'), 'w') as f:
|
|
json.dump(prs, f, ensure_ascii=False, indent=2)
|
|
|
|
# Return structured data for bash
|
|
pr_commands = []
|
|
for i, pr in enumerate(prs, 1):
|
|
title = pr.get('title', '(no title)')
|
|
body = (pr.get('body') or '').strip()
|
|
head_branch = f"test-import/pr-{i}"
|
|
base_branch = pr.get('base', 'master')
|
|
|
|
title_short = title[:80] + ('...' if len(title) > 80 else '')
|
|
print(f"[INFO] [{i}/{len(prs)}] {title_short}")
|
|
|
|
pr_commands.append({
|
|
'index': i,
|
|
'title': title,
|
|
'body': body,
|
|
'head': head_branch,
|
|
'base': base_branch,
|
|
'upstream_id': pr['id'],
|
|
'upstream_index': pr['index'],
|
|
})
|
|
|
|
print("__PYTHON_RESULT__")
|
|
print(json.dumps(pr_commands, ensure_ascii=False))
|
|
|
|
# =========================================================================
|
|
# PHASE: pr_create_one — create a single PR via CLI (called after git ops)
|
|
# =========================================================================
|
|
elif PHASE == 'pr_create_one':
|
|
# Read PR data from temp file (avoids shell quoting issues with body text)
|
|
pr_data_file = os.environ['PR_DATA_FILE']
|
|
with open(pr_data_file, 'r') as f:
|
|
pr = json.load(f)
|
|
|
|
title = pr['title']
|
|
body = pr.get('body', '')
|
|
head = pr['head']
|
|
base = pr['base']
|
|
|
|
create_args = [
|
|
'--owner', TEST_OWNER, '--repo', REPO,
|
|
'pr', '+create', '--format', 'json',
|
|
'--title', title,
|
|
'--head', head,
|
|
'--base', base,
|
|
]
|
|
if body:
|
|
create_args.extend(['--body', body])
|
|
|
|
try:
|
|
result = run_cli(*create_args, timeout=30)
|
|
pr_data = result.get('data', result)
|
|
created_number = pr_data.get('index') or pr_data.get('number') or pr_data.get('id')
|
|
print(f"[OK] Created PR #{created_number}")
|
|
print("__PYTHON_RESULT__")
|
|
print(json.dumps({'ok': True, 'number': str(created_number), 'title': title}))
|
|
except Exception as e:
|
|
print(f"[ERROR] Failed to create PR: {e}")
|
|
print("__PYTHON_RESULT__")
|
|
print(json.dumps({'ok': False, 'error': str(e)}))
|
|
|
|
# =========================================================================
|
|
# PHASE: build_manifest — final manifest construction
|
|
# =========================================================================
|
|
elif PHASE == 'build_manifest':
|
|
issues_json = os.environ['ISSUES_JSON']
|
|
prs_json = os.environ['PRS_JSON']
|
|
from datetime import datetime
|
|
|
|
issues = json.loads(issues_json)
|
|
prs = json.loads(prs_json)
|
|
|
|
manifest = {
|
|
'created_at': datetime.now().isoformat(),
|
|
'upstream': 'gitlink/gitlink-cli',
|
|
'test_repo': 'baoerjun/gitlink-cli',
|
|
'issues': issues,
|
|
'prs': prs,
|
|
'summary': {
|
|
'issues_imported': len([i for i in issues if 'test_db_id' in i]),
|
|
'issues_failed': len([i for i in issues if 'error' in i]),
|
|
'prs_created': len([p for p in prs if p.get('ok')]),
|
|
'prs_failed': len([p for p in prs if not p.get('ok')]),
|
|
}
|
|
}
|
|
|
|
manifest_path = os.environ['MANIFEST_PATH']
|
|
with open(manifest_path, 'w') as f:
|
|
json.dump(manifest, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"[OK] Manifest saved to {manifest_path}")
|
|
s = manifest['summary']
|
|
print(f"[INFO] Issues: {s['issues_imported']} imported, {s['issues_failed']} failed")
|
|
print(f"[INFO] PRs: {s['prs_created']} created, {s['prs_failed']} failed")
|
|
PYEOF
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Create a PR branch, push it, then call Python to create the PR via API
|
|
# ---------------------------------------------------------------------------
|
|
create_single_pr() {
|
|
local index="$1"
|
|
local branch="$2"
|
|
local title="$3"
|
|
local body="$4"
|
|
local base="$5"
|
|
|
|
# All diagnostic output goes to stderr so stdout contains only JSON
|
|
{
|
|
log_info " [$index/$PR_COUNT] Creating branch '$branch' from '$base'..."
|
|
|
|
# Clean up any previous attempt
|
|
git branch -D "$branch" 2>/dev/null || true
|
|
git push "$FORK_REMOTE" --delete "$branch" 2>/dev/null || true
|
|
|
|
git checkout master
|
|
git checkout -b "$branch"
|
|
|
|
# Create a dummy file as PR content
|
|
local marker_file="test-pr-import-${index}.md"
|
|
printf '%s\n' "# Test PR #${index}" "" "Imported from upstream gitlink/gitlink-cli." \
|
|
"Upstream PR title: ${title}" > "$marker_file"
|
|
git add "$marker_file"
|
|
git commit -m "test: import PR #${index} from upstream" \
|
|
-m "Source: gitlink/gitlink-cli" \
|
|
-m "This is test data for gitlink-cli testing." --allow-empty
|
|
|
|
log_info " [$index/$PR_COUNT] Pushing '$branch' to $FORK_REMOTE..."
|
|
git push "$FORK_REMOTE" "$branch"
|
|
} >&2
|
|
|
|
# Write PR data to temp file to avoid any shell quoting issues
|
|
local tmp_pr_data
|
|
tmp_pr_data=$(mktemp)
|
|
python3 -c "
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
json.dump({
|
|
'title': data['title'],
|
|
'body': data['body'],
|
|
'head': data['head'],
|
|
'base': data['base'],
|
|
}, sys.stdout, ensure_ascii=False)
|
|
" <<< "$pr_data" > "$tmp_pr_data"
|
|
|
|
# Now create the PR via Python (reads body from temp file, no shell interpolation)
|
|
log_info " [$index/$PR_COUNT] Creating PR via API..." >&2
|
|
export PR_DATA_FILE="$tmp_pr_data"
|
|
export PY_PHASE="pr_create_one"
|
|
local api_result
|
|
api_result="$(run_python_phase "pr_create")"
|
|
rm -f "$tmp_pr_data"
|
|
|
|
# Show progress from the Python output
|
|
echo "$api_result" | grep -E '^\[(OK|ERROR)\]' >&2 || true
|
|
|
|
# Extract the result JSON (this goes to stdout for capture by caller)
|
|
local pr_json
|
|
pr_json=$(echo "$api_result" | sed -n '/^__PYTHON_RESULT__$/,$p' | tail -n +2)
|
|
echo "$pr_json"
|
|
}
|
|
|
|
# ===========================================================================
|
|
# Main
|
|
# ===========================================================================
|
|
main() {
|
|
echo ""
|
|
echo -e "${CYAN}========================================${NC}"
|
|
echo -e "${CYAN} GitLink Test Data Import Script ${NC}"
|
|
echo -e "${CYAN}========================================${NC}"
|
|
echo ""
|
|
log_info "Upstream : $UPSTREAM_OWNER/$REPO"
|
|
log_info "Test repo: $TEST_OWNER/$REPO"
|
|
log_info "Issues to import : $ISSUE_COUNT"
|
|
log_info "PRs to import : $PR_COUNT"
|
|
echo ""
|
|
|
|
check_prereqs
|
|
check_existing_manifest
|
|
|
|
# Export env vars for Python subprocess
|
|
export CLI_PATH="$CLI"
|
|
export UPSTREAM_OWNER TEST_OWNER REPO ISSUE_COUNT PR_COUNT DATA_DIR MANIFEST_PATH="$MANIFEST"
|
|
|
|
# ===== Phase 1: Import Issues =====
|
|
echo ""
|
|
log_step "===== Phase 1/4: Import Issues ====="
|
|
export PY_PHASE="issues"
|
|
local issues_output
|
|
issues_output="$(run_python_phase "issues")"
|
|
|
|
# Show progress lines
|
|
echo "$issues_output" | grep -E '^\[(INFO|OK|WARN|ERROR)\]' || true
|
|
|
|
# Extract result JSON
|
|
local issues_json
|
|
issues_json="$(echo "$issues_output" | sed -n '/^__PYTHON_RESULT__$/,$p' | tail -n +2)"
|
|
|
|
if [ -z "$issues_json" ]; then
|
|
log_error "Issue import produced no output. Check CLI connectivity."
|
|
exit 1
|
|
fi
|
|
|
|
local ok_count
|
|
ok_count=$(echo "$issues_json" | python3 -c "import sys,json; d=json.load(sys.stdin); print(len([i for i in d if 'test_db_id' in i]))")
|
|
log_info "Issues created: $ok_count / $ISSUE_COUNT"
|
|
|
|
# ===== Phase 2: Fetch PR data from upstream =====
|
|
echo ""
|
|
log_step "===== Phase 2/4: Fetch PR data from upstream ====="
|
|
export PY_PHASE="prs_fetch"
|
|
local prs_output
|
|
prs_output="$(run_python_phase "prs_fetch")"
|
|
echo "$prs_output" | grep -E '^\[(INFO|OK|WARN|ERROR)\]' || true
|
|
|
|
local prs_json
|
|
prs_json="$(echo "$prs_output" | sed -n '/^__PYTHON_RESULT__$/,$p' | tail -n +2)"
|
|
|
|
if [ -z "$prs_json" ]; then
|
|
log_error "PR fetch produced no output."
|
|
exit 1
|
|
fi
|
|
|
|
local pr_count
|
|
pr_count=$(echo "$prs_json" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
|
|
log_info "PRs to create: $pr_count"
|
|
|
|
# ===== Phase 3: Create PR branches, push, and create PRs =====
|
|
echo ""
|
|
log_step "===== Phase 3/4: Create PRs (branches + API) ====="
|
|
|
|
save_git_state
|
|
# Ensure we start from a clean master
|
|
git checkout master 2>/dev/null || true
|
|
|
|
local pr_results_json="["
|
|
local first=true
|
|
for idx in $(seq 0 $((pr_count - 1))); do
|
|
local pr_data index branch title body base
|
|
pr_data=$(echo "$prs_json" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$idx]))")
|
|
index=$(echo "$pr_data" | python3 -c "import sys,json; print(json.load(sys.stdin)['index'])")
|
|
branch=$(echo "$pr_data" | python3 -c "import sys,json; print(json.load(sys.stdin)['head'])")
|
|
title=$(echo "$pr_data" | python3 -c "import sys,json; print(json.load(sys.stdin)['title'])")
|
|
body=$(echo "$pr_data" | python3 -c "import sys,json; print(json.load(sys.stdin)['body'])")
|
|
base=$(echo "$pr_data" | python3 -c "import sys,json; print(json.load(sys.stdin)['base'])")
|
|
|
|
local pr_result
|
|
pr_result=$(create_single_pr "$index" "$branch" "$title" "$body" "$base")
|
|
|
|
if [ "$first" = true ]; then
|
|
first=false
|
|
else
|
|
pr_results_json+=","
|
|
fi
|
|
|
|
# Parse the result and add to our collection
|
|
local pr_entry
|
|
pr_entry=$(echo "$pr_result" | python3 -c "
|
|
import sys, json
|
|
r = json.load(sys.stdin)
|
|
entry = {
|
|
'index': $index,
|
|
'branch': '${branch}',
|
|
'title': r.get('title', ''),
|
|
'ok': r.get('ok', False),
|
|
'number': r.get('number', ''),
|
|
'error': r.get('error', ''),
|
|
}
|
|
print(json.dumps(entry))
|
|
" 2>/dev/null || echo "{\"index\": $index, \"branch\": \"$branch\", \"ok\": false, \"error\": \"parse failed\"}")
|
|
pr_results_json+="$pr_entry"
|
|
done
|
|
pr_results_json+="]"
|
|
|
|
restore_git_state
|
|
|
|
# ===== Phase 4: Build Manifest =====
|
|
echo ""
|
|
log_step "===== Phase 4/4: Save Manifest ====="
|
|
|
|
# Normalize PR results JSON
|
|
pr_results_json=$(echo "$pr_results_json" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)))" 2>/dev/null || echo "$pr_results_json")
|
|
|
|
export ISSUES_JSON="$issues_json"
|
|
export PRS_JSON="$pr_results_json"
|
|
export PY_PHASE="build_manifest"
|
|
run_python_phase "build_manifest"
|
|
|
|
# ===== Summary =====
|
|
echo ""
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo -e "${GREEN} Import Complete! ${NC}"
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo ""
|
|
log_info "Manifest: $MANIFEST"
|
|
log_info "Verify with:"
|
|
echo ""
|
|
echo " ${CLI} --owner ${TEST_OWNER} --repo ${REPO} issue +list --limit 5"
|
|
echo " ${CLI} --owner ${TEST_OWNER} --repo ${REPO} pr +list --limit 5"
|
|
echo ""
|
|
}
|
|
|
|
main "$@"
|