From 86eff9524469541ddbaab85cf23d06a15120a01b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8B=97gogo?= Date: Wed, 8 Jul 2026 09:14:21 +0800 Subject: [PATCH] Update research-compliance workflow on zzx_branch Co-Authored-By: Claude Fable 5 --- workflows/academic/08-research-compliance.ps1 | 117 +++++++++++------- 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/workflows/academic/08-research-compliance.ps1 b/workflows/academic/08-research-compliance.ps1 index 3edd8c9..ab87d70 100644 --- a/workflows/academic/08-research-compliance.ps1 +++ b/workflows/academic/08-research-compliance.ps1 @@ -81,56 +81,71 @@ function Ensure-LocalClone { return $result } -# 1. Compliance scan +# 1. Compliance scan + build file inventory from local clone Log-Step "1/7 Compliance scan..." $cloneInfo = Ensure-LocalClone -Owner $Owner -Repo $Repo -LocalPath $LocalPath -if ($cloneInfo.ScanPath -and (Test-Path (Join-Path $cloneInfo.ScanPath ".git"))) { - Push-Location $cloneInfo.ScanPath +$scanPath = $cloneInfo.ScanPath + +# Build a recursive relative file list ($names) and load README text from the +# local clone. The remote `raw/...` and `/v1/.../sub_entries` content endpoints +# are NOT valid API routes on this GitLink server (they return the SPA HTML), +# so all file-based detection (README, deps, build, tests, license) reads from +# the clone that Ensure-LocalClone already provides. +$names = @() +$readmeText = "" +if ($scanPath -and (Test-Path $scanPath)) { + Push-Location $scanPath + try { + $names = @(Get-ChildItem -Recurse -File -Force -ErrorAction SilentlyContinue -Name | + Where-Object { $_ -notmatch '^\.git[\\/]' -and $_ -notmatch '[\\/]node_modules[\\/]' } | + ForEach-Object { ($_ -replace '\\', '/') }) + foreach ($rn in @("README.md","README.MD","README","README.rst","README.txt","README.zh-CN.md","readme.md")) { + if (Test-Path $rn) { $readmeText = Get-Content -Raw -Path $rn -Encoding UTF8 -ErrorAction SilentlyContinue; break } + } + } catch { } + Pop-Location +} else { + # Fallback: no local clone — use the working entries endpoint for a root file list + $defaultBranch = if ($repoJson.data.default_branch) { $repoJson.data.default_branch } else { "master" } + $entriesRes = Invoke-GLCheck @("api", "GET", "/$Owner/$Repo/entries?ref=$defaultBranch") + if ($entriesRes.ok -and $entriesRes.data.entries) { + $names = @($entriesRes.data.entries | ForEach-Object { if ($_.name) { ($_.name -replace '\\','/') } else { "" } }) + } +} + +# Secrets / PII via compliance +scan (runs inside the clone) +if ($scanPath -and (Test-Path (Join-Path $scanPath ".git"))) { + Push-Location $scanPath $compResult = Invoke-GLCheck @("compliance", "+scan") Pop-Location - if ($cloneInfo.NeedCleanup) { - # Git objects are read-only on Windows; strip attributes first - Get-ChildItem -Path $cloneInfo.ScanPath -Recurse -Force -ErrorAction SilentlyContinue | - ForEach-Object { $_.Attributes = 'Normal' } - Remove-Item -Recurse -Force $cloneInfo.ScanPath -ErrorAction SilentlyContinue - if (Test-Path $cloneInfo.ScanPath) { - # Fallback: let cmd handle stubborn files - cmd /c "rd /s /q `"$($cloneInfo.ScanPath)`"" 2>$null - } - if (-not (Test-Path $cloneInfo.ScanPath)) { - Log-Info "Cleaned up temp clone." - } else { - Log-Warn "Could not fully remove temp clone: $($cloneInfo.ScanPath)" - } - } - if ($compResult.ok) { - $licenseOk = if ($null -ne $compResult.data.license.status) { $compResult.data.license.status } else { "" } - if ($licenseOk -match "ok|clean|found") { $dLicense.Score = 1.0; $dLicense.Detail = "Licensed (OK)" } - elseif ($licenseOk -eq "warning") { $dLicense.Score = 0.5; $dLicense.Detail = "License non-standard" } - else { $dLicense.Score = 0; $dLicense.Detail = "No LICENSE file" } - - # Fix: proper PS5.1 null check instead of "if @()" - $secFindings = if ($compResult.data.secrets.findings) { $compResult.data.secrets.findings } else { @() } - $secCount = if ($secFindings -is [array]) { $secFindings.Count } else { 0 } - $piiFindings = if ($compResult.data.exposure.findings) { $compResult.data.exposure.findings } else { @() } - $piiCount = if ($piiFindings -is [array]) { $piiFindings.Count } else { 0 } - - if ($secCount -eq 0 -and $piiCount -eq 0) { $dNoSecret.Score = 1.0; $dNoSecret.Detail = "No secrets/PII found" } - elseif ($secCount + $piiCount -le 3) { $dNoSecret.Score = 0.5; $dNoSecret.Detail = "Few suspicious items found" } - else { $dNoSecret.Score = 0; $dNoSecret.Detail = "Multiple secret/PII leaks" } + # NOTE: compliance +scan has NO license module (modules = secrets/exposure/vocab), + # and findings are a FLAT array with a .module field — not nested under + # .data.secrets/.data.exposure. License is detected separately below. + if ($compResult.ok -and $compResult.data.findings) { + $allFindings = @($compResult.data.findings) + $secCount = @($allFindings | Where-Object { $_.module -eq 'secrets' }).Count + $piiCount = @($allFindings | Where-Object { $_.module -eq 'exposure' }).Count + $totalFindings = $secCount + $piiCount + if ($totalFindings -eq 0) { $dNoSecret.Score = 1.0; $dNoSecret.Detail = "No secrets/PII found" } + elseif ($totalFindings -le 3) { $dNoSecret.Score = 0.5; $dNoSecret.Detail = "Few suspicious items ($totalFindings)" } + else { $dNoSecret.Score = 0; $dNoSecret.Detail = "Multiple secret/PII leaks ($totalFindings)" } + } else { + $dNoSecret.Score = 1.0; $dNoSecret.Detail = "No secrets/PII found" } } else { - Log-Warn "Cannot access repo for compliance scan, skipping this dimension" - $dLicense.Detail = "Not scanned (repo inaccessible)" + Log-Warn "Cannot access repo for compliance scan, skipping secrets scan" $dNoSecret.Detail = "Not scanned (repo inaccessible)" } +# License: detected from file inventory (compliance scanner has no license module) +$licenseFile = $names | Where-Object { $_ -match '(^|/)(LICENSE|LICENCE|COPYING|UNLICENCE)(\.(md|txt))?$' } | Select-Object -First 1 +if ($licenseFile) { $dLicense.Score = 1.0; $dLicense.Detail = "License file found: $licenseFile" } +elseif ($readmeText -match 'MIT License|Apache License|GPL|BSD') { $dLicense.Score = 0.5; $dLicense.Detail = "License mentioned in README" } +else { $dLicense.Score = 0; $dLicense.Detail = "No LICENSE file" } + # 2. README Log-Step "2/7 README completeness..." -try { - $readmeResult = Invoke-GLCheck @("api", "GET", "raw/$Owner/$Repo/master/README.md") - if ($readmeResult.ok) { $readmeText = if ($null -ne $readmeResult.data) { $readmeResult.data } else { "" } } else { $readmeText = "" } -} catch { $readmeText = "" } +# README text was already loaded from the local clone in step 1. $sectionCount = 0 foreach ($kw in @("# ", "## ", "Install", "Usage", "License", "Contribut", "Citation")) { @@ -143,15 +158,11 @@ Log-Info " README sections: $sectionCount" # 3. Dependencies Log-Step "3/7 Dependency declaration..." -$subResult = Invoke-GLCheck @("api", "GET", "/v1/$Owner/$Repo/sub_entries?ref=master") +# $names (relative file paths from the clone) was built in step 1. $depFiles = 0 $depList = "" -$names = @() -if ($subResult.ok -and ($subResult.data -is [array])) { - $names = @($subResult.data | ForEach-Object { if ($null -ne $_.name) { $_.name } else { "" } }) - foreach ($df in @("package.json","go.mod","requirements.txt","pyproject.toml","Cargo.toml","CMakeLists.txt","pom.xml","build.gradle","Gemfile","Makefile")) { - if ($names -contains $df) { $depFiles++; $depList += "$df, " } - } +foreach ($df in @("package.json","go.mod","requirements.txt","pyproject.toml","setup.py","setup.cfg","Cargo.toml","CMakeLists.txt","pom.xml","build.gradle","Gemfile","Makefile")) { + if ($names | Where-Object { (Split-Path $_ -Leaf) -eq $df }) { $depFiles++; $depList += "$df, " } } if ($depFiles -ge 1) { $dDeps.Score = 1.0; $dDeps.Detail = "Standard dep file(s): $depList".TrimEnd(', ') } elseif ($readmeText -match "dependenc|requirement|install") { $dDeps.Score = 0.5; $dDeps.Detail = "Deps mentioned in README" } @@ -208,6 +219,18 @@ elseif ($dataScore -ge 1) { $dData.Score = 0.5; $dData.Detail = "Partial data st else { $dData.Score = 0; $dData.Detail = "No data availability statement" } Log-Info " Data score: $dataScore/3" +# Cleanup temp clone (deferred until after all file-based checks are done) +if ($cloneInfo -and $cloneInfo.NeedCleanup -and $cloneInfo.ScanPath -and (Test-Path $cloneInfo.ScanPath)) { + Get-ChildItem -Path $cloneInfo.ScanPath -Recurse -Force -ErrorAction SilentlyContinue | + ForEach-Object { $_.Attributes = 'Normal' } + Remove-Item -Recurse -Force $cloneInfo.ScanPath -ErrorAction SilentlyContinue + if (Test-Path $cloneInfo.ScanPath) { + cmd /c "rd /s /q `"$($cloneInfo.ScanPath)`"" 2>$null + } + if (-not (Test-Path $cloneInfo.ScanPath)) { Log-Info "Cleaned up temp clone." } + else { Log-Warn "Could not fully remove temp clone: $($cloneInfo.ScanPath)" } +} + # Total score $weights = @(0.15, 0.15, 0.15, 0.15, 0.10, 0.10, 0.10, 0.10) $scores = @($dLicense.Score, $dNoSecret.Score, $dReadme.Score, $dDeps.Score, $dBuild.Score, $dCI.Score, $dTest.Score, $dData.Score) @@ -385,7 +408,7 @@ if (-not $DryRun) { # Console summary Divider Write-Host "====== Reproducibility Scorecard ======" -ForegroundColor White -Write-Host " Total score: $totalScore/100 —$grade ($gradeLabel)" +Write-Host " Total score: $totalScore/100 [$grade] ($gradeLabel)" $dimNames = @("License","No Secrets/PII","README","Dependencies","Build","CI","Tests","Data") for ($i = 0; $i -lt 8; $i++) { $icon = if ($scores[$i] -eq 1.0) { "OK" } elseif ($scores[$i] -eq 0.5) { "~" } else { "!!" }