Merge pull request '修改脚本中获取网页数据的渠道与方法' (#30) from zzx_branch into master
This commit is contained in:
commit
dde0a6d0b4
|
|
@ -81,56 +81,71 @@ function Ensure-LocalClone {
|
|||
return $result
|
||||
}
|
||||
|
||||
# 1. Compliance scan
|
||||
# 1. Compliance scan + build file inventory from local clone
|
||||
Log-Step "1/7 Compliance scan..."
|
||||
$cloneInfo = Ensure-LocalClone -Owner $Owner -Repo $Repo -LocalPath $LocalPath
|
||||
if ($cloneInfo.ScanPath -and (Test-Path (Join-Path $cloneInfo.ScanPath ".git"))) {
|
||||
Push-Location $cloneInfo.ScanPath
|
||||
$scanPath = $cloneInfo.ScanPath
|
||||
|
||||
# Build a recursive relative file list ($names) and load README text from the
|
||||
# local clone. The remote `raw/...` and `/v1/.../sub_entries` content endpoints
|
||||
# are NOT valid API routes on this GitLink server (they return the SPA HTML),
|
||||
# so all file-based detection (README, deps, build, tests, license) reads from
|
||||
# the clone that Ensure-LocalClone already provides.
|
||||
$names = @()
|
||||
$readmeText = ""
|
||||
if ($scanPath -and (Test-Path $scanPath)) {
|
||||
Push-Location $scanPath
|
||||
try {
|
||||
$names = @(Get-ChildItem -Recurse -File -Force -ErrorAction SilentlyContinue -Name |
|
||||
Where-Object { $_ -notmatch '^\.git[\\/]' -and $_ -notmatch '[\\/]node_modules[\\/]' } |
|
||||
ForEach-Object { ($_ -replace '\\', '/') })
|
||||
foreach ($rn in @("README.md","README.MD","README","README.rst","README.txt","README.zh-CN.md","readme.md")) {
|
||||
if (Test-Path $rn) { $readmeText = Get-Content -Raw -Path $rn -Encoding UTF8 -ErrorAction SilentlyContinue; break }
|
||||
}
|
||||
} catch { }
|
||||
Pop-Location
|
||||
} else {
|
||||
# Fallback: no local clone — use the working entries endpoint for a root file list
|
||||
$defaultBranch = if ($repoJson.data.default_branch) { $repoJson.data.default_branch } else { "master" }
|
||||
$entriesRes = Invoke-GLCheck @("api", "GET", "/$Owner/$Repo/entries?ref=$defaultBranch")
|
||||
if ($entriesRes.ok -and $entriesRes.data.entries) {
|
||||
$names = @($entriesRes.data.entries | ForEach-Object { if ($_.name) { ($_.name -replace '\\','/') } else { "" } })
|
||||
}
|
||||
}
|
||||
|
||||
# Secrets / PII via compliance +scan (runs inside the clone)
|
||||
if ($scanPath -and (Test-Path (Join-Path $scanPath ".git"))) {
|
||||
Push-Location $scanPath
|
||||
$compResult = Invoke-GLCheck @("compliance", "+scan")
|
||||
Pop-Location
|
||||
if ($cloneInfo.NeedCleanup) {
|
||||
# Git objects are read-only on Windows; strip attributes first
|
||||
Get-ChildItem -Path $cloneInfo.ScanPath -Recurse -Force -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { $_.Attributes = 'Normal' }
|
||||
Remove-Item -Recurse -Force $cloneInfo.ScanPath -ErrorAction SilentlyContinue
|
||||
if (Test-Path $cloneInfo.ScanPath) {
|
||||
# Fallback: let cmd handle stubborn files
|
||||
cmd /c "rd /s /q `"$($cloneInfo.ScanPath)`"" 2>$null
|
||||
}
|
||||
if (-not (Test-Path $cloneInfo.ScanPath)) {
|
||||
Log-Info "Cleaned up temp clone."
|
||||
} else {
|
||||
Log-Warn "Could not fully remove temp clone: $($cloneInfo.ScanPath)"
|
||||
}
|
||||
}
|
||||
if ($compResult.ok) {
|
||||
$licenseOk = if ($null -ne $compResult.data.license.status) { $compResult.data.license.status } else { "" }
|
||||
if ($licenseOk -match "ok|clean|found") { $dLicense.Score = 1.0; $dLicense.Detail = "Licensed (OK)" }
|
||||
elseif ($licenseOk -eq "warning") { $dLicense.Score = 0.5; $dLicense.Detail = "License non-standard" }
|
||||
else { $dLicense.Score = 0; $dLicense.Detail = "No LICENSE file" }
|
||||
|
||||
# Fix: proper PS5.1 null check instead of "if @()"
|
||||
$secFindings = if ($compResult.data.secrets.findings) { $compResult.data.secrets.findings } else { @() }
|
||||
$secCount = if ($secFindings -is [array]) { $secFindings.Count } else { 0 }
|
||||
$piiFindings = if ($compResult.data.exposure.findings) { $compResult.data.exposure.findings } else { @() }
|
||||
$piiCount = if ($piiFindings -is [array]) { $piiFindings.Count } else { 0 }
|
||||
|
||||
if ($secCount -eq 0 -and $piiCount -eq 0) { $dNoSecret.Score = 1.0; $dNoSecret.Detail = "No secrets/PII found" }
|
||||
elseif ($secCount + $piiCount -le 3) { $dNoSecret.Score = 0.5; $dNoSecret.Detail = "Few suspicious items found" }
|
||||
else { $dNoSecret.Score = 0; $dNoSecret.Detail = "Multiple secret/PII leaks" }
|
||||
# NOTE: compliance +scan has NO license module (modules = secrets/exposure/vocab),
|
||||
# and findings are a FLAT array with a .module field — not nested under
|
||||
# .data.secrets/.data.exposure. License is detected separately below.
|
||||
if ($compResult.ok -and $compResult.data.findings) {
|
||||
$allFindings = @($compResult.data.findings)
|
||||
$secCount = @($allFindings | Where-Object { $_.module -eq 'secrets' }).Count
|
||||
$piiCount = @($allFindings | Where-Object { $_.module -eq 'exposure' }).Count
|
||||
$totalFindings = $secCount + $piiCount
|
||||
if ($totalFindings -eq 0) { $dNoSecret.Score = 1.0; $dNoSecret.Detail = "No secrets/PII found" }
|
||||
elseif ($totalFindings -le 3) { $dNoSecret.Score = 0.5; $dNoSecret.Detail = "Few suspicious items ($totalFindings)" }
|
||||
else { $dNoSecret.Score = 0; $dNoSecret.Detail = "Multiple secret/PII leaks ($totalFindings)" }
|
||||
} else {
|
||||
$dNoSecret.Score = 1.0; $dNoSecret.Detail = "No secrets/PII found"
|
||||
}
|
||||
} else {
|
||||
Log-Warn "Cannot access repo for compliance scan, skipping this dimension"
|
||||
$dLicense.Detail = "Not scanned (repo inaccessible)"
|
||||
Log-Warn "Cannot access repo for compliance scan, skipping secrets scan"
|
||||
$dNoSecret.Detail = "Not scanned (repo inaccessible)"
|
||||
}
|
||||
|
||||
# License: detected from file inventory (compliance scanner has no license module)
|
||||
$licenseFile = $names | Where-Object { $_ -match '(^|/)(LICENSE|LICENCE|COPYING|UNLICENCE)(\.(md|txt))?$' } | Select-Object -First 1
|
||||
if ($licenseFile) { $dLicense.Score = 1.0; $dLicense.Detail = "License file found: $licenseFile" }
|
||||
elseif ($readmeText -match 'MIT License|Apache License|GPL|BSD') { $dLicense.Score = 0.5; $dLicense.Detail = "License mentioned in README" }
|
||||
else { $dLicense.Score = 0; $dLicense.Detail = "No LICENSE file" }
|
||||
|
||||
# 2. README
|
||||
Log-Step "2/7 README completeness..."
|
||||
try {
|
||||
$readmeResult = Invoke-GLCheck @("api", "GET", "raw/$Owner/$Repo/master/README.md")
|
||||
if ($readmeResult.ok) { $readmeText = if ($null -ne $readmeResult.data) { $readmeResult.data } else { "" } } else { $readmeText = "" }
|
||||
} catch { $readmeText = "" }
|
||||
# README text was already loaded from the local clone in step 1.
|
||||
|
||||
$sectionCount = 0
|
||||
foreach ($kw in @("# ", "## ", "Install", "Usage", "License", "Contribut", "Citation")) {
|
||||
|
|
@ -143,15 +158,11 @@ Log-Info " README sections: $sectionCount"
|
|||
|
||||
# 3. Dependencies
|
||||
Log-Step "3/7 Dependency declaration..."
|
||||
$subResult = Invoke-GLCheck @("api", "GET", "/v1/$Owner/$Repo/sub_entries?ref=master")
|
||||
# $names (relative file paths from the clone) was built in step 1.
|
||||
$depFiles = 0
|
||||
$depList = ""
|
||||
$names = @()
|
||||
if ($subResult.ok -and ($subResult.data -is [array])) {
|
||||
$names = @($subResult.data | ForEach-Object { if ($null -ne $_.name) { $_.name } else { "" } })
|
||||
foreach ($df in @("package.json","go.mod","requirements.txt","pyproject.toml","Cargo.toml","CMakeLists.txt","pom.xml","build.gradle","Gemfile","Makefile")) {
|
||||
if ($names -contains $df) { $depFiles++; $depList += "$df, " }
|
||||
}
|
||||
foreach ($df in @("package.json","go.mod","requirements.txt","pyproject.toml","setup.py","setup.cfg","Cargo.toml","CMakeLists.txt","pom.xml","build.gradle","Gemfile","Makefile")) {
|
||||
if ($names | Where-Object { (Split-Path $_ -Leaf) -eq $df }) { $depFiles++; $depList += "$df, " }
|
||||
}
|
||||
if ($depFiles -ge 1) { $dDeps.Score = 1.0; $dDeps.Detail = "Standard dep file(s): $depList".TrimEnd(', ') }
|
||||
elseif ($readmeText -match "dependenc|requirement|install") { $dDeps.Score = 0.5; $dDeps.Detail = "Deps mentioned in README" }
|
||||
|
|
@ -208,6 +219,18 @@ elseif ($dataScore -ge 1) { $dData.Score = 0.5; $dData.Detail = "Partial data st
|
|||
else { $dData.Score = 0; $dData.Detail = "No data availability statement" }
|
||||
Log-Info " Data score: $dataScore/3"
|
||||
|
||||
# Cleanup temp clone (deferred until after all file-based checks are done)
|
||||
if ($cloneInfo -and $cloneInfo.NeedCleanup -and $cloneInfo.ScanPath -and (Test-Path $cloneInfo.ScanPath)) {
|
||||
Get-ChildItem -Path $cloneInfo.ScanPath -Recurse -Force -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { $_.Attributes = 'Normal' }
|
||||
Remove-Item -Recurse -Force $cloneInfo.ScanPath -ErrorAction SilentlyContinue
|
||||
if (Test-Path $cloneInfo.ScanPath) {
|
||||
cmd /c "rd /s /q `"$($cloneInfo.ScanPath)`"" 2>$null
|
||||
}
|
||||
if (-not (Test-Path $cloneInfo.ScanPath)) { Log-Info "Cleaned up temp clone." }
|
||||
else { Log-Warn "Could not fully remove temp clone: $($cloneInfo.ScanPath)" }
|
||||
}
|
||||
|
||||
# Total score
|
||||
$weights = @(0.15, 0.15, 0.15, 0.15, 0.10, 0.10, 0.10, 0.10)
|
||||
$scores = @($dLicense.Score, $dNoSecret.Score, $dReadme.Score, $dDeps.Score, $dBuild.Score, $dCI.Score, $dTest.Score, $dData.Score)
|
||||
|
|
@ -385,7 +408,7 @@ if (-not $DryRun) {
|
|||
# Console summary
|
||||
Divider
|
||||
Write-Host "====== Reproducibility Scorecard ======" -ForegroundColor White
|
||||
Write-Host " Total score: $totalScore/100 —$grade ($gradeLabel)"
|
||||
Write-Host " Total score: $totalScore/100 [$grade] ($gradeLabel)"
|
||||
$dimNames = @("License","No Secrets/PII","README","Dependencies","Build","CI","Tests","Data")
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
$icon = if ($scores[$i] -eq 1.0) { "OK" } elseif ($scores[$i] -eq 0.5) { "~" } else { "!!" }
|
||||
|
|
|
|||
Loading…
Reference in New Issue