folkslinux
/
CBL-Mariner
mirror of https://github.com/microsoft/CBL-Mariner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[dev]: Resolving package build issues. (#1488) 

* Fixing 'bind', 'clang', 'systemd' provides, and 'openssh' configuration.

* Enhancing RPM resolution.
This commit is contained in:
Pawel Winogrodzki 2021-10-04 00:46:55 -07:00 committed by GitHub
parent 8740bb5658
commit 2401ea067f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 190 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
SPECS/bind/bind.spec
View File

@ -9,7 +9,7 @@
Summary: Domain Name System software
Name: bind
Version: 9.16.15
Release: 2%{?dist}
Release: 3%{?dist}
License: ISC
Vendor: Microsoft Corporation
Distribution: Mariner
@ -53,6 +53,7 @@ BuildRequires: postgresql-devel
BuildRequires: python3
BuildRequires: python3-ply
BuildRequires: sqlite-devel
BuildRequires: systemd-rpm-macros
Requires: libuv
Requires: openssl
@ -614,6 +615,9 @@ fi;
%{_tmpfilesdir}/named.conf
%changelog
* Sat Oct 02 2021 Pawel Winogrodzki <pawelwi@microsoft.com> - 9.16.15-3
- Adding missing BR on 'systemd-rpm-macros'.
* Fri Aug 27 2021 Pawel Winogrodzki <pawelwi@microsoft.com> - 9.16.15-2
- Adding DBZ subpackages using Fedora 34 (license: MIT) specs as guidance.

10
SPECS/clang/clang.spec
View File

@ -120,20 +120,10 @@ A set of extra tools built using Clang's tooling API.
%prep
%setup -q -T -b 1 -n %{clang_tools_srcdir}
pathfix.py -i python3 -pn \
clang-tidy/tool/*.py \
clang-include-fixer/find-all-symbols/tool/run-find-all-symbols.py
%setup -q -n %{clang_srcdir}
mv ../%{clang_tools_srcdir} tools/extra
pathfix.py -i python3 -pn \
tools/clang-format/*.py \
tools/clang-format/git-clang-format \
utils/hmaptool/hmaptool \
tools/scan-view/bin/scan-view
%build
# Disable symbol generation
export CFLAGS="`echo " %{build_cflags} " | sed 's/ -g//'`"

0
SPECS/kpmcore-3.3.0/kpmcore.signatures.json → SPECS/kpmcore/kpmcore.signatures.json
View File

0
SPECS/kpmcore-3.3.0/kpmcore.spec → SPECS/kpmcore/kpmcore.spec
View File

8
SPECS/openssh/openssh.spec
View File

@ -1,15 +1,15 @@
%global openssh_ver 8.5p1
%global openssh_rel 4
%global openssh_rel 4%{?dist}
%global pam_ssh_agent_ver 0.10.3
%global pam_ssh_agent_rel 10
%global pam_ssh_agent_rel 10%{?dist}
%define systemd_units_rel 20191026
Summary: Free version of the SSH connectivity tools
Name: openssh
Version: %{openssh_ver}
Release: %{openssh_rel}%{?dist}
Release: %{openssh_rel}
License: BSD
Vendor: Microsoft Corporation
Distribution: Mariner
@ -72,7 +72,7 @@ This provides the ssh client utilities.
%package -n pam_ssh_agent_auth
Summary: PAM module for authentication with ssh-agent
Version: %{pam_ssh_agent_ver}
Release: %{pam_ssh_agent_rel}.%{openssh_rel}%{?dist}
Release: %{pam_ssh_agent_rel}.%{openssh_rel}
License: BSD
%description -n pam_ssh_agent_auth

13
SPECS/systemd/systemd.spec
View File

@ -1,7 +1,7 @@
Summary: Systemd-239
Name: systemd
Version: 239
Release: 41%{?dist}
Release: 42%{?dist}
License: LGPLv2+ AND GPLv2+ AND MIT
Vendor: Microsoft Corporation
Distribution: Mariner
@ -78,7 +78,10 @@ Requires: libgcrypt
Requires: lz4
Requires: pam
Requires: xz
Obsoletes: systemd-bootstrap
Provides: systemd-bootstrap = %{version}-%{release}
Provides: systemd-units = %{version}-%{release}
Provides: systemd-sysv = %{version}-%{release}
Provides: systemd-udev = %{version}-%{release}
@ -101,6 +104,10 @@ Just the definitions of rpm macros.
Summary: Development headers for systemd
Requires: %{name} = %{version}-%{release}
Requires: glib-devel
Obsoletes: systemd-bootstrap-devel
Provides: systemd-bootstrap-devel = %{version}-%{release}
Provides: systemd-libs = %{version}-%{release}
Provides: libudev-devel = %{version}-%{release}
Provides: libudev-devel%{?_isa} = %{version}-%{release}
@ -289,6 +296,10 @@ rm -rf %{buildroot}/*
%files lang -f %{name}.lang
%changelog
* Sat Oct 02 2021 Pawel Winogrodzki <pawelwi@microsoft.com> - 239-42
- Adding 'Obsoletes: systemd-bootstrap-devel' for the 'devel' subpackage.
- Making 'systemd' obsolete 'systemd-bootstrap' regardless of version and release.
* Wed Aug 18 2021 Jon Slobodzian <joslobo@microsoft.com> - 239-41
- Merge from 1.0 to dev branch
- nehaagarwal@microsoft.com, 2.39-38: CVE-2021-33910 fix

89
toolkit/tools/graphpkgfetcher/graphpkgfetcher.go
View File

@ -17,6 +17,7 @@ import (
"microsoft.com/pkggen/internal/packagerepo/repoutils"
"microsoft.com/pkggen/internal/pkggraph"
"microsoft.com/pkggen/internal/pkgjson"
"microsoft.com/pkggen/internal/rpm"
)
var (
@ -155,7 +156,7 @@ func resolveSingleNode(cloner *rpmrepocloner.RpmRepoCloner, node *pkggraph.PkgNo
logger.Log.Debugf("Searching for a package which supplies: %s", node.VersionedPkg.Name)
// Resolve nodes to exact package names so they can be referenced in the graph.
resolvedPackage, err := cloner.WhatProvides(node.VersionedPkg)
resolvedPackages, err := cloner.WhatProvides(node.VersionedPkg)
if err != nil {
msg := fmt.Sprintf("Failed to resolve (%s) to a package. Error: %s", node.VersionedPkg, err)
// It is not an error if an implicit node could not be resolved as it may become available later in the build.
@ -168,27 +169,77 @@ func resolveSingleNode(cloner *rpmrepocloner.RpmRepoCloner, node *pkggraph.PkgNo
return
}
if !fetchedPackages[resolvedPackage] {
desiredPackage := &pkgjson.PackageVer{
Name: resolvedPackage,
}
err = cloner.Clone(cloneDeps, desiredPackage)
if err != nil {
logger.Log.Errorf("Failed to clone '%s' from RPM repo. Error: %s", resolvedPackage, err)
return
}
fetchedPackages[resolvedPackage] = true
logger.Log.Infof("Fetched: %s", resolvedPackage)
if len(resolvedPackages) == 0 {
return fmt.Errorf("failed to find any packages providing '%v'", node.VersionedPkg)
}
for _, resolvedPackage := range resolvedPackages {
if !fetchedPackages[resolvedPackage] {
desiredPackage := &pkgjson.PackageVer{
Name: resolvedPackage,
}
err = cloner.Clone(cloneDeps, desiredPackage)
if err != nil {
logger.Log.Errorf("Failed to clone '%s' from RPM repo. Error: %s", resolvedPackage, err)
return
}
fetchedPackages[resolvedPackage] = true
logger.Log.Debugf("Fetched '%s' as potential candidate.", resolvedPackage)
}
}
err = assignRPMPath(node, outDir, resolvedPackages)
if err != nil {
logger.Log.Errorf("Failed to find an RPM to provide '%s'. Error: %s", node.VersionedPkg.Name, err)
return
}
// Construct the rpm path of the cloned package.
rpmName := fmt.Sprintf("%s.rpm", resolvedPackage)
// To calculate the architecture grab the last segment of the resolved name since it will be in the NVRA format.
rpmArch := resolvedPackage[strings.LastIndex(resolvedPackage, ".")+1:]
node.RpmPath = filepath.Join(outDir, rpmArch, rpmName)
node.State = pkggraph.StateCached
logger.Log.Infof("Choosing '%s' to provide '%s'.", filepath.Base(node.RpmPath), node.VersionedPkg.Name)
return
}
func assignRPMPath(node *pkggraph.PkgNode, outDir string, resolvedPackages []string) (err error) {
rpmPaths := []string{}
for _, resolvedPackage := range resolvedPackages {
rpmPaths = append(rpmPaths, rpmPackageToRPMPath(resolvedPackage, outDir))
}
node.RpmPath = rpmPaths[0]
if len(rpmPaths) > 1 {
var resolvedRPMs []string
logger.Log.Debugf("Found %d candidates. Resolving.", len(rpmPaths))
resolvedRPMs, err = rpm.ResolveCompetingPackages(rpmPaths...)
if err != nil {
logger.Log.Errorf("Failed while trying to pick an RPM providing '%s' from the following RPMs: %v", node.VersionedPkg.Name, rpmPaths)
return
}
resolvedRPMsCount := len(resolvedRPMs)
if resolvedRPMsCount == 0 {
logger.Log.Errorf("Failed while trying to pick an RPM providing '%s'. No RPM can be installed from the following: %v", node.VersionedPkg.Name, rpmPaths)
return
}
if resolvedRPMsCount > 1 {
logger.Log.Warnf("Found %d candidates to provide '%s'. Picking the first one.", resolvedRPMsCount, node.VersionedPkg.Name)
}
node.RpmPath = rpmPackageToRPMPath(resolvedRPMs[0], outDir)
}
return
}
func rpmPackageToRPMPath(rpmPackage, outDir string) string {
// Construct the rpm path of the cloned package.
rpmName := fmt.Sprintf("%s.rpm", rpmPackage)
// To calculate the architecture grab the last segment of the resolved name since it will be in the NVRA format.
rpmArch := rpmPackage[strings.LastIndex(rpmPackage, ".")+1:]
return filepath.Join(outDir, rpmArch, rpmName)
}

2
toolkit/tools/internal/packagerepo/repocloner/repocloner.go
View File

@ -27,7 +27,7 @@ type RepoCloner interface {
Initialize(destinationDir, tmpDir, workerTar, existingRpmsDir string, useUpdateRepo, usePreviewRepo bool, repoDefinitions []string) error
AddNetworkFiles(tlsClientCert, tlsClientKey string) error
Clone(cloneDeps bool, packagesToClone ...*pkgjson.PackageVer) error
WhatProvides(pkgVer *pkgjson.PackageVer) (packageName string, err error)
WhatProvides(pkgVer *pkgjson.PackageVer) (packageNames []string, err error)
ConvertDownloadedPackagesIntoRepo() error
ClonedRepoContents() (repoContents *RepoContents, err error)
CloneDirectory() string

85
toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go
View File

@ -21,6 +21,8 @@ import (
)
const (
allRepoIDs = "*"
builtRepoID = "local-repo"
cacheRepoID = "upstream-cache-repo"
squashChrootRunErrors = false
chrootDownloadDir = "/outputrpms"
@ -256,9 +258,6 @@ func (r *RpmRepoCloner) Clone(cloneDeps bool, packagesToClone ...*pkgjson.Packag
strictComparisonOperator = "="
lessThanOrEqualComparisonOperator = "<="
versionSuffixFormat = "-%s"
builtRepoID = "local-repo"
allRepoIDs = "*"
)
for _, pkg := range packagesToClone {
@ -291,60 +290,76 @@ func (r *RpmRepoCloner) Clone(cloneDeps bool, packagesToClone ...*pkgjson.Packag
return
}
// WhatProvides attempts to find a package which provides the requested PackageVer.
func (r *RpmRepoCloner) WhatProvides(pkgVer *pkgjson.PackageVer) (packageName string, err error) {
// WhatProvides attempts to find packages which provide the requested PackageVer.
func (r *RpmRepoCloner) WhatProvides(pkgVer *pkgjson.PackageVer) (packageNames []string, err error) {
provideQuery := convertPackageVersionToTdnfArg(pkgVer)
args := []string{
baseArgs := []string{
"provides",
provideQuery,
fmt.Sprintf("--disablerepo=%s", allRepoIDs),
}
if !r.useUpdateRepo {
args = append(args, fmt.Sprintf("--disablerepo=%s", updateRepoID))
}
foundPackages := make(map[string]bool)
// Consider the built (local) RPMs first, then the already cached (e.g. tooolchain), and finally all remote packages.
repoOrderList := []string{builtRepoID, cacheRepoID, allRepoIDs}
for _, repoID := range repoOrderList {
logger.Log.Debugf("Enabling repo ID: %s", repoID)
if !r.usePreviewRepo {
args = append(args, fmt.Sprintf("--disablerepo=%s", previewRepoID))
}
err = r.chroot.Run(func() (err error) {
completeArgs := append(baseArgs, fmt.Sprintf("--enablerepo=%s", repoID))
err = r.chroot.Run(func() (err error) {
if !r.usePreviewRepo {
completeArgs = append(completeArgs, fmt.Sprintf("--disablerepo=%s", previewRepoID))
}
if !r.usePreviewRepo {
args = append(args, fmt.Sprintf("--disablerepo=%s", previewRepoID))
}
if !r.useUpdateRepo {
completeArgs = append(completeArgs, fmt.Sprintf("--disablerepo=%s", updateRepoID))
}
stdout, stderr, err := shell.Execute("tdnf", args...)
logger.Log.Debugf("tdnf search for provide '%s':\n%s", pkgVer.Name, stdout)
stdout, stderr, err := shell.Execute("tdnf", completeArgs...)
logger.Log.Debugf("tdnf search for provide '%s':\n%s", pkgVer.Name, stdout)
if err != nil {
logger.Log.Errorf("Failed to lookup provide '%s', tdnf error: '%s'", pkgVer.Name, stderr)
return
}
splitStdout := strings.Split(stdout, "\n")
for _, line := range splitStdout {
matches := packageLookupNameMatchRegex.FindStringSubmatch(line)
if len(matches) == 0 {
continue
}
packageName := matches[1]
if !foundPackages[packageName] {
foundPackages[packageName] = true
logger.Log.Debugf("'%s' is available from package '%s'", pkgVer.Name, packageName)
}
}
return
})
if err != nil {
logger.Log.Errorf("Failed to lookup provide '%s', tdnf error: '%s'", pkgVer.Name, stderr)
return
}
splitStdout := strings.Split(stdout, "\n")
for _, line := range splitStdout {
matches := packageLookupNameMatchRegex.FindStringSubmatch(line)
if len(matches) == 0 {
continue
}
// Local sources are listed last, keep searching for the last possible match
packageName = matches[1]
logger.Log.Debugf("'%s' is available from package '%s'", pkgVer.Name, packageName)
if len(foundPackages) > 0 {
logger.Log.Debug("Found required package(s), skipping further search in other repos.")
break
}
return
})
if err != nil {
return
}
if packageName == "" {
if len(foundPackages) == 0 {
err = fmt.Errorf("could not resolve %s", pkgVer.Name)
return
}
logger.Log.Debugf("Translated '%s' to package '%s'", pkgVer.Name, packageName)
for packageName := range foundPackages {
packageNames = append(packageNames, packageName)
}
logger.Log.Debugf("Translated '%s' to package(s): %s", pkgVer.Name, strings.Join(packageNames, " "))
return
}

48
toolkit/tools/internal/rpm/rpm.go
View File

@ -5,6 +5,7 @@ package rpm
import (
"fmt"
"regexp"
"strings"
"microsoft.com/pkggen/internal/file"
@ -50,6 +51,17 @@ const (
rpmBuildProgram = "rpmbuild"
)
var (
// Output from 'rpm' prints installed RPMs in a line with the following format:
//
// D: ========== +++ [name]-[version]-[release].[distribution] [architecture]-linux [hex_value]
//
// Example:
//
// D: ========== +++ systemd-devel-239-42.cm2 x86_64-linux 0x0
installedRPMLineRegex = regexp.MustCompile(`^D: =+ \+{3} (\S+).*$`)
)
// SetMacroDir adds RPM_CONFIGDIR=$(newMacroDir) into the shell's environment for the duration of a program.
// To restore the environment the caller can use shell.SetEnvironment() with the returned origenv.
// On an empty string argument return success immediately and do not modify the environment.
@ -263,6 +275,42 @@ func QueryRPMProvides(rpmFile string) (provides []string, err error) {
return
}
// ResolveCompetingPackages takes in a list of RPMs and returns only the ones, which would
// end up being installed after resolving outdated, obsoleted, or conflicting packages.
func ResolveCompetingPackages(rpmPaths ...string) (resolvedRPMs []string, err error) {
const (
queryFormat = ""
installedRPMIndex = 1
squashErrors = true
)
args := []string{
"-Uvvh",
"--nodeps",
"--test",
}
args = append(args, rpmPaths...)
// Output of interest is printed to stderr.
_, stderr, err := shell.Execute(rpmProgram, args...)
if err != nil {
logger.Log.Warn(stderr)
return
}
splitStdout := strings.Split(stderr, "\n")
for _, line := range splitStdout {
matches := installedRPMLineRegex.FindStringSubmatch(line)
if len(matches) == 0 {
continue
}
resolvedRPMs = append(resolvedRPMs, matches[installedRPMIndex])
}
return
}
// SpecExclusiveArchIsCompatible verifies ExclusiveArch tag is compatible with the current machine's architecture.
func SpecExclusiveArchIsCompatible(specfile, sourcedir string, defines map[string]string) (isCompatible bool, err error) {
const (