From 2401ea067f6ed08d76f98f38211bf5691fb01b20 Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Mon, 4 Oct 2021 00:46:55 -0700 Subject: [PATCH] [dev]: Resolving package build issues. (#1488) * Fixing 'bind', 'clang', 'systemd' provides, and 'openssh' configuration. * Enhancing RPM resolution. --- SPECS/bind/bind.spec | 6 +- SPECS/clang/clang.spec | 10 --- .../kpmcore.signatures.json | 0 SPECS/{kpmcore-3.3.0 => kpmcore}/kpmcore.spec | 0 SPECS/openssh/openssh.spec | 8 +- SPECS/systemd/systemd.spec | 13 ++- .../tools/graphpkgfetcher/graphpkgfetcher.go | 89 +++++++++++++++---- .../packagerepo/repocloner/repocloner.go | 2 +- .../repocloner/rpmrepocloner/rpmrepocloner.go | 85 ++++++++++-------- toolkit/tools/internal/rpm/rpm.go | 48 ++++++++++ 10 files changed, 190 insertions(+), 71 deletions(-) rename SPECS/{kpmcore-3.3.0 => kpmcore}/kpmcore.signatures.json (100%) rename SPECS/{kpmcore-3.3.0 => kpmcore}/kpmcore.spec (100%) diff --git a/SPECS/bind/bind.spec b/SPECS/bind/bind.spec index b16200ad8f..c3366430e4 100644 --- a/SPECS/bind/bind.spec +++ b/SPECS/bind/bind.spec @@ -9,7 +9,7 @@ Summary: Domain Name System software Name: bind Version: 9.16.15 -Release: 2%{?dist} +Release: 3%{?dist} License: ISC Vendor: Microsoft Corporation Distribution: Mariner @@ -53,6 +53,7 @@ BuildRequires: postgresql-devel BuildRequires: python3 BuildRequires: python3-ply BuildRequires: sqlite-devel +BuildRequires: systemd-rpm-macros Requires: libuv Requires: openssl @@ -614,6 +615,9 @@ fi; %{_tmpfilesdir}/named.conf %changelog +* Sat Oct 02 2021 Pawel Winogrodzki - 9.16.15-3 +- Adding missing BR on 'systemd-rpm-macros'. + * Fri Aug 27 2021 Pawel Winogrodzki - 9.16.15-2 - Adding DBZ subpackages using Fedora 34 (license: MIT) specs as guidance. diff --git a/SPECS/clang/clang.spec b/SPECS/clang/clang.spec index 31d45818e9..7f03db8e82 100644 --- a/SPECS/clang/clang.spec +++ b/SPECS/clang/clang.spec @@ -120,20 +120,10 @@ A set of extra tools built using Clang's tooling API. %prep %setup -q -T -b 1 -n %{clang_tools_srcdir} -pathfix.py -i python3 -pn \ - clang-tidy/tool/*.py \ - clang-include-fixer/find-all-symbols/tool/run-find-all-symbols.py - %setup -q -n %{clang_srcdir} mv ../%{clang_tools_srcdir} tools/extra -pathfix.py -i python3 -pn \ - tools/clang-format/*.py \ - tools/clang-format/git-clang-format \ - utils/hmaptool/hmaptool \ - tools/scan-view/bin/scan-view - %build # Disable symbol generation export CFLAGS="`echo " %{build_cflags} " | sed 's/ -g//'`" diff --git a/SPECS/kpmcore-3.3.0/kpmcore.signatures.json b/SPECS/kpmcore/kpmcore.signatures.json similarity index 100% rename from SPECS/kpmcore-3.3.0/kpmcore.signatures.json rename to SPECS/kpmcore/kpmcore.signatures.json diff --git a/SPECS/kpmcore-3.3.0/kpmcore.spec b/SPECS/kpmcore/kpmcore.spec similarity index 100% rename from SPECS/kpmcore-3.3.0/kpmcore.spec rename to SPECS/kpmcore/kpmcore.spec diff --git a/SPECS/openssh/openssh.spec b/SPECS/openssh/openssh.spec index 5843e892a9..9238f53392 100644 --- a/SPECS/openssh/openssh.spec +++ b/SPECS/openssh/openssh.spec @@ -1,15 +1,15 @@ %global openssh_ver 8.5p1 -%global openssh_rel 4 +%global openssh_rel 4%{?dist} %global pam_ssh_agent_ver 0.10.3 -%global pam_ssh_agent_rel 10 +%global pam_ssh_agent_rel 10%{?dist} %define systemd_units_rel 20191026 Summary: Free version of the SSH connectivity tools Name: openssh Version: %{openssh_ver} -Release: %{openssh_rel}%{?dist} +Release: %{openssh_rel} License: BSD Vendor: Microsoft Corporation Distribution: Mariner @@ -72,7 +72,7 @@ This provides the ssh client utilities. %package -n pam_ssh_agent_auth Summary: PAM module for authentication with ssh-agent Version: %{pam_ssh_agent_ver} -Release: %{pam_ssh_agent_rel}.%{openssh_rel}%{?dist} +Release: %{pam_ssh_agent_rel}.%{openssh_rel} License: BSD %description -n pam_ssh_agent_auth diff --git a/SPECS/systemd/systemd.spec b/SPECS/systemd/systemd.spec index 50b52e4474..f4f197e084 100644 --- a/SPECS/systemd/systemd.spec +++ b/SPECS/systemd/systemd.spec @@ -1,7 +1,7 @@ Summary: Systemd-239 Name: systemd Version: 239 -Release: 41%{?dist} +Release: 42%{?dist} License: LGPLv2+ AND GPLv2+ AND MIT Vendor: Microsoft Corporation Distribution: Mariner @@ -78,7 +78,10 @@ Requires: libgcrypt Requires: lz4 Requires: pam Requires: xz + Obsoletes: systemd-bootstrap +Provides: systemd-bootstrap = %{version}-%{release} + Provides: systemd-units = %{version}-%{release} Provides: systemd-sysv = %{version}-%{release} Provides: systemd-udev = %{version}-%{release} @@ -101,6 +104,10 @@ Just the definitions of rpm macros. Summary: Development headers for systemd Requires: %{name} = %{version}-%{release} Requires: glib-devel + +Obsoletes: systemd-bootstrap-devel +Provides: systemd-bootstrap-devel = %{version}-%{release} + Provides: systemd-libs = %{version}-%{release} Provides: libudev-devel = %{version}-%{release} Provides: libudev-devel%{?_isa} = %{version}-%{release} @@ -289,6 +296,10 @@ rm -rf %{buildroot}/* %files lang -f %{name}.lang %changelog +* Sat Oct 02 2021 Pawel Winogrodzki - 239-42 +- Adding 'Obsoletes: systemd-bootstrap-devel' for the 'devel' subpackage. +- Making 'systemd' obsolete 'systemd-bootstrap' regardless of version and release. + * Wed Aug 18 2021 Jon Slobodzian - 239-41 - Merge from 1.0 to dev branch - nehaagarwal@microsoft.com, 2.39-38: CVE-2021-33910 fix diff --git a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go index 407dd8a818..d84468547e 100644 --- a/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go +++ b/toolkit/tools/graphpkgfetcher/graphpkgfetcher.go @@ -17,6 +17,7 @@ import ( "microsoft.com/pkggen/internal/packagerepo/repoutils" "microsoft.com/pkggen/internal/pkggraph" "microsoft.com/pkggen/internal/pkgjson" + "microsoft.com/pkggen/internal/rpm" ) var ( @@ -155,7 +156,7 @@ func resolveSingleNode(cloner *rpmrepocloner.RpmRepoCloner, node *pkggraph.PkgNo logger.Log.Debugf("Searching for a package which supplies: %s", node.VersionedPkg.Name) // Resolve nodes to exact package names so they can be referenced in the graph. - resolvedPackage, err := cloner.WhatProvides(node.VersionedPkg) + resolvedPackages, err := cloner.WhatProvides(node.VersionedPkg) if err != nil { msg := fmt.Sprintf("Failed to resolve (%s) to a package. Error: %s", node.VersionedPkg, err) // It is not an error if an implicit node could not be resolved as it may become available later in the build. @@ -168,27 +169,77 @@ func resolveSingleNode(cloner *rpmrepocloner.RpmRepoCloner, node *pkggraph.PkgNo return } - if !fetchedPackages[resolvedPackage] { - desiredPackage := &pkgjson.PackageVer{ - Name: resolvedPackage, - } - - err = cloner.Clone(cloneDeps, desiredPackage) - if err != nil { - logger.Log.Errorf("Failed to clone '%s' from RPM repo. Error: %s", resolvedPackage, err) - return - } - fetchedPackages[resolvedPackage] = true - - logger.Log.Infof("Fetched: %s", resolvedPackage) + if len(resolvedPackages) == 0 { + return fmt.Errorf("failed to find any packages providing '%v'", node.VersionedPkg) + } + + for _, resolvedPackage := range resolvedPackages { + if !fetchedPackages[resolvedPackage] { + desiredPackage := &pkgjson.PackageVer{ + Name: resolvedPackage, + } + + err = cloner.Clone(cloneDeps, desiredPackage) + if err != nil { + logger.Log.Errorf("Failed to clone '%s' from RPM repo. Error: %s", resolvedPackage, err) + return + } + fetchedPackages[resolvedPackage] = true + + logger.Log.Debugf("Fetched '%s' as potential candidate.", resolvedPackage) + } + } + + err = assignRPMPath(node, outDir, resolvedPackages) + if err != nil { + logger.Log.Errorf("Failed to find an RPM to provide '%s'. Error: %s", node.VersionedPkg.Name, err) + return } - // Construct the rpm path of the cloned package. - rpmName := fmt.Sprintf("%s.rpm", resolvedPackage) - // To calculate the architecture grab the last segment of the resolved name since it will be in the NVRA format. - rpmArch := resolvedPackage[strings.LastIndex(resolvedPackage, ".")+1:] - node.RpmPath = filepath.Join(outDir, rpmArch, rpmName) node.State = pkggraph.StateCached + logger.Log.Infof("Choosing '%s' to provide '%s'.", filepath.Base(node.RpmPath), node.VersionedPkg.Name) + return } + +func assignRPMPath(node *pkggraph.PkgNode, outDir string, resolvedPackages []string) (err error) { + rpmPaths := []string{} + for _, resolvedPackage := range resolvedPackages { + rpmPaths = append(rpmPaths, rpmPackageToRPMPath(resolvedPackage, outDir)) + } + + node.RpmPath = rpmPaths[0] + if len(rpmPaths) > 1 { + var resolvedRPMs []string + logger.Log.Debugf("Found %d candidates. Resolving.", len(rpmPaths)) + + resolvedRPMs, err = rpm.ResolveCompetingPackages(rpmPaths...) + if err != nil { + logger.Log.Errorf("Failed while trying to pick an RPM providing '%s' from the following RPMs: %v", node.VersionedPkg.Name, rpmPaths) + return + } + + resolvedRPMsCount := len(resolvedRPMs) + if resolvedRPMsCount == 0 { + logger.Log.Errorf("Failed while trying to pick an RPM providing '%s'. No RPM can be installed from the following: %v", node.VersionedPkg.Name, rpmPaths) + return + } + + if resolvedRPMsCount > 1 { + logger.Log.Warnf("Found %d candidates to provide '%s'. Picking the first one.", resolvedRPMsCount, node.VersionedPkg.Name) + } + + node.RpmPath = rpmPackageToRPMPath(resolvedRPMs[0], outDir) + } + + return +} + +func rpmPackageToRPMPath(rpmPackage, outDir string) string { + // Construct the rpm path of the cloned package. + rpmName := fmt.Sprintf("%s.rpm", rpmPackage) + // To calculate the architecture grab the last segment of the resolved name since it will be in the NVRA format. + rpmArch := rpmPackage[strings.LastIndex(rpmPackage, ".")+1:] + return filepath.Join(outDir, rpmArch, rpmName) +} diff --git a/toolkit/tools/internal/packagerepo/repocloner/repocloner.go b/toolkit/tools/internal/packagerepo/repocloner/repocloner.go index 3f6880c2ec..b03320a688 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/repocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/repocloner.go @@ -27,7 +27,7 @@ type RepoCloner interface { Initialize(destinationDir, tmpDir, workerTar, existingRpmsDir string, useUpdateRepo, usePreviewRepo bool, repoDefinitions []string) error AddNetworkFiles(tlsClientCert, tlsClientKey string) error Clone(cloneDeps bool, packagesToClone ...*pkgjson.PackageVer) error - WhatProvides(pkgVer *pkgjson.PackageVer) (packageName string, err error) + WhatProvides(pkgVer *pkgjson.PackageVer) (packageNames []string, err error) ConvertDownloadedPackagesIntoRepo() error ClonedRepoContents() (repoContents *RepoContents, err error) CloneDirectory() string diff --git a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go index ff86398edd..126c6ef33d 100644 --- a/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go +++ b/toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go @@ -21,6 +21,8 @@ import ( ) const ( + allRepoIDs = "*" + builtRepoID = "local-repo" cacheRepoID = "upstream-cache-repo" squashChrootRunErrors = false chrootDownloadDir = "/outputrpms" @@ -256,9 +258,6 @@ func (r *RpmRepoCloner) Clone(cloneDeps bool, packagesToClone ...*pkgjson.Packag strictComparisonOperator = "=" lessThanOrEqualComparisonOperator = "<=" versionSuffixFormat = "-%s" - - builtRepoID = "local-repo" - allRepoIDs = "*" ) for _, pkg := range packagesToClone { @@ -291,60 +290,76 @@ func (r *RpmRepoCloner) Clone(cloneDeps bool, packagesToClone ...*pkgjson.Packag return } -// WhatProvides attempts to find a package which provides the requested PackageVer. -func (r *RpmRepoCloner) WhatProvides(pkgVer *pkgjson.PackageVer) (packageName string, err error) { +// WhatProvides attempts to find packages which provide the requested PackageVer. +func (r *RpmRepoCloner) WhatProvides(pkgVer *pkgjson.PackageVer) (packageNames []string, err error) { provideQuery := convertPackageVersionToTdnfArg(pkgVer) - args := []string{ + baseArgs := []string{ "provides", provideQuery, + fmt.Sprintf("--disablerepo=%s", allRepoIDs), } - if !r.useUpdateRepo { - args = append(args, fmt.Sprintf("--disablerepo=%s", updateRepoID)) - } + foundPackages := make(map[string]bool) + // Consider the built (local) RPMs first, then the already cached (e.g. tooolchain), and finally all remote packages. + repoOrderList := []string{builtRepoID, cacheRepoID, allRepoIDs} + for _, repoID := range repoOrderList { + logger.Log.Debugf("Enabling repo ID: %s", repoID) - if !r.usePreviewRepo { - args = append(args, fmt.Sprintf("--disablerepo=%s", previewRepoID)) - } + err = r.chroot.Run(func() (err error) { + completeArgs := append(baseArgs, fmt.Sprintf("--enablerepo=%s", repoID)) - err = r.chroot.Run(func() (err error) { + if !r.usePreviewRepo { + completeArgs = append(completeArgs, fmt.Sprintf("--disablerepo=%s", previewRepoID)) + } - if !r.usePreviewRepo { - args = append(args, fmt.Sprintf("--disablerepo=%s", previewRepoID)) - } + if !r.useUpdateRepo { + completeArgs = append(completeArgs, fmt.Sprintf("--disablerepo=%s", updateRepoID)) + } - stdout, stderr, err := shell.Execute("tdnf", args...) - logger.Log.Debugf("tdnf search for provide '%s':\n%s", pkgVer.Name, stdout) + stdout, stderr, err := shell.Execute("tdnf", completeArgs...) + logger.Log.Debugf("tdnf search for provide '%s':\n%s", pkgVer.Name, stdout) + if err != nil { + logger.Log.Errorf("Failed to lookup provide '%s', tdnf error: '%s'", pkgVer.Name, stderr) + return + } + + splitStdout := strings.Split(stdout, "\n") + for _, line := range splitStdout { + matches := packageLookupNameMatchRegex.FindStringSubmatch(line) + if len(matches) == 0 { + continue + } + + packageName := matches[1] + if !foundPackages[packageName] { + foundPackages[packageName] = true + logger.Log.Debugf("'%s' is available from package '%s'", pkgVer.Name, packageName) + } + } + return + }) if err != nil { - logger.Log.Errorf("Failed to lookup provide '%s', tdnf error: '%s'", pkgVer.Name, stderr) return } - splitStdout := strings.Split(stdout, "\n") - for _, line := range splitStdout { - matches := packageLookupNameMatchRegex.FindStringSubmatch(line) - if len(matches) == 0 { - continue - } - // Local sources are listed last, keep searching for the last possible match - packageName = matches[1] - logger.Log.Debugf("'%s' is available from package '%s'", pkgVer.Name, packageName) + if len(foundPackages) > 0 { + logger.Log.Debug("Found required package(s), skipping further search in other repos.") + break } - return - }) - - if err != nil { - return } - if packageName == "" { + if len(foundPackages) == 0 { err = fmt.Errorf("could not resolve %s", pkgVer.Name) return } - logger.Log.Debugf("Translated '%s' to package '%s'", pkgVer.Name, packageName) + for packageName := range foundPackages { + packageNames = append(packageNames, packageName) + } + + logger.Log.Debugf("Translated '%s' to package(s): %s", pkgVer.Name, strings.Join(packageNames, " ")) return } diff --git a/toolkit/tools/internal/rpm/rpm.go b/toolkit/tools/internal/rpm/rpm.go index 0be66f8abb..b9c9efb390 100644 --- a/toolkit/tools/internal/rpm/rpm.go +++ b/toolkit/tools/internal/rpm/rpm.go @@ -5,6 +5,7 @@ package rpm import ( "fmt" + "regexp" "strings" "microsoft.com/pkggen/internal/file" @@ -50,6 +51,17 @@ const ( rpmBuildProgram = "rpmbuild" ) +var ( + // Output from 'rpm' prints installed RPMs in a line with the following format: + // + // D: ========== +++ [name]-[version]-[release].[distribution] [architecture]-linux [hex_value] + // + // Example: + // + // D: ========== +++ systemd-devel-239-42.cm2 x86_64-linux 0x0 + installedRPMLineRegex = regexp.MustCompile(`^D: =+ \+{3} (\S+).*$`) +) + // SetMacroDir adds RPM_CONFIGDIR=$(newMacroDir) into the shell's environment for the duration of a program. // To restore the environment the caller can use shell.SetEnvironment() with the returned origenv. // On an empty string argument return success immediately and do not modify the environment. @@ -263,6 +275,42 @@ func QueryRPMProvides(rpmFile string) (provides []string, err error) { return } +// ResolveCompetingPackages takes in a list of RPMs and returns only the ones, which would +// end up being installed after resolving outdated, obsoleted, or conflicting packages. +func ResolveCompetingPackages(rpmPaths ...string) (resolvedRPMs []string, err error) { + const ( + queryFormat = "" + installedRPMIndex = 1 + squashErrors = true + ) + + args := []string{ + "-Uvvh", + "--nodeps", + "--test", + } + args = append(args, rpmPaths...) + + // Output of interest is printed to stderr. + _, stderr, err := shell.Execute(rpmProgram, args...) + if err != nil { + logger.Log.Warn(stderr) + return + } + + splitStdout := strings.Split(stderr, "\n") + for _, line := range splitStdout { + matches := installedRPMLineRegex.FindStringSubmatch(line) + if len(matches) == 0 { + continue + } + + resolvedRPMs = append(resolvedRPMs, matches[installedRPMIndex]) + } + + return +} + // SpecExclusiveArchIsCompatible verifies ExclusiveArch tag is compatible with the current machine's architecture. func SpecExclusiveArchIsCompatible(specfile, sourcedir string, defines map[string]string) (isCompatible bool, err error) { const (