From 97687f4334017e14acbe3ad9edf7c88920f6b5db Mon Sep 17 00:00:00 2001 From: Pawel Winogrodzki Date: Thu, 13 Jan 2022 05:56:04 -0800 Subject: [PATCH] [main] Fixing image builds. (#1884) --- SPECS/mariner-repos/mariner-repos.spec | 55 ++++++++++++------- toolkit/docs/building/building.md | 2 +- .../manifests/package/pkggen_core_aarch64.txt | 3 +- .../manifests/package/pkggen_core_x86_64.txt | 3 +- .../manifests/package/toolchain_aarch64.txt | 21 +++---- .../manifests/package/toolchain_x86_64.txt | 21 +++---- .../manifests/package/update_manifests.sh | 1 + toolkit/scripts/imggen.mk | 7 ++- toolkit/tools/scheduler/scheduler.go | 2 +- .../scheduler/schedulerutils/buildlist.go | 40 +++++++++++++- 10 files changed, 106 insertions(+), 49 deletions(-) diff --git a/SPECS/mariner-repos/mariner-repos.spec b/SPECS/mariner-repos/mariner-repos.spec index 557252e4d3..da4ee624ca 100644 --- a/SPECS/mariner-repos/mariner-repos.spec +++ b/SPECS/mariner-repos/mariner-repos.spec @@ -1,7 +1,7 @@ Summary: CBL-Mariner repo files, gpg keys Name: mariner-repos Version: 2.0 -Release: 3%{?dist} +Release: 4%{?dist} License: MIT Vendor: Microsoft Corporation Distribution: Mariner @@ -20,11 +20,7 @@ Source9: mariner-microsoft-preview.repo Source10: mariner-official-base.repo Source11: mariner-official-preview.repo -Requires(post): gpgme -Requires(post): rpm - -Requires(preun): gpgme -Requires(preun): rpm +Requires: %{name}-shared = %{version}-%{release} BuildArch: noarch @@ -34,7 +30,7 @@ CBL-Mariner repo files and gpg keys %package debuginfo Summary: CBL-Mariner Debuginfo repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description debuginfo %{summary} @@ -42,7 +38,7 @@ Requires: %{name} = %{version}-%{release} %package debuginfo-preview Summary: CBL-Mariner Debuginfo preview repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description debuginfo-preview %{summary} @@ -50,7 +46,7 @@ Requires: %{name} = %{version}-%{release} %package extended Summary: CBL-Mariner Extended repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description extended %{summary} @@ -58,7 +54,7 @@ Requires: %{name} = %{version}-%{release} %package extended-preview Summary: CBL-Mariner Extended preview repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description extended-preview %{summary} @@ -66,7 +62,7 @@ Requires: %{name} = %{version}-%{release} %package extras Summary: CBL-Mariner Extras repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description extras %{summary} @@ -74,7 +70,7 @@ Requires: %{name} = %{version}-%{release} %package extras-preview Summary: CBL-Mariner Extras preview repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description extras-preview %{summary} @@ -82,7 +78,7 @@ Requires: %{name} = %{version}-%{release} %package microsoft Summary: CBL-Mariner Microsoft repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description microsoft %{summary} @@ -90,7 +86,7 @@ Requires: %{name} = %{version}-%{release} %package microsoft-preview Summary: CBL-Mariner Microsoft preview repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description microsoft-preview %{summary} @@ -98,11 +94,24 @@ Requires: %{name} = %{version}-%{release} %package preview Summary: CBL-Mariner preview repo file. Group: System Environment/Base -Requires: %{name} = %{version}-%{release} +Requires: %{name}-shared = %{version}-%{release} %description preview %{summary} +%package shared +Summary: Directories and files needed by all %{name} configurations. +Group: System Environment/Base + +Requires(post): gpgme +Requires(post): rpm + +Requires(preun): gpgme +Requires(preun): rpm + +%description shared +%{summary} + %install export REPO_DIRECTORY="%{buildroot}%{_sysconfdir}/yum.repos.d" install -d -m 755 $REPO_DIRECTORY @@ -123,11 +132,11 @@ install -d -m 755 $RPM_GPG_DIRECTORY install -m 644 %{SOURCE0} $RPM_GPG_DIRECTORY install -m 644 %{SOURCE1} $RPM_GPG_DIRECTORY -%posttrans +%posttrans shared gpg --import %{_sysconfdir}/pki/rpm-gpg/MICROSOFT-METADATA-GPG-KEY gpg --import %{_sysconfdir}/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY -%preun +%preun shared # Remove the MICROSOFT-METADATA-GPG-KEY gpg --batch --yes --delete-keys BC528686B50D79E339D3721CEB3E94ADBE1229CF # Remove the MICROSOFT-RPM-GPG-KEY @@ -135,9 +144,6 @@ gpg --batch --yes --delete-keys 2BC94FFF7015A5F28F1537AD0CD9FED33135CE90 %files %defattr(-,root,root,-) -%dir %{_sysconfdir}/yum.repos.d -%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY -%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-METADATA-GPG-KEY %config(noreplace) %{_sysconfdir}/yum.repos.d/mariner-official-base.repo %files debuginfo @@ -176,7 +182,16 @@ gpg --batch --yes --delete-keys 2BC94FFF7015A5F28F1537AD0CD9FED33135CE90 %defattr(-,root,root,-) %config(noreplace) %{_sysconfdir}/yum.repos.d/mariner-official-preview.repo +%files shared +%dir %{_sysconfdir}/yum.repos.d +%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY +%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-METADATA-GPG-KEY + %changelog +* Mon Jan 10 2022 Pawel Winogrodzki - 2.0-4 +- Creating a separate "mariner-repos-shared" subpackage to make repo configurations + independent of each other. + * Thu Dec 16 2021 Jon Slobodzian - 2.0-3 - Corrected Repo URLS for 2.0. diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 8e0d045002..b4a7d8af53 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -381,7 +381,7 @@ If that is not desired all remote sources can be disabled by clearing the follow > List of RPM repositories to pull packages from. These packages are used to satisfy dependencies during the build process, and to compose a final image. Locally available packages are always prioritized. The repos are prioritized based on the order they appear in the list: repos earlier in the list are higher priority. CBL-Mariner provides a set of pre-populated RPM repositories accessible inside the toolkit folder under `toolkit/repos`: > > - `mariner-official-base.repo` and `mariner-official-update.repo` - default, always-on CBL-Mariner repositories. -> - `mariner-preview.repo` - CBL-Mariner repository containing pre-release versions of RPMs **subject to change without notice**. Using this .repo file is equivallent to adding the [`USE_PREVIEW_REPO=y`](#use_preview_repoy) argument to your build command. +> - `mariner-preview.repo` - CBL-Mariner repository containing pre-release versions of RPMs **subject to change without notice**. Using this .repo file is equivalent to adding the [`USE_PREVIEW_REPO=y`](#use_preview_repoy) argument to your build command. > - `mariner-ui.repo` and `mariner-ui-preview.repo` - CBL-Mariner repository containing packages related to any UI components. The preview version serves the same purpose as the official preview repo. > - `mariner-extras.repo` and `mariner-extras-preview.repo` - CBL-Mariner repository containing proprietory RPMs with sources not viewable to the public. The preview version serves the same purpose as the official preview repo. > diff --git a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt index cf826209b7..a01f94920a 100644 --- a/toolkit/resources/manifests/package/pkggen_core_aarch64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_aarch64.txt @@ -217,7 +217,8 @@ npth-1.6-4.cm2.aarch64.rpm pinentry-1.2.0-1.cm2.aarch64.rpm gnupg2-2.3.3-1.cm2.aarch64.rpm gpgme-1.16.0-1.cm2.aarch64.rpm -mariner-repos-preview-2.0-3.cm2.noarch.rpm +mariner-repos-shared-2.0-4.cm2.noarch.rpm +mariner-repos-preview-2.0-4.cm2.noarch.rpm libffi-3.4.2-1.cm2.aarch64.rpm libffi-devel-3.4.2-1.cm2.aarch64.rpm libtasn1-4.14-3.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt index 30cd1513b0..8c647f278c 100644 --- a/toolkit/resources/manifests/package/pkggen_core_x86_64.txt +++ b/toolkit/resources/manifests/package/pkggen_core_x86_64.txt @@ -217,7 +217,8 @@ npth-1.6-4.cm2.x86_64.rpm pinentry-1.2.0-1.cm2.x86_64.rpm gnupg2-2.3.3-1.cm2.x86_64.rpm gpgme-1.16.0-1.cm2.x86_64.rpm -mariner-repos-preview-2.0-3.cm2.noarch.rpm +mariner-repos-shared-2.0-4.cm2.noarch.rpm +mariner-repos-preview-2.0-4.cm2.noarch.rpm libffi-3.4.2-1.cm2.x86_64.rpm libffi-devel-3.4.2-1.cm2.x86_64.rpm libtasn1-4.14-3.cm2.x86_64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index 03bce5bb88..cf18c663ae 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -216,16 +216,17 @@ make-4.3-1.cm2.aarch64.rpm make-debuginfo-4.3-1.cm2.aarch64.rpm mariner-check-macros-2.0-10.cm2.noarch.rpm mariner-release-2.0-3.cm2.noarch.rpm -mariner-repos-2.0-3.cm2.noarch.rpm -mariner-repos-debuginfo-2.0-3.cm2.noarch.rpm -mariner-repos-debuginfo-preview-2.0-3.cm2.noarch.rpm -mariner-repos-extended-2.0-3.cm2.noarch.rpm -mariner-repos-extended-preview-2.0-3.cm2.noarch.rpm -mariner-repos-extras-2.0-3.cm2.noarch.rpm -mariner-repos-extras-preview-2.0-3.cm2.noarch.rpm -mariner-repos-microsoft-2.0-3.cm2.noarch.rpm -mariner-repos-microsoft-preview-2.0-3.cm2.noarch.rpm -mariner-repos-preview-2.0-3.cm2.noarch.rpm +mariner-repos-2.0-4.cm2.noarch.rpm +mariner-repos-debuginfo-2.0-4.cm2.noarch.rpm +mariner-repos-debuginfo-preview-2.0-4.cm2.noarch.rpm +mariner-repos-extended-2.0-4.cm2.noarch.rpm +mariner-repos-extended-preview-2.0-4.cm2.noarch.rpm +mariner-repos-extras-2.0-4.cm2.noarch.rpm +mariner-repos-extras-preview-2.0-4.cm2.noarch.rpm +mariner-repos-microsoft-2.0-4.cm2.noarch.rpm +mariner-repos-microsoft-preview-2.0-4.cm2.noarch.rpm +mariner-repos-preview-2.0-4.cm2.noarch.rpm +mariner-repos-shared-2.0-4.cm2.noarch.rpm mariner-rpm-macros-2.0-10.cm2.noarch.rpm meson-0.60.2-1.cm2.noarch.rpm mpfr-4.1.0-1.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index b9b6a8fac9..83378e1e35 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -216,16 +216,17 @@ make-4.3-1.cm2.x86_64.rpm make-debuginfo-4.3-1.cm2.x86_64.rpm mariner-check-macros-2.0-10.cm2.noarch.rpm mariner-release-2.0-3.cm2.noarch.rpm -mariner-repos-2.0-3.cm2.noarch.rpm -mariner-repos-debuginfo-2.0-3.cm2.noarch.rpm -mariner-repos-debuginfo-preview-2.0-3.cm2.noarch.rpm -mariner-repos-extended-2.0-3.cm2.noarch.rpm -mariner-repos-extended-preview-2.0-3.cm2.noarch.rpm -mariner-repos-extras-2.0-3.cm2.noarch.rpm -mariner-repos-extras-preview-2.0-3.cm2.noarch.rpm -mariner-repos-microsoft-2.0-3.cm2.noarch.rpm -mariner-repos-microsoft-preview-2.0-3.cm2.noarch.rpm -mariner-repos-preview-2.0-3.cm2.noarch.rpm +mariner-repos-2.0-4.cm2.noarch.rpm +mariner-repos-debuginfo-2.0-4.cm2.noarch.rpm +mariner-repos-debuginfo-preview-2.0-4.cm2.noarch.rpm +mariner-repos-extended-2.0-4.cm2.noarch.rpm +mariner-repos-extended-preview-2.0-4.cm2.noarch.rpm +mariner-repos-extras-2.0-4.cm2.noarch.rpm +mariner-repos-extras-preview-2.0-4.cm2.noarch.rpm +mariner-repos-microsoft-2.0-4.cm2.noarch.rpm +mariner-repos-microsoft-preview-2.0-4.cm2.noarch.rpm +mariner-repos-preview-2.0-4.cm2.noarch.rpm +mariner-repos-shared-2.0-4.cm2.noarch.rpm mariner-rpm-macros-2.0-10.cm2.noarch.rpm meson-0.60.2-1.cm2.noarch.rpm mpfr-4.1.0-1.cm2.x86_64.rpm diff --git a/toolkit/resources/manifests/package/update_manifests.sh b/toolkit/resources/manifests/package/update_manifests.sh index 2e956be726..3066aa3ca3 100755 --- a/toolkit/resources/manifests/package/update_manifests.sh +++ b/toolkit/resources/manifests/package/update_manifests.sh @@ -283,6 +283,7 @@ generate_pkggen_core () { grep "^pinentry-" $TmpPkgGen grep "^gnupg2-" $TmpPkgGen grep "^gpgme-" $TmpPkgGen + grep "^mariner-repos-shared" $TmpPkgGen grep "^mariner-repos-preview" $TmpPkgGen grep "^libffi-" $TmpPkgGen grep "^libtasn1-" $TmpPkgGen diff --git a/toolkit/scripts/imggen.mk b/toolkit/scripts/imggen.mk index 6c3f7308bd..c6a7d4cab3 100644 --- a/toolkit/scripts/imggen.mk +++ b/toolkit/scripts/imggen.mk @@ -71,6 +71,7 @@ fetch-external-image-packages: $(image_external_package_cache_summary) # Changes to files located outside the base directory will not be detected. validate-image-config: $(validate-config) $(STATUS_FLAGS_DIR)/validate-image-config%.flag: $(go-imageconfigvalidator) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(config_other_files) + $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) $(go-imageconfigvalidator) \ --input=$(CONFIG_FILE) \ --dir=$(CONFIG_BASE_DIR) && \ @@ -86,7 +87,7 @@ ifeq ($(USE_PREVIEW_REPO),y) imagepkgfetcher_extra_flags += --use-preview-repo endif -$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(packagelist_files) $(RPMS_DIR) $(imggen_rpms) +$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms) $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) $(go-imagepkgfetcher) \ --input=$(CONFIG_FILE) \ @@ -109,7 +110,7 @@ $(imager_disk_output_dir): $(STATUS_FLAGS_DIR)/imager_disk_output.flag @touch $@ @echo Finished updating $@ -$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(packagelist_files) $(assets_files) $(imggen_packagelist_files) +$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(assets_files) $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) mkdir -p $(imager_disk_output_dir) && \ rm -rf $(imager_disk_output_dir)/* && \ @@ -142,7 +143,7 @@ image: $(imager_disk_output_dir) $(imager_disk_output_files) $(go-roast) $(depen --log-file=$(LOGS_DIR)/imggen/roast.log \ --image-tag=$(IMAGE_TAG) -$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) +$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=)) $(go-imagepkgfetcher) \ --input=$(CONFIG_FILE) \ diff --git a/toolkit/tools/scheduler/scheduler.go b/toolkit/tools/scheduler/scheduler.go index 6d52cd969d..20500e3bad 100644 --- a/toolkit/tools/scheduler/scheduler.go +++ b/toolkit/tools/scheduler/scheduler.go @@ -105,7 +105,7 @@ func main() { logger.Log.Fatalf("Can't ignore and force a rebuild of a package at the same time. Abusing packages: %v", ignoredAndRebuiltPackages) } - packageVersToBuild, err := schedulerutils.CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild, *imageConfig, *baseDirPath) + packageVersToBuild, err := schedulerutils.CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild, *inputGraphFile, *imageConfig, *baseDirPath) if err != nil { logger.Log.Fatalf("Unable to generate package build list, error: %s", err) } diff --git a/toolkit/tools/scheduler/schedulerutils/buildlist.go b/toolkit/tools/scheduler/schedulerutils/buildlist.go index 55ffd97460..204d55768c 100644 --- a/toolkit/tools/scheduler/schedulerutils/buildlist.go +++ b/toolkit/tools/scheduler/schedulerutils/buildlist.go @@ -7,12 +7,17 @@ import ( "microsoft.com/pkggen/imagegen/configuration" "microsoft.com/pkggen/imagegen/installutils" "microsoft.com/pkggen/internal/logger" + "microsoft.com/pkggen/internal/pkggraph" "microsoft.com/pkggen/internal/pkgjson" ) // CalculatePackagesToBuild generates a comprehensive list of all PackageVers that the scheduler should attempt to build. -// The build list is a superset of packagesNamesToBuild, packagesNamesToRebuild, packages listed in the image config, and kernels in the image config. -func CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild []string, imageConfig, baseDirPath string) (packageVersToBuild []*pkgjson.PackageVer, err error) { +// The build list is a superset of: +// - packagesNamesToBuild, +// - packagesNamesToRebuild, +// - local packages listed in the image config, and +// - kernels in the image config (if built locally). +func CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild []string, inputGraphFile, imageConfig, baseDirPath string) (packageVersToBuild []*pkgjson.PackageVer, err error) { packageVersToBuild = convertPackageNamesIntoPackageVers(packagesNamesToBuild) packageVersToBuild = append(packageVersToBuild, convertPackageNamesIntoPackageVers(packagesNamesToRebuild)...) @@ -25,6 +30,11 @@ func CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild []str return } + packageVersFromConfig, err = filterLocalPackagesOnly(packageVersFromConfig, inputGraphFile) + if err != nil { + return + } + packageVersToBuild = append(packageVersToBuild, packageVersFromConfig...) return } @@ -61,3 +71,29 @@ func extractPackagesFromConfig(configFile, baseDirPath string) (packageList []*p return } + +// filterLocalPackagesOnly returns the subset of packageVersionsInConfig that only contains local packages. +func filterLocalPackagesOnly(packageVersionsInConfig []*pkgjson.PackageVer, inputGraph string) (filteredPackages []*pkgjson.PackageVer, err error) { + logger.Log.Debug("Filtering out external packages from list of packages extracted from the image config file.") + + dependencyGraph := pkggraph.NewPkgGraph() + err = pkggraph.ReadDOTGraphFile(dependencyGraph, inputGraph) + if err != nil { + return + } + + for _, pkgVer := range packageVersionsInConfig { + pkgNode, _ := dependencyGraph.FindBestPkgNode(pkgVer) + + // A pkgNode for a local package has the following characteristics: + // 1) The pkgNode exists in the graph (is not nil). + // 2) The pkgNode doesn't have the 'StateUnresolved' or 'StateCached' state. These are reserved for external dependencies nodes. + if pkgNode != nil && pkgNode.RunNode.State != pkggraph.StateUnresolved && pkgNode.RunNode.State != pkggraph.StateCached { + filteredPackages = append(filteredPackages, pkgVer) + } else { + logger.Log.Debugf("Found external package to filter out: %v.", pkgVer) + } + } + + return +}