[main] Fixing image builds. (#1884)

This commit is contained in:
Pawel Winogrodzki 2022-01-13 05:56:04 -08:00 committed by GitHub
parent 0d3db879b0
commit 97687f4334
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 106 additions and 49 deletions

View File

@ -1,7 +1,7 @@
Summary: CBL-Mariner repo files, gpg keys
Name: mariner-repos
Version: 2.0
Release: 3%{?dist}
Release: 4%{?dist}
License: MIT
Vendor: Microsoft Corporation
Distribution: Mariner
@ -20,11 +20,7 @@ Source9: mariner-microsoft-preview.repo
Source10: mariner-official-base.repo
Source11: mariner-official-preview.repo
Requires(post): gpgme
Requires(post): rpm
Requires(preun): gpgme
Requires(preun): rpm
Requires: %{name}-shared = %{version}-%{release}
BuildArch: noarch
@ -34,7 +30,7 @@ CBL-Mariner repo files and gpg keys
%package debuginfo
Summary: CBL-Mariner Debuginfo repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description debuginfo
%{summary}
@ -42,7 +38,7 @@ Requires: %{name} = %{version}-%{release}
%package debuginfo-preview
Summary: CBL-Mariner Debuginfo preview repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description debuginfo-preview
%{summary}
@ -50,7 +46,7 @@ Requires: %{name} = %{version}-%{release}
%package extended
Summary: CBL-Mariner Extended repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description extended
%{summary}
@ -58,7 +54,7 @@ Requires: %{name} = %{version}-%{release}
%package extended-preview
Summary: CBL-Mariner Extended preview repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description extended-preview
%{summary}
@ -66,7 +62,7 @@ Requires: %{name} = %{version}-%{release}
%package extras
Summary: CBL-Mariner Extras repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description extras
%{summary}
@ -74,7 +70,7 @@ Requires: %{name} = %{version}-%{release}
%package extras-preview
Summary: CBL-Mariner Extras preview repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description extras-preview
%{summary}
@ -82,7 +78,7 @@ Requires: %{name} = %{version}-%{release}
%package microsoft
Summary: CBL-Mariner Microsoft repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description microsoft
%{summary}
@ -90,7 +86,7 @@ Requires: %{name} = %{version}-%{release}
%package microsoft-preview
Summary: CBL-Mariner Microsoft preview repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description microsoft-preview
%{summary}
@ -98,11 +94,24 @@ Requires: %{name} = %{version}-%{release}
%package preview
Summary: CBL-Mariner preview repo file.
Group: System Environment/Base
Requires: %{name} = %{version}-%{release}
Requires: %{name}-shared = %{version}-%{release}
%description preview
%{summary}
%package shared
Summary: Directories and files needed by all %{name} configurations.
Group: System Environment/Base
Requires(post): gpgme
Requires(post): rpm
Requires(preun): gpgme
Requires(preun): rpm
%description shared
%{summary}
%install
export REPO_DIRECTORY="%{buildroot}%{_sysconfdir}/yum.repos.d"
install -d -m 755 $REPO_DIRECTORY
@ -123,11 +132,11 @@ install -d -m 755 $RPM_GPG_DIRECTORY
install -m 644 %{SOURCE0} $RPM_GPG_DIRECTORY
install -m 644 %{SOURCE1} $RPM_GPG_DIRECTORY
%posttrans
%posttrans shared
gpg --import %{_sysconfdir}/pki/rpm-gpg/MICROSOFT-METADATA-GPG-KEY
gpg --import %{_sysconfdir}/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
%preun
%preun shared
# Remove the MICROSOFT-METADATA-GPG-KEY
gpg --batch --yes --delete-keys BC528686B50D79E339D3721CEB3E94ADBE1229CF
# Remove the MICROSOFT-RPM-GPG-KEY
@ -135,9 +144,6 @@ gpg --batch --yes --delete-keys 2BC94FFF7015A5F28F1537AD0CD9FED33135CE90
%files
%defattr(-,root,root,-)
%dir %{_sysconfdir}/yum.repos.d
%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-METADATA-GPG-KEY
%config(noreplace) %{_sysconfdir}/yum.repos.d/mariner-official-base.repo
%files debuginfo
@ -176,7 +182,16 @@ gpg --batch --yes --delete-keys 2BC94FFF7015A5F28F1537AD0CD9FED33135CE90
%defattr(-,root,root,-)
%config(noreplace) %{_sysconfdir}/yum.repos.d/mariner-official-preview.repo
%files shared
%dir %{_sysconfdir}/yum.repos.d
%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
%{_sysconfdir}/pki/rpm-gpg/MICROSOFT-METADATA-GPG-KEY
%changelog
* Mon Jan 10 2022 Pawel Winogrodzki <pawelwi@microsoft.com> - 2.0-4
- Creating a separate "mariner-repos-shared" subpackage to make repo configurations
independent of each other.
* Thu Dec 16 2021 Jon Slobodzian <joslobo@microsoft.com> - 2.0-3
- Corrected Repo URLS for 2.0.

View File

@ -381,7 +381,7 @@ If that is not desired all remote sources can be disabled by clearing the follow
> List of RPM repositories to pull packages from. These packages are used to satisfy dependencies during the build process, and to compose a final image. Locally available packages are always prioritized. The repos are prioritized based on the order they appear in the list: repos earlier in the list are higher priority. CBL-Mariner provides a set of pre-populated RPM repositories accessible inside the toolkit folder under `toolkit/repos`:
>
> - `mariner-official-base.repo` and `mariner-official-update.repo` - default, always-on CBL-Mariner repositories.
> - `mariner-preview.repo` - CBL-Mariner repository containing pre-release versions of RPMs **subject to change without notice**. Using this .repo file is equivallent to adding the [`USE_PREVIEW_REPO=y`](#use_preview_repoy) argument to your build command.
> - `mariner-preview.repo` - CBL-Mariner repository containing pre-release versions of RPMs **subject to change without notice**. Using this .repo file is equivalent to adding the [`USE_PREVIEW_REPO=y`](#use_preview_repoy) argument to your build command.
> - `mariner-ui.repo` and `mariner-ui-preview.repo` - CBL-Mariner repository containing packages related to any UI components. The preview version serves the same purpose as the official preview repo.
> - `mariner-extras.repo` and `mariner-extras-preview.repo` - CBL-Mariner repository containing proprietory RPMs with sources not viewable to the public. The preview version serves the same purpose as the official preview repo.
>

View File

@ -217,7 +217,8 @@ npth-1.6-4.cm2.aarch64.rpm
pinentry-1.2.0-1.cm2.aarch64.rpm
gnupg2-2.3.3-1.cm2.aarch64.rpm
gpgme-1.16.0-1.cm2.aarch64.rpm
mariner-repos-preview-2.0-3.cm2.noarch.rpm
mariner-repos-shared-2.0-4.cm2.noarch.rpm
mariner-repos-preview-2.0-4.cm2.noarch.rpm
libffi-3.4.2-1.cm2.aarch64.rpm
libffi-devel-3.4.2-1.cm2.aarch64.rpm
libtasn1-4.14-3.cm2.aarch64.rpm

View File

@ -217,7 +217,8 @@ npth-1.6-4.cm2.x86_64.rpm
pinentry-1.2.0-1.cm2.x86_64.rpm
gnupg2-2.3.3-1.cm2.x86_64.rpm
gpgme-1.16.0-1.cm2.x86_64.rpm
mariner-repos-preview-2.0-3.cm2.noarch.rpm
mariner-repos-shared-2.0-4.cm2.noarch.rpm
mariner-repos-preview-2.0-4.cm2.noarch.rpm
libffi-3.4.2-1.cm2.x86_64.rpm
libffi-devel-3.4.2-1.cm2.x86_64.rpm
libtasn1-4.14-3.cm2.x86_64.rpm

View File

@ -216,16 +216,17 @@ make-4.3-1.cm2.aarch64.rpm
make-debuginfo-4.3-1.cm2.aarch64.rpm
mariner-check-macros-2.0-10.cm2.noarch.rpm
mariner-release-2.0-3.cm2.noarch.rpm
mariner-repos-2.0-3.cm2.noarch.rpm
mariner-repos-debuginfo-2.0-3.cm2.noarch.rpm
mariner-repos-debuginfo-preview-2.0-3.cm2.noarch.rpm
mariner-repos-extended-2.0-3.cm2.noarch.rpm
mariner-repos-extended-preview-2.0-3.cm2.noarch.rpm
mariner-repos-extras-2.0-3.cm2.noarch.rpm
mariner-repos-extras-preview-2.0-3.cm2.noarch.rpm
mariner-repos-microsoft-2.0-3.cm2.noarch.rpm
mariner-repos-microsoft-preview-2.0-3.cm2.noarch.rpm
mariner-repos-preview-2.0-3.cm2.noarch.rpm
mariner-repos-2.0-4.cm2.noarch.rpm
mariner-repos-debuginfo-2.0-4.cm2.noarch.rpm
mariner-repos-debuginfo-preview-2.0-4.cm2.noarch.rpm
mariner-repos-extended-2.0-4.cm2.noarch.rpm
mariner-repos-extended-preview-2.0-4.cm2.noarch.rpm
mariner-repos-extras-2.0-4.cm2.noarch.rpm
mariner-repos-extras-preview-2.0-4.cm2.noarch.rpm
mariner-repos-microsoft-2.0-4.cm2.noarch.rpm
mariner-repos-microsoft-preview-2.0-4.cm2.noarch.rpm
mariner-repos-preview-2.0-4.cm2.noarch.rpm
mariner-repos-shared-2.0-4.cm2.noarch.rpm
mariner-rpm-macros-2.0-10.cm2.noarch.rpm
meson-0.60.2-1.cm2.noarch.rpm
mpfr-4.1.0-1.cm2.aarch64.rpm

View File

@ -216,16 +216,17 @@ make-4.3-1.cm2.x86_64.rpm
make-debuginfo-4.3-1.cm2.x86_64.rpm
mariner-check-macros-2.0-10.cm2.noarch.rpm
mariner-release-2.0-3.cm2.noarch.rpm
mariner-repos-2.0-3.cm2.noarch.rpm
mariner-repos-debuginfo-2.0-3.cm2.noarch.rpm
mariner-repos-debuginfo-preview-2.0-3.cm2.noarch.rpm
mariner-repos-extended-2.0-3.cm2.noarch.rpm
mariner-repos-extended-preview-2.0-3.cm2.noarch.rpm
mariner-repos-extras-2.0-3.cm2.noarch.rpm
mariner-repos-extras-preview-2.0-3.cm2.noarch.rpm
mariner-repos-microsoft-2.0-3.cm2.noarch.rpm
mariner-repos-microsoft-preview-2.0-3.cm2.noarch.rpm
mariner-repos-preview-2.0-3.cm2.noarch.rpm
mariner-repos-2.0-4.cm2.noarch.rpm
mariner-repos-debuginfo-2.0-4.cm2.noarch.rpm
mariner-repos-debuginfo-preview-2.0-4.cm2.noarch.rpm
mariner-repos-extended-2.0-4.cm2.noarch.rpm
mariner-repos-extended-preview-2.0-4.cm2.noarch.rpm
mariner-repos-extras-2.0-4.cm2.noarch.rpm
mariner-repos-extras-preview-2.0-4.cm2.noarch.rpm
mariner-repos-microsoft-2.0-4.cm2.noarch.rpm
mariner-repos-microsoft-preview-2.0-4.cm2.noarch.rpm
mariner-repos-preview-2.0-4.cm2.noarch.rpm
mariner-repos-shared-2.0-4.cm2.noarch.rpm
mariner-rpm-macros-2.0-10.cm2.noarch.rpm
meson-0.60.2-1.cm2.noarch.rpm
mpfr-4.1.0-1.cm2.x86_64.rpm

View File

@ -283,6 +283,7 @@ generate_pkggen_core () {
grep "^pinentry-" $TmpPkgGen
grep "^gnupg2-" $TmpPkgGen
grep "^gpgme-" $TmpPkgGen
grep "^mariner-repos-shared" $TmpPkgGen
grep "^mariner-repos-preview" $TmpPkgGen
grep "^libffi-" $TmpPkgGen
grep "^libtasn1-" $TmpPkgGen

View File

@ -71,6 +71,7 @@ fetch-external-image-packages: $(image_external_package_cache_summary)
# Changes to files located outside the base directory will not be detected.
validate-image-config: $(validate-config)
$(STATUS_FLAGS_DIR)/validate-image-config%.flag: $(go-imageconfigvalidator) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(config_other_files)
$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
$(go-imageconfigvalidator) \
--input=$(CONFIG_FILE) \
--dir=$(CONFIG_BASE_DIR) && \
@ -86,7 +87,7 @@ ifeq ($(USE_PREVIEW_REPO),y)
imagepkgfetcher_extra_flags += --use-preview-repo
endif
$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(packagelist_files) $(RPMS_DIR) $(imggen_rpms)
$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms)
$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
$(go-imagepkgfetcher) \
--input=$(CONFIG_FILE) \
@ -109,7 +110,7 @@ $(imager_disk_output_dir): $(STATUS_FLAGS_DIR)/imager_disk_output.flag
@touch $@
@echo Finished updating $@
$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(packagelist_files) $(assets_files) $(imggen_packagelist_files)
$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(assets_files)
$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
mkdir -p $(imager_disk_output_dir) && \
rm -rf $(imager_disk_output_dir)/* && \
@ -142,7 +143,7 @@ image: $(imager_disk_output_dir) $(imager_disk_output_files) $(go-roast) $(depen
--log-file=$(LOGS_DIR)/imggen/roast.log \
--image-tag=$(IMAGE_TAG)
$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config)
$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config)
$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
$(go-imagepkgfetcher) \
--input=$(CONFIG_FILE) \

View File

@ -105,7 +105,7 @@ func main() {
logger.Log.Fatalf("Can't ignore and force a rebuild of a package at the same time. Abusing packages: %v", ignoredAndRebuiltPackages)
}
packageVersToBuild, err := schedulerutils.CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild, *imageConfig, *baseDirPath)
packageVersToBuild, err := schedulerutils.CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild, *inputGraphFile, *imageConfig, *baseDirPath)
if err != nil {
logger.Log.Fatalf("Unable to generate package build list, error: %s", err)
}

View File

@ -7,12 +7,17 @@ import (
"microsoft.com/pkggen/imagegen/configuration"
"microsoft.com/pkggen/imagegen/installutils"
"microsoft.com/pkggen/internal/logger"
"microsoft.com/pkggen/internal/pkggraph"
"microsoft.com/pkggen/internal/pkgjson"
)
// CalculatePackagesToBuild generates a comprehensive list of all PackageVers that the scheduler should attempt to build.
// The build list is a superset of packagesNamesToBuild, packagesNamesToRebuild, packages listed in the image config, and kernels in the image config.
func CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild []string, imageConfig, baseDirPath string) (packageVersToBuild []*pkgjson.PackageVer, err error) {
// The build list is a superset of:
// - packagesNamesToBuild,
// - packagesNamesToRebuild,
// - local packages listed in the image config, and
// - kernels in the image config (if built locally).
func CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild []string, inputGraphFile, imageConfig, baseDirPath string) (packageVersToBuild []*pkgjson.PackageVer, err error) {
packageVersToBuild = convertPackageNamesIntoPackageVers(packagesNamesToBuild)
packageVersToBuild = append(packageVersToBuild, convertPackageNamesIntoPackageVers(packagesNamesToRebuild)...)
@ -25,6 +30,11 @@ func CalculatePackagesToBuild(packagesNamesToBuild, packagesNamesToRebuild []str
return
}
packageVersFromConfig, err = filterLocalPackagesOnly(packageVersFromConfig, inputGraphFile)
if err != nil {
return
}
packageVersToBuild = append(packageVersToBuild, packageVersFromConfig...)
return
}
@ -61,3 +71,29 @@ func extractPackagesFromConfig(configFile, baseDirPath string) (packageList []*p
return
}
// filterLocalPackagesOnly returns the subset of packageVersionsInConfig that only contains local packages.
func filterLocalPackagesOnly(packageVersionsInConfig []*pkgjson.PackageVer, inputGraph string) (filteredPackages []*pkgjson.PackageVer, err error) {
logger.Log.Debug("Filtering out external packages from list of packages extracted from the image config file.")
dependencyGraph := pkggraph.NewPkgGraph()
err = pkggraph.ReadDOTGraphFile(dependencyGraph, inputGraph)
if err != nil {
return
}
for _, pkgVer := range packageVersionsInConfig {
pkgNode, _ := dependencyGraph.FindBestPkgNode(pkgVer)
// A pkgNode for a local package has the following characteristics:
// 1) The pkgNode exists in the graph (is not nil).
// 2) The pkgNode doesn't have the 'StateUnresolved' or 'StateCached' state. These are reserved for external dependencies nodes.
if pkgNode != nil && pkgNode.RunNode.State != pkggraph.StateUnresolved && pkgNode.RunNode.State != pkggraph.StateCached {
filteredPackages = append(filteredPackages, pkgVer)
} else {
logger.Log.Debugf("Found external package to filter out: %v.", pkgVer)
}
}
return
}