Add mode for updating with minimal writing (RhBug:783480)

How does the number of writes change? I compared number of writes with and without minimal writing for upgrade from RHEL 7.0 to RHEL 7.1, from RHEL 7.1 to RHEL 7.2, ... In more details, the size of memory used for writing regular files. The following table contains in the second column percentages of files which can be only touched from all installed files. In the third column, there are percentages of sum of their sizes. | Touched | Touched | Updated files | Updated bytes ------------------------------------------------------------------------ RHEL 7.0 -> RHEL 7.1 | 63 % | 66 % RHEL 7.1 -> RHEL 7.2 | 53 % | 43 % RHEL 7.2 -> RHEL 7.3 | 60 % | 42 % ------------------------------------------------------------------------ F24 -> F25 | 63 % | 40 % F25-> Fraw | 49 % | 28 % Does the speed change? The update speed for classical disks or SSD almost does not change. How it works? If there is a file in the new package, which has the equal digest as the corresponding file in the old package and the same file on the disk (thus the contents are expected to be equal), rpm will not install the whole file, but rather only upgrade the file's meta data. In other cases, it will install the whole file.
2017-04-21 11:45:42 +02:00 · 2017-04-21 11:45:42 +02:00 · 29c48e14de
parent f7d4b2b726
commit 29c48e14de
7 changed files with 246 additions and 0 deletions
--- a/lib/rpmfi.c
+++ b/lib/rpmfi.c
@ -962,6 +962,56 @@ int rpmfilesCompare(rpmfiles afi, int aix, rpmfiles bfi, int bix)
    return 0;
 }

+int rpmfileContentsEqual(rpmfiles ofi, int oix, rpmfiles nfi, int nix)
+{
+    char * fn = rpmfilesFN(nfi, nix);
+    rpmFileTypes diskWhat, newWhat, oldWhat;
+    struct stat sb;
+    int equal = 0;
+
+    if (fn == NULL || (lstat(fn, &sb))) {
+	goto exit; /* The file doesn't exist on the disk */
+    }
+
+    if (rpmfilesFSize(nfi, nix) != sb.st_size) {
+	goto exit;
+    }
+
+    diskWhat = rpmfiWhatis((rpm_mode_t)sb.st_mode);
+    newWhat = rpmfiWhatis(rpmfilesFMode(nfi, nix));
+    oldWhat = rpmfiWhatis(rpmfilesFMode(ofi, oix));
+
+    if ((diskWhat == REG) && (newWhat == REG) && (oldWhat == REG)) {
+	int oalgo, nalgo;
+	size_t odiglen, ndiglen;
+	const unsigned char * odigest, * ndigest;
+	char buffer[1024];
+
+	odigest = rpmfilesFDigest(ofi, oix, &oalgo, &odiglen);
+	ndigest = rpmfilesFDigest(nfi, nix, &nalgo, &ndiglen);
+	/* See if the file in old pkg is identical to the one in new pkg */
+	if ((oalgo != nalgo) || (odiglen != ndiglen) || (!ndigest) ||
+	    (memcmp(odigest, ndigest, ndiglen) != 0)) {
+	    goto exit;
+	}
+
+	if (rpmDoDigest(nalgo, fn, 0, (unsigned char *)buffer, NULL) != 0) {
+	     goto exit;		/* assume file has been removed */
+	}
+
+	/* See if the file on disk is identical to the one in new pkg */
+	if (memcmp(ndigest, buffer, ndiglen) == 0) {
+	    equal = 1;
+	    goto exit;
+	}
+    }
+
+exit:
+    free(fn);
+    return equal;
+}
+
+
 rpmFileAction rpmfilesDecideFate(rpmfiles ofi, int oix,
 				   rpmfiles nfi, int nix,
 				   int skipMissing)
--- a/lib/rpmfi_internal.h
+++ b/lib/rpmfi_internal.h
@ -68,6 +68,18 @@ rpmsid rpmfilesODNId(rpmfiles fi, int jx);
 RPM_GNUC_INTERNAL
 struct fingerPrint_s *rpmfilesFps(rpmfiles fi);

+/** \ingroup rpmfi
+ * Check if the file in new package, in old package and on the disk have the same contents.
+ * @param 	new file info set
+ * @param 	new file index
+ * @param 	old file info set
+ * @param 	old file index
+ * @return	1 if the condition is satisfied, 0 otherwise
+ */
+RPM_GNUC_INTERNAL
+int rpmfileContentsEqual(rpmfiles ofi, int oix, rpmfiles nfi, int nix);
+
+
 RPM_GNUC_INTERNAL
 rpmFileAction rpmfilesDecideFate(rpmfiles ofi, int oix,
 				   rpmfiles nfi, int nix,
--- a/lib/rpmts.c
+++ b/lib/rpmts.c
@ -1056,6 +1056,8 @@ rpmts rpmtsCreate(void)

    ts->trigs2run = rpmtriggersCreate(10);

+    ts->min_writes = rpmExpandNumeric("%{_minimize_writes}");
+
    return rpmtsLink(ts);
 }

--- a/lib/rpmts_internal.h
+++ b/lib/rpmts_internal.h
@ -84,6 +84,8 @@ struct rpmts_s {
    int nrefs;			/*!< Reference count. */

    rpmtriggers trigs2run;   /*!< Transaction file triggers */
+
+    int min_writes;             /*!< macro minimize_writes used */
 };

 #ifdef __cplusplus
--- a/lib/transaction.c
+++ b/lib/transaction.c
@ -475,6 +475,14 @@ static void handleInstInstalledFile(const rpmts ts, rpmte p, rpmfiles fi, int fx
 	action = rpmfilesDecideFate(otherFi, ofx, fi, fx, skipMissing);
 	rpmfsSetAction(fs, fx, action);
    }
+
+    /* Skip already existing files - if 'minimize_writes' is set. */
+    if ((!isCfgFile) && (rpmfsGetAction(fs, fx) == FA_UNKNOWN)  && ts->min_writes) {
+	if (rpmfileContentsEqual(otherFi, ofx, fi, fx)) {
+	   rpmfsSetAction(fs, fx, FA_TOUCH);
+	}
+    }
+
    rpmfilesSetFReplacedSize(fi, fx, rpmfilesFSize(otherFi, ofx));
 }

--- a/macros.in
+++ b/macros.in
@ -688,6 +688,11 @@ package or when debugging this package.\
 %_vsflags_rebuilddb	0xc0c00
 %_vsflags_verify	%{__vsflags}

+# Set to 1 to minimize writing (at the cost of more reads) to
+# conserve eg SSD disks.
+%_minimize_writes      0
+
+
 #
 # Default output format string for rpm -qa
 #
--- a/tests/rpmverify.at
+++ b/tests/rpmverify.at
@ -193,3 +193,170 @@ runroot rpm -Va --nouser --nogroup
 [],
 [])
 AT_CLEANUP
+
+# ------------------------------
+# Upgraded package verification with min_writes 1
+AT_SETUP([Upgraded package verification with min_writes 1])
+AT_KEYWORDS([upgrade verify min_writes])
+AT_CHECK([
+RPMDB_CLEAR
+RPMDB_INIT
+tf="${RPMTEST}"/opt/foo
+rm -rf "${tf}" "${tf}".rpm*
+rm -rf "${TOPDIR}"
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 1.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 2.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 3.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 4.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpm -i /build/RPMS/noarch/replacetest-1.0-1.noarch.rpm
+cat "${tf}"
+touch -t 201703171717 ${tf}
+
+runroot rpm -U \
+	--define "_minimize_writes 1" \
+	 /build/RPMS/noarch/replacetest-2.0-1.noarch.rpm
+runroot rpm -Va --nouser --nogroup replacetest
+cat "${tf}"
+
+runroot rpm -U \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm
+runroot rpm -Va  --nouser --nogroup replacetest
+cat "${tf}"
+
+echo "xx" > "${tf}"
+cat "${tf}"
+
+runroot rpm -U \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-4.0-1.noarch.rpm
+runroot rpm -Va  --nouser --nogroup replacetest
+cat "${tf}"
+
+touch -t 201703171717 ${tf}
+
+runroot rpm -U --oldpackage \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm
+runroot rpm -Va  --nouser --nogroup replacetest
+cat "${tf}"
+],
+[0],
+[foo
+foo
+fox
+xx
+fox
+fox
+],
+[])
+AT_CLEANUP
+
+
+# ------------------------------
+# Upgraded package verification with min_writes 2
+AT_SETUP([Upgraded package verification with min_writes 2])
+AT_KEYWORDS([upgrade verify min_writes])
+AT_CHECK([
+RPMDB_CLEAR
+RPMDB_INIT
+tf="${RPMTEST}"/opt/foo
+rm -rf "${tf}" "${tf}".rpm*
+rm -rf "${TOPDIR}"
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 1.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 2.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 3.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 4.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpm -i /build/RPMS/noarch/replacetest-1.0-1.noarch.rpm
+cat "${tf}"
+touch -t 201703171717 ${tf}
+
+runroot rpm -Uvv --fsmdebug \
+	--define "_minimize_writes 1" \
+	 /build/RPMS/noarch/replacetest-2.0-1.noarch.rpm > output.txt 2>&1
+runroot rpm -Va --nouser --nogroup replacetest
+grep -c "touch" output.txt
+cat "${tf}"
+
+
+runroot rpm -Uvv --fsmdebug \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm  > output.txt 2>&1
+runroot rpm -Va  --nouser --nogroup replacetest
+grep -c "touch" output.txt
+cat "${tf}"
+echo "xx" > "${tf}"
+cat "${tf}"
+
+runroot rpm -Uvv --fsmdebug \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-4.0-1.noarch.rpm  > output.txt 2>&1
+runroot rpm -Va  --nouser --nogroup replacetest
+grep -c "touch" output.txt
+cat "${tf}"
+
+touch -t 201703171717 ${tf}
+
+runroot rpm -U -Uvv --fsmdebug  --oldpackage \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm  > output.txt 2>&1
+runroot rpm -Va  --nouser --nogroup replacetest
+grep -c  "touch" output.txt
+cat "${tf}"
+],
+[0],
+[foo
+2
+foo
+1
+fox
+xx
+1
+fox
+2
+fox
+],
+[])
+AT_CLEANUP