Add mode for updating with minimal writing (RhBug:783480)

How does the number of writes change?
I compared number of writes with and without minimal writing
for upgrade from RHEL 7.0 to RHEL 7.1, from RHEL 7.1 to RHEL 7.2, ...
In more details, the size of memory used for writing regular files.

The following table contains in the second column percentages of files
which can be only touched from all installed files. In the third column,
there are percentages of sum of their sizes.

                       |       Touched        |        Touched
                       |    Updated files     |     Updated bytes
------------------------------------------------------------------------
RHEL 7.0 -> RHEL 7.1   |         63 %         |          66 %
RHEL 7.1 -> RHEL 7.2   |         53 %         |          43 %
RHEL 7.2 -> RHEL 7.3   |         60 %         |          42 %
------------------------------------------------------------------------
F24 -> F25             |         63 %         |          40 %
F25-> Fraw             |         49 %         |          28 %

Does the speed change?
The update speed for classical disks or SSD almost does not change.

How it works?
If there is a file in the new package, which has the equal digest
as the corresponding file in the old package and the same file on
the disk (thus the contents are expected to be equal), rpm will not
install the whole file, but rather only upgrade the file's meta data.
In other cases, it will install the whole file.

lib/rpmfi.c

@@ -962,6 +962,56 @@ int rpmfilesCompare(rpmfiles afi, int aix, rpmfiles bfi, int bix)
     return 0;
 }
 
+int rpmfileContentsEqual(rpmfiles ofi, int oix, rpmfiles nfi, int nix)
+{
+    char * fn = rpmfilesFN(nfi, nix);
+    rpmFileTypes diskWhat, newWhat, oldWhat;
+    struct stat sb;
+    int equal = 0;
+
+    if (fn == NULL || (lstat(fn, &sb))) {
+	goto exit; /* The file doesn't exist on the disk */
+    }
+
+    if (rpmfilesFSize(nfi, nix) != sb.st_size) {
+	goto exit;
+    }
+
+    diskWhat = rpmfiWhatis((rpm_mode_t)sb.st_mode);
+    newWhat = rpmfiWhatis(rpmfilesFMode(nfi, nix));
+    oldWhat = rpmfiWhatis(rpmfilesFMode(ofi, oix));
+
+    if ((diskWhat == REG) && (newWhat == REG) && (oldWhat == REG)) {
+	int oalgo, nalgo;
+	size_t odiglen, ndiglen;
+	const unsigned char * odigest, * ndigest;
+	char buffer[1024];
+
+	odigest = rpmfilesFDigest(ofi, oix, &oalgo, &odiglen);
+	ndigest = rpmfilesFDigest(nfi, nix, &nalgo, &ndiglen);
+	/* See if the file in old pkg is identical to the one in new pkg */
+	if ((oalgo != nalgo) || (odiglen != ndiglen) || (!ndigest) ||
+	    (memcmp(odigest, ndigest, ndiglen) != 0)) {
+	    goto exit;
+	}
+
+	if (rpmDoDigest(nalgo, fn, 0, (unsigned char *)buffer, NULL) != 0) {
+	     goto exit;		/* assume file has been removed */
+	}
+
+	/* See if the file on disk is identical to the one in new pkg */
+	if (memcmp(ndigest, buffer, ndiglen) == 0) {
+	    equal = 1;
+	    goto exit;
+	}
+    }
+
+exit:
+    free(fn);
+    return equal;
+}
+
+
 rpmFileAction rpmfilesDecideFate(rpmfiles ofi, int oix,
 				   rpmfiles nfi, int nix,
 				   int skipMissing)

lib/rpmfi_internal.h

@@ -68,6 +68,18 @@ rpmsid rpmfilesODNId(rpmfiles fi, int jx);
 RPM_GNUC_INTERNAL
 struct fingerPrint_s *rpmfilesFps(rpmfiles fi);
 
+/** \ingroup rpmfi
+ * Check if the file in new package, in old package and on the disk have the same contents.
+ * @param 	new file info set
+ * @param 	new file index
+ * @param 	old file info set
+ * @param 	old file index
+ * @return	1 if the condition is satisfied, 0 otherwise
+ */
+RPM_GNUC_INTERNAL
+int rpmfileContentsEqual(rpmfiles ofi, int oix, rpmfiles nfi, int nix);
+
+
 RPM_GNUC_INTERNAL
 rpmFileAction rpmfilesDecideFate(rpmfiles ofi, int oix,
 				   rpmfiles nfi, int nix,

lib/rpmts.c

@@ -1056,6 +1056,8 @@ rpmts rpmtsCreate(void)
 
     ts->trigs2run = rpmtriggersCreate(10);
 
+    ts->min_writes = rpmExpandNumeric("%{_minimize_writes}");
+
     return rpmtsLink(ts);
 }
 

lib/rpmts_internal.h

@@ -84,6 +84,8 @@ struct rpmts_s {
     int nrefs;			/*!< Reference count. */
 
     rpmtriggers trigs2run;   /*!< Transaction file triggers */
+
+    int min_writes;             /*!< macro minimize_writes used */
 };
 
 #ifdef __cplusplus

lib/transaction.c

@@ -475,6 +475,14 @@ static void handleInstInstalledFile(const rpmts ts, rpmte p, rpmfiles fi, int fx
 	action = rpmfilesDecideFate(otherFi, ofx, fi, fx, skipMissing);
 	rpmfsSetAction(fs, fx, action);
     }
+
+    /* Skip already existing files - if 'minimize_writes' is set. */
+    if ((!isCfgFile) && (rpmfsGetAction(fs, fx) == FA_UNKNOWN)  && ts->min_writes) {
+	if (rpmfileContentsEqual(otherFi, ofx, fi, fx)) {
+	   rpmfsSetAction(fs, fx, FA_TOUCH);
+	}
+    }
+
     rpmfilesSetFReplacedSize(fi, fx, rpmfilesFSize(otherFi, ofx));
 }
 

macros.in

@@ -688,6 +688,11 @@ package or when debugging this package.\
 %_vsflags_rebuilddb	0xc0c00
 %_vsflags_verify	%{__vsflags}
 
+# Set to 1 to minimize writing (at the cost of more reads) to
+# conserve eg SSD disks.
+%_minimize_writes      0
+
+
 #
 # Default output format string for rpm -qa
 #

tests/rpmverify.at

@@ -193,3 +193,170 @@ runroot rpm -Va --nouser --nogroup
 [],
 [])
 AT_CLEANUP
+
+# ------------------------------
+# Upgraded package verification with min_writes 1
+AT_SETUP([Upgraded package verification with min_writes 1])
+AT_KEYWORDS([upgrade verify min_writes])
+AT_CHECK([
+RPMDB_CLEAR
+RPMDB_INIT
+tf="${RPMTEST}"/opt/foo
+rm -rf "${tf}" "${tf}".rpm*
+rm -rf "${TOPDIR}"
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 1.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 2.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 3.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 4.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpm -i /build/RPMS/noarch/replacetest-1.0-1.noarch.rpm
+cat "${tf}"
+touch -t 201703171717 ${tf}
+
+runroot rpm -U \
+	--define "_minimize_writes 1" \
+	 /build/RPMS/noarch/replacetest-2.0-1.noarch.rpm
+runroot rpm -Va --nouser --nogroup replacetest
+cat "${tf}"
+
+runroot rpm -U \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm
+runroot rpm -Va  --nouser --nogroup replacetest
+cat "${tf}"
+
+echo "xx" > "${tf}"
+cat "${tf}"
+
+runroot rpm -U \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-4.0-1.noarch.rpm
+runroot rpm -Va  --nouser --nogroup replacetest
+cat "${tf}"
+
+touch -t 201703171717 ${tf}
+
+runroot rpm -U --oldpackage \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm
+runroot rpm -Va  --nouser --nogroup replacetest
+cat "${tf}"
+],
+[0],
+[foo
+foo
+fox
+xx
+fox
+fox
+],
+[])
+AT_CLEANUP
+
+
+# ------------------------------
+# Upgraded package verification with min_writes 2
+AT_SETUP([Upgraded package verification with min_writes 2])
+AT_KEYWORDS([upgrade verify min_writes])
+AT_CHECK([
+RPMDB_CLEAR
+RPMDB_INIT
+tf="${RPMTEST}"/opt/foo
+rm -rf "${tf}" "${tf}".rpm*
+rm -rf "${TOPDIR}"
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 1.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 2.0" \
+	--define "filetype file" \
+	--define "filedata foo" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 3.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpmbuild --quiet -bb \
+        --define "ver 4.0" \
+	--define "filetype file" \
+	--define "filedata fox" \
+          /data/SPECS/replacetest.spec
+
+runroot rpm -i /build/RPMS/noarch/replacetest-1.0-1.noarch.rpm
+cat "${tf}"
+touch -t 201703171717 ${tf}
+
+runroot rpm -Uvv --fsmdebug \
+	--define "_minimize_writes 1" \
+	 /build/RPMS/noarch/replacetest-2.0-1.noarch.rpm > output.txt 2>&1
+runroot rpm -Va --nouser --nogroup replacetest
+grep -c "touch" output.txt
+cat "${tf}"
+
+
+runroot rpm -Uvv --fsmdebug \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm  > output.txt 2>&1
+runroot rpm -Va  --nouser --nogroup replacetest
+grep -c "touch" output.txt
+cat "${tf}"
+echo "xx" > "${tf}"
+cat "${tf}"
+
+runroot rpm -Uvv --fsmdebug \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-4.0-1.noarch.rpm  > output.txt 2>&1
+runroot rpm -Va  --nouser --nogroup replacetest
+grep -c "touch" output.txt
+cat "${tf}"
+
+touch -t 201703171717 ${tf}
+
+runroot rpm -U -Uvv --fsmdebug  --oldpackage \
+	--define "_minimize_writes 1" \
+	/build/RPMS/noarch/replacetest-3.0-1.noarch.rpm  > output.txt 2>&1
+runroot rpm -Va  --nouser --nogroup replacetest
+grep -c  "touch" output.txt
+cat "${tf}"
+],
+[0],
+[foo
+2
+foo
+1
+fox
+xx
+1
+fox
+2
+fox
+],
+[])
+AT_CLEANUP