Detect attempts to replace directories and report as conflicts

- Rpm cannot currently deal with directory changing to something else
  through upgrades. Until now we've barfed up an ugly error in the
  middle of transaction, leaving partially installed broken junk behind.
  It'd sure be nice to be able to actually handle this some day, but
  until then detecting and aborting early is a far better option than
  the former behavior.
- There are some "only in rpm" level bizarre quirks here: packages
  can work around this limitation by using a %pretrans scriptlet,
  which runs before the conflict detection. But this means a possible
  test-transaction (as done by eg yum) will still see the conflicts,
  as we obviously dont want to run disk-modifying scripts on
  test-transaction. So when looking at these removal conflicts, we filter
  them out on test-transaction IFF the package has a %pretrans script
  so there's a chance it might actually fix the conflict when we get
  to it in a real transaction. Obviously %pretrans from any package
  could in theory fix such issues, but as this is evil enough as it is,
  try to limit the damage... Without %pretrans, the only other option
  to get around these is manual intervention.

lib/transaction.c

@@ -281,6 +281,33 @@ static uint64_t countFiles(rpmts ts)
     return fc;
 }
 
+static int handleRemovalConflict(rpmfi fi, int fx, rpmfi ofi, int ofx)
+{
+    int rConflicts = 0; /* Removed files don't conflict, normally */
+    rpmFileTypes ft = rpmfiWhatis(rpmfiFModeIndex(fi, fx));
+    rpmFileTypes oft = rpmfiWhatis(rpmfiFModeIndex(ofi, ofx));
+    struct stat sb;
+
+    if (oft == XDIR) {
+	/* We can't handle directory changing to anything else */
+	if (ft != XDIR)
+	    rConflicts = 1;
+    }
+
+    /*
+     * ...but if the conflicting item is either not on disk, or has
+     * already been changed to the new type, we should be ok afterall.
+     */
+    if (rConflicts) {
+	char *fn = rpmfiFNIndex(fi, fx);
+	if (lstat(fn, &sb) || rpmfiWhatis(sb.st_mode) == ft)
+	    rConflicts = 0;
+	free(fn);
+    }
+
+    return rConflicts;
+}
+
 /**
  * handleInstInstalledFiles.
  * @param ts		transaction set
@@ -309,9 +336,21 @@ static void handleInstInstalledFile(const rpmts ts, rpmte p, rpmfi fi, int fx,
 	int rConflicts = 1;
 	char rState = RPMFILE_STATE_REPLACED;
 
-	/* Conflicts on to-be-removed files aren't normally an issue */
-	if (beingRemoved)
-	    rConflicts = 0;
+	/*
+	 * There are some removal conflicts we can't handle. However
+	 * if the package has a %pretrans scriptlet, it might be able to
+	 * fix the conflict. Let it through on test-transaction to allow
+	 * eg yum to get past it, if the conflict is present on the actual
+	 * transaction we'll abort. Behaving differently on test is nasty,
+	 * but its still better than barfing in middle of large transaction.
+	 */
+	if (beingRemoved) {
+	    rConflicts = handleRemovalConflict(fi, fx, otherFi, ofx);
+	    if (rConflicts && rpmteHaveTransScript(p, RPMTAG_PRETRANS)) {
+		if (rpmtsFlags(ts) & RPMTRANS_FLAG_TEST)
+		    rConflicts = 0;
+	    }
+	}
 
 	/* Resolve file conflicts to prefer Elf64 (if not forced). */
 	if (tscolor != 0 && FColor != 0 && oFColor != 0 && FColor != oFColor) {

tests/rpmconflict.at

@@ -238,7 +238,6 @@ AT_CLEANUP
 # Removal conflict on directory -> symlink change
 AT_SETUP([rpm -U replacing directory with symlink])
 AT_KEYWORDS([install])
-AT_XFAIL_IF([test $RPM_XFAIL -ne 2])
 AT_CHECK([
 RPMDB_CLEAR
 RPMDB_INIT

tests/rpmreplace.at

@@ -362,7 +362,6 @@ test -d "${tf}"
 AT_CLEANUP
 
 AT_SETUP([upgrade empty directory to regular file])
-AT_XFAIL_IF([test $RPM_XFAIL -ne 2])
 AT_KEYWORDS([install])
 AT_CHECK([
 RPMDB_CLEAR
@@ -424,7 +423,6 @@ test -L "${tf}" && test -d "${tf}"
 AT_CLEANUP
 
 AT_SETUP([upgrade empty directory to broken link])
-AT_XFAIL_IF([test $RPM_XFAIL -ne 2])
 AT_KEYWORDS([install])
 AT_CHECK([
 RPMDB_CLEAR
@@ -454,7 +452,6 @@ test -d "${tf}" && runroot rpm -U "${TOPDIR}"/RPMS/noarch/replacetest-2.0-1.noar
 AT_CLEANUP
 
 AT_SETUP([upgrade empty directory to file link])
-AT_XFAIL_IF([test $RPM_XFAIL -ne 2])
 AT_KEYWORDS([install])
 AT_CHECK([
 RPMDB_CLEAR