Add runtime settable custom memory allocation failure callback routine

- lets API users perform theirn own cleanup / exit through their own
  routines in case of allocation failure, or even try to free up
  some memory and retry allocation

rpmio/rpmmalloc.c

@@ -9,11 +9,31 @@
 #define	EXIT_FAILURE	1
 #endif
 
+static rpmMemFailFunc failfunc = NULL;
+static void *failfunc_data = NULL;
+
+/*
+ * Give memfail callback a chance to try to give us memory or perform
+ * it's own cleanup. If we dont get memory we die anyway as rpm doesn't 
+ * check for NULL returns from allocations.
+ */
 static void *vmefail(size_t size)
 {
-    fprintf(stderr, _("memory alloc (%u bytes) returned NULL.\n"), (unsigned)size);
-    exit(EXIT_FAILURE);
-    return NULL;
+    void *val = failfunc ? (*failfunc)(size, failfunc_data) : NULL;
+    if (val == NULL) {
+	fprintf(stderr, _("memory alloc (%u bytes) returned NULL.\n"),
+		(unsigned)size);
+	exit(EXIT_FAILURE);
+    }
+    return val;	
+}
+
+void * rpmSetMemFail(rpmMemFailFunc func, void *data)
+{
+    void *ofunc = failfunc;
+    failfunc = func;
+    failfunc_data = data;
+    return ofunc;
 }
 
 void * rmalloc (size_t size)

rpmio/rpmutil.h

@@ -127,4 +127,25 @@ char * rstrdup(const char *str);
 /* Rpm specific free() which returns NULL */
 void * rfree(void *ptr);
 
+/** \ingroup rpmutil
+ * Memory allocation failure callback prototype. When registered through
+ * rpmSetMemFail(), this gets called if memory allocation through rmalloc()
+ * and friends fails. If the application can somehow recover memory here,
+ * it can return a newly allocated memory block of requested size, otherwise
+ * it must return NULL after performing it's own shutdown deeds or 
+ * terminate itself.
+ * @param size		Size of allocation request in bytes
+ * @param data		User data (or NULL)
+ * @return		Allocated memory block of requested size or NULL
+ */
+typedef void * (*rpmMemFailFunc) (size_t size, void *data);
+
+/** \ingroup rpmutil
+ * Set memory allocation failure callback.
+ * @param func		Allocation failure callback function
+ * @param data		User data (or NULL)
+ * @return		Previous callback function
+ */
+void * rpmSetMemFail(rpmMemFailFunc func, void *data);
+
 #endif /* _RPMUTIL_H */