folkslinux
/
rpm
mirror of https://github.com/rpm-software-management/rpm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement a key-only rpmdb index iterator 

The regular index iterator grabs the associated data too, which we
don't always need. The data associated with indexes is relatively
lightweight, but as with everything, it adds up if in the millions scale.

Update all backends to allow for NULL set in the index retrieve to
signal key-only retrieval. Ndb actually had an earlier, abandoned
implementation of the same idea under slightly different API, lets
reuse the code-block.
This commit is contained in:
Panu Matilainen 2019-12-05 14:22:08 +02:00
parent 61ea5a8ea6
commit 4eb7900d54
6 changed files with 46 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/backend/db3.c
View File

@ -1008,7 +1008,7 @@ static rpmRC db3_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
dbiIndexSet *set, int searchType)
{
rpmRC rc = RPMRC_FAIL; /* assume failure */
if (dbi != NULL && dbc != NULL && set != NULL) {
if (dbi != NULL && dbc != NULL) {
int cflags = DB_NEXT;
int dbrc;
DBT data, key;
@ -1033,12 +1033,14 @@ static rpmRC db3_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
if (searchType == DBC_PREFIX_SEARCH &&
(key.size < keylen || memcmp(key.data, keyp, keylen) != 0))
break;
dbt2set(dbi, &data, &newset);
if (*set == NULL) {
*set = newset;
} else {
dbiIndexSetAppendSet(*set, newset, 0);
dbiIndexSetFree(newset);
if (set) {
dbt2set(dbi, &data, &newset);
if (*set == NULL) {
*set = newset;
} else {
dbiIndexSetAppendSet(*set, newset, 0);
dbiIndexSetFree(newset);
}
}
if (searchType != DBC_PREFIX_SEARCH)
break;
@ -1048,7 +1050,7 @@ static rpmRC db3_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
}
/* fixup result status for prefix search */
if (searchType == DBC_PREFIX_SEARCH) {
if (searchType == DBC_PREFIX_SEARCH && set) {
if (dbrc == DB_NOTFOUND && *set != NULL && (*set)->count > 0)
dbrc = 0;
else if (dbrc == 0 && (*set == NULL || (*set)->count == 0))

18
lib/backend/lmdb.c
View File

@ -574,7 +574,7 @@ static rpmRC lmdb_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
dbiIndexSet *set, int searchType)
{
rpmRC rc = RPMRC_FAIL; /* assume failure */
if (dbi != NULL && dbc != NULL && set != NULL) {
if (dbi != NULL && dbc != NULL) {
int cflags = MDB_NEXT;
int dbrc;
MDB_val key = { 0, NULL };
@ -598,12 +598,14 @@ static rpmRC lmdb_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
if (searchType == DBC_PREFIX_SEARCH &&
(key.mv_size < keylen || memcmp(key.mv_data, keyp, keylen) != 0))
break;
dbt2set(dbi, &data, &newset);
if (*set == NULL) {
*set = newset;
} else {
dbiIndexSetAppendSet(*set, newset, 0);
dbiIndexSetFree(newset);
if (set) {
dbt2set(dbi, &data, &newset);
if (*set == NULL) {
*set = newset;
} else {
dbiIndexSetAppendSet(*set, newset, 0);
dbiIndexSetFree(newset);
}
}
if (searchType != DBC_PREFIX_SEARCH)
break;
@ -613,7 +615,7 @@ static rpmRC lmdb_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
}
/* fixup result status for prefix search */
if (searchType == DBC_PREFIX_SEARCH) {
if (searchType == DBC_PREFIX_SEARCH && set) {
if (dbrc == MDB_NOTFOUND && *set != NULL && (*set)->count > 0)
dbrc = 0;
else if (dbrc == 0 && (*set == NULL || (*set)->count == 0))

6
lib/backend/ndb/glue.c
View File

@ -400,15 +400,15 @@ static rpmRC ndb_idxdbIter(dbiIndex dbi, dbiCursor dbc, dbiIndexSet *set)
}
k = dbc->listdata + dbc->list[dbc->ilist];
kl = dbc->list[dbc->ilist + 1];
#if 0
if (searchType == DBC_KEY_SEARCH) {
if (set == NULL) {
dbc->ilist += 2;
dbc->key = k;
dbc->keylen = kl;
rc = RPMRC_OK;
break;
}
#endif
pkglist = 0;
pkglistn = 0;
rc = rpmidxGet(dbc->dbi->dbi_db, k, kl, &pkglist, &pkglistn);

8
lib/backend/sqlite.c
View File

@ -578,15 +578,17 @@ static rpmRC sqlite_idxdbIter(dbiIndex dbi, dbiCursor dbc, const char *keyp, siz
rc = sqlite3_step(dbc->stmt);
if (rc == SQLITE_ROW) {
dbiCursor kc = dbiCursorInit(dbi, 0);
if (dbc->ctype == SQLITE_TEXT) {
dbc->key = sqlite3_column_text(dbc->stmt, 0);
} else {
dbc->key = sqlite3_column_blob(dbc->stmt, 0);
}
dbc->keylen = sqlite3_column_bytes(dbc->stmt, 0);
rc = sqlite_idxdbByKey(dbi, kc, dbc->key, dbc->keylen, set);
dbiCursorFree(dbi, kc);
if (set) {
dbiCursor kc = dbiCursorInit(dbi, 0);
rc = sqlite_idxdbByKey(dbi, kc, dbc->key, dbc->keylen, set);
dbiCursorFree(dbi, kc);
}
rc = RPMRC_OK;
} else if (rc == SQLITE_DONE) {
if (searchType == DBC_PREFIX_SEARCH && (*set))

11
lib/rpmdb.c
View File

@ -298,6 +298,7 @@ struct rpmdbIndexIterator_s {
dbiCursor ii_dbc;
dbiIndexSet ii_set;
unsigned int *ii_hdrNums;
int ii_skipdata;
};
static rpmdb rpmdbRock;
@ -1885,6 +1886,13 @@ rpmdbIndexIterator rpmdbIndexIteratorInit(rpmdb db, rpmDbiTag rpmtag)
return ii;
}
rpmdbIndexIterator rpmdbIndexKeyIteratorInit(rpmdb db, rpmDbiTag rpmtag)
{
rpmdbIndexIterator ki = rpmdbIndexIteratorInit(db, rpmtag);
ki->ii_skipdata = 1;
return ki;
}
int rpmdbIndexIteratorNext(rpmdbIndexIterator ii, const void ** key, size_t * keylen)
{
int rc;
@ -1899,7 +1907,8 @@ int rpmdbIndexIteratorNext(rpmdbIndexIterator ii, const void ** key, size_t * ke
/* free old data */
ii->ii_set = dbiIndexSetFree(ii->ii_set);
rc = idxdbGet(ii->ii_dbi, ii->ii_dbc, NULL, 0, &ii->ii_set, DBC_NORMAL_SEARCH);
rc = idxdbGet(ii->ii_dbi, ii->ii_dbc, NULL, 0,
ii->ii_skipdata ? NULL : &ii->ii_set, DBC_NORMAL_SEARCH);
*key = idxdbKey(ii->ii_dbi, ii->ii_dbc, &iikeylen);
*keylen = iikeylen;

8
lib/rpmdb.h
View File

@ -153,6 +153,14 @@ Header rpmdbNextIterator(rpmdbMatchIterator mi);
*/
rpmdbMatchIterator rpmdbFreeIterator(rpmdbMatchIterator mi);
/** \ingroup rpmdb
* Get an iterator for index keys
* @param db rpm database
* @param rpmtag the index to iterate over
* @return the index iterator
*/
rpmdbIndexIterator rpmdbIndexKeyIteratorInit(rpmdb db, rpmDbiTag rpmtag);
/** \ingroup rpmdb
* Get an iterator for an index
* @param db rpm database