"Fossies" - the Fresh Open Source Software Archive

Member "citadel/database.c" (5 Jun 2021, 21070 Bytes) of package /linux/www/citadel.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "database.c" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 9.01_vs_902.

    1 /*
    2  * This is a data store backend for the Citadel server which uses Berkeley DB.
    3  *
    4  * Copyright (c) 1987-2021 by the citadel.org team
    5  *
    6  * This program is open source software; you can redistribute it and/or
    7  * modify it under the terms of the GNU General Public License version 3.
    8  *
    9  * This program is distributed in the hope that it will be useful,
   10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   12  * GNU General Public License for more details.
   13  */
   14 
   15 /*****************************************************************************
   16        Tunable configuration parameters for the Berkeley DB back end
   17  *****************************************************************************/
   18 
   19 /* Citadel will checkpoint the db at the end of every session, but only if
   20  * the specified number of kilobytes has been written, or if the specified
   21  * number of minutes has passed, since the last checkpoint.
   22  */
   23 #define MAX_CHECKPOINT_KBYTES   256
   24 #define MAX_CHECKPOINT_MINUTES  15
   25 
   26 /*****************************************************************************/
   27 
   28 #include "sysdep.h"
   29 #include <stdlib.h>
   30 #include <unistd.h>
   31 #include <sys/stat.h>
   32 #include <stdio.h>
   33 #include <dirent.h>
   34 #include <zlib.h>
   35 
   36 #include <db.h>
   37 
   38 #if DB_VERSION_MAJOR < 5
   39 #error Citadel requires Berkeley DB v5.0 or newer.  Please upgrade.
   40 #endif
   41 
   42 #include <libcitadel.h>
   43 
   44 #include "ctdl_module.h"
   45 #include "control.h"
   46 #include "citserver.h"
   47 #include "config.h"
   48 
   49 static DB *dbp[MAXCDB];     /* One DB handle for each Citadel database */
   50 static DB_ENV *dbenv;       /* The DB environment (global) */
   51 
   52 
   53 void cdb_abort(void) {
   54     syslog(LOG_DEBUG, "db: citserver is stopping in order to prevent data loss. uid=%d gid=%d euid=%d egid=%d",
   55         getuid(), getgid(), geteuid(), getegid()
   56     );
   57     exit(CTDLEXIT_DB);
   58 }
   59 
   60 
   61 /* Verbose logging callback */
   62 void cdb_verbose_log(const DB_ENV * dbenv, const char *msg) {
   63     if (!IsEmptyStr(msg)) {
   64         syslog(LOG_DEBUG, "db: %s", msg);
   65     }
   66 }
   67 
   68 
   69 /* Verbose logging callback */
   70 void cdb_verbose_err(const DB_ENV * dbenv, const char *errpfx, const char *msg) {
   71     syslog(LOG_ERR, "db: %s", msg);
   72 }
   73 
   74 
   75 /* wrapper for txn_abort() that logs/aborts on error */
   76 static void txabort(DB_TXN *tid) {
   77     int ret;
   78 
   79     ret = tid->abort(tid);
   80 
   81     if (ret) {
   82         syslog(LOG_ERR, "db: txn_abort: %s", db_strerror(ret));
   83         cdb_abort();
   84     }
   85 }
   86 
   87 
   88 /* wrapper for txn_commit() that logs/aborts on error */
   89 static void txcommit(DB_TXN *tid) {
   90     int ret;
   91 
   92     ret = tid->commit(tid, 0);
   93 
   94     if (ret) {
   95         syslog(LOG_ERR, "db: txn_commit: %s", db_strerror(ret));
   96         cdb_abort();
   97     }
   98 }
   99 
  100 
  101 /* wrapper for txn_begin() that logs/aborts on error */
  102 static void txbegin(DB_TXN **tid) {
  103     int ret;
  104 
  105     ret = dbenv->txn_begin(dbenv, NULL, tid, 0);
  106 
  107     if (ret) {
  108         syslog(LOG_ERR, "db: txn_begin: %s", db_strerror(ret));
  109         cdb_abort();
  110     }
  111 }
  112 
  113 
  114 /* panic callback */
  115 static void dbpanic(DB_ENV * env, int errval) {
  116     syslog(LOG_ERR, "db: PANIC: %s", db_strerror(errval));
  117 }
  118 
  119 
  120 static void cclose(DBC * cursor) {
  121     int ret;
  122 
  123     if ((ret = cursor->c_close(cursor))) {
  124         syslog(LOG_ERR, "db: c_close: %s", db_strerror(ret));
  125         cdb_abort();
  126     }
  127 }
  128 
  129 
  130 static void bailIfCursor(DBC ** cursors, const char *msg) {
  131     int i;
  132 
  133     for (i = 0; i < MAXCDB; i++)
  134         if (cursors[i] != NULL) {
  135             syslog(LOG_ERR, "db: cursor still in progress on cdb %02x: %s", i, msg);
  136             cdb_abort();
  137         }
  138 }
  139 
  140 
  141 void cdb_check_handles(void) {
  142     bailIfCursor(TSD->cursors, "in check_handles");
  143 
  144     if (TSD->tid != NULL) {
  145         syslog(LOG_ERR, "db: transaction still in progress!");
  146         cdb_abort();
  147     }
  148 }
  149 
  150 
  151 /*
  152  * Request a checkpoint of the database.  Called once per minute by the thread manager.
  153  */
  154 void cdb_checkpoint(void) {
  155     int ret;
  156 
  157     syslog(LOG_DEBUG, "db: -- checkpoint --");
  158     ret = dbenv->txn_checkpoint(dbenv, MAX_CHECKPOINT_KBYTES, MAX_CHECKPOINT_MINUTES, 0);
  159 
  160     if (ret != 0) {
  161         syslog(LOG_ERR, "db: cdb_checkpoint() txn_checkpoint: %s", db_strerror(ret));
  162         cdb_abort();
  163     }
  164 
  165     /* After a successful checkpoint, we can cull the unused logs */
  166     if (CtdlGetConfigInt("c_auto_cull")) {
  167         ret = dbenv->log_set_config(dbenv, DB_LOG_AUTO_REMOVE, 1);
  168     }
  169     else {
  170         ret = dbenv->log_set_config(dbenv, DB_LOG_AUTO_REMOVE, 0);
  171     }
  172 }
  173 
  174 
  175 /*
  176  * Open the various databases we'll be using.  Any database which
  177  * does not exist should be created.  Note that we don't need a
  178  * critical section here, because there aren't any active threads
  179  * manipulating the database yet.
  180  */
  181 void open_databases(void) {
  182     int ret;
  183     int i;
  184     char dbfilename[32];
  185     u_int32_t flags = 0;
  186     int dbversion_major, dbversion_minor, dbversion_patch;
  187 
  188     syslog(LOG_DEBUG, "db: open_databases() starting");
  189     syslog(LOG_DEBUG, "db: Compiled libdb: %s", DB_VERSION_STRING);
  190     syslog(LOG_DEBUG, "db:   Linked libdb: %s", db_version(&dbversion_major, &dbversion_minor, &dbversion_patch));
  191     syslog(LOG_DEBUG, "db:    Linked zlib: %s", zlibVersion());
  192 
  193     /*
  194      * Silently try to create the database subdirectory.  If it's already there, no problem.
  195      */
  196     if ((mkdir(ctdl_db_dir, 0700) != 0) && (errno != EEXIST)) {
  197         syslog(LOG_ERR, "db: unable to create database directory [%s]: %m", ctdl_db_dir);
  198     }
  199     if (chmod(ctdl_db_dir, 0700) != 0) {
  200         syslog(LOG_ERR, "db: unable to set database directory permissions [%s]: %m", ctdl_db_dir);
  201     }
  202     if (chown(ctdl_db_dir, CTDLUID, (-1)) != 0) {
  203         syslog(LOG_ERR, "db: unable to set the owner for [%s]: %m", ctdl_db_dir);
  204     }
  205     syslog(LOG_DEBUG, "db: Setting up DB environment");
  206     // db_env_set_func_yield((int (*)(u_long,  u_long))sched_yield);
  207     ret = db_env_create(&dbenv, 0);
  208     if (ret) {
  209         syslog(LOG_ERR, "db: db_env_create: %s", db_strerror(ret));
  210         syslog(LOG_ERR, "db: exit code %d", ret);
  211         exit(CTDLEXIT_DB);
  212     }
  213     dbenv->set_errpfx(dbenv, "citserver");
  214     dbenv->set_paniccall(dbenv, dbpanic);
  215     dbenv->set_errcall(dbenv, cdb_verbose_err);
  216     dbenv->set_errpfx(dbenv, "ctdl");
  217     dbenv->set_msgcall(dbenv, cdb_verbose_log);
  218     dbenv->set_verbose(dbenv, DB_VERB_DEADLOCK, 1);
  219     dbenv->set_verbose(dbenv, DB_VERB_RECOVERY, 1);
  220 
  221     /*
  222      * We want to specify the shared memory buffer pool cachesize,
  223      * but everything else is the default.
  224      */
  225     ret = dbenv->set_cachesize(dbenv, 0, 64 * 1024, 0);
  226     if (ret) {
  227         syslog(LOG_ERR, "db: set_cachesize: %s", db_strerror(ret));
  228         dbenv->close(dbenv, 0);
  229         syslog(LOG_ERR, "db: exit code %d", ret);
  230         exit(CTDLEXIT_DB);
  231     }
  232 
  233     if ((ret = dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT))) {
  234         syslog(LOG_ERR, "db: set_lk_detect: %s", db_strerror(ret));
  235         dbenv->close(dbenv, 0);
  236         syslog(LOG_ERR, "db: exit code %d", ret);
  237         exit(CTDLEXIT_DB);
  238     }
  239 
  240     flags = DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE | DB_INIT_TXN | DB_INIT_LOCK | DB_THREAD | DB_INIT_LOG;
  241     syslog(LOG_DEBUG, "db: dbenv->open(dbenv, %s, %d, 0)", ctdl_db_dir, flags);
  242     ret = dbenv->open(dbenv, ctdl_db_dir, flags, 0);                // try opening the database cleanly
  243     if (ret == DB_RUNRECOVERY) {
  244         syslog(LOG_ERR, "db: dbenv->open: %s", db_strerror(ret));
  245         syslog(LOG_ERR, "db: attempting recovery...");
  246         flags |= DB_RECOVER;
  247         ret = dbenv->open(dbenv, ctdl_db_dir, flags, 0);            // try recovery
  248     }
  249     if (ret == DB_RUNRECOVERY) {
  250         syslog(LOG_ERR, "db: dbenv->open: %s", db_strerror(ret));
  251         syslog(LOG_ERR, "db: attempting catastrophic recovery...");
  252         flags &= ~DB_RECOVER;
  253         flags |= DB_RECOVER_FATAL;
  254         ret = dbenv->open(dbenv, ctdl_db_dir, flags, 0);            // try catastrophic recovery
  255     }
  256     if (ret) {
  257         syslog(LOG_ERR, "db: dbenv->open: %s", db_strerror(ret));
  258         dbenv->close(dbenv, 0);
  259         syslog(LOG_ERR, "db: exit code %d", ret);
  260         exit(CTDLEXIT_DB);
  261     }
  262 
  263     syslog(LOG_INFO, "db: mounting databases");
  264     for (i = 0; i < MAXCDB; ++i) {
  265         ret = db_create(&dbp[i], dbenv, 0);                 // Create a database handle
  266         if (ret) {
  267             syslog(LOG_ERR, "db: db_create: %s", db_strerror(ret));
  268             syslog(LOG_ERR, "db: exit code %d", ret);
  269             exit(CTDLEXIT_DB);
  270         }
  271 
  272         snprintf(dbfilename, sizeof dbfilename, "cdb.%02x", i);         // table names by number
  273         ret = dbp[i]->open(dbp[i], NULL, dbfilename, NULL, DB_BTREE, DB_CREATE | DB_AUTO_COMMIT | DB_THREAD, 0600);
  274         if (ret) {
  275             syslog(LOG_ERR, "db: db_open[%02x]: %s", i, db_strerror(ret));
  276             if (ret == ENOMEM) {
  277                 syslog(LOG_ERR, "db: You may need to tune your database; please read http://www.citadel.org/doku.php?id=faq:troubleshooting:out_of_lock_entries for more information.");
  278             }
  279             syslog(LOG_ERR, "db: exit code %d", ret);
  280             exit(CTDLEXIT_DB);
  281         }
  282     }
  283 }
  284 
  285 
  286 /*
  287  * Make sure we own all the files, because in a few milliseconds we're going to drop root privs.
  288  */
  289 void cdb_chmod_data(void) {
  290     DIR *dp;
  291     struct dirent *d;
  292     char filename[PATH_MAX];
  293 
  294     dp = opendir(ctdl_db_dir);
  295     if (dp != NULL) {
  296         while (d = readdir(dp), d != NULL) {
  297             if (d->d_name[0] != '.') {
  298                 snprintf(filename, sizeof filename, "%s/%s", ctdl_db_dir, d->d_name);
  299                 syslog(LOG_DEBUG, "db: chmod(%s, 0600) returned %d", filename, chmod(filename, 0600));
  300                 syslog(LOG_DEBUG, "db: chown(%s, CTDLUID, -1) returned %d",
  301                     filename, chown(filename, CTDLUID, (-1))
  302                 );
  303             }
  304         }
  305         closedir(dp);
  306     }
  307 }
  308 
  309 
  310 /*
  311  * Close all of the db database files we've opened.  This can be done
  312  * in a loop, since it's just a bunch of closes.
  313  */
  314 void close_databases(void) {
  315     int i;
  316     int ret;
  317 
  318     syslog(LOG_INFO, "db: performing final checkpoint");
  319     if ((ret = dbenv->txn_checkpoint(dbenv, 0, 0, 0))) {
  320         syslog(LOG_ERR, "db: txn_checkpoint: %s", db_strerror(ret));
  321     }
  322 
  323     syslog(LOG_INFO, "db: flushing the database logs");
  324     if ((ret = dbenv->log_flush(dbenv, NULL))) {
  325         syslog(LOG_ERR, "db: log_flush: %s", db_strerror(ret));
  326     }
  327 
  328     /* close the tables */
  329     syslog(LOG_INFO, "db: closing databases");
  330     for (i = 0; i < MAXCDB; ++i) {
  331         syslog(LOG_INFO, "db: closing database %02x", i);
  332         ret = dbp[i]->close(dbp[i], 0);
  333         if (ret) {
  334             syslog(LOG_ERR, "db: db_close: %s", db_strerror(ret));
  335         }
  336 
  337     }
  338 
  339     // This seemed nifty at the time but did anyone really look at it?
  340     // #ifdef DB_STAT_ALL
  341     // /* print some statistics... */
  342     // dbenv->lock_stat_print(dbenv, DB_STAT_ALL);
  343     // #endif
  344 
  345     /* Close the handle. */
  346     ret = dbenv->close(dbenv, 0);
  347     if (ret) {
  348         syslog(LOG_ERR, "db: DBENV->close: %s", db_strerror(ret));
  349     }
  350 }
  351 
  352 
  353 /*
  354  * Decompress a database item if it was compressed on disk
  355  */
  356 void cdb_decompress_if_necessary(struct cdbdata *cdb) {
  357     static int magic = COMPRESS_MAGIC;
  358 
  359     if ((cdb == NULL) || (cdb->ptr == NULL) || (cdb->len < sizeof(magic)) || (memcmp(cdb->ptr, &magic, sizeof(magic)))) {
  360         return;
  361     }
  362 
  363     /* At this point we know we're looking at a compressed item. */
  364 
  365     struct CtdlCompressHeader zheader;
  366     char *uncompressed_data;
  367     char *compressed_data;
  368     uLongf destLen, sourceLen;
  369     size_t cplen;
  370 
  371     memset(&zheader, 0, sizeof(struct CtdlCompressHeader));
  372     cplen = sizeof(struct CtdlCompressHeader);
  373     if (sizeof(struct CtdlCompressHeader) > cdb->len) {
  374         cplen = cdb->len;
  375     }
  376     memcpy(&zheader, cdb->ptr, cplen);
  377 
  378     compressed_data = cdb->ptr;
  379     compressed_data += sizeof(struct CtdlCompressHeader);
  380 
  381     sourceLen = (uLongf) zheader.compressed_len;
  382     destLen = (uLongf) zheader.uncompressed_len;
  383     uncompressed_data = malloc(zheader.uncompressed_len);
  384 
  385     if (uncompress((Bytef *) uncompressed_data,
  386                (uLongf *) & destLen, (const Bytef *) compressed_data, (uLong) sourceLen) != Z_OK) {
  387         syslog(LOG_ERR, "db: uncompress() error");
  388         cdb_abort();
  389     }
  390 
  391     free(cdb->ptr);
  392     cdb->len = (size_t) destLen;
  393     cdb->ptr = uncompressed_data;
  394 }
  395 
  396 
  397 /*
  398  * Store a piece of data.  Returns 0 if the operation was successful.  If a
  399  * key already exists it should be overwritten.
  400  */
  401 int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) {
  402 
  403     DBT dkey, ddata;
  404     DB_TXN *tid = NULL;
  405     int ret = 0;
  406     struct CtdlCompressHeader zheader;
  407     char *compressed_data = NULL;
  408     int compressing = 0;
  409     size_t buffer_len = 0;
  410     uLongf destLen = 0;
  411 
  412     memset(&dkey, 0, sizeof(DBT));
  413     memset(&ddata, 0, sizeof(DBT));
  414     dkey.size = ckeylen;
  415     dkey.data = (void *) ckey;
  416     ddata.size = cdatalen;
  417     ddata.data = cdata;
  418 
  419     /* Only compress Visit and UseTable records.  Everything else is uncompressed. */
  420     if ((cdb == CDB_VISIT) || (cdb == CDB_USETABLE)) {
  421         compressing = 1;
  422         zheader.magic = COMPRESS_MAGIC;
  423         zheader.uncompressed_len = cdatalen;
  424         buffer_len = ((cdatalen * 101) / 100) + 100 + sizeof(struct CtdlCompressHeader);
  425         destLen = (uLongf) buffer_len;
  426         compressed_data = malloc(buffer_len);
  427         if (compress2((Bytef *) (compressed_data + sizeof(struct CtdlCompressHeader)),
  428                   &destLen, (Bytef *) cdata, (uLongf) cdatalen, 1) != Z_OK) {
  429             syslog(LOG_ERR, "db: compress2() error");
  430             cdb_abort();
  431         }
  432         zheader.compressed_len = (size_t) destLen;
  433         memcpy(compressed_data, &zheader, sizeof(struct CtdlCompressHeader));
  434         ddata.size = (size_t) (sizeof(struct CtdlCompressHeader) + zheader.compressed_len);
  435         ddata.data = compressed_data;
  436     }
  437 
  438     if (TSD->tid != NULL) {
  439         ret = dbp[cdb]->put(dbp[cdb],   // db
  440                     TSD->tid,   // transaction ID
  441                     &dkey,  // key
  442                     &ddata, // data
  443                     0       // flags
  444         );
  445         if (ret) {
  446             syslog(LOG_ERR, "db: cdb_store(%d): %s", cdb, db_strerror(ret));
  447             cdb_abort();
  448         }
  449         if (compressing) {
  450             free(compressed_data);
  451         }
  452         return ret;
  453     } else {
  454         bailIfCursor(TSD->cursors, "attempt to write during r/o cursor");
  455 
  456           retry:
  457         txbegin(&tid);
  458 
  459         if ((ret = dbp[cdb]->put(dbp[cdb],  // db
  460                      tid,       // transaction ID
  461                      &dkey,     // key
  462                      &ddata,    // data
  463                      0))) {     // flags
  464             if (ret == DB_LOCK_DEADLOCK) {
  465                 txabort(tid);
  466                 goto retry;
  467             } else {
  468                 syslog(LOG_ERR, "db: cdb_store(%d): %s", cdb, db_strerror(ret));
  469                 cdb_abort();
  470             }
  471         } else {
  472             txcommit(tid);
  473             if (compressing) {
  474                 free(compressed_data);
  475             }
  476             return ret;
  477         }
  478     }
  479     return ret;
  480 }
  481 
  482 
  483 /*
  484  * Delete a piece of data.  Returns 0 if the operation was successful.
  485  */
  486 int cdb_delete(int cdb, void *key, int keylen) {
  487     DBT dkey;
  488     DB_TXN *tid;
  489     int ret;
  490 
  491     memset(&dkey, 0, sizeof dkey);
  492     dkey.size = keylen;
  493     dkey.data = key;
  494 
  495     if (TSD->tid != NULL) {
  496         ret = dbp[cdb]->del(dbp[cdb], TSD->tid, &dkey, 0);
  497         if (ret) {
  498             syslog(LOG_ERR, "db: cdb_delete(%d): %s", cdb, db_strerror(ret));
  499             if (ret != DB_NOTFOUND) {
  500                 cdb_abort();
  501             }
  502         }
  503     } else {
  504         bailIfCursor(TSD->cursors, "attempt to delete during r/o cursor");
  505 
  506           retry:
  507         txbegin(&tid);
  508 
  509         if ((ret = dbp[cdb]->del(dbp[cdb], tid, &dkey, 0)) && ret != DB_NOTFOUND) {
  510             if (ret == DB_LOCK_DEADLOCK) {
  511                 txabort(tid);
  512                 goto retry;
  513             } else {
  514                 syslog(LOG_ERR, "db: cdb_delete(%d): %s", cdb, db_strerror(ret));
  515                 cdb_abort();
  516             }
  517         } else {
  518             txcommit(tid);
  519         }
  520     }
  521     return ret;
  522 }
  523 
  524 
  525 static DBC *localcursor(int cdb) {
  526     int ret;
  527     DBC *curs;
  528 
  529     if (TSD->cursors[cdb] == NULL) {
  530         ret = dbp[cdb]->cursor(dbp[cdb], TSD->tid, &curs, 0);
  531     }
  532     else {
  533         ret = TSD->cursors[cdb]->c_dup(TSD->cursors[cdb], &curs, DB_POSITION);
  534     }
  535 
  536     if (ret) {
  537         syslog(LOG_ERR, "db: localcursor: %s", db_strerror(ret));
  538         cdb_abort();
  539     }
  540 
  541     return curs;
  542 }
  543 
  544 
  545 /*
  546  * Fetch a piece of data.  If not found, returns NULL.  Otherwise, it returns
  547  * a struct cdbdata which it is the caller's responsibility to free later on
  548  * using the cdb_free() routine.
  549  */
  550 struct cdbdata *cdb_fetch(int cdb, const void *key, int keylen) {
  551     struct cdbdata *tempcdb;
  552     DBT dkey, dret;
  553     int ret;
  554 
  555     memset(&dkey, 0, sizeof(DBT));
  556     dkey.size = keylen;
  557     dkey.data = (void *) key;
  558 
  559     if (TSD->tid != NULL) {
  560         memset(&dret, 0, sizeof(DBT));
  561         dret.flags = DB_DBT_MALLOC;
  562         ret = dbp[cdb]->get(dbp[cdb], TSD->tid, &dkey, &dret, 0);       // crashing here
  563     } else {
  564         DBC *curs;
  565 
  566         do {
  567             memset(&dret, 0, sizeof(DBT));
  568             dret.flags = DB_DBT_MALLOC;
  569             curs = localcursor(cdb);
  570             ret = curs->c_get(curs, &dkey, &dret, DB_SET);
  571             cclose(curs);
  572         }
  573         while (ret == DB_LOCK_DEADLOCK);
  574     }
  575 
  576     if ((ret != 0) && (ret != DB_NOTFOUND)) {
  577         syslog(LOG_ERR, "db: cdb_fetch(%d): %s", cdb, db_strerror(ret));
  578         cdb_abort();
  579     }
  580 
  581     if (ret != 0) {
  582         return NULL;
  583     }
  584 
  585     tempcdb = (struct cdbdata *) malloc(sizeof(struct cdbdata));
  586 
  587     if (tempcdb == NULL) {
  588         syslog(LOG_ERR, "db: cdb_fetch: Cannot allocate memory for tempcdb");
  589         cdb_abort();
  590         return NULL;    /* make it easier for static analysis... */
  591     } else {
  592         tempcdb->len = dret.size;
  593         tempcdb->ptr = dret.data;
  594         cdb_decompress_if_necessary(tempcdb);
  595         return (tempcdb);
  596     }
  597 }
  598 
  599 
  600 /*
  601  * Free a cdbdata item.
  602  *
  603  * Note that we only free the 'ptr' portion if it is not NULL.  This allows
  604  * other code to assume ownership of that memory simply by storing the
  605  * pointer elsewhere and then setting 'ptr' to NULL.  cdb_free() will then
  606  * avoid freeing it.
  607  */
  608 void cdb_free(struct cdbdata *cdb) {
  609     if (cdb->ptr) {
  610         free(cdb->ptr);
  611     }
  612     free(cdb);
  613 }
  614 
  615 
  616 void cdb_close_cursor(int cdb) {
  617     if (TSD->cursors[cdb] != NULL) {
  618         cclose(TSD->cursors[cdb]);
  619     }
  620 
  621     TSD->cursors[cdb] = NULL;
  622 }
  623 
  624 
  625 /* 
  626  * Prepare for a sequential search of an entire database.
  627  * (There is guaranteed to be no more than one traversal in
  628  * progress per thread at any given time.)
  629  */
  630 void cdb_rewind(int cdb) {
  631     int ret = 0;
  632 
  633     if (TSD->cursors[cdb] != NULL) {
  634         syslog(LOG_ERR, "db: cdb_rewind: must close cursor on database %d before reopening", cdb);
  635         cdb_abort();
  636         /* cclose(TSD->cursors[cdb]); */
  637     }
  638 
  639     /*
  640      * Now initialize the cursor
  641      */
  642     ret = dbp[cdb]->cursor(dbp[cdb], TSD->tid, &TSD->cursors[cdb], 0);
  643     if (ret) {
  644         syslog(LOG_ERR, "db: cdb_rewind: db_cursor: %s", db_strerror(ret));
  645         cdb_abort();
  646     }
  647 }
  648 
  649 
  650 /*
  651  * Fetch the next item in a sequential search.  Returns a pointer to a 
  652  * cdbdata structure, or NULL if we've hit the end.
  653  */
  654 struct cdbdata *cdb_next_item(int cdb) {
  655     DBT key, data;
  656     struct cdbdata *cdbret;
  657     int ret = 0;
  658 
  659     /* Initialize the key/data pair so the flags aren't set. */
  660     memset(&key, 0, sizeof(key));
  661     memset(&data, 0, sizeof(data));
  662     data.flags = DB_DBT_MALLOC;
  663 
  664     ret = TSD->cursors[cdb]->c_get(TSD->cursors[cdb], &key, &data, DB_NEXT);
  665 
  666     if (ret) {
  667         if (ret != DB_NOTFOUND) {
  668             syslog(LOG_ERR, "db: cdb_next_item(%d): %s", cdb, db_strerror(ret));
  669             cdb_abort();
  670         }
  671         cdb_close_cursor(cdb);
  672         return NULL;    /* presumably, end of file */
  673     }
  674 
  675     cdbret = (struct cdbdata *) malloc(sizeof(struct cdbdata));
  676     cdbret->len = data.size;
  677     cdbret->ptr = data.data;
  678     cdb_decompress_if_necessary(cdbret);
  679 
  680     return (cdbret);
  681 }
  682 
  683 
  684 /*
  685  * Transaction-based stuff.  I'm writing this as I bake cookies...
  686  */
  687 void cdb_begin_transaction(void) {
  688     bailIfCursor(TSD->cursors, "can't begin transaction during r/o cursor");
  689 
  690     if (TSD->tid != NULL) {
  691         syslog(LOG_ERR, "db: cdb_begin_transaction: ERROR: nested transaction");
  692         cdb_abort();
  693     }
  694 
  695     txbegin(&TSD->tid);
  696 }
  697 
  698 
  699 void cdb_end_transaction(void) {
  700     int i;
  701 
  702     for (i = 0; i < MAXCDB; i++)
  703         if (TSD->cursors[i] != NULL) {
  704             syslog(LOG_WARNING, "db: cdb_end_transaction: WARNING: cursor %d still open at transaction end", i);
  705             cclose(TSD->cursors[i]);
  706             TSD->cursors[i] = NULL;
  707         }
  708 
  709     if (TSD->tid == NULL) {
  710         syslog(LOG_ERR, "db: cdb_end_transaction: ERROR: txcommit(NULL) !!");
  711         cdb_abort();
  712     } else {
  713         txcommit(TSD->tid);
  714     }
  715 
  716     TSD->tid = NULL;
  717 }
  718 
  719 
  720 /*
  721  * Truncate (delete every record)
  722  */
  723 void cdb_trunc(int cdb) {
  724     /* DB_TXN *tid; */
  725     int ret;
  726     u_int32_t count;
  727 
  728     if (TSD->tid != NULL) {
  729         syslog(LOG_ERR, "db: cdb_trunc must not be called in a transaction.");
  730         cdb_abort();
  731     }
  732     else {
  733         bailIfCursor(TSD->cursors, "attempt to write during r/o cursor");
  734 
  735           retry:
  736         /* txbegin(&tid); */
  737 
  738         if ((ret = dbp[cdb]->truncate(dbp[cdb], /* db */
  739                           NULL, /* transaction ID */
  740                           &count,   /* #rows deleted */
  741                           0))) {    /* flags */
  742             if (ret == DB_LOCK_DEADLOCK) {
  743                 /* txabort(tid); */
  744                 goto retry;
  745             } else {
  746                 syslog(LOG_ERR, "db: cdb_truncate(%d): %s", cdb, db_strerror(ret));
  747                 if (ret == ENOMEM) {
  748                     syslog(LOG_ERR, "db: You may need to tune your database; please read http://www.citadel.org/doku.php?id=faq:troubleshooting:out_of_lock_entries for more information.");
  749                 }
  750                 exit(CTDLEXIT_DB);
  751             }
  752         } else {
  753             /* txcommit(tid); */
  754         }
  755     }
  756 }
  757 
  758 
  759 /*
  760  * compact (defragment) the database , possibly returning space back to the underlying filesystem
  761  */
  762 void cdb_compact(void) {
  763     int ret;
  764     int i;
  765 
  766     syslog(LOG_DEBUG, "db: cdb_compact() started");
  767     for (i = 0; i < MAXCDB; i++) {
  768         syslog(LOG_DEBUG, "db: compacting database %d", i);
  769         ret = dbp[i]->compact(dbp[i], NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL);
  770         if (ret) {
  771             syslog(LOG_ERR, "db: compact: %s", db_strerror(ret));
  772         }
  773     }
  774     syslog(LOG_DEBUG, "db: cdb_compact() finished");
  775 }
  776 
  777 
  778 // Has an item already been seen (is it in the CDB_USETABLE) ?
  779 // Returns 0 if it hasn't, 1 if it has
  780 // In either case, writes the item to the database for next time.
  781 int CheckIfAlreadySeen(StrBuf *guid) {
  782     int found = 0;
  783     struct UseTable ut;
  784     struct cdbdata *cdbut;
  785 
  786     syslog(LOG_DEBUG, "db: CheckIfAlreadySeen(%s)", ChrPtr(guid));
  787     cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid));
  788     if (cdbut != NULL) {
  789         found = 1;
  790         cdb_free(cdbut);
  791     }
  792 
  793     /* (Re)write the record, to update the timestamp.  Zeroing it out makes it compress better. */
  794     memset(&ut, 0, sizeof(struct UseTable));
  795     memcpy(ut.ut_msgid, SKEY(guid));
  796     ut.ut_timestamp = time(NULL);
  797     cdb_store(CDB_USETABLE, SKEY(guid), &ut, sizeof(struct UseTable));
  798     return (found);
  799 }
  800 
  801 
  802 CTDL_MODULE_INIT(database)
  803 {
  804     if (!threading) {
  805         // nothing to do here
  806     }
  807 
  808     /* return our module id for the log */
  809     return "database";
  810 }