Beispiel #1
0
int
jack_get_properties (jack_uuid_t subject,
                     jack_description_t* desc)
{
        DBT key;
        DBT data;
        DBC* cursor;
        int ret;
        size_t len1, len2;
        size_t cnt = 0;
        char ustr[JACK_UUID_STRING_SIZE];
        size_t props_size = 0;
        jack_property_t* prop;

        desc->properties = NULL;
        desc->property_cnt = 0;

        jack_uuid_unparse (subject, ustr);

        if (jack_property_init (NULL)) {
                return -1;
        }


        if ((ret = db->cursor (db, NULL, &cursor, 0)) != 0) {
                jack_error ("Cannot create cursor for metadata search (%s)", db_strerror (ret));
                return -1;
        }

        memset(&key, 0, sizeof(key));
	memset(&data, 0, sizeof(data));
        data.flags = DB_DBT_MALLOC;

        while ((ret = cursor->get(cursor, &key, &data, DB_NEXT)) == 0) {

                /* require 2 extra chars (data+null) for key,
                   which is composed of UUID str plus a key name
                */

                if (key.size < JACK_UUID_STRING_SIZE + 2) {
                        if (data.size > 0) {
                                free (data.data);
                        }
                        continue;
                }

                if (memcmp (ustr, key.data, JACK_UUID_STRING_SIZE) != 0) {
                        /* not relevant */
                        if (data.size > 0) {
                                free (data.data);
                        }
                        continue;
                }

                /* result must have at least 2 chars plus 2 nulls to be valid 
                 */
                
                if (data.size < 4) {
                        if (data.size > 0) {
                                free (data.data);
                        }
                        continue;
                }
 
                /* realloc array if necessary */
       
                if (cnt == props_size) {
                        if (props_size == 0) {
                                props_size = 8; /* a rough guess at a likely upper bound for the number of properties */
                        } else {
                                props_size *= 2;
                        }
                        
                        desc->properties = (jack_property_t*) realloc (desc->properties, sizeof (jack_property_t) * props_size);
                }

                prop = &desc->properties[cnt];

                /* store UUID/subject */

                jack_uuid_copy (desc->subject, subject);
                
                /* copy key (without leading UUID as subject */

                len1 = key.size - JACK_UUID_STRING_SIZE;
                prop->key = malloc (len1);
                memcpy ((char*) prop->key, key.data + JACK_UUID_STRING_SIZE, len1);
                
                /* copy data (which contains 1 or 2 null terminated strings, the value
                   and optionally a MIME type.
                 */

                len1 = strlen (data.data) + 1;
                prop->data = (char *) malloc (len1);
                memcpy ((char*) prop->data, data.data, len1);
                
                if (len1 < data.size) {
                        len2 = strlen (data.data+len1) + 1;
                        
                        prop->type= (char *) malloc (len2);
                        memcpy ((char*) prop->type, data.data+len1, len2);
                } else {
                        /* no type specified, assume default */
                        prop->type = NULL;
                }
                
                if (data.size) {
                        free (data.data);
                }

                ++cnt;
        }
        
        cursor->close (cursor);
        desc->property_cnt = cnt;

        return cnt;
}
Beispiel #2
0
int
jack_get_all_properties (jack_description_t** descriptions)
{
        DBT key;
        DBT data;
        DBC* cursor;
        int ret;
        size_t dcnt = 0;
        size_t dsize = 0;
        size_t n = 0;
        jack_description_t* desc = NULL;
        jack_uuid_t uuid;
        jack_description_t* current_desc = NULL;
        jack_property_t* current_prop = NULL;
        size_t len1, len2;

        if (jack_property_init (NULL)) {
                return -1;
        }

        if ((ret = db->cursor (db, NULL, &cursor, 0)) != 0) {
                jack_error ("Cannot create cursor for metadata search (%s)", db_strerror (ret));
                return -1;
        }

        memset(&key, 0, sizeof(key));
	memset(&data, 0, sizeof(data));
        data.flags = DB_DBT_MALLOC;

        dsize = 8; /* initial guess at number of descriptions we need */
        dcnt = 0;
        desc = (jack_description_t*) malloc (dsize * sizeof (jack_description_t));

        while ((ret = cursor->get(cursor, &key, &data, DB_NEXT)) == 0) {

                /* require 2 extra chars (data+null) for key,
                   which is composed of UUID str plus a key name
                */

                if (key.size < JACK_UUID_STRING_SIZE + 2) {
                        if (data.size > 0) {
                                free (data.data);
                        }
                        continue;
                }

                if (jack_uuid_parse (key.data, uuid) != 0) {
                        continue;
                }
                
                /* do we have an existing description for this UUID */

                for (n = 0; n < dcnt ; ++n) {
                        if (jack_uuid_compare (uuid, desc[n].subject) == 0) {
                                break;
                        }
                }
                
                if (n == dcnt) {
                        /* we do not have an existing description, so grow the array */

                        if (dcnt == dsize) {
                                dsize *= 2;
                                desc = (jack_description_t*) realloc (desc, sizeof (jack_description_t) * dsize);
                        } 

                        /* initialize */

                        desc[n].property_size = 0;
                        desc[n].property_cnt = 0;
                        desc[n].properties = NULL;
                        
                        /* set up UUID */

                        jack_uuid_copy (desc[n].subject, uuid);
                        dcnt++;
                }

                current_desc = &desc[n];

                /* see if there is room for the new property or if we need to realloc 
                 */

                if (current_desc->property_cnt == current_desc->property_size) {
                        if (current_desc->property_size == 0) {
                                current_desc->property_size = 8;
                        } else {
                                current_desc->property_size *= 2;
                        }

                        current_desc->properties = (jack_property_t*) realloc (current_desc->properties, sizeof (jack_property_t) * current_desc->property_size);
                }
                
                current_prop = &current_desc->properties[current_desc->property_cnt++];
                
                /* copy key (without leading UUID) */

                len1 = key.size - JACK_UUID_STRING_SIZE;
                current_prop->key = malloc (len1);
                memcpy ((char*) current_prop->key, key.data + JACK_UUID_STRING_SIZE, len1);
                
                /* copy data (which contains 1 or 2 null terminated strings, the value
                   and optionally a MIME type.
                 */

                len1 = strlen (data.data) + 1;
                current_prop->data = (char *) malloc (len1);
                memcpy ((char*) current_prop->data, data.data, len1);
                
                if (len1 < data.size) {
                        len2 = strlen (data.data+len1) + 1;
                        
                        current_prop->type= (char *) malloc (len2);
                        memcpy ((char*) current_prop->type, data.data+len1, len2);
                } else {
                        /* no type specified, assume default */
                        current_prop->type = NULL;
                }
                
                if (data.size) {
                        free (data.data);
                }
        }
        
        cursor->close (cursor);

        (*descriptions) = desc;

        return dcnt;
}
Beispiel #3
0
int input_load(DB *db, sqlite3* sql_db, char* sql_table){
    int ret;
    struct sqlite_workspace w;
    DB *sdb;
    DBC *cursor;
    DBT pkey;
    memset(&pkey, 0, sizeof(pkey));
    sqlite3_stmt *ppStmt;
    DbRecord recordp;
    DB_BTREE_STAT *stat;
    db_recno_t count=0;
    w.count = &count;
    db_recno_t max = 0;
    char big_block[128];

    if ((ret = db->cursor(db, NULL, &cursor, 0)) != 0) {
		dbenv->err(dbenv, ret, "DB->cursor");
		return (1);
	}
	memset(&(w.key), 0, sizeof(w.key));
	memset(&(w.data), 0, sizeof(w.data));
    
    if(DB_NOTFOUND ==
      (ret = cursor->get(cursor, &(w.key), &(w.data), DB_LAST)))
        w.primary_key = 0;
    else{
        w.primary_key = *(u_int32_t*)(w.data.data);
        w.primary_key++;
    }

    ret = cursor->close(cursor);

    sqlite3_prepare_v2(sql_db, sql_query, -1, &ppStmt, NULL);
    while(SQLITE_ROW == (ret=sqlite3_step(ppStmt))){
        memcpy(&recordp, &DbRecord_base, sizeof(DbRecord));
        build_record(ppStmt, &recordp, &w);
        DbRecord_write(&recordp, db, &w);
        if((count++ % 100000)==0){
            printf("%lu records processed...\n", (ulong)count);
            db->sync(db, 0);
        }
    }
    count = 0;
    sqlite3_finalize(ppStmt);
    

    db->cursor(db, NULL, &cursor, 0);
    cursor->get(cursor, &(w.key), &(w.data), DB_LAST);
    DbRecord_dump(w.data.data);

    db->stat(db, NULL, &stat, 0);
    printf("primary nkeys: %lu\n", (u_long)(stat->bt_nkeys));
    free(stat);

    sqlite_db_secondary_open(db, &sdb, "block_idx", 8*1024, DB_DUPSORT, blocking_callback, compare_uint32);
    sdb->stat(sdb, NULL, &stat, 0);
    printf("block_idx keys: %lu\n", (u_long)(stat->bt_nkeys));
    free(stat);
    sdb->cursor(sdb, NULL, &cursor, 0);
    
    while(DB_NOTFOUND != cursor->pget(cursor, &(w.key), &pkey, &(w.data), DB_NEXT)){
        cursor->count(cursor, &count, 0);
        if (count > max){
            max = count;
            memcpy(big_block, w.key.data, (size_t)w.key.size);
            big_block[w.key.size] = '\0';
        }
    }
    cursor->close(cursor);
    printf("Biggest block: %s\n", big_block);
    printf("%u records.\n", (size_t)max);
    
    count_blocks(sdb);
    sdb->close(sdb, 0);
    

    sqlite_db_secondary_open(db, &sdb, "idx", 8*1024, 0, index_callback, NULL);
    ret = sdb->stat(sdb, NULL, &stat, 0);
    printf("%d\n", ret);
    printf("idx_keys: %lu\n", (u_long)(stat->bt_nkeys));
    free(stat);
    sdb->close(sdb, 0);
    
    return(0);
}
Beispiel #4
0
nfsstat4 nfs_op_readdir(struct nfs_cxn *cxn, const READDIR4args *args,
		        struct list_head *writes, struct rpc_write **wr)
{
	nfsstat4 status = NFS4_OK;
	struct nfs_inode *ino = NULL;
	uint32_t dircount, maxcount, *status_p;
	struct readdir_info ri;
	uint64_t cookie, attr_request;
	const verifier4 *cookie_verf;
	DB_TXN *txn = NULL;
	DB *dirent = srv.fsdb.dirent;
	DB_ENV *dbenv = srv.fsdb.env;
	DBT pkey, pval;
	struct fsdb_de_key key;
	int cget_flags;
	DBC *curs = NULL;
	int rc;
	uint64_t dirent_inum, db_de;
	struct fsdb_de_key *rkey;

	cookie = args->cookie;
	cookie_verf = &args->cookieverf;
	dircount = args->dircount;
	maxcount = args->maxcount;
	attr_request = bitmap4_decode(&args->attr_request);

	status_p = WRSKIP(4);

	if (debugging) {
		applog(LOG_INFO, "op READDIR (COOKIE:%Lu DIR:%u MAX:%u MAP:%Lx)",
		       (unsigned long long) cookie,
		       dircount,
		       maxcount,
		       (unsigned long long) attr_request);

		print_fattr_bitmap("op READDIR", attr_request);
	}

	/* traditionally "." and "..", hardcoded */
	if (cookie == 1 || cookie == 2) {
		status = NFS4ERR_BAD_COOKIE;
		goto out;
	}
	/* don't permit request of write-only attrib */
	if (attr_request & fattr_write_only_mask) {
		status = NFS4ERR_INVAL;
		goto out;
	}

	/* FIXME: very, very, very poor verifier */
	if (cookie &&
	    memcmp(cookie_verf, &srv.instance_verf, sizeof(verifier4))) {
		status = NFS4ERR_NOT_SAME;
		goto out;
	}

	/* read inode of directory being read */
	status = dir_curfh(NULL, cxn, &ino, 0);
	if (status != NFS4_OK)
		goto out;
	if (ino->mode == 0) {
		status = NFS4ERR_ACCESS;
		goto out;
	}

	/* subtract READDIR4resok header and footer size */
	if (maxcount < 16) {
		status = NFS4ERR_TOOSMALL;
		goto out;
	}

	maxcount -= (8 + 4 + 4);

	/* verify within server limits */
	if (dircount > SRV_MAX_READ || maxcount > SRV_MAX_READ) {
		status = NFS4ERR_INVAL;
		goto out;
	}

	/* open transaction */
	rc = dbenv->txn_begin(dbenv, NULL, &txn, 0);
	if (rc) {
		status = NFS4ERR_IO;
		dbenv->err(dbenv, rc, "DB_ENV->txn_begin");
		goto out;
	}

	/* set up directory iteration */
	memset(&ri, 0, sizeof(ri));
	ri.cookie = cookie;
	ri.dircount = dircount;
	ri.maxcount = maxcount;
	ri.attr_request = attr_request;
	ri.status = NFS4_OK;
	ri.writes = writes;
	ri.wr = wr;
	ri.dir_pos = 3;
	ri.first_time = true;

	/* if dir is empty, skip directory interation loop completely */
	if (dir_is_empty(txn, ino)) {
		WRMEM(&srv.instance_verf, sizeof(verifier4));	/* cookieverf */

		ri.val_follows = WRSKIP(4);

		if (debugging)
			applog(LOG_DEBUG, "   READDIR: empty directory");

		goto the_finale;
	}

	/* otherwise, loop through each dirent attached to ino->inum */
	rc = dirent->cursor(dirent, txn, &curs, 0);
	if (rc) {
		status = NFS4ERR_IO;
		dirent->err(dirent, rc, "dirent->cursor");
		goto out_abort;
	}

	key.inum = inum_encode(ino->inum);

	memset(&pkey, 0, sizeof(pkey));
	pkey.data = &key;
	pkey.size = sizeof(key);
	pkey.flags = DB_DBT_MALLOC;

	memset(&pval, 0, sizeof(pval));
	pval.data = &db_de;
	pval.ulen = sizeof(db_de);
	pval.flags = DB_DBT_USERMEM;

	cget_flags = DB_SET_RANGE;
	while (1) {
		bool iter_rc;

		rc = curs->get(curs, &pkey, &pval, cget_flags);
		if (rc) {
			if (rc != DB_NOTFOUND)
				dirent->err(dirent, rc, "readdir curs->get");
			break;
		}

		cget_flags = DB_NEXT;

		rkey = pkey.data;
		if (inum_decode(rkey->inum) != ino->inum) {
			free(rkey);
			break;
		}

		dirent_inum = inum_decode(db_de);

		iter_rc = readdir_iter(txn, rkey, pkey.size, dirent_inum, &ri);
		free(rkey);

		if (iter_rc)
			break;
	}

	if (!ri.n_results) {
		if (debugging)
			applog(LOG_INFO, "           zero results, status %s",
			       ri.status <= NFS4ERR_CB_PATH_DOWN ?
			       		status2str(ri.status) : "n/a");

		if (ri.status == NFS4_OK) {
			WRMEM(&srv.instance_verf, sizeof(verifier4));	/* cookieverf */

			ri.val_follows = WRSKIP(4);
		}
	}

	rc = curs->close(curs);
	if (rc) {
		status = NFS4ERR_IO;
		dirent->err(dirent, rc, "dirent->cursor close");
		goto out_abort;
	}

the_finale:

	/* terminate final entry4.nextentry and dirlist4.entries */
	if (ri.val_follows)
		*ri.val_follows = htonl(0);

	if (ri.cookie_found && !ri.n_results && ri.hit_limit) {
		status = NFS4ERR_TOOSMALL;
		goto out_abort;
	}

	/* close transaction */
	rc = txn->commit(txn, 0);
	if (rc) {
		dbenv->err(dbenv, rc, "DB_ENV->txn_commit");
		status = NFS4ERR_IO;
		goto out;
	}

	WR32(ri.hit_limit ? 0 : 1);		/* reply eof */

out:
	*status_p = htonl(status);
	inode_free(ino);
	return status;

out_abort:
	if (txn->abort(txn))
		dbenv->err(dbenv, rc, "DB_ENV->txn_abort");
	goto out;
}
Beispiel #5
0
/* Internal function to add or remove key/value pairs.  When rm is zero,
 * the entry will be added; otherwise, it will be removed.  Even when
 * adding, a matching record will be removed.
 *
 * The part of the database record to match is found by trimming both the
 * provided value and the database value to at most trimlen bytes and
 * comparing what remains.  Note that this comparison must also match
 * in size, so trimming to any size longer than the provided value will
 * cause a mismatch if the database value is larger.  Negative values
 * of trimlen are treated as infinite, and will thus take full records
 * into account, including their lengths.
 *
 * Another comparison trick is the inclusion of the trailing NUL char
 * at the end of a PKCS #11 URI or validation expression string.
 *
 * The array mask4 can be used to indicate flags for the first four bytes,
 * which is a big-endian flag value in localid and trust databases.
 * When NULL is provided, it will match everything.
 *
 * The function returns 1 on succes, 0 on failure.  Note that it is not
 * an error if a record to remove had already gone, or when a matching
 * record had to be removed before adding one.  This is considered to
 * be the result of manual overrides.  Failure of this function solely
 * refers to issues of a technical nature, such as I/O problems or
 * running out of memory.
 *
 * All operations on the database are excuted within the current
 * transaction; the call from api.c has ensured that one exists.
 */
static int update_db (struct pulleyback_tlspool *self,
				dercursor *key, dercursor *value,
				int trimlen, const uint8_t *mask4,
				int rm) {
	int ok = 1;
	int gotcrs = 0;
	int nomore = 1;
	DBC *crs;
	DBT db_key;
	DBT db_val;
	DBT db_got;
	uint8_t my_mask4 [] = { 0xff, 0xff, 0xff, 0xff };
	if (mask4 == NULL) {
		// When no mask4 provided, match everything
		mask4 = my_mask4;
	}
	if (trimlen < 0) {
		// Standardise on positive values that act as "infinite"
		trimlen = value->derlen + 1;
	}
	gotcrs =
	ok = ok && (0 == self->db->cursor (self->db, self->txn, &crs, 0));
	memset (&db_key, 0, sizeof (db_key));
	db_key.data = key->derptr;
	db_key.size = key->derlen;
	memset (&db_val, 0, sizeof (db_val));
	db_val.data = value->derptr;
	db_val.size = value->derlen;
	memset (&db_got, 0, sizeof (db_got));
	//OLD// dbt_init_fixbuf (&db_key, key  ->derptr, key  ->derlen);
	//OLD// dbt_init_fixbuf (&db_val, value->derptr, value->derlen);
	//OLD// dbt_init_empty  (&db_got);
	nomore = crs->get (crs, &db_key, &db_got, DB_SET);
	while (!nomore) {
		int match = 1;
		int i;
		for (i = 0; match && (i < trimlen); i++) {
			match = match && (i < db_val.size);
			match = match && (i < db_got.size);
			if (!match) {
				// Final decision; only match for same sizes
				match = (db_val.size == db_got.size);
				break;
			} else {
				uint8_t m, a, b;
				m = (i < 4)? mask4 [i]: 0xff;
				a = m & ((uint8_t *) db_val.data) [i];
				b = m & ((uint8_t *) db_got.data) [i];
				match = (a == b);
			}
		}
		if (match) {
			crs->del (crs, 0);
		}
		nomore = crs->get (crs, &db_key, &db_got, DB_NEXT_DUP);
	}
	ok = ok && (nomore == DB_NOTFOUND);
	if (gotcrs) {
		if (0 != crs->close (crs)) {
			fprintf (stderr, "Failed to close cursor\n");
		}
	}
	if (!rm) {
		ok = ok && (0 == self->db->put (
		                 self->db, self->txn, &db_key, &db_val, 0));
	}
	//OLD// // Not ours, so don't free // dbt_free (&db_got);
	//OLD// // Static,   so don't free // dbt_free (&db_val);
	//OLD// // Static,   so don't free // dbt_free (&db_key);
	return ok;
}
Beispiel #6
0
bool BulkLoad::bulkLoadBody()
{
	if (m_cancel)
	{
		emit updateJob(JobList::BulkLoad, JobList::Cancelled, job->seq);
		return false;
	}

	emit updateJob(JobList::BulkLoad, JobList::Running, job->seq);

	NewsGroup*  ng = job->ng;
	HeaderList* hl = job->headerList;

	mphDeletionsList = 0;
	sphDeletionsList = 0;

    DBC *dbcp = 0;

    MultiPartHeader mph;
    SinglePartHeader sph;
    HeaderBase* hb = 0;

	memset(&ckey, 0, sizeof(ckey));
	memset(&cdata, 0, sizeof(cdata));

    cdata.data = (void *) dataBuffer;
    cdata.ulen = HEADER_BULK_BUFFER_LENGTH;
    cdata.flags = DB_DBT_USERMEM;

    ckey.data = (void *) keyBuffer;
    ckey.ulen = HEADER_BULK_BUFFER_LENGTH;
    ckey.flags = DB_DBT_USERMEM;

    size_t retklen, retdlen;
    void *retkey = 0, *retdata = 0;
    int ret, t_ret;
    void *p = 0;

    quint64 count=0;

	QTime start = QTime::currentTime();
    qDebug() << "Loading started: " << start.toString();

    /* Acquire a cursor for the database. */
    if ((ret = ng->getDb()->get_DB()->cursor(ng->getDb()->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0)
    {
    	ng->getDb()->err(ret, "DB->cursor");
        return false;
    }

    quint32 numIgnored = 0;
    bool    mphFound = false;

	for (;;)
	{
		/*
		 * Acquire the next set of key/data pairs.  This code
		 * does not handle single key/data pairs that won't fit
		 * in a BUFFER_LENGTH size buffer, instead returning
		 * DB_BUFFER_SMALL to our caller.
		 */
		if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0)
		{
			if (ret != DB_NOTFOUND)
				ng->getDb()->err(ret, "DBcursor->get");
			break;
		}

		for (DB_MULTIPLE_INIT(p, &cdata);;)
		{
			DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen);
			if (p == NULL)
				break;

            if (retdlen){;} // MD TODO compiler .... unused variable

			if (*((char *)retdata) == 'm')
			{                
                MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph);
                hb = (HeaderBase*)&mph;

                mphFound = true;
			}
            else if (*((char *)retdata) == 's')
            {
                SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph);
                hb = (HeaderBase*)&sph;

                mphFound = false;
                // qDebug() << "Single index = " << sph.getIndex();
            }
			else
			{
				// What have we found ?????
				qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata);
				continue;
			}

            if (hb->getStatus() & HeaderBase::MarkedForDeletion)
			{
				// ignore this header, garbage collection will remove it ...
				++numIgnored;
				++count; // ... but still count it as bulk delete will reduce the count

				if (numIgnored == 1)
				{
					// These two will be freed by bulk delete
					mphDeletionsList = new QList<QString>;
					sphDeletionsList = new QList<QString>;
				}

				if (mphFound)
					mphDeletionsList->append(hb->getIndex());
				else
					sphDeletionsList->append(hb->getIndex());

				continue;
			}

            if (m_cancel)
                break;

            hl->headerTreeModel->setupTopLevelItem(hb);

			++count;

            if (count % 50 == 0)
            {
                QString labelText = tr("Loaded ") + QString("%L1").arg(count) +
                        " of " + QString("%L1").arg(ng->getTotal()) +
                        " articles.";
                emit progress(job->seq, count, labelText);
                QCoreApplication::processEvents();
            }
		}

		if (m_cancel)
			break;
	}

	if ((t_ret = dbcp->close(dbcp)) != 0)
	{
		ng->getDb()->err(ret, "DBcursor->close");
		if (ret == 0)
			ret = t_ret;
	}

    emit loadReady(job->seq);

    QString labelText = tr("Loaded ") + QString("%L1").arg(count) +
            " of " + QString("%L1").arg(ng->getTotal()) +
            " articles.";
    emit progress(job->seq, count, labelText);
    QCoreApplication::processEvents();

	if (!m_cancel)
	{
		quint64 totalArticles = ng->getTotal();
        if (totalArticles != count || ng->unread() > count)
		{
			qint64 difference = count - totalArticles; // may be negative
			quint64 unread = (quint64)qMax((qint64)(ng->unread() + difference), (qint64)0);

            if (unread > count)
                unread = count;

			ng->setTotal(count);
			ng->setUnread(unread);

			emit updateArticleCounts(ng);
		}
	}

	qDebug() << "Loading finished. Seconds: " << start.secsTo(QTime::currentTime());
	qDebug() << "Ignored " << numIgnored << " articles";
    qDebug() << "Loaded " << count << " articles";

	if (mphDeletionsList) // need bulk deletion
        emit addBulkJob(JobList::BulkDelete, ng, hl, mphDeletionsList, sphDeletionsList);

	if (m_cancel)
	{
        emit updateJob(JobList::BulkLoad, JobList::Cancelled, job->seq);
		return false;
	}

	emit updateJob(JobList::BulkLoad, JobList::Finished_Ok, job->seq);
	return true;
}
Beispiel #7
0
int
ex_btrec()
{
	DB *dbp;
	DBC *dbcp;
	DBT key, data;
	DB_BTREE_STAT *statp;
	FILE *fp;
	db_recno_t recno;
	size_t len;
	int cnt, ret;
	char *p, *t, buf[1024], rbuf[1024];
	const char *progname = "ex_btrec";		/* Program name. */

	/* Open the word database. */
	if ((fp = fopen(WORDLIST, "r")) == NULL) {
		fprintf(stderr, "%s: open %s: %s\n",
		    progname, WORDLIST, db_strerror(errno));
		return (1);
	}

	/* Remove the previous database. */
	(void)remove(DATABASE);

	/* Create and initialize database object, open the database. */
	if ((ret = db_create(&dbp, NULL, 0)) != 0) {
		fprintf(stderr,
		    "%s: db_create: %s\n", progname, db_strerror(ret));
		return (1);
	}
	dbp->set_errfile(dbp, stderr);
	dbp->set_errpfx(dbp, progname);			/* 1K page sizes. */
	if ((ret = dbp->set_pagesize(dbp, 1024)) != 0) {
		dbp->err(dbp, ret, "set_pagesize");
		return (1);
	}						/* Record numbers. */
	if ((ret = dbp->set_flags(dbp, DB_RECNUM)) != 0) {
		dbp->err(dbp, ret, "set_flags: DB_RECNUM");
		return (1);
	}
	if ((ret = dbp->open(dbp,
	    NULL, DATABASE, NULL, DB_BTREE, DB_CREATE, 0664)) != 0) {
		dbp->err(dbp, ret, "open: %s", DATABASE);
		return (1);
	}

	/*
	 * Insert records into the database, where the key is the word
	 * preceded by its record number, and the data is the same, but
	 * in reverse order.
	 */
	memset(&key, 0, sizeof(DBT));
	memset(&data, 0, sizeof(DBT));
	for (cnt = 1; cnt <= 1000; ++cnt) {
		(void)sprintf(buf, "%04d_", cnt);
		if (fgets(buf + 4, sizeof(buf) - 4, fp) == NULL)
			break;
		len = strlen(buf);
		for (t = rbuf, p = buf + (len - 2); p >= buf;)
			*t++ = *p--;
		*t++ = '\0';

		key.data = buf;
		data.data = rbuf;
		data.size = key.size = (u_int32_t)len - 1;

		if ((ret =
		    dbp->put(dbp, NULL, &key, &data, DB_NOOVERWRITE)) != 0) {
			dbp->err(dbp, ret, "DB->put");
			if (ret != DB_KEYEXIST)
				goto err1;
		}
	}

	/* Close the word database. */
	(void)fclose(fp);

	/* Print out the number of records in the database. */
	if ((ret = dbp->stat(dbp, NULL, &statp, 0)) != 0) {
		dbp->err(dbp, ret, "DB->stat");
		goto err1;
	}
	printf("%s: database contains %lu records\n",
	    progname, (u_long)statp->bt_ndata);
	free(statp);

	/* Acquire a cursor for the database. */
	if ((ret = dbp->cursor(dbp, NULL, &dbcp, 0)) != 0) {
		dbp->err(dbp, ret, "DB->cursor");
		goto err1;
	}

	/*
	 * Prompt the user for a record number, then retrieve and display
	 * that record.
	 */
	for (;;) {
		/* Get a record number. */
		printf("recno #> ");
		fflush(stdout);
		if (fgets(buf, sizeof(buf), stdin) == NULL)
			break;
		recno = atoi(buf);

		/*
		 * Reset the key each time, the dbp->get() routine returns
		 * the key and data pair, not just the key!
		 */
		key.data = &recno;
		key.size = sizeof(recno);
		if ((ret = dbcp->get(dbcp, &key, &data, DB_SET_RECNO)) != 0)
			goto get_err;

		/* Display the key and data. */
		show("k/d\t", &key, &data);

		/* Move the cursor a record forward. */
		if ((ret = dbcp->get(dbcp, &key, &data, DB_NEXT)) != 0)
			goto get_err;

		/* Display the key and data. */
		show("next\t", &key, &data);

		/*
		 * Retrieve the record number for the following record into
		 * local memory.
		 */
		data.data = &recno;
		data.size = sizeof(recno);
		data.ulen = sizeof(recno);
		data.flags |= DB_DBT_USERMEM;
		if ((ret = dbcp->get(dbcp, &key, &data, DB_GET_RECNO)) != 0) {
get_err:		dbp->err(dbp, ret, "DBcursor->get");
			if (ret != DB_NOTFOUND && ret != DB_KEYEMPTY)
				goto err2;
		} else
			printf("retrieved recno: %lu\n", (u_long)recno);

		/* Reset the data DBT. */
		memset(&data, 0, sizeof(data));
	}

	if ((ret = dbcp->close(dbcp)) != 0) {
		dbp->err(dbp, ret, "DBcursor->close");
		goto err1;
	}
	if ((ret = dbp->close(dbp, 0)) != 0) {
		fprintf(stderr,
		    "%s: DB->close: %s\n", progname, db_strerror(ret));
		return (1);
	}

	return (0);

err2:	(void)dbcp->close(dbcp);
err1:	(void)dbp->close(dbp, 0);
	return (ret);

}
Beispiel #8
0
static int
kvs_cursor_insert(WT_CURSOR *wtcursor)
{
	CURSOR_SOURCE *cursor;
	DB *db;
	DBC *dbc;
	DBT *key, *value;
	WT_EXTENSION_API *wt_api;
	WT_SESSION *session;
	int ret = 0;

	session = wtcursor->session;
	cursor = (CURSOR_SOURCE *)wtcursor;
	wt_api = cursor->wt_api;

	dbc = cursor->dbc;
	db = cursor->db;
	key = &cursor->key;
	value = &cursor->value;

	if ((ret = copyin_key(wtcursor)) != 0)
		return (ret);
	copyin_value(wtcursor);

	if (cursor->config_append) {
		/*
		 * Berkeley DB cursors have no operation to append/create a
		 * new record and set the cursor; use the DB handle instead
		 * then set the cursor explicitly.
		 *
		 * When appending, we're allocating and returning a new record
		 * number.
		 */
		if ((ret = db->put(db, NULL, key, value, DB_APPEND)) != 0)
			ERET(wt_api,
			    session, WT_ERROR, "Db.put: %s", db_strerror(ret));
		wtcursor->recno = *(db_recno_t *)key->data;

		if ((ret = dbc->get(dbc, key, value, DB_SET)) != 0)
			ERET(wt_api, session, WT_ERROR,
			    "DbCursor.get: %s", db_strerror(ret));
	} else if (cursor->config_overwrite) {
		if ((ret = dbc->put(dbc, key, value, DB_KEYFIRST)) != 0)
			ERET(wt_api, session, WT_ERROR,
			    "DbCursor.put: %s", db_strerror(ret));
	} else {
		/*
		 * Berkeley DB cursors don't have a no-overwrite flag; use
		 * the DB handle instead then set the cursor explicitly.
		 */
		if ((ret =
		    db->put(db, NULL, key, value, DB_NOOVERWRITE)) != 0) {
			if (ret == DB_KEYEXIST)
				return (WT_DUPLICATE_KEY);
			ERET(wt_api,
			    session, WT_ERROR, "Db.put: %s", db_strerror(ret));
		}
		if ((ret = dbc->get(dbc, key, value, DB_SET)) != 0)
			ERET(wt_api, session, WT_ERROR,
			    "DbCursor.get: %s", db_strerror(ret));
	}

	return (0);
}
Beispiel #9
0
static int
kvs_cursor_search_near(WT_CURSOR *wtcursor, int *exact)
{
	CURSOR_SOURCE *cursor;
	DBC *dbc;
	DBT *key, *value;
	WT_EXTENSION_API *wt_api;
	WT_SESSION *session;
	size_t len;
	int ret = 0;

	session = wtcursor->session;
	cursor = (CURSOR_SOURCE *)wtcursor;
	wt_api = cursor->wt_api;

	dbc = cursor->dbc;
	key = &cursor->key;
	value = &cursor->value;

	if ((ret = copyin_key(wtcursor)) != 0)
		return (ret);

retry:	if ((ret = dbc->get(dbc, key, value, DB_SET_RANGE)) == 0) {
		/*
		 * WiredTiger returns the logically adjacent key (which might
		 * be less than, equal to, or greater than the specified key),
		 * Berkeley DB returns a key equal to or greater than the
		 * specified key.  Check for an exact match, otherwise Berkeley
		 * DB must have returned a larger key than the one specified.
		 */
		if (key->size == wtcursor->key.size &&
		    memcmp(key->data, wtcursor->key.data, key->size) == 0)
			*exact = 0;
		else
			*exact = 1;
		copyout_key(wtcursor);
		copyout_value(wtcursor);
		return (0);
	}

	/*
	 * Berkeley DB only returns keys equal to or greater than the specified
	 * key, while WiredTiger returns adjacent keys, that is, if there's a
	 * key smaller than the specified key, it's supposed to be returned.  In
	 * other words, WiredTiger only fails if the store is empty.  Read the
	 * last key in the store, and see if it's less than the specified key,
	 * in which case we have the right key to return.  If it's not less than
	 * the specified key, we're racing with some other thread, throw up our
	 * hands and try again.
	 */
	if ((ret = dbc->get(dbc, key, value, DB_LAST)) == 0) {
		len = key->size < wtcursor->key.size ?
		    key->size : wtcursor->key.size;
		if (memcmp(key->data, wtcursor->key.data, len) < 0) {
			*exact = -1;
			copyout_key(wtcursor);
			copyout_value(wtcursor);
			return (0);
		}
		goto retry;
	}

	if (ret == DB_NOTFOUND || ret == DB_KEYEMPTY)
		return (WT_NOTFOUND);
	ERET(wt_api, session, WT_ERROR, "DbCursor.get: %s", db_strerror(ret));
}
Beispiel #10
0
static void tags_cache_gc (struct tags_cache *c)
{
	DBC *cur;
	DBT key;
	DBT serialized_cache_rec;
	int ret;
	char *last_referenced = NULL;
	time_t last_referenced_atime = time (NULL) + 1;
	int nitems = 0;

	c->db->cursor (c->db, NULL, &cur, 0);

	memset (&key, 0, sizeof(key));
	memset (&serialized_cache_rec, 0, sizeof(serialized_cache_rec));

	key.flags = DB_DBT_MALLOC;
	serialized_cache_rec.flags = DB_DBT_MALLOC;

	while (true) {
		struct cache_record rec;

#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR < 6
		ret = cur->c_get (cur, &key, &serialized_cache_rec, DB_NEXT);
#else
		ret = cur->get (cur, &key, &serialized_cache_rec, DB_NEXT);
#endif

		if (ret != 0)
			break;

		if (cache_record_deserialize (&rec, serialized_cache_rec.data,
					serialized_cache_rec.size, 1)
				&& rec.atime < last_referenced_atime) {
			last_referenced_atime = rec.atime;

			if (last_referenced)
				free (last_referenced);
			last_referenced = (char *)xmalloc (key.size + 1);
			memcpy (last_referenced, key.data, key.size);
			last_referenced[key.size] = '\0';
		}

		// TODO: remove objects with serialization error.

		nitems++;

		free (key.data);
		free (serialized_cache_rec.data);
	}

	if (ret != DB_NOTFOUND)
		log_errno ("Searching for element to remove failed (cursor)", ret);

#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR < 6
	cur->c_close (cur);
#else
	cur->close (cur);
#endif

	debug ("Elements in cache: %d (limit %d)", nitems, c->max_items);

	if (last_referenced) {
		if (nitems >= c->max_items)
			tags_cache_remove_rec (c, last_referenced);
		free (last_referenced);
	}
	else
		debug ("Cache empty");
}
void loop_over(DB *db, char *regex, int limit){
    DBC *cursorp;
    DBT key,data;
    int ret; 

    fprintf(stderr, "INFO: Querying with regex '%s', limit of %d total results \n",
                    regex, limit);

    regex_t re;
    int res = regcomp(&re, regex, REG_EXTENDED);
    if(res) {
        char tmp[128];
        regerror(res, &re, tmp, sizeof(tmp));
        DPRINTF(DEBUG_ERROR, "regex error: %s \n", tmp);
        regfree(&re);
        return;
    }

    int count = 0;
    int matches = 0;
    ret = db->cursor(db, NULL, &cursorp,0);
    if(ret) {
        fprintf(stderr, "ERROR: error opening cursor\n");
        db->err(db, ret, "ERROR: Cursor open");
        exit(1);
    }

    memset(&key, 0, sizeof(key));
    memset(&data, 0, sizeof(data));
    while((ret = cursorp->get(cursorp,&key,&data, DB_NEXT)) == 0) {
        int match = !regexec(&re, key.data, 0,0,0);
        if(match) {
            printf("\n");
            printf("Start Host: '%s' \n", (char*)key.data);
            printf("\n");
            ssh_key_info_list *list = list_from_data(data.data, data.size,
                                                      SIGNATURE_LEN);
            print_key_info_list(stdout, list);
            printf("End Host\n");
            printf("\n");
            free_key_info_list(list);
            ++matches;
            if(matches == limit) {
                fprintf(stderr, "WARNING: Reached limit of %d results.  Exiting \n", limit);
                cursorp->close(cursorp);
                return;
            }
        }
        ++count;
    }
    fprintf(stderr, "INFO: done examining %d entries \n", count);

    if(ret != DB_NOTFOUND) {
        fprintf(stderr, "ERROR: some error iterating through db: %s \n",
            db_strerror(ret));
        db->err(db, ret, "ERROR: DB cursor");
    }

    if(cursorp != NULL)
      cursorp->close(cursorp);

}