int lmdbgo_mdb_cursor_putmulti(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, size_t vstride, unsigned int flags) { MDB_val key, val[2]; LMDBGO_SET_VAL(&key, kn, kdata); LMDBGO_SET_VAL(&(val[0]), vstride, vdata); LMDBGO_SET_VAL(&(val[1]), vn, 0); return mdb_cursor_put(cur, &key, &val[0], flags); }
bool DBPrivWriteCursorEntry(DBCursorPriv *cursor, const void *value, int value_size) { MDB_val data; int rc; cursor->pending_delete = false; data.mv_data = (void *)value; data.mv_size = value_size; if (cursor->curkv) { MDB_val curkey; curkey.mv_data = cursor->curkv; curkey.mv_size = sizeof(cursor->curkv); if ((rc = mdb_cursor_put(cursor->mc, &curkey, &data, MDB_CURRENT)) != MDB_SUCCESS) { Log(LOG_LEVEL_ERR, "Could not write cursor entry: %s", mdb_strerror(rc)); } } else { Log(LOG_LEVEL_ERR, "Could not write cursor entry: cannot find current key"); rc = MDB_INVALID; } return rc == MDB_SUCCESS; }
CAMLprim value caml_mdb_cursor_put(value cursor,value key,value data,value flags){ CAMLparam4(cursor,flags,key,data); MDB_val key_,data_; key_.mv_data=String_val(key); key_.mv_size=caml_string_length(key); data_.mv_data=String_val(data); data_.mv_size=caml_string_length(data); if(mdb_cursor_put((MDB_cursor*)cursor, &key_,&data_,Int_val(flags))){ caml_failwith("error in mdb_cursor_put"); } CAMLreturn0; }
auto except(std::exception_ptr ex) { ICY_EXCEPT(ex ? S_OK : E_INVALIDARG); auto record = debug_record{ ex }; auto txn = database_txn{ m_db, database_flag::write }; { const auto cursor = database_cursor{ txn, m_log }; auto now = clock::now(); auto key = MDB_val{ sizeof(now), &now }; auto val = MDB_val{ sizeof(record), &record }; ICY_DATABASE_EXCEPT(mdb_cursor_put(cursor, &key, &val, MDB_NOOVERWRITE)); } txn.commit(); }
int setIntIndex(MDB_cursor *indexCursor, long long elementId, int propertyKeyId, int propertyValue) { int rc; MDB_val key, data; IntIndexKeyStruct *intIndexKeyStruct = malloc(sizeof(IntIndexKeyStruct)); intIndexKeyStruct->propertyKeyId = (int) propertyKeyId; intIndexKeyStruct->elementId = (long long) elementId; intIndexKeyStruct->value = propertyValue; key.mv_size = sizeof(IntIndexKeyStruct); key.mv_data = intIndexKeyStruct; data.mv_size = sizeof(long long); data.mv_data = &elementId; rc = mdb_cursor_put(indexCursor, &key, &data, 0); free(intIndexKeyStruct); return rc; }
int setEdgePropertyInt(MDB_cursor *cursor, jlong edgeId, jint propertyKeyId, jint *propertyValue) { if (propertyValue != NULL) { MDB_val key, data; EdgeDbId id; id.edgeId = edgeId; id.coreOrPropertyEnum = EPROPERTY_KEY; id.propertykeyId = propertyKeyId; key.mv_size = sizeof(EdgeDbId); key.mv_data = &id; data.mv_size = sizeof(jint); data.mv_data = propertyValue; return mdb_cursor_put(cursor, &key, &data, 0); } else { return GLMDB_WRITE_NULL; } }
auto insert_server(const GUID& id, const crypto_key& password) { auto txn = database_txn{ m_db, database_flag::write }; { const auto cur = database_cursor{ txn, m_servers.dbi }; auto key = id; auto val = server{ password }; auto db_val = MDB_val{ sizeof(val), &val }; auto db_key = MDB_val{ sizeof(key), &key }; const auto error = mdb_cursor_put(cur, &db_key, &db_val, MDB_NOOVERWRITE); if (error == MDB_KEYEXIST) return false; ICY_DATABASE_EXCEPT(error); } txn.commit(); return true; }
int setVertexPropertyInt(MDB_cursor *cursor, jlong vertexId, jint propertyKeyId, jint *propertyValue) { if (propertyValue != NULL) { MDB_val key, data; VertexDbId id; id.vertexId = vertexId; id.coreOrPropertyEnum = VPROPERTY_KEY; id.propertykeyId = propertyKeyId; id.labelId = -1; id.edgeId = -1LL; key.mv_size = sizeof(VertexDbId); key.mv_data = &id; data.mv_size = sizeof(jint); data.mv_data = propertyValue; return mdb_cursor_put(cursor, &key, &data, 0); } else { return GLMDB_WRITE_NULL; } }
auto insert_client(const string_view login, const crypto_key& password) { if (login.bytes().size() > sizeof(client::login)) return false; auto txn = database_txn{ m_db, database_flag::write }; { const auto cur = database_cursor{ txn, m_clients.dbi }; auto key = crypto_login{ login }; auto val = client{ password, login }; auto db_val = MDB_val{ sizeof(val), &val }; auto db_key = MDB_val{ sizeof(key), &key }; const auto error = mdb_cursor_put(cur, &db_key, &db_val, MDB_NOOVERWRITE); if (error == MDB_KEYEXIST) return false; ICY_DATABASE_EXCEPT(error); } txn.commit(); return true; }
int cache_insert(MDB_txn *txn, MDB_dbi dbi, const knot_dname_t *name, struct entry *entry) { MDB_cursor *cursor = cursor_acquire(txn, dbi); if (cursor == NULL) { return KNOT_ERROR; } MDB_val key = pack_key(name); MDB_val data = { 0, malloc(ENTRY_MAXLEN) }; int ret = pack_entry(&data, entry); if (ret != KNOT_EOK) { free(data.mv_data); return ret; } ret = mdb_cursor_put(cursor, &key, &data, 0); free(data.mv_data); cursor_release(cursor); return ret; }
/* * Class: jmdb_DatabaseWrapper * Method: cursorPut * Signature: (J[BII[BIII)V */ JNIEXPORT void JNICALL Java_jmdb_DatabaseWrapper_cursorPut(JNIEnv *vm, jclass clazz, jlong cursorL, jbyteArray keyA, jint kofs, jint klen, jbyteArray valueA, jint vofs, jint vlen, jint flags) { MDB_cursor *cursorC = (MDB_cursor*) cursorL; enum { NONE, OOM, MDB } result = NONE; jint ret; MDB_val key, value; jbyte *keyC = (*vm)->GetPrimitiveArrayCritical(vm, keyA, NULL); jbyte *valueC = (*vm)->GetPrimitiveArrayCritical(vm, valueA, NULL); if (keyC == NULL || valueC == NULL) { result = OOM; ret = -1; } else { key.mv_size = klen; key.mv_data = keyC + kofs; value.mv_size = vlen; value.mv_data = valueC + vofs; ret = mdb_cursor_put(cursorC, &key, &value, (unsigned int) flags); if (ret) { result = MDB; } } (*vm)->ReleasePrimitiveArrayCritical(vm, valueA, valueC, 0); (*vm)->ReleasePrimitiveArrayCritical(vm, keyA, keyC, JNI_ABORT); switch (result) { case NONE: return; case OOM: throwNew(vm, "java/lang/OutOfMemoryError", ""); return; case MDB: throwDatabaseException(vm, ret); return; } }
PageDBError page_db_new(PageDB **db, const char *path) { PageDB *p = *db = malloc(sizeof(*p)); if (p == 0) return page_db_error_memory; p->error = error_new(); if (p->error == 0) { free(p); return page_db_error_memory; } p->persist = PAGE_DB_DEFAULT_PERSIST; p->domain_temp = 0; // create directory if not present yet const char *error = make_dir(path); if ((p->path = strdup(path)) == 0) { error = "could not duplicate path string"; } if (error != 0) { page_db_set_error(p, page_db_error_invalid_path, __func__); page_db_add_error(p, error); return p->error->code; } if (txn_manager_new(&p->txn_manager, 0) != 0) { page_db_set_error(p, page_db_error_internal, __func__); page_db_add_error(p, p->txn_manager? p->txn_manager->error->message: "NULL"); return p->error->code; } // initialize LMDB on the directory MDB_txn *txn; MDB_dbi dbi; int mdb_rc = 0; if ((mdb_rc = mdb_env_create(&p->txn_manager->env) != 0)) error = "creating environment"; else if ((mdb_rc = mdb_env_set_mapsize( p->txn_manager->env, PAGE_DB_DEFAULT_SIZE)) != 0) error = "setting map size"; else if ((mdb_rc = mdb_env_set_maxdbs(p->txn_manager->env, 5)) != 0) error = "setting number of databases"; else if ((mdb_rc = mdb_env_open( p->txn_manager->env, path, MDB_NOTLS | MDB_NOSYNC, 0664) != 0)) error = "opening environment"; else if ((mdb_rc = txn_manager_begin(p->txn_manager, 0, &txn)) != 0) error = "starting transaction"; // create all database else if ((mdb_rc = mdb_dbi_open(txn, "hash2info", MDB_CREATE | MDB_INTEGERKEY, &dbi)) != 0) error = "creating hash2info database"; else if ((mdb_rc = mdb_dbi_open(txn, "hash2idx", MDB_CREATE | MDB_INTEGERKEY, &dbi)) != 0) error = "creating hash2idx database"; else if ((mdb_rc = mdb_dbi_open(txn, "links", MDB_CREATE | MDB_INTEGERKEY, &dbi)) != 0) error = "creating links database"; else if ((mdb_rc = mdb_dbi_open(txn, "info", MDB_CREATE, &dbi)) != 0) error = "creating info database"; else { // initialize n_pages inside info database size_t n_pages = 0; MDB_val key = { .mv_size = sizeof(info_n_pages), .mv_data = info_n_pages }; MDB_val val = { .mv_size = sizeof(size_t), .mv_data = &n_pages }; switch (mdb_rc = mdb_put(txn, dbi, &key, &val, MDB_NOOVERWRITE)) { case MDB_KEYEXIST: case 0: if (txn_manager_commit(p->txn_manager, txn) != 0) error = p->txn_manager->error->message; break; // we good default: error = "could not initialize info.n_pages"; } } if (error != 0) { page_db_set_error(p, page_db_error_internal, __func__); page_db_add_error(p, error); page_db_add_error(p, mdb_strerror(mdb_rc)); mdb_env_close(p->txn_manager->env); } return p->error->code; } /** Store a new or updated @ref PageInfo for a crawled page. * * @param cur An open cursor to the hash2info database * @param key The key (hash) to the page * @param page * @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter * will be set with the error (otherwise is set to zero). * @return 0 if success, -1 if failure. */ static int page_db_add_crawled_page_info(MDB_cursor *cur, MDB_val *key, const CrawledPage *page, PageInfo **page_info, int *mdb_error) { MDB_val val; *page_info = 0; int mdb_rc = mdb_cursor_get(cur, key, &val, MDB_SET); int put_flags = 0; switch (mdb_rc) { case 0: if (!(*page_info = page_info_load(&val))) goto on_error; if ((page_info_update(*page_info, page) != 0)) goto on_error; put_flags = MDB_CURRENT; break; case MDB_NOTFOUND: if (!(*page_info = page_info_new_crawled(page))) goto on_error; break; default: goto on_error; } if ((page_info_dump(*page_info, &val) != 0)) goto on_error; if ((mdb_rc = mdb_cursor_put(cur, key, &val, put_flags)) != 0) goto on_error; free(val.mv_data); val.mv_data = 0; *mdb_error = 0; return 0; on_error: if (val.mv_data) free(val.mv_data); *mdb_error = mdb_rc; page_info_delete(*page_info); return -1; } /** Store a new or updated @ref PageInfo for an uncrawled link. * * @param cur An open cursor to the hash2info database * @param key The key (hash) to the page * @param url * @param mdb_error In case of failure, if the error occurs inside LMDB this output parameter * will be set with the error (otherwise is set to zero). * @return 0 if success, -1 if failure. */ static int page_db_add_link_page_info(MDB_cursor *cur, MDB_val *key, uint64_t linked_from, uint64_t depth, const LinkInfo *link, PageInfo **page_info, int *mdb_error) { MDB_val val; int mdb_rc = 0; PageInfo *pi = *page_info = page_info_new_link(link->url, linked_from, depth, link->score); if (!pi) goto on_error; if ((page_info_dump(pi, &val) != 0)) goto on_error; if ((mdb_rc = mdb_cursor_put(cur, key, &val, MDB_NOOVERWRITE)) != 0) goto on_error; free(val.mv_data); val.mv_data = 0; *mdb_error = 0; return 0; on_error: if (val.mv_data) free(val.mv_data); *mdb_error = mdb_rc; page_info_delete(pi); return -1; }
/* How new PageInfo are created: page_db_add ---------------> page_db_add_crawled_page_info | | | | v v page_db_add_link_page_info page_info_new_crawled | | | | | | +--------> page_info_new_link <---+ */ PageDBError page_db_add(PageDB *db, const CrawledPage *page, PageInfoList **page_info_list) { // check if page should be expanded if (page_db_expand(db) != 0) return db->error->code; MDB_txn *txn; MDB_cursor *cur_hash2info; MDB_cursor *cur_hash2idx; MDB_cursor *cur_links; MDB_cursor *cur_info; MDB_val key; MDB_val val; int mdb_rc; char *error = 0; uint64_t *diff_id = 0; uint64_t *same_id = 0; // start a new write transaction txn = 0; if ((txn_manager_begin(db->txn_manager, 0, &txn)) != 0) error = db->txn_manager->error->message; else if ((mdb_rc = page_db_open_hash2info(txn, &cur_hash2info)) != 0) error = "opening hash2info cursor"; else if ((mdb_rc = page_db_open_hash2idx(txn, &cur_hash2idx)) != 0) error = "opening hash2idx cursor"; else if ((mdb_rc = page_db_open_links(txn, &cur_links)) != 0) error = "opening links cursor"; else if ((mdb_rc = page_db_open_info(txn, &cur_info)) != 0) error = "opening info cursor"; if (error != 0) goto on_error; // get n_pages key.mv_size = sizeof(info_n_pages); key.mv_data = info_n_pages; if ((mdb_rc = mdb_cursor_get(cur_info, &key, &val, MDB_SET)) != 0) { error = "retrieving info.n_pages"; goto on_error; } size_t n_pages = *(size_t*)val.mv_data; uint64_t cp_hash = page_db_hash(page->url); key.mv_size = sizeof(uint64_t); key.mv_data = &cp_hash; if (db->domain_temp) { domain_temp_update(db->domain_temp, (float)page->time); domain_temp_heat(db->domain_temp, page_db_hash_get_domain(cp_hash)); } PageInfo *pi; if (page_db_add_crawled_page_info(cur_hash2info, &key, page, &pi, &mdb_rc) != 0) { error = "adding/updating page info"; goto on_error; } uint64_t link_depth = pi->depth + 1; if (page_info_list) { *page_info_list = page_info_list_new(pi, cp_hash); if (!*page_info_list) { error = "allocating new PageInfo list"; goto on_error; } } else { page_info_delete(pi); pi = 0; } size_t n_links = crawled_page_n_links(page); // store here links inside the same domain as the crawled page same_id = malloc((n_links + 1)*sizeof(*same_id)); // store here links outside the domain of the crawled page diff_id = malloc((n_links + 1)*sizeof(*diff_id)); // next link id is going to be written here uint64_t *id = diff_id; // number of id's in same_id and diff_id. The first element of diff_id // array is reserved for the id of the crawled page, so we start at 1. // The first element of same_id will be a copy of the last element of // diff_id, so we start at 1 too. uint64_t same_i = 1; uint64_t diff_i = 1; if (!same_id || !diff_id) { error = "could not malloc"; goto on_error; } // hash of the current URL uint64_t hash = cp_hash; for (size_t i=0; i <= n_links; ++i) { const LinkInfo *link = i > 0? crawled_page_get_link(page, i - 1): 0; if (link) { hash = page_db_hash(link->url); key.mv_size = sizeof(uint64_t); key.mv_data = &hash; id = same_domain(page->url, link->url)? same_id + same_i++: diff_id + diff_i++; } val.mv_size = sizeof(uint64_t); val.mv_data = &n_pages; switch (mdb_rc = mdb_cursor_put(cur_hash2idx, &key, &val, MDB_NOOVERWRITE)) { case MDB_KEYEXIST: // not really an error *id = *(uint64_t*)val.mv_data; break; case 0: *id = n_pages++; if (link) { if (page_db_add_link_page_info( cur_hash2info, &key, cp_hash, link_depth, link, &pi, &mdb_rc) != 0) { error = "adding/updating link info"; goto on_error; } if (page_info_list) { if (!(*page_info_list = page_info_list_cons(*page_info_list, pi, hash))) { error = "adding new PageInfo to list"; goto on_error; } } else { page_info_delete(pi); } } break; default: goto on_error; } } // store n_pages key.mv_size = sizeof(info_n_pages); key.mv_data = info_n_pages; val.mv_size = sizeof(size_t); val.mv_data = &n_pages; if ((mdb_rc = mdb_cursor_put(cur_info, &key, &val, 0)) != 0) { error = "storing n_pages"; goto on_error; } // store links and commit transaction // The format for the links is the following: // // KEY = ID of crawled page // VAL = Number of links to different domain, // diff link id 1, diff link id 2, ... // same link id 1, same link id 2, ... key.mv_size = sizeof(uint64_t); key.mv_data = diff_id; // remember that diff_id[0] is the id of the // crawled page // the links are stored as deltas starting from the 'from' page, encoded // using varint. uint8_t *buf = val.mv_data = malloc(MAX_VARINT_SIZE*(n_links + 1)); if (!buf) { error = "allocating memory to store links"; goto on_error; } // write number of diff links (substract 1 to take into account this page id) buf = varint_encode_uint64(diff_i - 1, buf); // write diff links for (size_t i=1; i<diff_i; ++i) buf = varint_encode_int64((int64_t)diff_id[i] - (int64_t)diff_id[i-1], buf); same_id[0] = diff_id[diff_i-1]; for (size_t i=1; i<same_i; ++i) buf = varint_encode_int64((int64_t)same_id[i] - (int64_t)same_id[i-1], buf); val.mv_size = (char*)buf - (char*)val.mv_data; if ((mdb_rc = mdb_cursor_put(cur_links, &key, &val, 0)) != 0) { error = "storing links"; goto on_error; } free(val.mv_data); free(same_id); free(diff_id); same_id = diff_id = 0; if (txn_manager_commit(db->txn_manager, txn) != 0) { error = db->txn_manager->error->message; goto on_error; } return db->error->code; on_error: if (same_id) free(same_id); if (diff_id) free(diff_id); if (txn) txn_manager_abort(db->txn_manager, txn); page_db_set_error(db, page_db_error_internal, __func__); page_db_add_error(db, error); if (mdb_rc != 0) page_db_add_error(db, mdb_strerror(mdb_rc)); return db->error->code; }
static int mdb_tool_next_id( Operation *op, MDB_txn *tid, Entry *e, struct berval *text, int hole ) { struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; struct berval dn = e->e_name; struct berval ndn = e->e_nname; struct berval pdn, npdn, nmatched; ID id, pid = 0; int rc; if (ndn.bv_len == 0) { e->e_id = 0; return 0; } rc = mdb_dn2id( op, tid, mcp, &ndn, &id, NULL, &nmatched ); if ( rc == MDB_NOTFOUND ) { if ( !be_issuffix( op->o_bd, &ndn ) ) { ID eid = e->e_id; dnParent( &ndn, &npdn ); if ( nmatched.bv_len != npdn.bv_len ) { dnParent( &dn, &pdn ); e->e_name = pdn; e->e_nname = npdn; rc = mdb_tool_next_id( op, tid, e, text, 1 ); e->e_name = dn; e->e_nname = ndn; if ( rc ) { return rc; } /* If parent didn't exist, it was created just now * and its ID is now in e->e_id. Make sure the current * entry gets added under the new parent ID. */ if ( eid != e->e_id ) { pid = e->e_id; } } else { pid = id; } } rc = mdb_next_id( op->o_bd, idcursor, &e->e_id ); if ( rc ) { snprintf( text->bv_val, text->bv_len, "next_id failed: %s (%d)", mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); return rc; } rc = mdb_dn2id_add( op, mcp, mcd, pid, e ); if ( rc ) { snprintf( text->bv_val, text->bv_len, "dn2id_add failed: %s (%d)", mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); } else if ( hole ) { MDB_val key, data; if ( nholes == nhmax - 1 ) { if ( holes == hbuf ) { holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); } else { holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); } nhmax *= 2; } ber_dupbv( &holes[nholes].dn, &ndn ); holes[nholes++].id = e->e_id; key.mv_size = sizeof(ID); key.mv_data = &e->e_id; data.mv_size = 0; data.mv_data = NULL; rc = mdb_cursor_put( idcursor, &key, &data, MDB_NOOVERWRITE ); if ( rc == MDB_KEYEXIST ) rc = 0; if ( rc ) { snprintf( text->bv_val, text->bv_len, "dummy id2entry add failed: %s (%d)", mdb_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, "=> mdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); } } } else if ( !hole ) { unsigned i, j; e->e_id = id; for ( i=0; i<nholes; i++) { if ( holes[i].id == e->e_id ) { free(holes[i].dn.bv_val); for (j=i;j<nholes;j++) holes[j] = holes[j+1]; holes[j].id = 0; nholes--; break; } else if ( holes[i].id > e->e_id ) { break; } } } return rc; }
static int mdb_tool_idl_flush_one( MDB_cursor *mc, AttrInfo *ai, mdb_tool_idl_cache *ic ) { mdb_tool_idl_cache_entry *ice; MDB_val key, data[2]; int i, rc; ID id, nid; /* Freshly allocated, ignore it */ if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) { return 0; } key.mv_data = ic->kstr.bv_val; key.mv_size = ic->kstr.bv_len; if ( ic->count > MDB_IDL_DB_SIZE ) { while ( ic->flags & WAS_FOUND ) { rc = mdb_cursor_get( mc, &key, data, MDB_SET ); if ( rc ) { /* FIXME: find out why this happens */ ic->flags = 0; break; } if ( ic->flags & WAS_RANGE ) { /* Skip lo */ rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); /* Get hi */ rc = mdb_cursor_get( mc, &key, data, MDB_NEXT_DUP ); /* Store range hi */ data[0].mv_data = &ic->last; rc = mdb_cursor_put( mc, &key, data, MDB_CURRENT ); } else { /* Delete old data, replace with range */ ic->first = *(ID *)data[0].mv_data; mdb_cursor_del( mc, MDB_NODUPDATA ); } break; } if ( !(ic->flags & WAS_RANGE)) { /* range, didn't exist before */ nid = 0; data[0].mv_size = sizeof(ID); data[0].mv_data = &nid; rc = mdb_cursor_put( mc, &key, data, 0 ); if ( rc == 0 ) { data[0].mv_data = &ic->first; rc = mdb_cursor_put( mc, &key, data, 0 ); if ( rc == 0 ) { data[0].mv_data = &ic->last; rc = mdb_cursor_put( mc, &key, data, 0 ); } } if ( rc ) { rc = -1; } } } else { /* Normal write */ int n; data[0].mv_size = sizeof(ID); rc = 0; i = ic->offset; for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { int end; if ( ice->next ) { end = IDBLOCK; } else { end = ic->count & (IDBLOCK-1); if ( !end ) end = IDBLOCK; } data[1].mv_size = end - i; data[0].mv_data = &ice->ids[i]; i = 0; rc = mdb_cursor_put( mc, &key, data, MDB_NODUPDATA|MDB_APPEND|MDB_MULTIPLE ); if ( rc ) { if ( rc == MDB_KEYEXIST ) { rc = 0; continue; } rc = -1; break; } } if ( ic->head ) { ic->tail->next = ai->ai_flist; ai->ai_flist = ic->head; } } ic->head = ai->ai_clist; ai->ai_clist = ic; return rc; }
/* We add two elements to the DN2ID database - a data item under the parent's * entryID containing the child's RDN and entryID, and an item under the * child's entryID containing the parent's entryID. */ int mdb_dn2id_add( Operation *op, MDB_cursor *mcp, MDB_cursor *mcd, ID pid, ID nsubs, int upsub, Entry *e ) { struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; MDB_val key, data; ID nid; int rc, rlen, nrlen; diskNode *d; char *ptr; Debug( LDAP_DEBUG_TRACE, "=> mdb_dn2id_add 0x%lx: \"%s\"\n", e->e_id, e->e_ndn ? e->e_ndn : "" ); nrlen = dn_rdnlen( op->o_bd, &e->e_nname ); if (nrlen) { rlen = dn_rdnlen( op->o_bd, &e->e_name ); } else { nrlen = e->e_nname.bv_len; rlen = e->e_name.bv_len; } d = op->o_tmpalloc(sizeof(diskNode) + rlen + nrlen + sizeof(ID), op->o_tmpmemctx); d->nrdnlen[1] = nrlen & 0xff; d->nrdnlen[0] = (nrlen >> 8) | 0x80; ptr = lutil_strncopy( d->nrdn, e->e_nname.bv_val, nrlen ); *ptr++ = '\0'; ptr = lutil_strncopy( ptr, e->e_name.bv_val, rlen ); *ptr++ = '\0'; memcpy( ptr, &e->e_id, sizeof( ID )); ptr += sizeof( ID ); memcpy( ptr, &nsubs, sizeof( ID )); key.mv_size = sizeof(ID); key.mv_data = &nid; nid = pid; /* Need to make dummy root node once. Subsequent attempts * will fail harmlessly. */ if ( pid == 0 ) { diskNode dummy = {{0, 0}, "", "", ""}; data.mv_data = &dummy; data.mv_size = sizeof(diskNode); mdb_cursor_put( mcp, &key, &data, MDB_NODUPDATA ); } data.mv_data = d; data.mv_size = sizeof(diskNode) + rlen + nrlen + sizeof( ID ); /* Add our child node under parent's key */ rc = mdb_cursor_put( mcp, &key, &data, MDB_NODUPDATA ); /* Add our own node */ if (rc == 0) { int flag = MDB_NODUPDATA; nid = e->e_id; /* drop subtree count */ data.mv_size -= sizeof( ID ); ptr -= sizeof( ID ); memcpy( ptr, &pid, sizeof( ID )); d->nrdnlen[0] ^= 0x80; if ((slapMode & SLAP_TOOL_MODE) || (e->e_id == mdb_read_nextid(mdb))) flag |= MDB_APPEND; rc = mdb_cursor_put( mcd, &key, &data, flag ); } op->o_tmpfree( d, op->o_tmpmemctx ); /* Add our subtree count to all superiors */ if ( rc == 0 && upsub && pid ) { ID subs; nid = pid; do { /* Get parent's RDN */ rc = mdb_cursor_get( mcp, &key, &data, MDB_SET ); if ( !rc ) { char *p2; ptr = (char *)data.mv_data + data.mv_size - sizeof( ID ); memcpy( &nid, ptr, sizeof( ID )); /* Get parent's node under grandparent */ d = data.mv_data; rlen = ( d->nrdnlen[0] << 8 ) | d->nrdnlen[1]; p2 = op->o_tmpalloc( rlen + 2, op->o_tmpmemctx ); memcpy( p2, data.mv_data, rlen+2 ); *p2 ^= 0x80; data.mv_data = p2; rc = mdb_cursor_get( mcp, &key, &data, MDB_GET_BOTH ); op->o_tmpfree( p2, op->o_tmpmemctx ); if ( !rc ) { /* Get parent's subtree count */ ptr = (char *)data.mv_data + data.mv_size - sizeof( ID ); memcpy( &subs, ptr, sizeof( ID )); subs += nsubs; p2 = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); memcpy( p2, data.mv_data, data.mv_size - sizeof( ID )); memcpy( p2+data.mv_size - sizeof( ID ), &subs, sizeof( ID )); data.mv_data = p2; rc = mdb_cursor_put( mcp, &key, &data, MDB_CURRENT ); op->o_tmpfree( p2, op->o_tmpmemctx ); } } if ( rc ) break; } while ( nid ); } Debug( LDAP_DEBUG_TRACE, "<= mdb_dn2id_add 0x%lx: %d\n", e->e_id, rc ); return rc; }
int mdb_idl_insert_keys( BackendDB *be, MDB_cursor *cursor, struct berval *keys, ID id ) { struct mdb_info *mdb = be->be_private; MDB_val key, data; ID lo, hi, *i; char *err; int rc = 0, k; unsigned int flag = MDB_NODUPDATA; #ifndef MISALIGNED_OK int kbuf[2]; #endif { char buf[16]; Debug( LDAP_DEBUG_ARGS, "mdb_idl_insert_keys: %lx %s\n", (long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 ); } assert( id != NOID ); #ifndef MISALIGNED_OK if (keys[0].bv_len & ALIGNER) kbuf[1] = 0; #endif for ( k=0; keys[k].bv_val; k++ ) { /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ #ifndef MISALIGNED_OK if (keys[k].bv_len & ALIGNER) { key.mv_size = sizeof(kbuf); key.mv_data = kbuf; memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len); } else #endif { key.mv_size = keys[k].bv_len; key.mv_data = keys[k].bv_val; } rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); err = "c_get"; if ( rc == 0 ) { i = data.mv_data; memcpy(&lo, data.mv_data, sizeof(ID)); if ( lo != 0 ) { /* not a range, count the number of items */ size_t count; rc = mdb_cursor_count( cursor, &count ); if ( rc != 0 ) { err = "c_count"; goto fail; } if ( count >= MDB_IDL_DB_MAX ) { /* No room, convert to a range */ lo = *i; rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP ); if ( rc != 0 && rc != MDB_NOTFOUND ) { err = "c_get last_dup"; goto fail; } i = data.mv_data; hi = *i; /* Update hi/lo if needed */ if ( id < lo ) { lo = id; } else if ( id > hi ) { hi = id; } /* delete the old key */ rc = mdb_cursor_del( cursor, MDB_NODUPDATA ); if ( rc != 0 ) { err = "c_del dups"; goto fail; } /* Store the range */ data.mv_size = sizeof(ID); data.mv_data = &id; id = 0; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put range"; goto fail; } id = lo; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put lo"; goto fail; } id = hi; rc = mdb_cursor_put( cursor, &key, &data, 0 ); if ( rc != 0 ) { err = "c_put hi"; goto fail; } } else { /* There's room, just store it */ if (id == mdb->mi_nextid) flag |= MDB_APPENDDUP; goto put1; } } else { /* It's a range, see if we need to rewrite * the boundaries */ lo = i[1]; hi = i[2]; if ( id < lo || id > hi ) { /* position on lo */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( rc != 0 ) { err = "c_get lo"; goto fail; } if ( id > hi ) { /* position on hi */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( rc != 0 ) { err = "c_get hi"; goto fail; } } data.mv_size = sizeof(ID); data.mv_data = &id; /* Replace the current lo/hi */ rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT ); if ( rc != 0 ) { err = "c_put lo/hi"; goto fail; } } } } else if ( rc == MDB_NOTFOUND ) { flag &= ~MDB_APPENDDUP; put1: data.mv_data = &id; data.mv_size = sizeof(ID); rc = mdb_cursor_put( cursor, &key, &data, flag ); /* Don't worry if it's already there */ if ( rc == MDB_KEYEXIST ) rc = 0; if ( rc ) { err = "c_put id"; goto fail; } } else { /* initial c_get failed, nothing was done */ fail: Debug( LDAP_DEBUG_ANY, "=> mdb_idl_insert_keys: " "%s failed: %s (%d)\n", err, mdb_strerror(rc), rc ); break; } } return rc; }
/* Write a frame or frames to the log. */ int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags) { PgHdr *p; MDB_val key, data; int rc; mdbinf* mdb; MDB_txn* txn; #if ATOMIC db_thread *thr = g_tsd_thread; db_connection* pCon = g_tsd_conn; #else db_thread* thr = enif_tsd_get(g_tsd_thread); db_connection* pCon = enif_tsd_get(g_tsd_conn); #endif #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif txn = mdb->txn; if (!mdb) return SQLITE_ERROR; key.mv_size = sizeof(u64); key.mv_data = (void*)&pWal->index; // Term/evnum must always be increasing if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) || (pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum)) return SQLITE_ERROR; track_time(2,thr); // ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>} for(p=pList; p; p=p->pDirty) { u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)]; u8 pagesBuf[PAGE_BUFF_SIZE]; int full_size = 0; int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf)); char fragment_index = 0; int skipped = 0; track_time(3,thr); DBG("Insert frame, actor=%lld, pgno=%u, " "term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d", pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum, isCommit,nTruncate,page_size); if (pCon->doReplicate) { u8 hdr[sizeof(u64)*2+sizeof(u32)*2]; put8byte(hdr, pWal->inProgressTerm); put8byte(hdr+sizeof(u64), pWal->inProgressEvnum); put4byte(hdr+sizeof(u64)*2, p->pgno); if (p->pDirty) put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0); else put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate); #ifndef _TESTAPP_ wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr)); #endif } memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; // Check if there are pages with the same or higher evnum/evterm. If there are, delete them. // This can happen if sqlite flushed some page to disk before commiting, because there were // so many pages that they could not be held in memory. Or it could happen if pages need to be // overwritten because there was a write that did not pass raft consensus. rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS) { size_t ndupl; mdb_cursor_count(mdb->cursorPages,&ndupl); rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP); if (rc == MDB_SUCCESS) { MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL}; u64 evnum, evterm; u8 frag = *((u8*)data.mv_data+sizeof(u64)*2); memcpy(&evterm, data.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64)); while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum)) //(pWal->inProgressTerm + pWal->inProgressEvnum) > 0) { DBG("Deleting pages higher or equal to current. " "Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld", evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl); if (pgDelKey.mv_data != NULL) { if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS) { DBG("Unable to cleanup page from pagedb %d",rc); break; } pgDelKey.mv_data = NULL; } mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT); // if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS) // { // DBG("Cant delete!"); // break; // } if (frag == 0) pWal->allPages--; ndupl--; if (!ndupl) break; rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP); if (rc != MDB_SUCCESS) break; memcpy(&evterm, data.mv_data, sizeof(u64)); memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64)); frag = *((u8*)data.mv_data+sizeof(u64)*2); } if (pgDelKey.mv_data != NULL) { if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS) { DBG("Unable to cleanup page from pagedb %d",rc); break; } pgDelKey.mv_data = NULL; } } memcpy(pagesKeyBuf, &pWal->index,sizeof(u64)); memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32)); key.mv_size = sizeof(pagesKeyBuf); key.mv_data = pagesKeyBuf; } track_time(4,thr); memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64)); memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64)); full_size = page_size + sizeof(u64)*2 + 1; if (full_size < thr->maxvalsize) fragment_index = 0; else { full_size = page_size; skipped = thr->maxvalsize - sizeof(u64)*2 - 1; full_size -= skipped; while(full_size > 0) { full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1); fragment_index++; } full_size = page_size + sizeof(u64)*2 +1; } pagesBuf[sizeof(u64)*2] = fragment_index; data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize; data.mv_data = pagesBuf; // fragment_index == 0 ? MDB_APPENDDUP : 0 if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS) { // printf("Cursor put failed to pages %d",rc); DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size); return SQLITE_ERROR; } fragment_index--; skipped = data.mv_size; while (fragment_index >= 0) { DBG("Insert fragment %d",(int)fragment_index); if (fragment_index == 0) data.mv_size = full_size - skipped + sizeof(u64)*2 + 1; else data.mv_size = thr->maxvalsize; data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1); memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64)); memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64)); pagesBuf[skipped-1] = fragment_index; if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS) { DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d", rc,full_size, skipped, (int)fragment_index); return SQLITE_ERROR; } skipped += data.mv_size - sizeof(u64)*2 - 1; fragment_index--; } thr->pagesChanged++; } // printf(""); // ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>} if (pWal->inProgressTerm > 0) { for(p=pList; p; p=p->pDirty) { u8 logKeyBuf[sizeof(u64)*3]; DBG("Inserting to log"); memcpy(logKeyBuf, &pWal->index, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64)); memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64)); key.mv_size = sizeof(logKeyBuf); key.mv_data = logKeyBuf; data.mv_size = sizeof(u32); data.mv_data = &p->pgno; if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS) { // printf("Cursor put failed to log"); DBG("ERROR: cursor put to log failed: %d",rc); return SQLITE_ERROR; } pWal->allPages++; } } else { DBG("Skipping log"); for(p=pList; p; p=p->pDirty) pWal->allPages++; } /** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64, LastCompleteTerm:64,LastCompleteEvnum:64, InprogressTerm:64,InProgressEvnum:64>>} */ { if (isCommit) { DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu", pWal->index, pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm, pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum); #ifndef _TESTAPP_ enif_mutex_lock(pWal->mtx); #endif pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ? pWal->inProgressTerm : pWal->lastCompleteTerm; pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ? pWal->inProgressEvnum : pWal->lastCompleteEvnum; if (pWal->firstCompleteTerm == 0) { pWal->firstCompleteTerm = pWal->inProgressTerm; pWal->firstCompleteEvnum = pWal->inProgressEvnum; } pWal->inProgressTerm = pWal->inProgressEvnum = 0; pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate; // pWal->changed = 0; thr->forceCommit = 1; pCon->dirty = 0; #ifndef _TESTAPP_ enif_mutex_unlock(pWal->mtx); #endif DBG("cur mxpage=%u",pWal->mxPage); } else { // pWal->changed = 1; pCon->dirty = 1; } thr->pagesChanged++; rc = storeinfo(pWal,0,0,NULL); if (rc != SQLITE_OK) return rc; track_time(5,thr); } return SQLITE_OK; }
int cursor_put(cursor_t cursor, MDB_val *key, MDB_val *data, unsigned int flags){ int ret = cursor->prev ? mdb_cursor_put(cursor->cursor, key, data, flags) : MDB_BAD_TXN; if(MDB_SUCCESS == ret) cursor->txn->updated = 1; return ret; }
int mdb_idl_delete_keys( BackendDB *be, MDB_cursor *cursor, struct berval *keys, ID id ) { int rc = 0, k; MDB_val key, data; ID lo, hi, tmp, *i; char *err; #ifndef MISALIGNED_OK int kbuf[2]; #endif { char buf[16]; Debug( LDAP_DEBUG_ARGS, "mdb_idl_delete_keys: %lx %s\n", (long) id, mdb_show_key( buf, keys->bv_val, keys->bv_len ), 0 ); } assert( id != NOID ); #ifndef MISALIGNED_OK if (keys[0].bv_len & ALIGNER) kbuf[1] = 0; #endif for ( k=0; keys[k].bv_val; k++) { /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ #ifndef MISALIGNED_OK if (keys[k].bv_len & ALIGNER) { key.mv_size = sizeof(kbuf); key.mv_data = kbuf; memcpy(key.mv_data, keys[k].bv_val, keys[k].bv_len); } else #endif { key.mv_size = keys[k].bv_len; key.mv_data = keys[k].bv_val; } rc = mdb_cursor_get( cursor, &key, &data, MDB_SET ); err = "c_get"; if ( rc == 0 ) { memcpy( &tmp, data.mv_data, sizeof(ID) ); i = data.mv_data; if ( tmp != 0 ) { /* Not a range, just delete it */ data.mv_data = &id; rc = mdb_cursor_get( cursor, &key, &data, MDB_GET_BOTH ); if ( rc != 0 ) { err = "c_get id"; goto fail; } rc = mdb_cursor_del( cursor, 0 ); if ( rc != 0 ) { err = "c_del id"; goto fail; } } else { /* It's a range, see if we need to rewrite * the boundaries */ lo = i[1]; hi = i[2]; if ( id == lo || id == hi ) { ID lo2 = lo, hi2 = hi; if ( id == lo ) { lo2++; } else if ( id == hi ) { hi2--; } if ( lo2 >= hi2 ) { /* The range has collapsed... */ rc = mdb_cursor_del( cursor, MDB_NODUPDATA ); if ( rc != 0 ) { err = "c_del dup"; goto fail; } } else { /* position on lo */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); if ( id == lo ) data.mv_data = &lo2; else { /* position on hi */ rc = mdb_cursor_get( cursor, &key, &data, MDB_NEXT_DUP ); data.mv_data = &hi2; } /* Replace the current lo/hi */ data.mv_size = sizeof(ID); rc = mdb_cursor_put( cursor, &key, &data, MDB_CURRENT ); if ( rc != 0 ) { err = "c_put lo/hi"; goto fail; } } } } } else { /* initial c_get failed, nothing was done */ fail: if ( rc == MDB_NOTFOUND ) rc = 0; if ( rc ) { Debug( LDAP_DEBUG_ANY, "=> mdb_idl_delete_key: " "%s failed: %s (%d)\n", err, mdb_strerror(rc), rc ); break; } } } return rc; }
static int storeinfo(Wal *pWal, u64 currentTerm, u8 votedForSize, u8 *votedFor) { MDB_val key = {0,NULL}, data = {0,NULL}; int rc; #if ATOMIC db_thread *thr = g_tsd_thread; #else db_thread* thr = enif_tsd_get(g_tsd_thread); #endif mdbinf* mdb; #if ATOMIC if (!g_tsd_wmdb) lock_wtxn(thr->nEnv); mdb = g_tsd_wmdb; #else mdb = enif_tsd_get(g_tsd_wmdb); if (!mdb) lock_wtxn(thr->nEnv); mdb = enif_tsd_get(g_tsd_wmdb); #endif if (!mdb) return SQLITE_ERROR; key.mv_size = sizeof(u64); key.mv_data = &pWal->index; if (votedFor == NULL) { rc = mdb_cursor_get(mdb->cursorInfo,&key,&data,MDB_SET_KEY); if (rc == MDB_SUCCESS && data.mv_size >= (1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1)) { memcpy(¤tTerm, (u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2, sizeof(u64)); votedForSize = (u8)((u8*)data.mv_data)[1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)]; //votedFor = data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1; votedFor = alloca(votedForSize); memcpy(votedFor,(u8*)data.mv_data+1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1, votedForSize); // DBG("Voted for %.*s",(int)votedForSize,(char*)votedFor)); } } key.mv_size = sizeof(u64); key.mv_data = &pWal->index; data.mv_data = NULL; data.mv_size = 1+sizeof(u64)*6+sizeof(u32)*2+sizeof(u64)+1+votedForSize; rc = mdb_cursor_put(mdb->cursorInfo,&key,&data,MDB_RESERVE); if (rc == MDB_SUCCESS) { u8 *infoBuf = data.mv_data; infoBuf[0] = 1; memcpy(infoBuf+1, &pWal->firstCompleteTerm,sizeof(u64)); memcpy(infoBuf+1+sizeof(u64), &pWal->firstCompleteEvnum,sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*2, &pWal->lastCompleteTerm, sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*3, &pWal->lastCompleteEvnum,sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*4, &pWal->inProgressTerm, sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*5, &pWal->inProgressEvnum, sizeof(u64)); memcpy(infoBuf+1+sizeof(u64)*6, &pWal->mxPage, sizeof(u32)); memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32), &pWal->allPages,sizeof(u32)); memcpy(infoBuf+1+sizeof(u64)*6+sizeof(u32)*2, ¤tTerm, sizeof(u64)); infoBuf[1+sizeof(u64)*7+sizeof(u32)*2] = votedForSize; memcpy(infoBuf+2+sizeof(u64)*7+sizeof(u32)*2, votedFor, votedForSize); thr->forceCommit = 1; return SQLITE_OK; } return SQLITE_ERROR; }
static int mdb_dn2id_upgrade( BackendDB *be ) { struct mdb_info *mi = (struct mdb_info *) be->be_private; MDB_txn *mt; MDB_cursor *mc = NULL; MDB_val key, data; char *ptr; int rc, writes=0, depth=0; int enable_meter = 0; ID id = 0, *num, count = 0; rec *stack; lutil_meter_t meter; if (!(mi->mi_flags & MDB_NEED_UPGRADE)) { Debug( LDAP_DEBUG_ANY, "database %s: No upgrade needed.\n", be->be_suffix[0].bv_val, 0, 0 ); return 0; } { MDB_stat st; mdb_stat(mdb_cursor_txn(cursor), mi->mi_dbis[MDB_ID2ENTRY], &st); if (!st.ms_entries) { /* Empty DB, nothing to upgrade? */ return 0; } if (isatty(2)) enable_meter = !lutil_meter_open(&meter, &lutil_meter_text_display, &lutil_meter_linear_estimator, st.ms_entries); } num = ch_malloc(STACKSIZ * (sizeof(ID) + sizeof(rec))); stack = (rec *)(num + STACKSIZ); rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } key.mv_size = sizeof(ID); /* post-order depth-first update */ for(;;) { size_t dkids; unsigned char *ptr; /* visit */ key.mv_data = &id; stack[depth].id = id; rc = mdb_cursor_get(mc, &key, &data, MDB_SET); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } num[depth] = 1; rc = mdb_cursor_count(mc, &dkids); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_count failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } if (dkids > 1) { rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); down: ptr = (unsigned char *)data.mv_data + data.mv_size - sizeof(ID); memcpy(&id, ptr, sizeof(ID)); depth++; memcpy(stack[depth].rdn, data.mv_data, data.mv_size); stack[depth].len = data.mv_size; continue; } /* pop: write updated count, advance to next node */ pop: /* update superior counts */ if (depth) num[depth-1] += num[depth]; key.mv_data = &id; id = stack[depth-1].id; data.mv_data = stack[depth].rdn; data.mv_size = stack[depth].len; rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(BOTH) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } data.mv_data = stack[depth].rdn; ptr = (unsigned char *)data.mv_data + data.mv_size; memcpy(ptr, &num[depth], sizeof(ID)); data.mv_size += sizeof(ID); rc = mdb_cursor_del(mc, 0); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_del failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_put(mc, &key, &data, 0); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_put failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } count++; #if 1 if (enable_meter) lutil_meter_update(&meter, count, 0); #else { int len; ptr = data.mv_data; len = (ptr[0] & 0x7f) << 8 | ptr[1]; printf("ID: %zu, %zu, %.*s\n", stack[depth].id, num[depth], len, ptr+2); } #endif writes++; if (writes == 1000) { mdb_cursor_close(mc); rc = mdb_txn_commit(mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_txn_begin(mi->mi_dbenv, NULL, 0, &mt); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_begin(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_open(mt, mi->mi_dbis[MDB_DN2ID], &mc); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_open(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } rc = mdb_cursor_get(mc, &key, &data, MDB_GET_BOTH); if (rc) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_cursor_get(2) failed, %s (%d)\n", mdb_strerror(rc), rc, 0 ); goto leave; } writes = 0; } depth--; rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT_DUP); if (rc == 0) goto down; rc = 0; if (depth) goto pop; else break; } leave: mdb_cursor_close(mc); if (mt) { int r2; r2 = mdb_txn_commit(mt); if (r2) { Debug(LDAP_DEBUG_ANY, "mdb_dn2id_upgrade: mdb_txn_commit(2) failed, %s (%d)\n", mdb_strerror(r2), r2, 0 ); if (!rc) rc = r2; } } ch_free(num); if (enable_meter) { lutil_meter_update(&meter, count, 1); lutil_meter_close(&meter); } return rc; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data, sdata; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; long kval; char *sval; srand(time(NULL)); E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); E(mdb_stat(txn, dbi, &mst)); sval = calloc(1, mst.ms_psize / 4); key.mv_size = sizeof(long); key.mv_data = &kval; sdata.mv_size = mst.ms_psize / 4 - 30; sdata.mv_data = sval; printf("Adding 12 values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5; sprintf(sval, "%08x", kval); data = sdata; (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } printf("Adding 12 more values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5+4; sprintf(sval, "%08x", kval); data = sdata; (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } printf("Adding 12 more values, should yield 3 splits\n"); for (i=0;i<12;i++) { kval = i*5+1; sprintf(sval, "%08x", kval); data = sdata; (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); } E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); do { printf("key: %p %s, data: %p %.*s\n", key.mv_data, mdb_dkey(&key, dkbuf), data.mv_data, (int) data.mv_size, (char *) data.mv_data); } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_commit(txn); #if 0 j=0; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(kval, "%03x", values[i & ~0x0f]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); mdb_dbi_close(env, dbi); #endif mdb_env_close(env); return 0; }
int main(int argc,char * argv[]) { int i = 0, j = 0, rc; MDB_env *env; MDB_dbi dbi; MDB_val key, data; MDB_txn *txn; MDB_stat mst; MDB_cursor *cursor; int count; int *values; char sval[32]; char kval[sizeof(int)]; srand(time(NULL)); memset(sval, 0, sizeof(sval)); count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i<count;i++) { values[i] = rand()%1024; } E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); E(mdb_env_set_maxdbs(env, 4)); E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); E(mdb_txn_begin(env, NULL, 0, &txn)); E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi)); E(mdb_cursor_open(txn, dbi, &cursor)); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i<count;i++) { if (!(i & 0x0f)) sprintf(kval, "%03x", values[i]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); if (RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA))) j++; } if (j) printf("%d duplicates skipped\n", j); mdb_cursor_close(cursor); E(mdb_txn_commit(txn)); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int) key.mv_size, (char *) key.mv_data, data.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_txn_abort(txn); j=0; for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); sprintf(kval, "%03x", values[i & ~0x0f]); sprintf(sval, "%03x %d foo bar", values[i], values[i]); key.mv_size = sizeof(int); key.mv_data = kval; data.mv_size = sizeof(sval); data.mv_data = sval; if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { j--; mdb_txn_abort(txn); } else { E(mdb_txn_commit(txn)); } } free(values); printf("Deleted %d values\n", j); E(mdb_env_stat(env, &mst)); E(mdb_txn_begin(env, NULL, 1, &txn)); E(mdb_cursor_open(txn, dbi, &cursor)); printf("Cursor next\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); printf("Cursor prev\n"); while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { printf("key: %.*s, data: %.*s\n", (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); } CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); mdb_cursor_close(cursor); mdb_close(env, dbi); mdb_txn_abort(txn); mdb_env_close(env); return 0; }
int main(int argc, char *argv[]) { int i, rc; MDB_env *env; MDB_txn *txn; MDB_cursor *mc; MDB_dbi dbi; char *envname; int envflags = 0, putflags = 0; int dohdr = 0; prog = argv[0]; if (argc < 2) { usage(); } /* -f: load file instead of stdin * -n: use NOSUBDIR flag on env_open * -s: load into named subDB * -N: use NOOVERWRITE on puts * -T: read plaintext * -V: print version and exit */ while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { switch(i) { case 'V': printf("%s\n", MDB_VERSION_STRING); exit(0); break; case 'f': if (freopen(optarg, "r", stdin) == NULL) { fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno)); exit(EXIT_FAILURE); } break; case 'n': envflags |= MDB_NOSUBDIR; break; case 's': subname = strdup(optarg); break; case 'N': putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; break; case 'T': mode |= NOHDR | PRINT; break; default: usage(); } } if (optind != argc - 1) usage(); dbuf.mv_size = 4096; dbuf.mv_data = malloc(dbuf.mv_size); if (!(mode & NOHDR)) readhdr(); envname = argv[optind]; rc = mdb_env_create(&env); if (rc) { fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); return EXIT_FAILURE; } mdb_env_set_maxdbs(env, 2); if (info.me_maxreaders) mdb_env_set_maxreaders(env, info.me_maxreaders); if (info.me_mapsize) mdb_env_set_mapsize(env, info.me_mapsize); if (info.me_mapaddr) envflags |= MDB_FIXEDMAP; rc = mdb_env_open(env, envname, envflags, 0664); if (rc) { fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; kbuf.mv_data = malloc(kbuf.mv_size); while(!Eof) { MDB_val key, data; int batch = 0; flags = 0; if (!dohdr) { dohdr = 1; } else if (!(mode & NOHDR)) readhdr(); rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc) { fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); if (rc) { fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } rc = mdb_cursor_open(txn, dbi, &mc); if (rc) { fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } while(1) { rc = readline(&key, &kbuf); if (rc) /* rc == EOF */ break; rc = readline(&data, &dbuf); if (rc) { fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno); goto txn_abort; } rc = mdb_cursor_put(mc, &key, &data, putflags); if (rc == MDB_KEYEXIST && putflags) continue; if (rc) { fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } batch++; if (batch == 100) { rc = mdb_txn_commit(txn); if (rc) { fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc) { fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } rc = mdb_cursor_open(txn, dbi, &mc); if (rc) { fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } batch = 0; } } rc = mdb_txn_commit(txn); txn = NULL; if (rc) { fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } mdb_dbi_close(env, dbi); } txn_abort: mdb_txn_abort(txn); env_close: mdb_env_close(env); return rc ? EXIT_FAILURE : EXIT_SUCCESS; }
/* mc must have been set by mdb_dn2id */ int mdb_dn2id_delete( Operation *op, MDB_cursor *mc, ID id, ID nsubs ) { ID nid; char *ptr; int rc; Debug( LDAP_DEBUG_TRACE, "=> mdb_dn2id_delete 0x%lx\n", id ); /* Delete our ID from the parent's list */ rc = mdb_cursor_del( mc, 0 ); /* Delete our ID from the tree. With sorted duplicates, this * will leave any child nodes still hanging around. This is OK * for modrdn, which will add our info back in later. */ if ( rc == 0 ) { MDB_val key, data; if ( nsubs ) { mdb_cursor_get( mc, &key, NULL, MDB_GET_CURRENT ); memcpy( &nid, key.mv_data, sizeof( ID )); } key.mv_size = sizeof(ID); key.mv_data = &id; rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); if ( rc == 0 ) rc = mdb_cursor_del( mc, 0 ); } /* Delete our subtree count from all superiors */ if ( rc == 0 && nsubs && nid ) { MDB_val key, data; ID subs; key.mv_data = &nid; key.mv_size = sizeof( ID ); do { rc = mdb_cursor_get( mc, &key, &data, MDB_SET ); if ( !rc ) { char *p2; diskNode *d; int rlen; ptr = (char *)data.mv_data + data.mv_size - sizeof( ID ); memcpy( &nid, ptr, sizeof( ID )); /* Get parent's node under grandparent */ d = data.mv_data; rlen = ( d->nrdnlen[0] << 8 ) | d->nrdnlen[1]; p2 = op->o_tmpalloc( rlen + 2, op->o_tmpmemctx ); memcpy( p2, data.mv_data, rlen+2 ); *p2 ^= 0x80; data.mv_data = p2; rc = mdb_cursor_get( mc, &key, &data, MDB_GET_BOTH ); op->o_tmpfree( p2, op->o_tmpmemctx ); if ( !rc ) { /* Get parent's subtree count */ ptr = (char *)data.mv_data + data.mv_size - sizeof( ID ); memcpy( &subs, ptr, sizeof( ID )); subs -= nsubs; p2 = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); memcpy( p2, data.mv_data, data.mv_size - sizeof( ID )); memcpy( p2+data.mv_size - sizeof( ID ), &subs, sizeof( ID )); data.mv_data = p2; rc = mdb_cursor_put( mc, &key, &data, MDB_CURRENT ); op->o_tmpfree( p2, op->o_tmpmemctx ); } } if ( rc ) break; } while ( nid ); } Debug( LDAP_DEBUG_TRACE, "<= mdb_dn2id_delete 0x%lx: %d\n", id, rc ); return rc; }
int lmdbgo_mdb_cursor_put2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, unsigned int flags) { MDB_val key, val; LMDBGO_SET_VAL(&key, kn, kdata); LMDBGO_SET_VAL(&val, vn, vdata); return mdb_cursor_put(cur, &key, &val, flags); }
FreqSchedulerError freq_scheduler_cursor_write(FreqScheduler *sch, MDB_cursor *cursor, uint64_t hash, float freq) { if (freq <= 0) return 0; ScheduleKey sk = { .score = 0, .hash = hash }; MDB_val key = { .mv_size = sizeof(sk), .mv_data = &sk, }; MDB_val val = { .mv_size = sizeof(float), .mv_data = &freq, }; int mdb_rc; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "adding page to schedule"); freq_scheduler_add_error(sch, mdb_strerror(mdb_rc)); } return sch->error->code; } FreqSchedulerError freq_scheduler_load_simple(FreqScheduler *sch, float freq_default, float freq_scale) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; HashInfoStream *st; if (hashinfo_stream_new(&st, sch->page_db) != 0) { error1 = "creating stream"; error2 = st? sch->page_db->error->message: "NULL"; goto on_error; } if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; StreamState ss; uint64_t hash; PageInfo *pi; while ((ss = hashinfo_stream_next(st, &hash, &pi)) == stream_state_next) { if ((pi->n_crawls > 0) && ((sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls)) && !page_info_is_seed(pi)){ float freq = freq_default; if (freq_scale > 0) { float rate = page_info_rate(pi); if (rate > 0) { freq = freq_scale * rate; } } if (freq_scheduler_cursor_write(sch, cursor, hash, freq) != 0) goto on_error; } page_info_delete(pi); } if (ss != stream_state_end) { error1 = "incorrect stream state"; error2 = 0; hashinfo_stream_delete(st); goto on_error; } hashinfo_stream_delete(st); if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_load_mmap(FreqScheduler *sch, MMapArray *freqs) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; if (txn_manager_expand( sch->txn_manager, 2*freqs->n_elements*freqs->element_size) != 0) { error1 = "resizing database"; error2 = sch->txn_manager->error->message; goto on_error; } if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; for (size_t i=0; i<freqs->n_elements; ++i) { PageFreq *f = mmap_array_idx(freqs, i); ScheduleKey sk = { .score = 1.0/f->freq, .hash = f->hash }; MDB_val key = { .mv_size = sizeof(sk), .mv_data = &sk, }; MDB_val val = { .mv_size = sizeof(float), .mv_data = &f->freq, }; int mdb_rc; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { error1 = "adding page to schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_request(FreqScheduler *sch, size_t max_requests, PageRequest **request) { char *error1 = 0; char *error2 = 0; MDB_cursor *cursor = 0; if (freq_scheduler_cursor_open(sch, &cursor) != 0) goto on_error; PageRequest *req = *request = page_request_new(max_requests); if (!req) { error1 = "allocating memory"; goto on_error; } int interrupt_requests = 0; while ((req->n_urls < max_requests) && !interrupt_requests) { MDB_val key; MDB_val val; ScheduleKey sk; float freq; int mdb_rc; int crawl = 0; switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, MDB_FIRST)) { case 0: // copy data before deleting cursor sk = *(ScheduleKey*)key.mv_data; freq = *(float*)val.mv_data; PageInfo *pi = 0; if (page_db_get_info(sch->page_db, sk.hash, &pi) != 0) { error1 = "retrieving PageInfo from PageDB"; error2 = sch->page_db->error->message; goto on_error; } if (pi) { if (sch->margin >= 0) { double elapsed = difftime(time(0), 0) - pi->last_crawl; if (elapsed < 1.0/(freq*(1.0 + sch->margin))) interrupt_requests = 1; } crawl = (sch->max_n_crawls == 0) || (pi->n_crawls < sch->max_n_crawls); } if (!interrupt_requests) { if ((mdb_rc = mdb_cursor_del(cursor, 0)) != 0) { error1 = "deleting head of schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } if (crawl) { if (page_request_add_url(req, pi->url) != 0) { error1 = "adding url to request"; goto on_error; } sk.score += 1.0/freq; val.mv_data = &freq; key.mv_data = &sk; if ((mdb_rc = mdb_cursor_put(cursor, &key, &val, 0)) != 0) { error1 = "moving element inside schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } } page_info_delete(pi); break; case MDB_NOTFOUND: // no more pages left interrupt_requests = 1; break; default: error1 = "getting head of schedule"; error2 = mdb_strerror(mdb_rc); goto on_error; } } if (freq_scheduler_cursor_commit(sch, cursor) != 0) goto on_error; return sch->error->code; on_error: freq_scheduler_cursor_abort(sch, cursor); freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, error1); freq_scheduler_add_error(sch, error2); return sch->error->code; } FreqSchedulerError freq_scheduler_add(FreqScheduler *sch, const CrawledPage *page) { if (page_db_add(sch->page_db, page, 0) != 0) { freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "adding crawled page"); freq_scheduler_add_error(sch, sch->page_db->error->message); } return sch->error->code; } void freq_scheduler_delete(FreqScheduler *sch) { mdb_env_close(sch->txn_manager->env); (void)txn_manager_delete(sch->txn_manager); if (!sch->persist) { char *data = build_path(sch->path, "data.mdb"); char *lock = build_path(sch->path, "lock.mdb"); remove(data); remove(lock); free(data); free(lock); remove(sch->path); } free(sch->path); error_delete(sch->error); free(sch); } FreqSchedulerError freq_scheduler_dump(FreqScheduler *sch, FILE *output) { MDB_cursor *cursor; if (freq_scheduler_cursor_open(sch, &cursor) != 0) return sch->error->code; int end = 0; MDB_cursor_op cursor_op = MDB_FIRST; do { int mdb_rc; MDB_val key; MDB_val val; ScheduleKey *key_data; float *val_data; switch (mdb_rc = mdb_cursor_get(cursor, &key, &val, cursor_op)) { case 0: key_data = (ScheduleKey*)key.mv_data; val_data = (float*)val.mv_data; fprintf(output, "%.2e %016"PRIx64" %.2e\n", key_data->score, key_data->hash, *val_data); break; case MDB_NOTFOUND: end = 1; break; default: freq_scheduler_set_error(sch, freq_scheduler_error_internal, __func__); freq_scheduler_add_error(sch, "iterating over database"); freq_scheduler_add_error(sch, mdb_strerror(mdb_rc)); end = 1; break; } cursor_op = MDB_NEXT; } while (!end); freq_scheduler_cursor_abort(sch, cursor); return sch->error->code; }
int main(int argc,char * argv[]) { int rc; MDB_env *env; MDB_txn *txn; MDB_dbi db; rc = mdb_env_create(&env); if (rc != 0) { return rc; } rc = mdb_env_set_mapsize(env, 107374182400); //100G if (rc != 0) { return rc; } rc = mdb_env_set_maxdbs(env, 27); if (rc != 0) { return rc; } rc = mdb_env_open(env, "/home/pieter/Downloads/thundergraph/src/main/native/testdb", MDB_NOSYNC, 0664); if (rc != 0) { return rc; } rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_open(txn, "testdb", MDB_CREATE | MDB_INTEGERKEY, &db); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_set_compare(txn, db, compareTestDbId); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_txn_commit(txn); if (rc != 0) { mdb_txn_abort(txn); return rc; } rc = mdb_txn_begin(env, NULL, 0, &txn); if (rc != 0) { mdb_txn_abort(txn); return rc; } int i= 0; for (i = 0; i < 1000; i++) { MDB_cursor *cursor; rc = mdb_cursor_open(txn, db, &cursor); if (rc != 0) { printf("open cursor failure = %i!\n", rc); goto fail; } //Create a core vertex MDB_val key, data; TestDbId testDbId; testDbId.testId = i; testDbId.coreOrPropertyEnum = 0; testDbId.labelId = -1; testDbId.propertykeyId = -1; testDbId.edgeId = -1LL; key.mv_size = sizeof(TestDbId); key.mv_data = &testDbId; data.mv_size = 50; char *value = malloc(5); char v[] = "12345"; memcpy(value, v, 5); data.mv_data = &v; mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE); free(value); //Create a vertex property MDB_val propertyKey, propertyData; TestDbId propertyTestDbId; propertyTestDbId.testId = i; propertyTestDbId.coreOrPropertyEnum = 1; propertyTestDbId.labelId = -1; propertyTestDbId.propertykeyId = -1; propertyTestDbId.edgeId = -1LL; propertyKey.mv_size = sizeof(TestDbId); propertyKey.mv_data = &propertyTestDbId; data.mv_size = 50; char *propertyValue = malloc(5); char propertyV[] = "12345"; memcpy(propertyValue, propertyV, 5); data.mv_data = &propertyV; mdb_cursor_put(cursor, &propertyKey, &propertyData, MDB_NOOVERWRITE); free(propertyValue); //Create a vertex in edge MDB_val inEdgeKey, inEdgeData; TestDbId inEdgeTestDbId; inEdgeTestDbId.testId = i; inEdgeTestDbId.coreOrPropertyEnum = 2; inEdgeTestDbId.labelId = -1; inEdgeTestDbId.propertykeyId = -1; inEdgeTestDbId.edgeId = -1LL; inEdgeKey.mv_size = sizeof(TestDbId); inEdgeKey.mv_data = &inEdgeTestDbId; data.mv_size = 50; char *inEdgeValue = malloc(5); char inEdgeV[] = "12345"; memcpy(inEdgeValue, inEdgeV, 5); data.mv_data = &inEdgeV; mdb_cursor_put(cursor, &inEdgeKey, &inEdgeData, MDB_NOOVERWRITE); mdb_cursor_close(cursor); free(inEdgeValue); } MDB_cursor *cursor; mdb_cursor_close(cursor); rc = mdb_cursor_open(txn, db, &cursor); if (rc != 0) { printf("open cursor failure = %i!\n", rc); goto fail; } MDB_val key, data; for (i = 0; i < 1000; i++) { TestDbId testDbId; testDbId.testId = i; testDbId.coreOrPropertyEnum = 0; testDbId.labelId = -1; testDbId.propertykeyId = -1; testDbId.edgeId = -1LL; key.mv_size = sizeof(TestDbId); key.mv_data = &testDbId; rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY); printf("mdb_cursor_get index = %i result = %i\n", i, rc); MDB_val keyDelete, dataDelete; TestDbId deleteTestDbId; deleteTestDbId.testId = i; deleteTestDbId.coreOrPropertyEnum = 0; deleteTestDbId.labelId = -1; deleteTestDbId.propertykeyId = -1; deleteTestDbId.edgeId = -1LL; keyDelete.mv_size = sizeof(TestDbId); keyDelete.mv_data = &deleteTestDbId; rc = mdb_del(txn, db, &keyDelete, &dataDelete); printf("mdb_del = %i\n", rc); MDB_val propertyKeyDelete, propertyDataDelete; TestDbId propertyDeleteTestDbId; propertyDeleteTestDbId.testId = i; propertyDeleteTestDbId.coreOrPropertyEnum = 1; propertyDeleteTestDbId.labelId = -1; propertyDeleteTestDbId.propertykeyId = -1; propertyDeleteTestDbId.edgeId = -1LL; propertyKeyDelete.mv_size = sizeof(TestDbId); propertyKeyDelete.mv_data = &propertyDeleteTestDbId; rc = mdb_del(txn, db, &propertyKeyDelete, &propertyDataDelete); printf("mdb_del = %i\n", rc); MDB_val inEdgeKeyDelete, inEdgeDataDelete; TestDbId inEdgeDeleteTestDbId; inEdgeDeleteTestDbId.testId = i; inEdgeDeleteTestDbId.coreOrPropertyEnum = 2; inEdgeDeleteTestDbId.labelId = -1; inEdgeDeleteTestDbId.propertykeyId = -1; inEdgeDeleteTestDbId.edgeId = -1LL; inEdgeKeyDelete.mv_size = sizeof(TestDbId); inEdgeKeyDelete.mv_data = &inEdgeDeleteTestDbId; rc = mdb_del(txn, db, &inEdgeKeyDelete, &inEdgeDataDelete); printf("mdb_del = %i\n", rc); } mdb_cursor_close(cursor); rc = mdb_cursor_open(txn, db, &cursor); if (rc != 0) { printf("open cursor failure = %i!\n", rc); goto fail; } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { TestDbId testDbId = *((TestDbId *) (key.mv_data)); printf("keyId = %llu\n", testDbId.testId); } mdb_cursor_close(cursor); fail: mdb_close(env, db); mdb_env_close(env); printf("closing graph!\n"); return 0; }
int lmdbgo_mdb_cursor_put1(MDB_cursor *cur, char *kdata, size_t kn, MDB_val *val, unsigned int flags) { MDB_val key; LMDBGO_SET_VAL(&key, kn, kdata); return mdb_cursor_put(cur, &key, val, flags); }