/*--------------------------------------------------------------* * ss_m::_create_mrbt_file() * *--------------------------------------------------------------*/ rc_t ss_m::_create_mrbt_file(vid_t vid, stid_t& fid, store_property_t property, shpid_t cluster_hint // = 0 ) { FUNC(ss_m::_create_mrbt_file); DBG( << "Attempting to create a file on volume " << vid.vol ); store_flag_t st_flag = _make_store_flag(property); extnum_t first_extent = extnum_t(cluster_hint? cluster_hint / ss_m::ext_sz : 0); DBGTHRD(<<"about to create a store starting about extent " << first_extent); W_DO( io->create_store(vid, 100/*unused*/, st_flag, fid, first_extent) ); DBGTHRD(<<"created first store " << fid << " now create 2nd..."); /* // create a store for holding large record pages // always allocates 1 extent -- otherwise // asserts fail elsewhere // If this fails, we have to by-hand back out the creation // of the first store */ stid_t lg_stid; w_rc_t rc= io->create_store(vid, 100/*unused*/, st_flag, lg_stid, first_extent, 1); if(rc.is_error()) { // it would be a problem if this didn't work, but // if all else fails, abort should work. DBGTHRD(<<"2nd create failed; destroying first= " << fid); W_DO( io->destroy_store(fid) ); return rc; }
// scans the heap file the records are created rc_t smthread_scanner_t::scan_the_file() { cout << "Scanning file " << _fid << endl; W_DO(ssm->begin_xct()); scan_file_i scan(_fid); pin_i* cursor(NULL); bool eof(false); int i(0); do { w_rc_t rc = scan.next(cursor, 0, eof); if(rc.is_error()) { cerr << "Error getting next: " << rc << endl; retval = rc.err_num(); return rc; } if(eof) break; cout << "Record " << i << "/" << _num_rec << " Rid " << cursor->rid() << endl; vec_t header (cursor->hdr(), cursor->hdr_size()); int hdrcontents; header.copy_to(&hdrcontents, sizeof(hdrcontents)); cout << "Record hdr " << hdrcontents << endl; const char *body = cursor->body(); cout << "Record body " << body << endl; cout << "Record body size " << cursor->body_size() << endl; i++; } while (!eof); W_DO(ssm->commit_xct()); return RCOK; }
rc_t lg_tag_chunks_h::update(uint4_t start_byte, const vec_t& data) const { FUNC(lg_tag_chunks_h::update); uint4_t amount; // amount to update on page uint4_t pid_count = start_byte/lgdata_p::data_sz; // first page uint4_t data_size = data.size(); lpid_t curr_pid(_page.pid().vol(), _cref.store, 0); uint4_t offset = start_byte%lgdata_p::data_sz; uint4_t num_bytes = 0; while (num_bytes < data_size) { amount = MIN(lgdata_p::data_sz-offset, data_size-num_bytes); curr_pid.page = _pid(pid_count); lgdata_p lgdata; W_DO( lgdata.fix(curr_pid, LATCH_EX) ); W_DO(lgdata.update(offset, data, num_bytes, amount)); offset = 0; num_bytes += amount; pid_count++; } // verify last page touched is same as calculated last page w_assert9(pid_count-1 == (start_byte+data.size()-1)/lgdata_p::data_sz); return RCOK; }
rc_t btree_impl::_sx_adopt_foster_all_core ( btree_page_h &parent, bool is_root, bool recursive) { // TODO this should use the improved tree-walk-through // See jira ticket:60 "Tree walk-through without more than 2 pages latched" (originally trac ticket:62) w_assert1 (xct()->is_sys_xct()); w_assert1 (parent.is_fixed()); w_assert1 (parent.latch_mode() == LATCH_EX); if (parent.is_node()) { w_assert1(parent.pid0()); W_DO(_sx_adopt_foster_sweep(parent)); if (recursive) { // also adopt at all children recursively for (int i = -1; i < parent.nrecs(); ++i) { btree_page_h child; PageID shpid_opaqueptr = i == -1 ? parent.get_foster_opaqueptr() : parent.child_opaqueptr(i); W_DO(child.fix_nonroot(parent, shpid_opaqueptr, LATCH_EX)); W_DO(_sx_adopt_foster_all_core(child, false, true)); } } } // after all adopts, if this parent is the root and has foster, // let's grow the tree if (is_root && parent.get_foster()) { W_DO(_sx_grow_tree(parent)); W_DO(_sx_adopt_foster_sweep(parent)); } w_assert3(parent.is_consistent(true, true)); return RCOK; }
rc_t smthread_user_t::do_work() { if (_initialize_device) W_DO(do_init()); else W_DO(no_init()); return RCOK; }
rc_t btree_impl::_sx_norec_alloc(btree_page_h &page, PageID &new_page_id) { sys_xct_section_t sxs(true); W_DO(sxs.check_error_on_start()); rc_t ret = _ux_norec_alloc_core(page, new_page_id); W_DO (sxs.end_sys_xct (ret)); return ret; }
// looks up file info in the root index rc_t smthread_main_t::find_file_info() { file_info_t info; W_DO(ssm->begin_xct()); bool found; stid_t _root_iid; W_DO(ss_m::vol_root_index(_vid, _root_iid)); smsize_t info_len = sizeof(info); const vec_t key_vec_tmp(file_info_t::key, strlen(file_info_t::key)); W_DO(ss_m::find_assoc(_root_iid, key_vec_tmp, &info, info_len, found)); if (!found) { cerr << "No file information found" <<endl; return RC(fcASSERT); } else { cerr << " found assoc " << file_info_t::key << " --> " << info << endl; } W_DO(ssm->commit_xct()); _first_rid = info.first_rid; _last_rid = info.last_rid; _fid = info.fid; _rec_size = info.rec_size; _num_rec = info.num_rec; return RCOK; }
rc_t ShoreYCSBEnv::xct_populate_db(const int /* xct_id */, populate_db_input_t& pin) { assert (_pssm); assert (_initialized); tuple_guard<ycsbtable_man_impl> tuple(ycsbtable_man); rep_row_t areprow(ycsbtable_man->ts()); rep_row_t areprowkey(ycsbtable_man->ts()); areprow.set(ycsbtable_man->table()->maxsize()); areprowkey.set(ycsbtable_man->table()->maxsize()); tuple->_rep = &areprow; tuple->_rep_key = &areprowkey; W_DO(db()->begin_xct()); uint64_t key = pin.firstKey; char field[FieldSize]; for (unsigned i = 0; i < pin.count; i++) { tuple->set_value(0, key); for (int j = 1; j <= FieldCount; j++) { fill_value(field); tuple->set_value(j, field); } W_DO(ycsbtable_man->add_tuple(_pssm, tuple)); key++; } W_DO(db()->commit_xct()); return RCOK; }
rc_t smthread_user_t::no_init() { cout << "Using already-existing device: " << _device_name << endl; // mount already existing device devid_t devid; u_int vol_cnt; w_rc_t rc = ssm->mount_dev(_device_name, vol_cnt, devid); if (rc.is_error()) { cerr << "Error: could not mount device: " << _device_name << endl; cerr << " Did you forget to run the server with -i?" << endl; return rc; } // find ID of the volume on the device lvid_t* lvid_list; u_int lvid_cnt; W_DO(ssm->list_volumes(_device_name, lvid_list, lvid_cnt)); if (lvid_cnt == 0) { cerr << "Error, device has no volumes" << endl; exit(1); } _lvid = lvid_list[0]; delete [] lvid_list; W_COERCE(find_file_info()); W_COERCE(scan_the_root_index()); W_DO(scan_the_file()); return RCOK; }
rc_t smthread_main_t::do_work() { if (_initialize_device) { W_DO(do_init()); } W_DO(no_init()); return RCOK; }
w_rc_t ShoreTPCBEnv::load_and_register_fids() { W_DO(branch_man->load_and_register_fid(db())); W_DO(teller_man->load_and_register_fid(db())); W_DO(account_man->load_and_register_fid(db())); W_DO(history_man->load_and_register_fid(db())); return (RCOK); }
/*--------------------------------------------------------------* * ss_m::_create_index() * *--------------------------------------------------------------*/ rc_t ss_m::_create_index( vid_t vid, ndx_t ntype, store_property_t property, const char* key_desc, concurrency_t cc, // = t_cc_kvl stid_t& stid ) { FUNC(ss_m::_create_index); DBG(<<" vid " << vid); uint4_t count = max_keycomp; key_type_s kcomp[max_keycomp]; lpid_t root; W_DO( key_type_s::parse_key_type(key_desc, count, kcomp) ); { DBG(<<"vid " << vid); W_DO( io->create_store(vid, 100/*unused*/, _make_store_flag(property), stid) ); DBG(<<" stid " << stid); } // Note: theoretically, some other thread could destroy // the above store before the following lock request // is granted. The only forseable way for this to // happen would be due to a bug in a vas causing // it to destroy the wrong store. We make no attempt // to prevent this. W_DO(lm->lock(stid, EX, t_long, WAIT_SPECIFIED_BY_XCT)); if( (cc != t_cc_none) && (cc != t_cc_file) && (cc != t_cc_kvl) && (cc != t_cc_modkvl) && (cc != t_cc_im) ) return RC(eBADCCLEVEL); switch (ntype) { case t_btree: case t_uni_btree: // compress prefixes only if the first part is compressed W_DO( bt->create(stid, root, kcomp[0].compressed != 0) ); break; default: return RC(eBADNDXTYPE); } sinfo_s sinfo(stid.store, t_index, 100/*unused*/, ntype, cc, root.page, count, kcomp); W_DO( dir->insert(stid, sinfo) ); return RCOK; }
w_rc_t table_desc_t::load_stids() { w_assert0(_db); stid_t cat_stid = get_catalog_stid(); W_DO(_primary_idx->load_stid(_db, cat_stid)); for (size_t i = 0; i < _indexes.size(); i++) { W_DO(_indexes[i]->load_stid(_db, cat_stid)); } return RCOK; }
// creates records and their associations for a regular mrbt design rc_t smthread_creator_t::fill_the_file_regular() { int num_inserted = 1; W_DO(ssm->begin_xct()); char* dummy = new char[_rec_size]; memset(dummy, '\0', _rec_size); vec_t data(dummy, _rec_size); rid_t rid; for(int j=_start_key; j <= _end_key; j++, num_inserted++) { if( j == _end_key && (_end_key != _num_rec || _test_no != 2)) { continue; // for test 2 we want to insert one more record and // it is the reason for all the weird continues here TODO: try to make this better } { w_ostrstream o(dummy, _rec_size); o << j << ends; w_assert1(o.c_str() == dummy); } // header contains record # int i = j; const vec_t hdr(&i, sizeof(i)); W_COERCE(ssm->create_rec(_fid, hdr, _rec_size, data, rid, _bIgnoreLocks)); cout << "Created rec " << j << " rid:" << rid << endl; if (j == 0) { _first_rid = rid; if(_test_no == 2) { continue; } } else if(j == _num_rec) { _last_rid = rid; continue; } el_filler_utest eg; vec_t key(&i, sizeof(i)); vec_t el((char*)(&rid), sizeof(rid_t)); eg._el.put(el); W_DO(ssm->create_mr_assoc(_index_id, key, eg, _bIgnoreLocks, _bIgnoreLatches)); cout << "Created assoc " << j << endl; // if we want to insert a lot of records then we run out of log space // to avoid it, we should flush the log after inserting some number of records if(num_inserted >= 20000) { W_DO(ssm->commit_xct()); num_inserted = 0; W_DO(ssm->begin_xct()); } } cout << "Created all. First rid " << _first_rid << " Last rid " << _last_rid << endl; delete [] dummy; W_DO(ssm->commit_xct()); return RCOK; }
// prints the btree rc_t smthread_main_t::print_the_index() { cout << "printing the mr index from store " << _index_id << endl; W_DO(ssm->begin_xct()); if(_design_no == 0) { W_DO(ssm->print_index(_index_id)); } else { W_DO(ssm->print_mr_index(_index_id)); } W_DO(ssm->commit_xct()); return RCOK; }
rc_t btree_impl::_ux_norec_alloc_core(btree_page_h &page, PageID &new_page_id) { // This is called only in REDO-only SSX, so no compensation logging. Just apply. w_assert1 (xct()->is_single_log_sys_xct()); w_assert1 (page.latch_mode() == LATCH_EX); W_DO(smlevel_0::vol->alloc_a_page(new_page_id)); btree_page_h new_page; w_rc_t rc; rc = new_page.fix_nonroot(page, new_page_id, LATCH_EX, false, true); if (rc.is_error()) { // if failed for any reason, we release the allocated page. W_DO(smlevel_0::vol ->deallocate_page(new_page_id)); return rc; } // The new page has an empty key range; parent's high to high. w_keystr_t fence, chain_high; page.copy_fence_high_key(fence); bool was_right_most = (page.get_chain_fence_high_length() == 0); page.copy_chain_fence_high_key(chain_high); if (was_right_most) { // this means there was no chain or the page was the right-most of it. // (so its high=high of chain) // upon the first foster split, we start setting the chain-high. page.copy_fence_high_key(chain_high); } #if W_DEBUG_LEVEL >= 3 lsn_t old_lsn = page.get_page_lsn(); #endif //W_DEBUG_LEVEL W_DO(log_btree_norec_alloc(page, new_page, new_page_id, fence, chain_high)); DBGOUT3(<< "btree_impl::_ux_norec_alloc_core, fence=" << fence << ", old-LSN=" << old_lsn << ", new-LSN=" << page.get_page_lsn() << ", PID=" << new_page_id); // initialize as an empty child: new_page.format_steal(page.get_page_lsn(), new_page_id, page.store(), page.root(), page.level(), 0, lsn_t::null, page.get_foster_opaqueptr(), page.get_foster_emlsn(), fence, fence, chain_high, false); page.accept_empty_child(page.get_page_lsn(), new_page_id, false /*not from redo*/); // in this operation, the log contains everything we need to recover without any // write-order-dependency. So, no registration for WOD. w_assert3(new_page.is_consistent(true, true)); w_assert1(new_page.is_fixed()); w_assert1(new_page.latch_mode() == LATCH_EX); w_assert3(page.is_consistent(true, true)); w_assert1(page.is_fixed()); return RCOK; }
rc_t btree_impl::_ux_lock_range(const StoreID& stid, btree_page_h& leaf, const void* keystr, size_t keylen, slotid_t slot, latch_mode_t latch_mode, const okvl_mode& exact_hit_lock_mode, const okvl_mode& miss_lock_mode, bool check_only) { w_assert1(slot >= -1 && slot <= leaf.nrecs()); w_assert1(exact_hit_lock_mode.get_gap_mode() == okvl_mode::N); w_assert1(miss_lock_mode.is_keylock_empty()); if (slot == -1) { // this means we should search it again bool found; leaf.search((const char *) keystr, keylen, found, slot); w_assert1(!found); // precondition } w_assert1(slot >= 0 && slot <= leaf.nrecs()); #if W_DEBUG_LEVEL > 1 w_keystr_t key, key_at_slot; key.construct_from_keystr(keystr, keylen); if (slot<leaf.nrecs()) { leaf.get_key(slot, key_at_slot); w_assert1(key_at_slot.compare(key)>0); } #endif // W_DEBUG_LEVEL > 1 slot--; // want range lock from previous key if (slot == -1 && w_keystr_t::compare_bin_str(keystr, keylen, leaf.get_fence_low_key(), leaf.get_fence_low_length()) == 0) { // We were searching for the low-fence key! then, we take key lock on it and // subsequent structural modification (e.g., merge) will add the low-fence as // ghost record to be aware of the lock. W_DO (_ux_lock_key(stid, leaf, leaf.get_fence_low_key(), leaf.get_fence_low_length(), latch_mode, exact_hit_lock_mode, check_only)); } else { w_keystr_t prevkey; if (slot == -1) { leaf.copy_fence_low_key(prevkey); } else { leaf.get_key(slot, prevkey); } #if W_DEBUG_LEVEL > 1 w_assert1(prevkey.compare(key) < 0); #endif // W_DEBUG_LEVEL > 1 W_DO (_ux_lock_key(stid, leaf, prevkey, latch_mode, miss_lock_mode, check_only)); } return RCOK; }
w_rc_t parallel_locks(ss_m*, test_volume_t *) { lock_worker_t workers[THREAD_COUNT]; for (int i = 0; i < THREAD_COUNT; ++i) { W_DO(workers[i].fork()); } for (int i = 0; i < THREAD_COUNT; ++i) { W_DO(workers[i].join()); EXPECT_FALSE(workers[i]._running) << i; EXPECT_FALSE(workers[i]._rc.is_error()) << i; } return RCOK; }
rc_t log_alloc_file_page(const lpid_t& pid, const lsn_t& rec_lsn) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled; if (should_log) { logrec_t* logrec; // fudge 2.4 W_DO(xd->get_logbuf(logrec, t_alloc_file_page)); new (logrec) alloc_file_page_log(pid, rec_lsn); W_DO(xd->give_logbuf(logrec)); } return RCOK; }
w_rc_t dosome(ss_m* ssm, test_volume_t *test_volume) { StoreID stid; PageID root_pid; W_DO(x_btree_create_index(ssm, test_volume, stid, root_pid)); w_keystr_t key; // XXXX should move the following out to a separate DataGen class typedef unsigned int size_t; size_t const domain = 100000; size_t const records = 100000; size_t produced = 0; // produced output records int ibuffer; // hold the random int ::srand(12345); // use fixed seed for repeatability and easier debugging W_DO(ssm->begin_xct()); test_env->set_xct_query_lock(); while ( produced < records ) { std::stringstream buffer, buffer2; ibuffer = rand () % domain; buffer << ibuffer; key.construct_regularkey(buffer.str().data(), buffer.str().size()); ibuffer = rand () % domain; buffer2 << ibuffer; vec_t data; data.set(buffer2.str().data(),buffer2.str().size()); rc_t rc = ssm->create_assoc(stid, key, data); if (rc.is_error()) { if (rc.err_num() != eDUPLICATE) { cerr << "unexpected error"; return rc; } } if (produced%500 == 1) { W_DO(ssm->commit_xct()); W_DO(ssm->begin_xct()); test_env->set_xct_query_lock(); printf("Processed %d keys\n",produced); } produced++; } W_DO(ssm->commit_xct()); return RCOK; }
rc_t smthread_main_t::mr_index_test1() { cout << endl; cout << " ------- TEST1 -------" << endl; cout << "To test adding inital partitions to MRBtree!" << endl; cout << endl; cout << "Creating multi rooted btree index." << endl; W_DO(ssm->begin_xct()); W_DO(create_the_index()); W_DO(ssm->commit_xct()); W_DO(ssm->begin_xct()); int key1 = (int) (0.7 * _num_rec); vec_t key_vec1(&key1, sizeof(key1)); cout << "add_partition_init: stid = " << _index_id << ", key = " << key1 << endl; W_DO(ssm->add_partition_init(_index_id, key_vec1, false)); W_DO(ssm->commit_xct()); W_DO(ssm->begin_xct()); int key2 = (int) (0.5 * _num_rec); vec_t key_vec2(&key2, sizeof(key2)); cout << "add_partition_init: stid = " << _index_id << ", key = " << key2 << endl; W_DO(ssm->add_partition_init(_index_id, key_vec2, false)); W_DO(ssm->commit_xct()); // create the records and their assocs threadptr creator_thread1 = new smthread_creator_t(_num_rec, _rec_size, _bIgnoreLocks, _bIgnoreLatches, _design_no, 0, key2, _index_id, 1); threadptr creator_thread2 = new smthread_creator_t(_num_rec, _rec_size, _bIgnoreLocks, _bIgnoreLatches, _design_no, key2, key1, _index_id, 1); threadptr creator_thread3 = new smthread_creator_t(_num_rec, _rec_size, _bIgnoreLocks, _bIgnoreLatches, _design_no, key1, _num_rec, _index_id, 1); creator_thread1->fork(); creator_thread2->fork(); creator_thread3->fork(); creator_thread1->join(); creator_thread2->join(); creator_thread3->join(); delete creator_thread1; delete creator_thread2; delete creator_thread3; W_DO(print_the_index()); return RCOK; }
rc_t log_free_pages_in_ext(const page_p& page, snum_t snum, extnum_t idx, const Pmap& pmap) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled; if (should_log) { logrec_t* logrec; // fudge 1.5 W_DO(xd->get_logbuf(logrec, t_free_pages_in_ext, &page)); new (logrec) free_pages_in_ext_log(page, snum, idx, pmap); W_DO(xd->give_logbuf(logrec, &page)); } else page.set_dirty(); return RCOK; }
rc_t log_xct_end() { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled && xd && xd->is_log_on(); if (should_log) { logrec_t* logrec; // fudge 0.0 W_DO(xd->get_logbuf(logrec, t_xct_end)); new (logrec) xct_end_log(); W_DO(xd->give_logbuf(logrec)); } return RCOK; }
rc_t log_compensate(const lsn_t& rec_lsn) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled && xd && xd->is_log_on(); if (should_log) { logrec_t* logrec; // fudge 0.0 W_DO(xd->get_logbuf(logrec, t_compensate)); new (logrec) compensate_log(rec_lsn); W_DO(xd->give_logbuf(logrec)); } return RCOK; }
rc_t log_comment(const char* msg) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled && xd && xd->is_log_on(); if (should_log) { logrec_t* logrec; // fudge 1.0 W_DO(xd->get_logbuf(logrec, t_comment)); new (logrec) comment_log(msg); W_DO(xd->give_logbuf(logrec)); } return RCOK; }
rc_t log_store_operation(const page_p& page, const store_operation_param& op) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled; if (should_log) { logrec_t* logrec; // fudge 1.0 W_DO(xd->get_logbuf(logrec, t_store_operation, &page)); new (logrec) store_operation_log(page, op); W_DO(xd->give_logbuf(logrec, &page)); } else page.set_dirty(); return RCOK; }
rc_t log_set_ext_next(const page_p& page, extnum_t ext, extnum_t new_next) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled; if (should_log) { logrec_t* logrec; // fudge 0.0 W_DO(xd->get_logbuf(logrec, t_set_ext_next, &page)); new (logrec) set_ext_next_log(page, ext, new_next); W_DO(xd->give_logbuf(logrec, &page)); } else page.set_dirty(); return RCOK; }
rc_t log_free_ext_list(const page_p& page, const stid_t& stid, extnum_t head, extnum_t count) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled; if (should_log) { logrec_t* logrec; // fudge 0.0 W_DO(xd->get_logbuf(logrec, t_free_ext_list, &page)); new (logrec) free_ext_list_log(page, stid, head, count); W_DO(xd->give_logbuf(logrec, &page)); } else page.set_dirty(); return RCOK; }
rc_t log_xct_prepare_fi(int numex, int numix, int numsix, int numextent, const lsn_t& first, int rsvd, int ready, int used) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled && xd && xd->is_log_on(); if (should_log) { logrec_t* logrec; // fudge 0.0 W_DO(xd->get_logbuf(logrec, t_xct_prepare_fi)); new (logrec) xct_prepare_fi_log(numex, numix, numsix, numextent, first, rsvd, ready, used); W_DO(xd->give_logbuf(logrec)); } return RCOK; }
rc_t log_xct_prepare_stores(int num, const stid_t* stids) { xct_t* xd = xct(); bool should_log = smlevel_1::log && smlevel_0::logging_enabled && xd && xd->is_log_on(); if (should_log) { logrec_t* logrec; // fudge 0.0 W_DO(xd->get_logbuf(logrec, t_xct_prepare_stores)); new (logrec) xct_prepare_stores_log(num, stids); W_DO(xd->give_logbuf(logrec)); } return RCOK; }