void smthread_scanner_t::run()
{
stid_t root_iid; // root index ID
// record file info in the root index : this stores some
// attributes of the file in general
W_COERCE(ss_m::vol_root_index(vid, root_iid));
file_info_t info2;
W_COERCE(find_file_info(vid, root_iid, info2 ));
append_only = info2.append_only;
rid_t first_rid = info2.first_rid;
stid_t fid = info2.fid;
num_rec = info2.num_rec;
outstream << "********** Created scanner for key "
<< key
<< (append_only?" append-only" : " any order")
<< endl;
sendout();
{
W_COERCE(ssm->begin_xct());
ss_m::concurrency_t cc = ss_m::t_cc_file;
this->scan_i_scan(fid, num_rec, cc);
W_COERCE(ssm->commit_xct());
}
}
void test_trace(mem_mgmt_t* mem)
{
std::map<size_t, std::list<char*>> slots;
long count = 0;
std::ifstream in("allocs.txt");
char type;
size_t len;
while (in >> type >> len) {
if (type == 'A') {
slot_t s(nullptr, 0);
W_COERCE(mem->allocate(len, s));
if (s.address) {
slots[s.length].push_back(s.address);
}
}
else if (type == 'D') {
auto& l = slots[len];
auto pos = rand() % l.size();
auto iter = l.begin();
for (size_t i = 0; i < pos; i++) {
iter++;
}
char* addr = *iter;
l.erase(iter);
W_COERCE(mem->free(slot_t{addr, len}));
}
if (count++ % 100000 == 0) {
std::cout << count / 100000 << std::endl;
}
}
std::cout << "Processed " << count << " lines" << std::endl;
}
rc_t
smthread_user_t::no_init()
{
cout << "Using already-existing device: " << _device_name << endl;
// mount already existing device
devid_t devid;
u_int vol_cnt;
w_rc_t rc = ssm->mount_dev(_device_name, vol_cnt, devid);
if (rc.is_error()) {
cerr << "Error: could not mount device: "
<< _device_name << endl;
cerr << " Did you forget to run the server with -i?"
<< endl;
return rc;
}
// find ID of the volume on the device
lvid_t* lvid_list;
u_int lvid_cnt;
W_DO(ssm->list_volumes(_device_name, lvid_list, lvid_cnt));
if (lvid_cnt == 0) {
cerr << "Error, device has no volumes" << endl;
exit(1);
}
_lvid = lvid_list[0];
delete [] lvid_list;
W_COERCE(find_file_info());
W_COERCE(scan_the_root_index());
W_DO(scan_the_file());
return RCOK;
}
virtual void work()
{
// Create the tables, if any partitioning is to be applied, that has already
// been set at update_partitioning()
W_COERCE(_env->db()->begin_xct());
W_COERCE(_env->ycsbtable_man->table()->create_physical_table(_env->db()));
W_COERCE(_env->db()->commit_xct());
}
void smthread_scanner_t::run()
{
// TODO: you can seperate index and file scan and make index scan for a range
rc_t rc = find_file_info();
if(!rc.is_error()) {
// scan the heap file
double heap_scan_current = 0;
double heap_scan_total = 0;
for(int i=0; !rc.is_error() && i<_scan_file; i++) {
stopwatch_t timer;
rc = scan_the_file();
heap_scan_current = timer.time();
heap_scan_total += heap_scan_current;
}
cout << "Avg file scan: " << (heap_scan_total/_scan_file) << endl;
if (rc.is_error()) {
cerr << "Error in file scan due to: " << endl;
cerr << rc << endl;
rc = RCOK; // force deletion of w_error_t info hanging off rc
// otherwise a leak for w_error_t will be reported
if(rc.is_error())
W_COERCE(rc); // avoid error not checked.
}
// scan the index file
double index_scan_current = 0;
double index_scan_total = 0;
for(int i=0; !rc.is_error() && i<_scan_file; i++) {
stopwatch_t timer;
rc = scan_the_index();
index_scan_current = timer.time();
index_scan_total += index_scan_current;
}
cout << "Avg index scan: " << (index_scan_total/_scan_file) << endl;
if (rc.is_error()) {
cerr << "Error in index scan due to: " << endl;
cerr << rc << endl;
rc = RCOK; // force deletion of w_error_t info hanging off rc
// otherwise a leak for w_error_t will be reported
if(rc.is_error())
W_COERCE(rc); // avoid error not checked.
}
} else {
cerr << "Could not perform scan due to: " << endl;
cerr << rc << endl;
rc = RCOK; // force deletion of w_error_t info hanging off rc
// otherwise a leak for w_error_t will be reported
if(rc.is_error())
W_COERCE(rc); // avoid error not checked.
}
return;
}
void smthread_main_t::run()
{
w_rc_t rc = handle_options();
if(rc.is_error()) {
retval = 1;
return;
}
// Now start a storage manager.
cout << "Starting SSM and performing recovery ..." << endl;
ssm = new ss_m();
if (!ssm) {
cerr << "Error: Out of memory for ss_m" << endl;
retval = 1;
return;
}
cout << "Getting SSM config info for record size ..." << endl;
sm_config_info_t config_info;
W_COERCE(ss_m::config_info(config_info));
// Subroutine to set up the device and volume and
// create the num_rec records of rec_size if initialization
// then perform the requested test
rc = do_work();
sm_stats_info_t stats;
W_COERCE(ss_m::gather_stats(stats));
cout << " SM Statistics : " << endl
<< stats << endl;
if (rc.is_error()) {
cerr << "Could not set up device/volume due to: " << endl;
cerr << rc << endl;
delete ssm;
rc = RCOK; // force deletion of w_error_t info hanging off rc
// otherwise a leak for w_error_t will be reported
retval = 1;
if(rc.is_error())
W_COERCE(rc); // avoid error not checked.
return;
}
// Clean up and shut down
cout << "\nShutting down SSM ..." << endl;
delete ssm;
cout << "Finished!" << endl;
return;
}
w_rc_t sthread_t::cold_startup()
{
_class_list = new sthread_list_t(W_LIST_ARG(sthread_t, _class_link),
&_class_list_lock);
if (_class_list == 0)
W_FATAL(fcOUTOFMEMORY);
// initialize the global RNG
struct timeval now;
gettimeofday(&now, NULL);
// Set the seed for the clib random-number generator, which
// we use to seed the per-thread RNG
::srand(now.tv_usec);
/*
* Boot the main thread onto the current (system) stack.
*/
sthread_main_t *main = new sthread_main_t;
if (!main)
W_FATAL(fcOUTOFMEMORY);
me_lval() = _main_thread = main;
W_COERCE( main->fork() );
if (me() != main)
W_FATAL(stINTERNAL);
#if defined(PURIFY)
/* The main thread is different from all other threads. */
purify_name_thread(me()->name());
#endif
return RCOK;
}
int
xct_t::collect( vtable_t& v, bool names_too)
{
int n=0;
W_COERCE(acquire_xlist_mutex());
{
w_list_i<xct_t, queue_based_lock_t> i(_xlist);
while (i.next()) { n++; }
}
if(names_too) n++;
// n: number of rows
// xct_last: number of attributes
// names_init.max_size(): maximum attribute length
if(v.init(n, xct_last, names_init.max_size())) {
release_xlist_mutex();
return -1;
}
vtable_func<xct_t> f(v);
if(names_too) f.insert_names();
{
w_list_i<xct_t, queue_based_lock_t> i(_xlist);
while (i.next()) {
f( *i.curr());
}
}
release_xlist_mutex();
return 0; //no error
}
// this is a copy of module::list_append, which allocates out of the
// global collecton instead of the module-specific collection.
int process_input()
{
REF(ModDecl) lpt;
Ref<Declaration> dpt;
// dump_str_tab();
if (!scheck_only)
W_COERCE(Shore::begin_transaction(3));
lineno = 1;
if (yyparse())
return -1;
if (scheck_only)
return 0;
if (g_module_list)
for (lpt = (REF(ModDecl) &)(g_module_list);
lpt != 0;
lpt = (REF(ModDecl) &)(lpt->next))
lpt->dmodule.update()->resolve_types();
if (sdl_errors) // don't commit
{
fprintf(stderr,"%d error%s found; module(s) not created\n",
sdl_errors, sdl_errors==1 ? "" : "s");
shrc rc ;
rc = Shore::abort_transaction();
if (rc)
; // always has an rc. du.
}
else
SH_DO(SH_COMMIT_TRANSACTION);
//W_COERCE(Shore::commit_transaction());
return sdl_errors;
}
void sysevent::log_append_extent(extent_id_t ext, lsn_t& prev_page_lsn)
{
logrec_t* lr = new append_extent_log(ext);
lr->set_page_prev_lsn(prev_page_lsn);
W_COERCE(smlevel_0::log->insert(*lr, &prev_page_lsn));
delete lr;
}
void sysevent::log_create_store(PageID root, StoreID stid, lsn_t& prev_page_lsn)
{
logrec_t* lr = new create_store_log(root, stid);
lr->set_page_prev_lsn(prev_page_lsn);
W_COERCE(smlevel_0::log->insert(*lr, &prev_page_lsn));
delete lr;
}
void sysevent::log_dealloc_page(PageID pid, lsn_t& prev_page_lsn)
{
logrec_t* lr = new dealloc_page_log(pid);
lr->set_page_prev_lsn(prev_page_lsn);
W_COERCE(smlevel_0::log->insert(*lr, &prev_page_lsn));
delete lr;
}
void chkpt_m::take()
{
chkpt_mutex.acquire_write();
DBGOUT1(<<"BEGIN chkpt_m::take");
INC_TSTAT(log_chkpt_cnt);
// Insert chkpt_begin log record.
logrec_t* logrec = new logrec_t;
lsn_t begin_lsn;
LOG_INSERT(chkpt_begin_log(lsn_t::null), &begin_lsn);
W_COERCE(ss_m::log->flush_all());
// Collect checkpoint information from log
curr_chkpt.scan_log();
// Serialize chkpt to file
fs::path fpath = smlevel_0::log->get_storage()->make_chkpt_path(lsn_t::null);
fs::path newpath = smlevel_0::log->get_storage()->make_chkpt_path(begin_lsn);
ofstream ofs(fpath.string(), ios::binary | ios::trunc);
curr_chkpt.serialize_binary(ofs);
ofs.close();
fs::rename(fpath, newpath);
_min_rec_lsn = curr_chkpt.get_min_rec_lsn();
_min_xct_lsn = curr_chkpt.get_min_xct_lsn();
// Release the 'write' mutex so the next checkpoint request can come in
chkpt_mutex.release_write();
delete logrec;
}
rc_t
lgdata_p::format(const lpid_t& pid, tag_t tag,
uint4_t flags, store_flag_t store_flags
)
{
w_assert9(tag == t_lgdata_p);
vec_t vec; // empty vector
// format, then create a 0-length slot
/* Do the formatting and insert w/o logging them */
W_DO( page_p::_format(pid, tag, flags, store_flags) );
// always set the store_flag here --see comments
// in bf::fix(), which sets the store flags to st_regular
// for all pages, and lets the type-specific store manager
// override (because only file pages can be insert_file)
// persistent_part().set_page_storeflags ( store_flags );
this->set_store_flags(store_flags); // through the page_p, through the bfcb_t
W_COERCE( page_p::insert_expand(0, 1, &vec, false/*logit*/) );
/* Now, log as one (combined) record: */
rc_t rc = log_page_format(*this, 0, 1, &vec); // lgdata_p
return rc;
}
void smthread_creator_t::run()
{
rc_t rc = find_file_info();
if(!rc.is_error()) {
if(_design_no == 0) {
if(_test_no == 7) { // bulk loading test
rc = fill_the_file_regular_bl();
} else {
// TODO: implement non-bulkloading option here
cout << "This test is not supported for this design option yet!" << endl;
rc = RC(fcASSERT);
}
}
else if(_design_no == 1) {
rc = fill_the_file_regular(); // mrbt regular
} else {
rc = fill_the_file_non_regular(); // mrbt part&leaf
}
}
if (rc.is_error()) {
cerr << "Could not create the records due to: " << endl;
cerr << rc << endl;
rc = RCOK; // force deletion of w_error_t info hanging off rc
// otherwise a leak for w_error_t will be reported
if(rc.is_error())
W_COERCE(rc); // avoid error not checked.
}
return;
}
link_all_modules()
// link all listed modules together.
{
Ref<sdlModule> bmod;
Set<Ref<sdlModule> > omods;
W_COERCE(Shore::begin_transaction(3));
Ref<sdlDeclaration> lpt;
for (lpt = g_module_list; lpt != NULL; lpt = lpt->next)
{
bmod = ((Ref<sdlModDecl> &)lpt)->dmodule;
omods.add(bmod);
}
sdl_linking = 1;
for(int i=0; i<omods.get_size(); i++)
{
bmod = omods.get_elt(i);
bmod.update()->resolve_types();
if (sdl_errors)
{
cerr << sdl_errors << " found linking module " << bmod->name << endl;
SH_DO(Shore::abort_transaction());
return sdl_errors;
}
}
sdl_linking = 0;
// this should be a readonly transacton, so never commit.
SH_DO(SH_COMMIT_TRANSACTION);
return 0;
}
void
ErrLog::_openlogfile(
const char *fn
)
{
const char *filename=fn;
if(strcmp(filename, "-")==0) {
// std::cerr << "log to stderr" << std::endl;
_destination = log_to_stderr;
_file = stderr;
return;
}
if(filename) {
_destination = log_to_unix_file;
if(strncmp(filename, "unix:", 5) == 0) {
filename += 5;
} else if (strncmp(filename, "shore:", 6) == 0) {
filename += 6;
}
_file = fopen(filename, "a+");
if(_file == NULL) {
w_rc_t e = RC(fcOS);
std::cerr << "Cannot fopen Unix file " << filename << std::endl;
std::cerr << e << std::endl;
W_COERCE(e);
}
} else {
std::cerr << "Unknown logging destination." << std::endl;
W_FATAL(fcINTERNAL);
}
}
int
main(int argc, char* argv[])
{
cout << "processing configuration options ..." << endl;
// pointers to options we will create for the grid server program
option_t* opt_server_host = 0;
option_t* opt_connect_port = 0;
const option_level_cnt = 3;
option_group_t options(option_level_cnt);
W_COERCE(options.add_option("connect_port", "1024 < integer < 65535",
"1234", "port for connecting to grid server",
false, option_t::set_value_long,
opt_connect_port));
W_COERCE(options.add_option("server_host", "host address",
"localhost", "address of host running server",
false, option_t::set_value_charstr,
opt_server_host));
if (init_config_options(options, "client", argc, argv)) {
usage(options);
exit(1);
}
// there should not be any other command line arguments
if (argc > 1) {
usage(options);
exit(1);
}
int port = strtol(opt_connect_port->value(), 0, 0);
cout << "trying to connect to server at port " << port<< endl;
if (!connect_to_server(opt_server_host->value(), port)) {
cerr << "Shutting down due to connection failure" << endl;
exit(1);
}
process_user_commands();
disconnect_from_server();
cout << "Finished!" << endl;
return 0;
}
int main(int argc, char **argv)
{
int i;
int threads;
if (parse_args(argc, argv) == -1)
return 1;
if (mix_it_up)
threads = NumFloatThreads + NumIntThreads;
else
threads = NumFloatThreads > NumIntThreads ?
NumFloatThreads : NumIntThreads;
ack = new int[threads];
if (!ack)
W_FATAL(fcOUTOFMEMORY);
worker = new sthread_t *[threads];
if (!worker)
W_FATAL(fcOUTOFMEMORY);
for (i=0; i<NumIntThreads; ++i) {
ack[i] = 0;
worker[i] = new int_thread_t(i);
w_assert1(worker[i]);
W_COERCE( worker[i]->fork() );
}
if (!mix_it_up)
harvest(NumIntThreads);
int base = mix_it_up ? NumIntThreads : 0;
for(i=base ; i < base + NumFloatThreads; ++i){
ack[i] = 0;
worker[i] = new float_thread_t(i);
w_assert1(worker[i]);
W_COERCE( worker[i]->fork() );
}
harvest(mix_it_up ? threads : NumFloatThreads);
delete [] worker;
delete [] ack;
return 0;
}
chkpt_m::~chkpt_m()
{
if (_chkpt_thread) {
_chkpt_thread->retire();
W_COERCE(_chkpt_thread->join());
delete _chkpt_thread;
}
}
void chkpt_m::wakeup_thread()
{
if (!_chkpt_thread) {
_chkpt_thread = new chkpt_thread_t(-1);
W_COERCE(_chkpt_thread->fork());
}
_chkpt_thread->awaken();
}
// TODO: implement something more general like specified in the comments
// creates records in a regular file to be later used in bulk loading ( can be used in the baseline system
// with or without mrbtrees and plp-regular )
// it's better not to do this record creation in parallel since we need a sorted order for bulk loading
// if records are not in sorted order then we should sort them but i'm leaving this as a TODO now
// WARNING: right now just testing simple bulk loading
// create the recs with one thread then perform bulk loading
rc_t smthread_creator_t::fill_the_file_regular_bl()
{
int num_inserted = 1;
W_DO(ssm->begin_xct());
char* dummy = new char[_rec_size];
memset(dummy, '\0', _rec_size);
vec_t data(dummy, _rec_size);
rid_t rid;
for(int j=_start_key; j <= _end_key; j++, num_inserted++) {
if( j == _end_key && (_end_key != _num_rec || _test_no != 2)) {
continue; // for test 2 we want to insert one more record and
// it is the reason for all the weird continues here TODO: try to make this better
}
{
w_ostrstream o(dummy, _rec_size);
o << j << ends;
w_assert1(o.c_str() == dummy);
}
// header contains record #
int i = j;
const vec_t hdr(&i, sizeof(i));
W_COERCE(ssm->create_rec(_fid, hdr, _rec_size, data, rid, _bIgnoreLocks));
cout << "Created rec " << j << " rid:" << rid << endl;
if (j == 0) {
_first_rid = rid;
if(_test_no == 2) {
continue;
}
} else if(j == _num_rec) {
_last_rid = rid;
continue;
}
// if we want to insert a lot of records then we run out of log space
// to avoid it, we should flush the log after inserting some number of records
if(num_inserted >= 20000) {
W_DO(ssm->commit_xct());
num_inserted = 0;
W_DO(ssm->begin_xct());
}
}
cout << "Created all. First rid " << _first_rid << " Last rid " << _last_rid << endl;
delete [] dummy;
W_DO(ssm->commit_xct());
// filled the file, now perform bulk-loading
W_DO(ssm->begin_xct());
sm_du_stats_t bl_stats;
W_DO(ssm->bulkld_index(_index_id, _fid, bl_stats));
W_DO(ssm->commit_xct());
return RCOK;
}
// path in module specific load
oql_rc_t oqlDatabase::LoadSdlModule()
{
// for shore, scan a module
#ifndef NO_SDL
Ref<sdlModule> m;
// the name can be 1: an sdl module
// 2: a registered, sdl object
// 3: a directory.
// get a ref.
m = lookup_module(_name);
if (m==0)
{
errstream() << "couldn't find module" << _name;
return OQL_OK;
}
Ref<sdlDeclaration> bpt;
for (bpt = m->decl_list; bpt != 0; bpt = bpt->next)
// insert types in db.
{
Type *nt;
if(bpt->kind==TypeName)
{
nt = AddShoreType(bpt->type);
if (nt!=0)
{
if (nt->isObject()) {
// Add to the list of extents
extents.add(bpt->name, new SetType(nt));
}
}
else
errstream()<< "create_Type faied for "
<< bpt->name.string() << "in module " << _name;
}
}
#endif
// Assume that the _cat is open. Assume that the
// mutex has been taken
#ifdef NO_SDL
AutoMutex myMutex(*(_cat->mutex));
W_COERCE(myMutex.acquire());
uint4 numExtents = _cat->dbCache.extentCnt;
uint4 i;
if (numExtents) assert(t_array = new Type*[numExtents]);
// Now go thru each extent...
for (i = 0; i < numExtents; i++)
Do(_cat->extentCache[i], t_array[i]);
// That's it...
if (t_array) delete [] t_array;
#endif
return OQL_OK;
}
void DBInspect::run()
{
// Build alloc_cache to get allocation status of pages
int fd;
int flags = smthread_t::OPEN_RDONLY;
W_COERCE(me()->open(file.c_str(), flags, 0744, fd));
filestat_t fs;
W_COERCE(me()->fstat(fd, fs));
uint64_t fsize = fs.st_size;
PageID max_pid = fsize / sizeof(generic_page);
// CS TODO: update for new alloc cache
// bf_fixed_m bf_fixed(NULL, fd, max_pid);
// cout << "Max pid = " << max_pid << endl;
// bf_fixed.init();
// alloc_cache_t alloc(&bf_fixed);
// alloc.load_by_scan(max_pid);
// Iterate and print info about pages
ifstream in(file, std::ifstream::binary);
generic_page page;
// Volume header page can be just printed out
in.seekg(0);
in.read((char*) &page, sizeof(generic_page));
cout << (char*) &page << endl;
PageID p = 1;
while (in) {
in.seekg(p * sizeof(generic_page));
in.read((char*) &page, sizeof(generic_page));
if (!in) { break; }
cout << "Page=" << p
<< " PID=" << page.pid
<< " LSN=" << page.lsn
<< " Checksum="
<< (page.checksum == page.calculate_checksum() ? "OK" : "WRONG")
// << " Alloc=" << (alloc.is_allocated_page(p) ? "YES" : "NO")
<< endl;
p++;
}
}
chkpt_m::chkpt_m(const sm_options& options)
: _chkpt_thread(NULL), _chkpt_count(0), _min_rec_lsn(0), _min_xct_lsn(0),
_last_end_lsn(0)
{
int interval = options.get_int_option("sm_chkpt_interval", -1);
if (interval >= 0) {
_chkpt_thread = new chkpt_thread_t(interval);
W_COERCE(_chkpt_thread->fork());
}
}
void sysevent::log(logrec_t::kind_t kind)
{
// this should use TLS allocator, so it's fast
// (see macro DEFINE_SM_ALLOC in allocator.h and logrec.cpp)
logrec_t* lr = new logrec_t();
lr->header._type = kind;
lr->header._cat = 0 | logrec_t::t_status;
lr->fill(0, 0);
W_COERCE(smlevel_0::log->insert(*lr, NULL));
delete lr;
}
/*********************************************************************
*
* chkpt_m::spawn_chkpt_thread()
*
* Fork the checkpoint thread.
*
*********************************************************************/
void
chkpt_m::spawn_chkpt_thread()
{
w_assert1(_chkpt_thread == 0);
if (smlevel_0::log) {
/* Create thread (1) to take checkpoints */
_chkpt_thread = new chkpt_thread_t;
if (! _chkpt_thread) W_FATAL(eOUTOFMEMORY);
W_COERCE(_chkpt_thread->fork());
}
}
/*********************************************************************
*
* chkpt_m::retire_chkpt_thread()
*
* Kill the checkpoint thread.
*
*********************************************************************/
void
chkpt_m::retire_chkpt_thread()
{
if (log) {
w_assert1(_chkpt_thread);
_chkpt_thread->retire();
W_COERCE( _chkpt_thread->join() ); // wait for it to end
delete _chkpt_thread;
_chkpt_thread = 0;
}
}
void sysevent::log_page_read(PageID shpid, uint32_t count)
{
logrec_t* lr = new logrec_t();
lr->header._type = logrec_t::t_page_read;
lr->header._cat = 0 | logrec_t::t_status;
memcpy(lr->data(), &shpid, sizeof(PageID));
memcpy(lr->_data + sizeof(PageID), &count, sizeof(uint32_t));
lr->fill(0, sizeof(PageID) + sizeof(uint32_t));
W_COERCE(smlevel_0::log->insert(*lr, NULL));
delete lr;
}
void smthread_user_t::start_ssm()
{
// get the logdir option for use in dump()
if(logdir==NULL) {
W_COERCE(_options->lookup("sm_logdir", false, logdir));
fprintf(stdout, "Found logdir %s\n", logdir->value());
}
// Now start a storage manager.
cout << "Starting SSM (with out_of_log_space and get_archived_log_file) ..." << endl;
ssm = new ss_m(out_of_log_space, get_archived_log_file);
}