static void
ldo_putref(ScarabSession * session, int reference, ScarabDatum * obj)
{
LDO_Session *s;
ScarabDatum *ref;
s = (LDO_Session *) session->encoder;
ref = scarab_new_integer(reference);
scarab_dict_put(s->refs, ref, obj);
scarab_free_datum(ref);
scarab_free_datum(obj);
}
Datum VariableRegistry::generateCodecDatum() {
ScarabDatum *codec = NULL;
shared_ptr<Variable> var;
boost::mutex::scoped_lock s_lock(lock);
int dictSize = master_variable_list.size();
codec = scarab_dict_new(dictSize, &scarab_dict_times2);
for(int i = 0; i < dictSize; i++) {
var = master_variable_list[i];
if(var == NULL) {
continue;
}
int codec_code = var->getCodecCode();
if(codec_code == RESERVED_CODEC_CODE) {
continue;
}
VariableProperties *props = var->getProperties();
if(props == NULL){
continue;
}
Datum serialized_var(props->operator Datum());
if(serialized_var.isUndefined()) {
mdebug("local parameter null value at param (%d)", i);
}
ScarabDatum *codec_key = scarab_new_integer(codec_code);
scarab_dict_put(codec, codec_key, serialized_var.getScarabDatum());
scarab_free_datum(codec_key);
}
Datum returnCodec(codec);
scarab_free_datum(codec);
return returnCodec;
}
EventWrapper DataFileIndexer::EventsIterator::getNextEvent() {
EventWrapper event;
while (current_event_block < matching_event_blocks.size()) {
if ((current_relative_event == dfi.events_per_block) || (current_datum == NULL)) {
// Advance to the next block
scarab_seek(dfi.session, matching_event_blocks[current_event_block]->blockOffset(), SEEK_SET);
current_relative_event = 0;
}
// Read through the event block
while (event.empty() && (current_relative_event < dfi.events_per_block) && (current_datum = scarab_read(dfi.session)))
{
if (!DataFileUtilities::isScarabEvent(current_datum)) {
// Skip invalid events
scarab_free_datum(current_datum);
continue;
}
MWTime event_time = DataFileUtilities::getScarabEventTime(current_datum);
// Check the time criterion
if (event_time >= lower_bound && event_time <= upper_bound) {
unsigned int event_code = DataFileUtilities::getScarabEventCode(current_datum);
// Check if the event code matches
if (event_codes_to_match.empty() ||
(event_codes_to_match.find(event_code) != event_codes_to_match.end()))
{
event = EventWrapper(current_datum);
}
}
scarab_free_datum(current_datum);
current_relative_event++;
}
if ((current_relative_event == dfi.events_per_block) || (current_datum == NULL))
current_event_block++;
if (!event.empty())
return event;
}
return event;
}
static ScarabDatum *
ldo_read_dict(ScarabSession * session)
{
int len;
ScarabDatum *dict;
len = ldo_readber(session);
dict = scarab_dict_new(len, scarab_dict_times2);
for (; len > 0; len--)
{
ScarabDatum *key = ldo_read(session);
ScarabDatum *value = ldo_read(session);
scarab_dict_put(dict,
key,
value);
scarab_free_datum(key);
scarab_free_datum(value);
}
return dict;
}
static ScarabDatum *
ldo_getref(ScarabSession * session, int reference)
{
LDO_Session *s;
ScarabDatum *ref;
ScarabDatum *obj;
s = (LDO_Session *) session->encoder;
ref = scarab_new_integer(reference);
obj = scarab_dict_get(s->refs, ref);
scarab_free_datum(ref);
return obj;
}
static ScarabDatum *
ldo_read_list(ScarabSession * session)
{
int i;
int len;
ScarabDatum *list;
ScarabDatum *list_element;
len = ldo_readber(session);
list = scarab_list_new(len);
for (i = 0; i < len; i++)
{
list_element = ldo_read(session);
scarab_list_put(list, i, list_element);
scarab_free_datum(list_element);
}
return list;
}
std::vector<ScarabDatumWrapper> DataFileIndexer::events(const std::vector<unsigned int> &event_codes_to_match,
const MonkeyWorksTime lower_bound,
const MonkeyWorksTime upper_bound) const {
std::vector<ScarabDatumWrapper> return_vector;
std::vector<boost::shared_ptr<EventBlock> > event_blocks_with_time = root->children(event_codes_to_match, lower_bound, upper_bound);
if (event_blocks_with_time.size() == 0) {
std::cerr << "Error in indexing - no blocks found to search for codecs. Short file? Bug: fix" << std::endl;
}
//std::cout << "EventBlock size" << event_blocks_with_time.size();
for(std::vector<boost::shared_ptr<EventBlock> >::const_iterator i = event_blocks_with_time.begin();
i != event_blocks_with_time.end();
++i) {
long int block_offset = (*i)->blockOffset();
scarab_seek(session, block_offset, SEEK_SET);
ScarabDatum *current_datum = NULL;
unsigned int current_relative_event = 0;
while((current_datum = scarab_read(session)) && current_relative_event < events_per_block) {
MonkeyWorksTime event_time = DataFileUtilities::getScarabEventTime(current_datum);
if(event_time >= lower_bound && event_time <= upper_bound) {
unsigned int event_code = DataFileUtilities::getScarabEventCode(current_datum);
for(std::vector<unsigned int>::const_iterator j = event_codes_to_match.begin();
j != event_codes_to_match.end();
++j) {
if(event_code == *j) {
return_vector.push_back(ScarabDatumWrapper(current_datum));
}
}
}
scarab_free_datum(current_datum);
current_relative_event++; //std::cout << "cur rel" << current_relative_event;
}
}
return return_vector;
}
void DataFileIndexer::buildIndex(unsigned int _events_per_block, unsigned int multiplication_factor_per_level) {
number_of_events = 0;
events_per_block = _events_per_block;
root.reset();
{
std::vector<boost::shared_ptr<EventBlock> > event_blocks;
{
std::set<unsigned int> event_codes_in_block;
MWTime max_time = MIN_MWORKS_TIME();
MWTime min_time = MAX_MWORKS_TIME();
long int previous_datum_location = scarab_tell(session);
ScarabDatum *datum = NULL;
while(datum = scarab_read(session)) {
if (!DataFileUtilities::isScarabEvent(datum)) {
// Ignore invalid events
scarab_free_datum(datum);
continue;
}
event_codes_in_block.insert(DataFileUtilities::getScarabEventCode(datum));
MWTime event_time = DataFileUtilities::getScarabEventTime(datum);
max_time = std::max(max_time, event_time);
min_time = std::min(min_time, event_time);
number_of_events++;
if(number_of_events % events_per_block == 0) {
boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
event_blocks.push_back(new_event_block);
event_codes_in_block.clear();
max_time = MIN_MWORKS_TIME();
min_time = MAX_MWORKS_TIME();
previous_datum_location = scarab_tell(session);
}
scarab_free_datum(datum);
}
// add in the remainder blocks
boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
event_blocks.push_back(new_event_block);
}
{
// build the tree
int events_per_node = events_per_block;
int number_of_levels = 1;
while(events_per_node < number_of_events) {
number_of_levels++;
events_per_node *= multiplication_factor_per_level;
}
std::vector <boost::shared_ptr<EventBlock> > blocks_at_next_level = event_blocks;
for(int i = 1; i < number_of_levels; ++i) {
std::vector <boost::shared_ptr<EventBlock> > blocks_at_current_level;
std::vector<boost::shared_ptr<EventBlock> >::const_iterator j = blocks_at_next_level.begin();
while(j != blocks_at_next_level.end()) {
std::vector <boost::shared_ptr<EventBlock> > children;
for(int k = 0; k < multiplication_factor_per_level && j != blocks_at_next_level.end(); ++k) {
children.push_back(*j);
++j;
}
boost::shared_ptr<EventBlock> new_block = boost::shared_ptr<EventBlock>(new EventBlock(children));
blocks_at_current_level.push_back(new_block);
}
blocks_at_next_level = blocks_at_current_level;
}
if(blocks_at_next_level.size() != 1) {
//badness
throw DataFileIndexerError("Something went wrong ... please abort and try again");
}
// DDC added a patch to fix failure to index small numbers of events
// force a mandatory level below root
boost::shared_ptr<EventBlock> subroot = blocks_at_next_level.at(0);
std::vector< boost::shared_ptr<EventBlock> > root_children;
root_children.push_back(subroot);
root = boost::shared_ptr<EventBlock>(new EventBlock(root_children));
}
}
}
int ScarabWriteConnection::service() {
boost::mutex::scoped_lock lock(connectLock);
static int n_written = 0;
shared_ptr <Clock> clock = Clock::instance();
static MWTime start_time = clock->getCurrentTimeUS();
// static int functionCount = 0;
// mdebug("entering write service");
// struct timespec time_to_sleep;
// time_to_sleep.tv_sec = 0;
// time_to_sleep.tv_nsec = 200000000; //200 ms
// while(1) {
// mdebug("start write while loop");
// if(servicing) {
// // we are already servicing this connection.
// mdebug("Already Servicing Writing");
// M_ISUNLOCK;
// continue;
// //1return 1;
// }
servicing = true;
// mdebug("Started servicing write function count = %d", functionCount);
// functionCount++;
if(getScarabError(pipe)) {
mwarning(M_NETWORK_MESSAGE_DOMAIN, "Session Failure on ID %ld", cid);
servicing = false;
if(sibling) {
sibling->setInterrupt(true);
}
term = true;
return -1;
}
// if we have been requested to get interrupted
// if we are interrupted while processing events there will be a
// slight delay while out task is re-scheduled
if(interrupt) {
mwarning(M_NETWORK_MESSAGE_DOMAIN,
"Write Service Routine Interrupted on id %ld", cid);
if(sibling) {
sibling->setInterrupt(true);
}
ScarabDatum * termEvent;
// puts a control event of type termination into a scarab package
// and sends the package
termEvent = scarab_list_new(SCARAB_EVENT_N_TOPLEVEL_ELEMENTS);
ScarabDatum *termCode = scarab_new_integer(RESERVED_TERMINATION_CODE);
scarab_list_put(termEvent, SCARAB_EVENT_CODEC_CODE_INDEX, termCode);
scarab_free_datum(termCode);
shared_ptr <Clock> clock = Clock::instance();
ScarabDatum *time = scarab_new_integer(clock->getCurrentTimeUS());
scarab_list_put(termEvent, SCARAB_EVENT_TIME_INDEX, time);
scarab_free_datum(time);
if(scarab_write(pipe, termEvent) == 0) {
// success
mdebug("Wrote termination message from id %ld", cid);
servicing = false;
term = true; // mark the connection for termination
scarab_free_datum(termEvent);
return 0;
} else {
mwarning(M_NETWORK_MESSAGE_DOMAIN,
"Failed to Write Termination Sequence on socket %ld", cid);
// even though there was an error we are going to
// terminate
term = true; // mark the connection for termination
scarab_free_datum(termEvent);
return -1;
}
scarab_free_datum(termEvent);
}
// while we have events to process but havent been requested for
// interruption.
while(buffer_reader->nextEventExists() && !interrupt) {
//MWTime event_time = newevent->getTime();
//mEventType type = newevent->getEventType();
//mVariable *var = newevent->getParam();
ScarabDatum * scarab_event;
#define USE_EXPLICIT_BUFFERING 1
#define BUFFER_HIGH_WATER_MARK 1
#define MAX_EVENTS_TO_BUFFER 10000
if(USE_EXPLICIT_BUFFERING && buffer_reader->hasAtLeastNEvents(BUFFER_HIGH_WATER_MARK)) {
scarab_force_buffering(pipe, 1);
// fill up the macro event
int numEventsBuffered = 0;
bool buffering = true;
do {
shared_ptr<Event> newevent = buffer_reader->getNextEvent();
if(newevent == NULL) {
// don't send anything if an event was NULL.
servicing = false;
scarab_free_datum(scarab_event);
return 1;
}
scarab_event = newevent->toScarabDatum();
// stop buffering when you run out of events, or you hit the max events
if(!(buffering = buffer_reader->nextEventExists() && numEventsBuffered<MAX_EVENTS_TO_BUFFER)) {
scarab_force_buffering(pipe, 0);
}
if(scarab_write(pipe, scarab_event) == 0) {
n_written++;
} else {
merror(M_SYSTEM_MESSAGE_DOMAIN, "scarab buffered write error");
servicing = false;
return -1;
}
scarab_free_datum(scarab_event);
++numEventsBuffered;
} while(buffering);
} else {
// Single event write
shared_ptr<Event> newevent = buffer_reader->getNextEvent();
if(newevent == NULL) {
// don't send anything if an event was NULL.
servicing = false;
return 1;
}
scarab_event = newevent->toScarabDatum();
if(scarab_write(pipe, scarab_event) == 0) {
n_written++;
} else {
merror(M_SYSTEM_MESSAGE_DOMAIN, "scarab write error");
servicing = false;
return -1;
}
scarab_free_datum(scarab_event);
}
// Some reporting
#define N 1000
static int last_nwritten = 0;
if(n_written - last_nwritten >= N){
last_nwritten = n_written;
MWTime now = clock->getCurrentTimeUS();
/*fprintf(stderr, "last: %lld, now: %lld\n",
start_time,
now
); */
// fprintf(stderr, "%g events / sec (%d event in %lld usec)\n",
// (double)N / howlong,
// N, now - start_time
// );
// fprintf(stderr, "Number of items unserviced: %d\n",
// buffer_reader->getNItemsUnserviced());
// fflush(stderr);
start_time = now;
}
}
servicing = false;
return 1;
// }
}
void scarab_free_datum(ScarabDatum *d){
int i;
if(d == NULL) return;
// Unnecessary checking? Actually accounts for 16% of the time in this
// call, and this call is made frequently...
//if(d->mutex != NULL){
pthread_mutex_lock(d->mutex); // lock just THIS mutex
//} else {
//???
//}
//scarab_lock_datum(d);
// One fewer reference to this object
d->ref_count--;
// If the thing is no longer referenced, kill it
if((d->ref_count) == 0){
switch (d->type)
{
case SCARAB_DICT:
// free sub-data
for(i = 0; i < d->data.dict->tablesize; i++){
scarab_free_datum(d->data.dict->keys[i]);
scarab_free_datum(d->data.dict->values[i]);
}
// Free the parts
scarab_mem_free(d->data.dict->keys);
scarab_mem_free(d->data.dict->values);
scarab_mem_free(d->data.dict);
break;
case SCARAB_LIST:
// free sub-data
for(i = 0; i < d->data.list->size; i++){
scarab_free_datum(d->data.list->values[i]);
}
// free parts
scarab_mem_free(d->data.list->values);
scarab_mem_free(d->data.list);
break;
case SCARAB_FLOAT_OPAQUE:
case SCARAB_OPAQUE:
// free opaque data
if(d->data.opaque.data != NULL){
scarab_mem_free(d->data.opaque.data );
}
break;
default:
break;
}
//scarab_unlock_datum(d);
//scarab_destroy_lock(d);
// Take the lock out of the ScarabDatum, we want to hang onto it
// until we're done
pthread_mutex_t *lock = d->mutex;
scarab_mem_free(d);
// now, release the lock and destroy it
pthread_mutex_unlock(lock);
pthread_mutex_destroy(lock);
free(lock);
return;
}
// Unnecessary checking?
// if(d->mutex != NULL){
pthread_mutex_unlock(d->mutex);
// } else {
//???
// }
//scarab_unlock_datum(d);
}
void VariableRegistry::updateFromCodecDatum(const Datum &codec) {
mprintf(M_SYSTEM_MESSAGE_DOMAIN,
"Received new codec, updating variable registry.");
if(!codec.isDictionary()) {
merror(M_SYSTEM_MESSAGE_DOMAIN,
"Invalid codec received. Registry is unchanged.");
return;
}
boost::mutex::scoped_lock s_lock(lock);
master_variable_list.clear();
// add the placeholders
//addPlaceholders();
//////////////////////////////////////////////////////////////////
// now add what's in the codec
ScarabDatum *datum = codec.getScarabDatum();
ScarabDatum ** keys = datum->data.dict->keys;
int size = datum->data.dict->tablesize;
int maxCodecCode = -1;
// find the maximum codec value
for(int i = 0; i < size; ++i) {
if(keys[i]) {
long long code = keys[i]->data.integer;
maxCodecCode = (maxCodecCode < code) ? code : maxCodecCode;
}
}
// add each variable in order to the registry
for(int i = N_RESERVED_CODEC_CODES; i<=maxCodecCode; ++i) {
ScarabDatum *key = scarab_new_integer(i);
ScarabDatum *serializedVariable = scarab_dict_get(datum, key);
scarab_free_datum(key);
if(!serializedVariable) {
shared_ptr<EmptyVariable> empty_var(new EmptyVariable);
master_variable_list.push_back(empty_var);
continue;
} else {
if(serializedVariable->type != SCARAB_DICT) {
// these must be placeholder datums in the package
// that we should ignore.
mwarning(M_SYSTEM_MESSAGE_DOMAIN,
"Bad variable received from network stream");
shared_ptr<EmptyVariable> empty_var(new EmptyVariable);
master_variable_list.push_back(empty_var);
continue;
}
VariableProperties *props =
new VariableProperties(serializedVariable);
if(props == NULL){
mwarning(M_SYSTEM_MESSAGE_DOMAIN,
"Bad variable received from network stream");
shared_ptr<EmptyVariable> empty_var(new EmptyVariable);
master_variable_list.push_back(empty_var);
continue;
}
shared_ptr<Variable> newvar(new GlobalVariable(props));
newvar->setCodecCode(i); //necessary? .. Yup
master_variable_list.push_back(newvar);
std::string tag = newvar->getVariableName();
if(!tag.empty()){
master_variable_dictionary[tag] = newvar;
}
}
}
}
void scarab_free_datum(ScarabDatum *d){
int i;
if(d == NULL) return;
#ifdef THREAD_SAFE_SCARAB_DATUM
pthread_mutex_lock(d->mutex);
#endif
// One fewer reference to this object
d->ref_count--;
// If the thing is no longer referenced, kill it
if((d->ref_count) == 0){
switch (d->type)
{
case SCARAB_DICT:
// free sub-data
for(i = 0; i < d->data.dict->tablesize; i++){
scarab_free_datum(d->data.dict->keys[i]);
scarab_free_datum(d->data.dict->values[i]);
}
// Free the parts
scarab_mem_free(d->data.dict->keys);
scarab_mem_free(d->data.dict->values);
scarab_mem_free(d->data.dict);
break;
case SCARAB_LIST:
// free sub-data
for(i = 0; i < d->data.list->size; i++){
scarab_free_datum(d->data.list->values[i]);
}
// free parts
scarab_mem_free(d->data.list->values);
scarab_mem_free(d->data.list);
break;
case SCARAB_OPAQUE:
// free opaque data
if(d->data.opaque.data != NULL){
scarab_mem_free(d->data.opaque.data );
}
break;
default:
break;
}
#ifdef THREAD_SAFE_SCARAB_DATUM
// Take the lock out of the ScarabDatum, we want to hang onto it
// until we're done
pthread_mutex_t *lock = d->mutex;
#endif
scarab_mem_free(d);
#ifdef THREAD_SAFE_SCARAB_DATUM
// now, release the lock and destroy it
pthread_mutex_unlock(lock);
pthread_mutex_destroy(lock);
scarab_mem_free(lock);
#endif
return;
}
#ifdef THREAD_SAFE_SCARAB_DATUM
pthread_mutex_unlock(d->mutex);
#endif
}
DataFileIndexer::DataFileIndexer(const boost::filesystem::path &data_file,
const unsigned int _events_per_block,
const unsigned int multiplication_factor_per_level,
const int number_of_indexing_threads) : events_per_block(_events_per_block) {
uri = "ldobinary:file://" + data_file.string();
// I hate myself for this
char *uri_temp = new char[uri.length() + 1];
strncpy(uri_temp, uri.c_str(), uri.length() + 1);
session = scarab_session_connect(uri_temp);
delete [] uri_temp;
{
std::vector<boost::shared_ptr<EventBlock> > event_blocks;
{
number_of_events = 0;
std::vector<unsigned int> event_codes_in_block;
MonkeyWorksTime max_time = MIN_MONKEY_WORKS_TIME();
MonkeyWorksTime min_time = MAX_MONKEY_WORKS_TIME();
long int previous_datum_location = scarab_tell(session);
ScarabDatum *datum = NULL;
while(datum = scarab_read(session)) {
event_codes_in_block.push_back(DataFileUtilities::getScarabEventCode(datum));
const MonkeyWorksTime event_time = DataFileUtilities::getScarabEventTime(datum);
max_time = max_time > event_time ? max_time : event_time;
min_time = min_time < event_time ? min_time : event_time;
number_of_events++;
//std::cout << number_of_events << " ";
if(number_of_events % events_per_block == 0) {
std::sort(event_codes_in_block.begin(), event_codes_in_block.end());
event_codes_in_block.erase(std::unique(event_codes_in_block.begin(),
event_codes_in_block.end()),
event_codes_in_block.end());
std::cout << "indexing block " << event_blocks.size() << " .. time : " << min_time << "LL - " << max_time << "LL" << std::endl;
// cerr << "new event block : num events : " << event_codes_in_block.size() << endl;
boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
event_blocks.push_back(new_event_block);
event_codes_in_block.clear();
max_time = MIN_MONKEY_WORKS_TIME();
min_time = MAX_MONKEY_WORKS_TIME();
previous_datum_location = scarab_tell(session);
}
scarab_free_datum(datum);
}
// add in the remainder blocks
boost::shared_ptr<EventBlock> new_event_block = boost::shared_ptr<EventBlock>(new EventBlock(previous_datum_location, min_time, max_time, event_codes_in_block));
event_blocks.push_back(new_event_block);
//std::cout << "size " << event_blocks.size();
}
{
// build the tree
int events_per_node = events_per_block;
int number_of_levels = 1;
while(events_per_node < number_of_events) {
number_of_levels++;
events_per_node *= multiplication_factor_per_level;
}
std::vector <boost::shared_ptr<EventBlock> > blocks_at_next_level = event_blocks;
for(int i = 1; i < number_of_levels; ++i) {
std::vector <boost::shared_ptr<EventBlock> > blocks_at_current_level;
std::vector<boost::shared_ptr<EventBlock> >::const_iterator j = blocks_at_next_level.begin();
while(j != blocks_at_next_level.end()) {
std::vector <boost::shared_ptr<EventBlock> > children;
for(int k = 0; k < multiplication_factor_per_level && j != blocks_at_next_level.end(); ++k) {
children.push_back(*j);
++j;
}
boost::shared_ptr<EventBlock> new_block = boost::shared_ptr<EventBlock>(new EventBlock(children));
blocks_at_current_level.push_back(new_block);
}
blocks_at_next_level = blocks_at_current_level;
}
if(blocks_at_next_level.size() != 1) {
//badness
std::cerr << "something went wrong...please abort and try again" << std::endl;
throw new std::exception;
}
root = blocks_at_next_level.at(0);
}
}
}
ScarabDatumWrapper::~ScarabDatumWrapper() {
scarab_free_datum(datum);
}
