/**
* Determine which snapshots in the state queue can be free'd because are placed before the current time barrier.
*
* Queues are cleaned by deleting all the events the timestamp of which is STRICTLY lower than the time barrier.
* Since state_pointer points to an event in queue_in, the state queue must be cleaned after the input queue.
*
* @author Francesco Quaglia
*
* @param lid The logical process' local identifier
* @param time_barrier The current barrier
*/
void fossil_collection(unsigned int lid, simtime_t time_barrier) {
state_t *state;
msg_t *last_kept_event;
double committed_events;
time_barrier = 0.7 * time_barrier;
// State list must be handled differently, as nodes point to malloc'd
// nodes. We therefore manually scan the list and free the memory.
while( (state = list_head(LPS[lid]->queue_states)) != NULL && state->lvt < time_barrier) {
log_delete(list_head(LPS[lid]->queue_states)->log);
state->last_event = (void *)0xDEADBABE;
list_pop(LPS[lid]->queue_states);
}
// Determine queue pruning horizon
last_kept_event = list_head(LPS[lid]->queue_states)->last_event;
// Truncate the input queue, accounting for the event which is pointed by the lastly kept state
committed_events = (double)list_trunc_before(LPS[lid]->queue_in, timestamp, last_kept_event->timestamp);
statistics_post_lp_data(lid, STAT_COMMITTED, committed_events);
// Truncate the output queue
list_trunc_before(LPS[lid]->queue_out, send_time, last_kept_event->timestamp);
}
/**
* This function creates a full log of the current simulation states and returns a pointer to it.
* The algorithm behind this function is based on packing of the really allocated memory chunks into
* a contiguous memory area, exploiting some threshold-based approach to fasten it up even more.
* The contiguous copy is performed precomputing the size of the full log, and then scanning it
* using a pointer for storing the relevant information.
*
* For further information, please see the paper:
* R. Toccaceli, F. Quaglia
* DyMeLoR: Dynamic Memory Logger and Restorer Library for Optimistic Simulation Objects
* with Generic Memory Layout
* Proceedings of the 22nd Workshop on Principles of Advanced and Distributed Simulation
* 2008
*
* To fully understand the changes in this function to support the incremental logging as well,
* please point to the paper:
* A. Pellegrini, R. Vitali, F. Quaglia
* Di-DyMeLoR: Logging only Dirty Chunks for Efficient Management of Dynamic Memory Based
* Optimistic Simulation Objects
* Proceedings of the 23rd Workshop on Principles of Advanced and Distributed Simulation
* 2009
*
* @author Roberto Toccaceli
* @author Francesco Quaglia
* @author Alessandro Pellegrini
* @author Roberto Vitali
*
* @param lid The logical process' local identifier
* @return A pointer to a malloc()'d memory area which contains the full log of the current simulation state,
* along with the relative meta-data which can be used to perform a restore operation.
*
* @todo must be declared static. This will entail changing the logic in gvt.c to save a state before rebuilding.
*/
void *log_full(int lid) {
void *ptr, *ckpt;
int i, j, k, idx, bitmap_blocks;
size_t size, chunk_size;
malloc_area * m_area;
// Timers for self-tuning of the simulation platform
timer checkpoint_timer;
timer_start(checkpoint_timer);
size = sizeof(malloc_state) + sizeof(seed_type) + m_state[lid]->busy_areas * sizeof(malloc_area) + m_state[lid]->bitmap_size + m_state[lid]->total_log_size;
// This code is in a malloc-wrapper package, so here we call the real malloc
ckpt = rsalloc(size);
if(ckpt == NULL)
rootsim_error(true, "(%d) Unable to acquire memory for ckptging the current state (memory exhausted?)");
ptr = ckpt;
// Copy malloc_state in the ckpt
memcpy(ptr, m_state[lid], sizeof(malloc_state));
ptr = (void *)((char *)ptr + sizeof(malloc_state));
((malloc_state*)ckpt)->timestamp = current_lvt;
((malloc_state*)ckpt)->is_incremental = 0;
// Copy the per-LP Seed State (to make the numerical library rollbackable and PWD)
memcpy(ptr, &LPS[lid]->seed, sizeof(seed_type));
ptr = (void *)((char *)ptr + sizeof(seed_type));
for(i = 0; i < m_state[lid]->num_areas; i++){
m_area = &m_state[lid]->areas[i];
// Copy the bitmap
bitmap_blocks = m_area->num_chunks / NUM_CHUNKS_PER_BLOCK;
if (bitmap_blocks < 1)
bitmap_blocks = 1;
// Check if there is at least one chunk used in the area
if(m_area->alloc_chunks == 0){
m_area->dirty_chunks = 0;
m_area->state_changed = 0;
if (m_area->use_bitmap != NULL) {
for(j = 0; j < bitmap_blocks; j++)
m_area->dirty_bitmap[j] = 0;
}
continue;
}
// Copy malloc_area into the ckpt
memcpy(ptr, m_area, sizeof(malloc_area));
ptr = (void*)((char*)ptr + sizeof(malloc_area));
memcpy(ptr, m_area->use_bitmap, bitmap_blocks * BLOCK_SIZE);
ptr = (void*)((char*)ptr + bitmap_blocks * BLOCK_SIZE);
chunk_size = m_area->chunk_size;
RESET_BIT_AT(chunk_size, 0); // ckpt Mode bit
RESET_BIT_AT(chunk_size, 1); // Lock bit
// Check whether the area should be completely copied (not on a per-chunk basis)
// using a threshold-based heuristic
if(CHECK_LOG_MODE_BIT(m_area)){
// If the malloc_area is almost (over a threshold) full, copy it entirely
memcpy(ptr, m_area->area, m_area->num_chunks * chunk_size);
ptr = (void*)((char*)ptr + m_area->num_chunks * chunk_size);
} else {
// Copy only the allocated chunks
for(j = 0; j < bitmap_blocks; j++){
// Check the allocation bitmap on a per-block basis, to enhance scan speed
if(m_area->use_bitmap[j] == 0) {
// Empty (no-chunks-allocated) block: skip to the next
continue;
} else {
// At least one chunk is allocated: per-bit scan of the block is required
for(k = 0; k < NUM_CHUNKS_PER_BLOCK; k++){
if(CHECK_BIT_AT(m_area->use_bitmap[j], k)){
idx = j * NUM_CHUNKS_PER_BLOCK + k;
memcpy(ptr, (void*)((char*)m_area->area + (idx * chunk_size)), chunk_size);
ptr = (void*)((char*)ptr + chunk_size);
}
}
}
}
}
// Reset Dirty Bitmap, as there is a full ckpt in the chain now
m_area->dirty_chunks = 0;
m_area->state_changed = 0;
bzero((void *)m_area->dirty_bitmap, bitmap_blocks * BLOCK_SIZE);
} // For each m_area in m_state
// Sanity check
if ((char *)ckpt + size != ptr){
rootsim_error(false, "Actual (full) ckpt size different from the estimated one! ckpt = %p size = %x (%d), ptr = %p\n", ckpt, size, size, ptr);
}
m_state[lid]->dirty_areas = 0;
m_state[lid]->dirty_bitmap_size = 0;
m_state[lid]->total_inc_size = 0;
int checkpoint_time = timer_value_micro(checkpoint_timer);
statistics_post_lp_data(lid, STAT_CKPT_TIME, (double)checkpoint_time);
statistics_post_lp_data(lid, STAT_CKPT_MEM, (double)size);
return ckpt;
}
/**
* Upon the decision of performing a rollback operation, this function is invoked by the simulation
* kernel to perform a restore operation.
* This function checks the mark in the malloc_state telling whether we're dealing with a full or
* partial log, and calls the proper function accordingly
*
* @author Alessandro Pellegrini
* @author Roberto Vitali
*
* @param lid The logical process' local identifier
* @param queue_node a pointer to the simulation state which must be restored in the logical process
*/
void log_restore(int lid, state_t *state_queue_node) {
statistics_post_lp_data(lid, STAT_RECOVERY, 1.0);
restore_full(lid, state_queue_node->log);
}
/**
* This function restores a full log in the address space where the logical process will be
* able to use it as the current state.
* Operations performed by the algorithm are mostly the opposite of the corresponding log_full
* function.
*
* For further information, please see the paper:
* R. Toccaceli, F. Quaglia
* DyMeLoR: Dynamic Memory Logger and Restorer Library for Optimistic Simulation Objects
* with Generic Memory Layout
* Proceedings of the 22nd Workshop on Principles of Advanced and Distributed Simulation
* 2008
*
* @author Roberto Toccaceli
* @author Francesco Quaglia
* @author Roberto Vitali
* @author Alessandro Pellegrini
*
* @param lid The logical process' local identifier
* @param queue_node a pointer to the simulation state which must be restored in the logical process
*/
void restore_full(int lid, void *ckpt) {
void * ptr;
int i, j, k, bitmap_blocks, idx, original_num_areas, restored_areas;
unsigned int bitmap;
size_t chunk_size;
malloc_area *m_area, *new_area;
// Timers for simulation platform self-tuning
timer recovery_timer;
timer_start(recovery_timer);
restored_areas = 0;
ptr = ckpt;
original_num_areas = m_state[lid]->num_areas;
new_area = m_state[lid]->areas;
// Restore malloc_state
memcpy(m_state[lid], ptr, sizeof(malloc_state));
ptr = (void*)((char*)ptr + sizeof(malloc_state));
// Restore the per-LP Seed State (to make the numerical library rollbackable and PWD)
memcpy(&LPS[lid]->seed, ptr, sizeof(seed_type));
ptr = (void *)((char *)ptr + sizeof(seed_type));
m_state[lid]->areas = new_area;
// Scan areas and chunk to restore them
for(i = 0; i < m_state[lid]->num_areas; i++){
m_area = &m_state[lid]->areas[i];
bitmap_blocks = m_area->num_chunks / NUM_CHUNKS_PER_BLOCK;
if(bitmap_blocks < 1)
bitmap_blocks = 1;
if(restored_areas == m_state[lid]->busy_areas || m_area->idx != ((malloc_area*)ptr)->idx){
m_area->dirty_chunks = 0;
m_area->state_changed = 0;
if (m_area->use_bitmap != NULL){
for(j = 0; j < bitmap_blocks; j++) {
m_area->dirty_bitmap[j] = 0;
}
}
m_area->alloc_chunks = 0;
m_area->next_chunk = 0;
RESET_LOG_MODE_BIT(m_area);
RESET_AREA_LOCK_BIT(m_area);
if (m_area->use_bitmap != NULL) {
for(j = 0; j < bitmap_blocks; j++)
m_area->use_bitmap[j] = 0;
for(j = 0; j < bitmap_blocks; j++)
m_area->dirty_bitmap[j] = 0;
}
m_area->last_access = m_state[lid]->timestamp;
continue;
}
// Restore the malloc_area
memcpy(m_area, ptr, sizeof(malloc_area));
ptr = (void*)((char*)ptr + sizeof(malloc_area));
restored_areas++;
// Restore use bitmap
memcpy(m_area->use_bitmap, ptr, bitmap_blocks * BLOCK_SIZE);
ptr = (void*)((char*)ptr + bitmap_blocks * BLOCK_SIZE);
// Reset dirty bitmap
bzero(m_area->dirty_bitmap, bitmap_blocks * BLOCK_SIZE);
m_area->dirty_chunks = 0;
m_area->state_changed = 0;
chunk_size = m_area->chunk_size;
RESET_BIT_AT(chunk_size, 0);
RESET_BIT_AT(chunk_size, 1);
// Check how the area has been logged
if(CHECK_LOG_MODE_BIT(m_area)){
// The area has been entirely logged
memcpy(m_area->area, ptr, m_area->num_chunks * chunk_size);
ptr = (void*)((char*)ptr + m_area->num_chunks * chunk_size);
} else {
// The area was partially logged.
// Logged chunks are the ones associated with a used bit whose value is 1
// Their number is in the alloc_chunks counter
for(j = 0; j < bitmap_blocks; j++){
bitmap = m_area->use_bitmap[j];
// Check the allocation bitmap on a per-block basis, to enhance scan speed
if(bitmap == 0){
// Empty (no-chunks-allocated) block: skip to the next
continue;
} else {
// Scan the bitmap on a per-bit basis
for(k = 0; k < NUM_CHUNKS_PER_BLOCK; k++){
if(CHECK_BIT_AT(bitmap, k)){
idx = j * NUM_CHUNKS_PER_BLOCK + k;
memcpy((void*)((char*)m_area->area + (idx * chunk_size)), ptr, chunk_size);
ptr = (void*)((char*)ptr + chunk_size);
}
}
}
}
}
}
// Check whether there are more allocated areas which are not present in the log
if(original_num_areas > m_state[lid]->num_areas){
for(i = m_state[lid]->num_areas; i < original_num_areas; i++){
m_area = &m_state[lid]->areas[i];
m_area->alloc_chunks = 0;
m_area->dirty_chunks = 0;
m_area->state_changed = 0;
m_area->next_chunk = 0;
m_area->last_access = m_state[lid]->timestamp;
m_state[lid]->areas[m_area->prev].next = m_area->idx;
RESET_LOG_MODE_BIT(m_area);
RESET_AREA_LOCK_BIT(m_area);
if (m_area->use_bitmap != NULL) {
bitmap_blocks = m_area->num_chunks / NUM_CHUNKS_PER_BLOCK;
if(bitmap_blocks < 1)
bitmap_blocks = 1;
for(j = 0; j < bitmap_blocks; j++)
m_area->use_bitmap[j] = 0;
for(j = 0; j < bitmap_blocks; j++)
m_area->dirty_bitmap[j] = 0;
}
}
m_state[lid]->num_areas = original_num_areas;
}
m_state[lid]->timestamp = -1;
m_state[lid]->is_incremental = -1;
m_state[lid]->dirty_areas = 0;
m_state[lid]->dirty_bitmap_size = 0;
m_state[lid]->total_inc_size = 0;
int recovery_time = timer_value_micro(recovery_timer);
statistics_post_lp_data(lid, STAT_RECOVERY_TIME, (double)recovery_time);
}
/**
* This function is the only log function which should be called from the simulation platform. Actually,
* it is a demultiplexer which calls the correct function depending on the current configuration of the
* platform. Note that this function only returns a pointer to a malloc'd area which contains the
* state buffers. This means that this memory area cannot be used as-is, but should be wrapped
* into a state_t structure, which gives information about the simulation state pointer (defined
* via <SetState>() by the application-level code and the lvt associated with the log.
* This is done implicitly by the <LogState>() function, which in turn connects the newly taken
* snapshot with the currencly-scheduled LP.
* Therefore, any point of the simulator which wants to take a (real) log, shouldn't call directly this
* function, rather <LogState>() should be used, after having correctly set current_lp and current_lvt.
*
* @author Alessandro Pellegrini
*
* @param lid The logical process' local identifier
* @return A pointer to a malloc()'d memory area which contains the log of the current simulation state,
* along with the relative meta-data which can be used to perform a restore operation.
*/
void *log_state(int lid) {
statistics_post_lp_data(lid, STAT_CKPT, 1.0);
return log_full(lid);
}
void serial_simulation(void) {
timer serial_event_execution;
timer serial_gvt_timer;
msg_t *event;
unsigned int completed = 0;
#ifdef EXTRA_CHECKS
unsigned long long hash1, hash2;
hash1 = hash2 = 0;
#endif
timer_start(serial_gvt_timer);
statistics_post_other_data(STAT_SIM_START, 0.0);
while(!serial_simulation_complete) {
event = (msg_t *)calqueue_get();
if(event == NULL) {
rootsim_error(true, "No events to process!\n");
}
#ifdef EXTRA_CHECKS
if(event->size > 0) {
hash1 = XXH64(event->event_content, event->size, current_lp);
}
#endif
current_lp = event->receiver;
current_lvt = event->timestamp;
timer_start(serial_event_execution);
ProcessEvent_light(current_lp, current_lvt, event->type, event->event_content, event->size, serial_states[current_lp]);
statistics_post_lp_data(current_lp, STAT_EVENT, 1.0);
statistics_post_lp_data(current_lp, STAT_EVENT_TIME, timer_value_seconds(serial_event_execution) );
#ifdef EXTRA_CHECKS
if(event->size > 0) {
hash2 = XXH64(event->event_content, event->size, current_lp);
}
if(hash1 != hash2) {
printf("hash1 = %llu, hash2= %llu\n", hash1, hash2);
rootsim_error(true, "Error, LP %d has modified the payload of event %d during its processing. Aborting...\n", current_lp, event->type);
}
#endif
current_lp = IDLE_PROCESS;
// Termination detection can happen only after the state is initialized
if(serial_states[event->receiver] != NULL) {
// Should we terminate the simulation?
if(!serial_completed_simulation[event->receiver] && OnGVT_light(event->receiver, serial_states[event->receiver])) {
completed++;
serial_completed_simulation[event->receiver] = true;
if(completed == n_prc_tot) {
serial_simulation_complete = true;
}
}
}
// Termination detection on reached LVT value
if(rootsim_config.simulation_time > 0 && event->timestamp >= rootsim_config.simulation_time) {
serial_simulation_complete = true;
}
// Simulate the execution of GVT protocol
if (timer_value_milli(serial_gvt_timer) > (int)rootsim_config.gvt_time_period) {
timer_restart(serial_gvt_timer);
printf("TIME BARRIER: %f\n", current_lvt);
statistics_post_other_data(STAT_GVT, 0.0);
statistics_post_other_data(STAT_GVT_TIME, current_lvt);
}
rsfree(event);
}
simulation_shutdown(EXIT_SUCCESS);
}
