ServerBlock* DiskBackedBlockMap::get_block_for_writing(const BlockId& block_id){ /** If block is not in block map, allocate space for it * Otherwise, if the block is not in memory, allocate space for it. * Set in_memory and dirty_flag */ ServerBlock* block = block_map_.block(block_id); size_t block_size = sip_tables_.block_size(block_id); if (block == NULL) { std::stringstream msg; msg << "S " << sip_mpi_attr_.global_rank(); msg << " : getting uninitialized block " << block_id << ". Creating zero block for writing"<< std::endl; SIP_LOG(std::cout << msg.str() << std::flush); block = allocate_block(NULL, block_size); block_map_.insert_block(block_id, block); } else { if (!block->is_in_memory()) block->allocate_in_memory_data(); } block->set_in_memory(); block->set_dirty(); policy_.touch(block_id); return block; }
ServerBlock* DiskBackedBlockMap::allocate_block(ServerBlock* block, size_t block_size, bool initialize){ /** If enough memory remains, allocates block and returns. * Otherwise, frees up memory by writing out dirty blocks * till enough memory has been obtained, then allocates * and returns block. */ std::size_t remaining_mem = max_allocatable_bytes_ - ServerBlock::allocated_bytes(); while (block_size * sizeof(double) > remaining_mem){ try{ BlockId bid = policy_.get_next_block_for_removal(); ServerBlock* blk = block_map_.block(bid); SIP_LOG(std::cout << "S " << sip_mpi_attr_.company_rank() << " : Freeing block " << bid << " and writing to disk to make space for new block" << std::endl); if(blk->is_dirty()){ write_block_to_disk(bid, blk); } blk->free_in_memory_data(); // Junmin's fix : // As a result of freeing up block memory, the remaining memory should // have increased. Otherwise it will go into an infinite loop. if (!(remaining_mem < max_allocatable_bytes_ - ServerBlock::allocated_bytes())) { throw std::out_of_range("Break now."); } remaining_mem = max_allocatable_bytes_ - ServerBlock::allocated_bytes(); } catch (const std::out_of_range& oor){ std::cerr << " In DiskBackedBlockMap::allocate_block" << std::endl; std::cerr << oor.what() << std::endl; std::cerr << *this << std::endl; fail(" Something got messed up in the internal data structures of the Server", current_line()); } catch(const std::bad_alloc& ba){ std::cerr << " In DiskBackedBlockMap::allocate_block" << std::endl; std::cerr << ba.what() << std::endl; std::cerr << *this << std::endl; fail(" Could not allocate ServerBlock, out of memory", current_line()); } } std::stringstream ss; ss << "S " << sip_mpi_attr_.company_rank() << " : Could not allocate memory for block of size " << block_size << ", Memory being used :" << ServerBlock::allocated_bytes() << std::endl; sip :: check (block_size <= max_allocatable_bytes_ - ServerBlock::allocated_bytes(), ss.str()); if (block == NULL) { block = new ServerBlock(block_size, initialize); } else { block->allocate_in_memory_data(); } return block; }
ServerBlock* DiskBackedBlockMap::get_block_for_reading(const BlockId& block_id){ /** If block is not in block map, there is an error !! * Otherwise, if the block is in memory, read and return or * if it is only on disk, read it in, store in block map and return. * Set in_memory flag. */ ServerBlock* block = block_map_.block(block_id); size_t block_size = sip_tables_.block_size(block_id); if (block == NULL) { // Error ! std::stringstream errmsg; errmsg << " S " << sip_mpi_attr_.global_rank(); errmsg << " : Asking for block " << block_id << ". It has not been put/prepared before !"<< std::endl; std::cout << errmsg.str() << std::flush; sip::fail(errmsg.str()); // WARNING DISABLED ! if (false){ std::stringstream msg; msg << "S " << sip_mpi_attr_.global_rank(); msg << " : getting uninitialized block " << block_id << ". Creating zero block "<< std::endl; std::cout << msg.str() << std::flush; block = allocate_block(NULL, block_size); block_map_.insert_block(block_id, block); } } else { if(!block->is_in_memory()) if (block->is_on_disk()){ read_block_from_disk(block, block_id, block_size); } else { sip::fail("get_block_for_reading : ServerBlock neither on memory or on disk !"); } } block->set_in_memory(); policy_.touch(block_id); sip::check(block != NULL, "Block is NULL in Server get_block_for_reading, should not happen !"); return block; }
/** Template specialization for ServerBlock. * * Regular LRU Array policy is to evict the * "first" block from the least recently used array. * For ServerBlocks however, since they are not * removed from the BlockMap, but are "emptied out", * the regular array policy won't work. * * This methods first chooses an array whose block * is to be evicted. It then finds one, skipping * over those blocks which have been "emptied" out. */ template<> BlockId LRUArrayPolicy<ServerBlock>::get_next_block_for_removal(){ /** Return an arbitrary block from the least recently used array */ if(lru_list_.empty()) throw std::out_of_range("No blocks have been touched, yet block requested for flushing"); while (!lru_list_.empty()) { int to_remove_array = lru_list_.back(); IdBlockMap<ServerBlock>::PerArrayMap* array_map = block_map_.per_array_map(to_remove_array); IdBlockMap<ServerBlock>::PerArrayMap::iterator it = array_map->begin(); for (; it != array_map->end(); it ++) { ServerBlock *blk = it->second; if (blk !=NULL && blk->get_data() != NULL) { return it->first; } } lru_list_.pop_back(); } throw std::out_of_range("No server blocks to remove - all empty or none present !"); }
//returns the number of doubles actually freed size_t DiskBackedBlockMap::backup_and_free_doubles(size_t requested_doubles_to_free) { size_t freed_count = 0; try { while (freed_count < requested_doubles_to_free) { //if requested_doubles_to_free <= 0, no iterations performed. ServerBlock* block; BlockId bid = policy_.get_next_block_for_removal(block); block->wait(); ServerBlock* blk = block_map_.block(bid); check(blk == block, "bug in free_doubles"); SIP_LOG( std::cout << "S " << sip_mpi_attr_.company_rank() << " : Freeing block " << bid << " and writing to disk to make space for new block" << std::endl); if (!blk->disk_state_.is_valid_on_disk()) { write_block_to_disk(bid, blk); blocks_to_disk_.inc(); blk->disk_state_.set_valid_on_disk(); } double* data_to_free = blk->get_data(); free_data(data_to_free, blk->size()); //this method updates remaining_doubles_ blk->disk_state_.unset_in_memory(); blk->data_ = NULL; freed_count += blk->size(); } } catch (const std::out_of_range& oor) { //ran out of blocks, just return what was freed. } return freed_count; }