//returns the number of doubles actually freed size_t DiskBackedBlockMap::backup_and_free_doubles(size_t requested_doubles_to_free) { size_t freed_count = 0; try { while (freed_count < requested_doubles_to_free) { //if requested_doubles_to_free <= 0, no iterations performed. ServerBlock* block; BlockId bid = policy_.get_next_block_for_removal(block); block->wait(); ServerBlock* blk = block_map_.block(bid); check(blk == block, "bug in free_doubles"); SIP_LOG( std::cout << "S " << sip_mpi_attr_.company_rank() << " : Freeing block " << bid << " and writing to disk to make space for new block" << std::endl); if (!blk->disk_state_.is_valid_on_disk()) { write_block_to_disk(bid, blk); blocks_to_disk_.inc(); blk->disk_state_.set_valid_on_disk(); } double* data_to_free = blk->get_data(); free_data(data_to_free, blk->size()); //this method updates remaining_doubles_ blk->disk_state_.unset_in_memory(); blk->data_ = NULL; freed_count += blk->size(); } } catch (const std::out_of_range& oor) { //ran out of blocks, just return what was freed. } return freed_count; }
void append_data_to_file(int inum, u8 *data, size_t size) { printf("append data file at num %i\n", inum); inode *in = inode_at_num(inum); printf("inode: num: %i, size: %i, blocks: %i, end addr: %i\n", in->num, in->size, in->blocks, in->end_block); block *cur_blk = block_from_num(in->end_block); printf("block addr: %i\n", in->end_block); int pos = in->size % BLOCK_SIZE; int blocks = (pos + size) / BLOCK_SIZE; int numbytes = BLOCK_SIZE - pos; write_data_to_block(cur_blk, data, size, pos); in->size += numbytes; //printf("%i:",pos); for (int i = 0; i < blocks; i++) { alloc_blocks(in, 1); cur_blk = block_from_num(in->end_block); int pos = in->size % BLOCK_SIZE; int numbytes = BLOCK_SIZE - pos; write_data_to_block(cur_blk, data, numbytes, pos); in->size += numbytes; } write_block_to_disk(cur_blk); in->size += size; write_inode_to_disk(in); free(in); free(cur_blk); }
ServerBlock* DiskBackedBlockMap::allocate_block(ServerBlock* block, size_t block_size, bool initialize){ /** If enough memory remains, allocates block and returns. * Otherwise, frees up memory by writing out dirty blocks * till enough memory has been obtained, then allocates * and returns block. */ std::size_t remaining_mem = max_allocatable_bytes_ - ServerBlock::allocated_bytes(); while (block_size * sizeof(double) > remaining_mem){ try{ BlockId bid = policy_.get_next_block_for_removal(); ServerBlock* blk = block_map_.block(bid); SIP_LOG(std::cout << "S " << sip_mpi_attr_.company_rank() << " : Freeing block " << bid << " and writing to disk to make space for new block" << std::endl); if(blk->is_dirty()){ write_block_to_disk(bid, blk); } blk->free_in_memory_data(); // Junmin's fix : // As a result of freeing up block memory, the remaining memory should // have increased. Otherwise it will go into an infinite loop. if (!(remaining_mem < max_allocatable_bytes_ - ServerBlock::allocated_bytes())) { throw std::out_of_range("Break now."); } remaining_mem = max_allocatable_bytes_ - ServerBlock::allocated_bytes(); } catch (const std::out_of_range& oor){ std::cerr << " In DiskBackedBlockMap::allocate_block" << std::endl; std::cerr << oor.what() << std::endl; std::cerr << *this << std::endl; fail(" Something got messed up in the internal data structures of the Server", current_line()); } catch(const std::bad_alloc& ba){ std::cerr << " In DiskBackedBlockMap::allocate_block" << std::endl; std::cerr << ba.what() << std::endl; std::cerr << *this << std::endl; fail(" Could not allocate ServerBlock, out of memory", current_line()); } } std::stringstream ss; ss << "S " << sip_mpi_attr_.company_rank() << " : Could not allocate memory for block of size " << block_size << ", Memory being used :" << ServerBlock::allocated_bytes() << std::endl; sip :: check (block_size <= max_allocatable_bytes_ - ServerBlock::allocated_bytes(), ss.str()); if (block == NULL) { block = new ServerBlock(block_size, initialize); } else { block->allocate_in_memory_data(); } return block; }
//this can only be used to shrink files void resize_file(int inum, int new_size) { inode *in = inode_at_num(inum); //remove 1 less block than we need to int blks2remove = in->blocks - (new_size / BLOCK_SIZE + 1); dealloc_blocks(in, blks2remove); //pos within the block block *endblock = block_from_num(in->end_block); int endpos = in->size - new_size; for (int i = in->size - 1 % BLOCK_SIZE ; i > endpos; ++i) endblock->data[i] = 0; in->size = new_size; write_inode_to_disk(in); write_block_to_disk(endblock); free(endblock); free(in); }