void commit(bool all) { if (edgedata == NULL || only_adjacency) return; assert(is_loaded); metrics_entry cm = m.start_time(); /** * This is an optimization that is relevant only if memory shard * has been used in a case where only out-edges are considered. * Out-edges are in a continuous "window", while in-edges are * scattered all over the shard */ if (all) { iomgr->managed_pwritea_now(edata_iosession, &edgedata, edatafilesize, 0); } else { size_t last = streaming_offset_edge_ptr; if (last == 0){ // rollback last = edatafilesize; } char * bufp = ((char*)edgedata + range_start_edge_ptr); iomgr->managed_pwritea_now(edata_iosession, &bufp, last - range_start_edge_ptr, range_start_edge_ptr); } m.stop_time(cm, "memshard_commit"); iomgr->managed_release(adj_session, &adjdata); iomgr->managed_release(edata_iosession, &edgedata); is_loaded = false; }
inline void check_curblock(size_t toread) { if (curblock == NULL || curblock->end < edataoffset+toread) { if (curblock != NULL) { if (!curblock->active) { curblock->release(iomgr); } } // Load next std::string blockfilename = filename_shard_edata_block(filename_edata, (int) (edataoffset / blocksize), blocksize); int edata_session = iomgr->open_session(blockfilename, false, true); sblock<ET> newblock(edata_session, edata_session, true, blockfilename); // We align blocks always to the blocksize, even if that requires // allocating and reading some unnecessary data. newblock.offset = (edataoffset / blocksize) * blocksize; // Align size_t correction = edataoffset - newblock.offset; newblock.end = std::min(edatafilesize, newblock.offset + blocksize); assert(newblock.end >= newblock.offset); int realsize = get_block_uncompressed_size(blockfilename, newblock.end - newblock.offset); iomgr->managed_malloc(edata_session, &newblock.data, realsize, newblock.offset); newblock.ptr = newblock.data + correction; activeblocks.push_back(newblock); curblock = &activeblocks[activeblocks.size()-1]; curblock->active = true; curblock->read_now(iomgr); } }
void releaseblocks() { for(int i=0; i < (int)loadedblocks.size(); i++) { delete(loadedblocks[i].dblock); iomgr->managed_release(loadedblocks[i].fd, &loadedblocks[i].data); iomgr->close_session(loadedblocks[i].fd); loadedblocks[i].data = NULL; loadedblocks[i].dblock = NULL; } loadedblocks.clear(); }
~memory_shard() { if (edata_iosession >= 0) { if (edgedata != NULL) iomgr->managed_release(edata_iosession, &edgedata); iomgr->close_session(edata_iosession); } if (adj_session >= 0) { if (adjdata != NULL) iomgr->managed_release(adj_session, &adjdata); iomgr->close_session(adj_session); } }
vdblock load_block(int blockid) { vdblock db(blockid); std::string blockfname = blockfilename(blockid); db.fd = iomgr->open_session(blockfname, false, true); int realsize = get_block_uncompressed_size(blockfname, -1); assert(realsize > 0); iomgr->managed_malloc(db.fd, &db.data, realsize, 0); iomgr->managed_preada_now(db.fd, &db.data, realsize, 0); db.dblock = new dynamicdata_block<VertexDataType>(verticesperblock, (uint8_t *)db.data, realsize); return db; }
virtual void open_file(std::string base_filename) { filename = filename_degree_data(base_filename); modified = false; if (!use_mmap) { iomgr->allow_preloading(filename); filedesc = iomgr->open_session(filename.c_str(), false); } else { mmap_length = get_filesize(filename); filedesc = open(filename.c_str(), O_RDWR); mmap_file = (degree *) mmap(NULL, mmap_length, PROT_READ | PROT_WRITE, MAP_SHARED, filedesc, 0); assert(mmap_file); } }
virtual ~degree_data() { if (!use_mmap) { if (loaded_chunk != NULL) { iomgr->managed_release(filedesc, &loaded_chunk); } iomgr->close_session(filedesc); } else { if (modified) { msync(mmap_file, mmap_length, MS_SYNC); } munmap(mmap_file, mmap_length); close(filedesc); } }
/** * Saves the current chunk of vertex values */ virtual void save(bool async=false) { if (!use_mmap) { assert(loaded_chunk != NULL); size_t datasize = (vertex_en - vertex_st + 1) * sizeof(VertexDataType); size_t datastart = vertex_st * sizeof(VertexDataType); if (async) { iomgr->managed_pwritea_async(filedesc, &loaded_chunk, datasize, datastart, false); } else { iomgr->managed_pwritea_now(filedesc, &loaded_chunk, datasize, datastart); } } else { // do nothing } }
virtual ~vertex_data_store() { if (!use_mmap) { iomgr->close_session(filedesc); iomgr->wait_for_writes(); if (loaded_chunk != NULL) { iomgr->managed_release(filedesc, &loaded_chunk); } } else { logstream(LOG_INFO) << "Syncing vertex data..." << std::endl; msync(mmap_file, mmap_length, MS_SYNC); munmap(mmap_file, mmap_length); close(filedesc); } }
/** * Loads a chunk of vertex degrees * @param vertex_st first vertex id * @param vertex_en last vertex id, inclusive */ virtual void load(vid_t _vertex_st, vid_t _vertex_en) { assert(_vertex_en >= _vertex_st); vertex_st = _vertex_st; vertex_en = _vertex_en; size_t datasize = (vertex_en - vertex_st + 1) * sizeof(degree); size_t datastart = vertex_st * sizeof(degree); if (loaded_chunk != NULL) { iomgr->managed_release(filedesc, &loaded_chunk); } iomgr->managed_malloc(filedesc, &loaded_chunk, datasize, datastart); iomgr->managed_preada_now(filedesc, &loaded_chunk, datasize, datastart); }
void save() { if (!use_mmap) { size_t datasize = (vertex_en - vertex_st + 1) * sizeof(degree); size_t datastart = vertex_st * sizeof(degree); iomgr->managed_pwritea_now(filedesc, &loaded_chunk, datasize, datastart); } }
void write_block(vdblock &block) { int realsize; uint8_t * outdata; block.dblock->write(&outdata, realsize); std::string blockfname = blockfilename(block.blockid); iomgr->managed_pwritea_now(block.fd, &outdata, realsize, 0); /* Need to write whole block in the compressed regime */ write_block_uncompressed_size(blockfname, realsize); }
// TODO: recycle ptr! void load() { is_loaded = true; adjfilesize = get_filesize(filename_adj); edatafilesize = get_filesize(filename_edata); bool async_inedgedata_loading = !svertex_t().computational_edges(); #ifdef SUPPORT_DELETIONS async_inedgedata_loading = false; // Currently we encode the deleted status of an edge into the edge value (should be changed!), // so we need the edge data while loading #endif //preada(adjf, adjdata, adjfilesize, 0); adj_session = iomgr->open_session(filename_adj, true); iomgr->managed_malloc(adj_session, &adjdata, adjfilesize, 0); adj_stream_session = streaming_task(iomgr, adj_session, adjfilesize, (char**) &adjdata); iomgr->launch_stream_reader(&adj_stream_session); /* Initialize edge data asynchonous reading */ if (!only_adjacency) { edata_iosession = iomgr->open_session(filename_edata, false); iomgr->managed_malloc(edata_iosession, &edgedata, edatafilesize, 0); if (async_inedgedata_loading) { iomgr->managed_preada_async(edata_iosession, &edgedata, edatafilesize, 0); } else { iomgr->managed_preada_now(edata_iosession, &edgedata, edatafilesize, 0); } } }
void ensure_size(vid_t maxid) { if (!use_mmap) { iomgr->truncate(filedesc, (1 + maxid) * sizeof(degree)); } else { munmap(mmap_file, mmap_length); ftruncate(filedesc, (1 + maxid) * sizeof(degree)); close(filedesc); open_file(base_filename); } }
virtual void open_file() { if (!use_mmap) { filedesc = iomgr->open_session(filename.c_str(), false); } else { mmap_length = get_filesize(filename); filedesc = open(filename.c_str(), O_RDWR); mmap_file = (VertexDataType *) mmap(NULL, mmap_length, PROT_WRITE | PROT_READ, MAP_SHARED, filedesc, 0); assert(mmap_file); } }
/** * Loads a chunk of vertex values * @param vertex_st first vertex id * @param vertex_en last vertex id, inclusive */ virtual void load(vid_t _vertex_st, vid_t _vertex_en) { if (!use_mmap) { assert(_vertex_en >= _vertex_st); vertex_st = _vertex_st; vertex_en = _vertex_en; size_t datasize = (vertex_en - vertex_st + 1)* sizeof(VertexDataType); size_t datastart = vertex_st * sizeof(VertexDataType); if (loaded_chunk != NULL) { iomgr->managed_release(filedesc, &loaded_chunk); } iomgr->managed_malloc(filedesc, &loaded_chunk, datasize, datastart); iomgr->managed_preada_now(filedesc, &loaded_chunk, datasize, datastart); } else { // Do nothing } }
inline void check_adjblock(size_t toread) { if (curadjblock == NULL || curadjblock->end <= adjoffset + toread) { if (curadjblock != NULL) { curadjblock->release(iomgr); delete curadjblock; curadjblock = NULL; } sblock<ET> * newblock = new sblock<ET>(0, adjfile_session); newblock->offset = adjoffset; newblock->end = std::min(adjfilesize, adjoffset+blocksize); assert(newblock->end > 0); assert(newblock->end >= newblock->offset); iomgr->managed_malloc(adjfile_session, &newblock->data, newblock->end - newblock->offset, adjoffset); newblock->ptr = newblock->data; metrics_entry me = m.start_time(); iomgr->managed_preada_now(adjfile_session, &newblock->data, newblock->end - newblock->offset, adjoffset); m.stop_time(me, "blockload"); curadjblock = newblock; } }
~sliding_shard() { release_prior_to_offset(true); if (curblock != NULL) { curblock->release(iomgr); delete curblock; curblock = NULL; } if (curadjblock != NULL) { curadjblock->release(iomgr); delete curadjblock; curadjblock = NULL; } iomgr->close_session(adjfile_session); }
sliding_shard(stripedio * iomgr, std::string _filename_edata, std::string _filename_adj, vid_t _range_st, vid_t _range_en, size_t _blocksize, metrics &_m, bool _disable_writes=false, bool onlyadj = false) : iomgr(iomgr), filename_edata(_filename_edata), filename_adj(_filename_adj), range_st(_range_st), range_end(_range_en), blocksize(_blocksize), m(_m), disable_writes(_disable_writes) { curvid = 0; adjoffset = 0; edataoffset = 0; disable_writes = false; only_adjacency = onlyadj; curblock = NULL; curadjblock = NULL; window_start_edataoffset = 0; disable_async_writes = false; while(blocksize % sizeof(int) != 0) blocksize++; assert(blocksize % sizeof(int)==0); adjfilesize = get_filesize(filename_adj); edatafilesize = get_shard_edata_filesize<int>(filename_edata); if (!only_adjacency) { logstream(LOG_DEBUG) << "Total edge data size: " << edatafilesize << std::endl; } else { // Nothing } adjfile_session = iomgr->open_session(filename_adj, true); save_offset(); async_edata_loading = false; // With dynamic edge data size, do not load }
virtual ~degree_data() { if (loaded_chunk != NULL) { iomgr->managed_release(filedesc, &loaded_chunk); } iomgr->close_session(filedesc); }
virtual void open_file(std::string base_filename) { filename = filename_degree_data(base_filename); iomgr->allow_preloading(filename); filedesc = iomgr->open_session(filename.c_str(), false); }
void ensure_size(vid_t maxid) { iomgr->truncate(filedesc, (1 + maxid) * sizeof(degree)); }
/** * Read out-edges for vertices. */ void read_next_vertices(int nvecs, vid_t start, std::vector<svertex_t> & prealloc, bool record_index=false, bool disable_writes=false) { metrics_entry me = m.start_time(); if (!record_index) move_close_to(start); /* Release the blocks we do not need anymore */ curblock = NULL; release_prior_to_offset(false, disable_writes); assert(activeblocks.size() <= 1); /* Read next */ if (!activeblocks.empty() && !only_adjacency) { curblock = &activeblocks[0]; } vid_t lastrec = start; window_start_edataoffset = edataoffset; for(int i=((int)curvid) - ((int)start); i<nvecs; i++) { if (adjoffset >= adjfilesize) break; // TODO: skip unscheduled vertices. int n; if (record_index && (size_t)(curvid - lastrec) >= (size_t) std::max((int)100000, nvecs/16)) { save_offset(); lastrec = curvid; } uint8_t ns = read_val<uint8_t>(); if (ns == 0x00) { curvid++; uint8_t nz = read_val<uint8_t>(); curvid += nz; i += nz; continue; } if (ns == 0xff) { n = read_val<uint32_t>(); } else { n = ns; } if (i<0) { // Just skipping skip(n, sizeof(vid_t)); } else { svertex_t& vertex = prealloc[i]; assert(vertex.id() == curvid); if (vertex.scheduled) { while(--n >= 0) { bool special_edge = false; vid_t target = (sizeof(ET) == sizeof(ETspecial) ? read_val<vid_t>() : translate_edge(read_val<vid_t>(), special_edge)); ET * evalue = read_edgeptr(); vertex.add_outedge(target, evalue, special_edge); if (!((target >= range_st && target <= range_end))) { logstream(LOG_ERROR) << "Error : " << target << " not in [" << range_st << " - " << range_end << "]" << std::endl; iomgr->print_session(adjfile_session); } assert(target >= range_st && target <= range_end); } } else { // This vertex was not scheduled, so we can just skip its edges. skip(n, sizeof(vid_t)); } } curvid++; } m.stop_time(me, "read_next_vertices"); curblock = NULL; }
virtual ~vertex_data_store() { iomgr->wait_for_writes(); releaseblocks(); }