// TODO: recycle ptr! void load() { is_loaded = true; adjfilesize = get_filesize(filename_adj); edatafilesize = get_filesize(filename_edata); bool async_inedgedata_loading = !svertex_t().computational_edges(); #ifdef SUPPORT_DELETIONS async_inedgedata_loading = false; // Currently we encode the deleted status of an edge into the edge value (should be changed!), // so we need the edge data while loading #endif //preada(adjf, adjdata, adjfilesize, 0); adj_session = iomgr->open_session(filename_adj, true); iomgr->managed_malloc(adj_session, &adjdata, adjfilesize, 0); adj_stream_session = streaming_task(iomgr, adj_session, adjfilesize, (char**) &adjdata); iomgr->launch_stream_reader(&adj_stream_session); /* Initialize edge data asynchonous reading */ if (!only_adjacency) { edata_iosession = iomgr->open_session(filename_edata, false); iomgr->managed_malloc(edata_iosession, &edgedata, edatafilesize, 0); if (async_inedgedata_loading) { iomgr->managed_preada_async(edata_iosession, &edgedata, edatafilesize, 0); } else { iomgr->managed_preada_now(edata_iosession, &edgedata, edatafilesize, 0); } } }
inline void check_curblock(size_t toread) { if (curblock == NULL || curblock->end < edataoffset+toread) { if (curblock != NULL) { if (!curblock->active) { curblock->release(iomgr); } } // Load next std::string blockfilename = filename_shard_edata_block(filename_edata, (int) (edataoffset / blocksize), blocksize); int edata_session = iomgr->open_session(blockfilename, false, true); sblock<ET> newblock(edata_session, edata_session, true, blockfilename); // We align blocks always to the blocksize, even if that requires // allocating and reading some unnecessary data. newblock.offset = (edataoffset / blocksize) * blocksize; // Align size_t correction = edataoffset - newblock.offset; newblock.end = std::min(edatafilesize, newblock.offset + blocksize); assert(newblock.end >= newblock.offset); int realsize = get_block_uncompressed_size(blockfilename, newblock.end - newblock.offset); iomgr->managed_malloc(edata_session, &newblock.data, realsize, newblock.offset); newblock.ptr = newblock.data + correction; activeblocks.push_back(newblock); curblock = &activeblocks[activeblocks.size()-1]; curblock->active = true; curblock->read_now(iomgr); } }
virtual void open_file() { if (!use_mmap) { filedesc = iomgr->open_session(filename.c_str(), false); } else { mmap_length = get_filesize(filename); filedesc = open(filename.c_str(), O_RDWR); mmap_file = (VertexDataType *) mmap(NULL, mmap_length, PROT_WRITE | PROT_READ, MAP_SHARED, filedesc, 0); assert(mmap_file); } }
vdblock load_block(int blockid) { vdblock db(blockid); std::string blockfname = blockfilename(blockid); db.fd = iomgr->open_session(blockfname, false, true); int realsize = get_block_uncompressed_size(blockfname, -1); assert(realsize > 0); iomgr->managed_malloc(db.fd, &db.data, realsize, 0); iomgr->managed_preada_now(db.fd, &db.data, realsize, 0); db.dblock = new dynamicdata_block<VertexDataType>(verticesperblock, (uint8_t *)db.data, realsize); return db; }
virtual void open_file(std::string base_filename) { filename = filename_degree_data(base_filename); modified = false; if (!use_mmap) { iomgr->allow_preloading(filename); filedesc = iomgr->open_session(filename.c_str(), false); } else { mmap_length = get_filesize(filename); filedesc = open(filename.c_str(), O_RDWR); mmap_file = (degree *) mmap(NULL, mmap_length, PROT_READ | PROT_WRITE, MAP_SHARED, filedesc, 0); assert(mmap_file); } }
sliding_shard(stripedio * iomgr, std::string _filename_edata, std::string _filename_adj, vid_t _range_st, vid_t _range_en, size_t _blocksize, metrics &_m, bool _disable_writes=false, bool onlyadj = false) : iomgr(iomgr), filename_edata(_filename_edata), filename_adj(_filename_adj), range_st(_range_st), range_end(_range_en), blocksize(_blocksize), m(_m), disable_writes(_disable_writes) { curvid = 0; adjoffset = 0; edataoffset = 0; disable_writes = false; only_adjacency = onlyadj; curblock = NULL; curadjblock = NULL; window_start_edataoffset = 0; disable_async_writes = false; while(blocksize % sizeof(int) != 0) blocksize++; assert(blocksize % sizeof(int)==0); adjfilesize = get_filesize(filename_adj); edatafilesize = get_shard_edata_filesize<int>(filename_edata); if (!only_adjacency) { logstream(LOG_DEBUG) << "Total edge data size: " << edatafilesize << std::endl; } else { // Nothing } adjfile_session = iomgr->open_session(filename_adj, true); save_offset(); async_edata_loading = false; // With dynamic edge data size, do not load }
virtual void open_file(std::string base_filename) { filename = filename_degree_data(base_filename); iomgr->allow_preloading(filename); filedesc = iomgr->open_session(filename.c_str(), false); }