void check_and_forward(FILE *fp, FILE *sfp, int s)
{
char buf[MAX_LINE];
char esc_char= ESC_CHAR;
int len=0, total_len=0, ack_len=0;
total_len=0;
/* reset the EOF */
clearerr(fp);
while(fgets(buf, sizeof(buf), fp)){
len = strlen(buf);
total_len += len;
if(buf[0]==ESC_CHAR || buf[0]==CMD_CHAR || buf[0]== '\n'){
total_len ++;
if(writen(s, &esc_char, 1)<1){
syslog(LOG_CRIT|FACILITY,
"sending messages failed: %s",strerror(errno));
sig_term(1);
}
}
if(writen(s, buf, len)<len){
syslog(LOG_CRIT|FACILITY,
"sending messages failed: %s",strerror(errno));
sig_term(1);
}
}
if(total_len>0){
/********************* IMPORTANT ************************/
/*** Here send one additional '\n' to ensure boundary ***/
/********************************************************/
/* send "segment boundary information"
solicit acknowledgement */
sprintf(buf, "\n%c\n", CMD_CHAR);
len= 3;
if(writen(s, buf, len)<len){
syslog(LOG_CRIT|FACILITY,
"sending messages failed: %s",strerror(errno));
sig_term(1);
}
if((ack_len= readline(s, buf, sizeof(buf)))<0){
syslog(LOG_CRIT|FACILITY,
"cannot get acks, %s",strerror(errno));
sig_term(1);
}
if(buf[0]==CMD_CHAR && atoi(buf+1)== total_len+3)
save_offset(sfp, ftell(fp));
else{ /* this should not happen */
syslog(LOG_CRIT|FACILITY,
"sending messages failed unexpectedly");
sig_term(1);
}
}
}
sliding_shard(stripedio * iomgr, std::string _filename_edata, std::string _filename_adj, vid_t _range_st, vid_t _range_en, size_t _blocksize, metrics &_m,
bool _disable_writes=false, bool onlyadj = false) :
iomgr(iomgr),
filename_edata(_filename_edata),
filename_adj(_filename_adj),
range_st(_range_st),
range_end(_range_en),
blocksize(_blocksize),
m(_m),
disable_writes(_disable_writes) {
curvid = 0;
adjoffset = 0;
edataoffset = 0;
disable_writes = false;
only_adjacency = onlyadj;
curblock = NULL;
curadjblock = NULL;
window_start_edataoffset = 0;
disable_async_writes = false;
while(blocksize % sizeof(int) != 0) blocksize++;
assert(blocksize % sizeof(int)==0);
adjfilesize = get_filesize(filename_adj);
edatafilesize = get_shard_edata_filesize<int>(filename_edata);
if (!only_adjacency) {
logstream(LOG_DEBUG) << "Total edge data size: " << edatafilesize << std::endl;
} else {
// Nothing
}
adjfile_session = iomgr->open_session(filename_adj, true);
save_offset();
async_edata_loading = false; // With dynamic edge data size, do not load
}
/**
* Read out-edges for vertices.
*/
void read_next_vertices(int nvecs, vid_t start, std::vector<svertex_t> & prealloc, bool record_index=false, bool disable_writes=false) {
metrics_entry me = m.start_time();
if (!record_index)
move_close_to(start);
/* Release the blocks we do not need anymore */
curblock = NULL;
release_prior_to_offset(false, disable_writes);
assert(activeblocks.size() <= 1);
/* Read next */
if (!activeblocks.empty() && !only_adjacency) {
curblock = &activeblocks[0];
}
vid_t lastrec = start;
window_start_edataoffset = edataoffset;
for(int i=((int)curvid) - ((int)start); i<nvecs; i++) {
if (adjoffset >= adjfilesize) break;
// TODO: skip unscheduled vertices.
int n;
if (record_index && (size_t)(curvid - lastrec) >= (size_t) std::max((int)100000, nvecs/16)) {
save_offset();
lastrec = curvid;
}
uint8_t ns = read_val<uint8_t>();
if (ns == 0x00) {
curvid++;
uint8_t nz = read_val<uint8_t>();
curvid += nz;
i += nz;
continue;
}
if (ns == 0xff) {
n = read_val<uint32_t>();
} else {
n = ns;
}
if (i<0) {
// Just skipping
skip(n, sizeof(vid_t));
} else {
svertex_t& vertex = prealloc[i];
assert(vertex.id() == curvid);
if (vertex.scheduled) {
while(--n >= 0) {
bool special_edge = false;
vid_t target = (sizeof(ET) == sizeof(ETspecial) ? read_val<vid_t>() : translate_edge(read_val<vid_t>(), special_edge));
ET * evalue = read_edgeptr();
vertex.add_outedge(target, evalue, special_edge);
if (!((target >= range_st && target <= range_end))) {
logstream(LOG_ERROR) << "Error : " << target << " not in [" << range_st << " - " << range_end << "]" << std::endl;
iomgr->print_session(adjfile_session);
}
assert(target >= range_st && target <= range_end);
}
} else {
// This vertex was not scheduled, so we can just skip its edges.
skip(n, sizeof(vid_t));
}
}
curvid++;
}
m.stop_time(me, "read_next_vertices");
curblock = NULL;
}