static int noit_jlog_push(eventer_t e, noit_jlog_closure_t *jcl) { jlog_message msg; int mask; u_int32_t n_count; n_count = htonl(jcl->count); if(Ewrite(&n_count, sizeof(n_count)) != sizeof(n_count)) return -1; while(jcl->count > 0) { int rv; struct { jlog_id chkpt; u_int32_t n_sec, n_usec, n_len; } payload; if(jlog_ctx_read_message(jcl->jlog, &jcl->start, &msg) == -1) return -1; /* Here we actually push the message */ payload.chkpt.log = htonl(jcl->start.log); payload.chkpt.marker = htonl(jcl->start.marker); payload.n_sec = htonl(msg.header->tv_sec); payload.n_usec = htonl(msg.header->tv_usec); payload.n_len = htonl(msg.mess_len); if((rv = Ewrite(&payload, sizeof(payload))) != sizeof(payload)) { noitL(noit_error, "Error writing jlog header over SSL %d != %d\n", rv, (int)sizeof(payload)); return -1; } if((rv = Ewrite(msg.mess, msg.mess_len)) != msg.mess_len) { noitL(noit_error, "Error writing jlog message over SSL %d != %d\n", rv, msg.mess_len); return -1; } /* Note what the client must checkpoint */ jcl->chkpt = jcl->start; JLOG_ID_ADVANCE(&jcl->start); jcl->count--; } return 0; }
static fq_msg *queue_jlog_dequeue(fqd_queue_impl_data f) { struct queue_jlog *d = (struct queue_jlog *)f; jlog_message msg; fq_msg *m; if(d->count == 0 && d->last_seen_nenqueued == d->nenqueued) return NULL; retry: if(d->count <= 0) { d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish); fq_debug(FQ_DEBUG_IO, "jlog read batch count -> %d\n", d->count); if(d->count < 0) { char idxfile[PATH_MAX]; fq_debug(FQ_DEBUG_IO, "jlog_ctx_read_interval: %s\n", jlog_ctx_err_string(d->reader)); switch (jlog_ctx_err(d->reader)) { case JLOG_ERR_FILE_CORRUPT: case JLOG_ERR_IDX_CORRUPT: jlog_repair_datafile(d->reader, d->start.log); jlog_repair_datafile(d->reader, d->start.log + 1); fq_debug(FQ_DEBUG_IO, "jlog reconstructed, deleting corresponding index.\n"); STRSETDATAFILE(d->reader, idxfile, d->start.log); strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile)); unlink(idxfile); STRSETDATAFILE(d->reader, idxfile, d->start.log + 1); strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile)); unlink(idxfile); break; default: break; } } if(d->count <= 0) return NULL; } if(jlog_ctx_read_message(d->reader, &d->start, &msg) == -1) { d->count = 0; return NULL; } if(d->last_dequeued.log > d->start.log || (d->last_dequeued.log == d->start.log && d->last_dequeued.marker > d->start.marker)) { d->count--; JLOG_ID_ADVANCE(&d->start); goto retry; } if(msg.mess_len < sizeof(fq_msg)-1) m = NULL; else { off_t expected_len; uint32_t payload_len; m = (fq_msg *)msg.mess; memcpy(&payload_len, &m->payload_len, sizeof(m->payload_len)); expected_len = offsetof(fq_msg, payload) + payload_len; if(expected_len != msg.mess_len) m = NULL; else { m = malloc(expected_len); memcpy(m, msg.mess, expected_len); m->sender_msgid.id.u32.p3 = d->start.log; m->sender_msgid.id.u32.p4 = d->start.marker; } } d->count--; fq_debug(FQ_DEBUG_IO, "jlog batch count -> %d\n", d->count); if(d->count == 0) { if(d->auto_chkpt) { jlog_ctx_read_checkpoint(d->reader, &d->start); } } d->last_dequeued = d->start; JLOG_ID_ADVANCE(&d->start); ck_pr_inc_uint(&d->last_seen_nenqueued); return m; }
void *reader(void *unused) { jlog_ctx *ctx; char subname[32]; int tcount = 0, fcount = 0; int prev_err = 0; int subno = (int)(uintptr_t)unused; snprintf(subname, sizeof(subname), "sub-%02d", subno); reader_retry: ctx = jlog_new(LOGNAME); if(jlog_ctx_open_reader(ctx, subname) != 0) { if(prev_err == 0) { prev_err = jlog_ctx_err(ctx); jlog_ctx_close(ctx); ctx = jlog_new(LOGNAME); if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) { fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno); if(jlog_ctx_open_writer(ctx) != 0) { fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) { fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { jlog_ctx_close(ctx); goto reader_retry; } } } } fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } fprintf(stderr, "[%02d] reader started\n", subno); while(1) { char begins[20], ends[20]; jlog_id begin, end; int count; jlog_message message; if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) { fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } jlog_snprint_logid(begins, sizeof(begins), &begin); jlog_snprint_logid(ends, sizeof(ends), &end); if(count > 0) { int i; //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count); for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) { end = begin; if(jlog_ctx_read_message(ctx, &begin, &message) != 0) { if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) { /* fine */ } else { fcount++; jlog_snprint_logid(begins, sizeof(begins), &begin); fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } } else { tcount++; jlog_snprint_logid(begins, sizeof(begins), &begin); /* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins, message.mess_len, (char *)message.mess); */ } } if(jlog_ctx_read_checkpoint(ctx, &end) != 0) { fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { fprintf(stderr, "[%02d] \tcheckpointed...\n", subno); } } else { if(writer_done == 1) break; } } fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount); jlog_ctx_close(ctx); return (void *)(uintptr_t)tcount; }