static int
noit_jlog_push(eventer_t e, noit_jlog_closure_t *jcl) {
jlog_message msg;
int mask;
u_int32_t n_count;
n_count = htonl(jcl->count);
if(Ewrite(&n_count, sizeof(n_count)) != sizeof(n_count))
return -1;
while(jcl->count > 0) {
int rv;
struct { jlog_id chkpt; u_int32_t n_sec, n_usec, n_len; } payload;
if(jlog_ctx_read_message(jcl->jlog, &jcl->start, &msg) == -1)
return -1;
/* Here we actually push the message */
payload.chkpt.log = htonl(jcl->start.log);
payload.chkpt.marker = htonl(jcl->start.marker);
payload.n_sec = htonl(msg.header->tv_sec);
payload.n_usec = htonl(msg.header->tv_usec);
payload.n_len = htonl(msg.mess_len);
if((rv = Ewrite(&payload, sizeof(payload))) != sizeof(payload)) {
noitL(noit_error, "Error writing jlog header over SSL %d != %d\n",
rv, (int)sizeof(payload));
return -1;
}
if((rv = Ewrite(msg.mess, msg.mess_len)) != msg.mess_len) {
noitL(noit_error, "Error writing jlog message over SSL %d != %d\n",
rv, msg.mess_len);
return -1;
}
/* Note what the client must checkpoint */
jcl->chkpt = jcl->start;
JLOG_ID_ADVANCE(&jcl->start);
jcl->count--;
}
return 0;
}
static fq_msg *queue_jlog_dequeue(fqd_queue_impl_data f) {
struct queue_jlog *d = (struct queue_jlog *)f;
jlog_message msg;
fq_msg *m;
if(d->count == 0 && d->last_seen_nenqueued == d->nenqueued) return NULL;
retry:
if(d->count <= 0) {
d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish);
fq_debug(FQ_DEBUG_IO, "jlog read batch count -> %d\n", d->count);
if(d->count < 0) {
char idxfile[PATH_MAX];
fq_debug(FQ_DEBUG_IO, "jlog_ctx_read_interval: %s\n",
jlog_ctx_err_string(d->reader));
switch (jlog_ctx_err(d->reader)) {
case JLOG_ERR_FILE_CORRUPT:
case JLOG_ERR_IDX_CORRUPT:
jlog_repair_datafile(d->reader, d->start.log);
jlog_repair_datafile(d->reader, d->start.log + 1);
fq_debug(FQ_DEBUG_IO,
"jlog reconstructed, deleting corresponding index.\n");
STRSETDATAFILE(d->reader, idxfile, d->start.log);
strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile));
unlink(idxfile);
STRSETDATAFILE(d->reader, idxfile, d->start.log + 1);
strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile));
unlink(idxfile);
break;
default:
break;
}
}
if(d->count <= 0) return NULL;
}
if(jlog_ctx_read_message(d->reader, &d->start, &msg) == -1) {
d->count = 0;
return NULL;
}
if(d->last_dequeued.log > d->start.log ||
(d->last_dequeued.log == d->start.log &&
d->last_dequeued.marker > d->start.marker)) {
d->count--;
JLOG_ID_ADVANCE(&d->start);
goto retry;
}
if(msg.mess_len < sizeof(fq_msg)-1)
m = NULL;
else {
off_t expected_len;
uint32_t payload_len;
m = (fq_msg *)msg.mess;
memcpy(&payload_len, &m->payload_len, sizeof(m->payload_len));
expected_len = offsetof(fq_msg, payload) + payload_len;
if(expected_len != msg.mess_len) m = NULL;
else {
m = malloc(expected_len);
memcpy(m, msg.mess, expected_len);
m->sender_msgid.id.u32.p3 = d->start.log;
m->sender_msgid.id.u32.p4 = d->start.marker;
}
}
d->count--;
fq_debug(FQ_DEBUG_IO, "jlog batch count -> %d\n", d->count);
if(d->count == 0) {
if(d->auto_chkpt) {
jlog_ctx_read_checkpoint(d->reader, &d->start);
}
}
d->last_dequeued = d->start;
JLOG_ID_ADVANCE(&d->start);
ck_pr_inc_uint(&d->last_seen_nenqueued);
return m;
}
void *reader(void *unused) {
jlog_ctx *ctx;
char subname[32];
int tcount = 0, fcount = 0;
int prev_err = 0;
int subno = (int)(uintptr_t)unused;
snprintf(subname, sizeof(subname), "sub-%02d", subno);
reader_retry:
ctx = jlog_new(LOGNAME);
if(jlog_ctx_open_reader(ctx, subname) != 0) {
if(prev_err == 0) {
prev_err = jlog_ctx_err(ctx);
jlog_ctx_close(ctx);
ctx = jlog_new(LOGNAME);
if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) {
fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno);
if(jlog_ctx_open_writer(ctx) != 0) {
fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
} else {
if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) {
fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
} else {
jlog_ctx_close(ctx);
goto reader_retry;
}
}
}
}
fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
croak();
}
fprintf(stderr, "[%02d] reader started\n", subno);
while(1) {
char begins[20], ends[20];
jlog_id begin, end;
int count;
jlog_message message;
if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) {
fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
croak();
}
jlog_snprint_logid(begins, sizeof(begins), &begin);
jlog_snprint_logid(ends, sizeof(ends), &end);
if(count > 0) {
int i;
//fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count);
for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) {
end = begin;
if(jlog_ctx_read_message(ctx, &begin, &message) != 0) {
if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) {
/* fine */
} else {
fcount++;
jlog_snprint_logid(begins, sizeof(begins), &begin);
fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
}
} else {
tcount++;
jlog_snprint_logid(begins, sizeof(begins), &begin);
/* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins,
message.mess_len, (char *)message.mess); */
}
}
if(jlog_ctx_read_checkpoint(ctx, &end) != 0) {
fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
} else {
fprintf(stderr, "[%02d] \tcheckpointed...\n", subno);
}
} else {
if(writer_done == 1) break;
}
}
fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount);
jlog_ctx_close(ctx);
return (void *)(uintptr_t)tcount;
}