int jlog_ctx_add_subscriber(jlog_ctx *ctx, const char *s, jlog_position whence) { jlog_id chkpt; jlog_ctx *tmpctx = NULL; jlog_file *jchkpt; ctx->last_error = JLOG_ERR_SUCCESS; jchkpt = __jlog_open_named_checkpoint(ctx, s, O_CREAT|O_EXCL); if(!jchkpt) { ctx->last_error = JLOG_ERR_SUBSCRIBER_EXISTS; ctx->last_errno = EEXIST; return -1; } jlog_file_close(jchkpt); if(whence == JLOG_BEGIN) { memset(&chkpt, 0, sizeof(chkpt)); jlog_ctx_first_log_id(ctx, &chkpt); if(__jlog_set_checkpoint(ctx, s, &chkpt) != 0) { ctx->last_error = JLOG_ERR_CHECKPOINT; ctx->last_errno = 0; return -1; } return 0; } if(whence == JLOG_END) { jlog_id start, finish; memset(&chkpt, 0, sizeof(chkpt)); if(__jlog_open_metastore(ctx) != 0) SYS_FAIL(JLOG_ERR_META_OPEN); if(__jlog_restore_metastore(ctx, 0)) SYS_FAIL(JLOG_ERR_META_OPEN); chkpt.log = ctx->storage.log; if(__jlog_set_checkpoint(ctx, s, &chkpt) != 0) SYS_FAIL(JLOG_ERR_CHECKPOINT); tmpctx = jlog_new(ctx->path); if(jlog_ctx_open_reader(tmpctx, s) < 0) goto finish; if(jlog_ctx_read_interval(tmpctx, &start, &finish) < 0) goto finish; jlog_ctx_close(tmpctx); tmpctx = NULL; if(__jlog_set_checkpoint(ctx, s, &finish) != 0) SYS_FAIL(JLOG_ERR_CHECKPOINT); return 0; } ctx->last_error = JLOG_ERR_NOT_SUPPORTED; finish: if(tmpctx) jlog_ctx_close(tmpctx); return -1; }
static fq_msg *queue_jlog_dequeue(fqd_queue_impl_data f) { struct queue_jlog *d = (struct queue_jlog *)f; jlog_message msg; fq_msg *m; if(d->count == 0 && d->last_seen_nenqueued == d->nenqueued) return NULL; retry: if(d->count <= 0) { d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish); fq_debug(FQ_DEBUG_IO, "jlog read batch count -> %d\n", d->count); if(d->count < 0) { char idxfile[PATH_MAX]; fq_debug(FQ_DEBUG_IO, "jlog_ctx_read_interval: %s\n", jlog_ctx_err_string(d->reader)); switch (jlog_ctx_err(d->reader)) { case JLOG_ERR_FILE_CORRUPT: case JLOG_ERR_IDX_CORRUPT: jlog_repair_datafile(d->reader, d->start.log); jlog_repair_datafile(d->reader, d->start.log + 1); fq_debug(FQ_DEBUG_IO, "jlog reconstructed, deleting corresponding index.\n"); STRSETDATAFILE(d->reader, idxfile, d->start.log); strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile)); unlink(idxfile); STRSETDATAFILE(d->reader, idxfile, d->start.log + 1); strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile)); unlink(idxfile); break; default: break; } } if(d->count <= 0) return NULL; } if(jlog_ctx_read_message(d->reader, &d->start, &msg) == -1) { d->count = 0; return NULL; } if(d->last_dequeued.log > d->start.log || (d->last_dequeued.log == d->start.log && d->last_dequeued.marker > d->start.marker)) { d->count--; JLOG_ID_ADVANCE(&d->start); goto retry; } if(msg.mess_len < sizeof(fq_msg)-1) m = NULL; else { off_t expected_len; uint32_t payload_len; m = (fq_msg *)msg.mess; memcpy(&payload_len, &m->payload_len, sizeof(m->payload_len)); expected_len = offsetof(fq_msg, payload) + payload_len; if(expected_len != msg.mess_len) m = NULL; else { m = malloc(expected_len); memcpy(m, msg.mess, expected_len); m->sender_msgid.id.u32.p3 = d->start.log; m->sender_msgid.id.u32.p4 = d->start.marker; } } d->count--; fq_debug(FQ_DEBUG_IO, "jlog batch count -> %d\n", d->count); if(d->count == 0) { if(d->auto_chkpt) { jlog_ctx_read_checkpoint(d->reader, &d->start); } } d->last_dequeued = d->start; JLOG_ID_ADVANCE(&d->start); ck_pr_inc_uint(&d->last_seen_nenqueued); return m; }
/* * returns -1 when the incoming checkpoint is out of range of the log * returns -2 if there was an error actually setting the checkpoint */ static int queue_log_add_checkpoint(fqd_queue_impl_data data, const char *name, const fq_msgid *id) { struct queue_jlog *d = (struct queue_jlog *)data; jlog_id jid = { .log = id->id.u32.p1, .marker = id->id.u32.p2 }; /* ensure the checkpoint makes sense */ jlog_id first = { .log = 0, .marker = 0 }; jlog_id last = { .log = 0, .marker = 0 }; jlog_ctx_first_log_id(d->reader, &first); jlog_ctx_last_log_id(d->reader, &last); if (! (jid.log >= first.log && jid.log <= last.log && jid.marker >= first.marker && jid.marker <= last.marker)) { return -1; } char **subs; int sub_count = jlog_ctx_list_subscribers(d->reader, &subs); int have_it = 0; for (int i = 0; i < sub_count; i++) { have_it += strcmp(subs[i], name) == 0 ? 1 : 0; } if (have_it == 0) { jlog_ctx_add_subscriber(d->reader, name, JLOG_BEGIN); } if (jlog_ctx_read_checkpoint(d->reader, &jid) == -1) { /* If we failed to checkpoint we are in a situation where the 'add_subscriber' call above put them at the beginning of the log so we have to remove the subscriber if we just added them However, if they already existed and had a previous good checkpoint, leave it alone */ if (have_it == 0) { jlog_ctx_remove_subscriber(d->reader, name); } return -2; } return 0; } /* * return -1 if the subscriber doesn't exist * return 0 on success */ static int queue_log_remove_checkpoint(fqd_queue_impl_data data, const char *name) { struct queue_jlog *d = (struct queue_jlog *)data; if (jlog_ctx_remove_subscriber(d->reader, name) == 0) { return -1; } return 0; } /* * return -1 if the subscriber doesn't exist * return -2 if we can't reset the checkpoint * return 0 on success */ static int queue_log_reset_to_checkpoint(fqd_queue_impl_data data, const char *name) { struct queue_jlog *d = (struct queue_jlog *)data; char **subs; int sub_count = jlog_ctx_list_subscribers(d->reader, &subs); int have_it = 0; for (int i = 0; i < sub_count; i++) { have_it += strcmp(subs[i], name) == 0 ? 1 : 0; } if (have_it == 0) { return -1; } jlog_id checkpoint; if (jlog_get_checkpoint(d->reader, name, &checkpoint) == -1) { return -2; } if (jlog_ctx_read_checkpoint(d->reader, &checkpoint) == -1) { return -2; } return 0; } static int write_sig(struct queue_jlog *d) { char sigfile[PATH_MAX]; int fd; snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath); fd = open(sigfile, O_CREAT|O_TRUNC|O_WRONLY, 0640); if(fd < 0) return -1; write(fd, d->uuid, 16); close(fd); return 0; } static int read_sig(struct queue_jlog *d, uuid_t out) { char sigfile[PATH_MAX]; int fd, rv; snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath); fd = open(sigfile, O_RDONLY); if(fd < 0) return -1; rv = read(fd, out, 16); close(fd); return (rv == 16) ? 0 : -1; } static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) { char qpath[PATH_MAX]; jlog_id chkpt; struct queue_jlog *d; d = calloc(1, sizeof(*d)); d->auto_chkpt = true; fqd_config_construct_queue_path(qpath, sizeof(qpath), qname); d->qpath = strdup(qpath); d->writer = jlog_new(d->qpath); jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024); jlog_ctx_set_multi_process(d->writer, 0); jlog_ctx_set_use_compression(d->writer, 1); if(jlog_ctx_open_writer(d->writer) != 0) { jlog_ctx_close(d->writer); d->writer = jlog_new(d->qpath); jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024); jlog_ctx_set_multi_process(d->writer, 0); jlog_ctx_set_use_compression(d->writer, 1); if(jlog_ctx_init(d->writer) != 0) { fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer)); goto bail; } jlog_ctx_close(d->writer); d->writer = jlog_new(d->qpath); jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024); jlog_ctx_set_multi_process(d->writer, 0); jlog_ctx_set_use_compression(d->writer, 1); if(jlog_ctx_open_writer(d->writer) != 0) { fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer)); goto bail; } } /* 128MB journal chunks */ jlog_ctx_alter_journal_size(d->writer, 128 * 1024 * 1024); d->reader = jlog_new(d->qpath); if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) { if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) { fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader)); goto bail; } } if(jlog_ctx_open_reader(d->reader, "fq") != 0) { fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader)); goto bail; } uuid_generate(d->uuid); write_sig(d); *count = d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish); (void)qname; return d; bail: if(d->writer) jlog_ctx_close(d->writer); if(d->reader) jlog_ctx_close(d->reader); free(d->qpath); free(d); return NULL; }
void * noit_jlog_thread_main(void *e_vptr) { int mask, bytes_read; eventer_t e = e_vptr; acceptor_closure_t *ac = e->closure; noit_jlog_closure_t *jcl = ac->service_ctx; char inbuff[sizeof(jlog_id)]; eventer_set_fd_blocking(e->fd); while(1) { jlog_id client_chkpt; int sleeptime = (ac->cmd == NOIT_JLOG_DATA_TEMP_FEED) ? 1 : DEFAULT_SECONDS_BETWEEN_BATCHES; jlog_get_checkpoint(jcl->jlog, ac->remote_cn, &jcl->chkpt); jcl->count = jlog_ctx_read_interval(jcl->jlog, &jcl->start, &jcl->finish); if(jcl->count < 0) { char idxfile[PATH_MAX]; noitL(noit_error, "jlog_ctx_read_interval: %s\n", jlog_ctx_err_string(jcl->jlog)); switch (jlog_ctx_err(jcl->jlog)) { case JLOG_ERR_FILE_CORRUPT: case JLOG_ERR_IDX_CORRUPT: jlog_repair_datafile(jcl->jlog, jcl->start.log); jlog_repair_datafile(jcl->jlog, jcl->start.log + 1); noitL(noit_error, "jlog reconstructed, deleting corresponding index.\n"); STRSETDATAFILE(jcl->jlog, idxfile, jcl->start.log); strlcat(idxfile, INDEX_EXT, sizeof(idxfile)); unlink(idxfile); STRSETDATAFILE(jcl->jlog, idxfile, jcl->start.log + 1); strlcat(idxfile, INDEX_EXT, sizeof(idxfile)); unlink(idxfile); goto alldone; break; default: goto alldone; } } if(jcl->count > MAX_ROWS_AT_ONCE) { /* Artificially set down the range to make the batches a bit easier * to handle on the stratcond/postgres end. * However, we must have more data, so drop the sleeptime to 0 */ jcl->count = MAX_ROWS_AT_ONCE; jcl->finish.marker = jcl->start.marker + jcl->count; sleeptime = 0; } if(jcl->count > 0) { if(noit_jlog_push(e, jcl)) { goto alldone; } /* Read our jlog_id accounting for possibly short reads */ bytes_read = 0; while(bytes_read < sizeof(jlog_id)) { int len; if((len = e->opset->read(e->fd, inbuff + bytes_read, sizeof(jlog_id) - bytes_read, &mask, e)) <= 0) goto alldone; bytes_read += len; } memcpy(&client_chkpt, inbuff, sizeof(jlog_id)); /* Fix the endian */ client_chkpt.log = ntohl(client_chkpt.log); client_chkpt.marker = ntohl(client_chkpt.marker); if(memcmp(&jcl->chkpt, &client_chkpt, sizeof(jlog_id))) { noitL(noit_error, "client %s submitted invalid checkpoint %u:%u expected %u:%u\n", ac->remote_cn, client_chkpt.log, client_chkpt.marker, jcl->chkpt.log, jcl->chkpt.marker); goto alldone; } gettimeofday(&jcl->feed_stats->last_checkpoint, NULL); jlog_ctx_read_checkpoint(jcl->jlog, &jcl->chkpt); } else { /* we have nothing to write -- maybe we have no checks configured... * If this is the case "forever", the remote might disconnect and * we would never know. Do the painful work of detecting a * disconnected client. */ struct pollfd pfd; pfd.fd = e->fd; pfd.events = POLLIN | POLLHUP | POLLRDNORM; pfd.revents = 0; if(poll(&pfd, 1, 0) != 0) { /* normally, we'd recv PEEK|DONTWAIT. However, the client should * not be writing to us. So, we know we can't have any legitimate * data on this socket (true even though this is SSL). So, if we're * here then "shit went wrong" */ noitL(noit_error, "jlog client %s disconnected while idle\n", ac->remote_cn); goto alldone; } } if(sleeptime) sleep(sleeptime); } alldone: e->opset->close(e->fd, &mask, e); noit_atomic_dec32(&jcl->feed_stats->connections); noit_jlog_closure_free(jcl); acceptor_closure_free(ac); return NULL; }
static int process_jlog(const char *file, const char *sub) { jlog_ctx *log = jlog_new(file); if (add_subscriber) { if (jlog_ctx_add_subscriber(log, add_subscriber, JLOG_BEGIN)) { fprintf(stderr, "Could not add subscriber '%s': %s\n", add_subscriber, jlog_ctx_err_string(log)); } else { OUT("Added subscriber '%s'\n", add_subscriber); } } if (remove_subscriber) { if (jlog_ctx_remove_subscriber(log, remove_subscriber) <= 0) { fprintf(stderr, "Could not erase subscriber '%s': %s\n", remove_subscriber, jlog_ctx_err_string(log)); } else { OUT("Erased subscriber '%s'\n", remove_subscriber); } } if (!sub) { if (jlog_ctx_open_writer(log)) { fprintf(stderr, "error opening '%s'\n", file); return 0; } } else { if (jlog_ctx_open_reader(log, sub)) { fprintf(stderr, "error opening '%s'\n", file); return 0; } } if (show_progress) { char buff[20], buff2[20], buff3[20]; jlog_id id, id2, id3; jlog_get_checkpoint(log, sub, &id); if (jlog_ctx_last_log_id(log, &id3)) { fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log)); fprintf(stderr, "error calling jlog_ctx_last_log_id\n"); } jlog_snprint_logid(buff, sizeof(buff), &id); jlog_snprint_logid(buff3, sizeof(buff3), &id3); OUT("--------------------\n" " Perspective of the '%s' subscriber\n" " current checkpoint: %s\n" " Last write: %s\n", sub, buff, buff3); if (jlog_ctx_read_interval(log, &id, &id2) < 0) { fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log)); } jlog_snprint_logid(buff, sizeof(buff), &id); jlog_snprint_logid(buff2, sizeof(buff2), &id2); OUT(" next interval: [%s, %s]\n" "--------------------\n\n", buff, buff2); } if (show_subscribers) { char **list; int i; jlog_ctx_list_subscribers(log, &list); for (i = 0; list[i]; i++) { char buff[20]; jlog_id id; jlog_get_checkpoint(log, list[i], &id); jlog_snprint_logid(buff, sizeof(buff), &id); OUT("\t%32s @ %s\n", list[i], buff); } jlog_ctx_list_subscribers_dispose(log, list); } if (show_files) { struct dirent *de; DIR *dir; dir = opendir(file); if (!dir) { fprintf(stderr, "error opening '%s'\n", file); return 0; } while ((de = readdir(dir)) != NULL) { uint32_t logid; if (is_datafile(de->d_name, &logid)) { char fullfile[MAXPATHLEN]; char fullidx[MAXPATHLEN]; struct stat sb; int readers; snprintf(fullfile, sizeof(fullfile), "%s/%s", file, de->d_name); snprintf(fullidx, sizeof(fullidx), "%s/%s" INDEX_EXT, file, de->d_name); if (stat(fullfile, &sb)) { OUT("\t%8s [error stat(2)ing file: %s\n", de->d_name, strerror(errno)); } else { readers = __jlog_pending_readers(log, logid); OUT("\t%8s [%ju bytes] %d pending readers\n", de->d_name, sb.st_size, readers); if (show_index_info) { if (stat(fullidx, &sb)) { OUT("\t\t idx: none\n"); } else { uint32_t marker; int closed; if (jlog_idx_details(log, logid, &marker, &closed)) { OUT("\t\t idx: error\n"); } else { OUT("\t\t idx: %u messages (%08x), %s\n", marker, marker, closed ? "closed" : "open"); } } } if (analyze_datafiles) { analyze_datafile(log, logid); } if (readers == 0 && cleanup) { unlink(fullfile); unlink(fullidx); } } } } closedir(dir); } jlog_ctx_close(log); }
void *reader(void *unused) { jlog_ctx *ctx; char subname[32]; int tcount = 0, fcount = 0; int prev_err = 0; int subno = (int)(uintptr_t)unused; snprintf(subname, sizeof(subname), "sub-%02d", subno); reader_retry: ctx = jlog_new(LOGNAME); if(jlog_ctx_open_reader(ctx, subname) != 0) { if(prev_err == 0) { prev_err = jlog_ctx_err(ctx); jlog_ctx_close(ctx); ctx = jlog_new(LOGNAME); if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) { fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno); if(jlog_ctx_open_writer(ctx) != 0) { fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) { fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { jlog_ctx_close(ctx); goto reader_retry; } } } } fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } fprintf(stderr, "[%02d] reader started\n", subno); while(1) { char begins[20], ends[20]; jlog_id begin, end; int count; jlog_message message; if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) { fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } jlog_snprint_logid(begins, sizeof(begins), &begin); jlog_snprint_logid(ends, sizeof(ends), &end); if(count > 0) { int i; //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count); for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) { end = begin; if(jlog_ctx_read_message(ctx, &begin, &message) != 0) { if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) { /* fine */ } else { fcount++; jlog_snprint_logid(begins, sizeof(begins), &begin); fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } } else { tcount++; jlog_snprint_logid(begins, sizeof(begins), &begin); /* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins, message.mess_len, (char *)message.mess); */ } } if(jlog_ctx_read_checkpoint(ctx, &end) != 0) { fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { fprintf(stderr, "[%02d] \tcheckpointed...\n", subno); } } else { if(writer_done == 1) break; } } fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount); jlog_ctx_close(ctx); return (void *)(uintptr_t)tcount; }