static int rest_show_feed(noit_http_rest_closure_t *restc, int npats, char **pats) { noit_http_session_ctx *ctx = restc->http_ctx; const char *err = "unknown error"; const char *jpath_with_sub; char jlogpath[PATH_MAX], *cp, **subs = NULL; int nsubs, i; noit_log_stream_t feed; jlog_ctx *jctx = NULL; xmlDocPtr doc = NULL; xmlNodePtr root = NULL, subnodes; feed = noit_log_stream_find("feed"); if(!feed) { err = "cannot find feed"; goto error; } jpath_with_sub = noit_log_stream_get_path(feed); strlcpy(jlogpath, jpath_with_sub, sizeof(jlogpath)); cp = strchr(jlogpath, '('); if(cp) *cp = '\0'; jctx = jlog_new(jlogpath); if((nsubs = jlog_ctx_list_subscribers(jctx, &subs)) == -1) { err = jlog_ctx_err_string(jctx); goto error; } doc = xmlNewDoc((xmlChar *)"1.0"); root = xmlNewDocNode(doc, NULL, (xmlChar *)"feed", NULL); xmlDocSetRootElement(doc, root); subnodes = xmlNewNode(NULL, (xmlChar *)"subscribers"); for(i=0; i<nsubs; i++) { xmlNewChild(subnodes, NULL, (xmlChar *)"subscriber", (xmlChar *)subs[i]); } xmlAddChild(root, subnodes); noit_http_response_ok(restc->http_ctx, "text/xml"); noit_http_response_xml(restc->http_ctx, doc); noit_http_response_end(restc->http_ctx); if(subs) jlog_ctx_list_subscribers_dispose(jctx, subs); xmlFreeDoc(doc); jlog_ctx_close(jctx); return 0; error: if(doc) xmlFreeDoc(doc); if(subs) jlog_ctx_list_subscribers_dispose(jctx, subs); noit_http_response_server_error(ctx, "text/plain"); noit_http_response_append(ctx, err, strlen(err)); noit_http_response_end(ctx); if(jctx) jlog_ctx_close(jctx); return 0; }
static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) { char qpath[PATH_MAX]; jlog_id chkpt; struct queue_jlog *d; d = calloc(1, sizeof(*d)); d->auto_chkpt = true; fqd_config_construct_queue_path(qpath, sizeof(qpath), qname); d->qpath = strdup(qpath); d->writer = jlog_new(d->qpath); if(jlog_ctx_open_writer(d->writer) != 0) { jlog_ctx_close(d->writer); d->writer = jlog_new(d->qpath); if(jlog_ctx_init(d->writer) != 0) { fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer)); goto bail; } jlog_ctx_close(d->writer); d->writer = jlog_new(d->qpath); if(jlog_ctx_open_writer(d->writer) != 0) { fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer)); goto bail; } } d->reader = jlog_new(d->qpath); if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) { if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) { fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader)); goto bail; } } if(jlog_ctx_open_reader(d->reader, "fq") != 0) { fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader)); goto bail; } uuid_generate(d->uuid); write_sig(d); *count = 0; (void)qname; return d; bail: if(d->writer) jlog_ctx_close(d->writer); if(d->reader) jlog_ctx_close(d->reader); free(d->qpath); free(d); return NULL; }
void *writer(void *unused) { jlog_ctx *ctx; int i; char foo[72]; ctx = jlog_new(LOGNAME); memset(foo, 'X', sizeof(foo)-1); foo[sizeof(foo)-1] = '\0'; if(jlog_ctx_open_writer(ctx) != 0) { fprintf(stderr, "jlog_ctx_open_writer failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } #ifdef TEST_UNIT_LIMIT newsize = 1024*1024 + (((intptr_t)unused) % (1024 * 1024)); fprintf(stderr, "writer setting new unit_limit to %d\n", (int)newsize); jlog_ctx_alter_journal_size(ctx, newsize); #endif for(i=0;i<10000;i++) { int rv; rv = jlog_ctx_write(ctx, foo, strlen(foo)); if(rv != 0) { fprintf(stderr, "jlog_ctx_write_message failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); /* abort(); */ } usleep(100); } fprintf(stderr, "writer thinks unit_limit is %d\n", ctx->meta->unit_limit); jlog_ctx_close(ctx); writer_done = 1; return 0; }
int jlog_ctx_add_subscriber(jlog_ctx *ctx, const char *s, jlog_position whence) { jlog_id chkpt; jlog_ctx *tmpctx = NULL; jlog_file *jchkpt; ctx->last_error = JLOG_ERR_SUCCESS; jchkpt = __jlog_open_named_checkpoint(ctx, s, O_CREAT|O_EXCL); if(!jchkpt) { ctx->last_error = JLOG_ERR_SUBSCRIBER_EXISTS; ctx->last_errno = EEXIST; return -1; } jlog_file_close(jchkpt); if(whence == JLOG_BEGIN) { memset(&chkpt, 0, sizeof(chkpt)); jlog_ctx_first_log_id(ctx, &chkpt); if(__jlog_set_checkpoint(ctx, s, &chkpt) != 0) { ctx->last_error = JLOG_ERR_CHECKPOINT; ctx->last_errno = 0; return -1; } return 0; } if(whence == JLOG_END) { jlog_id start, finish; memset(&chkpt, 0, sizeof(chkpt)); if(__jlog_open_metastore(ctx) != 0) SYS_FAIL(JLOG_ERR_META_OPEN); if(__jlog_restore_metastore(ctx, 0)) SYS_FAIL(JLOG_ERR_META_OPEN); chkpt.log = ctx->storage.log; if(__jlog_set_checkpoint(ctx, s, &chkpt) != 0) SYS_FAIL(JLOG_ERR_CHECKPOINT); tmpctx = jlog_new(ctx->path); if(jlog_ctx_open_reader(tmpctx, s) < 0) goto finish; if(jlog_ctx_read_interval(tmpctx, &start, &finish) < 0) goto finish; jlog_ctx_close(tmpctx); tmpctx = NULL; if(__jlog_set_checkpoint(ctx, s, &finish) != 0) SYS_FAIL(JLOG_ERR_CHECKPOINT); return 0; } ctx->last_error = JLOG_ERR_NOT_SUPPORTED; finish: if(tmpctx) jlog_ctx_close(tmpctx); return -1; }
static int jlog_logio_close(noit_log_stream_t ls) { if(ls->op_ctx) { jlog_asynch_ctx *actx = ls->op_ctx; jlog_ctx_close(actx->log); ls->op_ctx = NULL; } return 0; }
static int jlog_logio_close(noit_log_stream_t ls) { if(ls->op_ctx) { asynch_log_ctx *actx = ls->op_ctx; jlog_ctx *log = actx->userdata; jlog_ctx_close(log); ls->op_ctx = NULL; } return 0; }
static void FREE_JLOG_OBJ(jlog_obj *intern) { if(intern) { if(intern->ctx) { jlog_ctx_close(intern->ctx); intern->ctx = NULL; } if(intern->path) { free(intern->path); } } }
void noit_jlog_closure_free(noit_jlog_closure_t *jcl) { if(jcl->jlog) { if(jcl->subscriber) { if(jcl->subscriber[0] == '~') jlog_ctx_remove_subscriber(jcl->jlog, jcl->subscriber); free(jcl->subscriber); } jlog_ctx_close(jcl->jlog); } free(jcl); }
static int rest_delete_feed(noit_http_rest_closure_t *restc, int npats, char **pats) { noit_http_session_ctx *ctx = restc->http_ctx; const char *err = "unknown error"; const char *jpath_with_sub; char jlogpath[PATH_MAX], *cp; int rv; noit_log_stream_t feed; jlog_ctx *jctx; feed = noit_log_stream_find("feed"); if(!feed) { err = "cannot find feed"; goto error; } jpath_with_sub = noit_log_stream_get_path(feed); strlcpy(jlogpath, jpath_with_sub, sizeof(jlogpath)); cp = strchr(jlogpath, '('); if(cp) *cp = '\0'; jctx = jlog_new(jlogpath); rv = jlog_ctx_remove_subscriber(jctx, pats[0]); jlog_ctx_close(jctx); if(rv < 0) { err = jlog_ctx_err_string(jctx); goto error; } /* removed or note, we should do a sweeping cleanup */ jlog_clean(jlogpath); if(rv == 0) { noit_http_response_not_found(ctx, "text/plain"); noit_http_response_end(ctx); return 0; } noit_http_response_standard(ctx, 204, "OK", "text/plain"); noit_http_response_end(ctx); return 0; error: noit_http_response_server_error(ctx, "text/plain"); noit_http_response_append(ctx, err, strlen(err)); noit_http_response_end(ctx); return 0; }
void jcreate(jlog_safety s) { jlog_ctx *ctx; const char *label = NULL; switch (s) { case JLOG_ALMOST_SAFE: label = "almost safe"; break; case JLOG_UNSAFE: label = "unsafe"; break; case JLOG_SAFE: label = "safe"; break; } fprintf(stderr, "jcreate %s in %s mode\n", LOGNAME, label); ctx = jlog_new(LOGNAME); jlog_ctx_alter_journal_size(ctx, 102400); jlog_ctx_alter_safety(ctx, s); if(jlog_ctx_init(ctx) != 0) { fprintf(stderr, "jlog_ctx_init failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); if(jlog_ctx_err(ctx) != JLOG_ERR_CREATE_EXISTS) exit(0); } else { jlog_ctx_add_subscriber(ctx, MASTER, JLOG_BEGIN); } jlog_ctx_close(ctx); }
/* * returns -1 when the incoming checkpoint is out of range of the log * returns -2 if there was an error actually setting the checkpoint */ static int queue_log_add_checkpoint(fqd_queue_impl_data data, const char *name, const fq_msgid *id) { struct queue_jlog *d = (struct queue_jlog *)data; jlog_id jid = { .log = id->id.u32.p1, .marker = id->id.u32.p2 }; /* ensure the checkpoint makes sense */ jlog_id first = { .log = 0, .marker = 0 }; jlog_id last = { .log = 0, .marker = 0 }; jlog_ctx_first_log_id(d->reader, &first); jlog_ctx_last_log_id(d->reader, &last); if (! (jid.log >= first.log && jid.log <= last.log && jid.marker >= first.marker && jid.marker <= last.marker)) { return -1; } char **subs; int sub_count = jlog_ctx_list_subscribers(d->reader, &subs); int have_it = 0; for (int i = 0; i < sub_count; i++) { have_it += strcmp(subs[i], name) == 0 ? 1 : 0; } if (have_it == 0) { jlog_ctx_add_subscriber(d->reader, name, JLOG_BEGIN); } if (jlog_ctx_read_checkpoint(d->reader, &jid) == -1) { /* If we failed to checkpoint we are in a situation where the 'add_subscriber' call above put them at the beginning of the log so we have to remove the subscriber if we just added them However, if they already existed and had a previous good checkpoint, leave it alone */ if (have_it == 0) { jlog_ctx_remove_subscriber(d->reader, name); } return -2; } return 0; } /* * return -1 if the subscriber doesn't exist * return 0 on success */ static int queue_log_remove_checkpoint(fqd_queue_impl_data data, const char *name) { struct queue_jlog *d = (struct queue_jlog *)data; if (jlog_ctx_remove_subscriber(d->reader, name) == 0) { return -1; } return 0; } /* * return -1 if the subscriber doesn't exist * return -2 if we can't reset the checkpoint * return 0 on success */ static int queue_log_reset_to_checkpoint(fqd_queue_impl_data data, const char *name) { struct queue_jlog *d = (struct queue_jlog *)data; char **subs; int sub_count = jlog_ctx_list_subscribers(d->reader, &subs); int have_it = 0; for (int i = 0; i < sub_count; i++) { have_it += strcmp(subs[i], name) == 0 ? 1 : 0; } if (have_it == 0) { return -1; } jlog_id checkpoint; if (jlog_get_checkpoint(d->reader, name, &checkpoint) == -1) { return -2; } if (jlog_ctx_read_checkpoint(d->reader, &checkpoint) == -1) { return -2; } return 0; } static int write_sig(struct queue_jlog *d) { char sigfile[PATH_MAX]; int fd; snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath); fd = open(sigfile, O_CREAT|O_TRUNC|O_WRONLY, 0640); if(fd < 0) return -1; write(fd, d->uuid, 16); close(fd); return 0; } static int read_sig(struct queue_jlog *d, uuid_t out) { char sigfile[PATH_MAX]; int fd, rv; snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath); fd = open(sigfile, O_RDONLY); if(fd < 0) return -1; rv = read(fd, out, 16); close(fd); return (rv == 16) ? 0 : -1; } static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) { char qpath[PATH_MAX]; jlog_id chkpt; struct queue_jlog *d; d = calloc(1, sizeof(*d)); d->auto_chkpt = true; fqd_config_construct_queue_path(qpath, sizeof(qpath), qname); d->qpath = strdup(qpath); d->writer = jlog_new(d->qpath); jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024); jlog_ctx_set_multi_process(d->writer, 0); jlog_ctx_set_use_compression(d->writer, 1); if(jlog_ctx_open_writer(d->writer) != 0) { jlog_ctx_close(d->writer); d->writer = jlog_new(d->qpath); jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024); jlog_ctx_set_multi_process(d->writer, 0); jlog_ctx_set_use_compression(d->writer, 1); if(jlog_ctx_init(d->writer) != 0) { fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer)); goto bail; } jlog_ctx_close(d->writer); d->writer = jlog_new(d->qpath); jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024); jlog_ctx_set_multi_process(d->writer, 0); jlog_ctx_set_use_compression(d->writer, 1); if(jlog_ctx_open_writer(d->writer) != 0) { fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer)); goto bail; } } /* 128MB journal chunks */ jlog_ctx_alter_journal_size(d->writer, 128 * 1024 * 1024); d->reader = jlog_new(d->qpath); if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) { if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) { fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader)); goto bail; } } if(jlog_ctx_open_reader(d->reader, "fq") != 0) { fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader)); goto bail; } uuid_generate(d->uuid); write_sig(d); *count = d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish); (void)qname; return d; bail: if(d->writer) jlog_ctx_close(d->writer); if(d->reader) jlog_ctx_close(d->reader); free(d->qpath); free(d); return NULL; }
void *reader(void *unused) { jlog_ctx *ctx; char subname[32]; int tcount = 0, fcount = 0; int prev_err = 0; int subno = (int)(uintptr_t)unused; snprintf(subname, sizeof(subname), "sub-%02d", subno); reader_retry: ctx = jlog_new(LOGNAME); if(jlog_ctx_open_reader(ctx, subname) != 0) { if(prev_err == 0) { prev_err = jlog_ctx_err(ctx); jlog_ctx_close(ctx); ctx = jlog_new(LOGNAME); if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) { fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno); if(jlog_ctx_open_writer(ctx) != 0) { fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) { fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { jlog_ctx_close(ctx); goto reader_retry; } } } } fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } fprintf(stderr, "[%02d] reader started\n", subno); while(1) { char begins[20], ends[20]; jlog_id begin, end; int count; jlog_message message; if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) { fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } jlog_snprint_logid(begins, sizeof(begins), &begin); jlog_snprint_logid(ends, sizeof(ends), &end); if(count > 0) { int i; //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count); for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) { end = begin; if(jlog_ctx_read_message(ctx, &begin, &message) != 0) { if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) { /* fine */ } else { fcount++; jlog_snprint_logid(begins, sizeof(begins), &begin); fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } } else { tcount++; jlog_snprint_logid(begins, sizeof(begins), &begin); /* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins, message.mess_len, (char *)message.mess); */ } } if(jlog_ctx_read_checkpoint(ctx, &end) != 0) { fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); } else { fprintf(stderr, "[%02d] \tcheckpointed...\n", subno); } } else { if(writer_done == 1) break; } } fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount); jlog_ctx_close(ctx); return (void *)(uintptr_t)tcount; }
static int process_jlog(const char *file, const char *sub) { jlog_ctx *log = jlog_new(file); if (add_subscriber) { if (jlog_ctx_add_subscriber(log, add_subscriber, JLOG_BEGIN)) { fprintf(stderr, "Could not add subscriber '%s': %s\n", add_subscriber, jlog_ctx_err_string(log)); } else { OUT("Added subscriber '%s'\n", add_subscriber); } } if (remove_subscriber) { if (jlog_ctx_remove_subscriber(log, remove_subscriber) <= 0) { fprintf(stderr, "Could not erase subscriber '%s': %s\n", remove_subscriber, jlog_ctx_err_string(log)); } else { OUT("Erased subscriber '%s'\n", remove_subscriber); } } if (!sub) { if (jlog_ctx_open_writer(log)) { fprintf(stderr, "error opening '%s'\n", file); return 0; } } else { if (jlog_ctx_open_reader(log, sub)) { fprintf(stderr, "error opening '%s'\n", file); return 0; } } if (show_progress) { char buff[20], buff2[20], buff3[20]; jlog_id id, id2, id3; jlog_get_checkpoint(log, sub, &id); if (jlog_ctx_last_log_id(log, &id3)) { fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log)); fprintf(stderr, "error calling jlog_ctx_last_log_id\n"); } jlog_snprint_logid(buff, sizeof(buff), &id); jlog_snprint_logid(buff3, sizeof(buff3), &id3); OUT("--------------------\n" " Perspective of the '%s' subscriber\n" " current checkpoint: %s\n" " Last write: %s\n", sub, buff, buff3); if (jlog_ctx_read_interval(log, &id, &id2) < 0) { fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log)); } jlog_snprint_logid(buff, sizeof(buff), &id); jlog_snprint_logid(buff2, sizeof(buff2), &id2); OUT(" next interval: [%s, %s]\n" "--------------------\n\n", buff, buff2); } if (show_subscribers) { char **list; int i; jlog_ctx_list_subscribers(log, &list); for (i = 0; list[i]; i++) { char buff[20]; jlog_id id; jlog_get_checkpoint(log, list[i], &id); jlog_snprint_logid(buff, sizeof(buff), &id); OUT("\t%32s @ %s\n", list[i], buff); } jlog_ctx_list_subscribers_dispose(log, list); } if (show_files) { struct dirent *de; DIR *dir; dir = opendir(file); if (!dir) { fprintf(stderr, "error opening '%s'\n", file); return 0; } while ((de = readdir(dir)) != NULL) { uint32_t logid; if (is_datafile(de->d_name, &logid)) { char fullfile[MAXPATHLEN]; char fullidx[MAXPATHLEN]; struct stat sb; int readers; snprintf(fullfile, sizeof(fullfile), "%s/%s", file, de->d_name); snprintf(fullidx, sizeof(fullidx), "%s/%s" INDEX_EXT, file, de->d_name); if (stat(fullfile, &sb)) { OUT("\t%8s [error stat(2)ing file: %s\n", de->d_name, strerror(errno)); } else { readers = __jlog_pending_readers(log, logid); OUT("\t%8s [%ju bytes] %d pending readers\n", de->d_name, sb.st_size, readers); if (show_index_info) { if (stat(fullidx, &sb)) { OUT("\t\t idx: none\n"); } else { uint32_t marker; int closed; if (jlog_idx_details(log, logid, &marker, &closed)) { OUT("\t\t idx: error\n"); } else { OUT("\t\t idx: %u messages (%08x), %s\n", marker, marker, closed ? "closed" : "open"); } } } if (analyze_datafiles) { analyze_datafile(log, logid); } if (readers == 0 && cleanup) { unlink(fullfile); unlink(fullidx); } } } } closedir(dir); } jlog_ctx_close(log); }
int main(int argc, char **argv) { int i; char *toremove = NULL; jlog_safety safety = JLOG_ALMOST_SAFE; pthread_t tid[THRCNT], wtid[WTHRCNT]; void *foo; #if _WIN32 mem_init(); #endif if(argc == 3) { if(!strcmp(argv[1], "safety")) { if(!strcmp(argv[2], "unsafe")) safety = JLOG_UNSAFE; else if(!strcmp(argv[2], "almost_safe")) safety = JLOG_ALMOST_SAFE; else if(!strcmp(argv[2], "safe")) safety = JLOG_SAFE; else { fprintf(stderr, "invalid safety option\n"); usage(); } } else if(!strcmp(argv[1], "only")) { if(!strcmp(argv[2], "read")) only_read = 1; else if(!strcmp(argv[2], "write")) only_write = 1; else usage(); } else if(!strcmp(argv[1], "remove")) { toremove = argv[2]; } else { usage(); } } else if(argc < 3 || argc > 3) { usage(); } jcreate(safety); if(toremove) { jlog_ctx *ctx; ctx = jlog_new(LOGNAME); if(jlog_ctx_open_writer(ctx) != 0) { fprintf(stderr, "jlog_ctx_open_writer failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); croak(); } jlog_ctx_remove_subscriber(ctx, argv[2]); jlog_ctx_close(ctx); exit(0); } if(!only_write) { for(i=0; i<THRCNT; i++) { pthread_create(&tid[i], NULL, reader, (void *)(uintptr_t)i); fprintf(stderr, "[%d] started reader\n", (int)(uintptr_t)tid[i]); } } if(!only_read) { fprintf(stderr, "starting writers..\n"); for(i=0; i<WTHRCNT; i++) { pthread_create(&wtid[i], NULL, writer, (void *)(uintptr_t)i); fprintf(stderr, "[%d] started writer\n", (int)(uintptr_t)wtid[i]); } } else { sleep(5); writer_done = 1; } if(!only_write) { for(i=0; i<THRCNT; i++) { pthread_join(tid[i], &foo); fprintf(stderr, "[%d] joined, read %d\n", i, (int)(uintptr_t)foo); } } if(!only_read) { for(i=0; i<WTHRCNT; i++) { pthread_join(wtid[i], &foo); fprintf(stderr, "[%d] joined, write %d\n", i, (int)(uintptr_t)foo); } } if(error) fprintf(stderr, "errors occurred\n"); return 0; }
static int jlog_logio_open(noit_log_stream_t ls) { char path[PATH_MAX], *sub, **subs, *p; jlog_asynch_ctx *actx; jlog_ctx *log = NULL; pthread_attr_t tattr; int i, listed, found; if(jlog_lspath_to_fspath(ls, path, sizeof(path), &sub) <= 0) return -1; log = jlog_new(path); if(!log) return -1; jlog_set_error_func(log, noit_log_jlog_err, ls); /* Open the writer. */ if(jlog_ctx_open_writer(log)) { /* If that fails, we'll give one attempt at initiailizing it. */ /* But, since we attempted to open it as a writer, it is tainted. */ /* path: close, new, init, close, new, writer, add subscriber */ jlog_ctx_close(log); log = jlog_new(path); jlog_set_error_func(log, noit_log_jlog_err, ls); if(jlog_ctx_init(log)) { noitL(noit_error, "Cannot init jlog writer: %s\n", jlog_ctx_err_string(log)); jlog_ctx_close(log); return -1; } /* After it is initialized, we can try to reopen it as a writer. */ jlog_ctx_close(log); log = jlog_new(path); jlog_set_error_func(log, noit_log_jlog_err, ls); if(jlog_ctx_open_writer(log)) { noitL(noit_error, "Cannot open jlog writer: %s\n", jlog_ctx_err_string(log)); jlog_ctx_close(log); return -1; } } /* Add or remove subscribers according to the current configuration. */ listed = jlog_ctx_list_subscribers(log, &subs); if(listed == -1) { noitL(noit_error, "Cannot list jlog subscribers: %s\n", jlog_ctx_err_string(log)); return -1; } if(sub) { /* Match all configured subscribers against jlog's list. */ for(p=strtok(sub, ",");p;p=strtok(NULL, ",")) { for(i=0;i<listed;i++) { if((subs[i]) && (strcmp(p, subs[i]) == 0)) { free(subs[i]); subs[i] = NULL; break; } } if(i == listed) jlog_ctx_add_subscriber(log, p, JLOG_BEGIN); } /* Remove all unmatched subscribers. */ for(i=0;i<listed;i++) { if(subs[i]) { jlog_ctx_remove_subscriber(log, subs[i]); free(subs[i]); subs[i] = NULL; } } free(subs); subs = NULL; } else { /* Remove all subscribers other than DEFAULT_JLOG_SUBSCRIBER. */ found = 0; for(i=0;i<listed;i++) { if((subs[i]) && (strcmp(DEFAULT_JLOG_SUBSCRIBER, subs[i]) == 0)) { found = 1; continue; } jlog_ctx_remove_subscriber(log, subs[i]); } /* Add DEFAULT_JLOG_SUBSCRIBER if it wasn't already on the jlog's list. */ if(!found) jlog_ctx_add_subscriber(log, DEFAULT_JLOG_SUBSCRIBER, JLOG_BEGIN); jlog_ctx_list_subscribers_dispose(log, subs); } actx = calloc(1, sizeof(*actx)); actx->log = log; ls->op_ctx = actx; pthread_attr_init(&tattr); pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED); if(pthread_create(&actx->writer, NULL, jlog_logio_asynch_writer, ls) != 0) return -1; /* We do this to clean things up */ jlog_logio_reopen(ls); return 0; }
int noit_jlog_handler(eventer_t e, int mask, void *closure, struct timeval *now) { eventer_t newe; pthread_t tid; pthread_attr_t tattr; int newmask = EVENTER_READ | EVENTER_EXCEPTION; acceptor_closure_t *ac = closure; noit_jlog_closure_t *jcl = ac->service_ctx; char errbuff[256]; const char *errstr = "unknown error"; if(mask & EVENTER_EXCEPTION || (jcl && jcl->wants_shutdown)) { int len, nlen; socket_error: /* Exceptions cause us to simply snip the connection */ len = strlen(errstr); nlen = htonl(0 - len); e->opset->write(e->fd, &nlen, sizeof(nlen), &newmask, e); e->opset->write(e->fd, errstr, strlen(errstr), &newmask, e); eventer_remove_fd(e->fd); e->opset->close(e->fd, &newmask, e); if(jcl) noit_jlog_closure_free(jcl); acceptor_closure_free(ac); return 0; } if(!ac->service_ctx) { noit_log_stream_t ls; const char *logname, *type; int first_attempt = 1; char path[PATH_MAX], subscriber[256], *sub; jcl = ac->service_ctx = noit_jlog_closure_alloc(); if(!noit_hash_retr_str(ac->config, "log_transit_feed_name", strlen("log_transit_feed_name"), &logname)) { errstr = "No 'log_transit_feed_name' specified in log_transit."; noitL(noit_error, "%s\n", errstr); goto socket_error; } ls = noit_log_stream_find(logname); if(!ls) { snprintf(errbuff, sizeof(errbuff), "Could not find log '%s' for log_transit.", logname); errstr = errbuff; noitL(noit_error, "%s\n", errstr); goto socket_error; } type = noit_log_stream_get_type(ls); if(!type || strcmp(type, "jlog")) { snprintf(errbuff, sizeof(errbuff), "Log '%s' for log_transit is not a jlog.", logname); errstr = errbuff; noitL(noit_error, "%s\n", errstr); goto socket_error; } if(ac->cmd == NOIT_JLOG_DATA_FEED) { if(!ac->remote_cn) { errstr = "jlog transit started to unidentified party."; noitL(noit_error, "%s\n", errstr); goto socket_error; } strlcpy(subscriber, ac->remote_cn, sizeof(subscriber)); jcl->feed_stats = noit_jlog_feed_stats(subscriber); } else { jcl->feed_stats = noit_jlog_feed_stats("~"); snprintf(subscriber, sizeof(subscriber), "~%07d", noit_atomic_inc32(&tmpfeedcounter)); } jcl->subscriber = strdup(subscriber); strlcpy(path, noit_log_stream_get_path(ls), sizeof(path)); sub = strchr(path, '('); if(sub) { char *esub = strchr(sub, ')'); if(esub) { *esub = '\0'; *sub++ = '\0'; } } jcl->jlog = jlog_new(path); if(ac->cmd == NOIT_JLOG_DATA_TEMP_FEED) { add_sub: if(jlog_ctx_add_subscriber(jcl->jlog, jcl->subscriber, JLOG_END) == -1) { snprintf(errbuff, sizeof(errbuff), "jlog reader[%s] error: %s", jcl->subscriber, jlog_ctx_err_string(jcl->jlog)); errstr = errbuff; noitL(noit_error, "%s\n", errstr); } } if(jlog_ctx_open_reader(jcl->jlog, jcl->subscriber) == -1) { if(sub && !strcmp(sub, "*")) { if(first_attempt) { jlog_ctx_close(jcl->jlog); jcl->jlog = jlog_new(path); first_attempt = 0; goto add_sub; } } snprintf(errbuff, sizeof(errbuff), "jlog reader[%s] error: %s", jcl->subscriber, jlog_ctx_err_string(jcl->jlog)); errstr = errbuff; noitL(noit_error, "%s\n", errstr); goto socket_error; } } /* The jlog stuff is disk I/O and can block us. * We'll create a new thread to just handle this connection. */ eventer_remove_fd(e->fd); newe = eventer_alloc(); memcpy(newe, e, sizeof(*e)); pthread_attr_init(&tattr); pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED); gettimeofday(&jcl->feed_stats->last_connection, NULL); noit_atomic_inc32(&jcl->feed_stats->connections); if(pthread_create(&tid, &tattr, noit_jlog_thread_main, newe) == 0) { return 0; } /* Undo our dup */ eventer_free(newe); /* Creating the thread failed, close it down and deschedule. */ e->opset->close(e->fd, &newmask, e); return 0; }
static int rest_add_feed(mtev_http_rest_closure_t *restc, int npats, char **pats) { mtev_http_session_ctx *ctx = restc->http_ctx; xmlXPathObjectPtr pobj = NULL; xmlDocPtr doc = NULL, indoc = NULL; xmlNodePtr node, root; acceptor_closure_t *ac = restc->ac; int error_code = 500, complete = 0, mask = 0, rv; const char *error = "internal error", *logname; char *name, *copy_from; mtev_log_stream_t feed; const char *jpath_with_sub; char jlogpath[PATH_MAX], *cp; jlog_ctx *jctx = NULL; jlog_id chkpt; if(npats != 0) goto error; indoc = rest_get_xml_upload(restc, &mask, &complete); if(!complete) return mask; if(indoc == NULL) { error = "xml parse error"; goto error; } if(!mtev_hash_retr_str(ac->config, "log_transit_feed_name", strlen("log_transit_feed_name"), &logname)) { goto error; } feed = mtev_log_stream_find("feed"); if(!feed) { error = "couldn't find feed"; goto error; } jpath_with_sub = mtev_log_stream_get_path(feed); strlcpy(jlogpath, jpath_with_sub, sizeof(jlogpath)); cp = strchr(jlogpath, '('); if(cp) *cp = '\0'; node = xmlDocGetRootElement(indoc); name = (char*)xmlGetProp(node, (xmlChar*)"name"); copy_from = (char*)xmlGetProp(node, (xmlChar*)"checkpoint_copy"); jctx = jlog_new(jlogpath); if (!jctx) { error = "couldn't open logpath"; goto error; } if (!jlog_get_checkpoint(jctx, name, &chkpt)) { error = "subscriber already exists, can't add"; goto error; } if (copy_from) { rv = jlog_ctx_add_subscriber_copy_checkpoint(jctx, name, copy_from); } else { rv = jlog_ctx_add_subscriber(jctx, name, JLOG_END); } if (rv == -1) { error = "couldn't add subscriber"; goto error; } mtev_http_response_ok(restc->http_ctx, "text/xml"); mtev_http_response_end(restc->http_ctx); goto cleanup; error: mtev_http_response_standard(ctx, error_code, "ERROR", "text/xml"); doc = xmlNewDoc((xmlChar *)"1.0"); root = xmlNewDocNode(doc, NULL, (xmlChar *)"error", NULL); xmlDocSetRootElement(doc, root); xmlNodeAddContent(root, (xmlChar *)error); mtev_http_response_xml(ctx, doc); mtev_http_response_end(ctx); cleanup: if (jctx) { jlog_ctx_close(jctx); } if(pobj) xmlXPathFreeObject(pobj); if(doc) xmlFreeDoc(doc); return 0; }