static int
jlog_logio_reopen(noit_log_stream_t ls) {
  char **subs;
  jlog_asynch_ctx *actx = ls->op_ctx;
  pthread_rwlock_t *lock = ls->lock;
  pthread_attr_t tattr;
  int i;
  /* reopening only has the effect of removing temporary subscriptions */
  /* (they start with ~ in our hair-brained model */

  if(lock) pthread_rwlock_wrlock(lock);
  if(jlog_ctx_list_subscribers(actx->log, &subs) == -1)
    goto bail;

  for(i=0;subs[i];i++)
    if(subs[i][0] == '~')
      jlog_ctx_remove_subscriber(actx->log, subs[i]);

  jlog_ctx_list_subscribers_dispose(actx->log, subs);
  jlog_logio_cleanse(ls);
 bail:
  if(lock) pthread_rwlock_unlock(lock);

  pthread_attr_init(&tattr);
  pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);
  if(pthread_create(&actx->writer, NULL, jlog_logio_asynch_writer, ls) != 0)
    return -1;
  
  return 0;
}
void
noit_jlog_closure_free(noit_jlog_closure_t *jcl) {
  if(jcl->jlog) {
    if(jcl->subscriber) {
      if(jcl->subscriber[0] == '~')
        jlog_ctx_remove_subscriber(jcl->jlog, jcl->subscriber);
      free(jcl->subscriber);
    }
    jlog_ctx_close(jcl->jlog);
  }
  free(jcl);
}
static int rest_delete_feed(noit_http_rest_closure_t *restc,
                            int npats, char **pats) {
  noit_http_session_ctx *ctx = restc->http_ctx;
  const char *err = "unknown error";
  const char *jpath_with_sub;
  char jlogpath[PATH_MAX], *cp;
  int rv;
  noit_log_stream_t feed;
  jlog_ctx *jctx;

  feed = noit_log_stream_find("feed");
  if(!feed) { err = "cannot find feed"; goto error; }

  jpath_with_sub = noit_log_stream_get_path(feed);
  strlcpy(jlogpath, jpath_with_sub, sizeof(jlogpath));
  cp = strchr(jlogpath, '(');
  if(cp) *cp = '\0';

  jctx = jlog_new(jlogpath);
  rv = jlog_ctx_remove_subscriber(jctx, pats[0]);
  jlog_ctx_close(jctx);
  if(rv < 0) {
    err = jlog_ctx_err_string(jctx);
    goto error;
  }

  /* removed or note, we should do a sweeping cleanup */
  jlog_clean(jlogpath);

  if(rv == 0) {
    noit_http_response_not_found(ctx, "text/plain");
    noit_http_response_end(ctx);
    return 0;
  }

  noit_http_response_standard(ctx, 204, "OK", "text/plain");
  noit_http_response_end(ctx);
  return 0;

 error:
  noit_http_response_server_error(ctx, "text/plain");
  noit_http_response_append(ctx, err, strlen(err));
  noit_http_response_end(ctx);
  return 0;
}
static int
jlog_logio_open(noit_log_stream_t ls) {
  char path[PATH_MAX], *sub, **subs, *p;
  jlog_asynch_ctx *actx;
  jlog_ctx *log = NULL;
  pthread_attr_t tattr;
  int i, listed, found;

  if(jlog_lspath_to_fspath(ls, path, sizeof(path), &sub) <= 0) return -1;
  log = jlog_new(path);
  if(!log) return -1;
  jlog_set_error_func(log, noit_log_jlog_err, ls);
  /* Open the writer. */
  if(jlog_ctx_open_writer(log)) {
    /* If that fails, we'll give one attempt at initiailizing it. */
    /* But, since we attempted to open it as a writer, it is tainted. */
    /* path: close, new, init, close, new, writer, add subscriber */
    jlog_ctx_close(log);
    log = jlog_new(path);
    jlog_set_error_func(log, noit_log_jlog_err, ls);
    if(jlog_ctx_init(log)) {
      noitL(noit_error, "Cannot init jlog writer: %s\n",
            jlog_ctx_err_string(log));
      jlog_ctx_close(log);
      return -1;
    }
    /* After it is initialized, we can try to reopen it as a writer. */
    jlog_ctx_close(log);
    log = jlog_new(path);
    jlog_set_error_func(log, noit_log_jlog_err, ls);
    if(jlog_ctx_open_writer(log)) {
      noitL(noit_error, "Cannot open jlog writer: %s\n",
            jlog_ctx_err_string(log));
      jlog_ctx_close(log);
      return -1;
    }
  }

  /* Add or remove subscribers according to the current configuration. */
  listed = jlog_ctx_list_subscribers(log, &subs);
  if(listed == -1) {
    noitL(noit_error, "Cannot list jlog subscribers: %s\n",
          jlog_ctx_err_string(log));
    return -1;
  }

  if(sub) {
    /* Match all configured subscribers against jlog's list. */
    for(p=strtok(sub, ",");p;p=strtok(NULL, ",")) {
      for(i=0;i<listed;i++) {
        if((subs[i]) && (strcmp(p, subs[i]) == 0)) {
          free(subs[i]);
          subs[i] = NULL;
          break;
        }
      }
      if(i == listed)
        jlog_ctx_add_subscriber(log, p, JLOG_BEGIN);
    }

    /* Remove all unmatched subscribers. */
    for(i=0;i<listed;i++) {
      if(subs[i]) {
        jlog_ctx_remove_subscriber(log, subs[i]);
        free(subs[i]);
        subs[i] = NULL;
      }
    }

    free(subs);
    subs = NULL;
  } else {
    /* Remove all subscribers other than DEFAULT_JLOG_SUBSCRIBER. */
    found = 0;
    for(i=0;i<listed;i++) {
      if((subs[i]) && (strcmp(DEFAULT_JLOG_SUBSCRIBER, subs[i]) == 0)) {
        found = 1;
        continue;
      }
      jlog_ctx_remove_subscriber(log, subs[i]);
    }

    /* Add DEFAULT_JLOG_SUBSCRIBER if it wasn't already on the jlog's list. */
    if(!found)
      jlog_ctx_add_subscriber(log, DEFAULT_JLOG_SUBSCRIBER, JLOG_BEGIN);

    jlog_ctx_list_subscribers_dispose(log, subs);
  }

  actx = calloc(1, sizeof(*actx));
  actx->log = log;
  ls->op_ctx = actx;

  pthread_attr_init(&tattr);
  pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);
  if(pthread_create(&actx->writer, NULL, jlog_logio_asynch_writer, ls) != 0)
    return -1;

  /* We do this to clean things up */
  jlog_logio_reopen(ls);
  return 0;
}
Exemple #5
0
/*
 * returns -1 when the incoming checkpoint is out of range of the log
 * returns -2 if there was an error actually setting the checkpoint
 */
static int queue_log_add_checkpoint(fqd_queue_impl_data data, const char *name, const fq_msgid *id)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  jlog_id jid = {
    .log = id->id.u32.p1,
    .marker = id->id.u32.p2
  };

  /* ensure the checkpoint makes sense */
  jlog_id first = {
    .log = 0,
    .marker = 0
  };
  jlog_id last = {
    .log = 0,
    .marker = 0
  };
  jlog_ctx_first_log_id(d->reader, &first);
  jlog_ctx_last_log_id(d->reader, &last);

  if (! (jid.log >= first.log && jid.log <= last.log &&
         jid.marker >= first.marker && jid.marker <= last.marker)) {
    return -1;
  }

  char **subs;
  int sub_count = jlog_ctx_list_subscribers(d->reader, &subs);

  int have_it = 0;
  for (int i = 0; i < sub_count; i++) {
    have_it += strcmp(subs[i], name) == 0 ? 1 : 0;
  }

  if (have_it == 0) {
    jlog_ctx_add_subscriber(d->reader, name, JLOG_BEGIN);
  }

  if (jlog_ctx_read_checkpoint(d->reader, &jid) == -1) {
    /*
       If we failed to checkpoint we are in a situation where the 'add_subscriber' call above put
       them at the beginning of the log so we have to remove the subscriber if we just added them

       However, if they already existed and had a previous good checkpoint, leave it alone
    */
    if (have_it == 0) {
      jlog_ctx_remove_subscriber(d->reader, name);
    }
    return -2;
  }
  return 0;
}

/*
 * return -1 if the subscriber doesn't exist
 * return 0 on success
 */
static int queue_log_remove_checkpoint(fqd_queue_impl_data data, const char *name)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  if (jlog_ctx_remove_subscriber(d->reader, name) == 0) {
    return -1;
  }
  return 0;
}

/*
 * return -1 if the subscriber doesn't exist
 * return -2 if we can't reset the checkpoint
 * return 0 on success
 */
static int queue_log_reset_to_checkpoint(fqd_queue_impl_data data, const char *name)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  char **subs;
  int sub_count = jlog_ctx_list_subscribers(d->reader, &subs);

  int have_it = 0;
  for (int i = 0; i < sub_count; i++) {
    have_it += strcmp(subs[i], name) == 0 ? 1 : 0;
  }

  if (have_it == 0) {
    return -1;
  }

  jlog_id checkpoint;
  if (jlog_get_checkpoint(d->reader, name, &checkpoint) == -1) {
    return -2;
  }

  if (jlog_ctx_read_checkpoint(d->reader, &checkpoint) == -1) {
    return -2;
  }
  return 0;
}


static int write_sig(struct queue_jlog *d) {
  char sigfile[PATH_MAX];
  int fd;
  snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath);
  fd = open(sigfile, O_CREAT|O_TRUNC|O_WRONLY, 0640);
  if(fd < 0) return -1;
  write(fd, d->uuid, 16);
  close(fd);
  return 0;
}
static int read_sig(struct queue_jlog *d, uuid_t out) {
  char sigfile[PATH_MAX];
  int fd, rv;
  snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath);
  fd = open(sigfile, O_RDONLY);
  if(fd < 0) return -1;
  rv = read(fd, out, 16);
  close(fd);
  return (rv == 16) ? 0 : -1;
}
static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) {
  char qpath[PATH_MAX];
  jlog_id chkpt;
  struct queue_jlog *d;

  d = calloc(1, sizeof(*d));
  d->auto_chkpt = true;
  fqd_config_construct_queue_path(qpath, sizeof(qpath), qname);
  d->qpath = strdup(qpath);
  d->writer = jlog_new(d->qpath);

  jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
  jlog_ctx_set_multi_process(d->writer, 0);
  jlog_ctx_set_use_compression(d->writer, 1);

  if(jlog_ctx_open_writer(d->writer) != 0) {
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
    jlog_ctx_set_multi_process(d->writer, 0);
    jlog_ctx_set_use_compression(d->writer, 1);
    if(jlog_ctx_init(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
    jlog_ctx_set_multi_process(d->writer, 0);
    jlog_ctx_set_use_compression(d->writer, 1);
    if(jlog_ctx_open_writer(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
  }

  /* 128MB journal chunks */
  jlog_ctx_alter_journal_size(d->writer, 128 * 1024 * 1024);

  d->reader = jlog_new(d->qpath);
  if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) {
    if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader));
      goto bail;
    }
  }
  if(jlog_ctx_open_reader(d->reader, "fq") != 0) {
    fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader));
    goto bail;
  }
  uuid_generate(d->uuid);
  write_sig(d);
  *count = d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish);
  (void)qname;
  return d;

 bail:
  if(d->writer) jlog_ctx_close(d->writer);
  if(d->reader) jlog_ctx_close(d->reader);
  free(d->qpath);
  free(d);
  return NULL;
}
Exemple #6
0
static int
process_jlog(const char *file, const char *sub)
{
  jlog_ctx *log = jlog_new(file);

  if (add_subscriber) {
    if (jlog_ctx_add_subscriber(log, add_subscriber, JLOG_BEGIN)) {
      fprintf(stderr, "Could not add subscriber '%s': %s\n", add_subscriber,
              jlog_ctx_err_string(log));
    } else {
      OUT("Added subscriber '%s'\n", add_subscriber);
    }
  }

  if (remove_subscriber) {
    if (jlog_ctx_remove_subscriber(log, remove_subscriber) <= 0) {
      fprintf(stderr, "Could not erase subscriber '%s': %s\n",
              remove_subscriber, jlog_ctx_err_string(log));
    } else {
      OUT("Erased subscriber '%s'\n", remove_subscriber);
    }
  }

  if (!sub) {
    if (jlog_ctx_open_writer(log)) {
      fprintf(stderr, "error opening '%s'\n", file);
      return 0;
    }
  } else {
    if (jlog_ctx_open_reader(log, sub)) {
      fprintf(stderr, "error opening '%s'\n", file);
      return 0;
    }
  }

  if (show_progress) {
    char buff[20], buff2[20], buff3[20];
    jlog_id id, id2, id3;

    jlog_get_checkpoint(log, sub, &id);

    if (jlog_ctx_last_log_id(log, &id3)) {
      fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log));
      fprintf(stderr, "error calling jlog_ctx_last_log_id\n");
    }

    jlog_snprint_logid(buff, sizeof(buff), &id);
    jlog_snprint_logid(buff3, sizeof(buff3), &id3);
    OUT("--------------------\n"
        "  Perspective of the '%s' subscriber\n"
        "    current checkpoint: %s\n"
        "    Last write: %s\n", sub, buff, buff3);

    if (jlog_ctx_read_interval(log, &id, &id2) < 0) {
      fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log));
    }

    jlog_snprint_logid(buff, sizeof(buff), &id);
    jlog_snprint_logid(buff2, sizeof(buff2), &id2);
    OUT("    next interval: [%s, %s]\n"
        "--------------------\n\n", buff, buff2);
  }

  if (show_subscribers) {
    char **list;
    int i;

    jlog_ctx_list_subscribers(log, &list);

    for (i = 0; list[i]; i++) {
      char buff[20];
      jlog_id id;

      jlog_get_checkpoint(log, list[i], &id);
      jlog_snprint_logid(buff, sizeof(buff), &id);
      OUT("\t%32s @ %s\n", list[i], buff);
    }

    jlog_ctx_list_subscribers_dispose(log, list);
  }

  if (show_files) {
    struct dirent *de;
    DIR *dir;

    dir = opendir(file);

    if (!dir) {
      fprintf(stderr, "error opening '%s'\n", file);
      return 0;
    }

    while ((de = readdir(dir)) != NULL) {
      uint32_t logid;

      if (is_datafile(de->d_name, &logid)) {
        char fullfile[MAXPATHLEN];
        char fullidx[MAXPATHLEN];
        struct stat sb;
        int readers;

        snprintf(fullfile, sizeof(fullfile), "%s/%s", file, de->d_name);
        snprintf(fullidx, sizeof(fullidx), "%s/%s" INDEX_EXT, file, de->d_name);

        if (stat(fullfile, &sb)) {
          OUT("\t%8s [error stat(2)ing file: %s\n", de->d_name, strerror(errno));
        } else {
          readers = __jlog_pending_readers(log, logid);

          OUT("\t%8s [%ju bytes] %d pending readers\n", de->d_name, sb.st_size, readers);

          if (show_index_info) {
            if (stat(fullidx, &sb)) {
              OUT("\t\t idx: none\n");
            } else {
              uint32_t marker;
              int closed;

              if (jlog_idx_details(log, logid, &marker, &closed)) {
                OUT("\t\t idx: error\n");
              } else {
                OUT("\t\t idx: %u messages (%08x), %s\n", marker, marker, closed ? "closed" : "open");
              }
            }
          }
          
          if (analyze_datafiles) {
            analyze_datafile(log, logid);
          }

          if (readers == 0 && cleanup) {
            unlink(fullfile);
            unlink(fullidx);
          }
        }
      }
    }

    closedir(dir);
  }

  jlog_ctx_close(log);
}
Exemple #7
0
int main(int argc, char **argv) {
  int i;
  char *toremove = NULL;
  jlog_safety safety = JLOG_ALMOST_SAFE;
  pthread_t tid[THRCNT], wtid[WTHRCNT];
  void *foo;
 
#if _WIN32 
  mem_init();
#endif

  if(argc == 3) {
    if(!strcmp(argv[1], "safety")) {
      if(!strcmp(argv[2], "unsafe"))
        safety = JLOG_UNSAFE;
      else if(!strcmp(argv[2], "almost_safe"))
        safety = JLOG_ALMOST_SAFE;
      else if(!strcmp(argv[2], "safe"))
        safety = JLOG_SAFE;
      else {
        fprintf(stderr, "invalid safety option\n");
        usage();
      }
    } else if(!strcmp(argv[1], "only")) {
      if(!strcmp(argv[2], "read")) only_read = 1;
      else if(!strcmp(argv[2], "write")) only_write = 1;
      else usage();
    } else if(!strcmp(argv[1], "remove")) {
      toremove = argv[2];
    } else {
      usage();
    }
  } else if(argc < 3 || argc > 3) {
    usage();
  }

  jcreate(safety);

  if(toremove) {
    jlog_ctx *ctx;
    ctx = jlog_new(LOGNAME);
    if(jlog_ctx_open_writer(ctx) != 0) {
      fprintf(stderr, "jlog_ctx_open_writer failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      croak();
    }
    jlog_ctx_remove_subscriber(ctx, argv[2]);
    jlog_ctx_close(ctx);
    exit(0);
  }
  if(!only_write) {
    for(i=0; i<THRCNT; i++) {
      pthread_create(&tid[i], NULL, reader, (void *)(uintptr_t)i);
      fprintf(stderr, "[%d] started reader\n", (int)(uintptr_t)tid[i]);
    }
  }
  if(!only_read) {
    fprintf(stderr, "starting writers..\n");
    for(i=0; i<WTHRCNT; i++) {
      pthread_create(&wtid[i], NULL, writer, (void *)(uintptr_t)i);
      fprintf(stderr, "[%d] started writer\n", (int)(uintptr_t)wtid[i]);
    }
  } else {
    sleep(5);
    writer_done = 1;
  }
  if(!only_write) {
    for(i=0; i<THRCNT; i++) {
      pthread_join(tid[i], &foo);
      fprintf(stderr, "[%d] joined, read %d\n", i, (int)(uintptr_t)foo);
    }
  }
  if(!only_read) {
    for(i=0; i<WTHRCNT; i++) {
      pthread_join(wtid[i], &foo);
      fprintf(stderr, "[%d] joined, write %d\n", i, (int)(uintptr_t)foo);
    }
  }
  if(error) fprintf(stderr, "errors occurred\n");
  return 0;
}