Example #1
0
static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) {
  char qpath[PATH_MAX];
  jlog_id chkpt;
  struct queue_jlog *d;

  d = calloc(1, sizeof(*d));
  d->auto_chkpt = true;
  fqd_config_construct_queue_path(qpath, sizeof(qpath), qname);
  d->qpath = strdup(qpath);
  d->writer = jlog_new(d->qpath);
  if(jlog_ctx_open_writer(d->writer) != 0) {
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    if(jlog_ctx_init(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    if(jlog_ctx_open_writer(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
  }
  d->reader = jlog_new(d->qpath);
  if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) {
    if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader));
      goto bail;
    }
  }
  if(jlog_ctx_open_reader(d->reader, "fq") != 0) {
    fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader));
    goto bail;
  }
  uuid_generate(d->uuid);
  write_sig(d);
  *count = 0;
  (void)qname;
  return d;

 bail:
  if(d->writer) jlog_ctx_close(d->writer);
  if(d->reader) jlog_ctx_close(d->reader);
  free(d->qpath);
  free(d);
  return NULL;
}
Example #2
0
int jlog_ctx_add_subscriber(jlog_ctx *ctx, const char *s, jlog_position whence) {
  jlog_id chkpt;
  jlog_ctx *tmpctx = NULL;
  jlog_file *jchkpt;
  ctx->last_error = JLOG_ERR_SUCCESS;

  jchkpt = __jlog_open_named_checkpoint(ctx, s, O_CREAT|O_EXCL);
  if(!jchkpt) {
    ctx->last_error = JLOG_ERR_SUBSCRIBER_EXISTS;
    ctx->last_errno = EEXIST;
    return -1;
  }
  jlog_file_close(jchkpt);
  
  if(whence == JLOG_BEGIN) {
    memset(&chkpt, 0, sizeof(chkpt));
    jlog_ctx_first_log_id(ctx, &chkpt);
    if(__jlog_set_checkpoint(ctx, s, &chkpt) != 0) {
      ctx->last_error = JLOG_ERR_CHECKPOINT;
      ctx->last_errno = 0;
      return -1;
    }
    return 0;
  }
  if(whence == JLOG_END) {
    jlog_id start, finish;
    memset(&chkpt, 0, sizeof(chkpt));
    if(__jlog_open_metastore(ctx) != 0) SYS_FAIL(JLOG_ERR_META_OPEN);
    if(__jlog_restore_metastore(ctx, 0))
      SYS_FAIL(JLOG_ERR_META_OPEN);
    chkpt.log = ctx->storage.log;
    if(__jlog_set_checkpoint(ctx, s, &chkpt) != 0)
      SYS_FAIL(JLOG_ERR_CHECKPOINT);
    tmpctx = jlog_new(ctx->path);
    if(jlog_ctx_open_reader(tmpctx, s) < 0) goto finish;
    if(jlog_ctx_read_interval(tmpctx, &start, &finish) < 0) goto finish;
    jlog_ctx_close(tmpctx);
    tmpctx = NULL;
    if(__jlog_set_checkpoint(ctx, s, &finish) != 0)
      SYS_FAIL(JLOG_ERR_CHECKPOINT);
    return 0;
  }
  ctx->last_error = JLOG_ERR_NOT_SUPPORTED;
 finish:
  if(tmpctx) jlog_ctx_close(tmpctx);
  return -1;
}
Example #3
0
/*
 * returns -1 when the incoming checkpoint is out of range of the log
 * returns -2 if there was an error actually setting the checkpoint
 */
static int queue_log_add_checkpoint(fqd_queue_impl_data data, const char *name, const fq_msgid *id)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  jlog_id jid = {
    .log = id->id.u32.p1,
    .marker = id->id.u32.p2
  };

  /* ensure the checkpoint makes sense */
  jlog_id first = {
    .log = 0,
    .marker = 0
  };
  jlog_id last = {
    .log = 0,
    .marker = 0
  };
  jlog_ctx_first_log_id(d->reader, &first);
  jlog_ctx_last_log_id(d->reader, &last);

  if (! (jid.log >= first.log && jid.log <= last.log &&
         jid.marker >= first.marker && jid.marker <= last.marker)) {
    return -1;
  }

  char **subs;
  int sub_count = jlog_ctx_list_subscribers(d->reader, &subs);

  int have_it = 0;
  for (int i = 0; i < sub_count; i++) {
    have_it += strcmp(subs[i], name) == 0 ? 1 : 0;
  }

  if (have_it == 0) {
    jlog_ctx_add_subscriber(d->reader, name, JLOG_BEGIN);
  }

  if (jlog_ctx_read_checkpoint(d->reader, &jid) == -1) {
    /*
       If we failed to checkpoint we are in a situation where the 'add_subscriber' call above put
       them at the beginning of the log so we have to remove the subscriber if we just added them

       However, if they already existed and had a previous good checkpoint, leave it alone
    */
    if (have_it == 0) {
      jlog_ctx_remove_subscriber(d->reader, name);
    }
    return -2;
  }
  return 0;
}

/*
 * return -1 if the subscriber doesn't exist
 * return 0 on success
 */
static int queue_log_remove_checkpoint(fqd_queue_impl_data data, const char *name)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  if (jlog_ctx_remove_subscriber(d->reader, name) == 0) {
    return -1;
  }
  return 0;
}

/*
 * return -1 if the subscriber doesn't exist
 * return -2 if we can't reset the checkpoint
 * return 0 on success
 */
static int queue_log_reset_to_checkpoint(fqd_queue_impl_data data, const char *name)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  char **subs;
  int sub_count = jlog_ctx_list_subscribers(d->reader, &subs);

  int have_it = 0;
  for (int i = 0; i < sub_count; i++) {
    have_it += strcmp(subs[i], name) == 0 ? 1 : 0;
  }

  if (have_it == 0) {
    return -1;
  }

  jlog_id checkpoint;
  if (jlog_get_checkpoint(d->reader, name, &checkpoint) == -1) {
    return -2;
  }

  if (jlog_ctx_read_checkpoint(d->reader, &checkpoint) == -1) {
    return -2;
  }
  return 0;
}


static int write_sig(struct queue_jlog *d) {
  char sigfile[PATH_MAX];
  int fd;
  snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath);
  fd = open(sigfile, O_CREAT|O_TRUNC|O_WRONLY, 0640);
  if(fd < 0) return -1;
  write(fd, d->uuid, 16);
  close(fd);
  return 0;
}
static int read_sig(struct queue_jlog *d, uuid_t out) {
  char sigfile[PATH_MAX];
  int fd, rv;
  snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath);
  fd = open(sigfile, O_RDONLY);
  if(fd < 0) return -1;
  rv = read(fd, out, 16);
  close(fd);
  return (rv == 16) ? 0 : -1;
}
static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) {
  char qpath[PATH_MAX];
  jlog_id chkpt;
  struct queue_jlog *d;

  d = calloc(1, sizeof(*d));
  d->auto_chkpt = true;
  fqd_config_construct_queue_path(qpath, sizeof(qpath), qname);
  d->qpath = strdup(qpath);
  d->writer = jlog_new(d->qpath);

  jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
  jlog_ctx_set_multi_process(d->writer, 0);
  jlog_ctx_set_use_compression(d->writer, 1);

  if(jlog_ctx_open_writer(d->writer) != 0) {
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
    jlog_ctx_set_multi_process(d->writer, 0);
    jlog_ctx_set_use_compression(d->writer, 1);
    if(jlog_ctx_init(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
    jlog_ctx_set_multi_process(d->writer, 0);
    jlog_ctx_set_use_compression(d->writer, 1);
    if(jlog_ctx_open_writer(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
  }

  /* 128MB journal chunks */
  jlog_ctx_alter_journal_size(d->writer, 128 * 1024 * 1024);

  d->reader = jlog_new(d->qpath);
  if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) {
    if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader));
      goto bail;
    }
  }
  if(jlog_ctx_open_reader(d->reader, "fq") != 0) {
    fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader));
    goto bail;
  }
  uuid_generate(d->uuid);
  write_sig(d);
  *count = d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish);
  (void)qname;
  return d;

 bail:
  if(d->writer) jlog_ctx_close(d->writer);
  if(d->reader) jlog_ctx_close(d->reader);
  free(d->qpath);
  free(d);
  return NULL;
}
int
noit_jlog_handler(eventer_t e, int mask, void *closure,
                     struct timeval *now) {
  eventer_t newe;
  pthread_t tid;
  pthread_attr_t tattr;
  int newmask = EVENTER_READ | EVENTER_EXCEPTION;
  acceptor_closure_t *ac = closure;
  noit_jlog_closure_t *jcl = ac->service_ctx;
  char errbuff[256];
  const char *errstr = "unknown error";

  if(mask & EVENTER_EXCEPTION || (jcl && jcl->wants_shutdown)) {
    int len, nlen;
socket_error:
    /* Exceptions cause us to simply snip the connection */
    len = strlen(errstr);
    nlen = htonl(0 - len);
    e->opset->write(e->fd, &nlen, sizeof(nlen), &newmask, e);
    e->opset->write(e->fd, errstr, strlen(errstr), &newmask, e);
    eventer_remove_fd(e->fd);
    e->opset->close(e->fd, &newmask, e);
    if(jcl) noit_jlog_closure_free(jcl);
    acceptor_closure_free(ac);
    return 0;
  }

  if(!ac->service_ctx) {
    noit_log_stream_t ls;
    const char *logname, *type;
    int first_attempt = 1;
    char path[PATH_MAX], subscriber[256], *sub;
    jcl = ac->service_ctx = noit_jlog_closure_alloc();
    if(!noit_hash_retr_str(ac->config,
                           "log_transit_feed_name",
                           strlen("log_transit_feed_name"),
                           &logname)) {
      errstr = "No 'log_transit_feed_name' specified in log_transit.";
      noitL(noit_error, "%s\n", errstr);
      goto socket_error;
    }
    ls = noit_log_stream_find(logname);
    if(!ls) {
      snprintf(errbuff, sizeof(errbuff),
               "Could not find log '%s' for log_transit.", logname);
      errstr = errbuff;
      noitL(noit_error, "%s\n", errstr);
      goto socket_error;
    }
    type = noit_log_stream_get_type(ls);
    if(!type || strcmp(type, "jlog")) {
      snprintf(errbuff, sizeof(errbuff),
               "Log '%s' for log_transit is not a jlog.", logname);
      errstr = errbuff;
      noitL(noit_error, "%s\n", errstr);
      goto socket_error;
    }
    if(ac->cmd == NOIT_JLOG_DATA_FEED) {
      if(!ac->remote_cn) {
        errstr = "jlog transit started to unidentified party.";
        noitL(noit_error, "%s\n", errstr);
        goto socket_error;
      }
      strlcpy(subscriber, ac->remote_cn, sizeof(subscriber));
      jcl->feed_stats = noit_jlog_feed_stats(subscriber);
    }
    else {
      jcl->feed_stats = noit_jlog_feed_stats("~");
      snprintf(subscriber, sizeof(subscriber),
               "~%07d", noit_atomic_inc32(&tmpfeedcounter));
    }
    jcl->subscriber = strdup(subscriber);

    strlcpy(path, noit_log_stream_get_path(ls), sizeof(path));
    sub = strchr(path, '(');
    if(sub) {
      char *esub = strchr(sub, ')');
      if(esub) {
        *esub = '\0';
        *sub++ = '\0';
      }
    }

    jcl->jlog = jlog_new(path);
    if(ac->cmd == NOIT_JLOG_DATA_TEMP_FEED) {
 add_sub:
      if(jlog_ctx_add_subscriber(jcl->jlog, jcl->subscriber, JLOG_END) == -1) {
        snprintf(errbuff, sizeof(errbuff),
                 "jlog reader[%s] error: %s", jcl->subscriber,
                 jlog_ctx_err_string(jcl->jlog));
        errstr = errbuff;
        noitL(noit_error, "%s\n", errstr);
      }
    }
    if(jlog_ctx_open_reader(jcl->jlog, jcl->subscriber) == -1) {
      if(sub && !strcmp(sub, "*")) {
        if(first_attempt) {
          jlog_ctx_close(jcl->jlog);
          jcl->jlog = jlog_new(path);
          first_attempt = 0;
          goto add_sub;
        }
      }
      snprintf(errbuff, sizeof(errbuff),
               "jlog reader[%s] error: %s", jcl->subscriber,
               jlog_ctx_err_string(jcl->jlog));
      errstr = errbuff;
      noitL(noit_error, "%s\n", errstr);
      goto socket_error;
    }
  }

  /* The jlog stuff is disk I/O and can block us.
   * We'll create a new thread to just handle this connection.
   */
  eventer_remove_fd(e->fd);
  newe = eventer_alloc();
  memcpy(newe, e, sizeof(*e));
  pthread_attr_init(&tattr);
  pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_DETACHED);
  gettimeofday(&jcl->feed_stats->last_connection, NULL);
  noit_atomic_inc32(&jcl->feed_stats->connections);
  if(pthread_create(&tid, &tattr, noit_jlog_thread_main, newe) == 0) {
    return 0;
  }

  /* Undo our dup */
  eventer_free(newe);
  /* Creating the thread failed, close it down and deschedule. */
  e->opset->close(e->fd, &newmask, e);
  return 0;
}
Example #5
0
static int
process_jlog(const char *file, const char *sub)
{
  jlog_ctx *log = jlog_new(file);

  if (add_subscriber) {
    if (jlog_ctx_add_subscriber(log, add_subscriber, JLOG_BEGIN)) {
      fprintf(stderr, "Could not add subscriber '%s': %s\n", add_subscriber,
              jlog_ctx_err_string(log));
    } else {
      OUT("Added subscriber '%s'\n", add_subscriber);
    }
  }

  if (remove_subscriber) {
    if (jlog_ctx_remove_subscriber(log, remove_subscriber) <= 0) {
      fprintf(stderr, "Could not erase subscriber '%s': %s\n",
              remove_subscriber, jlog_ctx_err_string(log));
    } else {
      OUT("Erased subscriber '%s'\n", remove_subscriber);
    }
  }

  if (!sub) {
    if (jlog_ctx_open_writer(log)) {
      fprintf(stderr, "error opening '%s'\n", file);
      return 0;
    }
  } else {
    if (jlog_ctx_open_reader(log, sub)) {
      fprintf(stderr, "error opening '%s'\n", file);
      return 0;
    }
  }

  if (show_progress) {
    char buff[20], buff2[20], buff3[20];
    jlog_id id, id2, id3;

    jlog_get_checkpoint(log, sub, &id);

    if (jlog_ctx_last_log_id(log, &id3)) {
      fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log));
      fprintf(stderr, "error calling jlog_ctx_last_log_id\n");
    }

    jlog_snprint_logid(buff, sizeof(buff), &id);
    jlog_snprint_logid(buff3, sizeof(buff3), &id3);
    OUT("--------------------\n"
        "  Perspective of the '%s' subscriber\n"
        "    current checkpoint: %s\n"
        "    Last write: %s\n", sub, buff, buff3);

    if (jlog_ctx_read_interval(log, &id, &id2) < 0) {
      fprintf(stderr, "jlog_error: %s\n", jlog_ctx_err_string(log));
    }

    jlog_snprint_logid(buff, sizeof(buff), &id);
    jlog_snprint_logid(buff2, sizeof(buff2), &id2);
    OUT("    next interval: [%s, %s]\n"
        "--------------------\n\n", buff, buff2);
  }

  if (show_subscribers) {
    char **list;
    int i;

    jlog_ctx_list_subscribers(log, &list);

    for (i = 0; list[i]; i++) {
      char buff[20];
      jlog_id id;

      jlog_get_checkpoint(log, list[i], &id);
      jlog_snprint_logid(buff, sizeof(buff), &id);
      OUT("\t%32s @ %s\n", list[i], buff);
    }

    jlog_ctx_list_subscribers_dispose(log, list);
  }

  if (show_files) {
    struct dirent *de;
    DIR *dir;

    dir = opendir(file);

    if (!dir) {
      fprintf(stderr, "error opening '%s'\n", file);
      return 0;
    }

    while ((de = readdir(dir)) != NULL) {
      uint32_t logid;

      if (is_datafile(de->d_name, &logid)) {
        char fullfile[MAXPATHLEN];
        char fullidx[MAXPATHLEN];
        struct stat sb;
        int readers;

        snprintf(fullfile, sizeof(fullfile), "%s/%s", file, de->d_name);
        snprintf(fullidx, sizeof(fullidx), "%s/%s" INDEX_EXT, file, de->d_name);

        if (stat(fullfile, &sb)) {
          OUT("\t%8s [error stat(2)ing file: %s\n", de->d_name, strerror(errno));
        } else {
          readers = __jlog_pending_readers(log, logid);

          OUT("\t%8s [%ju bytes] %d pending readers\n", de->d_name, sb.st_size, readers);

          if (show_index_info) {
            if (stat(fullidx, &sb)) {
              OUT("\t\t idx: none\n");
            } else {
              uint32_t marker;
              int closed;

              if (jlog_idx_details(log, logid, &marker, &closed)) {
                OUT("\t\t idx: error\n");
              } else {
                OUT("\t\t idx: %u messages (%08x), %s\n", marker, marker, closed ? "closed" : "open");
              }
            }
          }
          
          if (analyze_datafiles) {
            analyze_datafile(log, logid);
          }

          if (readers == 0 && cleanup) {
            unlink(fullfile);
            unlink(fullidx);
          }
        }
      }
    }

    closedir(dir);
  }

  jlog_ctx_close(log);
}
Example #6
0
void *reader(void *unused) {
  jlog_ctx *ctx;
  char subname[32];
  int tcount = 0, fcount = 0;
  int prev_err = 0;
  int subno = (int)(uintptr_t)unused;
  snprintf(subname, sizeof(subname), "sub-%02d", subno);
reader_retry:
  ctx = jlog_new(LOGNAME);
  if(jlog_ctx_open_reader(ctx, subname) != 0) {
    if(prev_err == 0) {
      prev_err = jlog_ctx_err(ctx);
      jlog_ctx_close(ctx);
      ctx = jlog_new(LOGNAME);
      if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) {
        fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno);
        if(jlog_ctx_open_writer(ctx) != 0) {
          fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
        } else {
          if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) {
            fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
          } else {
            jlog_ctx_close(ctx);
            goto reader_retry;
          }
        }
      }
    }
    fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
    croak();
  }
  fprintf(stderr, "[%02d] reader started\n", subno);
  while(1) {
    char begins[20], ends[20];
    jlog_id begin, end;
    int count;
    jlog_message message;
    if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) {
      fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      croak();
    }
    jlog_snprint_logid(begins, sizeof(begins), &begin);
    jlog_snprint_logid(ends, sizeof(ends), &end);
    if(count > 0) {
      int i;
      //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count);
      for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) {
        end = begin;
        if(jlog_ctx_read_message(ctx, &begin, &message) != 0) {
          if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) {
            /* fine */
          } else {
            fcount++;
            jlog_snprint_logid(begins, sizeof(begins), &begin);
            fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
          }
        } else {
          tcount++;
          jlog_snprint_logid(begins, sizeof(begins), &begin);
          /* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins,
                  message.mess_len, (char *)message.mess); */
        }
      }
      if(jlog_ctx_read_checkpoint(ctx, &end) != 0) {
        fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      } else {
        fprintf(stderr, "[%02d] \tcheckpointed...\n", subno);
      }
    } else {
      if(writer_done == 1) break;
    }
  }
  fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount);
  jlog_ctx_close(ctx);
  return (void *)(uintptr_t)tcount;
}