Exemple #1
0
static fq_msg *queue_jlog_dequeue(fqd_queue_impl_data f) {
  struct queue_jlog *d = (struct queue_jlog *)f;
  jlog_message msg;
  fq_msg *m;
  if(d->count == 0 && d->last_seen_nenqueued == d->nenqueued) return NULL;
 retry:
  if(d->count <= 0) {
    d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish);
    fq_debug(FQ_DEBUG_IO, "jlog read batch count -> %d\n", d->count);
    if(d->count < 0) {
      char idxfile[PATH_MAX];
      fq_debug(FQ_DEBUG_IO, "jlog_ctx_read_interval: %s\n",
               jlog_ctx_err_string(d->reader));
      switch (jlog_ctx_err(d->reader)) {
        case JLOG_ERR_FILE_CORRUPT:
        case JLOG_ERR_IDX_CORRUPT:
          jlog_repair_datafile(d->reader, d->start.log);
          jlog_repair_datafile(d->reader, d->start.log + 1);
          fq_debug(FQ_DEBUG_IO,
                "jlog reconstructed, deleting corresponding index.\n");
          STRSETDATAFILE(d->reader, idxfile, d->start.log);
          strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile));
          unlink(idxfile);
          STRSETDATAFILE(d->reader, idxfile, d->start.log + 1);
          strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile));
          unlink(idxfile);
          break;
        default:
          break;
      }
    }
    if(d->count <= 0) return NULL;
  }
  if(jlog_ctx_read_message(d->reader, &d->start, &msg) == -1) {
    d->count = 0;
    return NULL;
  }
  if(d->last_dequeued.log > d->start.log ||
     (d->last_dequeued.log == d->start.log &&
      d->last_dequeued.marker > d->start.marker)) {
    d->count--;
    JLOG_ID_ADVANCE(&d->start);
    goto retry;
  }
  if(msg.mess_len < sizeof(fq_msg)-1)
    m = NULL;
  else {
    off_t expected_len;
    uint32_t payload_len;
    m = (fq_msg *)msg.mess;
    memcpy(&payload_len, &m->payload_len, sizeof(m->payload_len));
    expected_len = offsetof(fq_msg, payload) + payload_len;
    if(expected_len != msg.mess_len) m = NULL;
    else {
      m = malloc(expected_len);
      memcpy(m, msg.mess, expected_len);
      m->sender_msgid.id.u32.p3 = d->start.log;
      m->sender_msgid.id.u32.p4 = d->start.marker;
    }
  }
  d->count--;
  fq_debug(FQ_DEBUG_IO, "jlog batch count -> %d\n", d->count);
  if(d->count == 0) {
    if(d->auto_chkpt) {
      jlog_ctx_read_checkpoint(d->reader, &d->start);
    }
  }
  d->last_dequeued = d->start;
  JLOG_ID_ADVANCE(&d->start);
  ck_pr_inc_uint(&d->last_seen_nenqueued);
  return m;
}
void *
noit_jlog_thread_main(void *e_vptr) {
  int mask, bytes_read;
  eventer_t e = e_vptr;
  acceptor_closure_t *ac = e->closure;
  noit_jlog_closure_t *jcl = ac->service_ctx;
  char inbuff[sizeof(jlog_id)];

  eventer_set_fd_blocking(e->fd);

  while(1) {
    jlog_id client_chkpt;
    int sleeptime = (ac->cmd == NOIT_JLOG_DATA_TEMP_FEED) ?
                      1 : DEFAULT_SECONDS_BETWEEN_BATCHES;
    jlog_get_checkpoint(jcl->jlog, ac->remote_cn, &jcl->chkpt);
    jcl->count = jlog_ctx_read_interval(jcl->jlog, &jcl->start, &jcl->finish);
    if(jcl->count < 0) {
      char idxfile[PATH_MAX];
      noitL(noit_error, "jlog_ctx_read_interval: %s\n",
            jlog_ctx_err_string(jcl->jlog));
      switch (jlog_ctx_err(jcl->jlog)) {
        case JLOG_ERR_FILE_CORRUPT:
        case JLOG_ERR_IDX_CORRUPT:
          jlog_repair_datafile(jcl->jlog, jcl->start.log);
          jlog_repair_datafile(jcl->jlog, jcl->start.log + 1);
          noitL(noit_error,
                "jlog reconstructed, deleting corresponding index.\n");
          STRSETDATAFILE(jcl->jlog, idxfile, jcl->start.log);
          strlcat(idxfile, INDEX_EXT, sizeof(idxfile));
          unlink(idxfile);
          STRSETDATAFILE(jcl->jlog, idxfile, jcl->start.log + 1);
          strlcat(idxfile, INDEX_EXT, sizeof(idxfile));
          unlink(idxfile);
          goto alldone;
          break;
        default:
          goto alldone;
      }
    }
    if(jcl->count > MAX_ROWS_AT_ONCE) {
      /* Artificially set down the range to make the batches a bit easier
       * to handle on the stratcond/postgres end.
       * However, we must have more data, so drop the sleeptime to 0
       */
      jcl->count = MAX_ROWS_AT_ONCE;
      jcl->finish.marker = jcl->start.marker + jcl->count;
      sleeptime = 0;
    }
    if(jcl->count > 0) {
      if(noit_jlog_push(e, jcl)) {
        goto alldone;
      }
      /* Read our jlog_id accounting for possibly short reads */
      bytes_read = 0;
      while(bytes_read < sizeof(jlog_id)) {
        int len;
        if((len = e->opset->read(e->fd, inbuff + bytes_read,
                                 sizeof(jlog_id) - bytes_read,
                                 &mask, e)) <= 0)
          goto alldone;
        bytes_read += len;
      }
      memcpy(&client_chkpt, inbuff, sizeof(jlog_id));
      /* Fix the endian */
      client_chkpt.log = ntohl(client_chkpt.log);
      client_chkpt.marker = ntohl(client_chkpt.marker);
  
      if(memcmp(&jcl->chkpt, &client_chkpt, sizeof(jlog_id))) {
        noitL(noit_error,
              "client %s submitted invalid checkpoint %u:%u expected %u:%u\n",
              ac->remote_cn, client_chkpt.log, client_chkpt.marker,
              jcl->chkpt.log, jcl->chkpt.marker);
        goto alldone;
      }
      gettimeofday(&jcl->feed_stats->last_checkpoint, NULL);
      jlog_ctx_read_checkpoint(jcl->jlog, &jcl->chkpt);
    }
    else {
      /* we have nothing to write -- maybe we have no checks configured...
       * If this is the case "forever", the remote might disconnect and
       * we would never know. Do the painful work of detecting a
       * disconnected client.
       */
      struct pollfd pfd;
      pfd.fd = e->fd;
      pfd.events = POLLIN | POLLHUP | POLLRDNORM;
      pfd.revents = 0;
      if(poll(&pfd, 1, 0) != 0) {
        /* normally, we'd recv PEEK|DONTWAIT.  However, the client should
         * not be writing to us.  So, we know we can't have any legitimate
         * data on this socket (true even though this is SSL). So, if we're
         * here then "shit went wrong"
         */
        noitL(noit_error, "jlog client %s disconnected while idle\n",
              ac->remote_cn);
        goto alldone;
      }
    }
    if(sleeptime) sleep(sleeptime);
  }

 alldone:
  e->opset->close(e->fd, &mask, e);
  noit_atomic_dec32(&jcl->feed_stats->connections);
  noit_jlog_closure_free(jcl);
  acceptor_closure_free(ac);
  return NULL;
}
Exemple #3
0
/*
 * returns -1 when the incoming checkpoint is out of range of the log
 * returns -2 if there was an error actually setting the checkpoint
 */
static int queue_log_add_checkpoint(fqd_queue_impl_data data, const char *name, const fq_msgid *id)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  jlog_id jid = {
    .log = id->id.u32.p1,
    .marker = id->id.u32.p2
  };

  /* ensure the checkpoint makes sense */
  jlog_id first = {
    .log = 0,
    .marker = 0
  };
  jlog_id last = {
    .log = 0,
    .marker = 0
  };
  jlog_ctx_first_log_id(d->reader, &first);
  jlog_ctx_last_log_id(d->reader, &last);

  if (! (jid.log >= first.log && jid.log <= last.log &&
         jid.marker >= first.marker && jid.marker <= last.marker)) {
    return -1;
  }

  char **subs;
  int sub_count = jlog_ctx_list_subscribers(d->reader, &subs);

  int have_it = 0;
  for (int i = 0; i < sub_count; i++) {
    have_it += strcmp(subs[i], name) == 0 ? 1 : 0;
  }

  if (have_it == 0) {
    jlog_ctx_add_subscriber(d->reader, name, JLOG_BEGIN);
  }

  if (jlog_ctx_read_checkpoint(d->reader, &jid) == -1) {
    /*
       If we failed to checkpoint we are in a situation where the 'add_subscriber' call above put
       them at the beginning of the log so we have to remove the subscriber if we just added them

       However, if they already existed and had a previous good checkpoint, leave it alone
    */
    if (have_it == 0) {
      jlog_ctx_remove_subscriber(d->reader, name);
    }
    return -2;
  }
  return 0;
}

/*
 * return -1 if the subscriber doesn't exist
 * return 0 on success
 */
static int queue_log_remove_checkpoint(fqd_queue_impl_data data, const char *name)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  if (jlog_ctx_remove_subscriber(d->reader, name) == 0) {
    return -1;
  }
  return 0;
}

/*
 * return -1 if the subscriber doesn't exist
 * return -2 if we can't reset the checkpoint
 * return 0 on success
 */
static int queue_log_reset_to_checkpoint(fqd_queue_impl_data data, const char *name)
{
  struct queue_jlog *d = (struct queue_jlog *)data;

  char **subs;
  int sub_count = jlog_ctx_list_subscribers(d->reader, &subs);

  int have_it = 0;
  for (int i = 0; i < sub_count; i++) {
    have_it += strcmp(subs[i], name) == 0 ? 1 : 0;
  }

  if (have_it == 0) {
    return -1;
  }

  jlog_id checkpoint;
  if (jlog_get_checkpoint(d->reader, name, &checkpoint) == -1) {
    return -2;
  }

  if (jlog_ctx_read_checkpoint(d->reader, &checkpoint) == -1) {
    return -2;
  }
  return 0;
}


static int write_sig(struct queue_jlog *d) {
  char sigfile[PATH_MAX];
  int fd;
  snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath);
  fd = open(sigfile, O_CREAT|O_TRUNC|O_WRONLY, 0640);
  if(fd < 0) return -1;
  write(fd, d->uuid, 16);
  close(fd);
  return 0;
}
static int read_sig(struct queue_jlog *d, uuid_t out) {
  char sigfile[PATH_MAX];
  int fd, rv;
  snprintf(sigfile, sizeof(sigfile), "%s/.sig", d->qpath);
  fd = open(sigfile, O_RDONLY);
  if(fd < 0) return -1;
  rv = read(fd, out, 16);
  close(fd);
  return (rv == 16) ? 0 : -1;
}
static fqd_queue_impl_data queue_jlog_setup(fq_rk *qname, uint32_t *count) {
  char qpath[PATH_MAX];
  jlog_id chkpt;
  struct queue_jlog *d;

  d = calloc(1, sizeof(*d));
  d->auto_chkpt = true;
  fqd_config_construct_queue_path(qpath, sizeof(qpath), qname);
  d->qpath = strdup(qpath);
  d->writer = jlog_new(d->qpath);

  jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
  jlog_ctx_set_multi_process(d->writer, 0);
  jlog_ctx_set_use_compression(d->writer, 1);

  if(jlog_ctx_open_writer(d->writer) != 0) {
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
    jlog_ctx_set_multi_process(d->writer, 0);
    jlog_ctx_set_use_compression(d->writer, 1);
    if(jlog_ctx_init(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog init: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
    jlog_ctx_close(d->writer);
    d->writer = jlog_new(d->qpath);
    jlog_ctx_set_pre_commit_buffer_size(d->writer, 1024 * 1024);
    jlog_ctx_set_multi_process(d->writer, 0);
    jlog_ctx_set_use_compression(d->writer, 1);
    if(jlog_ctx_open_writer(d->writer) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog writer: %s\n", jlog_ctx_err_string(d->writer));
      goto bail;
    }
  }

  /* 128MB journal chunks */
  jlog_ctx_alter_journal_size(d->writer, 128 * 1024 * 1024);

  d->reader = jlog_new(d->qpath);
  if(jlog_get_checkpoint(d->reader, "fq", &chkpt) != 0) {
    if(jlog_ctx_add_subscriber(d->reader, "fq", JLOG_BEGIN) != 0) {
      fq_debug(FQ_DEBUG_IO, "jlog add sub: %s\n", jlog_ctx_err_string(d->reader));
      goto bail;
    }
  }
  if(jlog_ctx_open_reader(d->reader, "fq") != 0) {
    fq_debug(FQ_DEBUG_IO, "jlog: %s\n", jlog_ctx_err_string(d->reader));
    goto bail;
  }
  uuid_generate(d->uuid);
  write_sig(d);
  *count = d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish);
  (void)qname;
  return d;

 bail:
  if(d->writer) jlog_ctx_close(d->writer);
  if(d->reader) jlog_ctx_close(d->reader);
  free(d->qpath);
  free(d);
  return NULL;
}
Exemple #4
0
void *reader(void *unused) {
  jlog_ctx *ctx;
  char subname[32];
  int tcount = 0, fcount = 0;
  int prev_err = 0;
  int subno = (int)(uintptr_t)unused;
  snprintf(subname, sizeof(subname), "sub-%02d", subno);
reader_retry:
  ctx = jlog_new(LOGNAME);
  if(jlog_ctx_open_reader(ctx, subname) != 0) {
    if(prev_err == 0) {
      prev_err = jlog_ctx_err(ctx);
      jlog_ctx_close(ctx);
      ctx = jlog_new(LOGNAME);
      if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) {
        fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno);
        if(jlog_ctx_open_writer(ctx) != 0) {
          fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
        } else {
          if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) {
            fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
          } else {
            jlog_ctx_close(ctx);
            goto reader_retry;
          }
        }
      }
    }
    fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
    croak();
  }
  fprintf(stderr, "[%02d] reader started\n", subno);
  while(1) {
    char begins[20], ends[20];
    jlog_id begin, end;
    int count;
    jlog_message message;
    if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) {
      fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      croak();
    }
    jlog_snprint_logid(begins, sizeof(begins), &begin);
    jlog_snprint_logid(ends, sizeof(ends), &end);
    if(count > 0) {
      int i;
      //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count);
      for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) {
        end = begin;
        if(jlog_ctx_read_message(ctx, &begin, &message) != 0) {
          if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) {
            /* fine */
          } else {
            fcount++;
            jlog_snprint_logid(begins, sizeof(begins), &begin);
            fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
          }
        } else {
          tcount++;
          jlog_snprint_logid(begins, sizeof(begins), &begin);
          /* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins,
                  message.mess_len, (char *)message.mess); */
        }
      }
      if(jlog_ctx_read_checkpoint(ctx, &end) != 0) {
        fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      } else {
        fprintf(stderr, "[%02d] \tcheckpointed...\n", subno);
      }
    } else {
      if(writer_done == 1) break;
    }
  }
  fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount);
  jlog_ctx_close(ctx);
  return (void *)(uintptr_t)tcount;
}