Пример #1
0
static int
noit_jlog_push(eventer_t e, noit_jlog_closure_t *jcl) {
  jlog_message msg;
  int mask;
  u_int32_t n_count;
  n_count = htonl(jcl->count);
  if(Ewrite(&n_count, sizeof(n_count)) != sizeof(n_count))
    return -1;
  while(jcl->count > 0) {
    int rv;
    struct { jlog_id chkpt; u_int32_t n_sec, n_usec, n_len; } payload;
    if(jlog_ctx_read_message(jcl->jlog, &jcl->start, &msg) == -1)
      return -1;

    /* Here we actually push the message */
    payload.chkpt.log = htonl(jcl->start.log);
    payload.chkpt.marker = htonl(jcl->start.marker);
    payload.n_sec  = htonl(msg.header->tv_sec);
    payload.n_usec = htonl(msg.header->tv_usec);
    payload.n_len  = htonl(msg.mess_len);
    if((rv = Ewrite(&payload, sizeof(payload))) != sizeof(payload)) {
      noitL(noit_error, "Error writing jlog header over SSL %d != %d\n",
            rv, (int)sizeof(payload));
      return -1;
    }
    if((rv = Ewrite(msg.mess, msg.mess_len)) != msg.mess_len) {
      noitL(noit_error, "Error writing jlog message over SSL %d != %d\n",
            rv, msg.mess_len);
      return -1;
    }
    /* Note what the client must checkpoint */
    jcl->chkpt = jcl->start;

    JLOG_ID_ADVANCE(&jcl->start);
    jcl->count--;
  }
  return 0;
}
Пример #2
0
static fq_msg *queue_jlog_dequeue(fqd_queue_impl_data f) {
  struct queue_jlog *d = (struct queue_jlog *)f;
  jlog_message msg;
  fq_msg *m;
  if(d->count == 0 && d->last_seen_nenqueued == d->nenqueued) return NULL;
 retry:
  if(d->count <= 0) {
    d->count = jlog_ctx_read_interval(d->reader, &d->start, &d->finish);
    fq_debug(FQ_DEBUG_IO, "jlog read batch count -> %d\n", d->count);
    if(d->count < 0) {
      char idxfile[PATH_MAX];
      fq_debug(FQ_DEBUG_IO, "jlog_ctx_read_interval: %s\n",
               jlog_ctx_err_string(d->reader));
      switch (jlog_ctx_err(d->reader)) {
        case JLOG_ERR_FILE_CORRUPT:
        case JLOG_ERR_IDX_CORRUPT:
          jlog_repair_datafile(d->reader, d->start.log);
          jlog_repair_datafile(d->reader, d->start.log + 1);
          fq_debug(FQ_DEBUG_IO,
                "jlog reconstructed, deleting corresponding index.\n");
          STRSETDATAFILE(d->reader, idxfile, d->start.log);
          strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile));
          unlink(idxfile);
          STRSETDATAFILE(d->reader, idxfile, d->start.log + 1);
          strncpy(idxfile + strlen(idxfile), INDEX_EXT, sizeof(idxfile) - strlen(idxfile));
          unlink(idxfile);
          break;
        default:
          break;
      }
    }
    if(d->count <= 0) return NULL;
  }
  if(jlog_ctx_read_message(d->reader, &d->start, &msg) == -1) {
    d->count = 0;
    return NULL;
  }
  if(d->last_dequeued.log > d->start.log ||
     (d->last_dequeued.log == d->start.log &&
      d->last_dequeued.marker > d->start.marker)) {
    d->count--;
    JLOG_ID_ADVANCE(&d->start);
    goto retry;
  }
  if(msg.mess_len < sizeof(fq_msg)-1)
    m = NULL;
  else {
    off_t expected_len;
    uint32_t payload_len;
    m = (fq_msg *)msg.mess;
    memcpy(&payload_len, &m->payload_len, sizeof(m->payload_len));
    expected_len = offsetof(fq_msg, payload) + payload_len;
    if(expected_len != msg.mess_len) m = NULL;
    else {
      m = malloc(expected_len);
      memcpy(m, msg.mess, expected_len);
      m->sender_msgid.id.u32.p3 = d->start.log;
      m->sender_msgid.id.u32.p4 = d->start.marker;
    }
  }
  d->count--;
  fq_debug(FQ_DEBUG_IO, "jlog batch count -> %d\n", d->count);
  if(d->count == 0) {
    if(d->auto_chkpt) {
      jlog_ctx_read_checkpoint(d->reader, &d->start);
    }
  }
  d->last_dequeued = d->start;
  JLOG_ID_ADVANCE(&d->start);
  ck_pr_inc_uint(&d->last_seen_nenqueued);
  return m;
}
Пример #3
0
void *reader(void *unused) {
  jlog_ctx *ctx;
  char subname[32];
  int tcount = 0, fcount = 0;
  int prev_err = 0;
  int subno = (int)(uintptr_t)unused;
  snprintf(subname, sizeof(subname), "sub-%02d", subno);
reader_retry:
  ctx = jlog_new(LOGNAME);
  if(jlog_ctx_open_reader(ctx, subname) != 0) {
    if(prev_err == 0) {
      prev_err = jlog_ctx_err(ctx);
      jlog_ctx_close(ctx);
      ctx = jlog_new(LOGNAME);
      if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) {
        fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno);
        if(jlog_ctx_open_writer(ctx) != 0) {
          fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
        } else {
          if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) {
            fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
          } else {
            jlog_ctx_close(ctx);
            goto reader_retry;
          }
        }
      }
    }
    fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
    croak();
  }
  fprintf(stderr, "[%02d] reader started\n", subno);
  while(1) {
    char begins[20], ends[20];
    jlog_id begin, end;
    int count;
    jlog_message message;
    if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) {
      fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      croak();
    }
    jlog_snprint_logid(begins, sizeof(begins), &begin);
    jlog_snprint_logid(ends, sizeof(ends), &end);
    if(count > 0) {
      int i;
      //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count);
      for(i=0; i<count; i++, JLOG_ID_ADVANCE(&begin)) {
        end = begin;
        if(jlog_ctx_read_message(ctx, &begin, &message) != 0) {
          if(jlog_ctx_err(ctx) == JLOG_ERR_CLOSE_LOGID) {
            /* fine */
          } else {
            fcount++;
            jlog_snprint_logid(begins, sizeof(begins), &begin);
            fprintf(stderr, "[%02d] read failed @ %s: %d %s\n", subno, begins, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
          }
        } else {
          tcount++;
          jlog_snprint_logid(begins, sizeof(begins), &begin);
          /* fprintf(stderr, "[%02d] read: [%s]\n\t'%.*s'\n", subno, begins,
                  message.mess_len, (char *)message.mess); */
        }
      }
      if(jlog_ctx_read_checkpoint(ctx, &end) != 0) {
        fprintf(stderr, "[%02d] checkpoint failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx));
      } else {
        fprintf(stderr, "[%02d] \tcheckpointed...\n", subno);
      }
    } else {
      if(writer_done == 1) break;
    }
  }
  fprintf(stderr, "[%02d] reader read %d, failed %d\n", subno, tcount, fcount);
  jlog_ctx_close(ctx);
  return (void *)(uintptr_t)tcount;
}