Example #1
0
void *
fqd_worker_thread(void *arg)
{
  struct incoming_message *m;
  int tindex = *(int*) arg;

  ck_fifo_spsc_t *q = &work_queues[tindex];
  uint32_t *backlog = &work_queue_backlogs[tindex];

  while (!worker_thread_shutdown) {
    if (!CK_FIFO_SPSC_ISEMPTY(q)) {
      if (ck_fifo_spsc_dequeue(q, &m)) {
        if (m != NULL) {
          ck_pr_dec_32(backlog);
          fq_msg *copy = fq_msg_alloc_BLANK(m->msg->payload_len);
          if (copy == NULL) {
            continue;
          }
          memcpy(copy, m->msg, sizeof(fq_msg) + m->msg->payload_len);
          /* reset the refcnt on the copy since the memcpy above will have overwritten it */
          copy->refcnt = 1;
          /* the copy will be freed as normal so eliminate cleanup_stack pointer */
          copy->cleanup_stack = NULL;

          /* we are done with the incoming message, deref */
          fq_msg_deref(m->msg);

          fqd_inject_message(m->client, copy);
	  fqd_remote_client_deref((remote_client *)m->client);
          free(m);
        }
      }
    } else {
      usleep(1000);
    }
  }

  /* drain the queue before exiting */
  if (!CK_FIFO_SPSC_ISEMPTY(q)) {
    ck_fifo_spsc_dequeue_lock(q);
    if (ck_fifo_spsc_dequeue(q, &m)) {
      if (m != NULL) {
        ck_pr_dec_32(backlog);
        fq_msg *copy = fq_msg_alloc_BLANK(m->msg->payload_len);
        memcpy(copy, m->msg, sizeof(fq_msg) + m->msg->payload_len);
        /* the copy will be freed as normal so eliminate cleanup_stack pointer */
        copy->cleanup_stack = NULL;
        /* we are done with the incoming message, drop it on it's cleanup stack */
        fqd_inject_message(m->client, copy);
        fq_msg_deref(m->msg);
        free(m);
      }
    }
    ck_fifo_spsc_dequeue_unlock(q);
  }
  usleep(10);
  return NULL;
}
noit_metric_message_t *noit_metric_director_lane_next() {
  noit_metric_message_t *msg = NULL;
  if(my_lane.fifo == NULL)
    return NULL;
  ck_fifo_spsc_dequeue_lock(my_lane.fifo);
  if(ck_fifo_spsc_dequeue(my_lane.fifo, &msg) == false) {
    msg = NULL;
  }
  ck_fifo_spsc_dequeue_unlock(my_lane.fifo);
  return msg;
}
Example #3
0
static void *
fq_conn_worker(void *u) {
  int backoff = 0, i;
  bool zero;
  fq_conn_s *conn_s = (fq_conn_s *)u;
  cmd_instr *last_entries[MAX_PENDING] = { NULL };
  int last_entry_idx = 0;
  int next_entry_idx = 0;

#define SAVE_ENTRY(e) do { \
  if(last_entries[next_entry_idx] != NULL) { \
    CONNERR(conn_s, "exceed max cmd pipeline"); \
    goto restart; \
  } \
  last_entries[next_entry_idx++] = e; \
  next_entry_idx = next_entry_idx % MAX_PENDING; \
} while(0)
#define last_entry last_entries[last_entry_idx]
#define PROCESS_ENTRY(e, should_free) do { \
  fq_assert(last_entries[last_entry_idx] == e); \
  if(should_free) free(last_entries[last_entry_idx]); \
  last_entries[last_entry_idx++] = NULL; \
  last_entry_idx = last_entry_idx % MAX_PENDING; \
} while(0)

  cmd_instr *entry;

  ck_pr_inc_uint(&conn_s->thrcnt);

  while(conn_s->stop == 0) {
    int wait_ms = 50;
    if(fq_client_connect_internal(conn_s) == 0) {
      backoff = 0; /* we're good, restart our backoff */
    }
    while(conn_s->data_ready && conn_s->stop == 0) {
      hrtime_t t;
      unsigned long long hb_us;
      struct timeval tv;
      int rv;

      ck_fifo_spsc_dequeue_lock(&conn_s->cmdq);
      while(ck_fifo_spsc_dequeue(&conn_s->cmdq, &entry) == true) {
#ifdef DEBUG
        fq_debug(FQ_DEBUG_CONN, "client acting on user req 0x%04x\n", entry->cmd);
#endif
        switch(entry->cmd) {
          case FQ_PROTO_STATUSREQ:
            fq_debug(FQ_DEBUG_CONN, "sending status request\n");
            if(fq_write_uint16(conn_s->cmd_fd, entry->cmd)) {
              free(entry);
              CONNERR(conn_s, "write failed (statusreq)");
              goto restart;
            }
            SAVE_ENTRY(entry);
            break;
          case FQ_PROTO_HBREQ:
            fq_debug(FQ_DEBUG_CONN, "sending heartbeat request\n");
            if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
               fq_write_uint16(conn_s->cmd_fd, entry->data.heartbeat.ms)) {
              free(entry);
              CONNERR(conn_s, "write failed (hbreq)");
              goto restart;
            }
            conn_s->cmd_hb_ms = entry->data.heartbeat.ms;
            tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
            tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
            if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_RCVTIMEO,
                          &tv, sizeof(tv)))
              CONNERR(conn_s, strerror(errno));
            tv.tv_sec = (unsigned long)entry->data.heartbeat.ms / 1000;
            tv.tv_usec = 1000UL * (entry->data.heartbeat.ms % 1000);
            if(setsockopt(conn_s->cmd_fd, SOL_SOCKET, SO_SNDTIMEO,
                          &tv, sizeof(tv)))
              CONNERR(conn_s, strerror(errno));
            conn_s->cmd_hb_last = fq_gethrtime();
            free(entry);
            break;
          case FQ_PROTO_BINDREQ:
            {
              unsigned short flags = entry->data.bind->flags;
              if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
                 fq_write_uint16(conn_s->cmd_fd, flags) ||
                 fq_write_short_cmd(conn_s->cmd_fd,
                                    entry->data.bind->exchange.len,
                                    entry->data.bind->exchange.name) < 0 ||
                 fq_write_short_cmd(conn_s->cmd_fd,
                                    strlen(entry->data.bind->program),
                                    entry->data.bind->program) < 0) {
	CONNERR(conn_s, "write failed (bindreq)");
                goto restart;
              }
              SAVE_ENTRY(entry);
            }
            break;
          case FQ_PROTO_UNBINDREQ:
            {
              if(fq_write_uint16(conn_s->cmd_fd, entry->cmd) ||
                 fq_write_uint32(conn_s->cmd_fd, entry->data.unbind->route_id) ||
                 fq_write_short_cmd(conn_s->cmd_fd,
                                    entry->data.unbind->exchange.len,
                                    entry->data.unbind->exchange.name) < 0) {
	CONNERR(conn_s, "write failed (unbindreq)");
                goto restart;
              }
              SAVE_ENTRY(entry);
            }
            break;
          default:
            CONNERR(conn_s, "unknown user-side cmd");
            free(entry);
        }
      }
      ck_fifo_spsc_dequeue_unlock(&conn_s->cmdq);

      if(conn_s->cmd_hb_needed) {
#ifdef DEBUG
          fq_debug(FQ_DEBUG_CONN, "-> heartbeat\n");
#endif
        if(fq_write_uint16(conn_s->cmd_fd, FQ_PROTO_HB)) {
          CONNERR(conn_s, "write failed (hb)");
          break;
        }
        conn_s->cmd_hb_needed = 0;
      }

      rv = fq_client_wfrw_internal(conn_s->cmd_fd, 1, 0, wait_ms, NULL);
      if(rv == 0) {
        wait_ms = (wait_ms >> 2) + wait_ms;
        if(conn_s->cmd_hb_ms && wait_ms > conn_s->cmd_hb_ms)
          wait_ms = conn_s->cmd_hb_ms;
        if(wait_ms > 1000) wait_ms = 1000;
      }
      else wait_ms = 50;
      fq_debug(FQ_DEBUG_CONN, "fq_client_wfrw_internal(cmd:%d) -> %d\n", conn_s->cmd_fd, rv);
      t = fq_gethrtime();
      hb_us = (unsigned long long)conn_s->cmd_hb_ms * 3 * 1000000ULL;
      if(conn_s->cmd_hb_last && hb_us &&
         conn_s->cmd_hb_last < (unsigned int) (t - hb_us)) {
        char errbuf[256];
        snprintf(errbuf, sizeof(errbuf), "heartbeat failed [%llu - %llu = %llu]",
                 (unsigned long long)t, (unsigned long long)conn_s->cmd_hb_last,
                 (unsigned long long)(t - conn_s->cmd_hb_last));
        CONNERR(conn_s, errbuf);
        break;
      }
      if(rv < 0) {
        CONNERR(conn_s, strerror(errno));
        break;
      }
      if(rv > 0) {
        bool handled = false;
        uint16_t hb;
        if(fq_read_uint16(conn_s->cmd_fd, &hb)) break;
        switch(hb) {
          case FQ_PROTO_HB:
#ifdef DEBUG
            fq_debug(FQ_DEBUG_CONN, "<- heartbeat\n");
#endif
            conn_s->cmd_hb_last = fq_gethrtime();
            conn_s->cmd_hb_needed = 1;
            break;
          case FQ_PROTO_STATUS:
            if(!last_entry || last_entry->cmd != FQ_PROTO_STATUSREQ) {
              CONNERR(conn_s, "protocol violation (status unexpected)");
              goto restart;
            }
            if(fq_read_status(conn_s->cmd_fd,
                              last_entry->data.status.callback,
                              last_entry->data.status.closure))
              goto restart;
            PROCESS_ENTRY(last_entry, 1);
            break;
          case FQ_PROTO_BIND:
            if(!last_entry || last_entry->cmd != FQ_PROTO_BINDREQ) {
              CONNERR(conn_s, "protocol violation (bind unexpected)");
              goto restart;
            }
            if(fq_read_uint32(conn_s->cmd_fd,
                              &last_entry->data.bind->out__route_id)) {
              if(conn_s->bind_hook) {
                if(conn_s->sync_hooks) {
                  enqueue_cmd_hook_req(conn_s, last_entry);
	  PROCESS_ENTRY(last_entry, 0);
	  handled = true;
                }
                else conn_s->bind_hook((fq_client)conn_s, last_entry->data.bind);
              }
              CONNERR(conn_s, "read failed (bind)");
              goto restart;
            }
            if(conn_s->bind_hook) {
              if(conn_s->sync_hooks) {
                enqueue_cmd_hook_req(conn_s, last_entry);
	PROCESS_ENTRY(last_entry, 0);
	handled = true;
              }
              else conn_s->bind_hook((fq_client)conn_s, last_entry->data.bind);
            }
            if(!handled) PROCESS_ENTRY(last_entry, 1);
            break;
          case FQ_PROTO_UNBIND:
            if(!last_entry || last_entry->cmd != FQ_PROTO_UNBINDREQ) {
              CONNERR(conn_s, "protocol violation (unbind unexpected)");
              goto restart;
            }
            if(fq_read_uint32(conn_s->cmd_fd,
                              &last_entry->data.unbind->out__success)) {
              if(conn_s->unbind_hook) {
                if(conn_s->sync_hooks) {
                  enqueue_cmd_hook_req(conn_s, last_entry);
                  PROCESS_ENTRY(last_entry, 0);
	  handled = true;
                }
                conn_s->unbind_hook((fq_client)conn_s, last_entry->data.unbind);
              }
              CONNERR(conn_s, "read failed (unbind)");
              goto restart;
            }
            if(conn_s->unbind_hook) {
              if(conn_s->sync_hooks) {
                enqueue_cmd_hook_req(conn_s, last_entry);
                PROCESS_ENTRY(last_entry, 0);
	handled = true;
                last_entry = NULL;
              }
              conn_s->unbind_hook((fq_client)conn_s, last_entry->data.unbind);
            }
            if(!handled) PROCESS_ENTRY(last_entry,1);
            break;
          default:
            CONNERR(conn_s, "protocol violation");
            goto restart;
            break;
        }
      }
    }
Example #4
0
static void
fq_data_worker_loop(fq_conn_s *conn_s) {
  buffered_msg_reader *ctx = NULL;
  ctx = fq_buffered_msg_reader_alloc(conn_s->data_fd, 1);
  while(conn_s->cmd_fd >= 0 && conn_s->data_fd >= 0 && conn_s->stop == 0) {
    int rv;
    int wait_ms = 500, needs_write = 0, mask, write_rv;
    if(conn_s->tosend) goto the_thick_of_it;
    ck_fifo_spsc_dequeue_lock(&conn_s->q);
    while(ck_fifo_spsc_dequeue(&conn_s->q, &conn_s->tosend) == true) {
      conn_s->tosend_offset = 0;
      ck_pr_dec_uint(&conn_s->qlen);
     the_thick_of_it:
#ifdef DEBUG
      fq_debug(FQ_DEBUG_MSG, "dequeue message to submit to server\n");
#endif
      write_rv = fq_client_write_msg(conn_s->data_fd, conn_s->peermode,
                                     conn_s->tosend, conn_s->tosend_offset, NULL);
      if(write_rv > 0) {
        conn_s->tosend_offset += write_rv;
        wait_ms = 0;
        break;
      }
      if(write_rv < 0) {
        if(errno == EAGAIN) {
          needs_write = 1;
          break;
        }
        if(conn_s->errorlog) {
          char errbuf[128];
          snprintf(errbuf, sizeof(errbuf), "data write error: %s\n", strerror(errno));
          conn_s->errorlog(conn_s, errbuf);
        }
        ck_fifo_spsc_dequeue_unlock(&conn_s->q);
        goto finish;
      }
      fq_msg_deref(conn_s->tosend);
      conn_s->tosend = NULL;
      conn_s->tosend_offset = 0;
      wait_ms = 0;
    }
    ck_fifo_spsc_dequeue_unlock(&conn_s->q);

    rv = fq_client_wfrw_internal(conn_s->data_fd, 1, needs_write, wait_ms, &mask);
    fq_debug(FQ_DEBUG_CONN, "fq_client_wfrw_internal(data:%d) -> %d\n", conn_s->data_fd, rv);
    if(rv < 0) {
      if(conn_s->errorlog) {
        char errbuf[128];
        snprintf(errbuf, sizeof(errbuf), "data read error: %s\n", strerror(errno));
        conn_s->errorlog(conn_s, errbuf);
      }
      goto finish;
    }
    if(rv > 0 && (mask & POLLIN)) {
      if(fq_buffered_msg_read(ctx, fq_client_read_complete, conn_s) < 0) {
        if(conn_s->errorlog) conn_s->errorlog(conn_s, "data read: end-of-line\n");
        goto finish;
      }
    }
  }
finish:
  fq_clear_message_cleanup_stack();
  if(ctx) fq_buffered_msg_reader_free(ctx);
#ifdef DEBUG
  fq_debug(FQ_DEBUG_CONN, "cmd_fd -> %d, stop -> %d\n", conn_s->cmd_fd, conn_s->stop);
#endif
}