예제 #1
0
void *spr_worker(void *data) {
  int rc=-1, nc;
  void *sp=NULL, *set;
  size_t len;
  char *buf;

  set = kv_set_new();
  sp = kv_spoolreader_new(CF.dir);
  if (!sp) goto done;

  while (kv_spool_read(sp,set,1) > 0) {
    if (set_to_buf(set,&buf,&len) < 0) goto done;
    assert(len < MAX_BUF);
    nc = nn_send(CF.ingress_socket_push, buf, len, 0);
    free(buf);
    if (nc < 0) {
      fprintf(stderr,"nn_send: %s\n", nn_strerror(errno));
      goto done;
    }
    ts_add(CF.spr_msgs_ts, CF.now, NULL);
    if (CF.shutdown) break;
  }

 done:
  CF.shutdown = 1;
  kv_set_free(set);
  if (sp) kv_spoolreader_free(sp);
  return NULL;
}
예제 #2
0
/******************************************************************************
 * kvsp-spr: a spool reader, for manually inspecting contents of a spool
 *****************************************************************************/
int main(int argc, char *argv[]) {

  char *exe = argv[0];
  int opt;

  while ( (opt = getopt(argc, argv, "fv+")) != -1) {
    switch (opt) {
      case 'v': verbose++; break;
      case 'f': mode=mode_nop; break;
      default: usage(exe); break;
    }
  }
  if (optind < argc) dir=argv[optind++];
  else usage(exe);

  void *sp = kv_spoolreader_new(dir);
  if (!sp) exit(-1);


  void *set = kv_set_new();
  while (kv_spool_read(sp,set,1) > 0) {
    switch(mode) {
      case mode_nop: continue;
      case mode_out: kv_set_dump(set,stdout); printf("\n"); break;
      default: assert(0); break;
    }
  }

 done:
  kv_spoolreader_free(sp);
  kv_set_free(set);
  return 0;
}
예제 #3
0
파일: kvsp-bpub.c 프로젝트: JHUAPL/kvspool
int main(int argc, char *argv[]) {
  void *sp=NULL;
  void *set=NULL;
  int opt,rc=-1;
  char *config_file;
  set = kv_set_new();
  utarray_new(output_keys, &ut_str_icd);
  utarray_new(output_defaults, &ut_str_icd);
  utarray_new(output_types,&ut_int_icd);
  utstring_new(tmp);

  while ( (opt = getopt(argc, argv, "v+d:b:s")) != -1) {
    switch (opt) {
      case 'v': verbose++; break;
      case 's': push_mode++; break;
      case 'd': spool=strdup(optarg); break;
      case 'b': config_file=strdup(optarg); break;
      default: usage(argv[0]); break;
    }
  }
  if (optind < argc) pub_transport = argv[optind++];
  if (!pub_transport) usage(argv[0]);
  if (spool == NULL) usage(argv[0]);
  if (parse_config(config_file) < 0) goto done;

  if ( !(pub_context = zmq_init(1))) goto done;
  if ( !(pub_socket = zmq_socket(pub_context, push_mode?ZMQ_PUSH:ZMQ_PUB))) goto done;
  if (zmq_setsockopt(pub_socket, ZMQ_SNDHWM, &hwm, sizeof(hwm))) goto done;
  if (zmq_bind(pub_socket, pub_transport) == -1) goto done;

  sp = kv_spoolreader_new(spool);
  if (!sp) goto done;

  while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */
    zmq_msg_t part;
    if (set_to_binary(set,&part) < 0) goto done;
    rc = zmq_sendmsg(pub_socket, &part, 0);
    zmq_msg_close(&part);
    if (rc == -1) goto done;
  }

  rc = 0;

 done:
  if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno));
  if (pub_socket) zmq_close(pub_socket);
  if (pub_context) zmq_term(pub_context);
  if (sp) kv_spoolreader_free(sp);
  kv_set_free(set);
  utarray_free(output_keys);
  utarray_free(output_defaults);
  utarray_free(output_types);
  utstring_free(tmp);

  return 0;
}
예제 #4
0
파일: kvsp-sub.c 프로젝트: JHUAPL/kvspool
int main(int argc, char *argv[]) {

  zmq_rcvmore_t more; size_t more_sz = sizeof(more);
  char *exe = argv[0], *filter = "";
  int part_num,opt,rc=-1;
  void *msg_data, *sp, *set=NULL;
  char **endpoint;
  UT_array *endpoints;
  size_t msg_len;
  zmq_msg_t part;

  utarray_new(endpoints,&ut_str_icd);

  while ( (opt = getopt(argc, argv, "sd:f:v+")) != -1) {
    switch (opt) {
      case 'v': verbose++; break;
      case 's': pull_mode++; break;
      case 'd': dir=strdup(optarg); break;
      case 'f': remotes_file=strdup(optarg); break;
      default: usage(exe); break;
    }
  }
  if (!dir) usage(exe);

  sp = kv_spoolwriter_new(dir);
  if (!sp) usage(exe);
  set = kv_set_new();

  /* connect socket to each publisher. yes, zeromq lets you connect n times */
  if ( !(context = zmq_init(1))) goto done;
  if ( !(socket = zmq_socket(context, pull_mode?ZMQ_PULL:ZMQ_SUB))) goto done;
  if (remotes_file) if (read_lines(remotes_file,endpoints)) goto done;
  while (optind < argc) utarray_push_back(endpoints,&argv[optind++]);
  endpoint=NULL;
  if (utarray_len(endpoints) == 0) usage(argv[0]);
  while ( (endpoint=(char**)utarray_next(endpoints,endpoint))) {
    if (verbose) fprintf(stderr,"connecting to %s\n", *endpoint);
    if (zmq_connect(socket, *endpoint)) goto done;
  }
  if (!pull_mode) {
    if (zmq_setsockopt(socket, ZMQ_SUBSCRIBE, filter, strlen(filter))) goto done;
  }

  while(1) {

    /* receive a multi-part message */
    part_num=1;
    do {
      if ( (rc= zmq_msg_init(&part))) goto done;
      if ( ((rc= zmq_recvmsg(socket, &part, 0)) == -1) || 
           ((rc= zmq_getsockopt(socket, ZMQ_RCVMORE, &more,&more_sz)) != 0)) {
        zmq_msg_close(&part);
        goto done;
      }

      msg_data = zmq_msg_data(&part);
      msg_len = zmq_msg_size(&part);
       
      switch(part_num) {  /* part 1 has serialized frame */
        case 1: if (json_to_frame(sp,set,msg_data,msg_len)) goto done; break;
        default: assert(0); 
      }

      zmq_msg_close(&part);
      part_num++;
    } while(more);
  }
  rc = 0; /* not reached TODO under clean shutdown on signal */


 done:
  if (rc) fprintf(stderr,"%s: %s\n", exe, zmq_strerror(errno));
  if(socket) zmq_close(socket);
  if(context) zmq_term(context);
  kv_spoolwriter_free(sp);
  if (set) kv_set_free(set);
  utarray_free(endpoints);
  return rc;
}
예제 #5
0
파일: test6.c 프로젝트: JHUAPL/kvspool
/******************************************************************************
 * the test itself
 *****************************************************************************/
int main(int argc, char *argv[]) {

  char *dir = mktmpdir();

  void *sp = kv_spoolwriter_new(dir);
  if (!sp) exit(-1);

  void *set = kv_set_new();
  kv_adds(set, "hello", "again");
  kv_adds(set, "second", "life");
  kv_spool_write(sp,set);

  /* scan spool to validate expected file creation */
  scan_spool(0);

  /* replace a value in the set, spool the set out. it should append
   * to the spool file previously created. */
  printf("spooling second frame\n");
  kv_adds(set, "second", "time");
  kv_spool_write(sp,set);
  kv_spoolwriter_free(sp);

  /* verify spool has updated according to expectation */
  scan_spool(0);

  printf("clear set\n");
  kv_set_clear(set);
  /* loop over them */
  kv_t *kv = NULL;
  while ( (kv = kv_next(set, kv))) {
    printf("key [%.*s], val [%.*s]\n", kv->klen, kv->key, kv->vlen, kv->val);
  }

  printf("reading from spool\n");
  /* now try reading the spool */
  sp = kv_spoolreader_new(dir);
  int rc;
  while ( (rc=kv_spool_read(sp, set, 0)) == 1) {
    printf("reader read frame:\n");
    kv = NULL;
    while ( (kv = kv_next(set, kv))) {
      printf(" key [%.*s], val [%.*s]\n", kv->klen, kv->key, kv->vlen, kv->val);
    }
  }
  printf("kv_spool_read returned %d\n", rc);
  kv_spoolreader_free(sp);

  /* now reset it and read again */
  printf("resetting the spool directory for another round of reading\n");
  sp_reset(dir);
  sp = kv_spoolreader_new(dir);
  while ( (rc=kv_spool_read(sp, set, 0)) == 1) {
    printf("reader read frame:\n");
    kv = NULL;
    while ( (kv = kv_next(set, kv))) {
      printf(" key [%.*s], val [%.*s]\n", kv->klen, kv->key, kv->vlen, kv->val);
    }
  }
  printf("kv_spool_read returned %d\n", rc);
  kv_spoolreader_free(sp);

  kv_set_free(set);
  scan_spool(1);
  return 0;
}
예제 #6
0
파일: kvsp-tpub.c 프로젝트: JHUAPL/kvspool
int main(int argc, char *argv[]) {
  int opt, rc, n, *fd;
  cfg.prog = argv[0];
  utarray_new(cfg.clients,&ut_int_icd);
  utarray_new(cfg.outbufs,&ut_ptr_icd);
  utarray_new(cfg.outidxs, &ut_int_icd);
  cfg.set = kv_set_new();
  struct epoll_event ev;
  UT_string **s;

  utstring_new(cfg.s);
  utarray_new(output_keys, &ut_str_icd);
  utarray_new(output_defaults, &ut_str_icd);
  utarray_new(output_types,&ut_int_icd);

  while ( (opt=getopt(argc,argv,"vb:p:m:d:rS:h")) != -1) {
    switch(opt) {
      case 'v': cfg.verbose++; break;
      case 'p': cfg.listener_port=atoi(optarg); break; 
      case 'm': cfg.mb_per_client=atoi(optarg); break; 
      case 'd': cfg.dir=strdup(optarg); break; 
      case 'r': cfg.mode=round_robin; break; 
      case 'b': cfg.config_file=strdup(optarg); break;
      case 'S': cfg.stats_file=strdup(optarg); break;
      case 'h': default: usage(); break;
    }
  }
  if (cfg.listener_port==0) usage();
  if (setup_client_listener()) goto done;
  if (cfg.config_file==NULL) goto done;
  if (parse_config(cfg.config_file) < 0) goto done;
  if ( !(cfg.sp = kv_spoolreader_new(cfg.dir))) goto done;

  /* block all signals. we take signals synchronously via signalfd */
  sigset_t all;
  sigfillset(&all);
  sigprocmask(SIG_SETMASK,&all,NULL);

  /* a few signals we'll accept via our signalfd */
  sigset_t sw;
  sigemptyset(&sw);
  for(n=0; n < sizeof(sigs)/sizeof(*sigs); n++) sigaddset(&sw, sigs[n]);

  /* create the signalfd for receiving signals */
  cfg.signal_fd = signalfd(-1, &sw, 0);
  if (cfg.signal_fd == -1) {
    fprintf(stderr,"signalfd: %s\n", strerror(errno));
    goto done;
  }

  /* set up the epoll instance */
  cfg.epoll_fd = epoll_create(1); 
  if (cfg.epoll_fd == -1) {
    fprintf(stderr,"epoll: %s\n", strerror(errno));
    goto done;
  }

  /* add descriptors of interest */
  if (new_epoll(EPOLLIN, cfg.listener_fd)) goto done; // new client connections
  if (new_epoll(EPOLLIN, cfg.signal_fd))   goto done; // signal socket

  alarm(1);
  while (epoll_wait(cfg.epoll_fd, &ev, 1, -1) > 0) {
    if (cfg.verbose > 1)  fprintf(stderr,"epoll reports fd %d\n", ev.data.fd);
    if      (ev.data.fd == cfg.signal_fd)   { if (handle_signal() < 0) goto done; }
    else if (ev.data.fd == cfg.listener_fd) { if (accept_client() < 0) goto done; }
    else    feed_client(ev.data.fd, ev.events);
  }

done:
  /* free the clients: close and deep free their buffers */
  fd=NULL; s=NULL;
  while ( (fd=(int*)utarray_prev(cfg.clients,fd))) {
    s=(UT_string**)utarray_prev(cfg.outbufs,s);
    close(*fd);
    utstring_free(*s);
  }
  utarray_free(cfg.clients);
  utarray_free(cfg.outbufs);
  utarray_free(cfg.outidxs);
  utarray_free(output_keys);
  utarray_free(output_defaults);
  utarray_free(output_types);
  utstring_free(cfg.s);
  if (cfg.listener_fd) close(cfg.listener_fd);
  if (cfg.signal_fd) close(cfg.signal_fd);
  if (cfg.sp) kv_spoolreader_free(cfg.sp);
  if (cfg.set) kv_set_free(cfg.set);
  return 0;
}