Ejemplo n.º 1
0
/////////////////////////////// destroy double linked list ////////////////////
static void destroy_dlist_node(dlist_node *p_node)
{
   if (NULL != p_node)
   {
      ds_free(p_node->p);
      ds_free(p_node);
   }
}
Ejemplo n.º 2
0
/*
 * Free current thread data structures etc..
 */
void exit_thread(void)
{
	/* The process may have allocated an io port bitmap... nuke it. */
	if (unlikely(test_thread_flag(TIF_IO_BITMAP))) {
		struct task_struct *tsk = current;
		struct thread_struct *t = &tsk->thread;
		int cpu = get_cpu();
		struct tss_struct *tss = &per_cpu(init_tss, cpu);

		kfree(t->io_bitmap_ptr);
		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, tss->io_bitmap_max);
		t->io_bitmap_max = 0;
		tss->io_bitmap_owner = NULL;
		tss->io_bitmap_max = 0;
		tss->x86_tss.io_bitmap_base = INVALID_IO_BITMAP_OFFSET;
		put_cpu();
	}
#ifdef CONFIG_X86_DS
	/* Free any DS contexts that have not been properly released. */
	if (unlikely(current->thread.ds_ctx)) {
		/* we clear debugctl to make sure DS is not used. */
		update_debugctlmsr(0);
		ds_free(current->thread.ds_ctx);
	}
#endif /* CONFIG_X86_DS */
}
Ejemplo n.º 3
0
/** Removes a remote from the server
 *  \relates adbus_Server
 */
void adbus_remote_disconnect(adbus_Remote* r)
{
    if (r == NULL)
        return;

    adbus_Server* s = r->server;

    dl_remove(Remote, r, &r->hl);

    // Free the matches
    struct Match* m = r->matches.next;
    while (m) {
        struct Match* next = m->hl.next;
        adbusI_serv_freematch(m);
        m = next;
    }
    dl_clear(Match, &r->matches);

    while (dv_size(&r->services) > 0) {
        adbusI_serv_releasename(s, r, dv_a(&r->services, 0)->name);
    }
    dv_free(Service, &r->services);


    adbus_buf_free(r->msg);
    adbus_buf_free(r->dispatch);
    ds_free(&r->unique);
    free(r);
}
Ejemplo n.º 4
0
int ds_set(delim_str** ds, const char* str, int len) {
    if (ds_resize(ds,len) != len) {
        ds_free(ds);
        return 0;
    }
    memcpy((*ds)->str, str, len);
    return len;
}
Ejemplo n.º 5
0
void cleanupMotionDetection(MotionDetect* md) {
  if(md->fields) {
    ds_free(md->fields);
    md->fields=0;
  }
  if (md->prev) {
    ds_free(md->prev);
    md->prev = NULL;
  }
  if (md->curr) {
    ds_free(md->curr);
    md->curr = NULL;
  }
  if (md->currtmp) {
    ds_free(md->currtmp);
    md->currtmp = NULL;
  }
  md->initialized = 0;
}
Ejemplo n.º 6
0
void adbusI_logreply(const char* header, const adbus_Reply* r)
{
    if (!adbusI_log_enabled())
        return;
    d_String str;
    ZERO(&str);
    ReplyString(&str, r);
    adbusI_addheader(&str, "%-10s ", header);
    adbusI_klog(&str);
    ds_free(&str);
}
Ejemplo n.º 7
0
void adbusI_logmatch(const char* header, const adbus_Match* m)
{
    if (!adbusI_log_enabled())
        return;
    d_String str;
    ZERO(&str);
    MatchString(&str, m);
    adbusI_addheader(&str, "%-10s ", header);
    adbusI_klog(&str);
    ds_free(&str);
}
Ejemplo n.º 8
0
void adbusI_logbind(const char* header, const adbus_Bind* b)
{
    if (!adbusI_log_enabled())
        return;
    d_String str;
    ZERO(&str);
    BindString(&str, b);
    adbusI_addheader(&str, "%-10s ", header);
    adbusI_klog(&str);
    ds_free(&str);
}
Ejemplo n.º 9
0
void adbusI_logmsg(const char* header, const adbus_Message* msg)
{
    if (!adbusI_log_enabled())
        return;
    d_String str;
    ZERO(&str);
    MsgSummary(&str, msg);
    adbusI_addheader(&str, "%-10s ", header);
    adbusI_klog(&str);
    ds_free(&str);
}
Ejemplo n.º 10
0
void boxblurYUV(unsigned char* dest, const unsigned char* src,
        unsigned char* buffer, const DSFrameInfo* fi,
        unsigned int size, BoxBlurColorMode colormode){
  int localbuffer=0;
  int size2;
  int offset = fi->width * fi->height;
  if(size<2){
    if(dest!=buffer)
      memcpy(dest,src,fi->framesize);
    return;
  }
  if(buffer==0){
    buffer=(unsigned char*) ds_malloc(fi->framesize);
    localbuffer=1;
  }
  // odd and larger than 2 and maximally half of smaller image dimension
  size  = DS_CLAMP((size/2)*2+1,3,DS_MIN(fi->height/2,fi->width/2));
  //printf("%i\n",size);

  // luminance
  boxblur_hori_C(buffer,  src, fi->width, fi->height, fi->strive, size);
  boxblur_vert_C(dest, buffer, fi->width, fi->height, fi->strive, size);

  size2 = size/2+1;                        // odd and larger than 0
  switch (colormode){
  case BoxBlurColor:
    // color
    if(size2>1){
      boxblur_hori_C(buffer+offset, src+offset,
             fi->width/2, fi->height/2, fi->strive/2, size2);
      boxblur_hori_C(buffer+5*offset/4, src+5*offset/4,
             fi->width/2, fi->height/2, fi->strive/2, size2);

      boxblur_vert_C(dest+offset, buffer+offset,
             fi->width/2, fi->height/2, fi->strive/2, size2);
      boxblur_vert_C(dest+5*offset/4, buffer+5*offset/4,
             fi->width/2, fi->height/2, fi->strive/2, size2);
    }
    break;
  case BoxBlurKeepColor:
    // copy both color channels
    memcpy(dest+offset, src+offset, 2*(fi->strive / 2) * (fi->height / 2));
  case BoxBlurNoColor: // do nothing
  default:
    break;
  }

  if(localbuffer)
    ds_free(buffer);
}
Ejemplo n.º 11
0
/** Frees a message factory.
 *  \relates adbus_MsgFactory
 */
void adbus_msg_free(adbus_MsgFactory* m)
{
    if (!m)
        return;

    adbus_buf_free(m->buf);
    adbus_buf_free(m->argbuf);
    ds_free(&m->path);
    ds_free(&m->interface);
    ds_free(&m->member);
    ds_free(&m->error);
    ds_free(&m->destination);
    ds_free(&m->sender);
    free(m);
}
Ejemplo n.º 12
0
int adbusI_introspect(adbus_CbData* d)
{
    adbus_check_end(d);

    // If no reply is wanted, we're done
    if (!d->ret) {
        return 0;
    }

    struct ObjectPath* p = (struct ObjectPath*) d->user2;

    d_String out;
    ZERO(&out);
    IntrospectNode(p, &out);

    adbus_msg_setsig(d->ret, "s", 1);
    adbus_msg_string(d->ret, ds_cstr(&out), ds_size(&out));
    adbus_msg_end(d->ret);

    ds_free(&out);

    return 0;
}
Ejemplo n.º 13
0
Archivo: control.c Proyecto: bholt/tmux
static void
control_write_attached_session_change_cb(
    struct client *c, unused void *user_data)
{
	if (c->flags & CLIENT_SESSION_CHANGED) {
		struct dstring ds;
		ds_init(&ds);
		ds_appendf(&ds, "%%session-changed %d %s\n",
				   c->session->id, c->session->name);
		control_write_str(c, ds.buffer);
		ds_free(&ds);
		c->flags &= ~CLIENT_SESSION_CHANGED;
	}
	if (session_changed_flags &
	    (SESSION_CHANGE_ADDREMOVE | SESSION_CHANGE_RENAME)) {
		control_write_str(c, "%sessions-changed\n");
	}
	if ((session_changed_flags & SESSION_CHANGE_RENAME) &&
		(c->session->flags & SESSION_RENAMED)) {
		control_write_printf(c, "%%session-renamed %s\n",
				     c->session->name);
	}
}
Ejemplo n.º 14
0
__unused void destroy()
{
	ds_free(root.child, 1);
	root.child = NULL;
}
Ejemplo n.º 15
0
/*
 * Public functions
 */
int cu_rrd_create_file(const char *filename, /* {{{ */
                       const data_set_t *ds, const value_list_t *vl,
                       const rrdcreate_config_t *cfg) {
  char **argv;
  int argc;
  char **rra_def = NULL;
  int rra_num;
  char **ds_def = NULL;
  int ds_num;
  int status = 0;
  time_t last_up;
  unsigned long stepsize;

  if (check_create_dir(filename))
    return -1;

  if ((rra_num = rra_get(&rra_def, vl, cfg)) < 1) {
    P_ERROR("cu_rrd_create_file failed: Could not calculate RRAs");
    return -1;
  }

  if ((ds_num = ds_get(&ds_def, ds, vl, cfg)) < 1) {
    P_ERROR("cu_rrd_create_file failed: Could not calculate DSes");
    rra_free(rra_num, rra_def);
    return -1;
  }

  argc = ds_num + rra_num;

  if ((argv = malloc(sizeof(*argv) * (argc + 1))) == NULL) {
    P_ERROR("cu_rrd_create_file failed: %s", STRERRNO);
    rra_free(rra_num, rra_def);
    ds_free(ds_num, ds_def);
    return -1;
  }

  memcpy(argv, ds_def, ds_num * sizeof(char *));
  memcpy(argv + ds_num, rra_def, rra_num * sizeof(char *));
  argv[ds_num + rra_num] = NULL;

  last_up = CDTIME_T_TO_TIME_T(vl->time);
  if (last_up <= 0)
    last_up = time(NULL);
  last_up -= 1;

  if (cfg->stepsize > 0)
    stepsize = cfg->stepsize;
  else
    stepsize = (unsigned long)CDTIME_T_TO_TIME_T(vl->interval);

  if (cfg->async) {
    status = srrd_create_async(filename, stepsize, last_up, argc,
                               (const char **)argv);
    if (status != 0)
      P_WARNING("cu_rrd_create_file: srrd_create_async (%s) "
                "returned status %i.",
                filename, status);
  } else /* synchronous */
  {
    status = lock_file(filename);
    if (status != 0) {
      if (status == EEXIST)
        P_NOTICE("cu_rrd_create_file: File \"%s\" is already being created.",
                 filename);
      else
        P_ERROR("cu_rrd_create_file: Unable to lock file \"%s\".", filename);
    } else {
      status =
          srrd_create(filename, stepsize, last_up, argc, (const char **)argv);

      if (status != 0) {
        P_WARNING("cu_rrd_create_file: srrd_create (%s) returned status %i.",
                  filename, status);
      } else {
        DEBUG("cu_rrd_create_file: Successfully created RRD file \"%s\".",
              filename);
      }
      unlock_file(filename);
    }
  }

  free(argv);
  ds_free(ds_num, ds_def);
  rra_free(rra_num, rra_def);

  return status;
} /* }}} int cu_rrd_create_file */
Ejemplo n.º 16
0
static int ds_get(char ***ret, /* {{{ */
                  const data_set_t *ds, const value_list_t *vl,
                  const rrdcreate_config_t *cfg) {
  char **ds_def;
  size_t ds_num;

  char min[32];
  char max[32];
  char buffer[128];

  assert(ds->ds_num > 0);

  ds_def = calloc(ds->ds_num, sizeof(*ds_def));
  if (ds_def == NULL) {
    P_ERROR("ds_get: calloc failed: %s", STRERRNO);
    return -1;
  }

  for (ds_num = 0; ds_num < ds->ds_num; ds_num++) {
    data_source_t *d = ds->ds + ds_num;
    const char *type;
    int status;

    ds_def[ds_num] = NULL;

    if (d->type == DS_TYPE_COUNTER)
      type = "COUNTER";
    else if (d->type == DS_TYPE_GAUGE)
      type = "GAUGE";
    else if (d->type == DS_TYPE_DERIVE)
      type = "DERIVE";
    else if (d->type == DS_TYPE_ABSOLUTE)
      type = "ABSOLUTE";
    else {
      P_ERROR("ds_get: Unknown DS type: %i", d->type);
      break;
    }

    if (isnan(d->min)) {
      sstrncpy(min, "U", sizeof(min));
    } else
      snprintf(min, sizeof(min), "%f", d->min);

    if (isnan(d->max)) {
      sstrncpy(max, "U", sizeof(max));
    } else
      snprintf(max, sizeof(max), "%f", d->max);

    status = snprintf(
        buffer, sizeof(buffer), "DS:%s:%s:%i:%s:%s", d->name, type,
        (cfg->heartbeat > 0) ? cfg->heartbeat
                             : (int)CDTIME_T_TO_TIME_T(2 * vl->interval),
        min, max);
    if ((status < 1) || ((size_t)status >= sizeof(buffer)))
      break;

    ds_def[ds_num] = sstrdup(buffer);
  } /* for ds_num = 0 .. ds->ds_num */

  if (ds_num != ds->ds_num) {
    ds_free(ds_num, ds_def);
    return -1;
  }

  if (ds_num == 0) {
    sfree(ds_def);
    return 0;
  }

  *ret = ds_def;
  return ds_num;
} /* }}} int ds_get */
Ejemplo n.º 17
0
void gm2s_test() {
    int size = 128;
    delim_str* ds1 = NULL;
    char* str = "I wish I were a jellyfish that cannot fall down stairs.";;
    char* name = "jellyfish_string";
    
#if 0
    //delim_str testing stuff
    ds_print(ds1);
    ds_resize(&ds1,size);
    ds_print(ds1);
    memcpy(ds1->str,str,strlen(str));
    ds_print(ds1);
    ds_free(&ds1);
    printf("ds1 has been free'd\n");
    ds_print(ds1);
#endif

#if 0
    //named_data testing stuff (print)
    named_data* nd1 = NULL;
    nd_print(nd1);
    nd_resize_name(&nd1,128);
    nd_print(nd1);
    memcpy(nd1->name,str,strlen(str));
    nd_print(nd1);
    nd_resize_data(&nd1,128);
    nd_print(nd1);
    memcpy(nd1->data,name,strlen(name));
    nd_print(nd1);
    nd_free(&nd1);
    printf("nd1 is free'd\n");
    nd_print(nd1);
#endif

#if 0
    //test named data list stuff
    int count = 1024;
    named_data* nda[count];
    named_data* list = NULL;
    for (int i=0; i<count; i++) {
        nda[i] = NULL;
        nd_resize_data(&nda[i],strlen(str)+1);
        memcpy(nda[i]->data,str,strlen(str)); 
        nd_resize_name(&nda[i],strlen(name)+1);
        memcpy(nda[i]->name,name,strlen(name)); 
        nda[i]->order = (int)(((double)rand()/RAND_MAX)*100.0);
        nd_add_sorted(&list,&nda[i]);
        //nd_print_all(list);
    }

    nd_print_all(list);
    nd_free_all(&list);

#endif

#if 0
    //test subtitle stuff (printing, list op)
    int count = 110;
    subtitle* sa[count];
    subtitle* list = NULL;
    for (int i=0; i<count; i++) {
        sa[i] = NULL;
        sub_resize(&sa[i],128);
        memcpy(sa[i]->txt->str,str,strlen(str)); 
        sa[i]->txt_color = 0x11223344;
        sa[i]->bord_color = 0x66778899;
        sa[i]->start_frame = (int)(((double)rand()/RAND_MAX)*100.0);
        sa[i]->end_frame = 123;
        sa[i]->x = 40;
        sa[i]->y = 80;
        sub_add_sorted(&list,&sa[i]);
        sub_print_all(list);
    }

    sub_free_all(&list);
#endif

}
Ejemplo n.º 18
0
int dt_split_on_node(dt_node *node, data_set *train_data, int depth, split_criterion criterion) {
    if(!dt_should_split(train_data)) {
        // all y values are the same, so make a leaf!
        node->is_leaf = 1;
        node->prediction_value = train_data->y_data[0];
        return 1;
    }
    else if(train_data->rowcount < 1) {
        // this is generally a bad place to be
        // should never happen
        fprintf(stderr, "No rows left in training set!\n");
        return 1;
    }

    // pick the best column based in info gain
    unsigned int col = dt_pick_best_column(train_data, criterion);

    // split on the mean of the column
    node->split_value = ds_col_mean(train_data, col);
    node->split_col = col;

    // make a new data set for all of the rows less than the mean
    data_set *lesser_data = ds_new(train_data->colcount, 1);

    // add all rows < mean
    for(int i = 0; i < train_data->rowcount; i++) {
        float val = train_data->x_data[i][col];
        if(val < node->split_value) {
            ds_add_item(lesser_data, train_data->x_data[i], train_data->y_data[i]);
        }
    }

    int c1 = 0;
    if(lesser_data->rowcount > 0) {
        // if we have data that was less than the mean (should always happen)
        // then recurse on that new data set
        dt_node *left_node = dt_new_node();
        left_node->is_lesser = 1;
        node->left = left_node;
        c1 = dt_split_on_node(left_node, lesser_data, depth+1, criterion);
    }
    else {
        node->left = NULL;
    }
    ds_free(lesser_data);

    // make a data set for values >= mean
    data_set *greater_data = ds_new(train_data->colcount, 1);

    for(int i = 0; i < train_data->rowcount; i++) {
        float val = train_data->x_data[i][col];
        if(val >= node->split_value) {
            ds_add_item(greater_data, train_data->x_data[i], train_data->y_data[i]);
        }
    }

    int c2 = 0;
    if(greater_data->rowcount > 0) {
        // recurse on the new data set
        dt_node *right_node = dt_new_node();
        node->right = right_node;
        right_node->is_lesser = 0;
        c2 = dt_split_on_node(right_node, greater_data, depth+1, criterion);
    }
    else {
        node->right = NULL;
    }
    ds_free(greater_data);

    // return a count of all of the decendent nodes for the current node
    return c1+c2;
}
Ejemplo n.º 19
0
static int ds_get (char ***ret, /* {{{ */
                   const data_set_t *ds, const value_list_t *vl,
                   const rrdcreate_config_t *cfg)
{
    char **ds_def;
    int ds_num;

    char min[32];
    char max[32];
    char buffer[128];

    ds_def = (char **) malloc (ds->ds_num * sizeof (char *));
    if (ds_def == NULL)
    {
        char errbuf[1024];
        ERROR ("rrdtool plugin: malloc failed: %s",
               sstrerror (errno, errbuf, sizeof (errbuf)));
        return (-1);
    }
    memset (ds_def, '\0', ds->ds_num * sizeof (char *));

    for (ds_num = 0; ds_num < ds->ds_num; ds_num++)
    {
        data_source_t *d = ds->ds + ds_num;
        char *type;
        int status;

        ds_def[ds_num] = NULL;

        if (d->type == DS_TYPE_COUNTER)
            type = "COUNTER";
        else if (d->type == DS_TYPE_GAUGE)
            type = "GAUGE";
        else if (d->type == DS_TYPE_DERIVE)
            type = "DERIVE";
        else if (d->type == DS_TYPE_ABSOLUTE)
            type = "ABSOLUTE";
        else
        {
            ERROR ("rrdtool plugin: Unknown DS type: %i",
                   d->type);
            break;
        }

        if (isnan (d->min))
        {
            sstrncpy (min, "U", sizeof (min));
        }
        else
            ssnprintf (min, sizeof (min), "%f", d->min);

        if (isnan (d->max))
        {
            sstrncpy (max, "U", sizeof (max));
        }
        else
            ssnprintf (max, sizeof (max), "%f", d->max);

        status = ssnprintf (buffer, sizeof (buffer),
                            "DS:%s:%s:%i:%s:%s",
                            d->name, type,
                            (cfg->heartbeat > 0) ? cfg->heartbeat : (2 * vl->interval),
                            min, max);
        if ((status < 1) || ((size_t) status >= sizeof (buffer)))
            break;

        ds_def[ds_num] = sstrdup (buffer);
    } /* for ds_num = 0 .. ds->ds_num */

    if (ds_num != ds->ds_num)
    {
        ds_free (ds_num, ds_def);
        return (-1);
    }

    *ret = ds_def;
    return (ds_num);
} /* }}} int ds_get */
Ejemplo n.º 20
0
/**
 * Executes a product-maker.
 *
 * This function is thread-compatible but not thread-safe.
 *
 * @retval (void*)0    The FIFO was closed.
 * @retval (void*)1    Usage failure. \c log_start() called.
 * @retval (void*)2    O/S failure. \c log_start() called.
 * @retval (void*)-1   Retransmission failure. \c log_start() called.
 */
void* pmStart(
    void* const         arg)          /**< [in/out] Pointer to the
                                        *  product-maker to be executed */
{
    ProductMaker* const productMaker = (ProductMaker*)arg;
    int                 status;
    Fifo* const         fifo = productMaker->fifo;
    unsigned char*      buf = productMaker->buf;
    sbn_struct*         sbn = &productMaker->sbn;
    pdh_struct*         pdh = &productMaker->pdh;
    psh_struct*         psh = &productMaker->psh;
    pdb_struct*         pdb = &productMaker->pdb;
    ccb_struct*         ccb = &productMaker->ccb;
    unsigned long       last_sbn_seqno;
    unsigned long       last_sbn_runno = ULONG_MAX;
    int                 PNGINIT = 0;
    char*               memheap = NULL;
    MD5_CTX*            md5ctxp = productMaker->md5ctxp;
    int                 logResync = 1;
    prodstore           prod;

#ifdef RETRANS_SUPPORT
    long cpio_addr_tmp;
    int cpio_fd_tmp;
 /*   char transfer_type[10]={0};*/
    int retrans_val,idx;
    long retrans_tbl_size;
    time_t orig_arrive_time;
    int new_key; /* shm key */
    ACQ_TABLE *acq_tbl = NULL;
    long num_prod_discards=0;
    int save_prod = 1;
    int discard_prod = 0;
    long proc_orig_prod_seqno_last_save=0;
#endif


    prod.head = NULL;
    prod.tail = NULL;

	/*** For Retranmission Purpose  ***/
#ifdef RETRANS_SUPPORT
	if (ulogIsDebug())
         udebug(" retrans_xmit_enable = %d   transfer_type = %s sbn_channel_name=%s \n",retrans_xmit_enable,transfer_type,sbn_channel_name);

       if((retrans_xmit_enable == OPTION_ENABLE) && (!strcmp(transfer_type,"MHS") || !strcmp(transfer_type,"mhs"))){
                idx = get_cpio_addr(mcastAddr);
                if( idx >= 0 && idx < NUM_CPIO_ENTRIES){
                    global_cpio_fd = cpio_tbl[idx].cpio_fd;
                    global_cpio_addr = cpio_tbl[idx].cpio_addr;
	            if (ulogIsDebug())
 	              udebug("Global cpio_addr  = 0x%x Global cpio_fd = %d \n",global_cpio_addr,global_cpio_fd);
                }else{
                    uerror("Invalid multicast address provided");
	  	    return (void*)-1;
                 }

		 retrans_tbl_size = sizeof(PROD_RETRANS_TABLE);

		/** Modified to setup retrans table on per channel basis - Sathya - 14-Mar'2013 **/

		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * GET_RETRANS_CHANNEL_ENTRIES(sbn_type));

                /****   
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC1);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC2);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NMC3);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_GOES_EAST);
		 retrans_tbl_size += (sizeof(PROD_RETRANS_ENTRY) * DEFAULT_RETRANS_ENTRIES_NOAAPORT_OPT);
		****/


		p_prod_retrans_table = (PROD_RETRANS_TABLE *) malloc (retrans_tbl_size);
		if(p_prod_retrans_table == NULL){
		   uerror("Unable to allocate memory for retrans table..Quitting.\n");
	  	   return (void*)-1;
		}

		if( init_retrans(&p_prod_retrans_table) < 0 ){
		  uerror("Error in initializing retrans table \n");
		  if(p_prod_retrans_table)
			free(p_prod_retrans_table);
	  	  return (void*)-1;
		}	

       GET_SHMPTR(global_acq_tbl,ACQ_TABLE,ACQ_TABLE_SHMKEY,DEBUGGETSHM);

	if (ulogIsDebug())
	  udebug("Global acquisition table = 0x%x cpio_fd = %d \n",global_acq_tbl,global_cpio_fd);
	acq_tbl = &global_acq_tbl[global_cpio_fd];

	if (ulogIsDebug())
	    udebug("Obtained acquisition table = 0x%x \n",acq_tbl);

	/* ACQ_TABLE already initialized in acq_ldm_getshm */
	/*
	if(init_acq_table(acq_tbl) < 0){
		uerror("Unable to initialize acq table\n");
		exit(-1);
	}
	*/
	
	 buff_hdr = (BUFF_HDR *) malloc(sizeof(BUFF_HDR));
	 if(init_buff_hdr(buff_hdr) < 0){
		uerror("Unalbe to initialize buffer header \n");
		if(acq_tbl)
			free(acq_tbl);
		if(p_prod_retrans_table)
		  free(p_prod_retrans_table);
	  	return (void*)-1;
	 }

	 acq_tbl->pid = getpid();
	 acq_tbl->link_id = global_cpio_fd;
	 acq_tbl->link_addr = global_cpio_addr;
	 if(ulogIsVerbose()){
 	  uinfo("Initialized acq_tbl  = 0x%x & buff_hdr = 0x%x pid = %d \n",acq_tbl,buff_hdr,acq_tbl->pid);
	  uinfo("Global link id = %d  Global link addr = %ld \n",acq_tbl->link_id,acq_tbl->link_addr);
	  uinfo("acq_tbl->read_distrib_enable = 0x%x \n",acq_tbl->read_distrib_enable);
	 }
    }
	   /*** For Retranmission Purpose  ***/
#endif

    for (;;) {
        unsigned char       b1;
        long                IOFF;
        int                 NWSTG;
        int                 GOES;
        int                 PROD_COMPRESSED;
        size_t              heapcount;
        size_t              heapsize;
        char                PROD_NAME[1024];
        int                 dataoff;
        int                 datalen;
        datastore*          pfrag;
        int                 nscan;
        int                 deflen;
        static const char*  FOS_TRAILER = "\015\015\012\003";
        int                 cnt;

        /* Look for first byte == 255  and a valid SBN checksum */
        if ((status = fifoRead(fifo, buf, 1)) != 0) {
            if (3 == status)
                status = 0;
            break;
        }
        if ((b1 = (unsigned char)buf[0]) != 255) {
            if (logResync) {
                uinfo("Trying to resync %u", b1);
                logResync = 0;
            }
            continue;
        }
        logResync = 1;

        if (fifoRead(fifo, buf + 1, 15) != 0) {
            if (ulogIsDebug())
                udebug("couldn't read 16 bytes for sbn");
            continue;
        }

        while ((status = readsbn(buf, sbn)) != 0) {
            if (ulogIsDebug())
                udebug("Not SBN start");

            IOFF = 1;

            while ((IOFF < 16) && 
                    ((b1 = (unsigned char) buf[IOFF]) != 255))
                IOFF++;

            if (IOFF > 15) {
                break;
            }
            else {
                int ch;

                for (ch = IOFF; ch < 16; ch++)
                    buf[ch - IOFF] = buf[ch];

                if (fifoRead(fifo, buf + 16 - IOFF, IOFF)
                        != 0) {
                    if (ulogIsDebug())
                        udebug("Couldn't read bytes for SBN, resync");
                    break;
                }
            }
        }

        if (status != 0) {
            if (ulogIsDebug())
                udebug("SBN status continue");
            continue;
        }

        IOFF = 0;

        if (fifoRead(fifo, buf + 16, 16) != 0) {
            if (ulogIsDebug())
                udebug("error reading Product Definition Header");
            continue;
        }

#ifdef RETRANS_SUPPORT
		/* Update acq table stats - Begin */
		if(retrans_xmit_enable == OPTION_ENABLE){
			buff_hdr->read_channel_type = sbn->datastream;
		}
		/* Update acq table stats - End */
#endif

        if (ulogIsDebug())
            udebug("***********************************************");
        if (last_sbn_runno != sbn->runno) {
            last_sbn_runno = sbn->runno;
        }
        else {
            unsigned long   delta = sbn->seqno - last_sbn_seqno;
#           define          MAX_SEQNO 0xFFFFFFFFu

            if (0 == delta || MAX_SEQNO/2 < delta) {
                uwarn("Retrograde packet number: previous=%lu, latest=%lu, "
                        "difference=%lu", last_sbn_seqno, sbn->seqno, 
                        0 == delta ? 0ul : MAX_SEQNO - delta + 1);
            }
            else {
                if (1 != delta) {
                    unsigned long   gap = delta - 1;

                    uwarn("Gap in packet sequence: %lu to %lu [skipped %lu]",
                             last_sbn_seqno, sbn->seqno, gap);

                    (void)pthread_mutex_lock(&productMaker->mutex);
                    productMaker->nmissed += gap;
                    (void)pthread_mutex_unlock(&productMaker->mutex);
                }

                (void)pthread_mutex_lock(&productMaker->mutex);
                productMaker->npackets++;
                (void)pthread_mutex_unlock(&productMaker->mutex);
            }                           /* non-retrograde packet number */
        }                               /* "last_sbn_seqno" initialized */
        last_sbn_seqno = sbn->seqno;

        if (ulogIsVerbose())
            uinfo("SBN seqnumber %ld", sbn->seqno);
        if (ulogIsVerbose())
            uinfo("SBN datastream %d command %d", sbn->datastream,
                sbn->command);
        if (ulogIsDebug())
            udebug("SBN version %d length offset %d", sbn->version, sbn->len);
        if (((sbn->command != 3) && (sbn->command != 5)) || 
                (sbn->version != 1)) {
            uerror("Unknown sbn command/version %d PUNT", sbn->command);
            continue;
        }

        switch (sbn->datastream) {
        case 5:       /* nwstg */
        case 6:       /* nwtg2 */
        case 7:       /* polarsat */
        case 8:       /* NOAA Weather Wire Service (NWWS) */
        case 9:
        case 10:
        case 11:
        case 12:      /* GOES-R */
        case 13:      /* GOES-R */
            NWSTG = 1;
            GOES = 0;
            break;
        case 1:       /* GINI GOES */
        case 2:       /* GINI GOES */
        case 4:       /* OCONUS */
            NWSTG = 0;
            GOES = 1;
            break;
        default:
            uerror("Unknown NOAAport channel %d PUNT", sbn->datastream);
            continue;
        }

        /* End of SBN version low 4 bits */

        if (readpdh(buf + IOFF + sbn->len, pdh) == -1) {
            uerror("problem with pdh, PUNT");
            continue;
        }
        if (pdh->len > 16) {
            if (fifoRead(fifo, buf + sbn->len + 16,
                        pdh->len - 16) != 0)
                continue;
        }

        if (ulogIsDebug())
            udebug("Product definition header version %d pdhlen %d",
                pdh->version, pdh->len);

        if (pdh->version != 1) {
            uerror("Error: PDH transfer type %u, PUNT", pdh->transtype);
            continue;
        }
        else if (ulogIsDebug()) {
            udebug("PDH transfer type %u", pdh->transtype);
        }

        if ((pdh->transtype & 8) > 0)
            uerror("Product transfer flag error %u", pdh->transtype);
        if ((pdh->transtype & 32) > 0)
            uerror("Product transfer flag error %u", pdh->transtype);

        if ((pdh->transtype & 16) > 0) {
            PROD_COMPRESSED = 1;

            if (ulogIsDebug())
                udebug("Product transfer flag compressed %u", pdh->transtype);
        }
        else {
            PROD_COMPRESSED = 0;
        }

        if (ulogIsDebug())
            udebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen,
                pdh->pshlen);
        if (ulogIsDebug())
            udebug("blocks per record %ld records per block %ld\n",
                pdh->blocks_per_record, pdh->records_per_block);
        if (ulogIsDebug())
            udebug("product seqnumber %ld block number %ld data block size "
                "%ld", pdh->seqno, pdh->dbno, pdh->dbsize);

        /* Stop here if no psh */
        if ((pdh->pshlen == 0) && (pdh->transtype == 0)) {
            IOFF = IOFF + sbn->len + pdh->len;
            continue;
        }

#ifdef RETRANS_SUPPORT
		/** Update acquisition table statistics  **/
		if(retrans_xmit_enable == OPTION_ENABLE){
				acq_tbl->read_tot_buff_read++;
		}
#endif
        if (pdh->pshlen != 0) {
            if (fifoRead(fifo, buf + sbn->len + pdh->len,
                        pdh->pshlen) != 0) {
                uerror("problem reading psh");
                continue;
            }
            else {
                if (ulogIsDebug())
                    udebug("read psh %d", pdh->pshlen);
            }

            /* Timing block */
            if (sbn->command == 5) {
                if (ulogIsDebug())
                    udebug("Timing block recieved %ld %ld\0", psh->olen,
                        pdh->len);
                /*
                 * Don't step on our psh of a product struct of prod in
                 * progress.
                 */
                continue;
            }

            if (readpsh(buf + IOFF + sbn->len + pdh->len, psh) == -1) {
                uerror("problem with readpsh");
                continue;
            }
            if (psh->olen != pdh->pshlen) {
                uerror("ERROR in calculation of psh len %ld %ld", psh->olen,
                    pdh->len);
                continue;
            }
            if (ulogIsDebug())
                udebug("len %ld", psh->olen);
            if (ulogIsDebug())
                udebug("product header flag %d, version %d", psh->hflag,
                    psh->version);
            if (ulogIsDebug())
                udebug("prodspecific data length %ld", psh->psdl);
            if (ulogIsDebug())
                udebug("bytes per record %ld", psh->bytes_per_record);
            if (ulogIsDebug())
                udebug("Fragments = %ld category %d ptype %d code %d",
                    psh->frags, psh->pcat, psh->ptype, psh->pcode);
            if (psh->frags < 0)
                uerror("check psh->frags %d", psh->frags);
            if (psh->origrunid != 0)
                uerror("original runid %d", psh->origrunid);
            if (ulogIsDebug())
                udebug("next header offset %ld", psh->nhoff);
            if (ulogIsDebug())
                udebug("original seq number %ld", psh->seqno);
            if (ulogIsDebug())
                udebug("receive time %ld", psh->rectime);
            if (ulogIsDebug())
                udebug("transmit time %ld", psh->transtime);
            if (ulogIsDebug())
                udebug("run ID %ld", psh->runid);
            if (ulogIsDebug())
                udebug("original run id %ld", psh->origrunid);

#ifdef RETRANS_SUPPORT
				/* Update acq table stats - Begin */
			if(retrans_xmit_enable == OPTION_ENABLE){
			
				buff_hdr->buff_data_length = pdh->dbsize;
				if(pdh->dbno == 0) {
						/* Assume first block */
					acq_tbl->proc_base_prod_type_last = psh->ptype;
					acq_tbl->proc_base_prod_cat_last = psh->pcat;
					acq_tbl->proc_base_prod_code_last = psh->pcode;
					acq_tbl->proc_prod_NCF_rcv_time = (time_t)psh->rectime;
					acq_tbl->proc_prod_NCF_xmit_time = (time_t)psh->transtime;
					if(psh->hflag & XFR_PROD_RETRANSMIT){
					   acq_tbl->proc_orig_prod_seqno_last = psh->seqno;
					   acq_tbl->proc_orig_prod_run_id = psh->origrunid;
					   if(ulogIsDebug())
					     udebug("ORIG SEQ# = %ld	CURR SEQ#: %ld \n",acq_tbl->proc_orig_prod_seqno_last,pdh->seqno);
						}else{
						   acq_tbl->proc_orig_prod_seqno_last = 0;
						   acq_tbl->proc_orig_prod_run_id = 0;
						}
					acq_tbl->proc_prod_run_id = psh->runid;
					buff_hdr->buff_datahdr_length = psh->psdl;
					time(&acq_tbl->proc_prod_start_time);
					acq_tbl->proc_tot_prods_handled++;
				}else{
						buff_hdr->buff_datahdr_length = 0;
				 }
				buff_hdr->proc_prod_seqno= pdh->seqno;
				buff_hdr->proc_blkno = pdh->dbno;
				buff_hdr->proc_sub_code = 0;
				buff_hdr->proc_prod_flag = pdh->transtype;
					
				acq_tbl->proc_base_channel_type_last = buff_hdr->read_channel_type;
				buff_hdr->proc_prod_type = acq_tbl->proc_base_prod_type_last;
				buff_hdr->proc_prod_code = acq_tbl->proc_base_prod_code_last;
				buff_hdr->proc_prod_cat = acq_tbl->proc_base_prod_cat_last;
					
				acq_tbl->proc_prod_bytes_read = buff_hdr->buff_data_length;
						
				/* Check prod_seqno for lost products */
				if((buff_hdr->proc_prod_seqno - acq_tbl->proc_base_prod_seqno_last) != 1){
					do_prod_lost(buff_hdr,acq_tbl);
				}
				retrans_val = prod_retrans_ck(acq_tbl, buff_hdr, &orig_arrive_time);
				log_buff[0] = '\0'; 		
				if((retrans_val == PROD_DUPLICATE_DISCARD) ||
					((retrans_val == PROD_DUPLICATE_MATCH) &&
					(acq_tbl->proc_retransmit_ctl_flag & ENABLE_RETRANS_DUP_MATCH_DISCARD)) ||
					((retrans_val == PROD_DUPLICATE_NOMATCH) &&
					(acq_tbl->proc_retransmit_ctl_flag & ENABLE_RETRANS_DUP_NOMATCH_DISCARD))){
						/* Log product details and discard the product */
						strcpy(log_buff,"DISCARD");
						if(acq_tbl->proc_orig_prod_seqno_last != 0){
							strcat(log_buff, "/RETRANS");
						}
								
						log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last,
									buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno,
									buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,orig_arrive_time);
						save_prod = 0;
						acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
						/* Current prod discarded and continue with next */
						/*continue; */
					}else{
						if(retrans_val == PROD_DUPLICATE_NOMATCH){
							strcpy(log_buff,"SAVE RETRANS");
							log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last,
							buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno,
							buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,acq_tbl->proc_prod_start_time);
					  }
				   }
			  }
#endif

            if (prod.head != NULL) {
                uerror("OOPS, start of new product [%ld ] with unfinished "
                    "product %ld", pdh->seqno, prod.seqno);

#ifdef RETRANS_SUPPORT
				/* Request retrans when prod is partially received but before completion */
				/* if there is frame error and continue with different prod then, we need */
				/* to abort the old prod and clear retrans table. */
				if((retrans_xmit_enable == OPTION_ENABLE) /*&& (pdh->dbno != 0)*/){
				 acq_tbl->proc_acqtab_prodseq_errs++;
				 if(proc_orig_prod_seqno_last_save != acq_tbl->proc_orig_prod_seqno_last){
				 /* Clear retrans table for the orig prod if the previous prod is retrans */
				 /* of original prod  */
				   prod_retrans_abort_entry(acq_tbl, proc_orig_prod_seqno_last_save, RETRANS_RQST_CAUSE_RCV_ERR);
				 }
				 prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
				 /* Update Statistics */
				 acq_tbl->proc_tot_prods_lost_errs++;
				  /* For now, generate retrans request only for non-imagery products */
				 if(!((buff_hdr->proc_prod_cat == PROD_CAT_IMAGE) && 
					   (PROD_TYPE_NESDIS_HDR_TRUE(buff_hdr->proc_prod_type)))){
				       generate_retrans_rqst(acq_tbl,prod.seqno , prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
				 }
				   acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
				}
#endif
                ds_free();

                prod.head = NULL;
                prod.tail = NULL;

                if (PNGINIT != 0) {
                    pngout_end();
                    PNGINIT = 0;
                }

                uerror("Product definition header version %d pdhlen %d",
                        pdh->version, pdh->len);
                uerror("PDH transfer type %u", pdh->transtype);

                if ((pdh->transtype & 8) > 0)
                    uerror("Product transfer flag error %u", pdh->transtype);
                if ((pdh->transtype & 32) > 0)
                    uerror("Product transfer flag error %u", pdh->transtype);

                uerror("header length %ld [pshlen = %d]",
                    pdh->len + pdh->pshlen, pdh->pshlen);
                uerror("blocks per record %ld records per block %ld",
                    pdh->blocks_per_record, pdh->records_per_block);
                uerror("product seqnumber %ld block number %ld data block "
                    "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize);
                uerror("product header flag %d", psh->hflag);
                uerror("prodspecific data length %ld", psh->psdl);
                uerror("bytes per record %ld", psh->bytes_per_record);
                uerror("Fragments = %ld category %d", psh->frags, psh->pcat);

                if (psh->frags < 0)
                    uerror("check psh->frags %d", psh->frags);
                if (psh->origrunid != 0)
                    uerror("original runid %d", psh->origrunid);

                uerror("next header offset %ld", psh->nhoff);
                uerror("original seq number %ld", psh->seqno);
                uerror("receive time %ld", psh->rectime);
                uerror("transmit time %ld", psh->transtime);
                uerror("run ID %ld", psh->runid);
                uerror("original run id %ld", psh->origrunid);
            }

            prod.seqno = pdh->seqno;
            prod.nfrag = psh->frags;

            ds_init(prod.nfrag);

            /* NWSTG CCB = dataoff, WMO = dataoff + 24 */

            if (fifoRead(fifo, buf + sbn->len + pdh->len + 
                        pdh->pshlen, pdh->dbsize) != 0) {
                uerror("problem reading datablock");
                continue;
            }
            if (sbn->datastream == 4) {
                if (psh->pcat != 3) {
                    GOES = 0;
                    NWSTG = 1;
                }
            }

            heapcount = 0;

            MD5Init(md5ctxp);

            if (GOES == 1) {
                if (readpdb(buf + IOFF + sbn->len + pdh->len + 
                            pdh->pshlen,
                        psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) {
                    uerror("Error reading pdb, punt");
                    continue;
                }

                (void)memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                if (ulogIsDebug())
                    udebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len,
                        pdh->pshlen, sbn->len + pdh->len + pdh->pshlen,
                        pdb->len);

                /* Data starts at first block after pdb */
                ccb->len = 0;
                heapsize = prodalloc(psh->frags, 5152, &memheap);
            }
            if (NWSTG == 1) {
                memset(psh->pname, 0, sizeof(psh->pname));

                if (readccb(buf + IOFF + sbn->len + pdh->len + 
                            pdh->pshlen,
                        ccb, psh, pdh->dbsize) == -1)
                    uerror("Error reading ccb, using default name");
                if (ulogIsDebug())
                    udebug("look at ccb start %d %d", ccb->b1, ccb->len);

                if (ulogIsVerbose())
                    uinfo("%s", psh->pname);

                memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                heapsize = prodalloc(psh->frags, 4000 + 15, &memheap);
                /*
                 * We will only compute md5 checksum on the data, 11 FOS
                 * characters at start
                 */
                sprintf(memheap, "\001\015\015\012%03d\040\015\015\012",
                    ((int) pdh->seqno) % 1000);

                heapcount += 11;

                if (psh->metaoff > 0)
                    psh->metaoff = psh->metaoff + 11;
            }
        }
        else {
            /* If a continuation record...don't let psh->pcat get missed */
            if ((sbn->datastream == 4) && (psh->pcat != 3)) {
                GOES = 0;
                NWSTG = 1;
            }

            ccb->len = 0;

            if (ulogIsDebug())
                udebug("continuation record");

#ifdef RETRANS_SUPPORT
			if(retrans_xmit_enable == OPTION_ENABLE){
					 buff_hdr->buff_data_length = pdh->dbsize;
					 buff_hdr->buff_datahdr_length = 0;
					 buff_hdr->proc_prod_seqno= pdh->seqno;
					 buff_hdr->proc_blkno = pdh->dbno;
					 buff_hdr->proc_sub_code = 0;
					 buff_hdr->proc_prod_flag = pdh->transtype;
					 
					 acq_tbl->proc_base_channel_type_last = buff_hdr->read_channel_type;
					 buff_hdr->proc_prod_type = acq_tbl->proc_base_prod_type_last;
					 buff_hdr->proc_prod_code = acq_tbl->proc_base_prod_code_last;
					 buff_hdr->proc_prod_cat = acq_tbl->proc_base_prod_cat_last;
				 
					 acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;
					 
			  }
#endif

            if ((pdh->transtype & 4) > 0) {
                psh->frags = 0;
            }
            if (fifoRead(fifo, buf + sbn->len + pdh->len + 
                        pdh->pshlen, pdh->dbsize) != 0) {
                uerror("problem reading datablock (cont)");
                continue;
            }
            if (prod.head == NULL) {
                if (ulogIsVerbose())
                    uinfo("found data block before header, "
                        "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno);
                continue;
            }
        }

        /* Get the data */
        dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len;
        datalen = pdh->dbsize - ccb->len;

        if (ulogIsDebug())
            udebug("look at datalen %d", datalen);

        pfrag = ds_alloc();
        pfrag->seqno = pdh->seqno;
        pfrag->fragnum = pdh->dbno;
        pfrag->recsiz = datalen;
        pfrag->offset = heapcount;
        pfrag->next = NULL;

        if (GOES == 1) {
            if (pfrag->fragnum > 0) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) || 
                        (pfrag->seqno != prod.seqno)) {
                    uerror("Missing GOES fragment in sequence, "
                        "last %d/%d this %d/%d\0", prod.tail->fragnum,
                        prod.seqno, pfrag->fragnum, pfrag->seqno);

#ifdef RETRANS_SUPPORT
					if(retrans_xmit_enable == OPTION_ENABLE){
					  acq_tbl->proc_acqtab_prodseq_errs++;
					  do_prod_mismatch(acq_tbl,buff_hdr);
					  acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
					}
#endif

					ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }

                if ((PNGINIT != 1) && (!PROD_COMPRESSED)) {
                    uerror("failed pnginit %d %d %s", sbn->datastream,
                            psh->pcat, PROD_NAME);
                    continue;
                }
                if (pdh->records_per_block < 1) {
                    uerror("records_per_block %d blocks_per_record %d "
                        "nx %d ny %d", pdh->records_per_block,
                        pdh->blocks_per_record, pdb->nx, pdb->ny);
                    uerror("source %d sector %d channel %d", pdb->source,
                        pdb->sector, pdb->channel);
                    uerror("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                        "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year,
                        pdb->month, pdb->day, pdb->hour, pdb->minute,
                        pdb->second, pdb->sechunds);
                    uerror("pshname %s", psh->pname);
                }
                if (!PROD_COMPRESSED) {
                    for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) {
                        if (ulogIsDebug())
                            udebug("png write nscan %d", nscan);
                        if (nscan >= pdh->records_per_block) {
                            uerror("nscan exceeding records per block %d [%d "
                                "%d %d]", pdh->records_per_block, nscan,
                                pdb->nx, pdh->dbsize);
                        }
                        else {
                          pngwrite(buf + dataoff + (nscan * pdb->nx));
                        }
                    }
                }
                else {
                    memcpy(memheap + heapcount, buf + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
            }
            else {
                if (!PROD_COMPRESSED) {
                    png_set_memheap(memheap, md5ctxp);
                    png_header(buf + dataoff, datalen);
                    /*
                     * Add 1 to number of scanlines, image ends with 
                     * f0f0f0f0...
                     */
                    pngout_init(pdb->nx, pdb->ny + 1);

                    PNGINIT = 1;
                }
                else {
                    memcpy(memheap + heapcount, buf + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
                unotice("records_per_block %d blocks_per_record %d nx %d ny %d",
                    pdh->records_per_block, pdh->blocks_per_record, pdb->nx,
                    pdb->ny);
                unotice("source %d sector %d channel %d", pdb->source,
                    pdb->sector, pdb->channel);
                unotice("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                    "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month,
                    pdb->day, pdb->hour, pdb->minute, pdb->second,
                    pdb->sechunds);
                unotice("pshname %s", psh->pname);
            }
            deflen = 0;
#ifdef RETRANS_SUPPORT
			if(retrans_xmit_enable == OPTION_ENABLE){
			   if(buff_hdr->proc_blkno != 0){
		          /*acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;*/
				  acq_tbl->proc_prod_bytes_read += datalen;
	           }
            }
#endif
       	}
        else {
            /* If the product already has a FOS trailer, don't add
             * another....this will match what pqing(SDI) sees
             */
            if ((prod.nfrag != 0) && (prod.tail != NULL)) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) ||
                        (pfrag->seqno != prod.seqno)) {
                    uerror("Missing fragment in sequence, last %d/%d this "
                        "%d/%d\0", prod.tail->fragnum, prod.seqno,
                        pfrag->fragnum, pfrag->seqno);

#ifdef RETRANS_SUPPORT
                                      if(retrans_xmit_enable == OPTION_ENABLE){
                                         acq_tbl->proc_acqtab_prodseq_errs++;
                                         if(ulogIsDebug())
                                            udebug("do_prod_mismatch() proc_base_prod_seqno_last = %d \n",
					                                    acq_tbl->proc_base_prod_seqno_last);
                                            do_prod_mismatch(acq_tbl,buff_hdr);
                                            acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
                                      }
#endif

                    ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }
            }
            if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
                char testme[4];

                while (datalen > 4) {
                    memcpy(testme, buf + (dataoff + datalen - 4), 4);

                    if (memcmp(testme, FOS_TRAILER, 4) == 0) {
                        datalen -= 4;

                        if (ulogIsDebug())
                            udebug("removing FOS trailer from %s", PROD_NAME);
                    }
                    else {
                        break;
                    }
                }
            }
            if (heapcount + datalen > heapsize) {
                /*
                 * this above wasn't big enough heapsize =
                 * prodalloc(psh->frags,4000+15,&memheap);
                 */
                uerror("Error in heapsize %d product size %d [%d %d], Punt!\0",
                    heapsize, (heapcount + datalen), heapcount, datalen);
				
#ifdef RETRANS_SUPPORT
				if(retrans_xmit_enable == OPTION_ENABLE){ 	
			      /* Update Statistics */
					acq_tbl->proc_tot_prods_lost_errs++;
					/*  Abort entry and request retransmission */
					prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
					generate_retrans_rqst(acq_tbl, prod.seqno, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
					if(acq_tbl->proc_orig_prod_seqno_last != 0){
						strcpy(log_buff, "RETRANS");
					}
					log_prod_end(log_buff, acq_tbl->proc_orig_prod_seqno_last,
								 buff_hdr->proc_prod_seqno,buff_hdr->proc_blkno,
								 buff_hdr->proc_prod_code, acq_tbl->proc_prod_bytes_read,
								 acq_tbl->proc_prod_start_time);

		      		acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
	      		}
#endif

                continue;
            }

            memcpy(memheap + heapcount, buf + dataoff, datalen);

            deflen = datalen;

            MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                deflen);

#ifdef RETRANS_SUPPORT
			if(retrans_xmit_enable == OPTION_ENABLE){
	  		  if(buff_hdr->proc_blkno != 0){
				/*acq_tbl->proc_prod_bytes_read += buff_hdr->buff_data_length;*/
				acq_tbl->proc_prod_bytes_read += datalen;
			  }
	  		}
#endif
        }

        pfrag->recsiz = deflen;
        heapcount += deflen;

        if (prod.head == NULL) {
            prod.head = pfrag;
            prod.tail = pfrag;
        }
        else {
            prod.tail->next = pfrag;
            prod.tail = pfrag;
        }
 
#ifdef RETRANS_SUPPORT
		 if(((prod.nfrag == 0) || (prod.nfrag >= (pfrag->fragnum +1))) && (save_prod == 0)){
		   if(ulogIsVerbose())
			  uinfo("Do not save prod [seqno=%ld] as its retrans dup fragnum/total fragments =[%d of %d] save_prod=[%d] \n",
			   prod.seqno,pfrag->fragnum,prod.nfrag,save_prod);
		   ds_free ();
		   prod.head = NULL;
		   prod.tail = NULL;
		   PNGINIT = 0;
		}else{
#endif
          if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
            if (GOES == 1) {
                if (PNGINIT == 1) {
                    pngout_end();
                    heapcount = png_get_prodlen();
                }
                else {
                    if (ulogIsDebug())
                        udebug("GOES product already compressed %d", heapcount);
                }
            }
            if (ulogIsVerbose())
              uinfo("we should have a complete product %ld %ld/%ld %ld /heap "
                  "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum,
                 (long) heapcount);
            if ((NWSTG == 1) && (heapcount > 4)) {
                cnt = 4;                /* number of bytes to add for TRAILER */

                /*
                 * Do a DDPLUS vs HDS check for NWSTG channel only
                 */
                if (sbn->datastream == 5) {
                    /* nwstg channel */
                    switch (psh->pcat) {
                    case 1:
                    case 7:
                      /* Do a quick check for non-ascii text products */
                      if (!prod_isascii(PROD_NAME, memheap, heapcount))
                        psh->pcat += 100;       /* call these HDS */
                      break;
                    }
                }

                if (cnt > 0) {
                    memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        cnt);
                    heapcount += cnt;
                }
            }

#ifdef RETRANS_SUPPORT
			if((retrans_xmit_enable == OPTION_ENABLE) && (acq_tbl->read_distrib_enable & READ_CTL_DISCARD)){
						num_prod_discards++;
						/* Set discard_prod to 1; Otherwise already stored prod may be requested for retransmit */
						discard_prod=1;
						if(ulogIsVerbose())
						  uinfo("No of products discarded = %ld prod.seqno=%ld \n ",num_prod_discards,prod.seqno);
						prod_retrans_abort_entry(acq_tbl, prod.seqno, RETRANS_RQST_CAUSE_RCV_ERR);
						acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno -1 ;
						ds_free ();
						prod.head = NULL;
						prod.tail = NULL;
						PNGINIT = 0;
			}else{
				/* Do not insert prod into queue if its a duplicate product */
				if(save_prod != 0)
#endif
                                   process_prod(prod, PROD_NAME, memheap, heapcount,
                                                md5ctxp, productMaker->ldmProdQueue, psh, sbn);
#ifdef RETRANS_SUPPORT
				/* Update acq table with last processed seqno -Begin */
				if(retrans_xmit_enable == OPTION_ENABLE){
					acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
					uinfo(" prod with seqno processed = %ld\n",acq_tbl->proc_base_prod_seqno_last);
				}
				/* Update acq table with last processed seqno -End */
#endif

            ds_free();

            prod.head = NULL;
            prod.tail = NULL;
            PNGINIT = 0;

            (void)pthread_mutex_lock(&productMaker->mutex);
            productMaker->nprods++;
            (void)pthread_mutex_unlock(&productMaker->mutex);
#ifdef RETRANS_SUPPORT
        }
#endif
       }
        else {
            if (ulogIsDebug())
                udebug("processing record %ld [%ld %ld]", prod.seqno,
                    prod.nfrag, pfrag->fragnum);
            if ((pdh->transtype & 4) > 0) {
                uerror("Hmmm....should call completed product %ld [%ld %ld]",
                    prod.seqno, prod.nfrag, pfrag->fragnum);
            }
        }

#ifdef RETRANS_SUPPORT
	   if(retrans_xmit_enable == OPTION_ENABLE){
	     if(!(acq_tbl->read_distrib_enable & READ_CTL_DISCARD))
	   	  if(!discard_prod){
		    acq_tbl->proc_base_prod_seqno_last = buff_hdr->proc_prod_seqno;
		    discard_prod = 0;
	      }
	   }  
#endif
		
#ifdef RETRANS_SUPPORT
	}

        save_prod = 1;
#endif
        IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize);

        if (ulogIsDebug())
            udebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen,
                deflen);
#ifdef RETRANS_SUPPORT
		if(retrans_xmit_enable == OPTION_ENABLE){
		  total_prods_retrans_rcvd = acq_tbl->proc_tot_prods_retrans_rcvd;     /* prods retrans rcvd by proc */
		  total_prods_retrans_rcvd_lost = acq_tbl->proc_tot_prods_retrans_rcvd_lost; /* prods retrans rcvd lost */
		  total_prods_retrans_rcvd_notlost = acq_tbl->proc_tot_prods_retrans_rcvd_notlost; /* prods rcvd not lost */
		  total_prods_retrans_rqstd = acq_tbl->proc_tot_prods_retrans_rqstd;    /* prods retrans requested */
		  total_prods_handled = acq_tbl->proc_tot_prods_handled;    /* prods retrans requested */
		  total_prods_lost_err = acq_tbl->proc_tot_prods_lost_errs;    /* prods retrans requested */
		  total_frame_cnt = acq_tbl->read_tot_buff_read;
		  total_frame_err = acq_tbl->read_frame_tot_lost_errs;
		  proc_orig_prod_seqno_last_save = acq_tbl->proc_orig_prod_seqno_last;
        }
#endif
    }

    if (NULL != memheap)
        free(memheap);

    productMaker->status = status;

    return NULL;
}
Ejemplo n.º 21
0
/*
 * Public functions
 */
int cu_rrd_create_file (const char *filename, /* {{{ */
    const data_set_t *ds, const value_list_t *vl,
    const rrdcreate_config_t *cfg)
{
  char **argv;
  int argc;
  char **rra_def;
  int rra_num;
  char **ds_def;
  int ds_num;
  int status = 0;
  time_t last_up;
  unsigned long stepsize;

  if (check_create_dir (filename))
    return (-1);

  if ((rra_num = rra_get (&rra_def, vl, cfg)) < 1)
  {
    ERROR ("cu_rrd_create_file failed: Could not calculate RRAs");
    return (-1);
  }

  if ((ds_num = ds_get (&ds_def, ds, vl, cfg)) < 1)
  {
    ERROR ("cu_rrd_create_file failed: Could not calculate DSes");
    return (-1);
  }

  argc = ds_num + rra_num;

  if ((argv = (char **) malloc (sizeof (char *) * (argc + 1))) == NULL)
  {
    char errbuf[1024];
    ERROR ("cu_rrd_create_file failed: %s",
        sstrerror (errno, errbuf, sizeof (errbuf)));
    return (-1);
  }

  memcpy (argv, ds_def, ds_num * sizeof (char *));
  memcpy (argv + ds_num, rra_def, rra_num * sizeof (char *));
  argv[ds_num + rra_num] = NULL;

  last_up = CDTIME_T_TO_TIME_T (vl->time);
  if (last_up <= 0)
    last_up = time (NULL);
  last_up -= 1;

  if (cfg->stepsize > 0)
    stepsize = cfg->stepsize;
  else
    stepsize = (unsigned long) CDTIME_T_TO_TIME_T (vl->interval);

  if (cfg->async)
  {
    status = srrd_create_async (filename, stepsize, last_up,
        argc, (const char **) argv);
    if (status != 0)
      WARNING ("cu_rrd_create_file: srrd_create_async (%s) "
          "returned status %i.",
          filename, status);
  }
  else /* synchronous */
  {
    status = srrd_create (filename, stepsize, last_up,
        argc, (const char **) argv);

    if (status != 0)
    {
      WARNING ("cu_rrd_create_file: srrd_create (%s) returned status %i.",
          filename, status);
    }
    else
    {
      DEBUG ("cu_rrd_create_file: Successfully created RRD file \"%s\".",
          filename);
    }
  }

  free (argv);
  ds_free (ds_num, ds_def);
  rra_free (rra_num, rra_def);

  return (status);
} /* }}} int cu_rrd_create_file */
Ejemplo n.º 22
0
/* select only the best 'maxfields' fields
   first calc contrasts then select from each part of the
   frame a some fields
*/
DSVector selectfields(MotionDetect* md, contrastSubImgFunc contrastfunc) {
  int i, j;
  DSVector goodflds;
  contrast_idx *ci =
    (contrast_idx*) ds_malloc(sizeof(contrast_idx) * md->fieldNum);
  ds_vector_init(&goodflds, md->fieldNum);

  // we split all fields into row+1 segments and take from each segment
  // the best fields
  int numsegms = (md->fieldRows + 1);
  int segmlen = md->fieldNum / (md->fieldRows + 1) + 1;
  // split the frame list into rows+1 segments
  contrast_idx *ci_segms =
    (contrast_idx*) ds_malloc(sizeof(contrast_idx) * md->fieldNum);
  int remaining = 0;
  // calculate contrast for each field
  // #pragma omp parallel for shared(ci,md) no speedup because to short
  for (i = 0; i < md->fieldNum; i++) {
    ci[i].contrast = contrastfunc(md, &md->fields[i]);
    ci[i].index = i;
    if (ci[i].contrast < md->contrastThreshold)
      ci[i].contrast = 0;
    // else printf("%i %lf\n", ci[i].index, ci[i].contrast);
  }

  memcpy(ci_segms, ci, sizeof(contrast_idx) * md->fieldNum);
  // get best fields from each segment
  for (i = 0; i < numsegms; i++) {
    int startindex = segmlen * i;
    int endindex = segmlen * (i + 1);
    endindex = endindex > md->fieldNum ? md->fieldNum : endindex;
    //printf("Segment: %i: %i-%i\n", i, startindex, endindex);

    // sort within segment
    qsort(ci_segms + startindex, endindex - startindex,
      sizeof(contrast_idx), cmp_contrast_idx);
    // take maxfields/numsegms
    for (j = 0; j < md->maxFields / numsegms; j++) {
      if (startindex + j >= endindex)
        continue;
      // printf("%i %lf\n", ci_segms[startindex+j].index,
      //                    ci_segms[startindex+j].contrast);
      if (ci_segms[startindex + j].contrast > 0) {
        ds_vector_append_dup(&goodflds, &ci[ci_segms[startindex+j].index],
                           sizeof(contrast_idx));
        // don't consider them in the later selection process
        ci_segms[startindex + j].contrast = 0;
      }
    }
  }
  // check whether enough fields are selected
  // printf("Phase2: %i\n", ds_list_size(goodflds));
  remaining = md->maxFields - ds_vector_size(&goodflds);
  if (remaining > 0) {
    // take the remaining from the leftovers
    qsort(ci_segms, md->fieldNum, sizeof(contrast_idx), cmp_contrast_idx);
    for (j = 0; j < remaining; j++) {
      if (ci_segms[j].contrast > 0) {
        ds_vector_append_dup(&goodflds, &ci_segms[j], sizeof(contrast_idx));
      }
    }
  }
  // printf("Ende: %i\n", ds_list_size(goodflds));
  ds_free(ci);
  ds_free(ci_segms);
  return goodflds;
}
Ejemplo n.º 23
0
/**
 * Reads NOAAPORT data from a shared-memory FIFO or a file, creates LDM
 * data-products, and inserts the data-products into an LDM product-queue.
 *
 * Usage:
 *
 *     readnoaaport [-nvx] [-q <em>queue</em>] [-u <em>n</em>] [-m mcastAddr] [path]\n
 *
 * Where:
 * <dl>
 *      <dt>-l <em>log</em></dt>
 *      <dd>Log to \e log. if \e log is "-", then logging occurs to the 
 *      standard error stream; otherwise, \e log is the pathname of a file to
 *      which logging will occur. If not specified, then log messages will go
 *      to the system logging daemon. </dd>
 *
 *      <dt>-m <em>mcastAddr</em></dt>
 *      <dd>Use the shared-memory FIFO associated with the UDP
 *      multicast address \e mcastAddr.</dd>
 *
 *      <dt>-n</dt>
 *      <dd>Log messages of level NOTICE and higher priority.</dd>
 *
 *      <dt>-q <em>queue</em></dt>
 *      <dd>Use \e queue as the pathname of the LDM product-queue. The default
 *      is to use the default LDM pathname of the product-queue.</dd>
 *
 *      <dt>-u <em>n</em></dt>
 *      <dd>If logging is to the system logging daemon, then use facility 
 *      <b>local</b><em>n</em>. The default is to use the LDM facility.
 *
 *      <dt>-v</dt>
 *      <dd>Log messages of level INFO and higher priority. Each data-product
 *      will generate a log message.</dd>
 *
 *      <dt>-x</dt>
 *      <dd>Log messages of level DEBUG and higher priority.</dd>
 *
 *      <dt><em>path</em></dt>
 *      <dd>Pathname of the file from which to read data. The default is to use
 *      a shared-memory FIFO.</dd>
 * </dl>
 *
 * @retval 0 if successful.
 * @retval 1 if an error occurred. At least one error-message is logged.
 */
int main(
     const int          argc,
     char* const        argv[])
{
#ifdef HAVE_GET_QUEUE_PATH
    const char*         pqfname = getQueuePath();
#else
    const char*         pqfname = DEFAULT_QUEUE;
#endif
    int                 fd;
    char*               prodmmap;
    char*               memheap = NULL;
    size_t              heapsize;
    size_t              heapcount;
    unsigned char       b1;
    int                 cnt, dataoff, datalen, deflen;
    int                 nscan;
    long                IOFF;
    int                 NWSTG, GOES, PNGINIT = 0, PROD_COMPRESSED;
    long                last_sbn_seqno = (-1);
    char                PROD_NAME[1024];
    int                 status;
    prodstore           prod;
    sbn_struct*         sbn;
    pdh_struct*         pdh;
    psh_struct*         psh;
    ccb_struct*         ccb;
    pdb_struct*         pdb;
    datastore*          pfrag;
    extern int          optind;
    extern int          opterr;
    extern char*        optarg;
    int                 ch;
    int                 logmask = LOG_MASK(LOG_ERR);
    const char*         logfname = NULL;        /* use system logging daemon */
    unsigned            logOptions = LOG_CONS | LOG_PID;
    unsigned            logFacility = LOG_LDM;  /* use default LDM facility */
    const char* const   progName = ubasename(argv[0]);
    MD5_CTX*            md5ctxp = NULL;
    /*unsigned char *compr;
    long                comprLen = 10000 * sizeof (int);*/
    int                 pid_channel = -1;

    /*compr = (unsigned char *) calloc (comprLen, 1);*/

    /* Initialize the logger. */
    (void)setulogmask(logmask);
    (void)openulog(progName, logOptions, logFacility, logfname);

    opterr = 1;
    while ((ch = getopt(argc, argv, "nvxl:q:u:m:")) != EOF) {
        switch (ch) {
        case 'v':
            logmask |= LOG_MASK(LOG_INFO);
            (void)setulogmask(logmask);
            break;
        case 'x':
            logmask |= LOG_MASK(LOG_DEBUG);
            (void)setulogmask(logmask);
            break;
        case 'n':
            logmask |= LOG_MASK(LOG_NOTICE);
            (void)setulogmask(logmask);
            break;
        case 'l':
            if (optarg[0] == '-' && optarg[1] != 0) {
                nplError("logfile \"%s\" ??\n", optarg);
                usage(argv[0]);
            }
            /* else */
            logfname = optarg;
            (void)openulog(progName, logOptions, logFacility, logfname);
            break;
        case 'q':
            pqfname = optarg;
            break;
        case 'u': {
            int         i = atoi(optarg);

            if (0 <= i && 7 >= i) {
                static int  logFacilities[] = {LOG_LOCAL0, LOG_LOCAL1,
                    LOG_LOCAL2, LOG_LOCAL3, LOG_LOCAL4, LOG_LOCAL5, LOG_LOCAL6,
                    LOG_LOCAL7};

                logFacility = logFacilities[i];

                (void)openulog(progName, logOptions, logFacility, logfname);
            }

            break;
        }
        case 'm':
            sscanf(optarg, "%*d.%*d.%*d.%d", &pid_channel);	
            if ((pid_channel < 1) || (pid_channel > MAX_DVBS_PID)) {
                pid_channel = -1;
            }
            else {  
                shm = shmfifo_new();
                cnt = 0;

                while (((status = shmfifo_shm_from_key(shm, 
                          s_port[pid_channel - 1])) == -3) && (cnt < 30)) {
                    nplInfo("Trying to get shared-memory FIFO");
                    cnt++;
                    sleep(1);
                }

                if (0 != status) {
                    nplError("Couldn't get shared-memory FIFO. "
                            "Check associated dvbs_multicast(1) process.");
                    shmfifo_free(shm);
                    shm = NULL;
                }
                else {
                    nplInfo("Got shared-memory FIFO");
                }
            }
            break;
        case '?':
            usage(argv[0]);
            break;
        }
    }

    if (argc - optind < 0)
        usage(argv[0]);

    nplNotice("Starting Up %s", PACKAGE_VERSION);

    fd = ((argc - optind) == 0)
        ? fileno(stdin)
        : open(argv[optind], O_RDONLY, 0);

    if ((!shm) && (fd == -1)) {
        nplError("could not open input file");
        exit(0);
    }

    /*
     * Set up signal handlers
     */
    set_sigactions();

    /*
     * Register atexit routine
     */
    if (atexit(cleanup) != 0) {
        nplSerror("atexit");
        exit(-1);
    }

    sbn = (sbn_struct*)malloc(sizeof(sbn_struct));
    pdh = (pdh_struct*)malloc(sizeof(pdh_struct));
    psh = (psh_struct*)malloc(sizeof(psh_struct));
    ccb = (ccb_struct*)malloc(sizeof(ccb_struct));
    pdb = (pdb_struct*)malloc(sizeof(pdb_struct));
    prodmmap = (char*)malloc(10000);

    if (prodmmap == NULL) {
        nplError("could not allocate read buffer");
        exit(-1);
    }

    md5ctxp = new_MD5_CTX();
    prod.head = NULL;
    prod.tail = NULL;

    if (lpqGet(pqfname, &ldmProdQueue) != 0) {
        NPL_ADD1("Couldn't open LDM product-queue \"%s\"", pqfname);
        exit(1);
    }

    while (DONE == 0) {
        /* See if any stats need to be logged */
        if (logstats) {
            logstats = 0;
            dump_stats();
        }

        /* Look for first byte == 255  and a valid SBN checksum */
        if ((status = bufread(fd, prodmmap, 1)) != 0) {
            if (-3 == status)
                break;
            abort();
        }
        if ((b1 = (unsigned char)prodmmap[0]) != 255) {
            if (ulogIsVerbose())
                nplInfo("trying to resync %u", b1);
            if (ulogIsDebug())
                nplDebug("bufread loop");
            continue;
        }

        if (bufread(fd, prodmmap + 1, 15) != 0) {
            if (ulogIsDebug())
                nplDebug("couldn't read 16 bytes for sbn");
            continue;
        }

        while ((status = readsbn(prodmmap, sbn)) != 0) {
            if (ulogIsDebug())
                nplDebug("Not SBN start");

            IOFF = 1;

            while ((IOFF < 16) && ((b1 = (unsigned char) prodmmap[IOFF]) !=
                        255))
                IOFF++;

            if (IOFF > 15) {
                break;
            }
            else {
                for (ch = IOFF; ch < 16; ch++)
                    prodmmap[ch - IOFF] = prodmmap[ch];

                if (bufread(fd, prodmmap + 16 - IOFF, IOFF) != 0) {
                    if (ulogIsDebug())
                        nplDebug("Couldn't read bytes for SBN, resync");
                    break;
                }
            }
        }

        if (status != 0) {
            if (ulogIsDebug())
                nplDebug("SBN status continue");
            continue;
        }

        IOFF = 0;

        if (bufread(fd, prodmmap + 16, 16) != 0) {
            if (ulogIsDebug())
                nplDebug("error reading Product Definition Header");
            continue;
        }

        if (ulogIsDebug())
            nplDebug("***********************************************");
        if (last_sbn_seqno != -1) {
            if (sbn->seqno != last_sbn_seqno + 1) {
                nplNotice("Gap in SBN sequence number %ld to %ld [skipped %ld]",
                         last_sbn_seqno, sbn->seqno,
                         sbn->seqno - last_sbn_seqno - 1);
                if ( sbn->seqno > last_sbn_seqno )
                    nmissed = nmissed + 
                        (unsigned long)(sbn->seqno - last_sbn_seqno - 1);
            }
        }

        last_sbn_seqno = sbn->seqno;

        if (ulogIsVerbose())
            nplInfo("SBN seqnumber %ld", sbn->seqno);
        if (ulogIsVerbose())
            nplInfo("SBN datastream %d command %d", sbn->datastream,
                sbn->command);
        if (ulogIsDebug())
            nplDebug("SBN version %d length offset %d", sbn->version, sbn->len);
        if (((sbn->command != 3) && (sbn->command != 5)) || 
                (sbn->version != 1)) {
            nplError("Unknown sbn command/version %d PUNT", sbn->command);
            continue;
        }

        switch (sbn->datastream) {
        case 7:	      /* test */
        case 6:       /* was reserved...now nwstg2 */
        case 5:
            NWSTG = 1;
            GOES = 0;
            break;
        case 1:
        case 2:
        case 4:
            NWSTG = 0;
            GOES = 1;
            break;
        default:
            nplError("Unknown NOAAport channel %d PUNT", sbn->datastream);
            continue;
        }

        /* End of SBN version low 4 bits */

        if (readpdh(prodmmap + IOFF + sbn->len, pdh) == -1) {
            nplError("problem with pdh, PUNT");
            continue;
        }
        if (pdh->len > 16) {
            bufread(fd, prodmmap + sbn->len + 16, pdh->len - 16);
        }

        if (ulogIsDebug())
            nplDebug("Product definition header version %d pdhlen %d",
                pdh->version, pdh->len);

        if (pdh->version != 1) {
            nplError("Error: PDH transfer type %u, PUNT", pdh->transtype);
            continue;
        }
        else if (ulogIsDebug()) {
            nplDebug("PDH transfer type %u", pdh->transtype);
        }

        if ((pdh->transtype & 8) > 0)
            nplError("Product transfer flag error %u", pdh->transtype);
        if ((pdh->transtype & 32) > 0)
            nplError("Product transfer flag error %u", pdh->transtype);

        if ((pdh->transtype & 16) > 0) {
            PROD_COMPRESSED = 1;

            if (ulogIsDebug())
                nplDebug("Product transfer flag compressed %u", pdh->transtype);
        }
        else {
            PROD_COMPRESSED = 0;
        }

        if (ulogIsDebug())
            nplDebug("header length %ld [pshlen = %d]", pdh->len + pdh->pshlen,
                pdh->pshlen);
        if (ulogIsDebug())
            nplDebug("blocks per record %ld records per block %ld\n",
                pdh->blocks_per_record, pdh->records_per_block);
        if (ulogIsDebug())
            nplDebug("product seqnumber %ld block number %ld data block size "
                "%ld", pdh->seqno, pdh->dbno, pdh->dbsize);

        /* Stop here if no psh */
        if ((pdh->pshlen == 0) && (pdh->transtype == 0)) {
            IOFF = IOFF + sbn->len + pdh->len;
            continue;
        }

        if (pdh->pshlen != 0) {
            if (bufread(fd, prodmmap + sbn->len + pdh->len, pdh->pshlen) != 0) {
                nplError("problem reading psh");
                continue;
            }
            else {
                if (ulogIsDebug())
                    nplDebug("read psh %d", pdh->pshlen);
            }

            /* Timing block */
            if (sbn->command == 5) {
                if (ulogIsDebug())
                    nplDebug("Timing block recieved %ld %ld\0", psh->olen,
                        pdh->len);
                /*
                 * Don't step on our psh of a product struct of prod in
                 * progress.
                 */
                continue;
            }

            if (readpsh(prodmmap + IOFF + sbn->len + pdh->len, psh) == -1) {
                nplError("problem with readpsh");
                continue;
            }
            if (psh->olen != pdh->pshlen) {
                nplError("ERROR in calculation of psh len %ld %ld", psh->olen,
                    pdh->len);
                continue;
            }
            if (ulogIsDebug())
                nplDebug("len %ld", psh->olen);
            if (ulogIsDebug())
                nplDebug("product header flag %d, version %d", psh->hflag,
                    psh->version);
            if (ulogIsDebug())
                nplDebug("prodspecific data length %ld", psh->psdl);
            if (ulogIsDebug())
                nplDebug("bytes per record %ld", psh->bytes_per_record);
            if (ulogIsDebug())
                nplDebug("Fragments = %ld category %d ptype %d code %d",
                    psh->frags, psh->pcat, psh->ptype, psh->pcode);
            if (psh->frags < 0)
                nplError("check psh->frags %d", psh->frags);
            if (psh->origrunid != 0)
                nplError("original runid %d", psh->origrunid);
            if (ulogIsDebug())
                nplDebug("next header offset %ld", psh->nhoff);
            if (ulogIsDebug())
                nplDebug("original seq number %ld", psh->seqno);
            if (ulogIsDebug())
                nplDebug("receive time %ld", psh->rectime);
            if (ulogIsDebug())
                nplDebug("transmit time %ld", psh->transtime);
            if (ulogIsDebug())
                nplDebug("run ID %ld", psh->runid);
            if (ulogIsDebug())
                nplDebug("original run id %ld", psh->origrunid);
            if (prod.head != NULL) {
                nplError("OOPS, start of new product [%ld ] with unfinished "
                    "product %ld", pdh->seqno, prod.seqno);

                ds_free();

                prod.head = NULL;
                prod.tail = NULL;

                if (PNGINIT != 0) {
                    pngout_end();
                    PNGINIT = 0;
                }

                nplError("Product definition header version %d pdhlen %d",
                        pdh->version, pdh->len);
                nplError("PDH transfer type %u", pdh->transtype);

                if ((pdh->transtype & 8) > 0)
                    nplError("Product transfer flag error %u", pdh->transtype);
                if ((pdh->transtype & 32) > 0)
                    nplError("Product transfer flag error %u", pdh->transtype);

                nplError("header length %ld [pshlen = %d]",
                    pdh->len + pdh->pshlen, pdh->pshlen);
                nplError("blocks per record %ld records per block %ld",
                    pdh->blocks_per_record, pdh->records_per_block);
                nplError("product seqnumber %ld block number %ld data block "
                    "size %ld", pdh->seqno, pdh->dbno, pdh->dbsize);
                nplError("product header flag %d", psh->hflag);
                nplError("prodspecific data length %ld", psh->psdl);
                nplError("bytes per record %ld", psh->bytes_per_record);
                nplError("Fragments = %ld category %d", psh->frags, psh->pcat);

                if (psh->frags < 0)
                    nplError("check psh->frags %d", psh->frags);
                if (psh->origrunid != 0)
                    nplError("original runid %d", psh->origrunid);

                nplError("next header offset %ld", psh->nhoff);
                nplError("original seq number %ld", psh->seqno);
                nplError("receive time %ld", psh->rectime);
                nplError("transmit time %ld", psh->transtime);
                nplError("run ID %ld", psh->runid);
                nplError("original run id %ld", psh->origrunid);
            }

            prod.seqno = pdh->seqno;
            prod.nfrag = psh->frags;

            ds_init(prod.nfrag);

            /* NWSTG CCB = dataoff, WMO = dataoff + 24 */

            if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen,
                    pdh->dbsize) != 0) {
                nplError("problem reading datablock");
                continue;
            }
            if (sbn->datastream == 4) {
                if (psh->pcat != 3) {
                    GOES = 0;
                    NWSTG = 1;
                }
            }

            heapcount = 0;

            MD5Init(md5ctxp);

            if (GOES == 1) {
                if (readpdb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen,
                        psh, pdb, PROD_COMPRESSED, pdh->dbsize) == -1) {
                    nplError("Error reading pdb, punt");
                    continue;
                }

                memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                if (ulogIsDebug())
                    nplDebug("Read GOES %d %d %d [%d] %d", sbn->len, pdh->len,
                        pdh->pshlen, sbn->len + pdh->len + pdh->pshlen,
                        pdb->len);

                /* Data starts at first block after pdb */
                ccb->len = 0;
                heapsize = prodalloc(psh->frags, 5152, &memheap);
            }
            if (NWSTG == 1) {
                memset(psh->pname, 0, sizeof(psh->pname));

                if (readccb(prodmmap + IOFF + sbn->len + pdh->len + pdh->pshlen,
                        ccb, psh, pdh->dbsize) == -1)
                    nplError("Error reading ccb, using default name");
                if (ulogIsDebug())
                    nplDebug("look at ccb start %d %d", ccb->b1, ccb->len);

                /*
                   cnt = 0;
                   memset(psh->pname,0,sizeof(psh->pname));
                   while ((b1 = (unsigned char)prodmmap[
                           IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len +
                           cnt]) >= 32) {
                       psh->pname[cnt] = prodmmap[
                           IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len +
                           cnt];
                       cnt++;
                   } 
                   if(cnt > 0)
                 */
                if (ulogIsVerbose())
                    nplInfo("%s", psh->pname);

                memcpy(PROD_NAME, psh->pname, sizeof(PROD_NAME));

                heapsize = prodalloc(psh->frags, 4000 + 15, &memheap);
                /*
                 * We will only compute md5 checksum on the data, 11 FOS
                 * characters at start
                 */
                /*
                 * sprintf(memheap,"\001\015\015\012%04d\015\015\012",
                 * ((int)pdh->seqno)%10000);
                 */
                sprintf(memheap, "\001\015\015\012%03d\040\015\015\012",
                    ((int) pdh->seqno) % 1000);

                heapcount += 11;

                if (psh->metaoff > 0)
                    psh->metaoff = psh->metaoff + 11;
            }
        }
        else {
            /* If a continuation record...don't let psh->pcat get missed */
            if ((sbn->datastream == 4) && (psh->pcat != 3)) {
                GOES = 0;
                NWSTG = 1;
            }

            ccb->len = 0;

            if (ulogIsDebug())
                nplDebug("continuation record");
            if ((pdh->transtype & 4) > 0) {
                psh->frags = 0;
            }
            if (bufread(fd, prodmmap + sbn->len + pdh->len + pdh->pshlen,
                    pdh->dbsize) != 0) {
                nplError("problem reading datablock (cont)");
                continue;
            }
            if (prod.head == NULL) {
                if (ulogIsVerbose())
                    nplInfo("found data block before header, "
                        "skipping sequence %d frag #%d", pdh->seqno, pdh->dbno);
                continue;
            }
        }

        /* Get the data */
        dataoff = IOFF + sbn->len + pdh->len + pdh->pshlen + ccb->len;
        datalen = pdh->dbsize - ccb->len;

        if (ulogIsDebug())
            nplDebug("look at datalen %d", datalen);

        pfrag = ds_alloc();
        pfrag->seqno = pdh->seqno;
        pfrag->fragnum = pdh->dbno;
        pfrag->recsiz = datalen;
        pfrag->offset = heapcount;
        pfrag->next = NULL;

        /*memcpy(memheap+heapcount,prodmmap+dataoff,datalen);
        MD5Update(md5ctxp, (unsigned char *)(memheap+heapcount), datalen);
        test_deflate(compr,comprLen,(unsigned char *)(memheap+heapcount),
        datalen);*/

        if (GOES == 1) {
            if (pfrag->fragnum > 0) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) || 
                        (pfrag->seqno != prod.seqno)) {
                    nplError("Missing GOES fragment in sequence, "
                        "last %d/%d this %d/%d\0", prod.tail->fragnum,
                        prod.seqno, pfrag->fragnum, pfrag->seqno);
                    ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }

                if ((PNGINIT != 1) && (!PROD_COMPRESSED)) {
                    nplError("failed pnginit %d %d %s", sbn->datastream,
                            psh->pcat, PROD_NAME);
                    continue;
                }
                if (pdh->records_per_block < 1) {
                    nplError("records_per_block %d blocks_per_record %d "
                        "nx %d ny %d", pdh->records_per_block,
                        pdh->blocks_per_record, pdb->nx, pdb->ny);
                    nplError("source %d sector %d channel %d", pdb->source,
                        pdb->sector, pdb->channel);
                    nplError("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                        "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year,
                        pdb->month, pdb->day, pdb->hour, pdb->minute,
                        pdb->second, pdb->sechunds);
                    nplError("pshname %s", psh->pname);
                }
                if (!PROD_COMPRESSED) {
                    for (nscan = 0; (nscan * pdb->nx) < pdh->dbsize; nscan++) {
                        if (ulogIsDebug())
                            nplDebug("png write nscan %d", nscan);
                        if (nscan >= pdh->records_per_block) {
                            nplError("nscan exceeding records per block %d [%d "
                                "%d %d]", pdh->records_per_block, nscan,
                                pdb->nx, pdh->dbsize);
                        }
                        else {
                          pngwrite(prodmmap + dataoff + (nscan * pdb->nx));
                        }
                    }
                }
                else {
                    memcpy(memheap + heapcount, prodmmap + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
            }
            else {
                if (!PROD_COMPRESSED) {
                    png_set_memheap(memheap, md5ctxp);
                    png_header(prodmmap + dataoff, datalen);
                    /*
                     * Add 1 to number of scanlines, image ends with 
                     * f0f0f0f0...
                     */
                    pngout_init(pdb->nx, pdb->ny + 1);

                    PNGINIT = 1;
                }
                else {
                    memcpy(memheap + heapcount, prodmmap + dataoff, datalen);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        datalen);
                    heapcount += datalen;
                }
                nplNotice("records_per_block %d blocks_per_record %d nx %d ny %d",
                    pdh->records_per_block, pdh->blocks_per_record, pdb->nx,
                    pdb->ny);
                nplNotice("source %d sector %d channel %d", pdb->source,
                    pdb->sector, pdb->channel);
                nplNotice("nrec %d recsize %d date %02d%02d%02d %02d%02d "
                    "%02d.%02d", pdb->nrec, pdb->recsize, pdb->year, pdb->month,
                    pdb->day, pdb->hour, pdb->minute, pdb->second,
                    pdb->sechunds);
                nplNotice("pshname %s", psh->pname);
            }
            deflen = 0;
        }
        else {
            /*
             * test_deflate(memheap+heapcount,heapsize-heapcount,(unsigned char
             * *)(prodmmap+dataoff),datalen,&deflen);
             */
            /* If the product already has a FOS trailer, don't add
             * another....this will match what pqing(SDI) sees
             */
            if ((prod.nfrag != 0) && (prod.tail != NULL)) {
                if ((pfrag->fragnum != prod.tail->fragnum + 1) ||
                        (pfrag->seqno != prod.seqno)) {
                    nplError("Missing fragment in sequence, last %d/%d this "
                        "%d/%d\0", prod.tail->fragnum, prod.seqno,
                        pfrag->fragnum, pfrag->seqno);
                    ds_free();

                    prod.head = NULL;
                    prod.tail = NULL;

                    continue;
                }
            }
            if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
                char testme[4];

                while (datalen > 4) {
                    memcpy(testme, prodmmap + (dataoff + datalen - 4), 4);

                    if (memcmp(testme, FOS_TRAILER, 4) == 0) {
                        datalen -= 4;

                        if (ulogIsDebug())
                            nplDebug("removing FOS trailer from %s", PROD_NAME);
                    }
                    else {
                        break;
                    }
                }
            }
            if (heapcount + datalen > heapsize) {
                /*
                 * this above wasn't big enough heapsize =
                 * prodalloc(psh->frags,4000+15,&memheap);
                 */
                nplError("Error in heapsize %d product size %d [%d %d], Punt!\0",
                    heapsize, (heapcount + datalen), heapcount, datalen);
                continue;
            }

            memcpy(memheap + heapcount, prodmmap + dataoff, datalen);

            deflen = datalen;

            MD5Update(md5ctxp, (unsigned char *) (memheap + heapcount),
                deflen);
        }

        pfrag->recsiz = deflen;
        /*heapcount += datalen;*/
        heapcount += deflen;

        if (prod.head == NULL) {
            prod.head = pfrag;
            prod.tail = pfrag;
        }
        else {
            prod.tail->next = pfrag;
            prod.tail = pfrag;
        }

        if ((prod.nfrag == 0) || (prod.nfrag == (pfrag->fragnum + 1))) {
            if (GOES == 1) {
                if (PNGINIT == 1) {
                    pngout_end();
                    heapcount = png_get_prodlen();
                }
                else {
                    if (ulogIsDebug())
                        nplDebug("GOES product already compressed %d", heapcount);
                }
            }
            if (ulogIsVerbose())
              nplInfo("we should have a complete product %ld %ld/%ld %ld /heap "
                  "%ld", prod.seqno, pfrag->seqno, prod.nfrag, pfrag->fragnum,
                 (long) heapcount);
            if ((NWSTG == 1) && (heapcount > 4)) {
                cnt = 4;		/* number of bytes to add for TRAILER */

                /*
                 * Do a DDPLUS vs HDS check for NWSTG channel only
                 */
                if (sbn->datastream == 5) {
                    /* nwstg channel */
                    switch (psh->pcat) {
                    case 1:
                    case 7:
                      /* Do a quick check for non-ascii text products */
                      if (!prod_isascii(PROD_NAME, memheap, heapcount))
                        psh->pcat += 100;	/* call these HDS */
                      /* else {
                         ** call these DDPLUS **
                         if (memheap[heapcount-1] == FOS_TRAILER[3]) **
                             ETX check **
                             cnt = 0; ** no need to add extra ETX pqing doesn't
                             see it **
                         }
                       */
                      break;
                    }
                }

                if (cnt > 0) {
                    memcpy(memheap + heapcount, FOS_TRAILER + 4 - cnt, cnt);
                    MD5Update(md5ctxp, (unsigned char*)(memheap + heapcount),
                        cnt);
                    heapcount += cnt;
                }
            }

            process_prod(prod, PROD_NAME, memheap, heapcount,
                md5ctxp, ldmProdQueue, psh, sbn);
            ds_free();

            prod.head = NULL;
            prod.tail = NULL;
            PNGINIT = 0;
        }
        else {
            if (ulogIsDebug())
                nplDebug("processing record %ld [%ld %ld]", prod.seqno,
                    prod.nfrag, pfrag->fragnum);
            if ((pdh->transtype & 4) > 0) {
                nplError("Hmmm....should call completed product %ld [%ld %ld]",
                    prod.seqno, prod.nfrag, pfrag->fragnum);
            }
        }

        IOFF += (sbn->len + pdh->len + pdh->pshlen + pdh->dbsize);

        if (ulogIsDebug())
            nplDebug("look IOFF %ld datalen %ld (deflate %ld)", IOFF, datalen,
                deflen);
    }

    if (fd != -1)
       (void)close(fd);

    exit(0);
}
Ejemplo n.º 24
0
/*
 * Public functions
 */
int cu_rrd_create_file (const char *filename, /* {{{ */
                        const data_set_t *ds, const value_list_t *vl,
                        const rrdcreate_config_t *cfg)
{
    char **argv;
    int argc;
    char **rra_def;
    int rra_num;
    char **ds_def;
    int ds_num;
    int status = 0;

    if (check_create_dir (filename))
        return (-1);

    if ((rra_num = rra_get (&rra_def, vl, cfg)) < 1)
    {
        ERROR ("cu_rrd_create_file failed: Could not calculate RRAs");
        return (-1);
    }

    if ((ds_num = ds_get (&ds_def, ds, vl, cfg)) < 1)
    {
        ERROR ("cu_rrd_create_file failed: Could not calculate DSes");
        return (-1);
    }

    argc = ds_num + rra_num;

    if ((argv = (char **) malloc (sizeof (char *) * (argc + 1))) == NULL)
    {
        char errbuf[1024];
        ERROR ("cu_rrd_create_file failed: %s",
               sstrerror (errno, errbuf, sizeof (errbuf)));
        return (-1);
    }

    memcpy (argv, ds_def, ds_num * sizeof (char *));
    memcpy (argv + ds_num, rra_def, rra_num * sizeof (char *));
    argv[ds_num + rra_num] = NULL;

    status = srrd_create (filename,
                          (cfg->stepsize > 0) ? cfg->stepsize : vl->interval,
                          (vl->time > 10) ? (vl->time - 10) : vl->time,
                          argc, (const char **) argv);

    free (argv);
    ds_free (ds_num, ds_def);
    rra_free (rra_num, rra_def);

    if (status != 0)
    {
        WARNING ("cu_rrd_create_file: srrd_create (%s) returned status %i.",
                 filename, status);
    }
    else
    {
        DEBUG ("cu_rrd_create_file: Successfully created RRD file \"%s\".",
               filename);
    }

    return (status);
} /* }}} int cu_rrd_create_file */
Ejemplo n.º 25
0
// pick the best column to split on, based on the information gain metric
int dt_pick_best_column(data_set *data, split_criterion criterion) {
    float *gains = malloc(data->colcount * sizeof(float));

    for(int col = 0; col < data->colcount; col++) {
        // divide up the data based on the mean of the chosen column
        float mean = ds_col_mean(data, col);

        data_set *lesser = ds_new(data->colcount, 1);
        data_set *greater = ds_new(data->colcount, 1);

        for(int row = 0; row < data->rowcount; row++) {
            if(data->x_data[row][col] < mean) {
                ds_add_item(lesser, data->x_data[row], data->y_data[row]);
            }
            else {
                ds_add_item(greater, data->x_data[row], data->y_data[row]);
            }
        }

        float main_splitscore;
        float lesser_splitscore;
        float greater_splitscore;

        if(criterion == CR_ENTROPY) {
            // entropy estimation for the whole data set and the two splits
            main_splitscore = ds_entropy(data);
            lesser_splitscore = ds_entropy(lesser);
            greater_splitscore = ds_entropy(greater);
        }
        else if(criterion == CR_GINI) {
            main_splitscore = ds_gini(data);
            lesser_splitscore = ds_gini(lesser);
            greater_splitscore = ds_gini(greater);
        }
        else {
            fprintf(stderr, "Unknown criterion %d!\n", criterion);
            return 0;
        }

        // ratios for split data sets
        float lesser_frac = ((float)lesser->rowcount) / data->rowcount;
        float greater_frac = ((float)greater->rowcount) / data->rowcount;

        // this is either information gain if the splitscore is entropy
        // or it is the total population diversity score if using gini
        float gain;
        if(criterion == CR_ENTROPY) {
            gain = main_splitscore - ((lesser_frac * lesser_splitscore) +
                    (greater_frac * greater_splitscore));
        }
        else if(criterion == CR_GINI) {
            gain = (lesser_frac * lesser_splitscore) +
                (greater_frac * greater_splitscore);
        }
        else {
            fprintf(stderr, "Unknown criterion %d!\n", criterion);
            return 0;
        }

        gains[col] = gain;
        ds_free(lesser);
        ds_free(greater);
    }

    // pick the best gain
    float best = gains[0];
    int bestcol = 0;
    for(int i = 0; i < data->colcount; i++) {
        if(gains[i] > best) {
            best = gains[i];
            bestcol = i;
        }
    }

    free(gains);
    return bestcol;
}