END_TEST
START_TEST (test_make_ports)
{
UT_string str;
utstring_init(&str);
uint16_t ports[] = {htons(53), htons(80), htons(443)};
size_t count = sizeof(ports) / sizeof(ports[0]);
zclient_rules_make_ports(&str, PROTO_TCP, ACCESS_ALLOW, ports, count);
fail_if(0 != strcmp(utstring_body(&str), "ports.allow.tcp.53.80.443"), "make ports str fail");
utstring_clear(&str);
zclient_rules_make_ports(&str, PROTO_UDP, ACCESS_ALLOW, ports, count);
fail_if(0 != strcmp(utstring_body(&str), "ports.allow.udp.53.80.443"), "make ports str fail");
utstring_clear(&str);
zclient_rules_make_ports(&str, PROTO_TCP, ACCESS_DENY, ports, count);
fail_if(0 != strcmp(utstring_body(&str), "ports.deny.tcp.53.80.443"), "make ports str fail");
utstring_clear(&str);
zclient_rules_make_ports(&str, PROTO_UDP, ACCESS_DENY, ports, count);
fail_if(0 != strcmp(utstring_body(&str), "ports.deny.udp.53.80.443"), "make ports str fail");
utstring_done(&str);
}
END_TEST
START_TEST (test_make_fwd)
{
UT_string str;
utstring_init(&str);
zfwd_rule_t rule;
rule.port = htons(80);
rule.fwd_ip = 0x04030201;
rule.fwd_port = htons(83);
zclient_rules_make_fwd(&str, PROTO_TCP, &rule);
fail_if(0 != strcmp(utstring_body(&str), "fwd.tcp.80.1.2.3.4:83"), "make fwd str fail");
utstring_clear(&str);
zclient_rules_make_fwd(&str, PROTO_UDP, &rule);
fail_if(0 != strcmp(utstring_body(&str), "fwd.udp.80.1.2.3.4:83"), "make fwd str fail");
utstring_clear(&str);
rule.fwd_port = 0;
zclient_rules_make_fwd(&str, PROTO_UDP, &rule);
fail_if(0 != strcmp(utstring_body(&str), "fwd.udp.80.1.2.3.4"), "make fwd str fail");
utstring_done(&str);
}
// TODO: finish to test this one
int o_bin_dbcreate(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *dbname, int dbtype) {
UT_string *s;
int i;
debug_note(o, ORIENT_INFO, "executing binary method: DB_CREATE\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
// check the dbname and dbtype parameters
if ((dbname == NULL) || (strlen(dbname) == 0) || ((dbtype != O_DBTYPE_LOCAL) && (dbtype != O_DBTYPE_MEMORY))) return -1;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_DB_CREATE);
// send db name
utstring_printf(s, dbname);
i = o_putstring(o, c, s);
utstring_clear(s);
switch (dbtype) {
case O_DBTYPE_LOCAL:
utstring_printf(s, "local");
break;
case O_DBTYPE_MEMORY:
utstring_printf(s, "memory");
break;
default:
break;
}
// send db type
i = o_putstring(o, c, s);
utstring_clear(s);
/*
i = o_sendcredentials(o, c, ORIENT_USER);
*/
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
utstring_free(s);
return 0;
}
short o_bin_dataclusteradd(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, const char *type, const char *name, const char *filename, int initialsize) {
UT_string *s;
int i;
short x;
debug_note(o, ORIENT_INFO, "executing binary method: DATACLUSTER_ADD\n");
i = get_con_fd(o, c);
if (i == -1) return -1;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_DATACLUSTER_ADD);
// send cluster type
utstring_printf(s, type);
i = o_putstring(o, c, s);
utstring_clear(s);
// send cluster name
utstring_printf(s, name);
i = o_putstring(o, c, s);
utstring_clear(s);
if (strcmp(type, O_CLUSTER_PHYSICAL) == 0) { // the physical cluster need filename and initialsize, while memory and logical do not need them.
// send cluster filename
utstring_printf(s, filename);
i = o_putstring(o, c, s);
utstring_clear(s);
// send cluster initialsize in bytes
i = o_putint(o, c, initialsize);
}
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return -1;
}
// get the new cluster id
x = o_getshort(o, c, timeout);
utstring_free(s);
return x;
}
/* report to all configured destinations */
void report_status(pmtr_t *cfg) {
int rc;
time_t now = time(NULL);
/* construct msg */
utstring_clear(cfg->s);
utstring_printf(cfg->s, "report %s\n", cfg->report_id);
job_t *j = NULL;
while ( (j=(job_t*)utarray_next(cfg->jobs,j))) {
if (j->respawn == 0) continue; /* don't advertise one-time jobs */
utstring_printf(cfg->s, "%s %c %u %d %s\n", j->name, j->disabled?'d':'e',
(unsigned)(now - j->start_ts), (int)j->pid,
*((char**)utarray_front(&j->cmdv)));
}
/* send to all dests */
int *fd=NULL;
while ( (fd=(int*)utarray_next(cfg->report,fd))) {
rc = write(*fd,utstring_body(cfg->s),utstring_len(cfg->s));
if (rc < 0 && errno != ECONNREFUSED)
syslog(LOG_INFO,"write error: %s", strerror(errno));
if (rc >= 0 && rc < utstring_len(cfg->s)) {
syslog(LOG_INFO,"incomplete write %d/%d", rc, utstring_len(cfg->s));
}
}
}
int set_to_binary(void *set, UT_string *bin) {
uint32_t l, u, a,b,c,d, abcd;
uint16_t s;
uint8_t g;
double h;
utstring_clear(bin);
l=0; utstring_bincpy(bin,&l,sizeof(l)); // placeholder for size prefix
int rc=-1,i=0,*t;
kv_t *kv, kvdef;
char **k=NULL,**def;
while( (k=(char**)utarray_next(output_keys,k))) {
kv = kv_get(set,*k);
t = (int*)utarray_eltptr(output_types,i); assert(t);
def = (char**)utarray_eltptr(output_defaults,i); assert(def);
if (kv==NULL) { /* no such key */
kv=&kvdef;
if (*def) {kv->val=*def; kv->vlen=strlen(*def);} /* default */
else if (*t == str) {kv->val=NULL; kv->vlen=0;} /* zero len string */
else {
fprintf(stderr,"required key %s not present in spool frame\n", *k);
goto done;
}
}
switch(*t) {
case d64: h=atof(kv->val); utstring_bincpy(bin,&h,sizeof(h)); break;
case i8: g=atoi(kv->val); utstring_bincpy(bin,&g,sizeof(g)); break;
case i16: s=atoi(kv->val); utstring_bincpy(bin,&s,sizeof(s)); break;
case i32: u=atoi(kv->val); utstring_bincpy(bin,&u,sizeof(u)); break;
case str:
l=kv->vlen; utstring_bincpy(bin,&l,sizeof(l)); /* length prefix */
utstring_bincpy(bin,kv->val,kv->vlen); /* string itself */
break;
case ipv4:
if ((sscanf(kv->val,"%u.%u.%u.%u",&a,&b,&c,&d) != 4) ||
(a > 255 || b > 255 || c > 255 || d > 255)) {
fprintf(stderr,"invalid IP for key %s: %s\n",*k,kv->val);
goto done;
}
abcd = (a << 24) | (b << 16) | (c << 8) | d;
abcd = htonl(abcd);
utstring_bincpy(bin,&abcd,sizeof(abcd));
break;
default: assert(0); break;
}
i++;
}
uint32_t len = utstring_len(bin); len -= sizeof(len); // length does not include itself
char *length_prefix = utstring_body(bin);
memcpy(length_prefix, &len, sizeof(len));
rc = 0;
done:
return rc;
}
int main(int argc, char * argv[]) {
int opt,verbose=0;
long dirmax=0;
/* input spool */
char *dir=NULL,unit,*sz="10GB";
UT_string *s;
utstring_new(s);
while ( (opt = getopt(argc, argv, "v+s:")) != -1) {
switch (opt) {
default: usage(argv[0]); break;
case 'v': verbose++; break;
case 's':
sz = strdup(optarg);
switch (sscanf(sz, "%ld%c", &dirmax, &unit)) {
case 2: /* check unit */
switch (unit) {
case 't': case 'T': break;
case 'g': case 'G': break;
case 'm': case 'M': break;
case 'k': case 'K': break;
case '\r': case '\n': case ' ': case '\t': break;
default: usage(argv[0]); break;
}
case 1: /* just a number in bytes */ break;
default: usage(argv[0]); break;
}
break;
}
}
if (optind >= argc) usage(argv[0]);
if (!dirmax) usage(argv[0]);
kv_spool_options.dir_max = dirmax;
while (optind < argc) {
dir = argv[optind++];
utstring_clear(s);
utstring_printf(s,"%s/limits", dir);
char *p = utstring_body(s);
FILE *f = fopen(p, "w");
if (f == NULL) {
fprintf(stderr,"cannot open %s: %s\n", p, strerror(errno));
continue;
}
fprintf(f, "%s", sz);
fclose(f);
sp_attrition(dir);
}
done:
utstring_free(s);
return 0;
}
static void lsscan(char *pathpat, time_t s_lo, time_t s_hi, JsonNode *obj, int reverse)
{
struct dirent **namelist;
int i, n;
JsonNode *jarr = json_mkarray();
static UT_string *path = NULL;
if (obj == NULL || obj->tag != JSON_OBJECT)
return;
utstring_renew(path);
/* Set global t_ values */
t_lo = s_lo;
t_hi = s_hi;
if ((n = scandir(pathpat, &namelist, filter_filename, cmp)) < 0) {
json_append_member(obj, "error", json_mkstring("Cannot lsscan requested directory"));
return;
}
if (reverse) {
for (i = n - 1; i >= 0; i--) {
utstring_clear(path);
utstring_printf(path, "%s/%s", pathpat, namelist[i]->d_name);
json_append_element(jarr, json_mkstring(UB(path)));
free(namelist[i]);
}
} else {
for (i = 0; i < n; i++) {
utstring_clear(path);
utstring_printf(path, "%s/%s", pathpat, namelist[i]->d_name);
json_append_element(jarr, json_mkstring(UB(path)));
free(namelist[i]);
}
}
free(namelist);
json_append_member(obj, "results", jarr);
}
////////////////////////////////////////////////////////////////
//
// output_read
//
// This function reads a fastq record from one file and writes
// it to another. It assumes the header in the first line of
// record has already been stored in fwd. The last 3 lines of
// the record are read from input and the record is written
// unaltered to output. A read will also be written to
// mates_output if mates_output is not null. If rev is not null
// this read will be written to mates_output with reversed
// sequence and quality.
//
void output_read(UT_string *fwd, UT_string *rev, UT_string *tmp, FILE *input, FILE *output, FILE *mates_output, int fmt_fasta) {
if (mates_output) {
if (rev) {
utstring_clear(rev);
utstring_concat(rev, fwd); // head
ss_get_utstring(input, tmp); // seq
if (output) {
utstring_concat(fwd, tmp);
}
ss_trunc_utstring(tmp, 1); // remove newline
ss_rev_utstring(tmp);
ss_strcat_utstring(tmp, "\n"); // add newline back
utstring_concat(rev, tmp);
if (!fmt_fasta) {
ss_get_utstring(input, tmp); // extra
if (output) {
utstring_concat(fwd, tmp);
}
utstring_concat(rev, tmp);
ss_get_utstring(input, tmp); // qual
if (output) {
utstring_concat(fwd, tmp);
}
ss_trunc_utstring(tmp, 1); // remove newline
ss_rev_utstring(tmp);
ss_strcat_utstring(tmp, "\n"); // add newline back
utstring_concat(rev, tmp);
}
fputs(utstring_body(rev), mates_output);
if (output) {
fputs(utstring_body(fwd), output);
}
} else {
ss_get_cat_utstring(input, fwd);
if (!fmt_fasta) {
ss_get_cat_utstring(input, fwd);
ss_get_cat_utstring(input, fwd);
}
fputs(utstring_body(fwd), mates_output);
if (output) {
fputs(utstring_body(fwd), output);
}
}
} else {
ss_get_cat_utstring(input, fwd);
if (!fmt_fasta) {
ss_get_cat_utstring(input, fwd);
ss_get_cat_utstring(input, fwd);
}
fputs(utstring_body(fwd), output);
}
}
/**
* prepare the given user, its password and its role to be used for new connections
*/
int o_prepare_user(orientdb *o, int role, const char *username, const char *password) {
OM_DEFINE_OBJECT(o_handler,oh);
// Checking o parameter
if (o == NULL) return O_WRONG_PARAM;
oh = get_oh(o);
debug_note(o, ORIENT_DEBUG, "preparing for setting user and password\n");
// TODO: a would have nice feature: check for local security policies (pass is too short, it is not robust enough, user is banned, etc...)
// checking which type of authentication role we are going to set up
switch (role) {
case ORIENT_ADMIN:
debug_note(o, ORIENT_INFO, "setting user \"%s\" (ADMIN) and its password\n", username);
OM_MUTEX_LOCK(o_prepared, oprep);
utstring_clear(oh->admin->username);
utstring_clear(oh->admin->password);
utstring_printf(oh->admin->username, username);
utstring_printf(oh->admin->password, password);
OM_MUTEX_UNLOCK(o_prepared, oprep);
break;
case ORIENT_USER:
debug_note(o, ORIENT_INFO, "setting user \"%s\" (USER) and its password\n", username);
OM_MUTEX_LOCK(o_prepared, oprep);
utstring_clear(oh->user->username);
utstring_clear(oh->user->password);
utstring_printf(oh->user->username, username);
utstring_printf(oh->user->password, password);
OM_MUTEX_UNLOCK(o_prepared, oprep);
break;
default:
debug_note(o, ORIENT_CRITICAL, "unknown role was given setting user \"%s\" and its password\n", username);
return O_WRONG_ROLE;
break;
}
return O_OK;
}
int
packing_append_tree(struct packing *pack, const char *treepath,
const char *newroot)
{
FTS *fts = NULL;
FTSENT *fts_e = NULL;
size_t treelen;
UT_string *sb;
char *paths[2] = { __DECONST(char *, treepath), NULL };
treelen = strlen(treepath);
fts = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL);
if (fts == NULL)
goto cleanup;
utstring_new(sb);
while ((fts_e = fts_read(fts)) != NULL) {
switch(fts_e->fts_info) {
case FTS_D:
case FTS_DEFAULT:
case FTS_F:
case FTS_SL:
case FTS_SLNONE:
/* Entries not within this tree are irrelevant. */
if (fts_e->fts_pathlen <= treelen)
break;
utstring_clear(sb);
/* Strip the prefix to obtain the target path */
if (newroot) /* Prepend a root if one is specified */
utstring_printf(sb, "%s", newroot);
/* +1 = skip trailing slash */
utstring_printf(sb, "%s", fts_e->fts_path + treelen + 1);
packing_append_file_attr(pack, fts_e->fts_name,
utstring_body(sb), NULL, NULL, 0, 0);
break;
case FTS_DC:
case FTS_DNR:
case FTS_ERR:
case FTS_NS:
/* XXX error cases, check fts_e->fts_errno and
* bubble up the call chain */
break;
default:
break;
}
}
utstring_free(sb);
cleanup:
fts_close(fts);
return EPKG_OK;
}
END_TEST
START_TEST (test_make_p2p_policer)
{
UT_string str;
utstring_init(&str);
crules_make_p2p_policy(&str, 0);
fail_if(0 != strcmp(utstring_body(&str), "p2p_policy.0"), "make p2p_policy str fail");
utstring_clear(&str);
crules_make_p2p_policy(&str, 1);
fail_if(0 != strcmp(utstring_body(&str), "p2p_policy.1"), "make p2p_policy str fail");
utstring_done(&str);
}
END_TEST
START_TEST (test_make_bw)
{
UT_string str;
utstring_init(&str);
zclient_rules_make_bw(&str, 524288, DIR_DOWN);
fail_if(0 != strcmp(utstring_body(&str), "bw.4096KBit.down"), "make bw down str fail");
utstring_clear(&str);
zclient_rules_make_bw(&str, 524288, DIR_UP);
fail_if(0 != strcmp(utstring_body(&str), "bw.4096KBit.up"), "make bw up str fail");
utstring_done(&str);
}
/* PLAYING WITH EXCEPTIONS */
int o_getexceptions(orientdb *o, orientdb_con *c, struct timeval *timeout, UT_string *str) {
int i,len=0;
utstring_clear(str);
debug_note(o, ORIENT_DEBUG, "get a exception\n");
// if i it's == 1 it follows an exception with a cause (2 strings), else the exception is over
while ((i = o_getbyte(o, c, timeout)) == 1) {
if (len > 0) { utstring_printf(str, " | "); len += 3;}
// read the first string
len += o_getstring(o, c, timeout, str);
utstring_printf(str, ": "); len += 2;
// read the second string (it get automatically concatenated due to utstring_printf() macro in utstring.h)
len += o_getstring(o, c, timeout, str);
}
return len;
}
int get_files(UT_array *files, UT_array *stats) {
struct dirent *dent;
char *name, *path;
file_stat_t fsb;
int rc=-1,i=0;
DIR *d;
UT_string *s;
utstring_new(s);
utarray_clear(files);
utarray_clear(stats);
if ( (d = opendir(cf.dir)) == NULL) {
syslog(LOG_ERR,"failed to opendir [%s]: %s\n", cf.dir, strerror(errno));
goto done;
}
while ( (dent = readdir(d)) != NULL) {
if (dent->d_type != DT_REG) continue;
if (dent->d_name[0] == '.') continue; /* skip dot files */
// ok, fully qualify it and push it
utstring_clear(s);
utstring_printf(s, "%s/%s", cf.dir, dent->d_name);
path = utstring_body(s);
if (stat(path,&fsb.sb) == -1) {
syslog(LOG_ERR,"can't stat %s: %s", path, strerror(errno));
continue;
}
fsb.file_idx = i++;
utarray_push_back(files, &path);
utarray_push_back(stats, &fsb);
}
rc = 0; // success
done:
utstring_free(s);
if (d) closedir(d);
return rc;
}
int main(int argc, char *argv[]) {
#ifndef _OPENMP
fprintf(stderr, "\nERROR: Program built with compiler lacking OpenMP support.\n");
fprintf(stderr, "See SEAStAR README file for information about suitable compilers.\n");
exit(EXIT_FAILURE);
#endif
///////////////////////////
// Variable declarations
///////////////////////////
// Input filenames
UT_string *in_read1_fq_fn, *in_read2_fq_fn, *in_single1_fq_fn, *in_single2_fq_fn;
utstring_new(in_read1_fq_fn);
utstring_new(in_read2_fq_fn);
utstring_new(in_single1_fq_fn);
utstring_new(in_single2_fq_fn);
// Output filenames
UT_string *out_read1_fn, *out_read2_fn, *out_single1_fn, *out_single2_fn, *out_mates_fn, *out_filetype;
utstring_new(out_filetype);
utstring_new(out_read1_fn);
utstring_new(out_read2_fn);
utstring_new(out_single1_fn);
utstring_new(out_single2_fn);
utstring_new(out_mates_fn);
// Read name prefix
UT_string *out_read_prefix;
utstring_new(out_read_prefix);
// Flags
int singles_flag = 0; // 1 when two output singles files being written
int num_input_singles_files = 0;
// Read counters
unsigned long int mp_org = 0, R1_org = 0, R2_org = 0, singlet1_org = 0, singlet2_org = 0;
unsigned long int mp_cnt = 0, R1_cnt = 0, R2_cnt = 0, singlet1_cnt = 0, singlet2_cnt = 0, s1_cnt = 0, s2_cnt = 0;
unsigned long int comp_r1 = 0, comp_r2 = 0, comp_s1 = 0, comp_s2 = 0;
unsigned long int read1_singlet_cnt = 0, read2_singlet_cnt = 0;
////////////////////////////////////////////////////////////////////////
// All done with variable declarations!!
///////////////////////////////////
// Command line argtable settings
///////////////////////////////////
struct arg_lit *gzip = arg_lit0("z", "gzip", "Output converted files in gzip compressed format. [NULL]");
struct arg_lit *inv_singles = arg_lit0("v", "invert_singles", "Causes singles output to be the inverse of the input. 2->1 or 1->2 [NULL]");
struct arg_lit *num_singles = arg_lit0("s", "singles", "Write two singlet files, one for each mate-paired input file. [NULL]");
struct arg_rem *sing_rem = arg_rem(NULL, "Note! -v is only valid when there are input singlet reads. -s is only valid when there are NO input singlet reads.");
struct arg_str *pre_read_id = arg_str0(NULL, "prefix", "<string>", "Prefix to add to read identifiers. [out_prefix]");
struct arg_lit *no_pre = arg_lit0(NULL, "no_prefix", "Do not change the read names in any way. [NULL]");
struct arg_lit *pre_read_len = arg_lit0(NULL, "add_len", "Add the final trimmed length value to the read id prefix. [length not added]");
struct arg_dbl *prob = arg_dbl0("p","correct_prob","<d>","Probability that output reads are correct. 0.0 disables quality trimming. [0.5]");
struct arg_int *fixed_len = arg_int0("f","fixed_len","<u>","Trim all reads to a fixed length, still filtering on quality [no fixed length]");
struct arg_int *len = arg_int0("l","min_read_len","<u>","Minimum length of a singlet or longest-mate in nucleotides [24]");
struct arg_int *mate_len = arg_int0("m","min_mate_len","<u>","Minimum length of the shortest mate in nucleotides [min_read_len]");
struct arg_dbl *entropy = arg_dbl0("e","entropy_filter","<d>","Remove reads with per position information below given value (in bits per dinucleotide) [No filter]");
struct arg_lit *entropy_strict = arg_lit0(NULL, "entropy_strict", "Reject reads for low entropy overall, not just the retained part after trimming [NULL]");
struct arg_lit *mates = arg_lit0(NULL, "mates_file", "Produce a Velvet compatible interleaved paired read output file (e.g. <out_prefix>_mates.fastq). [NULL]");
struct arg_lit *no_rev = arg_lit0(NULL, "no_rev", "By default, the second read in each pair is reversed for colorspace --mate-file output. --no_rev disables reversing. [rev]");
struct arg_lit *only_mates = arg_lit0(NULL, "only_mates", "Supress writing .read1 and .read2 outputs. Requires --mates_file. [NULL]");
struct arg_lit *fasta = arg_lit0(NULL, "fasta", "Write FASTA format files instead of FASTQ for all outputs (e.g. <out_prefix>.<read_type>.fasta). [FASTQ]");
struct arg_file *input = arg_file1(NULL, NULL, "<in_prefix>", "Input file prefix: (e.g. <in_prefix>_single.fastq [<in_prefix>_read1.fastq <in_prefix>_read2.fastq]) ");
struct arg_file *output = arg_file1(NULL, NULL, "<out_prefix>", "Output file prefix: (e.g. <out_prefix>_single.fastq [<out_prefix>_read1.fastq <out_prefix>_read2.fastq]) ");
struct arg_lit *version = arg_lit0(NULL,"version","Print the build version and exit.");
struct arg_lit *h = arg_lit0("h", "help", "Request help.");
struct arg_end *end = arg_end(20);
void *argtable[] = {h,version,gzip,inv_singles,num_singles,sing_rem,prob,len,mate_len,fixed_len,pre_read_id,pre_read_len,no_pre,entropy,entropy_strict,mates,no_rev,only_mates,fasta,input,output,end};
int arg_errors = 0;
////////////////////////////////////////////////////////////////////////
// Handle command line processing (via argtable2 library)
////////////////////////////////////////////////////////////////////////
arg_errors = arg_parse(argc, argv, argtable);
if (version->count) {
fprintf(stderr, "%s version: %s\n", argv[0], SS_BUILD_VERSION);
exit(EXIT_SUCCESS);
}
if (h->count) {
fprintf(stderr,"\ntrim_fastq is a utility for performing quality and information-based\n");
fprintf(stderr,"trimming on paired or unpaired, nucleotide or SOLiD colorspace reads. \n\n");
arg_print_syntaxv(stderr, argtable, "\n\n");
arg_print_glossary(stderr, argtable, "%-25s %s\n");
fprintf(stderr, "\nInput and output \"prefixes\" are the part of the filename before:\n");
fprintf(stderr, "_single.fastq [_read1.fastq _read2.fastq] A singlets (single) file\n");
fprintf(stderr, "is required. Mate-paired read files are automatically used if present.\n");
fprintf(stderr, "Multiple output files only produced for mate-paired inputs.\n");
fprintf(stderr, "\nNote! Input and output files may be gzipped, and outputs can be written\n");
fprintf(stderr, "as either FASTQ or FASTA format files.\n");
exit(EXIT_FAILURE);
}
if (arg_errors) {
arg_print_errors(stderr, end, "trimfastq");
arg_print_syntaxv(stderr, argtable, "\n");
exit(EXIT_FAILURE);
}
// Validate entropy
if (entropy->count) {
entropy_cutoff = entropy->dval[0];
if ((entropy_cutoff < 0.0) || (entropy_cutoff > 4.0)) {
fprintf(stderr, "entropy_filter must be [0.0 - 4.0] \n");
exit(EXIT_FAILURE);
}
strict_ent = entropy_strict->count;
} else {
if (entropy_strict->count) {
fprintf(stderr, "Error: --entropy_strict requires --entropy_filter.\n");
exit(EXIT_FAILURE);
}
entropy_cutoff = -1.0;
}
// Validate error_prob
if (prob->count) {
err_prob = prob->dval[0];
if ((err_prob < 0.0) || (err_prob > 1.0)) {
fprintf(stderr, "--correct_prob (-p) must be 0.0 - 1.0 inclusive\n");
exit(EXIT_FAILURE);
}
} else {
err_prob = 0.5;
}
// Validate min read len
if (len->count) {
min_len = len->ival[0];
if (min_len <= 0) {
fprintf(stderr, "min_read_len must be > 0\n");
exit(EXIT_FAILURE);
}
} else {
min_len = 24;
}
// Validate min mate len
if (mate_len->count) {
min_mate_len = mate_len->ival[0];
if (min_mate_len <= 0) {
fprintf(stderr, "min_mate_len must be > 0\n");
exit(EXIT_FAILURE);
}
if (min_mate_len > min_len) {
fprintf(stderr, "min_mate_len must be <= min_len\n");
exit(EXIT_FAILURE);
}
} else {
min_mate_len = min_len;
}
if (fixed_len->count) {
fix_len = min_mate_len = min_len = fixed_len->ival[0];
if ((mate_len->count) || (len->count)) {
fprintf(stderr, "fixed_len cannot be used with min_read_len or min_mate_len\n");
exit(EXIT_FAILURE);
}
if (fix_len <= 0) {
fprintf(stderr, "fixed_len must be > 0\n");
exit(EXIT_FAILURE);
}
} else {
fix_len = 0;
}
if (pre_read_id->count) {
if (no_pre->count) {
fprintf(stderr, "Error: Both --prefix and --no_prefix were specified.\n");
exit(EXIT_FAILURE);
}
if (! strlen(pre_read_id->sval[0])) {
fprintf(stderr, "Read ID prefix may not be zero length.\n");
exit(EXIT_FAILURE);
}
if (strchr(pre_read_id->sval[0], ':') || strchr(pre_read_id->sval[0], '|') || strchr(pre_read_id->sval[0], '+') || strchr(pre_read_id->sval[0], '/')) {
fprintf(stderr, "Read ID prefix '%s' may not contain the characters ':', '|', '+' or '/'.\n", pre_read_id->sval[0]);
exit(EXIT_FAILURE);
}
// Build default read ID prefix
ss_strcat_utstring(out_read_prefix, pre_read_id->sval[0]);
} else {
if (!no_pre->count) {
if (strchr(output->filename[0], ':') || strchr(output->filename[0], '|') || strchr(output->filename[0], '+') || strchr(output->filename[0], '/')) {
fprintf(stderr, "Read ID prefix '%s' (from output prefix) may not contain the characters ':', '|', '+' or '/'.\n", output->filename[0]);
fprintf(stderr, "Hint: Use the --prefix parameter if the output file prefix contains path information.\n");
exit(EXIT_FAILURE);
}
// Build default read ID prefix
ss_strcat_utstring(out_read_prefix, output->filename[0]);
}
}
if ((only_mates->count) && (!mates->count)) {
fprintf(stderr, "--only_mates requires --mates.\n");
exit(EXIT_FAILURE);
}
if ((no_rev->count) && (!mates->count)) {
fprintf(stderr, "--no_rev requires --mates.\n");
exit(EXIT_FAILURE);
}
// Check for null string prefixes
if (!(strlen(input->filename[0]) && strlen(output->filename[0]))) {
fprintf(stderr, "Error: NULL prefix strings are not permitted.\n");
exit(EXIT_FAILURE);
}
// Construct input filenames
utstring_printf(in_read1_fq_fn, "%s.read1.fastq", input->filename[0]);
utstring_printf(in_read2_fq_fn, "%s.read2.fastq", input->filename[0]);
utstring_printf(in_single1_fq_fn, "%s.single.fastq", input->filename[0]);
FILE *in_read_file = NULL;
num_input_singles_files = 1;
// Try to open a singlet fastq file
// Check singlet output options -s and -v
// Set input singlet names to
// - *.single.fastq or
// - *.single1.fastq and *.single2.fastq
if (!(in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r"))) {
utstring_clear(in_single1_fq_fn);
utstring_printf(in_single1_fq_fn, "%s.single1.fastq", input->filename[0]);
utstring_printf(in_single2_fq_fn, "%s.single2.fastq", input->filename[0]);
num_input_singles_files = 2;
if ((in_read_file = ss_get_gzFile(utstring_body(in_single1_fq_fn), "r")) || (in_read_file = ss_get_gzFile(utstring_body(in_single2_fq_fn), "r"))) {
singles_flag = 1; // Two singlet outputs
} else {
singles_flag = num_singles->count; // Number of singlet outputs set by -s parm
if (inv_singles->count) {
fprintf(stderr, "Error: Invalid option -v, No input singlet file(s) found. Use -s to select multiple output singlet files.\n");
exit(EXIT_FAILURE);
}
}
}
if (in_read_file) {
gzclose(in_read_file);
if (num_singles->count) {
fprintf(stderr, "Error: Invalid option -s, Input singlet file(s) found, use -v to change the number of output singlet files.\n");
exit(EXIT_FAILURE);
}
}
// singles->count inverts the current singles file input scheme
singles_flag = (singles_flag ^ inv_singles->count);
// Check if input fastq is colorspace
// If some files are colorspace and some are basespace, throw an error
int fcount = 0;
int cscount = 0;
fcount += ss_is_fastq(utstring_body(in_read1_fq_fn));
fcount += ss_is_fastq(utstring_body(in_read2_fq_fn));
fcount += ss_is_fastq(utstring_body(in_single1_fq_fn));
fcount += ss_is_fastq(utstring_body(in_single2_fq_fn));
cscount += (ss_is_fastq(utstring_body(in_read1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read1_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_read2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_read2_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_single1_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single1_fq_fn)));
cscount += (ss_is_fastq(utstring_body(in_single2_fq_fn)) && ss_is_colorspace_fastq(utstring_body(in_single2_fq_fn)));
if (cscount && (cscount != fcount)) {
printf("Error: Mixed colorspace and basespace FASTQ files detected\n");
exit(EXIT_FAILURE);
}
colorspace_flag = cscount ? 1 : 0;
// Output filenames
if (fasta->count) {
ss_strcat_utstring(out_filetype, "fasta");
read_count_divisor = 2;
} else {
ss_strcat_utstring(out_filetype, "fastq");
read_count_divisor = 4;
}
if (!only_mates->count) {
utstring_printf(out_read1_fn, "%s.read1.%s", output->filename[0], utstring_body(out_filetype));
utstring_printf(out_read2_fn, "%s.read2.%s", output->filename[0], utstring_body(out_filetype));
}
if (singles_flag == 1) {
utstring_printf(out_single1_fn, "%s.single1.%s", output->filename[0], utstring_body(out_filetype));
utstring_printf(out_single2_fn, "%s.single2.%s", output->filename[0], utstring_body(out_filetype));
} else {
utstring_printf(out_single1_fn, "%s.single.%s", output->filename[0], utstring_body(out_filetype));
}
if (mates->count) {
utstring_printf(out_mates_fn, "%s.mates.%s", output->filename[0], utstring_body(out_filetype));
}
////////////////////////////////////////////////////////////////////////////////////////////////
// Begin processing!
#ifdef _OPENMP
omp_set_num_threads(10);
#endif
// This is the value of a non-valid pipe descriptor
#define NO_PIPE 0
int r1_pipe[2];
int r2_pipe[2];
int s1_pipe[2];
int s2_pipe[2];
pipe(r1_pipe);
pipe(r2_pipe);
pipe(s1_pipe);
pipe(s2_pipe);
int r1_out_pipe[2];
int r2_out_pipe[2];
int mates_out_pipe[2];
int s1_out_pipe[2];
int s2_out_pipe[2];
pipe(r1_out_pipe);
pipe(r2_out_pipe);
pipe(mates_out_pipe);
pipe(s1_out_pipe);
pipe(s2_out_pipe);
#pragma omp parallel sections default(shared)
{
#pragma omp section
{ // Read1 reader
fq_stream_trimmer(in_read1_fq_fn, r1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r1, &R1_org, '\0', fasta->count);
}
#pragma omp section
{ // Read1 writer
R1_cnt = ss_stream_writer(out_read1_fn, r1_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Read2 reader
fq_stream_trimmer(in_read2_fq_fn, r2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_r2, &R2_org, '\0', fasta->count);
}
#pragma omp section
{ // Read2 writer
R2_cnt = ss_stream_writer(out_read2_fn, r2_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Single1 reader
// When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
if ((singles_flag) && (num_input_singles_files == 1)) {
singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '1', fasta->count);
} else {
singlet1_cnt = fq_stream_trimmer(in_single1_fq_fn, s1_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s1, &singlet1_org, '\0', fasta->count);
}
}
#pragma omp section
{ // Single1 writer
s1_cnt = ss_stream_writer(out_single1_fn, s1_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Single2 reader
// When there is only one input singles file, but two output singles files, then supply which mate to use for this stream in the split parameter
if ((singles_flag) && (num_input_singles_files == 1)) {
singlet2_cnt = fq_stream_trimmer(in_single1_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '2', fasta->count);
} else {
singlet2_cnt = fq_stream_trimmer(in_single2_fq_fn, s2_pipe[1], out_read_prefix, no_pre->count, pre_read_len->count, &comp_s2, &singlet2_org, '\0', fasta->count);
}
}
#pragma omp section
{ // Single2 writer
s2_cnt = ss_stream_writer(out_single2_fn, s2_out_pipe[0], gzip->count) / read_count_divisor;
}
#pragma omp section
{ // Velvet mates writer
// Divide count by 2 because both R1 and R2 reads go through this writer
mp_cnt = ss_stream_writer(out_mates_fn, mates_out_pipe[0], gzip->count) / 2 / read_count_divisor;
}
#pragma omp section
{ // Dispatcher
// Allocate data buffer strings
UT_string *r1_data;
utstring_new(r1_data);
UT_string *r2_data;
utstring_new(r2_data);
UT_string *s1_data;
utstring_new(s1_data);
UT_string *s2_data;
utstring_new(s2_data);
UT_string *rev_tmp;
utstring_new(rev_tmp);
UT_string *rev_data;
utstring_new(rev_data);
// Pipes
FILE *r1_in = fdopen(r1_pipe[0],"r");
FILE *r2_in = fdopen(r2_pipe[0],"r");
FILE *s1_in = fdopen(s1_pipe[0],"r");
FILE *s2_in = fdopen(s2_pipe[0],"r");
FILE *mates_out = fdopen(mates_out_pipe[1],"w");
FILE *r1_out = fdopen(r1_out_pipe[1],"w");
FILE *r2_out = fdopen(r2_out_pipe[1],"w");
FILE *s1_out = fdopen(s1_out_pipe[1],"w");
FILE *s2_out = fdopen(s2_out_pipe[1],"w");
if (!singles_flag) {
fclose(s2_out);
s2_out = s1_out;
}
// Flags for data left in single files
int single1_hungry = 1;
int single2_hungry = 1;
// Handle read1 and read2 files
while (ss_get_utstring(r1_in, r1_data)) {
if (!ss_get_utstring(r2_in, r2_data)) {
fprintf(stderr, "Error: Input read1 and read2 files are not synced\n");
exit(EXIT_FAILURE);
}
if (keep_read(r1_data)) {
if (keep_read(r2_data)) {
// Output both read1 and read2
if (mates->count) {
if (only_mates->count) {
// Interleaved velvet output only
output_read(r1_data, NULL, NULL, r1_in, NULL, mates_out, fasta->count);
if (no_rev->count || !colorspace_flag) {
output_read(r2_data, NULL, NULL, r2_in, NULL, mates_out, fasta->count);
} else {
output_read(r2_data, rev_data, rev_tmp, r2_in, NULL, mates_out, fasta->count);
}
} else {
// Interleaved velvet output and normal read file output
output_read(r1_data, NULL, NULL, r1_in, r1_out, mates_out, fasta->count);
if (no_rev->count || !colorspace_flag) {
output_read(r2_data, NULL, NULL, r2_in, r2_out, mates_out, fasta->count);
} else {
output_read(r2_data, rev_data, rev_tmp, r2_in, r2_out, mates_out, fasta->count);
}
}
} else {
// No interleaved velvet output
output_read(r1_data, NULL, NULL, r1_in, r1_out, NULL, fasta->count);
output_read(r2_data, NULL, NULL, r2_in, r2_out, NULL, fasta->count);
}
} else {
// Discard read2, output read1 as singlet
output_read(r1_data, NULL, NULL, r1_in, s1_out, NULL, fasta->count);
read1_singlet_cnt++;
}
} else {
if (keep_read(r2_data)) {
// Discard read1, output read2 as singlet
output_read(r2_data, NULL, NULL, r2_in, s2_out, NULL, fasta->count);
read2_singlet_cnt++;
}
}
// Process reads from singles here to take advantage of
// parallelism
if (single1_hungry || single2_hungry) {
if (single1_hungry) {
if (ss_get_utstring(s1_in, s1_data)) {
if (keep_read(s1_data)) {
output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
}
} else {
single1_hungry = 0;
}
}
if (single2_hungry) {
if (ss_get_utstring(s2_in, s2_data)) {
if (keep_read(s2_data)) {
output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
}
} else {
single2_hungry = 0;
}
}
}
}
while (single1_hungry || single2_hungry) {
if (single1_hungry) {
if (ss_get_utstring(s1_in, s1_data)) {
if (keep_read(s1_data)) {
output_read(s1_data, NULL, NULL, s1_in, s1_out, NULL, fasta->count);
}
} else {
single1_hungry = 0;
}
}
if (single2_hungry) {
if (ss_get_utstring(s2_in, s2_data)) {
if (keep_read(s2_data)) {
output_read(s2_data, NULL, NULL, s2_in, s2_out, NULL, fasta->count);
}
} else {
single2_hungry = 0;
}
}
}
fclose(r1_in);
fclose(r2_in);
fclose(s1_in);
fclose(s2_in);
fclose(mates_out);
fclose(r1_out);
fclose(r2_out);
fclose(s1_out);
if (singles_flag) {
fclose(s2_out);
}
// Free buffers
utstring_free(r1_data);
utstring_free(r2_data);
utstring_free(s1_data);
utstring_free(s2_data);
utstring_free(rev_tmp);
utstring_free(rev_data);
}
}
if (!(R1_org+singlet1_org+singlet2_org)) {
fprintf(stderr, "ERROR! No reads found in input files, or input(s) not found.\n");
exit(EXIT_FAILURE);
}
if (R1_org != R2_org) {
fprintf(stderr, "\nWarning! read1 and read2 fastq files did not contain an equal number of reads. %lu %lu\n", R1_org, R2_org);
}
if ((R1_org + R2_org) && !(singlet1_cnt + singlet2_cnt)) {
fprintf(stderr, "\nWarning! read1/read2 files were processed, but no corresponding input singlets were found.\n");
}
if (entropy->count) {
printf("\nLow complexity reads discarded: Read1: %lu, Read2: %lu, Singlets: %lu %lu\n", comp_r1, comp_r2, comp_s1, comp_s2);
}
mp_org = R1_org;
if (!only_mates->count) {
mp_cnt = R1_cnt;
}
printf("\nMatepairs: Before: %lu, After: %lu\n", mp_org, mp_cnt);
printf("Singlets: Before: %lu %lu After: %lu %lu\n", singlet1_org, singlet2_org, s1_cnt, s2_cnt);
printf("Read1 singlets: %lu, Read2 singlets: %lu, Original singlets: %lu %lu\n", read1_singlet_cnt, read2_singlet_cnt, singlet1_cnt, singlet2_cnt);
printf("Total Reads Processed: %lu, Reads retained: %lu\n", 2*mp_org+singlet1_org+singlet2_org, 2*mp_cnt+s1_cnt+s2_cnt);
utstring_free(in_read1_fq_fn);
utstring_free(in_read2_fq_fn);
utstring_free(in_single1_fq_fn);
utstring_free(in_single2_fq_fn);
utstring_free(out_read1_fn);
utstring_free(out_read2_fn);
utstring_free(out_single1_fn);
utstring_free(out_single2_fn);
utstring_free(out_mates_fn);
utstring_free(out_filetype);
utstring_free(out_read_prefix);
exit(EXIT_SUCCESS);
}
int qk_start(struct qk *qk) {
utvector_clear(&qk->keys);
utstring_clear(&qk->tmp);
}
void cxStringClean(cxString string)
{
if(cxStringLength(string) > 0){
utstring_clear(&string->strptr);
}
}
/*******************************************************************************
* ethernet frame: | 6 byte dst MAC | 6 byte src MAC | 2 byte type | data
* IP datagram: | 1 byte v/len | 1 byte TOS | 2 byte len | 16 more bytes | data
******************************************************************************/
void cb(u_char *unused, const struct pcap_pkthdr *hdr, const u_char *pkt) {
/* data link: ethernet frame */
const uint8_t *dst_mac=pkt, *src_mac=pkt+6, *typep=pkt+12;
const uint8_t *data = pkt+14, *tci_p;
uint16_t type,tci,vid;
// need at least a MAC pair, ethertype and IP header
if (hdr->caplen < 12+2+20) return;
again:
if (hdr->caplen < ((typep - pkt) + 2)) return;
memcpy(&type, typep, sizeof(uint16_t));
type = ntohs(type);
/* 802.2/802.3 encapsulation (RFC 1042). Skip LLC/SNAP to reach ethertype. */
if (type <= 1500) {
typep += 6;
goto again;
}
if (type == 0x8100) /* vlan */ {
/* vlan tags are 'interjected' before the ethertype; they are known by their
* own ethertype (0x8100) followed by a TCI, after which the real etherype
* (or another double VLAN tag) occurs. Each VLAN tag adds two bytes to the
* frame size. The TCI (tag control identifier) contains the VLAN ID (vid).*/
tci_p = typep + 2;
if (hdr->caplen < ((tci_p - pkt)+2)) return;
memcpy(&tci, tci_p, sizeof(uint16_t));
tci = ntohs(tci);
vid = tci & 0xfff; // vlan VID is in the low 12 bits of the TCI
typep = tci_p + 2;
goto again;
}
data = typep + 2;
/*****************************************************************************
* IP datagram
****************************************************************************/
const uint8_t *ip_datagram, *ip_hl, *ip_tos, *ip_len, *ip_id, *ip_fo, *ip_ttl,
*ip_proto, *ip_sum, *ip_src, *ip_dst, *ip_opt, *ip_data;
uint8_t ip_version, ip_hdr_len, *ap;
uint16_t ip_lenh, ip_idh, ip_opts_len;
uint32_t ip_srch, ip_dsth;
if (type == 0x0800) {
ip_datagram = data;
if (hdr->caplen < ((ip_datagram - pkt) + 20)) return;
ip_hl = data;
ip_hdr_len = (*ip_hl & 0x0f) * 4; assert(ip_hdr_len >= 20);
ip_opts_len = ip_hdr_len - 20;
ip_version = (*ip_hl & 0xf0) >> 4;
ip_tos = data + 1;
ip_len = data + 2;
ip_id = data + 4;
ip_fo = data + 6;
ip_ttl = data + 8;
ip_proto=data + 9;
ip_sum = data + 10;
ip_src = data + 12;
ip_dst = data + 16;
ip_opt = data + 20;
ip_data= data + 20 + ip_opts_len;
memcpy(&ip_lenh, ip_len, sizeof(uint16_t)); ip_lenh = ntohs(ip_lenh);
memcpy(&ip_idh, ip_id, sizeof(uint16_t)); ip_idh = ntohs(ip_idh);
memcpy(&ip_srch, ip_src, sizeof(uint32_t)); ip_srch = ntohl(ip_srch);
memcpy(&ip_dsth, ip_dst, sizeof(uint32_t)); ip_dsth = ntohl(ip_dsth);
utstring_clear(cfg.label);
utstring_printf(cfg.label, "%d.%d.%d.%d->%d.%d.%d.%d",
(ip_srch & 0xff000000) >> 24,
(ip_srch & 0x00ff0000) >> 16,
(ip_srch & 0x0000ff00) >> 8,
(ip_srch & 0x000000ff) >> 0,
(ip_dsth & 0xff000000) >> 24,
(ip_dsth & 0x00ff0000) >> 16,
(ip_dsth & 0x0000ff00) >> 8,
(ip_dsth & 0x000000ff) >> 0);
data = ip_data;
char *label = utstring_body(cfg.label);
abtop_hit(cfg.abtop, label, cfg.now, ip_lenh, 0); /* len excludes frame */ // FIXME ab ba
}
void _utstring_clear(void *_buf, unsigned num) {
UT_string *s = (UT_string*)_buf;
while(num--) utstring_clear(&s[num]);
}
static void JsonParser_internalReset(struct ParserInternal *pi)
{
pi->quote_begin = pi->is_value = 0;
utstring_clear(pi->key);
utstring_clear(pi->value);
}
static int
pkg_create_from_dir(struct pkg *pkg, const char *root,
struct packing *pkg_archive)
{
char fpath[MAXPATHLEN];
struct pkg_file *file = NULL;
struct pkg_dir *dir = NULL;
int ret;
struct stat st;
int64_t flatsize = 0;
int64_t nfiles;
const char *relocation;
hardlinks_t *hardlinks;
if (pkg_is_valid(pkg) != EPKG_OK) {
pkg_emit_error("the package is not valid");
return (EPKG_FATAL);
}
relocation = pkg_kv_get(&pkg->annotations, "relocated");
if (relocation == NULL)
relocation = "";
if (pkg_rootdir != NULL)
relocation = pkg_rootdir;
/*
* Get / compute size / checksum if not provided in the manifest
*/
nfiles = kh_count(pkg->filehash);
counter_init("file sizes/checksums", nfiles);
hardlinks = kh_init_hardlinks();
while (pkg_files(pkg, &file) == EPKG_OK) {
snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "",
relocation, file->path);
if (lstat(fpath, &st) == -1) {
pkg_emit_error("file '%s' is missing", fpath);
kh_destroy_hardlinks(hardlinks);
return (EPKG_FATAL);
}
if (file->size == 0)
file->size = (int64_t)st.st_size;
if (st.st_nlink == 1 || !check_for_hardlink(hardlinks, &st)) {
flatsize += file->size;
}
file->sum = pkg_checksum_generate_file(fpath,
PKG_HASH_TYPE_SHA256_HEX);
if (file->sum == NULL) {
kh_destroy_hardlinks(hardlinks);
return (EPKG_FATAL);
}
counter_count();
}
kh_destroy_hardlinks(hardlinks);
counter_end();
pkg->flatsize = flatsize;
if (pkg->type == PKG_OLD_FILE) {
pkg_emit_error("Cannot create an old format package");
return (EPKG_FATAL);
}
/*
* Register shared libraries used by the package if
* SHLIBS enabled in conf. Deletes shlib info if not.
*/
UT_string *b;
utstring_new(b);
pkg_emit_manifest_buf(pkg, b, PKG_MANIFEST_EMIT_COMPACT, NULL);
packing_append_buffer(pkg_archive, utstring_body(b), "+COMPACT_MANIFEST", utstring_len(b));
utstring_clear(b);
pkg_emit_manifest_buf(pkg, b, 0, NULL);
packing_append_buffer(pkg_archive, utstring_body(b), "+MANIFEST", utstring_len(b));
utstring_free(b);
counter_init("packing files", nfiles);
while (pkg_files(pkg, &file) == EPKG_OK) {
snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "",
relocation, file->path);
ret = packing_append_file_attr(pkg_archive, fpath, file->path,
file->uname, file->gname, file->perm, file->fflags);
if (developer_mode && ret != EPKG_OK)
return (ret);
counter_count();
}
counter_end();
nfiles = kh_count(pkg->dirhash);
counter_init("packing directories", nfiles);
while (pkg_dirs(pkg, &dir) == EPKG_OK) {
snprintf(fpath, sizeof(fpath), "%s%s%s", root ? root : "",
relocation, dir->path);
ret = packing_append_file_attr(pkg_archive, fpath, dir->path,
dir->uname, dir->gname, dir->perm, dir->fflags);
if (developer_mode && ret != EPKG_OK)
return (ret);
counter_count();
}
counter_end();
return (EPKG_OK);
}
int main() {
UT_string *s,*t;
char V_TestStr[] = "There are two needle\0s in this \0haystack with needle\0s.";
char V_NeedleStr[] = "needle\0s";
long *V_KMP_Table;
long V_FindPos;
size_t V_StartPos;
size_t V_FindCnt;
utstring_new(s);
utstring_new(t);
utstring_bincpy(s, V_TestStr, sizeof(V_TestStr)-1);
printf("\"%s\" len=%u\n", utstring_body(s), utstring_len(s));
utstring_bincpy(t, V_NeedleStr, sizeof(V_NeedleStr)-1);
printf("\"%s\" len=%u\n", utstring_body(t), utstring_len(t));
V_KMP_Table = (long *)malloc(sizeof(long) * (utstring_len(t) + 1));
if (V_KMP_Table != NULL)
{
_utstring_BuildTableR(utstring_body(t), utstring_len(t), V_KMP_Table);
V_FindCnt = 0;
V_FindPos = 0;
V_StartPos = utstring_len(s) - 1;
do
{
V_FindPos = _utstring_findR(utstring_body(s),
V_StartPos + 1,
utstring_body(t),
utstring_len(t),
V_KMP_Table);
if (V_FindPos >= 0)
{
V_FindCnt++;
V_StartPos = V_FindPos - 1;
}
printf("utstring_find()=%ld\n", V_FindPos);
} while (V_FindPos >= 0);
printf("FindCnt=%u\n", V_FindCnt);
free(V_KMP_Table);
}
else
{
printf("malloc() failed...\n");
}
utstring_free(t);
utstring_clear(s);
utstring_printf(s,"ABC ABCDAB ABCDABCDABDE");
int o;
o=utstring_find( s, -9, "ABC", 3 ) ; printf("expect 15 %d\n",o);
o=utstring_find( s, 3, "ABC", 3 ) ; printf("expect 4 %d\n",o);
o=utstring_find( s, 16, "ABC", 3 ) ; printf("expect -1 %d\n",o);
o=utstring_findR( s, -9, "ABC", 3 ) ; printf("expect 11 %d\n",o);
o=utstring_findR( s, 12, "ABC", 3 ) ; printf("expect 4 %d\n",o);
o=utstring_findR( s, 13, "ABC", 3 ) ; printf("expect 11 %d\n",o);
o=utstring_findR( s, 2, "ABC", 3 ) ; printf("expect 0 %d\n",o);
utstring_free(s);
return 0;
}
ODOC_OBJECT *o_bin_recordload(orientdb *o, orientdb_con *c, struct timeval *timeout, int nonblocking, short clusterid, long position, const char *fetchplan) {
UT_string *s;
int i;
char status;
int record_version;
char record_type;
ODOC_OBJECT *odoc = NULL;
debug_note(o, ORIENT_INFO, "executing binary method: RECORD_LOAD\n");
i = get_con_fd(o, c);
if (i == -1) return NULL;
utstring_new(s);
// send the operation we want to execute to the server
i = o_sendrequest(o, c, O_RECORD_LOAD);
// send the cluster id
i = o_putshort(o, c, clusterid);
// send the cluster position of the record
o_putlong(o, c, position);
if (!fetchplan) {
utstring_printf(s, "%s", O_DEFAULT_FETCHPLAN);
} else {
utstring_printf(s, "%s", fetchplan);
}
// send the fetch plan
i = o_putstring(o, c, s);
utstring_clear(s);
i = o_flushsendbuffer(o, c, timeout);
i = o_getresponse(o, c, timeout);
if (i == 1) { /* EXCEPTION!! */
debug_note(o, ORIENT_INFO, "received an exception\n");
i = o_getexceptions(o, c, timeout, s);
debug_note(o, ORIENT_INFO, "E: %s\n", utstring_body(s));
utstring_free(s);
return NULL;
}
ODOC_INITIALIZE_DOCUMENT(odoc, "");
// getting the status
while ((status = o_getbyte(o, c, timeout)) >0) {
if (status > 2) {
utstring_free(s);
// protocol stream broken. something weird happened
ODOC_FREE_DOCUMENT(odoc);
return NULL;
}
// getting the document
i = o_getbytes(o, c, timeout, s);
record_version = o_getint(o, c, timeout);
record_type = o_getbyte(o, c, timeout);
debug_note(o, ORIENT_DEBUG, "Fetched record: %s, version: %i, type: %c\n", utstring_body(s), record_version, record_type);
odoc_setraw_utstring(odoc,s);
/*
switch (record_type) {
case O_RECORD_DOC: // this is a record document
// create a doc parser object
docparser = odoc_parser_create(s);
if (!docparser) {
utstring_free(s);
ODOC_FREE_DOCUMENT(odoc);
return NULL;
}
// parse the document and return a o_doc object
doc = odoc_parse(o, docparser);
if (!doc) {
odoc_parser_free(docparser);
utstring_free(s);
ODOC_FREE_DOCUMENT(odoc);
return NULL;
}
// drop the bundled document created by ODOC_INITIALIZE_DOCUMENT since we have our doc
odoc_free((OM_OBJECT_TYPE(o_doc) *)odoc->value);
// attach our document instead
odoc->value = (void *)doc;
odoc_parser_free(docparser);
break;
case O_RECORD_FLAT:
utstring_free(s);
ODOC_FREE_DOCUMENT(odoc);
return NULL;
break;
case O_RECORD_RAW:
utstring_free(s);
ODOC_FREE_DOCUMENT(odoc);
return NULL;
break;
}
*/
}
utstring_free(s);
return odoc;
}
/* flush as much pending output to the client as it can handle. */
void feed_client(int ready_fd, int events) {
int *fd=NULL, rc, pos, rv, *p;
char *buf, tmp[100];
size_t len;
UT_string **s=NULL;
/* find the fd in our list */
while ( (fd=(int*)utarray_next(cfg.clients,fd))) {
s=(UT_string**)utarray_next(cfg.outbufs,s); assert(s);
pos = utarray_eltidx(cfg.clients, fd);
if (ready_fd == *fd) break;
}
assert(fd);
if (cfg.verbose > 1) {
fprintf(stderr, "pollout:%c pollin: %c\n", (events & EPOLLOUT)?'1':'0',
(events & EPOLLIN) ?'1':'0');
}
/* before we write to the client, drain any input or closure */
rv = recv(*fd, tmp, sizeof(tmp), MSG_DONTWAIT);
if (rv == 0) {
fprintf(stderr,"client closed (eof)\n");
close(*fd); /* deletes epoll instances on *fd too */
discard_client_buffers(pos);
return;
}
if ((events & EPOLLOUT) == 0) return;
/* send the pending buffer to the client */
p = (int*)utarray_eltptr(cfg.outidxs,pos);
buf = utstring_body(*s) + *p;
len = utstring_len(*s) - *p;
rc = send(*fd, buf, len, MSG_DONTWAIT);
if (cfg.verbose) fprintf(stderr,"sent %d/%d bytes\n", rc, (int)len);
/* test for client closure or error. */
if (rc < 0) {
if ((errno == EWOULDBLOCK) || (errno == EAGAIN)) return;
fprintf(stderr,"client closed (%s)\n", strerror(errno));
close(*fd); /* deletes all epoll instances on *fd too */
discard_client_buffers(pos);
return;
}
/* advance output index in the output buffer; we wrote rc bytes */
if (rc < len) {
*p += rc;
cfg.obpp += rc;
} else {
*p = 0;
utstring_clear(*s); // buffer emptied
mod_epoll(EPOLLIN,*fd); // remove EPOLLOUT
}
#if 1
shift_buffers();
int kc;
while (have_capacity()) {
kc = kv_spool_read(cfg.sp,cfg.set,0);
if (kc < 0) goto done; // error
if (kc == 0) break; // no data
cfg.ompp++;
if (set_to_binary(cfg.set, cfg.s)) goto done;
append_to_client_buf(cfg.s);
}
mark_writable();
done:
return;
#endif
}
static void _nbt_record_clear(void *_r, unsigned num) {
struct nbt_record *r = (struct nbt_record *)_r;
while(num--) { utstring_clear(&r->fqname); r++; }
}
unsigned long int fq_stream_trimmer(UT_string *fq_fn, int pipe_fd, UT_string *out_prefix, int no_pre, int len_pre, unsigned long int *comp_cnt, unsigned long int *org, char split, int fmt_fasta) {
UT_string *new_head_data;
utstring_new(new_head_data);
UT_string *head_data;
utstring_new(head_data);
UT_string *seq_data;
utstring_new(seq_data);
UT_string *extra_data;
utstring_new(extra_data);
UT_string *qual_data;
utstring_new(qual_data);
unsigned long int cnt = 0;
char *start = NULL;
char *end = NULL;
char *suffix = NULL;
FILE *fq_file = NULL;
FILE *pipe_in = fdopen(pipe_fd, "w");
if (!(utstring_len(fq_fn))) {
fclose(pipe_in);
return(0);
}
// Try to open the fastq file
if (!(fq_file = gzopen(utstring_body(fq_fn), "r"))) {
utstring_printf(fq_fn, ".gz");
if (!(fq_file = gzopen(utstring_body(fq_fn), "r"))) {
fclose(pipe_in);
return(0);
}
}
int x = 0;
char head_char = '@';
if (fmt_fasta) {
head_char = '>';
}
while (ss_gzget_utstring(fq_file, head_data)) {
ss_gzget_utstring(fq_file, seq_data);
ss_gzget_utstring(fq_file, extra_data);
ss_gzget_utstring(fq_file, qual_data);
if (!split || ((suffix = strchr(utstring_body(head_data), '/')) && (suffix[1] == split))) {
(*org)++;
if ((x = trimmer(utstring_body(qual_data))) >= min_len) { // Keep at least some of read
// Reject read if complexity is too low
if ((entropy_cutoff < 0.0) || (entropy_calc(utstring_body(seq_data), x) >= entropy_cutoff)) {
// Truncate sequence
ss_trim_utstring(seq_data, x);
ss_strcat_utstring(seq_data, "\n");
if (!fmt_fasta) {
ss_trim_utstring(qual_data, x);
ss_strcat_utstring(qual_data, "\n");
}
// Fixup the read name
utstring_clear(new_head_data);
end = strchr(utstring_body(head_data), ':');
if (no_pre) {
if ((start = strchr(utstring_body(head_data), '|'))) {
start++;
} else {
if (colorspace_flag) {
start = utstring_body(head_data) + 4;
} else {
start = utstring_body(head_data) + 1;
}
}
*end = '\0';
} else {
start = utstring_body(out_prefix);
}
end++;
if (colorspace_flag) {
if (len_pre) {
utstring_printf(new_head_data, "%c%.2s+%u|%s:%s",head_char,utstring_body(head_data)+1,x,start,end);
} else {
utstring_printf(new_head_data, "%c%.2s+%s:%s",head_char,utstring_body(head_data)+1,start,end);
}
} else {
if (len_pre) {
utstring_printf(new_head_data, "%c%u|%s:%s",head_char,x,start,end);
} else {
utstring_printf(new_head_data, "%c%s:%s",head_char,start,end);
}
}
fputs(utstring_body(new_head_data), pipe_in);
fputs(utstring_body(seq_data), pipe_in);
if (!fmt_fasta) {
fputs(utstring_body(extra_data), pipe_in);
fputs(utstring_body(qual_data), pipe_in);
}
cnt++;
} else {
// rejected by entropy filter
// Send along placeholder read to be discarded, keeping read1 and read2 in sync
// Empty fastq header is a read to be rejected by consumer threads
(*comp_cnt)++;
fputs("\n", pipe_in);
}
} else {
// rejected by minimum length cutoff
// Send along placeholder read to be discarded, keeping read1 and read2 in sync
// Empty fastq header is a read to be rejected by consumer threads
fputs("\n", pipe_in);
}
}
}
fclose(pipe_in);
gzclose(fq_file);
utstring_free(new_head_data);
utstring_free(head_data);
utstring_free(seq_data);
utstring_free(extra_data);
utstring_free(qual_data);
return(cnt);
}
