void test_seq_nonacgt(){
seq_t S;
seq_init( &S, "11ACGTNN7682394689NNTGCA11", "name");
seq_subject_init( &S);
g_assert_cmpstr(S.S, ==, "ACGTTGCA");
g_assert_cmpuint(S.len, ==, 8 );
g_assert( FLAGS & F_NON_ACGT);
g_assert_cmpstr(S.RS, ==, "TGCAACGT#ACGTTGCA");
g_assert_cmpuint(S.RSlen, ==, 8*2+1);
g_assert( S.gc == 0.5);
seq_free( &S);
FLAGS = F_NONE;
seq_init( &S, "@ACGT_!0TGCA ", "name");
seq_subject_init( &S);
g_assert_cmpstr(S.S, ==, "ACGT!TGCA");
g_assert_cmpuint(S.len, ==, 9 );
g_assert( FLAGS & F_NON_ACGT);
g_assert_cmpstr(S.RS, ==, "TGCA;ACGT#ACGT!TGCA");
g_assert_cmpuint(S.RSlen, ==, 9*2+1);
seq_free( &S);
FLAGS = F_NONE;
}
void aln_free(alignment *a) {
size_t i;
for (i = 0; i != a->seq_num; ++i)
seq_free(&a->seqs[i]);
free(a->seqs);
free(a->key_positions);
}
void* pique_thread(void* arg)
{
pique_ctx_t* ctx = arg;
seq_t* seq = seq_create();
twobit_t* tb = twobit_alloc();
rng_t* rng = rng_alloc(1234);
bool r;
while (true) {
pthread_mutex_lock(ctx->f_mutex);
if (ctx->fmt == INPUT_FMT_FASTA) r = fasta_read(ctx->f, seq);
else if (ctx->fmt == INPUT_FMT_FASTQ) r = fastq_read(ctx->f, seq);
pthread_mutex_unlock(ctx->f_mutex);
if (!r) break;
/* TODO: remove sequences with Ns? */
twobit_copy_str_n(tb, seq->seq.s, seq->seq.n);
dbg_add_twobit_seq(ctx->G, rng, tb);
}
rng_free(rng);
seq_free(seq);
return NULL;
}
void fastq_grep(FILE* fin, FILE* fout, FILE* mismatch_file, pcre* re)
{
int rc;
int ovector[3];
size_t count = 0;
fastq_t* fqf = fastq_create(fin);
seq_t* seq = seq_create();
while (fastq_read(fqf, seq)) {
rc = pcre_exec(re, /* pattern */
NULL, /* extra data */
id_flag ? seq->id1.s : seq->seq.s,
id_flag ? seq->id1.n : seq->seq.n,
0, /* subject offset */
0, /* options */
ovector, /* output vector */
3 ); /* output vector length */
if ((invert_flag && rc == PCRE_ERROR_NOMATCH) || (!invert_flag && rc >= 0)) {
if (count_flag) count++;
else fastq_print_maybe_trim(fout, seq, ovector);
}
else if (mismatch_file) {
fastq_print(mismatch_file, seq);
}
}
seq_free(seq);
fastq_free(fqf);
if (count_flag) fprintf(fout, "%zu\n", count);
}
/* count the number of entries in a fastq file */
unsigned long count_entries(fastq_t* fqf)
{
seq_t* seq = seq_create();
unsigned long n = 0;
while (fastq_read(fqf, seq)) ++n;
seq_free(seq);
return n;
}
/** Frees the array and all sequences stored within. */
void dsa_free(dsa_t *A) {
size_t i;
for (i = 0; i < A->size; i++) {
seq_free(&A->data[i]);
}
free(A->data);
*A = (dsa_t){};
}
void test_seq_basic(){
seq_t S;
seq_init( &S, "ACGT", "name");
g_assert_cmpstr(S.S, ==, "ACGT");
g_assert_cmpstr(S.name, ==, "name");
g_assert_cmpuint(S.len, ==, 4);
seq_free( &S);
}
/* n-way merge sort to stdout */
void merge_sort(const seq_dumps_t* d, int (*cmp)(const void*, const void*))
{
FILE** files = malloc_or_die(d->n * sizeof(FILE*));
size_t i;
for (i = 0; i < d->n; ++i) {
files[i] = fopen(d->fns[i], "rb");
if (files[i] == NULL) {
fprintf(stderr, "Cannot open temporary file %s for reading.\n",
d->fns[i]);
exit(EXIT_FAILURE);
}
}
fastq_t** fs = malloc_or_die(d->n * sizeof(fastq_t*));
seq_t** seqs = malloc_or_die(d->n * sizeof(seq_t*));
for (i = 0; i < d->n; ++i) {
fs[i] = fastq_create(files[i]);
seqs[i] = seq_create();
}
/* A binary heap of indexes to fs. We use this to repeatedly pop the
* smallest fastq entry. */
size_t* heap = malloc_or_die(d->n * sizeof(size_t));
/* heap size */
size_t m = 0;
for (i = 0; i < d->n; ++i) {
if (fastq_read(fs[i], seqs[i])) {
heap_push(heap, d->n, &m, seqs, cmp, i);
}
}
while (m > 0) {
i = heap_pop(heap, &m, seqs, cmp);
fastq_print(stdout, seqs[i]);
if (fastq_read(fs[i], seqs[i])) {
heap_push(heap, d->n, &m, seqs, cmp, i);
}
}
for (i = 0; i < d->n; ++i) {
seq_free(seqs[i]);
fastq_free(fs[i]);
fclose(files[i]);
}
free(files);
free(fs);
}
void test_seq_full(){
seq_t S;
seq_init( &S, "ACGTTGCA", "name");
int check = seq_subject_init( &S);
g_assert_cmpint(check, ==, 0);
g_assert_cmpstr(S.RS, ==, "TGCAACGT#ACGTTGCA");
g_assert_cmpuint(S.RSlen, ==, 8*2+1);
g_assert( S.gc == 0.5);
seq_free( &S);
}
void count_fastq_kmers(FILE* fin, uint32_t* cs)
{
seq_t* seq = seq_create();
fastq_t* fqf = fastq_create(fin);
int i;
int n;
uint32_t kmer;
while (fastq_read(fqf, seq)) {
n = (int)seq->seq.n - k + 1;
for (i = 0; i < n; i++) {
if( packkmer(seq->seq.s + i, &kmer, k) ) {
cs[kmer]++;
}
}
}
seq_free(seq);
fastq_free(fqf);
}
int main(int argc, char **argv) {
Seq *seq;
char *seq_str;
gzFile f;
if(argc != 2) {
fprintf(stderr, "usage: %s <nuc_str>\n", argv[0]);
exit(2);
}
seq = seq_new();
seq_read_str(seq, argv[1]);
fprintf(stderr, "seq->len=%ld\n", seq->len);
seq_str = seq_get_seqstr(seq);
fprintf(stderr, "%s\n", seq_str);
my_free(seq_str);
fprintf(stderr, "reading from fasta\n");
f = gzopen("test.fa.gz", "wb");
seq_write_fasta_record(seq, f);
gzclose(f);
fprintf(stderr, "writing to fasta\n");
f = gzopen("test.fa.gz", "rb");
seq_read_fasta_record(seq, f);
gzclose(f);
fprintf(stderr, "seq->len=%ld\n", seq->len);
seq_str = seq_get_seqstr(seq);
fprintf(stderr, "%s\n", seq_str);
my_free(seq_str);
seq_free(seq);
return 0;
}
void fastq_print_maybe_trim(FILE* fout, seq_t* seq, int* ovector)
{
if (!trim_before_flag && !trim_after_flag) {
fastq_print(fout, seq);
return;
}
// trimming
seq_t* trimmed = seq_create();
int trimmed_start = 0;
int trimmed_end = 0;
int match_start = ovector[0];
int match_end = ovector[1];
if (trim_before_flag) {
trimmed_end = seq->seq.n;
trimmed_start = trim_match_flag ? match_end : match_start;
} else if (trim_after_flag) {
trimmed_start = 0;
trimmed_end = trim_match_flag ? match_start : match_end;
}
seq_trim(seq, trimmed, trimmed_start, trimmed_end);
fastq_print(fout, trimmed);
seq_free(trimmed);
}
int main(int argc, char ** argv)
{
const char *progname = "sand_compress_reads";
FILE * infile;
FILE * outfile;
int quiet_mode = 0;
struct seq *s;
struct cseq *c;
signed char d;
int clip = 0;
int internal = 0;
char tmp_id[128];
int count = 0;
while((d=getopt(argc,argv,"cvqhi")) > -1) {
switch(d) {
case 'c':
clip = 1;
break;
case 'i':
internal = 1;
break;
case 'q':
quiet_mode = 1;
break;
case 'v':
cctools_version_print(stdout, progname);
exit(0);
break;
case 'h':
default:
show_help(progname);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if( optind<argc ) {
infile = fopen(argv[optind], "r");
if(!infile) {
fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind],strerror(errno));
return 1;
}
optind++;
} else {
infile = stdin;
}
if( optind<argc ) {
outfile = fopen(argv[optind],"w");
if(!outfile) {
fprintf(stderr,"%s: couldn't open %s: %s\n",progname,argv[optind],strerror(errno));
return 1;
}
optind++;
} else {
outfile = stdout;
}
while((s = seq_read(infile))) {
if(clip != 0 || internal != 0){
strcpy(tmp_id, s->name);
strcpy(s->name, strtok(tmp_id,","));
if(internal != 0){
strcpy(s->name, strtok(NULL,","));
}
}
c = seq_compress(s);
cseq_write(outfile,c);
cseq_free(c);
seq_free(s);
count++;
}
if(!quiet_mode) {
fprintf(stderr,"%d sequences compressed.\n",count);
}
fclose(infile);
fclose(outfile);
return 0;
}
void fastq_sample(unsigned long rng_seed,
const char* prefix, const char* cprefix,
FILE* file1, FILE* file2, unsigned long k, double p)
{
/*
* The basic idea is this:
*
* 1. Count the number of lines in the file, n.
*
* 2a. If sampling with replacement, generate k random integers in [0, n-1].
*
* 2b. If sampling without replacement, generate a list of integers 0..(n-1),
* shuffle with fisher-yates, then consider the first k.
*
* 3. Sort the integer list.
*
* 3. Read through the file again, when the number at the front of the integer
* list matches the index of the fastq etry, print the entry, and pop the
* number.
*/
unsigned long n, n2;
fastq_t* f1 = fastq_create(file1);
fastq_t* f2 = file2 == NULL ? NULL : fastq_create(file2);
n = count_entries(f1);
if (f2 != NULL) {
n2 = count_entries(f2);
if (n != n2) {
fprintf(stderr, "Input files have differing numbers of entries (%lu != %lu).\n", n, n2);
exit(1);
}
}
fastq_rewind(f1);
if (f2 != NULL) fastq_rewind(f2);
if (p > 0.0) {
k = (unsigned long) round(p * (double) n);
if (!replacement_flag && k > n) k = n;
}
rng_t* rng = fastq_rng_alloc();
fastq_rng_seed(rng, rng_seed);
unsigned long* xs;
if (replacement_flag) xs = index_with_replacement(rng, n, k);
else xs = index_without_replacement(rng, n);
qsort(xs, k, sizeof(unsigned long), cmpul);
/* open output */
FILE* fout1;
FILE* fout2;
char* output_name;
size_t output_len;
if (file2 == NULL) {
output_len = strlen(prefix) + 7;
output_name = malloc_or_die((output_len + 1) * sizeof(char));
snprintf(output_name, output_len, "%s.fastq", prefix);
fout1 = open_without_clobber(output_name);
if (fout1 == NULL) {
fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
exit(1);
}
fout2 = NULL;
free(output_name);
}
else {
output_len = strlen(prefix) + 9;
output_name = malloc_or_die((output_len + 1) * sizeof(char));
snprintf(output_name, output_len, "%s.1.fastq", prefix);
fout1 = open_without_clobber(output_name);
if (fout1 == NULL) {
fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
exit(1);
}
snprintf(output_name, output_len, "%s.2.fastq", prefix);
fout1 = open_without_clobber(output_name);
if (fout1 == NULL) {
fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
exit(1);
}
free(output_name);
}
/* open complement output */
FILE* cfout1 = NULL;
FILE* cfout2 = NULL;
if (cprefix != NULL && file2 == NULL) {
output_len = strlen(cprefix) + 7;
output_name = malloc_or_die((output_len + 1) * sizeof(char));
snprintf(output_name, output_len, "%s.fastq", cprefix);
cfout1 = fopen(output_name, "wb");
if (cfout1 == NULL) {
fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
exit(1);
}
cfout2 = NULL;
free(output_name);
}
else if (cprefix != NULL) {
output_len = strlen(cprefix) + 9;
output_name = malloc_or_die((output_len + 1) * sizeof(char));
snprintf(output_name, output_len, "%s.1.fastq", cprefix);
cfout1 = fopen(output_name, "wb");
if (cfout1 == NULL) {
fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
exit(1);
}
snprintf(output_name, output_len, "%s.2.fastq", cprefix);
cfout2 = fopen(output_name, "wb");
if (cfout1 == NULL) {
fprintf(stderr, "Cannot open file %s for writing.\n", output_name);
exit(1);
}
free(output_name);
}
unsigned long i = 0; // read number
unsigned long j = 0; // index into xs
int ret;
seq_t* seq1 = seq_create();
seq_t* seq2 = seq_create();
while (j < k && fastq_read(f1, seq1)) {
if (f2 != NULL) {
ret = fastq_read(f2, seq2);
if (ret == 0) {
fputs("Input files have differing numbers of entries.\n", stderr);
exit(1);
}
}
if (xs[j] == i) {
while (j < k && xs[j] == i) {
fastq_print(fout1, seq1);
if (f2 != NULL) fastq_print(fout2, seq2);
++j;
}
}
else if (cfout1 != NULL) {
fastq_print(cfout1, seq1);
if (f2 != NULL) fastq_print(cfout2, seq2);
}
++i;
}
seq_free(seq1);
seq_free(seq2);
fastq_free(f1);
if (f2 != NULL) fastq_free(f2);
fclose(fout1);
if (fout2 != NULL) fclose(fout2);
if (cfout1 != NULL) fclose(cfout1);
if (cfout2 != NULL) fclose(cfout2);
fastq_rng_free(rng);
free(xs);
}
int
restore(
FILE *file,
const char *filename)
{
char *path_p;
struct stat st;
uid_t uid;
gid_t gid;
mode_t mask, flags;
struct do_set_args args = { };
int line = 0, backup_line;
int error, status = 0;
int chmod_required = 0;
memset(&st, 0, sizeof(st));
for(;;) {
backup_line = line;
error = read_acl_comments(file, &line, &path_p, &uid, &gid,
&flags);
if (error < 0) {
error = -error;
goto fail;
}
if (error == 0)
return status;
if (path_p == NULL) {
if (filename) {
fprintf(stderr, _("%s: %s: No filename found "
"in line %d, aborting\n"),
progname, xquote(filename, "\n\r"),
backup_line);
} else {
fprintf(stderr, _("%s: No filename found in "
"line %d of standard input, "
"aborting\n"),
progname, backup_line);
}
status = 1;
goto getout;
}
if (!(args.seq = seq_init()))
goto fail_errno;
if (seq_append_cmd(args.seq, CMD_REMOVE_ACL, ACL_TYPE_ACCESS) ||
seq_append_cmd(args.seq, CMD_REMOVE_ACL, ACL_TYPE_DEFAULT))
goto fail_errno;
error = read_acl_seq(file, args.seq, CMD_ENTRY_REPLACE,
SEQ_PARSE_WITH_PERM |
SEQ_PARSE_DEFAULT |
SEQ_PARSE_MULTI,
&line, NULL);
if (error != 0) {
fprintf(stderr, _("%s: %s: %s in line %d\n"),
progname, xquote(filename, "\n\r"), strerror(errno),
line);
status = 1;
goto getout;
}
error = stat(path_p, &st);
if (opt_test && error != 0) {
fprintf(stderr, "%s: %s: %s\n", progname,
xquote(path_p, "\n\r"), strerror(errno));
status = 1;
}
args.mode = 0;
error = do_set(path_p, &st, 0, &args);
if (error != 0) {
status = 1;
goto resume;
}
if (uid != ACL_UNDEFINED_ID && uid != st.st_uid)
st.st_uid = uid;
else
st.st_uid = -1;
if (gid != ACL_UNDEFINED_ID && gid != st.st_gid)
st.st_gid = gid;
else
st.st_gid = -1;
if (!opt_test &&
(st.st_uid != -1 || st.st_gid != -1)) {
if (chown(path_p, st.st_uid, st.st_gid) != 0) {
fprintf(stderr, _("%s: %s: Cannot change "
"owner/group: %s\n"),
progname, xquote(path_p, "\n\r"),
strerror(errno));
status = 1;
}
/* chown() clears setuid/setgid so force a chmod if
* S_ISUID/S_ISGID was expected */
if ((st.st_mode & flags) & (S_ISUID | S_ISGID))
chmod_required = 1;
}
mask = S_ISUID | S_ISGID | S_ISVTX;
if (chmod_required || ((st.st_mode & mask) != (flags & mask))) {
if (!args.mode)
args.mode = st.st_mode;
args.mode &= (S_IRWXU | S_IRWXG | S_IRWXO);
if (chmod(path_p, flags | args.mode) != 0) {
fprintf(stderr, _("%s: %s: Cannot change "
"mode: %s\n"),
progname, xquote(path_p, "\n\r"),
strerror(errno));
status = 1;
}
}
resume:
if (path_p) {
free(path_p);
path_p = NULL;
}
if (args.seq) {
seq_free(args.seq);
args.seq = NULL;
}
}
getout:
if (path_p) {
free(path_p);
path_p = NULL;
}
if (args.seq) {
seq_free(args.seq);
args.seq = NULL;
}
return status;
fail_errno:
error = errno;
fail:
fprintf(stderr, "%s: %s: %s\n", progname, xquote(filename, "\n\r"),
strerror(error));
status = 1;
goto getout;
}
int main(int argc, char **argv) {
char **fasta_files;
int seed_len, i, n_fasta_files;
ChrTable *chr_tab;
SeedTable *seed_tab;
Seq *seq;
gzFile gzf, out_gzf;
char *out_filename;
if(argc < 4) {
fprintf(stderr, "usage: %s <seed_len> <chromInfo.txt> "
"<output_seed_index.gz> [chr1.fa.gz [chr2.fa.gz [...]]]\n",
argv[0]);
exit(2);
}
seed_len = util_parse_long(argv[1]);
fasta_files = &argv[4];
n_fasta_files = argc - 4;
out_filename = argv[3];
/* read chromosomes and make table containing offsets for both
* forward and reverse strands (so we can represent genomic
* coordinates with a single long integer).
*/
chr_tab = chr_table_read(argv[2]);
fprintf(stderr, "there are %d chromosomes, total length: %u\n",
chr_tab->n_chr, chr_tab->total_chr_len);
/* create a table to hold seed matches */
fprintf(stderr, "initializing seed table\n");
seed_tab = seed_table_new(seed_len);
/* open an output file to write seed table to */
if(util_file_exists(out_filename)) {
my_err("output file %s already exists\n", out_filename);
exit(2);
}
out_gzf = util_must_gzopen(out_filename, "wb");
/*
* first pass: count number of matches to each seed
*/
seq = seq_new();
for(i = 0; i < n_fasta_files; i++) {
fprintf(stderr, "reading sequence from file %s\n", fasta_files[i]);
gzf = util_must_gzopen(fasta_files[i], "rb");
while(seq_read_fasta_record(seq, gzf)) {
fprintf(stderr, "%s %ld\n", seq->name, seq->len);
fprintf(stderr, "counting seed matches\n");
count_matches(chr_tab, seed_tab, seq);
}
gzclose(gzf);
}
/*
* second pass: store location of each match
*/
for(i = 0; i < n_fasta_files; i++) {
fprintf(stderr, "reading sequence from file %s\n", fasta_files[i]);
gzf = util_must_gzopen(fasta_files[i], "rb");
while(seq_read_fasta_record(seq, gzf)) {
fprintf(stderr, "%s %ld\n", seq->name, seq->len);
fprintf(stderr, "recording seed match positions\n");
add_matches(chr_tab, seed_tab, seq);
}
gzclose(gzf);
}
seq_free(seq);
/* write table to file in binary format */
fprintf(stderr, "writing seed table to file %s\n", out_filename);
seed_table_write(seed_tab, out_gzf);
gzclose(out_gzf);
chr_table_free(chr_tab);
seed_table_free(seed_tab);
return 0;
}
/*
* reorder the TCP packets
*/
int tcp_order(order_t *ord, seq_t *new_seq, BOOL src){
seq_t **plist=NULL, **plist_last=NULL, *pre=NULL, *cp=NULL, *aft=NULL;
u_int32_t fr=0, bk=0;
if(src == TRUE){
plist = &(ord->src);
plist_last = &(ord->last_src);
}else{
plist = &(ord->dst);
plist_last = &(ord->last_dst);
}
if( (*plist) == NULL ){
/* first packet */
(*plist) = new_seq;
(*plist_last) = new_seq;
ord->num++;
return 0;
}else{
/* set some variables fr,pre,cp,aft,bk to add a new seq to order */
if((*plist_last)->nxt_seq == new_seq->seq){
fr = ((*plist_last)->seq);
pre = (*plist_last);
cp = tcp_cont_seq((*plist_last));
aft = cp->next;
bk = bk = cp->nxt_seq - 1;
}
else{
cp = (*plist);
/* search position */
pre = cp;
while(cp != NULL && (cp->nxt_seq) <= (new_seq->seq)){ /* not (cp->nxt_seq) <= (new_seq->seq) */
pre = cp;
cp = cp->next;
}
if( cp == NULL){
/* at list end */
pre->next = new_seq;
(*plist_last) = new_seq;
ord->num++;
return 0;
}else{
fr = pre->seq; /* start of continuous segment */
cp = tcp_cont_seq(pre); /* the last seq_t in a continuous segment */
aft = cp->next;
bk = cp->nxt_seq - 1; /* end of continuous segment */
}
}
/* add new seq to the right position */
if((new_seq->seq) >= fr && (new_seq->seq) <= bk ){
if( (new_seq->nxt_seq -1) <= bk){
/* retransmission */
seq_free(new_seq);
return 1;
}else{
/* adjust the packet */
u_int32_t delta = (new_seq->nxt_seq-1) - bk;
if(new_seq->pkt != NULL){
if(new_seq->pkt->tcp_data != NULL){ /* check if we store the packet payload */
new_seq->pkt->tcp_data = new_seq->pkt->tcp_data + (new_seq->pkt->tcp_dl - delta);
}
new_seq->pkt->tcp_dl = delta;
}
new_seq->seq = bk + 1;
/* update order */
cp->next = new_seq;
(*plist_last) = new_seq;
if( aft != NULL ){
new_seq->next = aft;
/* Connect with follower */
if(new_seq->nxt_seq <= aft->seq){
new_seq->next = aft;
(*plist_last) = new_seq;
ord->num++;
return 0;
}else{
/* partially overlapped */
if(new_seq->pkt != NULL){
new_seq->pkt->tcp_dl -= (new_seq->nxt_seq - aft->seq);
}
new_seq->nxt_seq = aft->seq;
new_seq->next = aft;
(*plist_last) = new_seq;
ord->num++;
return 0;
}
}
return 0;
}
}else{
cp->next = new_seq;
if(aft != NULL){
if(new_seq->nxt_seq <= aft->seq){
new_seq->next = aft;
(*plist_last) = new_seq;
ord->num++;
return 0;
}else{
/* partially overlapped */
if(new_seq->pkt != NULL){
new_seq->pkt->tcp_dl -= (new_seq->nxt_seq - aft->seq);
}
new_seq->nxt_seq = aft->seq;
new_seq->next = aft;
(*plist_last) = new_seq;
ord->num++;
return 0;
}
}
return 0;
}
}
}
int main(int argc, char *argv[])
{
int opt;
int saw_files = 0;
int status = 0;
FILE *file;
int which;
int lineno;
int error;
seq_t seq = NULL;
int seq_cmd, parse_mode;
progname = basename(argv[0]);
#if POSIXLY_CORRECT
cmd_line_options = POSIXLY_CMD_LINE_OPTIONS;
cmd_line_spec = _(POSIXLY_CMD_LINE_SPEC);
#else
if (getenv(POSIXLY_CORRECT_STR))
posixly_correct = 1;
if (!posixly_correct) {
cmd_line_options = CMD_LINE_OPTIONS;
cmd_line_spec = _(CMD_LINE_SPEC);
} else {
cmd_line_options = POSIXLY_CMD_LINE_OPTIONS;
cmd_line_spec = _(POSIXLY_CMD_LINE_SPEC);
}
#endif
setlocale(LC_CTYPE, "");
setlocale(LC_MESSAGES, "");
bindtextdomain(PACKAGE, LOCALEDIR);
textdomain(PACKAGE);
while ((opt = getopt_long(argc, argv, cmd_line_options,
long_options, NULL)) != -1) {
/* we remember the two REMOVE_ACL commands of the set
operations because we may later need to delete them. */
cmd_t seq_remove_default_acl_cmd = NULL;
cmd_t seq_remove_acl_cmd = NULL;
if (opt != '\1' && saw_files) {
if (seq) {
seq_free(seq);
seq = NULL;
}
saw_files = 0;
}
if (seq == NULL) {
if (!(seq = seq_init()))
ERRNO_ERROR(1);
}
switch (opt) {
case 'b': /* remove all extended entries */
if (seq_append_cmd(seq, CMD_REMOVE_EXTENDED_ACL,
ACL_TYPE_ACCESS) ||
seq_append_cmd(seq, CMD_REMOVE_ACL,
ACL_TYPE_DEFAULT))
ERRNO_ERROR(1);
break;
case 'k': /* remove default ACL */
if (seq_append_cmd(seq, CMD_REMOVE_ACL,
ACL_TYPE_DEFAULT))
ERRNO_ERROR(1);
break;
case 'n': /* do not recalculate mask */
opt_recalculate = -1;
break;
case 'r': /* force recalculate mask */
opt_recalculate = 1;
break;
case 'd': /* operations apply to default ACL */
opt_promote = 1;
break;
case 's': /* set */
if (seq_append_cmd(seq, CMD_REMOVE_ACL,
ACL_TYPE_ACCESS))
ERRNO_ERROR(1);
seq_remove_acl_cmd = seq->s_last;
if (seq_append_cmd(seq, CMD_REMOVE_ACL,
ACL_TYPE_DEFAULT))
ERRNO_ERROR(1);
seq_remove_default_acl_cmd = seq->s_last;
seq_cmd = CMD_ENTRY_REPLACE;
parse_mode = SEQ_PARSE_WITH_PERM |
SEQ_PARSE_NO_RELATIVE;
goto set_modify_delete;
case 'm': /* modify */
seq_cmd = CMD_ENTRY_REPLACE;
parse_mode = SEQ_PARSE_WITH_PERM;
#if POSIXLY_CORRECT || 1
parse_mode |= SEQ_PARSE_NO_RELATIVE;
#else
if (posixly_correct)
parse_mode |= SEQ_PARSE_NO_RELATIVE;
else
parse_mode |= SEQ_PARSE_ANY_RELATIVE;
#endif
goto set_modify_delete;
case 'x': /* delete */
seq_cmd = CMD_REMOVE_ENTRY;
parse_mode = SEQ_PARSE_NO_RELATIVE;
#if POSIXLY_CORRECT
parse_mode |= SEQ_PARSE_ANY_PERM;
#else
if (posixly_correct)
parse_mode |= SEQ_PARSE_ANY_PERM;
else
parse_mode |= SEQ_PARSE_NO_PERM;
#endif
goto set_modify_delete;
set_modify_delete:
if (!posixly_correct)
parse_mode |= SEQ_PARSE_DEFAULT;
if (opt_promote)
parse_mode |= SEQ_PROMOTE_ACL;
if (parse_acl_seq(seq, optarg, &which,
seq_cmd, parse_mode) != 0) {
if (which < 0 ||
(size_t) which >= strlen(optarg)) {
fprintf(stderr, _(
"%s: Option "
"-%c incomplete\n"),
progname, opt);
} else {
fprintf(stderr, _(
"%s: Option "
"-%c: %s near "
"character %d\n"),
progname, opt,
strerror(errno),
which+1);
}
status = 2;
goto cleanup;
}
break;
case 'S': /* set from file */
if (seq_append_cmd(seq, CMD_REMOVE_ACL,
ACL_TYPE_ACCESS))
ERRNO_ERROR(1);
seq_remove_acl_cmd = seq->s_last;
if (seq_append_cmd(seq, CMD_REMOVE_ACL,
ACL_TYPE_DEFAULT))
ERRNO_ERROR(1);
seq_remove_default_acl_cmd = seq->s_last;
seq_cmd = CMD_ENTRY_REPLACE;
parse_mode = SEQ_PARSE_WITH_PERM |
SEQ_PARSE_NO_RELATIVE;
goto set_modify_delete_from_file;
case 'M': /* modify from file */
seq_cmd = CMD_ENTRY_REPLACE;
parse_mode = SEQ_PARSE_WITH_PERM;
#if POSIXLY_CORRECT || 1
parse_mode |= SEQ_PARSE_NO_RELATIVE;
#else
if (posixly_correct)
parse_mode |= SEQ_PARSE_NO_RELATIVE;
else
parse_mode |= SEQ_PARSE_ANY_RELATIVE;
#endif
goto set_modify_delete_from_file;
case 'X': /* delete from file */
seq_cmd = CMD_REMOVE_ENTRY;
parse_mode = SEQ_PARSE_NO_RELATIVE;
#if POSIXLY_CORRECT
parse_mode |= SEQ_PARSE_ANY_PERM;
#else
if (posixly_correct)
parse_mode |= SEQ_PARSE_ANY_PERM;
else
parse_mode |= SEQ_PARSE_NO_PERM;
#endif
goto set_modify_delete_from_file;
set_modify_delete_from_file:
if (!posixly_correct)
parse_mode |= SEQ_PARSE_DEFAULT;
if (opt_promote)
parse_mode |= SEQ_PROMOTE_ACL;
if (strcmp(optarg, "-") == 0) {
file = stdin;
} else {
file = fopen(optarg, "r");
if (file == NULL) {
fprintf(stderr, "%s: %s: %s\n",
progname,
xquote(optarg),
strerror(errno));
status = 2;
goto cleanup;
}
}
lineno = 0;
error = read_acl_seq(file, seq, seq_cmd,
parse_mode, &lineno, NULL);
if (file != stdin) {
fclose(file);
}
if (error) {
if (!errno)
errno = EINVAL;
if (file != stdin) {
fprintf(stderr, _(
"%s: %s in line "
"%d of file %s\n"),
progname,
strerror(errno),
lineno,
xquote(optarg));
} else {
fprintf(stderr, _(
"%s: %s in line "
"%d of standard "
"input\n"), progname,
strerror(errno),
lineno);
}
status = 2;
goto cleanup;
}
break;
case '\1': /* file argument */
if (seq_empty(seq))
goto synopsis;
saw_files = 1;
status = next_file(optarg, seq);
break;
case 'B': /* restore ACL backup */
saw_files = 1;
if (strcmp(optarg, "-") == 0)
file = stdin;
else {
file = fopen(optarg, "r");
if (file == NULL) {
fprintf(stderr, "%s: %s: %s\n",
progname,
xquote(optarg),
strerror(errno));
status = 2;
goto cleanup;
}
}
status = restore(file,
(file == stdin) ? NULL : optarg);
if (file != stdin)
fclose(file);
if (status != 0)
goto cleanup;
break;
case 'R': /* recursive */
opt_recursive = 1;
break;
case 'L': /* follow symlinks */
opt_walk_logical = 1;
opt_walk_physical = 0;
break;
case 'P': /* do not follow symlinks */
opt_walk_logical = 0;
opt_walk_physical = 1;
break;
case 't': /* test mode */
opt_test = 1;
break;
case 'v': /* print version and exit */
printf("%s " VERSION "\n", progname);
status = 0;
goto cleanup;
case 'h': /* help! */
help();
status = 0;
goto cleanup;
case ':': /* option missing */
case '?': /* unknown option */
default:
goto synopsis;
}
if (seq_remove_acl_cmd) {
/* This was a set operation. Check if there are
actually entries of ACL_TYPE_ACCESS; if there
are none, we need to remove this command! */
if (!has_any_of_type(seq_remove_acl_cmd->c_next,
ACL_TYPE_ACCESS))
seq_delete_cmd(seq, seq_remove_acl_cmd);
}
if (seq_remove_default_acl_cmd) {
/* This was a set operation. Check if there are
actually entries of ACL_TYPE_DEFAULT; if there
are none, we need to remove this command! */
if (!has_any_of_type(seq_remove_default_acl_cmd->c_next,
ACL_TYPE_DEFAULT))
seq_delete_cmd(seq, seq_remove_default_acl_cmd);
}
}
while (optind < argc) {
if (seq_empty(seq))
goto synopsis;
saw_files = 1;
status = next_file(argv[optind++], seq);
}
if (!saw_files)
goto synopsis;
goto cleanup;
synopsis:
fprintf(stderr, _("Usage: %s %s\n"),
progname, cmd_line_spec);
fprintf(stderr, _("Try `%s --help' for more information.\n"),
progname);
status = 2;
goto cleanup;
errno_error:
fprintf(stderr, "%s: %s\n", progname, strerror(errno));
goto cleanup;
cleanup:
if (seq)
seq_free(seq);
return status;
}
int
restore(
FILE *file,
const char *filename)
{
char *path_p;
struct stat stat;
uid_t uid;
gid_t gid;
seq_t seq = NULL;
int line = 0, backup_line;
int error, status = 0;
memset(&stat, 0, sizeof(stat));
for(;;) {
backup_line = line;
error = read_acl_comments(file, &line, &path_p, &uid, &gid);
if (error < 0)
goto fail;
if (error == 0)
return 0;
if (path_p == NULL) {
if (filename) {
fprintf(stderr, _("%s: %s: No filename found "
"in line %d, aborting\n"),
progname, xquote(filename),
backup_line);
} else {
fprintf(stderr, _("%s: No filename found in "
"line %d of standard input, "
"aborting\n"),
progname, backup_line);
}
goto getout;
}
if (!(seq = seq_init()))
goto fail;
if (seq_append_cmd(seq, CMD_REMOVE_ACL, ACL_TYPE_ACCESS) ||
seq_append_cmd(seq, CMD_REMOVE_ACL, ACL_TYPE_DEFAULT))
goto fail;
error = read_acl_seq(file, seq, CMD_ENTRY_REPLACE,
SEQ_PARSE_WITH_PERM |
SEQ_PARSE_NO_RELATIVE |
SEQ_PARSE_DEFAULT |
SEQ_PARSE_MULTI,
&line, NULL);
if (error != 0) {
fprintf(stderr, _("%s: %s: %s in line %d\n"),
progname, xquote(filename), strerror(errno),
line);
goto getout;
}
error = lstat(path_p, &stat);
if (opt_test && error != 0) {
fprintf(stderr, "%s: %s: %s\n", progname,
xquote(path_p), strerror(errno));
status = 1;
}
stat.st_uid = uid;
stat.st_gid = gid;
error = do_set(path_p, &stat, seq);
if (error != 0) {
status = 1;
goto resume;
}
if (!opt_test &&
(uid != ACL_UNDEFINED_ID || gid != ACL_UNDEFINED_ID)) {
if (chown(path_p, uid, gid) != 0) {
fprintf(stderr, _("%s: %s: Cannot change "
"owner/group: %s\n"),
progname, xquote(path_p),
strerror(errno));
status = 1;
}
}
resume:
if (path_p) {
free(path_p);
path_p = NULL;
}
if (seq) {
seq_free(seq);
seq = NULL;
}
}
getout:
if (path_p) {
free(path_p);
path_p = NULL;
}
if (seq) {
seq_free(seq);
seq = NULL;
}
return status;
fail:
fprintf(stderr, "%s: %s: %s\n", progname, xquote(filename),
strerror(errno));
status = 1;
goto getout;
}
/*
* sequencer() - Sequencer main thread entry point.
*/
void sequencer (void *arg) /* ptr to original (global) state program table */
{
PROG *sp = (PROG *)arg;
unsigned nss;
size_t threadLen;
char threadName[THREAD_NAME_SIZE+10];
/* Get this thread's id */
sp->ss->threadId = epicsThreadGetIdSelf();
/* Add the program to the program list */
seqAddProg(sp);
createOrAttachPvSystem(sp);
if (!pvSysIsDefined(sp->pvSys))
{
sp->die = TRUE;
goto exit;
}
/* Call sequencer init function to initialize variables. */
sp->initFunc(sp);
/* Initialize state set variables. In safe mode, copy variable
block to state set buffers. Must do all this before connecting. */
if (optTest(sp, OPT_SAFE))
{
for (nss = 0; nss < sp->numSS; nss++)
{
SSCB *ss = sp->ss + nss;
memcpy(ss->var, sp->var, sp->varSize);
}
}
/* Attach to PV system */
pvSysAttach(sp->pvSys);
/* Initiate connect & monitor requests to database channels, waiting
for all connections to be established if the option is set. */
if (seq_connect(sp, optTest(sp, OPT_CONN) != pvStatOK))
goto exit;
/* Emulate the 'first monitor event' for anonymous PVs */
if (optTest(sp, OPT_SAFE))
{
unsigned nch;
for (nch=0; nch<sp->numChans; nch++)
if (sp->chan[nch].syncedTo && !sp->chan[nch].dbch)
seq_efSet(sp->ss, sp->chan[nch].syncedTo);
}
/* Call program entry function if defined.
Treat as if called from 1st state set. */
if (sp->entryFunc) sp->entryFunc(sp->ss);
/* Create each additional state set task (additional state set thread
names are derived from the first ss) */
epicsThreadGetName(sp->ss->threadId, threadName, sizeof(threadName));
threadLen = strlen(threadName);
for (nss = 1; nss < sp->numSS; nss++)
{
SSCB *ss = sp->ss + nss;
epicsThreadId tid;
/* Form thread name from program name + state set number */
sprintf(threadName+threadLen, "_%d", nss);
/* Spawn the task */
tid = epicsThreadCreate(
threadName, /* thread name */
sp->threadPriority, /* priority */
sp->stackSize, /* stack size */
ss_entry, /* entry point */
ss); /* parameter */
DEBUG("Spawning additional state set thread %p: \"%s\"\n", tid, threadName);
}
/* First state set jumps directly to entry point */
ss_entry(sp->ss);
DEBUG(" Wait for other state sets to exit\n");
for (nss = 1; nss < sp->numSS; nss++)
{
SSCB *ss = sp->ss + nss;
epicsEventMustWait(ss->dead);
}
/* Call program exit function if defined.
Treat as if called from 1st state set. */
if (sp->exitFunc) sp->exitFunc(sp->ss);
exit:
DEBUG(" Disconnect all channels\n");
seq_disconnect(sp);
DEBUG(" Remove program instance from list\n");
seqDelProg(sp);
errlogSevPrintf(errlogInfo,
"Instance %d of sequencer program \"%s\" terminated\n",
sp->instance, sp->progName);
/* Free all allocated memory */
seq_free(sp);
}
int main(int argc, char ** argv)
{
FILE * input;
struct seq *s1=0, *s2=0;
char ori;
char c;
int fileindex;
int del_input=0;
while((c = getopt(argc, argv, "a:o:k:m:q:xd:vh")) != (char) -1) {
switch (c) {
case 'a':
align_type = optarg;
break;
case 'o':
output_format = optarg;
break;
case 'm':
min_align = atoi(optarg);
break;
case 'q':
min_qual = atof(optarg);
break;
case 'x':
del_input = 1;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
default:
case 'h':
show_help(argv[0]);
exit(0);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
fileindex = optind;
if ((argc - optind) == 1) {
input = fopen(argv[fileindex], "r");
if (!input) {
fprintf(stderr, "sand_align_kernel: couldn't open %s: %s\n",argv[fileindex],strerror(errno));
exit(1);
}
} else {
input = stdin;
}
struct cseq *c1, *c2;
if(!strcmp(output_format,"ovl") || !strcmp(output_format, "ovl_new")) {
overlap_write_begin(stdout);
}
// outer loop: read first sequence in comparison list
while((c1=cseq_read(input))) {
s1 = cseq_uncompress(c1);
cseq_free(c1);
// inner loop: read sequences until null (indicating end of list)
// then continue again with outer loop. (two nulls to halt.)
while((c2=cseq_read(input))) {
s2 = cseq_uncompress(c2);
cseq_free(c2);
int dir = 0;
int start1 = 0;
int start2 = 0;
char* tmp = strdup(s2->metadata);
int metadata_valid = 0;
char* token = strtok(tmp, " ");
start2 = atoi(token);
metadata_valid++;
while((token = strtok(NULL, " ")))
{
dir = start1;
start1 = start2;
start2 = atoi(token);
metadata_valid++;
}
if(metadata_valid>=1 && dir==-1) {
seq_reverse_complement(s2);
ori = 'I';
} else {
ori = 'N';
}
struct matrix *m = matrix_create(s1->num_bases,s2->num_bases);
if(!m) {
fprintf(stderr,"sand_align_kernel: out of memory when creating alignment matrix.\n");
exit(1);
}
struct alignment *aln;
if(!strcmp(align_type,"sw")) {
aln = align_smith_waterman(m,s1->data,s2->data);
} else if(!strcmp(align_type,"ps")) {
aln = align_prefix_suffix(m,s1->data,s2->data, min_align);
} else if(!strcmp(align_type,"banded")) {
if(metadata_valid<3) {
fprintf(stderr,"sand_align_kernel: sequence %s did not indicate start positions for the banded alignment.\n",s2->name);
exit(1);
}
/* The width of the band is proportional to the desired quality of the match. */
int k = 2 + min_qual * MIN(s1->num_bases,s2->num_bases) / 2.0;
if(k<5) k = 5;
aln = align_banded(m,s1->data, s2->data, start1, start2, k);
} else {
fprintf(stderr,"unknown alignment type: %s\n",align_type);
exit(1);
}
aln->ori = ori;
if(aln->quality <= min_qual) {
if(!strcmp(output_format,"ovl")) {
overlap_write_v5(stdout, aln, s1->name, s2->name);
} else if(!strcmp(output_format, "ovl_new")) {
overlap_write_v7(stdout, aln, s1->name, s2->name);
} else if(!strcmp(output_format,"matrix")) {
printf("*** %s alignment of sequences %s and %s (quality %lf):\n\n",align_type,s1->name,s2->name,aln->quality);
matrix_print(m,s1->data,s2->data);
} else if(!strcmp(output_format,"align")) {
printf("*** %s alignment of sequences %s and %s (quality %lf):\n\n",align_type,s1->name,s2->name,aln->quality);
alignment_print(stdout,s1->data,s2->data,aln);
} else {
printf("unknown output format '%s'\n",output_format);
exit(1);
}
}
matrix_delete(m);
seq_free(s2);
alignment_delete(aln);
}
seq_free(s1);
}
fclose(input);
if(!strcmp(output_format,"ovl") || !strcmp(output_format, "ovl_new")) {
overlap_write_end(stdout);
}
if ((argc - optind) == 1 && del_input == 1)
{
remove(argv[fileindex]);
}
return 0;
}
// create a test sequence which never uses more than max_used_memory
// and allocates a total of max_used_memory*allocation_factor
SEQLIST *generate_sequence(int max_used_memory, int allocation_factor) {
int used_memory = 0;
int total_allocated = 0;
int next_block_size = 0;
int allocated_blocks = 0;
int actual_max_used_memory = 0;
SEQLIST *test_sequence = (SEQLIST *) 0;
SEQLIST *tail_sequence = (SEQLIST *) 0;
SEQLIST *tofree = (SEQLIST *) 0;
unsigned char *new_block_ref;
while (total_allocated < allocation_factor * max_used_memory) {
next_block_size = random_block_size(max_used_memory);
// first see if we need to free anything in order to
// accommodate the new allocation
while (used_memory + next_block_size > max_used_memory) {
// randomly pick a block to free
SEQLIST *tofree =
find_nth_allocated_block(test_sequence, random_int(allocated_blocks));
// add the free
tail_sequence = seq_set_next_free(tofree, tail_sequence);
// reclaim the memory
used_memory -= seq_size(tofree);
allocated_blocks--;
// mark the old block as something that has been freed
seq_free(tofree);
}
// allocate a reference buffer for the new block
new_block_ref = allocate_and_fill(next_block_size);
// now allocate that block
if (seq_null(test_sequence)) {
// special case for first allocation
test_sequence = seq_add_front(next_block_size, new_block_ref, (SEQLIST *) 0);
tail_sequence = test_sequence;
}
else {
// typical case we add at the end
tail_sequence =
seq_set_next_allocate(next_block_size, new_block_ref, tail_sequence);
}
// debug
//seq_print(tail_sequence); // just prints the new one
total_allocated += next_block_size;
used_memory += next_block_size;
if (used_memory > actual_max_used_memory)
actual_max_used_memory = used_memory;
allocated_blocks++;
}
// just so can manually see this is doing something sensible
printf("Actual maximum memory usage %d (%f)\n", actual_max_used_memory,
((double) actual_max_used_memory / (double) max_used_memory));
return test_sequence;
}
