boost::uint64_t thread_pool_attached_executor<Scheduler>::num_pending_closures(
error_code& ec) const
{
if (&ec != &throws)
ec = make_success_code();
return get_thread_count() - get_thread_count(terminated);
}
void find(const bfs::path& cdb_path, const options& opt)
{
config cfg = load_config(cdb_path / "config");
const bfs::path cdb_root = cdb_path.parent_path();
const bfs::path search_root = bfs::initial_path();
std::size_t prefix_size = 0;
std::string file_match;
if(opt.m_options.count("-a") == 0 && search_root != cdb_root)
{
file_match = "^" + escape_regex(
search_root.generic_string().substr(cdb_root.string().size() + 1) + "/") + ".*";
prefix_size = search_root.string().size() - cdb_root.string().size();
}
file_lock lock(cdb_path / "lock");
lock.lock_sharable();
const char* find_regex_options =
opt.m_options.count("-i") ? "i" : "";
const char* file_regex_options =
cfg.get_value("nocase-file-match") == "on" ? "i" : "";
bool trim = cfg.get_value("find-trim-ws") == "on";
unsigned thread_count = get_thread_count(cfg);
unsigned buffer_count = get_buffer_count(thread_count);
for(unsigned i = 0; i != opt.m_args.size(); ++i)
{
database_ptr db = open_database(cdb_path / "db");
std::string pattern = opt.m_args[i];
if(opt.m_options.count("-v"))
pattern = escape_regex(pattern);
mt_search mts(*db, trim, prefix_size, buffer_count);
auto worker = [&]
{
mts.search_db(compile_regex(pattern, 0, find_regex_options),
compile_regex(file_match, 0, file_regex_options));
};
boost::thread_group workers;
for(unsigned i = 0; i != thread_count-1; ++i)
workers.create_thread(worker);
worker();
workers.join_all();
}
}
size_t fastq_paired_two_files_for_each_parallel(string& file1, string& file2, function<void(Alignment&, Alignment&)> lambda) {
gzFile fp1 = (file1 != "-") ? gzopen(file1.c_str(), "r") : gzdopen(fileno(stdin), "r");
gzFile fp2 = (file2 != "-") ? gzopen(file2.c_str(), "r") : gzdopen(fileno(stdin), "r");
size_t len = 2 << 18; // 256k
size_t nLines = 0;
int thread_count = get_thread_count();
//vector<Alignment> alns; alns.resize(thread_count);
vector<char*> bufs; bufs.resize(thread_count);
for (auto& buf : bufs) {
buf = new char[len];
}
bool more_data = true;
#pragma omp parallel shared(fp1, fp2, more_data, bufs)
{
int tid = omp_get_thread_num();
while (more_data) {
Alignment mate1, mate2;
char* buf = bufs[tid];
bool got_anything = false;
#pragma omp critical (fastq_input)
{
if (more_data) {
got_anything = more_data = get_next_alignment_pair_from_fastqs(fp1, fp2, buf, len, mate1, mate2);
nLines++;
}
}
if (got_anything) {
lambda(mate1, mate2);
}
}
}
for (auto& buf : bufs) {
delete buf;
}
gzclose(fp1);
gzclose(fp2);
return nLines;
}
bool UDTPPeer::remove_thread()
{
if(get_thread_count() <= 0) return false;
UDTPThreadFlow *removeFlowThread = front_flow_thread();
UDTPThreadProcess *removeProcessThread = front_process_thread();
removeFlowThread->kill();
removeProcessThread->kill();
sem_post(&_semProcessChunkThread); /*Have to post one!*/
close(removeFlowThread->flow_socket()); /****FIND A WAY TO CLOSE BUT KEEP WHATEVER IS REMAINING INSIDE!*/
removeFlowThread = NULL;
removeProcessThread = NULL;
std::cout << "PEER " << get_unique_id() << " has removed a flow and process thread!" << std::endl;
decrement_thread_count();
pop_thread();
return true;
}
int hts_for_each_parallel(string& filename, function<void(Alignment&)> lambda) {
samFile *in = hts_open(filename.c_str(), "r");
if (in == NULL) return 0;
bam_hdr_t *hdr = sam_hdr_read(in);
map<string, string> rg_sample;
parse_rg_sample_map(hdr->text, rg_sample);
int thread_count = get_thread_count();
vector<bam1_t*> bs; bs.resize(thread_count);
for (auto& b : bs) {
b = bam_init1();
}
bool more_data = true;
#pragma omp parallel shared(in, hdr, more_data, rg_sample)
{
int tid = omp_get_thread_num();
while (more_data) {
bam1_t* b = bs[tid];
#pragma omp critical (hts_input)
if (more_data) {
more_data = sam_read1(in, hdr, b) >= 0;
}
if (more_data) {
Alignment a = bam_to_alignment(b, rg_sample);
lambda(a);
}
}
}
for (auto& b : bs) bam_destroy1(b);
bam_hdr_destroy(hdr);
hts_close(in);
return 1;
}
vector<edge_t> find_edges_to_prune(const HandleGraph& graph, size_t k, size_t edge_max) {
// for each position on the forward and reverse of the graph
//unordered_set<edge_t> edges_to_prune;
vector<vector<edge_t> > edges_to_prune;
edges_to_prune.resize(get_thread_count());
graph.for_each_handle([&](const handle_t& h) {
// for the forward and reverse of this handle
// walk k bases from the end, so that any kmer starting on the node will be represented in the tree we build
for (auto handle_is_rev : { false, true }) {
//cerr << "###########################################" << endl;
handle_t handle = handle_is_rev ? graph.flip(h) : h;
list<walk_t> walks;
// for each position in the node, set up a kmer with that start position and the node end or kmer length as the end position
// determine next positions
id_t handle_id = graph.get_id(handle);
size_t handle_length = graph.get_length(handle);
string handle_seq = graph.get_sequence(handle);
for (size_t i = 0; i < handle_length; ++i) {
pos_t begin = make_pos_t(handle_id, handle_is_rev, handle_length);
pos_t end = make_pos_t(handle_id, handle_is_rev, min(handle_length, i+k));
walk_t walk = walk_t(offset(end)-offset(begin), begin, end, handle, 0);
if (walk.length < k) {
// are we branching over more than one edge?
size_t next_count = 0;
graph.follow_edges(walk.curr, false, [&](const handle_t& next) { ++next_count; });
graph.follow_edges(walk.curr, false, [&](const handle_t& next) {
if (next_count > 1 && edge_max == walk.forks) { // our next step takes us over the max
int tid = omp_get_thread_num();
edges_to_prune[tid].push_back(graph.edge_handle(walk.curr, next));
} else {
walks.push_back(walk);
auto& todo = walks.back();
todo.curr = next;
if (next_count > 1) {
++todo.forks;
}
}
});
} else {
walks.push_back(walk);
}
}
// now expand the kmers until they reach k
while (!walks.empty()) {
// first we check which ones have reached length k in the current handle; for each of these we run lambda and remove them from our list
auto walks_end = walks.end();
for (list<walk_t>::iterator q = walks.begin(); q != walks_end; ++q) {
auto& walk = *q;
// did we reach our target length?
if (walk.length >= k) {
q = walks.erase(q);
} else {
id_t curr_id = graph.get_id(walk.curr);
size_t curr_length = graph.get_length(walk.curr);
bool curr_is_rev = graph.get_is_reverse(walk.curr);
size_t take = min(curr_length, k-walk.length);
walk.end = make_pos_t(curr_id, curr_is_rev, take);
walk.length += take;
if (walk.length < k) {
// if not, we need to expand through the node then follow on
size_t next_count = 0;
graph.follow_edges(walk.curr, false, [&](const handle_t& next) { ++next_count; });
graph.follow_edges(walk.curr, false, [&](const handle_t& next) {
if (next_count > 1 && edge_max == walk.forks) { // our next step takes us over the max
int tid = omp_get_thread_num();
edges_to_prune[tid].push_back(graph.edge_handle(walk.curr, next));
} else {
walks.push_back(walk);
auto& todo = walks.back();
todo.curr = next;
if (next_count > 1) {
++todo.forks;
}
}
});
q = walks.erase(q);
} else {
// nothing, we'll remove it next time around
}
}
}
}
}
}, true);
uint64_t total_edges = 0;
for (auto& v : edges_to_prune) total_edges += v.size();
vector<edge_t> merged; merged.reserve(total_edges);
for (auto& v : edges_to_prune) {
merged.insert(merged.end(), v.begin(), v.end());
}
// duplicates are assumed to be dealt with externally
return merged;
}
void * echo_server(void * arg) {
int status;
char buffer[MAX_BUFFER_LEN];
ServerTag * server_tag = arg;
int c_socket = server_tag->c_socket;
ssize_t num_read;
struct timeval time_out;
char * error_msg;
/* Free struct allocated by calling function before we do
anything that might cause us to exit() */
free(server_tag);
DINCREMENT_THREAD_COUNT();
DFPRINTF ((stderr, "Entering thread - number of active threads is %d.\n",
get_thread_count()));
/* Detach thread, the main server doesn't wait for it */
status = pthread_detach(pthread_self());
if ( status != 0 ) {
mk_errmsg("Error detaching thread", &error_msg);
fprintf(stderr, "%s\n", error_msg);
free(error_msg);
exit(EXIT_FAILURE);
}
/* Loop over input lines */
while ( 1 ) {
/* Note: Linux modifies the struct timeval on a call to
select(), so we need to reset it before each call. */
time_out.tv_sec = time_out_secs;
time_out.tv_usec = time_out_usecs;
num_read = socket_readline_timeout_r(c_socket, buffer,
MAX_BUFFER_LEN, &time_out, &error_msg);
if ( num_read < 0 ) {
fprintf(stderr, "%s\n", error_msg);
free(error_msg);
exit(EXIT_FAILURE);
}
if ( num_read == 0 ) {
/* We've timed out getting a line of input */
DFPRINTF ((stderr, "No input available.\n"));
if ( socket_writeline_r(c_socket, time_out_msg,
strlen(time_out_msg), &error_msg) < 0 ) {
fprintf(stderr, "%s\n", error_msg);
free(error_msg);
exit(EXIT_FAILURE);
}
break;
}
/* Echo the line of input */
DFPRINTF ((stderr, "Echoing input.\n"));
if ( socket_writeline_r(c_socket, buffer,
strlen(buffer), &error_msg) < 0 ) {
fprintf(stderr, "%s\n", error_msg);
free(error_msg);
exit(EXIT_FAILURE);
}
}
if ( close(c_socket) == - 1 ) {
mk_errmsg("Error closing socket", &error_msg);
fprintf(stderr, "%s\n", error_msg);
free(error_msg);
exit(EXIT_FAILURE);
}
DFPRINTF ((stderr, "Exiting from thread.\n"));
DDECREMENT_THREAD_COUNT();
return NULL;
}
