void assign_io_interrupt_to_cpu(long vector, int32 newCPU)
{
ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ);
int32 oldCPU = sVectors[vector].assigned_cpu->cpu;
if (newCPU == -1)
newCPU = assign_cpu();
if (newCPU == oldCPU)
return;
ASSERT(oldCPU != -1);
cpu_ent* cpu = &gCPU[oldCPU];
SpinLocker locker(cpu->irqs_lock);
sVectors[vector].assigned_cpu->cpu = -1;
list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu);
locker.Unlock();
cpu = &gCPU[newCPU];
locker.SetTo(cpu->irqs_lock, false);
sVectors[vector].assigned_cpu->cpu = newCPU;
arch_int_assign_to_cpu(vector, newCPU);
list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu);
}
void list_filter(ADTList list, int (*filter)(void *)) {
for (ADTListItem it = list_head(list), next; it; it = next) {
next = list_next(it);
if (filter(list_value(it)) == 0)
list_remove_item(list, it);
}
}
void progressive_display_add_rect(struct xrdp_screen * self)
{
int i, j;
struct list* update_rects = self->update_rects;
struct update_rect* urect;
struct update_rect* fu_rect;
struct update_rect* ur;
struct xrdp_rect intersection;
struct update_rect* tmp;
struct list* l_tmp = list_create();
l_tmp->auto_free = 1;
bool no_inter = true;
while (!fifo_is_empty(self->candidate_update_rects))
{
fu_rect = (struct update_rect*) fifo_pop(self->candidate_update_rects);
if (update_rects->count > 0)
{
no_inter = true;
for (i = update_rects->count - 1; i >= 0; i--)
{
urect = (struct update_rect*) list_get_item(update_rects, i);
if (!rect_equal(&urect->rect, &fu_rect->rect))
{
if (rect_intersect(&urect->rect, &fu_rect->rect, &intersection))
{
no_inter = false;
progressive_display_rect_union(fu_rect, urect, l_tmp);
list_remove_item(update_rects, i);
for (j = 0; j < l_tmp->count; j++)
{
ur = (struct update_rect*) list_get_item(l_tmp, j);
tmp = (struct update_rect*) g_malloc(sizeof(struct update_rect), 0);
g_memcpy(tmp, ur, sizeof(struct update_rect));
fifo_push(self->candidate_update_rects, tmp);
}
list_clear(l_tmp);
break;
}
}
else
{
no_inter = false;
urect->quality = fu_rect->quality;
urect->quality_already_send = fu_rect->quality_already_send;
break;
}
}
if (no_inter)
{
list_add_progressive_display_rect(update_rects, fu_rect->rect.left, fu_rect->rect.top, fu_rect->rect.right, fu_rect->rect.bottom, fu_rect->quality, fu_rect->quality_already_send);
}
}
else
{
list_add_progressive_display_rect(update_rects, fu_rect->rect.left, fu_rect->rect.top, fu_rect->rect.right, fu_rect->rect.bottom, fu_rect->quality, fu_rect->quality_already_send);
}
}
list_delete(l_tmp);
}
region_table_t *parse_regions_from_gff_file(char *filename, const char *url, const char *species, const char *version) {
gff_file_t *file = gff_open(filename);
if (file == NULL) {
return NULL;
}
region_table_t *regions_table = create_table(url, species, version);
int ret_code = 0;
size_t max_batches = 20;
size_t batch_size = 2000;
list_t *read_list = (list_t*) malloc (sizeof(list_t));
list_init("batches", 1, max_batches, read_list);
#pragma omp parallel sections
{
// The producer reads the GFF file
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the GFF file\n", omp_get_thread_num());
ret_code = gff_read_batches(read_list, batch_size, file);
list_decr_writers(read_list);
if (ret_code) {
LOG_FATAL_F("Error while reading GFF file %s (%d)\n", filename, ret_code);
}
}
// The consumer inserts regions in the structure
#pragma omp section
{
list_item_t *item = NULL, *batch_item = NULL;
gff_batch_t *batch;
gff_record_t *record;
while ( (item = list_remove_item(read_list)) != NULL ) {
batch = item->data_p;
// For each record in the batch, generate a new region
for (batch_item = batch->first_p; batch_item != NULL; batch_item = batch_item->next_p) {
record = batch_item->data_p;
region_t *region = (region_t*) malloc (sizeof(region_t));
region->chromosome = (char*) calloc ((strlen(record->sequence)+1), sizeof(char));
strncat(region->chromosome, record->sequence, strlen(record->sequence));
region->start_position = record->start;
region->end_position = record->end;
LOG_DEBUG_F("region '%s:%u-%u'\n", region->chromosome, region->start_position, region->end_position);
insert_region(region, regions_table);
}
gff_batch_free(item->data_p);
list_item_free(item);
}
}
}
gff_close(file, 0);
return regions_table;
}
snet_buffer*
snb_fetch(struct list* l, uint16 size)
{
snet_buffer* item = NULL;
snet_buffer* newitem = NULL;
if (!list_is_empty(l))
while ((item = (snet_buffer*)list_get_next_item(l, item)) != NULL) {
if (item->allocatedSize >= size) {
// This one is for us
break;
}
}
newitem = snb_attempt_reuse(item, size);
/* the resulting reused one is the same
* as we fetched? => remove it from list
*/
if (item == newitem) {
list_remove_item(l, item);
}
return newitem;
}
/* Broadcast a message to all active streams */
static bool actl_stream_broadcast_callback(struct stream *str,
struct str_broadcast_data *sbd)
{
switch (sbd->cmd)
{
case STREAM_PLAY:
case STREAM_PAUSE:
break;
case STREAM_STOP:
if (sbd->data != 0)
{
actl_lock();
list_remove_item(stream_mgr.actl, str);
list_add_item(stream_mgr.strl, str);
actl_unlock();
sbd->data = 0;
}
break;
default:
return false;
}
str_send_msg(str, sbd->cmd, sbd->data);
return true;
}
void *list_pop_random(ADTList list) {
ADTListItem item = list_random_item(list);
if (item) {
void *value = list_value(item);
list_remove_item(list, item);
return value;
}
return NULL;
}
/* Add a stream to the playback pool */
void stream_add_stream(struct stream *str)
{
actl_lock();
list_remove_item(stream_mgr.strl, str);
list_add_item(stream_mgr.strl, str);
actl_unlock();
}
vcf_batch_t *fetch_vcf_batch(vcf_file_t *file) {
assert(file);
list_item_t *item = list_remove_item(file->record_batches);
if (item) {
vcf_batch_t *batch = item->data_p;
list_item_free(item);
return batch;
}
return NULL;
}
/* Clear list items after the head item */
void list_clear_all(struct list_item *head_item)
{
struct list_item *curr = head_item->next;
while (curr != NULL)
{
list_remove_item(curr);
curr = head_item->next;
}
}
/*! Removes the interface from its domain, and deletes it.
You need to hold the domain's lock when calling this function.
*/
status_t
remove_interface_from_domain(net_interface* interface)
{
net_domain_private* domain = (net_domain_private*)interface->domain;
list_remove_item(&domain->interfaces, interface);
notify_interface_removed(interface);
delete_interface((net_interface_private*)interface);
return B_OK;
}
void write_output_body(enum ASSOC_task task, list_t* output_list, FILE *fd) {
assert(fd);
list_item_t* item = NULL;
if (task == CHI_SQUARE) {
while (item = list_remove_item(output_list)) {
assoc_basic_result_t *result = item->data_p;
double freq_a1 = (result->affected1 + result->affected2 > 0) ? (double) result->affected1 / (result->affected1 + result->affected2) : 0.0f;
double freq_u1 = (result->unaffected1 + result->unaffected2 > 0) ? (double) result->unaffected1 / (result->unaffected1 + result->unaffected2) : 0.0f;
double freq_a2 = (result->affected1 + result->affected2 > 0) ? (double) result->affected2 / (result->affected1 + result->affected2) : 0.0f;
double freq_u2 = (result->unaffected1 + result->unaffected2 > 0) ? (double) result->unaffected2 / (result->unaffected1 + result->unaffected2) : 0.0f;
fprintf(fd, "%s\t%8ld\t%s\t%s\t%3d\t%3d\t%6f\t%6f\t%s\t%3d\t%3d\t%6f\t%6f\t%6f\t%6f\t%6f\n",
result->chromosome, result->position, result->id,
result->reference, result->affected1, result->unaffected1, freq_a1, freq_u1,
result->alternate, result->affected2, result->unaffected2, freq_a2, freq_u2,
result->odds_ratio, result->chi_square, result->p_value);
assoc_basic_result_free(result);
list_item_free(item);
}
} else if (task == FISHER) {
while (item = list_remove_item(output_list)) {
assoc_fisher_result_t *result = item->data_p;
double freq_a1 = (result->affected1 + result->affected2 > 0) ? (double) result->affected1 / (result->affected1 + result->affected2) : 0.0f;
double freq_u1 = (result->unaffected1 + result->unaffected2 > 0) ? (double) result->unaffected1 / (result->unaffected1 + result->unaffected2) : 0.0f;
double freq_a2 = (result->affected1 + result->affected2 > 0) ? (double) result->affected2 / (result->affected1 + result->affected2) : 0.0f;
double freq_u2 = (result->unaffected1 + result->unaffected2 > 0) ? (double) result->unaffected2 / (result->unaffected1 + result->unaffected2) : 0.0f;
fprintf(fd, "%s\t%8ld\t%s\t%s\t%3d\t%3d\t%6f\t%6f\t%s\t%3d\t%3d\t%6f\t%6f\t%6f\t%6f\n",
result->chromosome, result->position, result->id,
result->reference, result->affected1, result->unaffected1, freq_a1, freq_u1,
result->alternate, result->affected2, result->unaffected2, freq_a2, freq_u2,
result->odds_ratio, result->p_value);
assoc_fisher_result_free(result);
list_item_free(item);
}
}
}
void list_remove(list_t *list, void *data) {
list_item_t *item;
// Find the item
for (item = list_first(list); item != NULL; item = item->next) {
if (item->data == data) {
list_remove_item(list, item);
break;
}
}
}
/* Callback for various list-moving operations */
static bool strl_enum_callback(struct stream *str, intptr_t data)
{
actl_lock();
list_remove_item(stream_mgr.strl, str);
if (data == 1)
list_add_item(stream_mgr.actl, str);
actl_unlock();
return true;
}
void precalculate_aux_values_for_annotation(int calculate_stats, int calculate_dp, int calculate_mq, vcf_record_t *record,
variant_stats_t **variant_stats, file_stats_t *file_stats, list_t *stats_list,
int *dp, int *mq0, double *mq) {
// Variant statistics
if (calculate_stats) {
get_variants_stats(&record, 1, NULL, NULL,0, stats_list, file_stats);
list_item_t *item = list_remove_item(stats_list);
*variant_stats = item->data_p;
list_item_free(item);
}
// Read depth
if (calculate_dp) {
int dp_pos = -1;
*dp = 0;
for (int j = 0; j < record->samples->size; j++) {
char *aux = strndup(record->format, record->format_len);
dp_pos = get_field_position_in_format("DP", aux);
free(aux);
if (dp_pos >= 0) {
aux = strdup((char*) array_list_get(j, record->samples));
*dp += atoi(get_field_value_in_sample(aux, dp_pos));
free(aux);
}
}
}
// Mapping quality
if (calculate_mq) {
for (int j = 0; j < record->samples->size; j++) {
char *aux = strndup(record->format, record->format_len);
int mq_pos = get_field_position_in_format("GQ", aux);
free(aux);
if (mq_pos < 0) {
continue;
}
aux = strdup((char*) array_list_get(j, record->samples));
int cur_gq = atoi(get_field_value_in_sample(aux, mq_pos));
free(aux);
// printf("sample = %s\tmq_pos = %d\tvalue = %d\n", array_list_get(j, record->samples), mq_pos,
// atoi(get_field_value_in_sample(strdup((char*) array_list_get(j, record->samples)), mq_pos)));
if (cur_gq == 0) {
(*mq0)++;
} else {
*mq += cur_gq * cur_gq;
}
}
*mq = sqrt(*mq / record->samples->size);
}
}
/*!
The socket has been connected. It will be moved to the connected queue
of its parent socket.
*/
status_t
socket_connected(net_socket *socket)
{
net_socket_private *parent = (net_socket_private *)socket->parent;
if (parent == NULL)
return B_BAD_VALUE;
mutex_lock(&parent->lock);
list_remove_item(&parent->pending_children, socket);
list_add_item(&parent->connected_children, socket);
mutex_unlock(&parent->lock);
return B_OK;
}
int
_callout_stop_safe(struct callout *c, int safe)
{
MutexLocker locker(sLock);
TRACE("_callout_stop_safe %p, func %p, arg %p\n", c, c->c_func, c->c_arg);
if (c->due <= 0)
return 0;
// this timer is scheduled, cancel it
list_remove_item(&sTimers, c);
c->due = 0;
return 1;
}
/* Remove a stream from the active list and return it to the pool */
static bool actl_stream_remove(struct stream *str)
{
bool retval;
actl_lock();
retval = list_remove_item(stream_mgr.actl, str);
if (retval)
list_add_item(stream_mgr.strl, str);
actl_unlock();
return retval;
}
void topic_remove_connection(topic_t* t, connection_t* con)
{
comms_disconnect(con->socketfd);
list_item_t* item = t->conns->begin;
while(item != NULL)
{
if(item->data == con)
{
connection_free(con);
list_remove_item(t->conns, item);
return;
}
item = item->next;
}
}
void progressive_display_add_update_order(struct xrdp_screen* self, struct list* p_display, struct list* update_list)
{
struct ip_image* desktop = self->screen;
int i;
update* up;
int bpp = (self->bpp + 7) / 8;
if (bpp == 3)
{
bpp = 4;
}
if (p_display->count > 0)
{
for (i = p_display->count - 1; i >= 0; i--)
{
struct update_rect* cur = (struct update_rect*) list_get_item(p_display, i);
up = (update*) g_malloc(sizeof(update), 1);
up->order_type = paint_rect;
up->x = cur->rect.left;
up->y = cur->rect.top;
int w = rect_width(&cur->rect);
int h = rect_height(&cur->rect);
up->cx = w;
up->cy = h;
up->width = w;
up->height = h;
up->srcx = 0;
up->srcy = 0;
up->quality = cur->quality;
up->data_len = up->cx * up->cy * bpp;
up->data = g_malloc(up->data_len, 0);
ip_image_crop(desktop, up->x, up->y, up->cx, up->cy, up->data);
list_add_item(update_list, (tbus) up);
if (cur->quality != 0)
{
struct update_rect* tmp = (struct update_rect*) g_malloc(sizeof(struct update_rect), 0);
g_memcpy(tmp, cur, sizeof(struct update_rect));
tmp->quality = 0;
tmp->quality_already_send = cur->quality;
fifo_push(self->candidate_update_rects, tmp);
}
list_remove_item(p_display, i);
}
}
}
static int APP_CC
xrdp_listen_delete_done_pro(struct xrdp_listen* self)
{
int i;
struct xrdp_process* pro;
for (i = self->process_list->count - 1; i >= 0; i--)
{
pro = (struct xrdp_process*)list_get_item(self->process_list, i);
if (pro != 0)
{
if (pro->status < 0)
{
xrdp_process_delete(pro);
list_remove_item(self->process_list, i);
}
}
}
return 0;
}
static void
guarded_heap_page_allocate(guarded_heap_area& area, size_t startPageIndex,
size_t pagesNeeded, size_t allocationSize, size_t alignment,
void* allocationBase)
{
if (pagesNeeded < 2) {
panic("need to allocate at least 2 pages, one for guard\n");
return;
}
guarded_heap_page* firstPage = NULL;
for (size_t i = 0; i < pagesNeeded; i++) {
guarded_heap_page& page = area.pages[startPageIndex + i];
page.flags = GUARDED_HEAP_PAGE_FLAG_USED;
if (i == 0) {
page.thread = find_thread(NULL);
page.allocation_size = allocationSize;
page.allocation_base = allocationBase;
page.alignment = alignment;
page.flags |= GUARDED_HEAP_PAGE_FLAG_FIRST;
firstPage = &page;
} else {
page.thread = firstPage->thread;
page.allocation_size = allocationSize;
page.allocation_base = allocationBase;
page.alignment = alignment;
}
list_remove_item(&area.free_list, &page);
if (i == pagesNeeded - 1) {
page.flags |= GUARDED_HEAP_PAGE_FLAG_GUARD;
guarded_heap_page_protect(area, startPageIndex + i, 0);
} else {
guarded_heap_page_protect(area, startPageIndex + i,
B_READ_AREA | B_WRITE_AREA);
}
}
}
int
thread_pool_pop_job(THREAD_POOL* pool)
{
int job = 0;
thread_pool_acquire_job(pool);
if (pool->job_queue->count == 0)
{
thread_pool_release_job(pool);
return 0;
}
job = list_get_item(pool->job_queue, 0);
list_remove_item(pool->job_queue, 0);
if (pool->job_queue->count == 0)
{
thread_pool_lock_thread(pool);
}
thread_pool_release_job(pool);
return job;
}
int APP_CC
xrdp_region_subtract_rect(struct xrdp_region* self,
struct xrdp_rect* rect)
{
struct xrdp_rect* r;
struct xrdp_rect rect1;
int i;
for (i = self->rects->count - 1; i >= 0; i--)
{
r = (struct xrdp_rect*)list_get_item(self->rects, i);
rect1 = *r;
r = &rect1;
if (rect->left <= r->left &&
rect->top <= r->top &&
rect->right >= r->right &&
rect->bottom >= r->bottom)
{ /* rect is not visible */
list_remove_item(self->rects, i);
}
else if (rect->right < r->left ||
rect->bottom < r->top ||
rect->top > r->bottom ||
rect->left > r->right)
{ /* rect are not related */
}
else if (rect->left <= r->left &&
rect->right >= r->right &&
rect->bottom < r->bottom &&
rect->top <= r->top)
{ /* partially covered(whole top) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, rect->bottom,
r->right, r->bottom);
}
else if (rect->top <= r->top &&
rect->bottom >= r->bottom &&
rect->right < r->right &&
rect->left <= r->left)
{ /* partially covered(left) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, rect->right, r->top,
r->right, r->bottom);
}
else if (rect->left <= r->left &&
rect->right >= r->right &&
rect->top > r->top &&
rect->bottom >= r->bottom)
{ /* partially covered(bottom) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
}
else if (rect->top <= r->top &&
rect->bottom >= r->bottom &&
rect->left > r->left &&
rect->right >= r->right)
{ /* partially covered(right) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
rect->left, r->bottom);
}
else if (rect->left <= r->left &&
rect->top <= r->top &&
rect->right < r->right &&
rect->bottom < r->bottom)
{ /* partially covered(top left) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, rect->right, r->top,
r->right, rect->bottom);
xrdp_region_insert_rect(self, i, r->left, rect->bottom,
r->right, r->bottom);
}
else if (rect->left <= r->left &&
rect->bottom >= r->bottom &&
rect->right < r->right &&
rect->top > r->top)
{ /* partially covered(bottom left) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
xrdp_region_insert_rect(self, i, rect->right, rect->top,
r->right, r->bottom);
}
else if (rect->left > r->left &&
rect->right >= r->right &&
rect->top <= r->top &&
rect->bottom < r->bottom)
{ /* partially covered(top right) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
rect->left, r->bottom);
xrdp_region_insert_rect(self, i, rect->left, rect->bottom,
r->right, r->bottom);
}
else if (rect->left > r->left &&
rect->right >= r->right &&
rect->top > r->top &&
rect->bottom >= r->bottom)
{ /* partially covered(bottom right) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
xrdp_region_insert_rect(self, i, r->left, rect->top,
rect->left, r->bottom);
}
else if (rect->left > r->left &&
rect->top <= r->top &&
rect->right < r->right &&
rect->bottom >= r->bottom)
{ /* 2 rects, one on each end */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
rect->left, r->bottom);
xrdp_region_insert_rect(self, i, rect->right, r->top,
r->right, r->bottom);
}
else if (rect->left <= r->left &&
rect->top > r->top &&
rect->right >= r->right &&
rect->bottom < r->bottom)
{ /* 2 rects, one on each end */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
xrdp_region_insert_rect(self, i, r->left, rect->bottom,
r->right, r->bottom);
}
else if (rect->left > r->left &&
rect->right < r->right &&
rect->top <= r->top &&
rect->bottom < r->bottom)
{ /* partially covered(top) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
rect->left, r->bottom);
xrdp_region_insert_rect(self, i, rect->left, rect->bottom,
rect->right, r->bottom);
xrdp_region_insert_rect(self, i, rect->right, r->top,
r->right, r->bottom);
}
else if (rect->top > r->top &&
rect->bottom < r->bottom &&
rect->left <= r->left &&
rect->right < r->right)
{ /* partially covered(left) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
xrdp_region_insert_rect(self, i, rect->right, rect->top,
r->right, rect->bottom);
xrdp_region_insert_rect(self, i, r->left, rect->bottom,
r->right, r->bottom);
}
else if (rect->left > r->left &&
rect->right < r->right &&
rect->bottom >= r->bottom &&
rect->top > r->top)
{ /* partially covered(bottom) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
rect->left, r->bottom);
xrdp_region_insert_rect(self, i, rect->left, r->top,
rect->right, rect->top);
xrdp_region_insert_rect(self, i, rect->right, r->top,
r->right, r->bottom);
}
else if (rect->top > r->top &&
rect->bottom < r->bottom &&
rect->right >= r->right &&
rect->left > r->left)
{ /* partially covered(right) */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
xrdp_region_insert_rect(self, i, r->left, rect->top,
rect->left, rect->bottom);
xrdp_region_insert_rect(self, i, r->left, rect->bottom,
r->right, r->bottom);
}
else if (rect->left > r->left &&
rect->top > r->top &&
rect->right < r->right &&
rect->bottom < r->bottom)
{ /* totally contained, 4 rects */
list_remove_item(self->rects, i);
xrdp_region_insert_rect(self, i, r->left, r->top,
r->right, rect->top);
xrdp_region_insert_rect(self, i, r->left, rect->top,
rect->left, rect->bottom);
xrdp_region_insert_rect(self, i, r->left, rect->bottom,
r->right, r->bottom);
xrdp_region_insert_rect(self, i, rect->right, rect->top,
r->right, rect->bottom);
}
else
{
g_writeln("error in xrdp_region_subtract_rect");
}
}
return 0;
}
region_table_t *parse_regions_from_gff_file(char *filename, const char *url, const char *species, const char *version) {
gff_file_t *file = gff_open(filename);
if (file == NULL) {
return NULL;
}
region_table_t *regions_table = new_region_table_from_ws(url, species, version);
int ret_code = 0;
size_t max_batches = 20, batch_size = 2000;
list_t *read_list = (list_t*) malloc (sizeof(list_t));
list_init("batches", 1, max_batches, read_list);
#pragma omp parallel sections
{
// The producer reads the GFF file
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the GFF file\n", omp_get_thread_num());
ret_code = gff_read_batches(read_list, batch_size, file);
list_decr_writers(read_list);
if (ret_code) {
LOG_FATAL_F("Error while reading GFF file %s (%d)\n", filename, ret_code);
}
}
// The consumer inserts regions in the structure
#pragma omp section
{
list_item_t *item = NULL;
gff_batch_t *batch;
gff_record_t *record;
region_t *regions_batch[REGIONS_CHUNKSIZE];
int avail_regions = 0;
while ( item = list_remove_item(read_list) ) {
batch = item->data_p;
// For each record in the batch, generate a new region
for (int i = 0; i < batch->records->size; i++) {
record = batch->records->items[i];
region_t *region = region_new(strndup(record->sequence, record->sequence_len),
record->start, record->end,
record->strand ? strndup(&record->strand, 1) : NULL,
record->feature ? strndup(record->feature, record->feature_len) : NULL);
LOG_DEBUG_F("region '%s:%u-%u'\n", region->chromosome, region->start_position, region->end_position);
regions_batch[avail_regions++] = region;
// Save when the recommended size is reached
if (avail_regions == REGIONS_CHUNKSIZE) {
insert_regions(regions_batch, avail_regions, regions_table);
for (int i = 0; i < avail_regions; i++) {
free(regions_batch[i]);
}
avail_regions = 0;
}
}
gff_batch_free(batch);
list_item_free(item);
}
// Save the remaining regions that did not fill a batch
if (avail_regions > 0) {
insert_regions(regions_batch, avail_regions, regions_table);
for (int i = 0; i < avail_regions; i++) {
free(regions_batch[i]);
}
avail_regions = 0;
}
}
}
finish_region_table_loading(regions_table);
list_free_deep(read_list, NULL);
gff_close(file, 1);
return regions_table;
}
int run_effect(char **urls, shared_options_data_t *shared_options_data, effect_options_data_t *options_data) {
int ret_code = 0;
double start, stop, total;
vcf_file_t *vcf_file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
if (!vcf_file) {
LOG_FATAL("VCF file does not exist!\n");
}
ped_file_t *ped_file = NULL;
if (shared_options_data->ped_filename) {
ped_file = ped_open(shared_options_data->ped_filename);
if (!ped_file) {
LOG_FATAL("PED file does not exist!\n");
}
LOG_INFO("About to read PED file...\n");
// Read PED file before doing any processing
ret_code = ped_read(ped_file);
if (ret_code != 0) {
LOG_FATAL_F("Can't read PED file: %s\n", ped_file->filename);
}
}
char *output_directory = shared_options_data->output_directory;
size_t output_directory_len = strlen(output_directory);
ret_code = create_directory(output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", output_directory);
}
// Remove all .txt files in folder
ret_code = delete_files_by_extension(output_directory, "txt");
if (ret_code != 0) {
return ret_code;
}
// Initialize environment for connecting to the web service
ret_code = init_http_environment(0);
if (ret_code != 0) {
return ret_code;
}
// Output file descriptors
static cp_hashtable *output_files = NULL;
// Lines of the output data in the main .txt files
static list_t *output_list = NULL;
// Consequence type counters (for summary, must be kept between web service calls)
static cp_hashtable *summary_count = NULL;
// Gene list (for genes-with-variants, must be kept between web service calls)
static cp_hashtable *gene_list = NULL;
// Initialize collections of file descriptors and summary counters
ret_code = initialize_output_files(output_directory, output_directory_len, &output_files);
if (ret_code != 0) {
return ret_code;
}
initialize_output_data_structures(shared_options_data, &output_list, &summary_count, &gene_list);
initialize_ws_buffers(shared_options_data->num_threads);
// Create job.status file
char job_status_filename[output_directory_len + 10];
sprintf(job_status_filename, "%s/job.status", output_directory);
FILE *job_status = new_job_status_file(job_status_filename);
if (!job_status) {
LOG_FATAL("Can't create job status file\n");
} else {
update_job_status_file(0, job_status);
}
#pragma omp parallel sections private(start, stop, total)
{
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
start = omp_get_wtime();
ret_code = vcf_read(vcf_file, 1,
(shared_options_data->batch_bytes > 0) ? shared_options_data->batch_bytes : shared_options_data->batch_lines,
shared_options_data->batch_bytes <= 0);
stop = omp_get_wtime();
total = stop - start;
if (ret_code) {
LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, vcf_file->filename);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
notify_end_parsing(vcf_file);
}
#pragma omp section
{
// Enable nested parallelism and set the number of threads the user has chosen
omp_set_nested(1);
LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num());
// Filters and files for filtering output
filter_t **filters = NULL;
int num_filters = 0;
if (shared_options_data->chain != NULL) {
filters = sort_filter_chain(shared_options_data->chain, &num_filters);
}
FILE *passed_file = NULL, *failed_file = NULL, *non_processed_file = NULL;
get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
// Pedigree information (used in some filters)
individual_t **individuals = NULL;
khash_t(ids) *sample_ids = NULL;
// Filename structure outdir/vcfname.errors
char *prefix_filename = calloc(strlen(shared_options_data->vcf_filename), sizeof(char));
get_filename_from_path(shared_options_data->vcf_filename, prefix_filename);
char *non_processed_filename = malloc((strlen(shared_options_data->output_directory) + strlen(prefix_filename) + 9) * sizeof(char));
sprintf(non_processed_filename, "%s/%s.errors", shared_options_data->output_directory, prefix_filename);
non_processed_file = fopen(non_processed_filename, "w");
free(non_processed_filename);
// Maximum size processed by each thread (never allow more than 1000 variants per query)
if (shared_options_data->batch_lines > 0) {
shared_options_data->entries_per_thread = MIN(MAX_VARIANTS_PER_QUERY,
ceil((float) shared_options_data->batch_lines / shared_options_data->num_threads));
} else {
shared_options_data->entries_per_thread = MAX_VARIANTS_PER_QUERY;
}
LOG_DEBUG_F("entries-per-thread = %d\n", shared_options_data->entries_per_thread);
int i = 0;
vcf_batch_t *batch = NULL;
int ret_ws_0 = 0, ret_ws_1 = 0, ret_ws_2 = 0;
start = omp_get_wtime();
while (batch = fetch_vcf_batch(vcf_file)) {
if (i == 0) {
// Add headers associated to the defined filters
vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
for (int j = 0; j < num_filters; j++) {
add_vcf_header_entry(filter_headers[j], vcf_file);
}
// Write file format, header entries and delimiter
if (passed_file != NULL) { write_vcf_header(vcf_file, passed_file); }
if (failed_file != NULL) { write_vcf_header(vcf_file, failed_file); }
if (non_processed_file != NULL) { write_vcf_header(vcf_file, non_processed_file); }
LOG_DEBUG("VCF header written\n");
if (ped_file) {
// Create map to associate the position of individuals in the list of samples defined in the VCF file
sample_ids = associate_samples_and_positions(vcf_file);
// Sort individuals in PED as defined in the VCF file
individuals = sort_individuals(vcf_file, ped_file);
}
}
// printf("batch loaded = '%.*s'\n", 50, batch->text);
// printf("batch text len = %zu\n", strlen(batch->text));
// if (i % 10 == 0) {
LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n",
i, omp_get_thread_num(),
batch->records->size, batch->records->capacity);
// }
int reconnections = 0;
int max_reconnections = 3; // TODO allow to configure?
// Write records that passed to a separate file, and query the WS with them as args
array_list_t *failed_records = NULL;
int num_variables = ped_file? get_num_variables(ped_file): 0;
array_list_t *passed_records = filter_records(filters, num_filters, individuals, sample_ids, num_variables, batch->records, &failed_records);
if (passed_records->size > 0) {
// Divide the list of passed records in ranges of size defined in config file
int num_chunks;
int *chunk_sizes;
int *chunk_starts = create_chunks(passed_records->size, shared_options_data->entries_per_thread, &num_chunks, &chunk_sizes);
do {
// OpenMP: Launch a thread for each range
#pragma omp parallel for num_threads(shared_options_data->num_threads)
for (int j = 0; j < num_chunks; j++) {
int tid = omp_get_thread_num();
LOG_DEBUG_F("[%d] WS invocation\n", tid);
LOG_DEBUG_F("[%d] -- effect WS\n", tid);
if (!reconnections || ret_ws_0) {
ret_ws_0 = invoke_effect_ws(urls[0], (vcf_record_t**) (passed_records->items + chunk_starts[j]),
chunk_sizes[j], options_data->excludes);
parse_effect_response(tid, output_directory, output_directory_len, output_files, output_list, summary_count, gene_list);
free(effect_line[tid]);
effect_line[tid] = (char*) calloc (max_line_size[tid], sizeof(char));
}
if (!options_data->no_phenotypes) {
if (!reconnections || ret_ws_1) {
LOG_DEBUG_F("[%d] -- snp WS\n", omp_get_thread_num());
ret_ws_1 = invoke_snp_phenotype_ws(urls[1], (vcf_record_t**) (passed_records->items + chunk_starts[j]), chunk_sizes[j]);
parse_snp_phenotype_response(tid, output_list);
free(snp_line[tid]);
snp_line[tid] = (char*) calloc (snp_max_line_size[tid], sizeof(char));
}
if (!reconnections || ret_ws_2) {
LOG_DEBUG_F("[%d] -- mutation WS\n", omp_get_thread_num());
ret_ws_2 = invoke_mutation_phenotype_ws(urls[2], (vcf_record_t**) (passed_records->items + chunk_starts[j]), chunk_sizes[j]);
parse_mutation_phenotype_response(tid, output_list);
free(mutation_line[tid]);
mutation_line[tid] = (char*) calloc (mutation_max_line_size[tid], sizeof(char));
}
}
}
LOG_DEBUG_F("*** %dth web services invocation finished\n", i);
if (ret_ws_0 || ret_ws_1 || ret_ws_2) {
if (ret_ws_0) {
LOG_ERROR_F("Effect web service error: %s\n", get_last_http_error(ret_ws_0));
}
if (ret_ws_1) {
LOG_ERROR_F("SNP phenotype web service error: %s\n", get_last_http_error(ret_ws_1));
}
if (ret_ws_2) {
LOG_ERROR_F("Mutations phenotype web service error: %s\n", get_last_http_error(ret_ws_2));
}
// In presence of errors, wait 4 seconds before retrying
reconnections++;
LOG_ERROR_F("Some errors ocurred, reconnection #%d\n", reconnections);
sleep(4);
} else {
free(chunk_starts);
free(chunk_sizes);
}
} while (reconnections < max_reconnections && (ret_ws_0 || ret_ws_1 || ret_ws_2));
}
// If the maximum number of reconnections was reached still with errors,
// write the non-processed batch to the corresponding file
if (reconnections == max_reconnections && (ret_ws_0 || ret_ws_1 || ret_ws_2)) {
#pragma omp critical
{
write_vcf_batch(batch, non_processed_file);
}
}
// Write records that passed and failed filters to separate files, and free them
write_filtering_output_files(passed_records, failed_records, passed_file, failed_file);
free_filtered_records(passed_records, failed_records, batch->records);
// Free batch and its contents
vcf_batch_free(batch);
i++;
}
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
// Free resources
if (passed_file) { fclose(passed_file); }
if (failed_file) { fclose(failed_file); }
if (non_processed_file) { fclose(non_processed_file); }
// Free filters
for (i = 0; i < num_filters; i++) {
filter_t *filter = filters[i];
filter->free_func(filter);
}
free(filters);
// Decrease list writers count
for (i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
}
#pragma omp section
{
// Thread which writes the results to all_variants, summary and one file per consequence type
int ret = 0;
char *line;
list_item_t* item = NULL;
FILE *fd = NULL;
FILE *all_variants_file = cp_hashtable_get(output_files, "all_variants");
FILE *snp_phenotype_file = cp_hashtable_get(output_files, "snp_phenotypes");
FILE *mutation_phenotype_file = cp_hashtable_get(output_files, "mutation_phenotypes");
while ((item = list_remove_item(output_list)) != NULL) {
line = item->data_p;
// Type greater than 0: consequence type identified by its SO code
// Type equals to -1: SNP phenotype
// Type equals to -2: mutation phenotype
if (item->type > 0) {
// Write entry in the consequence type file
fd = cp_hashtable_get(output_files, &(item->type));
int ret = fprintf(fd, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to file: '%s'\n", line);
}
// Write in all_variants
ret = fprintf(all_variants_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to all_variants: '%s'\n", line);
}
} else if (item->type == SNP_PHENOTYPE) {
ret = fprintf(snp_phenotype_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to snp_phenotypes: '%s'\n", line);
}
} else if (item->type == MUTATION_PHENOTYPE) {
ret = fprintf(mutation_phenotype_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to mutation_phenotypes: '%s'\n", line);
}
}
free(line);
list_item_free(item);
}
}
}
write_summary_file(summary_count, cp_hashtable_get(output_files, "summary"));
write_genes_with_variants_file(gene_list, output_directory);
write_result_file(shared_options_data, options_data, summary_count, output_directory);
free_output_data_structures(output_files, summary_count, gene_list);
free_ws_buffers(shared_options_data->num_threads);
free(output_list);
vcf_close(vcf_file);
update_job_status_file(100, job_status);
close_job_status_file(job_status);
return ret_code;
}
void list_remove_full_item(ADTList list, ADTListItem item, void (*release)(void *)) {
release(list_value(item));
list_remove_item(list, item);
}
void list_remove_at(ADTList list, int index) {
ADTListItem item = list_item_at(list, index);
list_remove_item(list, item);
}
void list_remove(ADTList list, void *value) {
ADTListItem item = list_find(list, value);
if (item)
list_remove_item(list, item);
}
void list_remove_cmp(ADTList list, int (*cmp)(void *, void *), void *value) {
ADTListItem item = list_find_cmp(list, cmp, value);
if (item)
list_remove_item(list, item);
}