size_t pack_version_msg(uint8_t *buf, size_t size, const uint64_t nonce) { const struct in6_addr in6_loop = IN6ADDR_LOOPBACK_INIT; const size_t my_user_agent_len = NELEMS(USER_AGENT) - 1; const time_t now = time(NULL); size_t len; size_t off; len = off = HDR_SIZE; PACKLE32(PROTOCOL_VERSION); PACKLE64(0); PACKLE64(now); PACKLE64(NODE_NETWORK); PACKN(&in6_loop, sizeof(in6_loop)); PACKBE16(MAINNET_PORT); PACKLE64(0); PACKN(&in6_loop, sizeof(in6_loop)); PACKBE16(MAINNET_PORT); PACKLE64(nonce); PACKCINT(my_user_agent_len); PACKN(USER_AGENT, my_user_agent_len); PACKLE32(0); PACK8(0); if (off == len) { (void)pack_header(buf, size, "version", buf + HDR_SIZE, len - HDR_SIZE); } return len; }
void MpegDemultiplexer::pack() { ASSERT(m_file->m_next_code == 0x01ba); pack_header(); if (m_mpeg1) // mpeg1 { // m_file->next_start_code(); while (m_file->m_next_code >= 0x1bc && m_file->m_next_code <= 0x1ff) { packet_mpeg1(); // m_file->next_start_code(); } } else // mpeg2 { // pack_header(); // m_file->next_start_code(); while (m_file->m_next_code != 0x1ba && m_file->m_next_code != 0x1b9) { PES_packet(); } #if 0 while (m_file->m_next_code == 0x1bc/* && m_file->m_next_code <= 0x1ff*/) { program_stream_map_mpeg2(); } #endif } }
char *pack_response(dns_packet_t *packet, size_t *rlen) { size_t header_len = sizeof(dns_header_t); size_t question_len = sizeof(dns_question_t); size_t answer_len = sizeof(dns_record_t); char *buf = malloc( header_len + \ packet->header.qdcount * question_len + \ packet->header.ancount * answer_len); char *start = buf; memcpy(buf, pack_header(&packet->header), header_len); buf += header_len; for (int i = 0; i < packet->header.qdcount; i++) { size_t question_len; char *question = pack_question(&packet->questions[i], &question_len); memcpy(buf, question, question_len); buf += question_len; free(question); } for (int i = 0; i < packet->header.ancount; i++) { size_t record_len; char *record = pack_record(&packet->answers[i], &record_len); memcpy(buf, record, record_len); buf += record_len; free(record); } *rlen = buf - start; return start; }
char *ddb_cons_finalize(struct ddb_cons *cons, uint64_t *length, uint64_t flags) { struct ddb_packed *pack = NULL; struct ddb_entry *order = NULL; char *buf = NULL; int disable_compression, err = 1; DDB_TIMER_DEF if (!(pack = buffer_init())) goto err; if (pack_header(pack, cons)) goto err; #ifdef DDB_PROFILE print_mem_usage(cons); #endif DDB_TIMER_START pack->head->hash_offs = pack->offs; if (!(order = pack_hash(pack, cons->keys_map))) goto err; DDB_TIMER_END("hash") DDB_TIMER_START pack->head->key2values_offs = pack->offs; if (pack_key2values(pack, order, cons->keys_map, flags & DDB_OPT_UNIQUE_ITEMS)) goto err; DDB_TIMER_END("key2values") DDB_TIMER_START flags |= maybe_disable_compression(cons); disable_compression = flags & DDB_OPT_DISABLE_COMPRESSION; pack->head->id2value_offs = pack->offs; if (pack_id2value(pack, cons->values_map, disable_compression)) goto err; else{ ddb_map_free(cons->values_map); cons->values_map = NULL; } DDB_TIMER_END("id2values") if (!disable_compression){ DDB_TIMER_START pack->head->codebook_offs = pack->offs; if (pack_codebook(pack)) goto err; DDB_TIMER_END("save_codebook") }
/* this function sends a chunk of data down the next appropriate fd and a corresponding chunk header down the control connection */ void send_chunk(FD_ARRAY *fdstate, uint8_t *data, size_t len) { CHUNK_HDR ourheader; PACKED_CHUNK packedchunk; int fd; assert(fdstate -> numfds > 0); /* make sure we have FDs ready */ fd = fdstate->fds[fdstate->nextidx]; /* which FD we use */ /* we prepare the chunk header */ ourheader.index = fdstate->indices[fdstate->nextidx]; ourheader.begin_off = fdstate->bytes[fdstate->nextidx]; ourheader.end_off = ourheader.begin_off + len; ourheader.seq = fdstate->nextseq; /* and allocate space for the packed version of it */ packedchunk.data = malloc(CHUNK_HDR_LEN); assert(packedchunk.data != NULL); /* and we pack it */ pack_header(&ourheader, &packedchunk); /* and we send it down the control connection */ if(send_all(fdstate->controlfd, packedchunk.data, packedchunk.len, 0) == -1) { perror("send error in control connection"); exit(-1); } /* it's sent, so we don't need its packed version */ free(packedchunk.data); /* ...and we send the corresponding data */ if(send_all(fd, data, len, 0) == -1) { perror("send error in data connection"); exit(-1); } /* here we update fdstate */ fdstate->bytes[fdstate->nextidx] += len; /* how many bytes we sent down that fd*/ /* we update nextidx while making sure it doesn't get too big */ fdstate->nextidx++; if (fdstate->nextidx > (fdstate->numfds - 1)) { fdstate->nextidx = 0; } /* increment the sequence number */ fdstate->nextseq++; }
void encode_vehicle_authorization_reply( uint8_t src_id, uint8_t dest_id, uint8_t msg_ttl, uint8_t seq_number, vehicle_authorization_reply_t* tx_msg, proto_msg_t* msg) { msg_offset start_offset, offset; msg->direction = Proto_Out; start_offset = msg->data; offset = msg->data; msg->header.node_src_id = src_id; msg->header.node_dest_id = dest_id; msg->header.message_ttl = msg_ttl; msg->header.sequence_number = seq_number; msg->header.message_length = 13; msg->header.message_type = Proto_Vehicle_Authorization_Reply; offset = pack_sync(offset); offset = pack_header(&msg->header, offset); offset = pack_vehicle_authorization_reply(tx_msg, offset); offset = pack_checksum(start_offset, offset); msg->tx_len = offset-start_offset; }
void encode_vehicle_termination_command( uint8_t src_id, uint8_t dest_id, uint8_t msg_ttl, uint8_t seq_number, vehicle_termination_command_t* tx_msg, proto_msg_t* msg) { msg_offset start_offset, offset; msg->direction = Proto_Out; start_offset = msg->data; offset = msg->data; msg->header.node_src_id = src_id; msg->header.node_dest_id = dest_id; msg->header.message_ttl = msg_ttl; msg->header.sequence_number = seq_number; msg->header.message_length = 11; msg->header.message_type = Proto_Vehicle_Termination_Command; offset = pack_sync(offset); offset = pack_header(&msg->header, offset); offset = pack_vehicle_termination_command(tx_msg, offset); offset = pack_checksum(start_offset, offset); msg->tx_len = offset-start_offset; }
void *_forward_thread(void *arg) { forward_msg_t *fwd_msg = (forward_msg_t *)arg; forward_struct_t *fwd_struct = fwd_msg->fwd_struct; Buf buffer = init_buf(BUF_SIZE); /* probably enough for header */ List ret_list = NULL; int fd = -1; ret_data_info_t *ret_data_info = NULL; char *name = NULL; hostlist_t hl = hostlist_create(fwd_msg->header.forward.nodelist); slurm_addr_t addr; char *buf = NULL; int steps = 0; int start_timeout = fwd_msg->timeout; /* repeat until we are sure the message was sent */ while ((name = hostlist_shift(hl))) { if (slurm_conf_get_addr(name, &addr) == SLURM_ERROR) { error("forward_thread: can't find address for host " "%s, check slurm.conf", name); slurm_mutex_lock(&fwd_struct->forward_mutex); mark_as_failed_forward(&fwd_struct->ret_list, name, SLURM_UNKNOWN_FORWARD_ADDR); free(name); if (hostlist_count(hl) > 0) { slurm_mutex_unlock(&fwd_struct->forward_mutex); continue; } goto cleanup; } if ((fd = slurm_open_msg_conn(&addr)) < 0) { error("forward_thread to %s: %m", name); slurm_mutex_lock(&fwd_struct->forward_mutex); mark_as_failed_forward( &fwd_struct->ret_list, name, SLURM_COMMUNICATIONS_CONNECTION_ERROR); free(name); if (hostlist_count(hl) > 0) { slurm_mutex_unlock(&fwd_struct->forward_mutex); /* Abandon tree. This way if all the * nodes in the branch are down we * don't have to time out for each * node serially. */ _forward_msg_internal(hl, NULL, fwd_struct, &fwd_msg->header, 0, hostlist_count(hl)); continue; } goto cleanup; } buf = hostlist_ranged_string_xmalloc(hl); xfree(fwd_msg->header.forward.nodelist); fwd_msg->header.forward.nodelist = buf; fwd_msg->header.forward.cnt = hostlist_count(hl); #if 0 info("sending %d forwards (%s) to %s", fwd_msg->header.forward.cnt, fwd_msg->header.forward.nodelist, name); #endif if (fwd_msg->header.forward.nodelist[0]) { debug3("forward: send to %s along with %s", name, fwd_msg->header.forward.nodelist); } else debug3("forward: send to %s ", name); pack_header(&fwd_msg->header, buffer); /* add forward data to buffer */ if (remaining_buf(buffer) < fwd_struct->buf_len) { int new_size = buffer->processed + fwd_struct->buf_len; new_size += 1024; /* padded for paranoia */ xrealloc_nz(buffer->head, new_size); buffer->size = new_size; } if (fwd_struct->buf_len) { memcpy(&buffer->head[buffer->processed], fwd_struct->buf, fwd_struct->buf_len); buffer->processed += fwd_struct->buf_len; } /* * forward message */ if (slurm_msg_sendto(fd, get_buf_data(buffer), get_buf_offset(buffer), SLURM_PROTOCOL_NO_SEND_RECV_FLAGS ) < 0) { error("forward_thread: slurm_msg_sendto: %m"); slurm_mutex_lock(&fwd_struct->forward_mutex); mark_as_failed_forward(&fwd_struct->ret_list, name, errno); free(name); if (hostlist_count(hl) > 0) { free_buf(buffer); buffer = init_buf(fwd_struct->buf_len); slurm_mutex_unlock(&fwd_struct->forward_mutex); slurm_close(fd); fd = -1; /* Abandon tree. This way if all the * nodes in the branch are down we * don't have to time out for each * node serially. */ _forward_msg_internal(hl, NULL, fwd_struct, &fwd_msg->header, 0, hostlist_count(hl)); continue; } goto cleanup; } /* These messages don't have a return message, but if * we got here things worked out so make note of the * list of nodes as success. */ if ((fwd_msg->header.msg_type == REQUEST_SHUTDOWN) || (fwd_msg->header.msg_type == REQUEST_RECONFIGURE) || (fwd_msg->header.msg_type == REQUEST_REBOOT_NODES)) { slurm_mutex_lock(&fwd_struct->forward_mutex); ret_data_info = xmalloc(sizeof(ret_data_info_t)); list_push(fwd_struct->ret_list, ret_data_info); ret_data_info->node_name = xstrdup(name); free(name); while ((name = hostlist_shift(hl))) { ret_data_info = xmalloc(sizeof(ret_data_info_t)); list_push(fwd_struct->ret_list, ret_data_info); ret_data_info->node_name = xstrdup(name); free(name); } goto cleanup; } if (fwd_msg->header.forward.cnt > 0) { static int message_timeout = -1; if (message_timeout < 0) message_timeout = slurm_get_msg_timeout() * 1000; if (!fwd_msg->header.forward.tree_width) fwd_msg->header.forward.tree_width = slurm_get_tree_width(); steps = (fwd_msg->header.forward.cnt+1) / fwd_msg->header.forward.tree_width; fwd_msg->timeout = (message_timeout*steps); /* info("got %d * %d = %d", message_timeout, */ /* steps, fwd_msg->timeout); */ steps++; fwd_msg->timeout += (start_timeout*steps); /* info("now + %d*%d = %d", start_timeout, */ /* steps, fwd_msg->timeout); */ } ret_list = slurm_receive_msgs(fd, steps, fwd_msg->timeout); /* info("sent %d forwards got %d back", */ /* fwd_msg->header.forward.cnt, list_count(ret_list)); */ if (!ret_list || (fwd_msg->header.forward.cnt != 0 && list_count(ret_list) <= 1)) { slurm_mutex_lock(&fwd_struct->forward_mutex); mark_as_failed_forward(&fwd_struct->ret_list, name, errno); free(name); FREE_NULL_LIST(ret_list); if (hostlist_count(hl) > 0) { free_buf(buffer); buffer = init_buf(fwd_struct->buf_len); slurm_mutex_unlock(&fwd_struct->forward_mutex); slurm_close(fd); fd = -1; continue; } goto cleanup; } else if ((fwd_msg->header.forward.cnt+1) != list_count(ret_list)) { /* this should never be called since the above should catch the failed forwards and pipe them back down, but this is here so we never have to worry about a locked mutex */ ListIterator itr = NULL; char *tmp = NULL; int first_node_found = 0; hostlist_iterator_t host_itr = hostlist_iterator_create(hl); error("We shouldn't be here. We forwarded to %d " "but only got %d back", (fwd_msg->header.forward.cnt+1), list_count(ret_list)); while ((tmp = hostlist_next(host_itr))) { int node_found = 0; itr = list_iterator_create(ret_list); while ((ret_data_info = list_next(itr))) { if (!ret_data_info->node_name) { first_node_found = 1; ret_data_info->node_name = xstrdup(name); } if (!xstrcmp(tmp, ret_data_info->node_name)) { node_found = 1; break; } } list_iterator_destroy(itr); if (!node_found) { mark_as_failed_forward( &fwd_struct->ret_list, tmp, SLURM_COMMUNICATIONS_CONNECTION_ERROR); } free(tmp); } hostlist_iterator_destroy(host_itr); if (!first_node_found) { mark_as_failed_forward( &fwd_struct->ret_list, name, SLURM_COMMUNICATIONS_CONNECTION_ERROR); } } break; } slurm_mutex_lock(&fwd_struct->forward_mutex); if (ret_list) { while ((ret_data_info = list_pop(ret_list)) != NULL) { if (!ret_data_info->node_name) { ret_data_info->node_name = xstrdup(name); } list_push(fwd_struct->ret_list, ret_data_info); debug3("got response from %s", ret_data_info->node_name); } FREE_NULL_LIST(ret_list); } free(name); cleanup: if ((fd >= 0) && slurm_close(fd) < 0) error ("close(%d): %m", fd); hostlist_destroy(hl); destroy_forward(&fwd_msg->header.forward); free_buf(buffer); slurm_cond_signal(&fwd_struct->notify); slurm_mutex_unlock(&fwd_struct->forward_mutex); xfree(fwd_msg); return (NULL); }
size_t pack_payloadless_msg(uint8_t *buf, size_t buf_size, const char *cmd) { return pack_header(buf, buf_size, cmd, NULL, 0); }
void MpegDemultiplexer::PES_packet() { int stream_id = m_file->m_next_code & 0xff; uint16 PES_packet_length = m_file->getnbits(16); m_file->m_countBits = 0; switch (stream_id) { case 0xbc/*program_stream_map*/: { ASSERT(0); } break; case 0xbf/*private-stream2*/: { /*m_file->m_bytepos =*/ m_file->SetBytePos(m_file->GetBytePos() + PES_packet_length); //m_file->m_nstacked = 0; } break; case 0xbe/*padding*/: { // Maybe check that all bytes are 0xFF /*m_file->m_bytepos =*/ m_file->SetBytePos(m_file->GetBytePos() + PES_packet_length); //m_file->m_nstacked = 0; } break; default: { int fill = m_file->getnbits(2); ASSERT(fill == 2); int PES_scrambling_control = m_file->getnbits(2);// bslbf int PES_priority = m_file->getbit();// bslbf int data_alignment_indicator = m_file->getbit();// bslbf int copyright = m_file->getbit();// bslbf int original_or_copy = m_file->getbit();// bslbf int PTS_DTS_flags = m_file->getnbits(2);// bslbf int ESCR_flag = m_file->getbit();// bslbf int ES_rate_flag = m_file->getbit();// bslbf int DSM_trick_mode_flag = m_file->getbit();// bslbf int additional_copy_info_flag = m_file->getbit();// bslbf int PES_CRC_flag = m_file->getbit();// bslbf int PES_extension_flag = m_file->getbit();// bslbf uint8 PES_header_data_length = m_file->getnbits(8); if (PTS_DTS_flags == 2) { int fill = m_file->getnbits(4);// '0010' 4 bslbf ASSERT(fill == 2); m_file->getnbits(3);//PTS [32..30] 3 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf m_file->getnbits(15);//PTS [29..15] 15 bslbf m_file->getmarkerbit();// 1 bslbf m_file->getnbits(15);//PTS [14..0] 15 bslbf m_file->getmarkerbit();// 1 bslbf } else if (PTS_DTS_flags == 3) { int fill = m_file->getnbits(4);// ASSERT(fill == 3);//'0011' 4 bslbf m_file->getnbits(3);//PTS [32..30] 3 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf m_file->getnbits(15);//PTS [29..15] 15 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf m_file->getnbits(15);//PTS [14..0] 15 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf fill = m_file->getnbits(4);//marker_bit 1 bslbf ASSERT(fill == 1);//'0001' 4 bslbf m_file->getnbits(3);//DTS [32..30] 3 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf m_file->getnbits(15);//DTS [29..15] 15 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf m_file->getnbits(15);//DTS [14..0] 15 bslbf m_file->getmarkerbit();//marker_bit 1 bslbf } if (ESCR_flag == 1) { ASSERT(0); /* reserved 2 bslbf ESCR_base[32..30] 3 bslbf marker_bit 1 bslbf ESCR_base[29..15] 15 bslbf marker_bit 1 bslbf ESCR_base[14..0] 15 bslbf marker_bit 1 bslbf ESCR_extension 9 uimsbf marker_bit 1 bslbf */ } if (ES_rate_flag == 1) { ASSERT(0); /* marker_bit 1 bslbf ES_rate 22 uimsbf marker_bit 1 bslbf */ } if (DSM_trick_mode_flag == 1) { ASSERT(0); /* trick_mode_control 3 uimsbf if ( trick_mode_control = = fast_forward ) { field_id 2 bslbf intra_slice_refresh 1 bslbf frequency_truncation 2 bslbf } else if ( trick_mode_control = = slow_motion ) { rep_cntrl 5 uimsbf } else if ( trick_mode_control = = freeze_frame ) { field_id 2 uimsbf reserved 3 bslbf } else if ( trick_mode_control = = fast_reverse ) { field_id 2 bslbf intra_slice_refresh 1 bslbf frequency_truncation 2 bslbf } else if ( trick_mode_control = = slow_reverse ) { rep_cntrl 5 uimsbf } else reserved 5 bslbf */ } if (additional_copy_info_flag == 1) { m_file->getmarkerbit();// 1 bslbf int additional_copy_info = m_file->getnbits(7);// bslbf } if (PES_CRC_flag == 1) { int previous_PES_packet_CRC = m_file->getnbits(16);// bslbf } if (PES_extension_flag == 1) { int PES_private_data_flag = m_file->getbit();// bslbf int pack_header_field_flag = m_file->getbit();// 1 bslbf int program_packet_sequence_counter_flag = m_file->getbit();// 1 bslbf int P_STD_buffer_flag = m_file->getnbits(1);// 1 bslbf m_file->getnbits(3);//reserved 3 bslbf int PES_extension_flag_2 = m_file->getbit();// 1 bslbf if ( PES_private_data_flag == 1) { m_file->skipnbits(128); //PES_private_data 128 bslbf } if (pack_header_field_flag == 1) { int pack_field_length = m_file->getnbits(8);// uimsbf pack_header(); } if (program_packet_sequence_counter_flag == 1) { ASSERT(0); /* marker_bit 1 bslbf program_packet_sequence_counter 7 uimsbf marker_bit 1 bslbf MPEG1_MPEG2_identifier 1 bslbf original_stuff_length 6 uimsbf */ } if (P_STD_buffer_flag == 1) { int fill = m_file->getnbits(2);//'01' 2 bslbf ASSERT(fill == 1); int P_STD_buffer_scale = m_file->getbit();//1 bslbf int P_STD_buffer_size = m_file->getnbits(13);//13 uimsbf } if (PES_extension_flag_2 == 1) { m_file->getmarkerbit();// 1 bslbf uint8 PES_extension_field_length = m_file->getnbits(7);// uimsbf for (int i = 0; i < PES_extension_field_length; i++) { //reserved 8 bslbf uint8 reserved = m_file->getnbits(8); } } } int i; ASSERT((m_file->m_countBits & 0x3) == 0); int n = (m_file->m_countBits >> 3); int N1 = n - (PES_header_data_length+3); int N2 = PES_packet_length - n; for (i = 0; i < N1; i++) { m_file->getnbits(8);//stuffing_byte 8 bslbf } MpegStream* pStream = FindStream(stream_id); if (pStream == NULL) // ??? { pStream = new MpegStream; pStream->m_pFilter = this; pStream->m_stream_id = stream_id; m_streams.push_back(pStream); } CPacket* pPacket = NULL; if (pStream) { pPacket = new CPacket; pPacket->m_pStream = pStream; pStream->m_packets.push_back(pPacket); } pPacket->m_byteOffset = m_file->GetBytePos(); pPacket->m_byteLength = N2; pStream->m_byteLength += pPacket->m_byteLength; m_file->SetBytePos(pPacket->m_byteOffset + N2); /* for (i = 0; i < N2; i++) { m_file->getnbits(8);//PES_packet_data_byte 8 bslbf } */ } break; } m_file->next_start_code(); }
/****************************************************************************** * * The main worker node function. * * int thread_id: the thread_id * char *fastq1: FIFO from which bowtie2 can get read1 * char *fastq2: FIFO from which bowtie2 can get read2 (if it exists) * *******************************************************************************/ void herd_worker_node(int thread_id, char *fastq1, char *fastq2) { int cmd_length = 1, max_qname = 0, status, strand; char *cmd, *last_qname = calloc(1, sizeof(char)); MPI_Header *packed_header; MPI_read *packed_read = calloc(1, sizeof(MPI_read)); bam_hdr_t *header; bam1_t *read1 = bam_init1(); bam1_t *read2 = bam_init1(); samFile *fp; #ifdef DEBUG MPI_Status stat; int current_p_size = 100; htsFile *of; bam_hdr_t *debug_header = bam_hdr_init(); bam1_t *debug_read = bam_init1(); global_header = bam_hdr_init(); void *p = calloc(100,1); char *oname = NULL; #else int i = 0; #endif time_t t0, t1; int swapped = 0; assert(last_qname); assert(packed_read); //Which strand should we be aligning to? if(config.directional) { strand = (thread_id-1) % 2; } else { strand = (thread_id-1) % 4; } packed_read->size = 0; packed_read->packed = NULL; //construct the bowtie2 command cmd_length += (int) strlen("bowtie2 -q --reorder") + 1; cmd_length += (int) strlen(config.bowtie2_options) + 1; cmd_length += (int) strlen("--norc -x") + 1; cmd_length += (int) strlen(config.genome_dir) + strlen("bisulfite_genome/CT_conversion/BS_CT") + 1; cmd_length += (int) 2*(strlen("-1 ") + strlen(fastq1)) + 3; if(config.paired) cmd_length += (int) strlen(fastq2); //This is likely unneeded. #ifdef DEBUG oname = malloc(sizeof(char) *(1+strlen(config.odir)+strlen(config.basename)+strlen("_X.bam"))); assert(oname); sprintf(oname, "%s%s_%i.bam", config.odir, config.basename, thread_id); if(!config.quiet) fprintf(stderr, "Writing output to %s\n", oname); of = sam_open(oname, "wb"); free(oname); #endif cmd = (char *) malloc(sizeof(char) * cmd_length); assert(cmd); if(strand == 0) { //OT Read#1 C->T, Read#2 G->A, Genome C->T only the + strand if(config.paired) { sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/CT_conversion/BS_CT -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2); } else { sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/CT_conversion/BS_CT -U %s", config.bowtie2_options, config.genome_dir, fastq1); } } else if(strand == 1) { //OB Read#1 C->T, Read#2 G->A, Genome G->A only the - strand if(config.paired) { sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/GA_conversion/BS_GA -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2); } else { sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/GA_conversion/BS_GA -U %s", config.bowtie2_options, config.genome_dir, fastq1); } } else if(strand == 2) { //CTOT Read#1 G->A, Read#2 C->T, Genome C->T, only the - strand if(config.paired) { sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/CT_conversion/BS_CT -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2); } else { sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/CT_conversion/BS_CT -U %s", config.bowtie2_options, config.genome_dir, fastq1); } } else if(strand == 3) { //CTOB Read#1 G->A, Read#2 C->T, Genome G->A, only the + strand if(config.paired) { sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/GA_conversion/BS_GA -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2); } else { sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/GA_conversion/BS_GA -U %s", config.bowtie2_options, config.genome_dir, fastq1); } } else { fprintf(stderr, "Oh shit, got strand %i!\n", strand); return; } //Start the process if(!config.quiet) fprintf(stderr, "Node %i executing: %s\n", thread_id, cmd); fflush(stderr); fp = sam_popen(cmd); header = sam_hdr_read(fp); #ifdef DEBUG sam_hdr_write(of, header); #endif #ifndef DEBUG packed_header = pack_header(header); if(thread_id == 1) { //Send the header MPI_Send((void *) &(packed_header->size), 1, MPI_INT, 0, 1, MPI_COMM_WORLD); status = MPI_Send((void *) packed_header->packed, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD); if(status != MPI_SUCCESS) { fprintf(stderr, "MPI_Send returned %i\n", status); fflush(stderr); } } #else packed_header = pack_header(header); void *tmp_pointer = malloc(packed_header->size); assert(tmp_pointer); MPI_Request request; MPI_Isend((void *) packed_header->packed, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD, &request); status = MPI_Recv(tmp_pointer, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD, &stat); if(status != MPI_SUCCESS) fprintf(stderr, "We seem to have not been able to send the message to ourselves!\n"); MPI_Wait(&request, &stat); unpack_header(debug_header, tmp_pointer); global_header = debug_header; free(tmp_pointer); #endif t0 = time(NULL); if(!config.quiet) fprintf(stderr, "Node %i began sending reads @%s", thread_id, ctime(&t0)); fflush(stderr); while(sam_read1(fp, header, read1) >= 0) { #ifdef DEBUG sam_write1(of, global_header, read1); #endif if(strcmp(bam_get_qname(read1), last_qname) == 0) { //Multimapper if(config.paired) { sam_read1(fp, header, read2); #ifdef DEBUG sam_write1(of, global_header, read2); #endif } continue; } else { if(read1->core.l_qname > max_qname) { max_qname = read1->core.l_qname + 10; last_qname = realloc(last_qname, sizeof(char) * max_qname); assert(last_qname); } strcpy(last_qname, bam_get_qname(read1)); } //Are paired-end reads in the wrong order? swapped = 0; if(config.paired) { if(read1->core.flag & BAM_FREAD2) { swapped = 1; sam_read1(fp, header, read2); packed_read = pack_read(read2, packed_read); #ifndef DEBUG MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD); #else sam_write1(of, global_header, read2); if(packed_read->size > current_p_size) { p = realloc(p, packed_read->size); assert(p); } MPI_Isend((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request); status = MPI_Recv(p, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat); MPI_Wait(&request, &stat); debug_read = unpack_read(debug_read, p); #endif } } //Send the read packed_read = pack_read(read1, packed_read); #ifndef DEBUG MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD); #else if(packed_read->size > current_p_size) { p = realloc(p, packed_read->size); assert(p); } MPI_Isend(packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request); status = MPI_Recv(p, packed_header->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat); MPI_Wait(&request, &stat); #endif //Deal with paired-end reads if(config.paired && !swapped) { sam_read1(fp, header, read2); packed_read = pack_read(read2, packed_read); #ifndef DEBUG MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD); #else sam_write1(of, global_header, read2); if(packed_read->size > current_p_size) { p = realloc(p, packed_read->size); assert(p); } MPI_Isend((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request); status = MPI_Recv(p, packed_header->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat); MPI_Wait(&request, &stat); debug_read = unpack_read(debug_read, p); #endif } #ifndef DEBUG i++; #endif } t1 = time(NULL); if(!config.quiet) fprintf(stderr, "Node %i finished sending reads @%s\t(%f sec elapsed)\n", thread_id, ctime(&t1), difftime(t1, t0)); fflush(stderr); //Notify the master node packed_read->size = 0; #ifndef DEBUG void *A = malloc(1); assert(A); MPI_Send(A, 1, MPI_BYTE, 0, 5, MPI_COMM_WORLD); free(A); #endif //Close things up bam_hdr_destroy(header); bam_destroy1(read1); bam_destroy1(read2); free(cmd); if(packed_read->packed != NULL) free(packed_read->packed); free(packed_read); if(packed_header->packed != NULL) free(packed_header->packed); free(packed_header); free(last_qname); sam_pclose(fp); //Remove the FIFO(s) unlink(fastq1); if(config.paired) unlink(fastq2); #ifdef DEBUG sam_close(of); bam_hdr_destroy(debug_header); bam_destroy1(debug_read); free(p); #endif if(!config.quiet) fprintf(stderr, "Exiting worker node %i\n", thread_id); fflush(stderr); };