size_t pack_version_msg(uint8_t *buf, size_t size, const uint64_t nonce)
{
const struct in6_addr in6_loop = IN6ADDR_LOOPBACK_INIT;
const size_t my_user_agent_len = NELEMS(USER_AGENT) - 1;
const time_t now = time(NULL);
size_t len;
size_t off;
len = off = HDR_SIZE;
PACKLE32(PROTOCOL_VERSION);
PACKLE64(0);
PACKLE64(now);
PACKLE64(NODE_NETWORK);
PACKN(&in6_loop, sizeof(in6_loop));
PACKBE16(MAINNET_PORT);
PACKLE64(0);
PACKN(&in6_loop, sizeof(in6_loop));
PACKBE16(MAINNET_PORT);
PACKLE64(nonce);
PACKCINT(my_user_agent_len);
PACKN(USER_AGENT, my_user_agent_len);
PACKLE32(0);
PACK8(0);
if (off == len) {
(void)pack_header(buf, size, "version",
buf + HDR_SIZE, len - HDR_SIZE);
}
return len;
}
void MpegDemultiplexer::pack()
{
ASSERT(m_file->m_next_code == 0x01ba);
pack_header();
if (m_mpeg1) // mpeg1
{
// m_file->next_start_code();
while (m_file->m_next_code >= 0x1bc && m_file->m_next_code <= 0x1ff)
{
packet_mpeg1();
// m_file->next_start_code();
}
}
else // mpeg2
{
// pack_header();
// m_file->next_start_code();
while (m_file->m_next_code != 0x1ba && m_file->m_next_code != 0x1b9)
{
PES_packet();
}
#if 0
while (m_file->m_next_code == 0x1bc/* && m_file->m_next_code <= 0x1ff*/)
{
program_stream_map_mpeg2();
}
#endif
}
}
char *pack_response(dns_packet_t *packet, size_t *rlen)
{
size_t header_len = sizeof(dns_header_t);
size_t question_len = sizeof(dns_question_t);
size_t answer_len = sizeof(dns_record_t);
char *buf = malloc(
header_len + \
packet->header.qdcount * question_len + \
packet->header.ancount * answer_len);
char *start = buf;
memcpy(buf, pack_header(&packet->header), header_len);
buf += header_len;
for (int i = 0; i < packet->header.qdcount; i++) {
size_t question_len;
char *question = pack_question(&packet->questions[i], &question_len);
memcpy(buf, question, question_len);
buf += question_len;
free(question);
}
for (int i = 0; i < packet->header.ancount; i++) {
size_t record_len;
char *record = pack_record(&packet->answers[i], &record_len);
memcpy(buf, record, record_len);
buf += record_len;
free(record);
}
*rlen = buf - start;
return start;
}
char *ddb_cons_finalize(struct ddb_cons *cons, uint64_t *length, uint64_t flags)
{
struct ddb_packed *pack = NULL;
struct ddb_entry *order = NULL;
char *buf = NULL;
int disable_compression, err = 1;
DDB_TIMER_DEF
if (!(pack = buffer_init()))
goto err;
if (pack_header(pack, cons))
goto err;
#ifdef DDB_PROFILE
print_mem_usage(cons);
#endif
DDB_TIMER_START
pack->head->hash_offs = pack->offs;
if (!(order = pack_hash(pack, cons->keys_map)))
goto err;
DDB_TIMER_END("hash")
DDB_TIMER_START
pack->head->key2values_offs = pack->offs;
if (pack_key2values(pack, order, cons->keys_map,
flags & DDB_OPT_UNIQUE_ITEMS))
goto err;
DDB_TIMER_END("key2values")
DDB_TIMER_START
flags |= maybe_disable_compression(cons);
disable_compression = flags & DDB_OPT_DISABLE_COMPRESSION;
pack->head->id2value_offs = pack->offs;
if (pack_id2value(pack, cons->values_map, disable_compression))
goto err;
else{
ddb_map_free(cons->values_map);
cons->values_map = NULL;
}
DDB_TIMER_END("id2values")
if (!disable_compression){
DDB_TIMER_START
pack->head->codebook_offs = pack->offs;
if (pack_codebook(pack))
goto err;
DDB_TIMER_END("save_codebook")
}
/* this function sends a chunk of data down the next appropriate fd and a
corresponding chunk header down the control connection */
void send_chunk(FD_ARRAY *fdstate, uint8_t *data, size_t len)
{
CHUNK_HDR ourheader;
PACKED_CHUNK packedchunk;
int fd;
assert(fdstate -> numfds > 0); /* make sure we have FDs ready */
fd = fdstate->fds[fdstate->nextidx]; /* which FD we use */
/* we prepare the chunk header */
ourheader.index = fdstate->indices[fdstate->nextidx];
ourheader.begin_off = fdstate->bytes[fdstate->nextidx];
ourheader.end_off = ourheader.begin_off + len;
ourheader.seq = fdstate->nextseq;
/* and allocate space for the packed version of it */
packedchunk.data = malloc(CHUNK_HDR_LEN);
assert(packedchunk.data != NULL);
/* and we pack it */
pack_header(&ourheader, &packedchunk);
/* and we send it down the control connection */
if(send_all(fdstate->controlfd, packedchunk.data, packedchunk.len, 0) == -1)
{
perror("send error in control connection");
exit(-1);
}
/* it's sent, so we don't need its packed version */
free(packedchunk.data);
/* ...and we send the corresponding data */
if(send_all(fd, data, len, 0) == -1)
{
perror("send error in data connection");
exit(-1);
}
/* here we update fdstate */
fdstate->bytes[fdstate->nextidx] += len; /* how many bytes we sent down that fd*/
/* we update nextidx while making sure it doesn't get too big */
fdstate->nextidx++;
if (fdstate->nextidx > (fdstate->numfds - 1))
{
fdstate->nextidx = 0;
}
/* increment the sequence number */
fdstate->nextseq++;
}
void encode_vehicle_authorization_reply(
uint8_t src_id,
uint8_t dest_id,
uint8_t msg_ttl,
uint8_t seq_number,
vehicle_authorization_reply_t* tx_msg,
proto_msg_t* msg)
{
msg_offset start_offset, offset;
msg->direction = Proto_Out;
start_offset = msg->data;
offset = msg->data;
msg->header.node_src_id = src_id;
msg->header.node_dest_id = dest_id;
msg->header.message_ttl = msg_ttl;
msg->header.sequence_number = seq_number;
msg->header.message_length = 13;
msg->header.message_type = Proto_Vehicle_Authorization_Reply;
offset = pack_sync(offset);
offset = pack_header(&msg->header, offset);
offset = pack_vehicle_authorization_reply(tx_msg, offset);
offset = pack_checksum(start_offset, offset);
msg->tx_len = offset-start_offset;
}
void encode_vehicle_termination_command(
uint8_t src_id,
uint8_t dest_id,
uint8_t msg_ttl,
uint8_t seq_number,
vehicle_termination_command_t* tx_msg,
proto_msg_t* msg)
{
msg_offset start_offset, offset;
msg->direction = Proto_Out;
start_offset = msg->data;
offset = msg->data;
msg->header.node_src_id = src_id;
msg->header.node_dest_id = dest_id;
msg->header.message_ttl = msg_ttl;
msg->header.sequence_number = seq_number;
msg->header.message_length = 11;
msg->header.message_type = Proto_Vehicle_Termination_Command;
offset = pack_sync(offset);
offset = pack_header(&msg->header, offset);
offset = pack_vehicle_termination_command(tx_msg, offset);
offset = pack_checksum(start_offset, offset);
msg->tx_len = offset-start_offset;
}
void *_forward_thread(void *arg)
{
forward_msg_t *fwd_msg = (forward_msg_t *)arg;
forward_struct_t *fwd_struct = fwd_msg->fwd_struct;
Buf buffer = init_buf(BUF_SIZE); /* probably enough for header */
List ret_list = NULL;
int fd = -1;
ret_data_info_t *ret_data_info = NULL;
char *name = NULL;
hostlist_t hl = hostlist_create(fwd_msg->header.forward.nodelist);
slurm_addr_t addr;
char *buf = NULL;
int steps = 0;
int start_timeout = fwd_msg->timeout;
/* repeat until we are sure the message was sent */
while ((name = hostlist_shift(hl))) {
if (slurm_conf_get_addr(name, &addr) == SLURM_ERROR) {
error("forward_thread: can't find address for host "
"%s, check slurm.conf", name);
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(&fwd_struct->ret_list, name,
SLURM_UNKNOWN_FORWARD_ADDR);
free(name);
if (hostlist_count(hl) > 0) {
slurm_mutex_unlock(&fwd_struct->forward_mutex);
continue;
}
goto cleanup;
}
if ((fd = slurm_open_msg_conn(&addr)) < 0) {
error("forward_thread to %s: %m", name);
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(
&fwd_struct->ret_list, name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
free(name);
if (hostlist_count(hl) > 0) {
slurm_mutex_unlock(&fwd_struct->forward_mutex);
/* Abandon tree. This way if all the
* nodes in the branch are down we
* don't have to time out for each
* node serially.
*/
_forward_msg_internal(hl, NULL, fwd_struct,
&fwd_msg->header, 0,
hostlist_count(hl));
continue;
}
goto cleanup;
}
buf = hostlist_ranged_string_xmalloc(hl);
xfree(fwd_msg->header.forward.nodelist);
fwd_msg->header.forward.nodelist = buf;
fwd_msg->header.forward.cnt = hostlist_count(hl);
#if 0
info("sending %d forwards (%s) to %s",
fwd_msg->header.forward.cnt,
fwd_msg->header.forward.nodelist, name);
#endif
if (fwd_msg->header.forward.nodelist[0]) {
debug3("forward: send to %s along with %s",
name, fwd_msg->header.forward.nodelist);
} else
debug3("forward: send to %s ", name);
pack_header(&fwd_msg->header, buffer);
/* add forward data to buffer */
if (remaining_buf(buffer) < fwd_struct->buf_len) {
int new_size = buffer->processed + fwd_struct->buf_len;
new_size += 1024; /* padded for paranoia */
xrealloc_nz(buffer->head, new_size);
buffer->size = new_size;
}
if (fwd_struct->buf_len) {
memcpy(&buffer->head[buffer->processed],
fwd_struct->buf, fwd_struct->buf_len);
buffer->processed += fwd_struct->buf_len;
}
/*
* forward message
*/
if (slurm_msg_sendto(fd,
get_buf_data(buffer),
get_buf_offset(buffer),
SLURM_PROTOCOL_NO_SEND_RECV_FLAGS ) < 0) {
error("forward_thread: slurm_msg_sendto: %m");
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(&fwd_struct->ret_list, name,
errno);
free(name);
if (hostlist_count(hl) > 0) {
free_buf(buffer);
buffer = init_buf(fwd_struct->buf_len);
slurm_mutex_unlock(&fwd_struct->forward_mutex);
slurm_close(fd);
fd = -1;
/* Abandon tree. This way if all the
* nodes in the branch are down we
* don't have to time out for each
* node serially.
*/
_forward_msg_internal(hl, NULL, fwd_struct,
&fwd_msg->header, 0,
hostlist_count(hl));
continue;
}
goto cleanup;
}
/* These messages don't have a return message, but if
* we got here things worked out so make note of the
* list of nodes as success.
*/
if ((fwd_msg->header.msg_type == REQUEST_SHUTDOWN) ||
(fwd_msg->header.msg_type == REQUEST_RECONFIGURE) ||
(fwd_msg->header.msg_type == REQUEST_REBOOT_NODES)) {
slurm_mutex_lock(&fwd_struct->forward_mutex);
ret_data_info = xmalloc(sizeof(ret_data_info_t));
list_push(fwd_struct->ret_list, ret_data_info);
ret_data_info->node_name = xstrdup(name);
free(name);
while ((name = hostlist_shift(hl))) {
ret_data_info =
xmalloc(sizeof(ret_data_info_t));
list_push(fwd_struct->ret_list, ret_data_info);
ret_data_info->node_name = xstrdup(name);
free(name);
}
goto cleanup;
}
if (fwd_msg->header.forward.cnt > 0) {
static int message_timeout = -1;
if (message_timeout < 0)
message_timeout =
slurm_get_msg_timeout() * 1000;
if (!fwd_msg->header.forward.tree_width)
fwd_msg->header.forward.tree_width =
slurm_get_tree_width();
steps = (fwd_msg->header.forward.cnt+1) /
fwd_msg->header.forward.tree_width;
fwd_msg->timeout = (message_timeout*steps);
/* info("got %d * %d = %d", message_timeout, */
/* steps, fwd_msg->timeout); */
steps++;
fwd_msg->timeout += (start_timeout*steps);
/* info("now + %d*%d = %d", start_timeout, */
/* steps, fwd_msg->timeout); */
}
ret_list = slurm_receive_msgs(fd, steps, fwd_msg->timeout);
/* info("sent %d forwards got %d back", */
/* fwd_msg->header.forward.cnt, list_count(ret_list)); */
if (!ret_list || (fwd_msg->header.forward.cnt != 0
&& list_count(ret_list) <= 1)) {
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(&fwd_struct->ret_list, name,
errno);
free(name);
FREE_NULL_LIST(ret_list);
if (hostlist_count(hl) > 0) {
free_buf(buffer);
buffer = init_buf(fwd_struct->buf_len);
slurm_mutex_unlock(&fwd_struct->forward_mutex);
slurm_close(fd);
fd = -1;
continue;
}
goto cleanup;
} else if ((fwd_msg->header.forward.cnt+1)
!= list_count(ret_list)) {
/* this should never be called since the above
should catch the failed forwards and pipe
them back down, but this is here so we
never have to worry about a locked
mutex */
ListIterator itr = NULL;
char *tmp = NULL;
int first_node_found = 0;
hostlist_iterator_t host_itr
= hostlist_iterator_create(hl);
error("We shouldn't be here. We forwarded to %d "
"but only got %d back",
(fwd_msg->header.forward.cnt+1),
list_count(ret_list));
while ((tmp = hostlist_next(host_itr))) {
int node_found = 0;
itr = list_iterator_create(ret_list);
while ((ret_data_info = list_next(itr))) {
if (!ret_data_info->node_name) {
first_node_found = 1;
ret_data_info->node_name =
xstrdup(name);
}
if (!xstrcmp(tmp,
ret_data_info->node_name)) {
node_found = 1;
break;
}
}
list_iterator_destroy(itr);
if (!node_found) {
mark_as_failed_forward(
&fwd_struct->ret_list,
tmp,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
}
free(tmp);
}
hostlist_iterator_destroy(host_itr);
if (!first_node_found) {
mark_as_failed_forward(
&fwd_struct->ret_list,
name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
}
}
break;
}
slurm_mutex_lock(&fwd_struct->forward_mutex);
if (ret_list) {
while ((ret_data_info = list_pop(ret_list)) != NULL) {
if (!ret_data_info->node_name) {
ret_data_info->node_name = xstrdup(name);
}
list_push(fwd_struct->ret_list, ret_data_info);
debug3("got response from %s",
ret_data_info->node_name);
}
FREE_NULL_LIST(ret_list);
}
free(name);
cleanup:
if ((fd >= 0) && slurm_close(fd) < 0)
error ("close(%d): %m", fd);
hostlist_destroy(hl);
destroy_forward(&fwd_msg->header.forward);
free_buf(buffer);
slurm_cond_signal(&fwd_struct->notify);
slurm_mutex_unlock(&fwd_struct->forward_mutex);
xfree(fwd_msg);
return (NULL);
}
size_t pack_payloadless_msg(uint8_t *buf, size_t buf_size, const char *cmd)
{
return pack_header(buf, buf_size, cmd, NULL, 0);
}
void MpegDemultiplexer::PES_packet()
{
int stream_id = m_file->m_next_code & 0xff;
uint16 PES_packet_length = m_file->getnbits(16);
m_file->m_countBits = 0;
switch (stream_id)
{
case 0xbc/*program_stream_map*/:
{
ASSERT(0);
}
break;
case 0xbf/*private-stream2*/:
{
/*m_file->m_bytepos =*/ m_file->SetBytePos(m_file->GetBytePos() + PES_packet_length);
//m_file->m_nstacked = 0;
}
break;
case 0xbe/*padding*/:
{
// Maybe check that all bytes are 0xFF
/*m_file->m_bytepos =*/ m_file->SetBytePos(m_file->GetBytePos() + PES_packet_length);
//m_file->m_nstacked = 0;
}
break;
default:
{
int fill = m_file->getnbits(2);
ASSERT(fill == 2);
int PES_scrambling_control = m_file->getnbits(2);// bslbf
int PES_priority = m_file->getbit();// bslbf
int data_alignment_indicator = m_file->getbit();// bslbf
int copyright = m_file->getbit();// bslbf
int original_or_copy = m_file->getbit();// bslbf
int PTS_DTS_flags = m_file->getnbits(2);// bslbf
int ESCR_flag = m_file->getbit();// bslbf
int ES_rate_flag = m_file->getbit();// bslbf
int DSM_trick_mode_flag = m_file->getbit();// bslbf
int additional_copy_info_flag = m_file->getbit();// bslbf
int PES_CRC_flag = m_file->getbit();// bslbf
int PES_extension_flag = m_file->getbit();// bslbf
uint8 PES_header_data_length = m_file->getnbits(8);
if (PTS_DTS_flags == 2)
{
int fill = m_file->getnbits(4);// '0010' 4 bslbf
ASSERT(fill == 2);
m_file->getnbits(3);//PTS [32..30] 3 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
m_file->getnbits(15);//PTS [29..15] 15 bslbf
m_file->getmarkerbit();// 1 bslbf
m_file->getnbits(15);//PTS [14..0] 15 bslbf
m_file->getmarkerbit();// 1 bslbf
}
else if (PTS_DTS_flags == 3)
{
int fill = m_file->getnbits(4);//
ASSERT(fill == 3);//'0011' 4 bslbf
m_file->getnbits(3);//PTS [32..30] 3 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
m_file->getnbits(15);//PTS [29..15] 15 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
m_file->getnbits(15);//PTS [14..0] 15 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
fill = m_file->getnbits(4);//marker_bit 1 bslbf
ASSERT(fill == 1);//'0001' 4 bslbf
m_file->getnbits(3);//DTS [32..30] 3 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
m_file->getnbits(15);//DTS [29..15] 15 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
m_file->getnbits(15);//DTS [14..0] 15 bslbf
m_file->getmarkerbit();//marker_bit 1 bslbf
}
if (ESCR_flag == 1)
{
ASSERT(0);
/*
reserved 2 bslbf
ESCR_base[32..30] 3 bslbf
marker_bit 1 bslbf
ESCR_base[29..15] 15 bslbf
marker_bit 1 bslbf
ESCR_base[14..0] 15 bslbf
marker_bit 1 bslbf
ESCR_extension 9 uimsbf
marker_bit 1 bslbf
*/
}
if (ES_rate_flag == 1)
{
ASSERT(0);
/*
marker_bit 1 bslbf
ES_rate 22 uimsbf
marker_bit 1 bslbf
*/
}
if (DSM_trick_mode_flag == 1)
{
ASSERT(0);
/*
trick_mode_control 3 uimsbf
if ( trick_mode_control = = fast_forward )
{
field_id 2 bslbf
intra_slice_refresh 1 bslbf
frequency_truncation 2 bslbf
}
else if ( trick_mode_control = = slow_motion )
{
rep_cntrl 5 uimsbf
}
else if ( trick_mode_control = = freeze_frame )
{
field_id 2 uimsbf
reserved 3 bslbf
}
else if ( trick_mode_control = = fast_reverse )
{
field_id 2 bslbf
intra_slice_refresh 1 bslbf
frequency_truncation 2 bslbf
}
else if ( trick_mode_control = = slow_reverse )
{
rep_cntrl 5 uimsbf
}
else
reserved 5 bslbf
*/
}
if (additional_copy_info_flag == 1)
{
m_file->getmarkerbit();// 1 bslbf
int additional_copy_info = m_file->getnbits(7);// bslbf
}
if (PES_CRC_flag == 1)
{
int previous_PES_packet_CRC = m_file->getnbits(16);// bslbf
}
if (PES_extension_flag == 1)
{
int PES_private_data_flag = m_file->getbit();// bslbf
int pack_header_field_flag = m_file->getbit();// 1 bslbf
int program_packet_sequence_counter_flag = m_file->getbit();// 1 bslbf
int P_STD_buffer_flag = m_file->getnbits(1);// 1 bslbf
m_file->getnbits(3);//reserved 3 bslbf
int PES_extension_flag_2 = m_file->getbit();// 1 bslbf
if ( PES_private_data_flag == 1)
{
m_file->skipnbits(128);
//PES_private_data 128 bslbf
}
if (pack_header_field_flag == 1)
{
int pack_field_length = m_file->getnbits(8);// uimsbf
pack_header();
}
if (program_packet_sequence_counter_flag == 1)
{
ASSERT(0);
/*
marker_bit 1 bslbf
program_packet_sequence_counter 7 uimsbf
marker_bit 1 bslbf
MPEG1_MPEG2_identifier 1 bslbf
original_stuff_length 6 uimsbf
*/
}
if (P_STD_buffer_flag == 1)
{
int fill = m_file->getnbits(2);//'01' 2 bslbf
ASSERT(fill == 1);
int P_STD_buffer_scale = m_file->getbit();//1 bslbf
int P_STD_buffer_size = m_file->getnbits(13);//13 uimsbf
}
if (PES_extension_flag_2 == 1)
{
m_file->getmarkerbit();// 1 bslbf
uint8 PES_extension_field_length = m_file->getnbits(7);// uimsbf
for (int i = 0; i < PES_extension_field_length; i++)
{
//reserved 8 bslbf
uint8 reserved = m_file->getnbits(8);
}
}
}
int i;
ASSERT((m_file->m_countBits & 0x3) == 0);
int n = (m_file->m_countBits >> 3);
int N1 = n - (PES_header_data_length+3);
int N2 = PES_packet_length - n;
for (i = 0; i < N1; i++)
{
m_file->getnbits(8);//stuffing_byte 8 bslbf
}
MpegStream* pStream = FindStream(stream_id);
if (pStream == NULL) // ???
{
pStream = new MpegStream;
pStream->m_pFilter = this;
pStream->m_stream_id = stream_id;
m_streams.push_back(pStream);
}
CPacket* pPacket = NULL;
if (pStream)
{
pPacket = new CPacket;
pPacket->m_pStream = pStream;
pStream->m_packets.push_back(pPacket);
}
pPacket->m_byteOffset = m_file->GetBytePos();
pPacket->m_byteLength = N2;
pStream->m_byteLength += pPacket->m_byteLength;
m_file->SetBytePos(pPacket->m_byteOffset + N2);
/*
for (i = 0; i < N2; i++)
{
m_file->getnbits(8);//PES_packet_data_byte 8 bslbf
}
*/
}
break;
}
m_file->next_start_code();
}
/******************************************************************************
*
* The main worker node function.
*
* int thread_id: the thread_id
* char *fastq1: FIFO from which bowtie2 can get read1
* char *fastq2: FIFO from which bowtie2 can get read2 (if it exists)
*
*******************************************************************************/
void herd_worker_node(int thread_id, char *fastq1, char *fastq2) {
int cmd_length = 1, max_qname = 0, status, strand;
char *cmd, *last_qname = calloc(1, sizeof(char));
MPI_Header *packed_header;
MPI_read *packed_read = calloc(1, sizeof(MPI_read));
bam_hdr_t *header;
bam1_t *read1 = bam_init1();
bam1_t *read2 = bam_init1();
samFile *fp;
#ifdef DEBUG
MPI_Status stat;
int current_p_size = 100;
htsFile *of;
bam_hdr_t *debug_header = bam_hdr_init();
bam1_t *debug_read = bam_init1();
global_header = bam_hdr_init();
void *p = calloc(100,1);
char *oname = NULL;
#else
int i = 0;
#endif
time_t t0, t1;
int swapped = 0;
assert(last_qname);
assert(packed_read);
//Which strand should we be aligning to?
if(config.directional) {
strand = (thread_id-1) % 2;
} else {
strand = (thread_id-1) % 4;
}
packed_read->size = 0;
packed_read->packed = NULL;
//construct the bowtie2 command
cmd_length += (int) strlen("bowtie2 -q --reorder") + 1;
cmd_length += (int) strlen(config.bowtie2_options) + 1;
cmd_length += (int) strlen("--norc -x") + 1;
cmd_length += (int) strlen(config.genome_dir) + strlen("bisulfite_genome/CT_conversion/BS_CT") + 1;
cmd_length += (int) 2*(strlen("-1 ") + strlen(fastq1)) + 3;
if(config.paired) cmd_length += (int) strlen(fastq2); //This is likely unneeded.
#ifdef DEBUG
oname = malloc(sizeof(char) *(1+strlen(config.odir)+strlen(config.basename)+strlen("_X.bam")));
assert(oname);
sprintf(oname, "%s%s_%i.bam", config.odir, config.basename, thread_id);
if(!config.quiet) fprintf(stderr, "Writing output to %s\n", oname);
of = sam_open(oname, "wb");
free(oname);
#endif
cmd = (char *) malloc(sizeof(char) * cmd_length);
assert(cmd);
if(strand == 0) { //OT Read#1 C->T, Read#2 G->A, Genome C->T only the + strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/CT_conversion/BS_CT -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/CT_conversion/BS_CT -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else if(strand == 1) { //OB Read#1 C->T, Read#2 G->A, Genome G->A only the - strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/GA_conversion/BS_GA -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/GA_conversion/BS_GA -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else if(strand == 2) { //CTOT Read#1 G->A, Read#2 C->T, Genome C->T, only the - strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/CT_conversion/BS_CT -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/CT_conversion/BS_CT -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else if(strand == 3) { //CTOB Read#1 G->A, Read#2 C->T, Genome G->A, only the + strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/GA_conversion/BS_GA -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/GA_conversion/BS_GA -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else {
fprintf(stderr, "Oh shit, got strand %i!\n", strand);
return;
}
//Start the process
if(!config.quiet) fprintf(stderr, "Node %i executing: %s\n", thread_id, cmd); fflush(stderr);
fp = sam_popen(cmd);
header = sam_hdr_read(fp);
#ifdef DEBUG
sam_hdr_write(of, header);
#endif
#ifndef DEBUG
packed_header = pack_header(header);
if(thread_id == 1) {
//Send the header
MPI_Send((void *) &(packed_header->size), 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
status = MPI_Send((void *) packed_header->packed, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD);
if(status != MPI_SUCCESS) {
fprintf(stderr, "MPI_Send returned %i\n", status);
fflush(stderr);
}
}
#else
packed_header = pack_header(header);
void *tmp_pointer = malloc(packed_header->size);
assert(tmp_pointer);
MPI_Request request;
MPI_Isend((void *) packed_header->packed, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD, &request);
status = MPI_Recv(tmp_pointer, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD, &stat);
if(status != MPI_SUCCESS) fprintf(stderr, "We seem to have not been able to send the message to ourselves!\n");
MPI_Wait(&request, &stat);
unpack_header(debug_header, tmp_pointer);
global_header = debug_header;
free(tmp_pointer);
#endif
t0 = time(NULL);
if(!config.quiet) fprintf(stderr, "Node %i began sending reads @%s", thread_id, ctime(&t0)); fflush(stderr);
while(sam_read1(fp, header, read1) >= 0) {
#ifdef DEBUG
sam_write1(of, global_header, read1);
#endif
if(strcmp(bam_get_qname(read1), last_qname) == 0) { //Multimapper
if(config.paired) {
sam_read1(fp, header, read2);
#ifdef DEBUG
sam_write1(of, global_header, read2);
#endif
}
continue;
} else {
if(read1->core.l_qname > max_qname) {
max_qname = read1->core.l_qname + 10;
last_qname = realloc(last_qname, sizeof(char) * max_qname);
assert(last_qname);
}
strcpy(last_qname, bam_get_qname(read1));
}
//Are paired-end reads in the wrong order?
swapped = 0;
if(config.paired) {
if(read1->core.flag & BAM_FREAD2) {
swapped = 1;
sam_read1(fp, header, read2);
packed_read = pack_read(read2, packed_read);
#ifndef DEBUG
MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
#else
sam_write1(of, global_header, read2);
if(packed_read->size > current_p_size) {
p = realloc(p, packed_read->size);
assert(p);
}
MPI_Isend((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request);
status = MPI_Recv(p, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat);
MPI_Wait(&request, &stat);
debug_read = unpack_read(debug_read, p);
#endif
}
}
//Send the read
packed_read = pack_read(read1, packed_read);
#ifndef DEBUG
MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
#else
if(packed_read->size > current_p_size) {
p = realloc(p, packed_read->size);
assert(p);
}
MPI_Isend(packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request);
status = MPI_Recv(p, packed_header->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat);
MPI_Wait(&request, &stat);
#endif
//Deal with paired-end reads
if(config.paired && !swapped) {
sam_read1(fp, header, read2);
packed_read = pack_read(read2, packed_read);
#ifndef DEBUG
MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
#else
sam_write1(of, global_header, read2);
if(packed_read->size > current_p_size) {
p = realloc(p, packed_read->size);
assert(p);
}
MPI_Isend((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request);
status = MPI_Recv(p, packed_header->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat);
MPI_Wait(&request, &stat);
debug_read = unpack_read(debug_read, p);
#endif
}
#ifndef DEBUG
i++;
#endif
}
t1 = time(NULL);
if(!config.quiet) fprintf(stderr, "Node %i finished sending reads @%s\t(%f sec elapsed)\n", thread_id, ctime(&t1), difftime(t1, t0)); fflush(stderr);
//Notify the master node
packed_read->size = 0;
#ifndef DEBUG
void *A = malloc(1);
assert(A);
MPI_Send(A, 1, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
free(A);
#endif
//Close things up
bam_hdr_destroy(header);
bam_destroy1(read1);
bam_destroy1(read2);
free(cmd);
if(packed_read->packed != NULL) free(packed_read->packed);
free(packed_read);
if(packed_header->packed != NULL) free(packed_header->packed);
free(packed_header);
free(last_qname);
sam_pclose(fp);
//Remove the FIFO(s)
unlink(fastq1);
if(config.paired) unlink(fastq2);
#ifdef DEBUG
sam_close(of);
bam_hdr_destroy(debug_header);
bam_destroy1(debug_read);
free(p);
#endif
if(!config.quiet) fprintf(stderr, "Exiting worker node %i\n", thread_id); fflush(stderr);
};
