void test_unpack_header_should_fail_if_the_header_is_the_wrong_size(void)
{
KineticPDUHeader header = {0};
KineticPDUHeader header_out = {0};
TEST_ASSERT_FALSE(unpack_header((uint8_t *)&header, sizeof(header) - 1, &header_out));
TEST_ASSERT_FALSE(unpack_header((uint8_t *)&header, sizeof(header) + 1, &header_out));
}
void test_unpack_header_should_reject_header_with_excessively_large_sizes(void)
{
KineticPDUHeader header = {0};
KineticPDUHeader header_out = {0};
header.protobufLength = PDU_PROTO_MAX_LEN + 1;
TEST_ASSERT_FALSE(unpack_header((uint8_t *)&header, sizeof(header), &header_out));
header.protobufLength = PDU_PROTO_MAX_LEN;
header.valueLength = PDU_PROTO_MAX_LEN + 1;
TEST_ASSERT_FALSE(unpack_header((uint8_t *)&header, sizeof(header), &header_out));
}
/* Shamelessly copied from JPilot (libplugin.c) */
static int read_header(FILE *pc_in, PC3RecordHeader *header) {
unsigned long l, len;
unsigned char packed_header[256];
int num;
num = fread(&l, sizeof(l), 1, pc_in);
if (feof(pc_in)) {
return -1;
}
if (num!=1) {
return num;
}
memcpy(packed_header, &l, sizeof(l));
len=ntohl(l);
if (len > 255) {
return -1;
}
num = fread(packed_header+sizeof(l), len-sizeof(l), 1, pc_in);
if (feof(pc_in)) {
return -1;
}
if (num!=1) {
return num;
}
unpack_header(header, packed_header);
return 1;
}
vector< vector<CVertex> > CSXFMap::s_height_map::load(const string fname)
{
unsigned v, u;
pnts.clear();
#ifdef EMBEDDED // Ограничение - только для МГК
header.width = 1000;
header.height = 1000;
for(v = 0; v < header.height; v++)
{
vector<CVertex> row;
for(u = 0; u < header.width; u++)
row.push_back(CVertex(u, v, 100));
pnts.push_back(row);
}
#else
unsigned t, elem_num;
float * p_buf;
shared_ptr<float> buf;
CFile fl(fname);
fl(& header, sizeof(header));
unpack_header();
check();
elem_num = header.height * header.width;
buf.reset(new float[elem_num], std::default_delete<float[]>());
p_buf = buf.get();
throw_null(p_buf);
fl(p_buf, elem_num * sizeof(float));
CFile::unpack<float>(p_buf, elem_num);
for(u = 0, t = 0; u < header.width; u++)
{
vector<CVertex> col;
for(v = 0; v < header.height; v++, t++)
col.push_back(CVertex(u, v, p_buf[t]));
pnts.push_back(col);
}
#endif
return pnts;
}
/*
* Fill the hostent from the given DNS packet.
*/
static int
hostent_from_packet(struct hostent *h, int action, char *pkt, size_t pktlen)
{
struct packed p;
struct header hdr;
struct query q;
struct rr rr;
int r;
packed_init(&p, pkt, pktlen);
unpack_header(&p, &hdr);
for(; hdr.qdcount; hdr.qdcount--)
unpack_query(&p, &q);
for(; hdr.ancount; hdr.ancount--) {
unpack_rr(&p, &rr);
if (rr.rr_class != C_IN)
continue;
switch (rr.rr_type) {
case T_CNAME:
if (action == ASR_GETHOSTBYNAME)
r = hostent_add_alias(h, rr.rr_dname, 1);
else
r = hostent_set_cname(h, rr.rr_dname, 1);
break;
case T_PTR:
if (action != ASR_GETHOSTBYADDR)
continue;
r = hostent_set_cname(h, rr.rr.ptr.ptrname, 1);
/* XXX See if we need MULTI_PTRS_ARE_ALIASES */
break;
case T_A:
if (h->h_addrtype != AF_INET)
break;
r = hostent_set_cname(h, rr.rr_dname, 1);
r = hostent_add_addr(h, &rr.rr.in_a.addr, 4);
break;
case T_AAAA:
if (h->h_addrtype != AF_INET6)
break;
r = hostent_set_cname(h, rr.rr_dname, 1);
r = hostent_add_addr(h, &rr.rr.in_aaaa.addr6, 16);
break;
}
}
return (0);
}
void test_unpack_header_should_unpack_header_fields_from_read_buf(void)
{
uint8_t read_buf[] = {
0xa0, // version prefix
0x00, 0x01, 0x23, 0x45, // protobuf length
0x00, 0x02, 0x34, 0x56, // value length
};
KineticPDUHeader header_out = {0};
TEST_ASSERT(unpack_header(read_buf, sizeof(header_out), &header_out));
TEST_ASSERT_EQUAL(0xa0, header_out.versionPrefix);
TEST_ASSERT_EQUAL(0x12345, header_out.protobufLength);
TEST_ASSERT_EQUAL(0x23456, header_out.valueLength);
}
static void
dns_dispatch_mx_preference(struct asr_result *ar, void *arg)
{
struct dns_session *s = arg;
struct unpack pack;
struct dns_header h;
struct dns_query q;
struct dns_rr rr;
char buf[512];
int error;
if (ar->ar_h_errno) {
if (ar->ar_rcode == NXDOMAIN)
error = DNS_ENONAME;
else if (ar->ar_h_errno == NO_RECOVERY
|| ar->ar_h_errno == NO_DATA)
error = DNS_EINVAL;
else
error = DNS_RETRY;
}
else {
error = DNS_ENOTFOUND;
unpack_init(&pack, ar->ar_data, ar->ar_datalen);
if (unpack_header(&pack, &h) != -1 &&
unpack_query(&pack, &q) != -1) {
for (; h.ancount; h.ancount--) {
if (unpack_rr(&pack, &rr) == -1)
break;
if (rr.rr_type != T_MX)
continue;
print_dname(rr.rr.mx.exchange, buf, sizeof(buf));
buf[strlen(buf) - 1] = '\0';
if (!strcasecmp(s->name, buf)) {
error = DNS_OK;
break;
}
}
}
}
free(ar->ar_data);
m_create(s->p, IMSG_MTA_DNS_MX_PREFERENCE, 0, 0, -1);
m_add_id(s->p, s->reqid);
m_add_int(s->p, error);
if (error == DNS_OK)
m_add_int(s->p, rr.rr.mx.preference);
m_close(s->p);
free(s);
}
static void
dns_dispatch_mx(struct asr_result *ar, void *arg)
{
struct dns_session *s = arg;
struct unpack pack;
struct dns_header h;
struct dns_query q;
struct dns_rr rr;
char buf[512];
size_t found;
if (ar->ar_h_errno && ar->ar_h_errno != NO_DATA) {
m_create(s->p, IMSG_MTA_DNS_HOST_END, 0, 0, -1);
m_add_id(s->p, s->reqid);
if (ar->ar_rcode == NXDOMAIN)
m_add_int(s->p, DNS_ENONAME);
else if (ar->ar_h_errno == NO_RECOVERY)
m_add_int(s->p, DNS_EINVAL);
else
m_add_int(s->p, DNS_RETRY);
m_close(s->p);
free(s);
free(ar->ar_data);
return;
}
found = 0;
unpack_init(&pack, ar->ar_data, ar->ar_datalen);
if (unpack_header(&pack, &h) == -1 || unpack_query(&pack, &q) == -1)
return;
for (; h.ancount; h.ancount--) {
if (unpack_rr(&pack, &rr) == -1)
break;
if (rr.rr_type != T_MX)
continue;
print_dname(rr.rr.mx.exchange, buf, sizeof(buf));
buf[strlen(buf) - 1] = '\0';
dns_lookup_host(s, buf, rr.rr.mx.preference);
found++;
}
free(ar->ar_data);
/* fallback to host if no MX is found. */
if (found == 0)
dns_lookup_host(s, s->name, 0);
}
int main(int argc, char *argv[]) {
// 入力ファイルの指定がなければ，エラー
if (argc < 2) {
ERROR("no file name input");
}
struct _TAG_OBJECT_INFO *object_info;
const char *input_file_name = argv[1];
unpack_header(input_file_name, object_info);
printf("type:%s\n", object_info->type);
printf("size:%s\n", object_info->size);
return 0;
}
void unpack_request(char *buf, dns_packet_t *packet)
{
dns_header_t header;
unpack_header(buf, &header);
packet->header = header;
char *qdata = buf + sizeof(dns_header_t);
packet->questions = calloc(packet->header.qdcount, sizeof(dns_question_t));
dns_question_t *questions = packet->questions;
for (int i = 0; i < packet->header.qdcount; i++) {
dns_question_t question;
unpack_question(qdata, &question);
questions[i] = question;
#ifdef DEBUG
printf("query for: %s\n", label_to_string(question->label, NULL));
#endif
}
packet->answers = NULL;
}
/*
* Fill the hostent from the given DNS packet.
*/
static struct hostent *
hostent_from_packet(int reqtype, int family, char *pkt, size_t pktlen)
{
struct hostent *h;
struct packed p;
struct header hdr;
struct query q;
struct rr rr;
if ((h = hostent_alloc(family)) == NULL)
return (NULL);
packed_init(&p, pkt, pktlen);
unpack_header(&p, &hdr);
for(; hdr.qdcount; hdr.qdcount--)
unpack_query(&p, &q);
for(; hdr.ancount; hdr.ancount--) {
unpack_rr(&p, &rr);
if (rr.rr_class != C_IN)
continue;
switch (rr.rr_type) {
case T_CNAME:
if (reqtype == ASR_GETHOSTBYNAME) {
if (hostent_add_alias(h, rr.rr_dname, 1) == -1)
goto fail;
} else {
if (hostent_set_cname(h, rr.rr_dname, 1) == -1)
goto fail;
}
break;
case T_PTR:
if (reqtype != ASR_GETHOSTBYADDR)
break;
if (hostent_set_cname(h, rr.rr.ptr.ptrname, 1) == -1)
goto fail;
/* XXX See if we need MULTI_PTRS_ARE_ALIASES */
break;
case T_A:
if (reqtype != ASR_GETHOSTBYNAME)
break;
if (family != AF_INET)
break;
if (hostent_set_cname(h, rr.rr_dname, 1) == -1)
goto fail;
if (hostent_add_addr(h, &rr.rr.in_a.addr, 4) == -1)
goto fail;
break;
case T_AAAA:
if (reqtype != ASR_GETHOSTBYNAME)
break;
if (family != AF_INET6)
break;
if (hostent_set_cname(h, rr.rr_dname, 1) == -1)
goto fail;
if (hostent_add_addr(h, &rr.rr.in_aaaa.addr6, 16) == -1)
goto fail;
break;
}
}
return (h);
fail:
free(h);
return (NULL);
}
int32_t parse_message(
uint8_t rx_byte,
parse_data_t* parse_data,
proto_msg_t* rx_msg)
{
switch(parse_data->parse_state){
case WAIT_SYNC_C :
{
if(rx_byte == 'C') parse_data->parse_state = WAIT_SYNC_P1;
parse_data->parse_sum = rx_byte;
rx_msg->data[0] = rx_byte;
break;
}
case WAIT_SYNC_P1 :
{
parse_data->parse_state = (rx_byte == 'P') ? WAIT_SYNC_P2 : WAIT_SYNC_C;
parse_data->parse_sum += rx_byte;
rx_msg->data[1] = rx_byte;
break;
}
case WAIT_SYNC_P2 :
{
parse_data->parse_state = (rx_byte == 'P') ? WAIT_SYNC_NULL : WAIT_SYNC_C;
parse_data->parse_sum += rx_byte;
rx_msg->data[2] = rx_byte;
break;
}
case WAIT_SYNC_NULL :
{
parse_data->parse_state = (rx_byte == 0) ? WAIT_MESSAGE_HEADER : WAIT_SYNC_C;
parse_data->parse_sum += rx_byte;
rx_msg->data[3] = rx_byte;
parse_data->parse_count = 0;
break;
}
case WAIT_MESSAGE_HEADER:
{
rx_msg->data[parse_data->parse_count+4] = rx_byte;
parse_data->parse_sum += rx_byte;
++parse_data->parse_count; /* Store parser count to keep track of header length */
if(parse_data->parse_count == sizeof(proto_header_t))
{
unpack_header(rx_msg->data+4, &rx_msg->header);
/* Checks if parsed message size is too large */
parse_data->parse_state = rx_msg->header.message_length <= 512 ? WAIT_MESSAGE_DATA : WAIT_SYNC_C;
parse_data->parse_count = 0;
}
break;
}
case WAIT_MESSAGE_DATA:
{
rx_msg->data[parse_data->parse_count+12] = rx_byte;
parse_data->parse_sum += rx_byte;
/* Increment parser count until the size of the data section is received */
parse_data->parse_state = parse_data->parse_count < rx_msg->header.message_length-1 ? WAIT_MESSAGE_DATA: WAIT_MESSAGE_CHECKSUM;
++parse_data->parse_count;
break;
}
case WAIT_MESSAGE_CHECKSUM:
{
rx_msg->checksum = rx_byte;
rx_msg->data[rx_msg->header.message_length+12] = rx_byte;
rx_msg->direction = Proto_In;
parse_data->parse_state = WAIT_SYNC_C;
/* The checksum is a two's complement, when added to the sum of bytes, valid messages evaluate to 0 */
if((rx_msg->checksum+parse_data->parse_sum) == 0)
{
return 1;
}
break;
}
default :
{
parse_data->parse_state = WAIT_SYNC_C;
break;
}
}
return 0;
}
/******************************************************************************
*
* The main worker node function.
*
* int thread_id: the thread_id
* char *fastq1: FIFO from which bowtie2 can get read1
* char *fastq2: FIFO from which bowtie2 can get read2 (if it exists)
*
*******************************************************************************/
void herd_worker_node(int thread_id, char *fastq1, char *fastq2) {
int cmd_length = 1, max_qname = 0, status, strand;
char *cmd, *last_qname = calloc(1, sizeof(char));
MPI_Header *packed_header;
MPI_read *packed_read = calloc(1, sizeof(MPI_read));
bam_hdr_t *header;
bam1_t *read1 = bam_init1();
bam1_t *read2 = bam_init1();
samFile *fp;
#ifdef DEBUG
MPI_Status stat;
int current_p_size = 100;
htsFile *of;
bam_hdr_t *debug_header = bam_hdr_init();
bam1_t *debug_read = bam_init1();
global_header = bam_hdr_init();
void *p = calloc(100,1);
char *oname = NULL;
#else
int i = 0;
#endif
time_t t0, t1;
int swapped = 0;
assert(last_qname);
assert(packed_read);
//Which strand should we be aligning to?
if(config.directional) {
strand = (thread_id-1) % 2;
} else {
strand = (thread_id-1) % 4;
}
packed_read->size = 0;
packed_read->packed = NULL;
//construct the bowtie2 command
cmd_length += (int) strlen("bowtie2 -q --reorder") + 1;
cmd_length += (int) strlen(config.bowtie2_options) + 1;
cmd_length += (int) strlen("--norc -x") + 1;
cmd_length += (int) strlen(config.genome_dir) + strlen("bisulfite_genome/CT_conversion/BS_CT") + 1;
cmd_length += (int) 2*(strlen("-1 ") + strlen(fastq1)) + 3;
if(config.paired) cmd_length += (int) strlen(fastq2); //This is likely unneeded.
#ifdef DEBUG
oname = malloc(sizeof(char) *(1+strlen(config.odir)+strlen(config.basename)+strlen("_X.bam")));
assert(oname);
sprintf(oname, "%s%s_%i.bam", config.odir, config.basename, thread_id);
if(!config.quiet) fprintf(stderr, "Writing output to %s\n", oname);
of = sam_open(oname, "wb");
free(oname);
#endif
cmd = (char *) malloc(sizeof(char) * cmd_length);
assert(cmd);
if(strand == 0) { //OT Read#1 C->T, Read#2 G->A, Genome C->T only the + strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/CT_conversion/BS_CT -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/CT_conversion/BS_CT -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else if(strand == 1) { //OB Read#1 C->T, Read#2 G->A, Genome G->A only the - strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/GA_conversion/BS_GA -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/GA_conversion/BS_GA -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else if(strand == 2) { //CTOT Read#1 G->A, Read#2 C->T, Genome C->T, only the - strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/CT_conversion/BS_CT -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --nofw -x %sbisulfite_genome/CT_conversion/BS_CT -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else if(strand == 3) { //CTOB Read#1 G->A, Read#2 C->T, Genome G->A, only the + strand
if(config.paired) {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/GA_conversion/BS_GA -1 %s -2 %s", config.bowtie2_options, config.genome_dir, fastq1, fastq2);
} else {
sprintf(cmd, "bowtie2 -q --reorder %s --norc -x %sbisulfite_genome/GA_conversion/BS_GA -U %s", config.bowtie2_options, config.genome_dir, fastq1);
}
} else {
fprintf(stderr, "Oh shit, got strand %i!\n", strand);
return;
}
//Start the process
if(!config.quiet) fprintf(stderr, "Node %i executing: %s\n", thread_id, cmd); fflush(stderr);
fp = sam_popen(cmd);
header = sam_hdr_read(fp);
#ifdef DEBUG
sam_hdr_write(of, header);
#endif
#ifndef DEBUG
packed_header = pack_header(header);
if(thread_id == 1) {
//Send the header
MPI_Send((void *) &(packed_header->size), 1, MPI_INT, 0, 1, MPI_COMM_WORLD);
status = MPI_Send((void *) packed_header->packed, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD);
if(status != MPI_SUCCESS) {
fprintf(stderr, "MPI_Send returned %i\n", status);
fflush(stderr);
}
}
#else
packed_header = pack_header(header);
void *tmp_pointer = malloc(packed_header->size);
assert(tmp_pointer);
MPI_Request request;
MPI_Isend((void *) packed_header->packed, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD, &request);
status = MPI_Recv(tmp_pointer, packed_header->size, MPI_BYTE, 0, 2, MPI_COMM_WORLD, &stat);
if(status != MPI_SUCCESS) fprintf(stderr, "We seem to have not been able to send the message to ourselves!\n");
MPI_Wait(&request, &stat);
unpack_header(debug_header, tmp_pointer);
global_header = debug_header;
free(tmp_pointer);
#endif
t0 = time(NULL);
if(!config.quiet) fprintf(stderr, "Node %i began sending reads @%s", thread_id, ctime(&t0)); fflush(stderr);
while(sam_read1(fp, header, read1) >= 0) {
#ifdef DEBUG
sam_write1(of, global_header, read1);
#endif
if(strcmp(bam_get_qname(read1), last_qname) == 0) { //Multimapper
if(config.paired) {
sam_read1(fp, header, read2);
#ifdef DEBUG
sam_write1(of, global_header, read2);
#endif
}
continue;
} else {
if(read1->core.l_qname > max_qname) {
max_qname = read1->core.l_qname + 10;
last_qname = realloc(last_qname, sizeof(char) * max_qname);
assert(last_qname);
}
strcpy(last_qname, bam_get_qname(read1));
}
//Are paired-end reads in the wrong order?
swapped = 0;
if(config.paired) {
if(read1->core.flag & BAM_FREAD2) {
swapped = 1;
sam_read1(fp, header, read2);
packed_read = pack_read(read2, packed_read);
#ifndef DEBUG
MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
#else
sam_write1(of, global_header, read2);
if(packed_read->size > current_p_size) {
p = realloc(p, packed_read->size);
assert(p);
}
MPI_Isend((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request);
status = MPI_Recv(p, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat);
MPI_Wait(&request, &stat);
debug_read = unpack_read(debug_read, p);
#endif
}
}
//Send the read
packed_read = pack_read(read1, packed_read);
#ifndef DEBUG
MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
#else
if(packed_read->size > current_p_size) {
p = realloc(p, packed_read->size);
assert(p);
}
MPI_Isend(packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request);
status = MPI_Recv(p, packed_header->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat);
MPI_Wait(&request, &stat);
#endif
//Deal with paired-end reads
if(config.paired && !swapped) {
sam_read1(fp, header, read2);
packed_read = pack_read(read2, packed_read);
#ifndef DEBUG
MPI_Send((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
#else
sam_write1(of, global_header, read2);
if(packed_read->size > current_p_size) {
p = realloc(p, packed_read->size);
assert(p);
}
MPI_Isend((void *) packed_read->packed, packed_read->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &request);
status = MPI_Recv(p, packed_header->size, MPI_BYTE, 0, 5, MPI_COMM_WORLD, &stat);
MPI_Wait(&request, &stat);
debug_read = unpack_read(debug_read, p);
#endif
}
#ifndef DEBUG
i++;
#endif
}
t1 = time(NULL);
if(!config.quiet) fprintf(stderr, "Node %i finished sending reads @%s\t(%f sec elapsed)\n", thread_id, ctime(&t1), difftime(t1, t0)); fflush(stderr);
//Notify the master node
packed_read->size = 0;
#ifndef DEBUG
void *A = malloc(1);
assert(A);
MPI_Send(A, 1, MPI_BYTE, 0, 5, MPI_COMM_WORLD);
free(A);
#endif
//Close things up
bam_hdr_destroy(header);
bam_destroy1(read1);
bam_destroy1(read2);
free(cmd);
if(packed_read->packed != NULL) free(packed_read->packed);
free(packed_read);
if(packed_header->packed != NULL) free(packed_header->packed);
free(packed_header);
free(last_qname);
sam_pclose(fp);
//Remove the FIFO(s)
unlink(fastq1);
if(config.paired) unlink(fastq2);
#ifdef DEBUG
sam_close(of);
bam_hdr_destroy(debug_header);
bam_destroy1(debug_read);
free(p);
#endif
if(!config.quiet) fprintf(stderr, "Exiting worker node %i\n", thread_id); fflush(stderr);
};
