void* compressZlib(void* buffin) { const size_t SIZEBUFF = 400000; z_stream defstream; defstream.zalloc = Z_NULL; defstream.zfree = Z_NULL; defstream.opaque = Z_NULL; ByteStreamPtr localBuffer[PACKET_NUM]; for(int n=0; n<NTIMES; n++) { // copy PACKET_NUM packets data locally pthread_mutex_lock(&lockp); for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr data = sharedDataAndInfoBuffer[sharedIndex].data; sharedIndex = (sharedIndex+1) % sharedDataAndInfoBuffer.size(); sizeMB += (data->size() / 1000000.0); #ifdef DEBUG std::cout << "data size " << data->size() << std::endl; std::cout << "tot size (MB) " << sizeMB << std::endl; #endif localBuffer[m] = data; } pthread_mutex_unlock(&lockp); int compbytes = 0; for(int m=0; m<PACKET_NUM; m++) { unsigned char outbuff[SIZEBUFF]; ByteStreamPtr data = localBuffer[m]; // compress defstream.avail_in = (uInt)data->size(); defstream.next_in = (Bytef *)data->getStream(); defstream.avail_out = (uInt) SIZEBUFF; defstream.next_out = (Bytef *)outbuff; deflateInit(&defstream, COMPRESSION_LEVEL); deflate(&defstream, Z_FINISH); deflateEnd(&defstream); // update local byte counter compbytes += ((unsigned char*) defstream.next_out - outbuff); } // update global byte counter pthread_mutex_lock(&lockp); totbytescomp += compbytes; pthread_mutex_unlock(&lockp); } return 0; }
void* decompressZlib(void* buffin) { const size_t SIZEBUFF = 400000; z_stream infstream; infstream.zalloc = Z_NULL; infstream.zfree = Z_NULL; infstream.opaque = Z_NULL; ByteStreamPtr localBuffer[PACKET_NUM]; for(int n=0; n<NTIMES; n++) { // copy PACKET_NUM packets data locally pthread_mutex_lock(&lockp); for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr data = sharedDataBuffer[sharedIndex]; sharedIndex = (sharedIndex+1) % sharedDataBuffer.size(); sizeMB += (data->size() / 1000000.0); #ifdef DEBUG std::cout << "data size " << data->size() << std::endl; std::cout << "tot size MB " << sizeMB << std::endl; #endif localBuffer[m] = data; } pthread_mutex_unlock(&lockp); int decompbytes = 0; for(int m=0; m<PACKET_NUM; m++) { unsigned char outbuff[SIZEBUFF]; ByteStreamPtr data = localBuffer[m]; infstream.avail_in = (uInt)data->size(); infstream.next_in = (Bytef *)data->getStream(); infstream.avail_out = (uInt)SIZEBUFF; infstream.next_out = (Bytef *)outbuff; // decompress inflateInit(&infstream); inflate(&infstream, Z_NO_FLUSH); inflateEnd(&infstream); decompbytes += ((unsigned char*) infstream.next_out - outbuff); } // update decompressed byte counter pthread_mutex_lock(&lockp); totbytesdecomp += decompbytes; pthread_mutex_unlock(&lockp); } return 0; }
void* extractWaveData(void* buffin) { unsigned short* maxres = new unsigned short[3000]; unsigned short* timeres = new unsigned short[3000]; // load packet type (once per thread) ByteStreamPtr localBuffer[PACKET_NUM]; int npix[PACKET_NUM]; int nsamp[PACKET_NUM]; for(int n=0; n<NTIMES; n++) { // copy PACKET_NUM packets data locally pthread_mutex_lock(&lockp); for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr data = sharedDataAndInfoBuffer[sharedIndex].data; localBuffer[m] = data; npix[m] = sharedDataAndInfoBuffer[sharedIndex].npix; nsamp[m] = sharedDataAndInfoBuffer[sharedIndex].nsamp; sharedIndex = (sharedIndex+1) % sharedDataAndInfoBuffer.size(); sizeMB += (data->size() / 1000000.0); #ifdef DEBUG std::cout << "data size " << data->size() << std::endl; std::cout << "tot size (MB) " << sizeMB << std::endl; #endif } pthread_mutex_unlock(&lockp); for(int m=0; m<PACKET_NUM; m++) { byte* rawdata = localBuffer[m]->getStream(); rawdata[0] = rand() % 100 + 50; #ifdef DEBUG std::cout << "npixels " << npix[m] << std::endl; std::cout << "nsamples " << nsamp[m] << std::endl; std::cout << "data size " << localBuffer[m]->size() << std::endl; std::cout << "tot size (MB) " << sizeMB << std::endl; #endif // extract waveform calcWaveformExtraction(rawdata, npix[m], nsamp[m], 6, maxres, timeres); maxres[0]++; timeres[0]++; } } return 0; }
bool File::writeByteStream(ByteStreamPtr b) throw(PacketExceptionIO*) { byte* stream = b->encode(); if(fwrite((void*)stream, b->size(), 1, fp)<1) throw new PacketExceptionIO("Can't write stream in the file."); b->endOutputStream(); return true; }
void* decompressLZ4(void* buffin) { ByteStreamPtr localBuffer[PACKET_NUM]; for(int n=0; n<NTIMES; n++) { // copy PACKET_NUM packets data locally pthread_mutex_lock(&lockp); for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr data = sharedDataBuffer[sharedIndex]; sharedIndex = (sharedIndex+1) % sharedDataBuffer.size(); sizeMB += (data->size() / 1000000.0); #ifdef DEBUG std::cout << "data size " << data->size() << std::endl; std::cout << "tot size (MB) " << sizeMB << std::endl; #endif localBuffer[m] = data; } pthread_mutex_unlock(&lockp); int decompbytes = 0; for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr data = localBuffer[m]; // decompress ByteStreamPtr datadecomp = data->decompress(LZ4, COMPRESSION_LEVEL, 400000); // update local byte counter decompbytes += datadecomp->size(); } // update global byte counter pthread_mutex_lock(&lockp); totbytesdecomp += decompbytes; pthread_mutex_unlock(&lockp); } return 0; }
bool PartOfPacket::setByteStream(ByteStreamPtr s) { decoded = false; /// If NULL is passed it exits if(s == NULL) return true; if(size() > s->size()) return false; if(!s->getMemAllocation()) stream->setStream(s->stream, s->size(), s->isBigendian()); /// The stream is assigned this->stream = s; //decode(); return true; }
std::vector<ByteStreamPtr> createZlibBuffer(PacketBufferV* buff) { std::vector<ByteStreamPtr> compbuff; for(int i=0; i<buff->size(); i++) { ByteStreamPtr rawPacket = buff->getNext(); Packet *p = ps->getPacket(rawPacket); #ifdef DEBUG if(p->getPacketID() == 0) { std::cerr << "No packet type recognized" << std::endl; continue; } #endif ByteStreamPtr data = p->getData(); z_stream defstream; defstream.zalloc = Z_NULL; defstream.zfree = Z_NULL; defstream.opaque = Z_NULL; defstream.avail_in = (uInt)data->size(); defstream.next_in = (Bytef *)data->getStream(); const size_t SIZEBUF = 400000; defstream.avail_out = (uInt) SIZEBUF; byte* outbuff = new byte[SIZEBUF]; defstream.next_out = (Bytef *)outbuff; deflateInit(&defstream, COMPRESSION_LEVEL); deflate(&defstream, Z_FINISH); deflateEnd(&defstream); size_t compSize = ((unsigned char*) defstream.next_out - outbuff); ByteStreamPtr out = ByteStreamPtr(new ByteStream(outbuff, compSize, data->isBigendian())); compbuff.push_back(out); } return compbuff; }
void* decodePacket(void* buffin) { PacketBufferV* buff = (PacketBufferV*) buffin; int npix_idx = 0; int nsamp_idx = 0; // load packet type (once per thread) PacketStream* ps; pthread_mutex_lock(&lockp); try { ps = new PacketStream(configFileName.c_str()); Packet *p = ps->getPacketType("triggered_telescope1_30GEN"); npix_idx = p->getPacketSourceDataField()->getFieldIndex("Number of pixels"); nsamp_idx = p->getPacketSourceDataField()->getFieldIndex("Number of samples"); } catch (PacketException* e) { cout << "Error during extractWavePacket: "; cout << e->geterror() << endl; } pthread_mutex_unlock(&lockp); ByteStreamPtr localBuffer[PACKET_NUM]; int npix[PACKET_NUM]; int nsamp[PACKET_NUM]; for(int n=0; n<NTIMES; n++) { // copy PACKET_NUM packets data locally pthread_mutex_lock(&lockp); for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr rawPacket = buff->getNext(); localBuffer[m] = rawPacket; } pthread_mutex_unlock(&lockp); for(int m=0; m<PACKET_NUM; m++) { ByteStreamPtr rawPacket = localBuffer[m]; Packet *p = ps->getPacket(rawPacket); // get npixel and nsamples int npix = p->getPacketSourceDataField()->getFieldValue(npix_idx); int nsamp = p->getPacketSourceDataField()->getFieldValue(nsamp_idx); ByteStreamPtr data = p->getData(); #ifdef ARCH_BIGENDIAN if(!data->isBigendian()) data->swapWord(); #else if(data->isBigendian()) data->swapWord(); #endif sizeMB += (data->size() / 1000000.0); byte* rawdata = data->getStream(); rawdata[0] = rand() % 100 + 50; #ifdef DEBUG std::cout << "npixels " << npix << std::endl; std::cout << "nsamples " << nsamp << std::endl; std::cout << "data size " << data->size() << std::endl; std::cout << "tot size (MB) " << sizeMB << std::endl; #endif } } delete ps; return 0; }