int untar_gui(const char *args) { char tarFile[1024] = ""; char destFile[1024] = ""; char outputFile[1024] = ""; char tarPos[100 * 1024]; if(sscanf(args, "%s %s %s %s", tarFile, destFile, tarPos, outputFile) != 4) { cerr << "untar: bad arguments" << endl; return(1); } cout << tarFile << endl; cout << destFile << endl; cout << tarPos << endl; cout << outputFile << endl; Tar tar; if(tar.tar_open(tarFile, O_RDONLY) == -1) { cerr << "untar: open file " << tarFile << " failed" << endl; return(1); } FILE *outputFileHandle = fopen(outputFile, "wb"); if(!outputFileHandle) { cerr << "untar: open output file " << outputFile << " failed" << endl; return(1); } tar.tar_read_save_parameters(outputFileHandle); tar.tar_read((string(destFile) + ".*").c_str(), destFile, 0, NULL, tarPos); fclose(outputFileHandle); return(0); }
Tar Reader::read_uncompressed(const uint8_t* data, size_t size) { if (size % SECTOR_SIZE not_eq 0) throw Tar_exception("Invalid size of tar file"); Tar tar; // Go through the whole tar file block by block for (size_t i = 0; i < size; i += SECTOR_SIZE) { Tar_header* header = (Tar_header*) ((const char*) data + i); Element element{*header}; if (element.name().empty()) { // Empty header name - continue continue; } // Check if this is a directory or not (typeflag) (directories have no content) // If typeflag not set -> is not a directory and has content if (not element.typeflag_is_set() or not element.is_dir()) { i += SECTOR_SIZE; // move past header element.set_content_start(data + i); i += (SECTOR_SIZE * (element.num_content_blocks() - 1)); // move to the end of the element } tar.add_element(element); } return tar; }
int TarQueue::write(int qtype, unsigned int time, data_t data) { stringstream tar_dir, tar_name; tar_dir << opt_chdir << "/" << setfill('0') << setw(4) << data.year << setw(1) << "-" << setw(2) << data.mon << setw(1) << "-" << setw(2) << data.day << setw(1) << "/" << setw(2) << data.hour << setw(1) << "/" << setw(2) << data.minute << setw(1) << "/" << setw(0) << qtype2strC(qtype); tar_name << tar_dir.str() << "/" << qtype2str(qtype) << "_" << setfill('0') << setw(4) << data.year << setw(1) << "-" << setw(2) << data.mon << setw(1) << "-" << setw(2) << data.day << setw(1) << "-" << setw(2) << data.hour << setw(1) << "-" << setw(2) << data.minute << ".tar"; switch(qtype) { case 1: switch(opt_pcap_dump_tar_compress_sip) { case 1: tar_name << ".gz"; break; case 2: tar_name << ".xz"; break; } break; case 2: switch(opt_pcap_dump_tar_compress_rtp) { case 1: tar_name << ".gz"; break; case 2: tar_name << ".xz"; break; } break; case 3: switch(opt_pcap_dump_tar_compress_graph) { case 1: tar_name << ".gz"; break; case 2: tar_name << ".xz"; break; } break; } mkdir_r(tar_dir.str(), 0777); //printf("tar_name %s\n", tar_name.str().c_str()); pthread_mutex_lock(&tarslock); Tar *tar = tars[tar_name.str()]; if(!tar) { tar = new FILE_LINE Tar; lock_okTarPointers(); okTarPointers[tar] = glob_last_packet_time; unlock_okTarPointers(); if(sverb.tar) syslog(LOG_NOTICE, "new tar %s\n", tar_name.str().c_str()); if(sverb.tar) syslog(LOG_NOTICE, "add tar pointer %lx\n", (long)tar); tars[tar_name.str()] = tar; pthread_mutex_unlock(&tarslock); tar->tar_open(tar_name.str(), O_WRONLY | O_CREAT | O_APPEND, 0777, TAR_GNU); tar->tar.qtype = qtype; tar->created_at = time; tar->year = data.year; tar->mon = data.mon; tar->day = data.day; tar->hour = data.hour; tar->minute = data.minute; if(sverb.tar > 2) { char dateTimeString[20]; sprintf(dateTimeString, "%4i-%02i-%02i %02i:%02i:00", data.year, data.mon, data.day, data.hour, data.minute); if(dateTimeString != sqlDateTimeString(tar->created_at)) { syslog(LOG_ERR, "BAD TAR set created_at: %s %lx %s %s %i %i", tar->pathname.c_str(), (long)tar, dateTimeString, sqlDateTimeString(tar->created_at).c_str(), time, data.buffer->getTime()); } } tar->thread_id = tarThreadCounter[qtype] % maxthreads; ++tarThreadCounter[qtype]; if(sverb.tar) { syslog(LOG_NOTICE, "tar %s to thread %i", tar->pathname.c_str(), tar->thread_id); } } else { pthread_mutex_unlock(&tarslock); } data.tar = tar; data.time = time; tarthreads[tar->thread_id].qlock(); // printf("push id:%u\n", tar->thread_id); tarthreads[tar->thread_id].queue_data[tar_name.str()].push_back(data); tarthreads[tar->thread_id].qunlock(); return 0; }
inline void TarQueue::tarthreads_t::processData(data_t *data, bool isClosed, size_t lenForProceed, size_t lenForProceedSafe) { #if TAR_PROF unsigned long long __prof_begin = rdtsc(); unsigned long long __prof_i1 = __prof_begin; unsigned long long __prof_i2 = __prof_begin; #endif Tar *tar = data->tar; tar->writing = 1; if(lenForProceedSafe) { data->buffer->addTarPosInCall(tar->tarLength); //reset and set header memset(&(tar->tar.th_buf), 0, sizeof(struct Tar::tar_header)); tar->th_set_type(0); //s->st_mode, 0 is regular file tar->th_set_user(0); //st_uid tar->th_set_group(0); //st_gid tar->th_set_mode(0444); //s->st_mode tar->th_set_mtime(data->time); tar->th_set_size(lenForProceedSafe); tar->th_set_path((char*)data->filename.c_str(), !isClosed); #if TAR_PROF __prof_i1 = rdtsc(); #endif // write header if (tar->th_write() == 0) { // if it's a regular file, write the contents as well #if TAR_PROF __prof_i2 = rdtsc(); #endif tar->tar_append_buffer(data->buffer, lenForProceedSafe); if(sverb.chunk_buffer > 2) { cout << " *** " << data->buffer->getName() << " " << lenForProceedSafe << endl; } } } tar->writing = 0; #if TAR_PROF unsigned long long __prof_i3 = rdtsc(); #endif if(isClosed && (!lenForProceed || lenForProceed > lenForProceedSafe)) { decreaseTartimemap(data->buffer->getTime()); if(sverb.tar > 2) { syslog(LOG_NOTICE, "tartimemap decrease1: %s %i %i %i %i", data->buffer->getName().c_str(), tar->created_at, tar->created_at - tar->created_at % TAR_MODULO_SECONDS, data->buffer->getTime(), data->buffer->getTime() - data->buffer->getTime() % TAR_MODULO_SECONDS); if(tar->created_at != (unsigned)(data->buffer->getTime() - data->buffer->getTime() % TAR_MODULO_SECONDS)) { syslog(LOG_ERR, "BAD TAR created_at - tar: %s %lx %i %i chunkbuffer: %s %lx %i %i", tar->pathname.c_str(), (long)tar, tar->created_at, tar->created_at - tar->created_at % TAR_MODULO_SECONDS, data->buffer->getName().c_str(), (long)data->buffer, data->buffer->getTime(), data->buffer->getTime() - data->buffer->getTime() % TAR_MODULO_SECONDS); } } delete data->buffer; //tar->incClosedPartCounter(); __sync_sub_and_fetch(&glob_tar_queued_files, 1); } #if TAR_PROF unsigned long long __prof_end = rdtsc(); __prof_processData_sum_1 += __prof_end - __prof_begin; __prof_processData_sum_2 += __prof_i1 - __prof_begin; __prof_processData_sum_3 += __prof_i2 - __prof_i1; __prof_processData_sum_4 += __prof_i3 - __prof_i2; __prof_processData_sum_5 += __prof_end - __prof_i3; #endif }
void *TarQueue::tarthreadworker(void *arg) { TarQueue *this2 = ((tarthreadworker_arg*)arg)->tq; tarthreads_t *tarthread = &this2->tarthreads[((tarthreadworker_arg*)arg)->i]; tarthread->thread_id = ((tarthreadworker_arg*)arg)->i; delete (tarthreadworker_arg*)arg; tarthread->threadId = get_unix_tid(); while(1) { int terminate_pass = terminated_tar_flush_queue; while(1) { bool doProcessData = false; if(tarthread->queue_data.empty()) { if(this2->terminate) { return NULL; } } else { /* Tar *maxTar = tarthread->getTarWithMaxLen(2, false); if(!maxTar) { maxTar = tarthread->getTarWithMaxLen(false, false); } if(!maxTar) { break; } Tar *processTar = maxtar; */ #if TAR_PROF static unsigned counter; ++counter; unsigned long long __prof_begin = rdtsc(); unsigned long long __prof_sum_1 = 0; unsigned long long __prof_sum_2 = 0; unsigned long long __prof_sum_3 = 0; unsigned long long __prof_sum_4 = 0; unsigned long long __prof_sum_5 = 0; unsigned long long __prof_sum_6 = 0; __prof_processData_sum_1 = 0; __prof_processData_sum_2 = 0; __prof_processData_sum_3 = 0; __prof_processData_sum_4 = 0; __prof_processData_sum_5 = 0; #endif tarthread->qlock(); list<string> listTars; std::map<string, tarthreads_tq>::iterator it = tarthread->queue_data.begin(); while(it != tarthread->queue_data.end()) { listTars.push_back(it->first); ++it; } tarthread->qunlock(); for(list<string>::iterator itTars = listTars.begin(); itTars != listTars.end(); itTars++) { string processTarName = *itTars; tarthreads_tq *processTarQueue = &tarthread->queue_data[*itTars]; bool doProcessDataTar = false; size_t index_list = 0; size_t length_list = processTarQueue->size(); size_t count_empty = 0; for(std::list<data_t>::iterator it = processTarQueue->begin(); index_list < length_list;) { if(index_list++) ++it; if(!it->buffer) { ++count_empty; continue; } data_t data = *it; /* if(data.buffer->isDecompressError()) { if(verbosity) { syslog(LOG_NOTICE, "tar: DECOMPRESS ERROR"); } //tarthread->queue[processTar].erase(tarthread->queue[processTar].begin() + index_list); //--length_list; //--index_list; data.buffer = NULL; tarthread->queue[processTar][index_list].buffer = NULL; ++count_empty; continue; } lock_okTarPointers(); if(okTarPointers.find(data.tar) == okTarPointers.end()) { if(verbosity) { syslog(LOG_NOTICE, "tar: BAD TAR"); } //tarthread->queue[processTar].erase(tarthread->queue[processTar].begin() + index_list); //--length_list; //--index_list; data.buffer = NULL; tarthread->queue[processTar][index_list].buffer = NULL; ++count_empty; unlock_okTarPointers(); continue; } unlock_okTarPointers(); */ bool isClosed = data.buffer->isClosed(); if(!isClosed && !data.buffer->isNewLastAddTimeForTar() && !data.buffer->isFull()) { continue; } data.buffer->copyLastAddTimeToTar(); unsigned int bufferLastTarTime = data.buffer->getLastTarTime(); if(!isClosed && bufferLastTarTime && bufferLastTarTime > glob_last_packet_time - 3 && !data.buffer->isFull()) { continue; } data.buffer->setLastTarTime(glob_last_packet_time); #if TAR_PROF unsigned long long __prof_begin2 = rdtsc(); #endif size_t lenForProceed = data.buffer->getChunkIterateLenForProceed(); size_t lenForProceedSafe = lenForProceed; #if TAR_PROF unsigned long long __prof_i1 = rdtsc(); #endif if(!isClosed && lenForProceedSafe > TAR_CHUNK_KB * 1024) { lenForProceedSafe = data.buffer->getChunkIterateSafeLimitLength(lenForProceedSafe); } #if TAR_PROF unsigned long long __prof_i2 = rdtsc(); #endif if(isClosed || lenForProceedSafe > TAR_CHUNK_KB * 1024) { doProcessData = true; doProcessDataTar = true; #if TAR_PROF unsigned long long __prof_i21 = rdtsc(); #endif tarthread->processData(&data, isClosed, lenForProceed, lenForProceedSafe); #if TAR_PROF unsigned long long __prof_i22 = rdtsc(); __prof_sum_5 += __prof_i22 - __prof_i21; #endif if(isClosed && (!lenForProceed || lenForProceed > lenForProceedSafe)) { //tarthread->queue[processTar].erase(tarthread->queue[processTar].begin() + index_list); //--length_list; //--index_list; data.buffer = NULL; it->buffer = NULL; ++count_empty; } #if TAR_PROF unsigned long long __prof_i23 = rdtsc(); __prof_sum_6 += __prof_i23 - __prof_i22; #endif } #if TAR_PROF unsigned long long __prof_end2 = rdtsc(); __prof_sum_1 += __prof_end2 - __prof_begin2; __prof_sum_2 += __prof_i1 - __prof_begin2; __prof_sum_3 += __prof_i2 - __prof_i1; __prof_sum_4 += __prof_end2 - __prof_i2; #endif } bool eraseTarQueueItem = false; //if(!tarthread->queue[processTar].size()) { if(processTarQueue->size() == count_empty) { pthread_mutex_lock(&this2->tarslock); if(this2->tars.find(processTarName) == this2->tars.end()) { tarthread->queue_data.erase(processTarName); eraseTarQueueItem = true; } pthread_mutex_unlock(&this2->tarslock); } if(!eraseTarQueueItem) { if(count_empty > processTarQueue->size() / 5) { tarthread->qlock(); for(std::list<data_t>::iterator it = processTarQueue->begin(); it != processTarQueue->end();) { if(!it->buffer) { processTarQueue->erase(it++); } else { it++; } } tarthread->qunlock(); } } if(!doProcessDataTar) { unsigned int lastAddTime = 0; if(!eraseTarQueueItem) { tarthread->qlock(); lastAddTime = processTarQueue->getLastAddTime(); tarthread->qunlock(); } if(!lastAddTime || lastAddTime < glob_last_packet_time - 30) { pthread_mutex_lock(&this2->tarslock); if(this2->tars.find(processTarName) != this2->tars.end()) { Tar *processTar = this2->tars[processTarName]; if(processTar->lastWriteTime && processTar->lastWriteTime < glob_last_packet_time - 30 && processTar->lastFlushTime < processTar->lastWriteTime - 30) { processTar->flush(); processTar->lastFlushTime = glob_last_packet_time; } } pthread_mutex_unlock(&this2->tarslock); } } } #if TAR_PROF unsigned long long __prof_end = rdtsc(); if(100 * __prof_sum_1 / (__prof_end - __prof_begin)) { cout << "**** " << counter << " : " << (100 * __prof_sum_1 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_sum_2 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_sum_3 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_sum_4 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_sum_5 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_sum_6 / (__prof_end - __prof_begin)) << "% " << " - " << (100 * __prof_processData_sum_1 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_processData_sum_2 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_processData_sum_3 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_processData_sum_4 / (__prof_end - __prof_begin)) << "% " << (100 * __prof_processData_sum_5 / (__prof_end - __prof_begin)) << "% " << endl; } #endif } if(!doProcessData) { break; } } // quque is empty - sleep before next run usleep(terminating ? 100000 : 250000); if(terminate_pass) { break; } } tarthread->threadEnd = true; return NULL; }
Format* GetType(void* file_pointer, size_t file_size, const string& filename) { if (depth > max_depth) return new Format(file_pointer, file_size); if (!filename.empty()) { size_t dot = filename.find_last_of('.'); if (dot != string::npos) { string ext = filename.substr(dot + 1); // toupper for (auto& c : ext) c &= ~0x20; if (ext == "HTML" || ext == "HTM" || ext == "JS" || ext == "CSS") { VerbosePrint(ext, " detected."); return new DataURI(file_pointer, file_size); } if (ext == "VCF" || ext == "VCARD") { VerbosePrint(ext, " detected."); return new Vcf(file_pointer, file_size); } if (ext == "MHT" || ext == "MHTML" || ext == "MIM" || ext == "MIME" || ext == "EML") { VerbosePrint(ext, " detected."); return new Mime(file_pointer, file_size); } } } if (memcmp(file_pointer, Png::header_magic, sizeof(Png::header_magic)) == 0) { VerbosePrint("PNG detected."); return new Png(file_pointer, file_size); } else if (memcmp(file_pointer, Jpeg::header_magic, sizeof(Jpeg::header_magic)) == 0) { VerbosePrint("JPEG detected."); return new Jpeg(file_pointer, file_size); } else if (memcmp(file_pointer, Lua::header_magic, sizeof(Lua::header_magic)) == 0) { VerbosePrint("Lua detected."); return new Lua(file_pointer, file_size); } else if (memcmp(file_pointer, Zip::header_magic, sizeof(Zip::header_magic)) == 0) { VerbosePrint("ZIP detected."); return new Zip(file_pointer, file_size); } else if (memcmp(file_pointer, Pe::header_magic, sizeof(Pe::header_magic)) == 0) { VerbosePrint("PE detected."); return new Pe(file_pointer, file_size); } else if (memcmp(file_pointer, Gz::header_magic, sizeof(Gz::header_magic)) == 0) { VerbosePrint("GZ detected."); return new Gz(file_pointer, file_size); } else if (memcmp(file_pointer, Ico::header_magic, sizeof(Ico::header_magic)) == 0) { VerbosePrint("ICO detected."); return new Ico(file_pointer, file_size); } else if (memcmp(file_pointer, Dwf::header_magic, sizeof(Dwf::header_magic)) == 0) { VerbosePrint("DWF detected."); return new Dwf(file_pointer, file_size); } else if (memcmp(file_pointer, Gft::header_magic, sizeof(Gft::header_magic)) == 0) { VerbosePrint("GFT detected."); return new Gft(file_pointer, file_size); } else if (memcmp(file_pointer, Rdb::header_magic, sizeof(Rdb::header_magic)) == 0) { VerbosePrint("RDB detected."); return new Rdb(file_pointer, file_size); } else if (memcmp(file_pointer, Swf::header_magic, sizeof(Swf::header_magic)) == 0 || memcmp(file_pointer, Swf::header_magic_deflate, sizeof(Swf::header_magic_deflate)) == 0 || memcmp(file_pointer, Swf::header_magic_lzma, sizeof(Swf::header_magic_lzma)) == 0) { VerbosePrint("SWF detected."); return new Swf(file_pointer, file_size); } else { // Search for vcard magic which might not be at the very beginning. const string vcard_magic = "BEGIN:VCARD"; const char* fp = static_cast<char*>(file_pointer); const char* search_end = fp + std::min(static_cast<size_t>(1024), file_size); if (std::search(fp, search_end, vcard_magic.begin(), vcard_magic.end()) < search_end) { VerbosePrint("VCF detected."); return new Vcf(file_pointer, file_size); } // tar file does not have header magic // ustar is optional { Tar* t = new Tar(file_pointer, file_size); // checking first record checksum if (t->IsValid()) { VerbosePrint("tar detected."); return t; } delete t; } // XML file does not have header magic // have to parse and see if there are any errors. { Xml* x = new Xml(file_pointer, file_size); if (x->IsValid()) { VerbosePrint("XML detected."); return x; } delete x; } } VerbosePrint("Format not supported!"); // for unsupported format, just memmove it. return new Format(file_pointer, file_size); }
Format *GetType(void *file_pointer, size_t file_size, const string& filename) { if (depth > max_depth) { return new Format(file_pointer, file_size); } if (!filename.empty()) { size_t dot = filename.find_last_of('.'); if (dot != string::npos) { string ext = filename.substr(dot + 1); // toupper for (auto &c : ext) c &= ~0x20; if (ext == "HTML" || ext == "HTM" || ext == "JS" || ext == "CSS") { if (is_verbose) { cout << ext << " detected." << endl; } return new DataURI(file_pointer, file_size); } } } if (memcmp(file_pointer, Png::header_magic, sizeof(Png::header_magic)) == 0) { if (is_verbose) { cout << "PNG detected." << endl; } return new Png(file_pointer, file_size); } else if (memcmp(file_pointer, Jpeg::header_magic, sizeof(Jpeg::header_magic)) == 0) { if (is_verbose) { cout << "JPEG detected." << endl; } return new Jpeg(file_pointer, file_size); } else if (memcmp(file_pointer, Lua::header_magic, sizeof(Lua::header_magic)) == 0) { if (is_verbose) { cout << "Lua detected." << endl; } return new Lua(file_pointer, file_size); } else if (memcmp(file_pointer, Zip::header_magic, sizeof(Zip::header_magic)) == 0) { if (is_verbose) { cout << "ZIP detected." << endl; } return new Zip(file_pointer, file_size); } else if (memcmp(file_pointer, Pe::header_magic, sizeof(Pe::header_magic)) == 0) { if (is_verbose) { cout << "PE detected." << endl; } return new Pe(file_pointer, file_size); } else if (memcmp(file_pointer, Gz::header_magic, sizeof(Gz::header_magic)) == 0) { if (is_verbose) { cout << "GZ detected." << endl; } return new Gz(file_pointer, file_size); } else if (memcmp(file_pointer, Ico::header_magic, sizeof(Ico::header_magic)) == 0) { if (is_verbose) { cout << "ICO detected." << endl; } return new Ico(file_pointer, file_size); } else if (memcmp(file_pointer, Dwf::header_magic, sizeof(Dwf::header_magic)) == 0) { if (is_verbose) { cout << "DWF detected." << endl; } return new Dwf(file_pointer, file_size); } else if (memcmp(file_pointer, Gft::header_magic, sizeof(Gft::header_magic)) == 0) { if (is_verbose) { cout << "GFT detected." << endl; } return new Gft(file_pointer, file_size); } else if (memcmp(file_pointer, Rdb::header_magic, sizeof(Rdb::header_magic)) == 0) { if (is_verbose) { cout << "RDB detected." << endl; } return new Rdb(file_pointer, file_size); } else if (memcmp(file_pointer, Swf::header_magic, sizeof(Swf::header_magic)) == 0 || memcmp(file_pointer, Swf::header_magic_deflate, sizeof(Swf::header_magic_deflate)) == 0 || memcmp(file_pointer, Swf::header_magic_lzma, sizeof(Swf::header_magic_lzma)) == 0) { if (is_verbose) { cout << "SWF detected." << endl; } return new Swf(file_pointer, file_size); } else { // tar file does not have header magic // ustar is optional { Tar *t = new Tar(file_pointer, file_size); // checking first record checksum if (t->IsValid()) { if (is_verbose) { cout << "tar detected." << endl; } return t; } delete t; } // XML file does not have header magic // have to parse and see if there are any errors. { Xml *x = new Xml(file_pointer, file_size); if (x->IsValid()) { if (is_verbose) { cout << "XML detected." << endl; } return x; } delete x; } } if (is_verbose) { cout << "Format not supported!" << endl; } // for unsupported format, just memmove it. return new Format(file_pointer, file_size); }