uint16_t recreate_original_payload(unsigned char* payload, uint16_t payload_len, struct ip* ip_hdr) { /* new_packet is already initialized */ assert(ip_hdr->ip_p == DEDUP_MIXED); memset(new_packet, 0, sizeof(unsigned char) * MTU); /* packet is mixed content */ /* iterate over it, read every 2B header and hash/unhash accordingly*/ uint16_t new_packed_upto = 0, unpacked_upto = 0, shim_header = 0, chunk_length=0; uint32_t hash_left = 0, hash_right = 0; uint64_t dedup_hash = 0; while(unpacked_upto < payload_len) { shim_header = ntohs(unpack_buffer(uint16_t, payload, unpacked_upto)); unpacked_upto += 2; if((shim_header & HASH_HDR) == HASH_HDR){ hash_left = ntohl(unpack_buffer(uint32_t, payload, unpacked_upto)); unpacked_upto += 4; hash_right = ntohl(unpack_buffer(uint32_t, payload, unpacked_upto)); unpacked_upto += 4; dedup_hash = hash_right + (((uint64_t)hash_left)<<32); /* unhash this value */ if(hash_memory.find(dedup_hash) == hash_memory.end()){ printlog(logfile, system_loglevel, LOG_CRITICAL, "** Hash not present: %llx\n", dedup_hash); continue; } else{ printlog(logfile, system_loglevel, LOG_CRITICAL, "** Hash hit: %llx\n", dedup_hash); memcpy(new_packet + new_packed_upto, hash_memory[dedup_hash]->chunk, hash_memory[dedup_hash]->chunk_length); new_packed_upto += hash_memory[dedup_hash]->chunk_length; } } else { /* it's a chunk in original text */ /* calculated length from shim_header and read the chunk*/ chunk_length = shim_header; memcpy(new_packet + new_packed_upto, payload + unpacked_upto, chunk_length); unpacked_upto += chunk_length; new_packed_upto += chunk_length; } } assert(unpacked_upto == payload_len); return new_packed_upto; }
/* -------------------- */ static struct finderinfo * unpack_finderinfo(struct vol *vol, struct path *path, struct adouble **adp, struct finderinfo *finfo, int islnk) { packed_finder buf; void *ptr; *adp = adl_lkup(vol, path, *adp); ptr = get_finderinfo(vol, path->u_name, *adp, &buf,islnk); return unpack_buffer(finfo, ptr); }
char *unpack_string(struct tmate_unpacker *uk) { const char *buf; char *alloc_buf; size_t len; unpack_buffer(uk, &buf, &len); alloc_buf = xmalloc(len + 1); memcpy(alloc_buf, buf, len); alloc_buf[len] = '\0'; return alloc_buf; }
void handle_packet(const gras::SBuffer &in_buff) { //extract info size_t seq = 0; size_t sid = 0; bool has_tsf = false; gras::item_index_t tsf = 0; bool is_ext = false; gras::SBuffer out_buff; unpack_buffer(in_buff, seq, sid, has_tsf, tsf, is_ext, out_buff); ASSERT(sid < _num_outs); //handle buffs if (not is_ext) { const size_t item_size = this->output_config(sid).item_size; ASSERT((out_buff.length % item_size) == 0); this->post_output_buffer(sid, out_buff); } //handle tags else if (has_tsf) { gras::Tag tag; tag.offset = tsf; tag.object = buffer_to_pmc(out_buff); this->post_output_tag(sid, tag); } //handle msgs else { PMCC msg = buffer_to_pmc(out_buff); this->post_output_msg(sid, msg); } }
int main(int argc, const char *argv[]) { int opt; poptContext pc; int debug_fd = -1; errno_t ret; int sysvol_gpt_version; int result; TALLOC_CTX *main_ctx = NULL; uint8_t *buf = NULL; ssize_t len = 0; struct input_buffer *ibuf = NULL; struct response *resp = NULL; size_t written; struct poptOption long_options[] = { POPT_AUTOHELP {"debug-level", 'd', POPT_ARG_INT, &debug_level, 0, _("Debug level"), NULL}, {"debug-timestamps", 0, POPT_ARG_INT, &debug_timestamps, 0, _("Add debug timestamps"), NULL}, {"debug-microseconds", 0, POPT_ARG_INT, &debug_microseconds, 0, _("Show timestamps with microseconds"), NULL}, {"debug-fd", 0, POPT_ARG_INT, &debug_fd, 0, _("An open file descriptor for the debug logs"), NULL}, {"debug-to-stderr", 0, POPT_ARG_NONE | POPT_ARGFLAG_DOC_HIDDEN, &debug_to_stderr, 0, _("Send the debug output to stderr directly."), NULL }, POPT_TABLEEND }; /* Set debug level to invalid value so we can decide if -d 0 was used. */ debug_level = SSSDBG_INVALID; pc = poptGetContext(argv[0], argc, argv, long_options, 0); while((opt = poptGetNextOpt(pc)) != -1) { switch(opt) { default: fprintf(stderr, "\nInvalid option %s: %s\n\n", poptBadOption(pc, 0), poptStrerror(opt)); poptPrintUsage(pc, stderr, 0); _exit(-1); } } poptFreeContext(pc); DEBUG_INIT(debug_level); debug_prg_name = talloc_asprintf(NULL, "[sssd[gpo_child[%d]]]", getpid()); if (debug_prg_name == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, "talloc_asprintf failed.\n"); goto fail; } if (debug_fd != -1) { ret = set_debug_file_from_fd(debug_fd); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, "set_debug_file_from_fd failed.\n"); } } DEBUG(SSSDBG_TRACE_FUNC, "gpo_child started.\n"); main_ctx = talloc_new(NULL); if (main_ctx == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, "talloc_new failed.\n"); talloc_free(discard_const(debug_prg_name)); goto fail; } talloc_steal(main_ctx, debug_prg_name); buf = talloc_size(main_ctx, sizeof(uint8_t)*IN_BUF_SIZE); if (buf == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, "talloc_size failed.\n"); goto fail; } ibuf = talloc_zero(main_ctx, struct input_buffer); if (ibuf == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, "talloc_zero failed.\n"); goto fail; } DEBUG(SSSDBG_TRACE_FUNC, "context initialized\n"); errno = 0; len = sss_atomic_read_s(STDIN_FILENO, buf, IN_BUF_SIZE); if (len == -1) { ret = errno; DEBUG(SSSDBG_CRIT_FAILURE, "read failed [%d][%s].\n", ret, strerror(ret)); goto fail; } close(STDIN_FILENO); ret = unpack_buffer(buf, len, ibuf); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, "unpack_buffer failed.[%d][%s].\n", ret, strerror(ret)); goto fail; } DEBUG(SSSDBG_TRACE_FUNC, "performing smb operations\n"); result = perform_smb_operations(ibuf->cached_gpt_version, ibuf->smb_server, ibuf->smb_share, ibuf->smb_path, ibuf->smb_cse_suffix, &sysvol_gpt_version); if (result != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, "perform_smb_operations failed.[%d][%s].\n", result, strerror(result)); goto fail; } ret = prepare_response(main_ctx, sysvol_gpt_version, result, &resp); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, "prepare_response failed. [%d][%s].\n", ret, strerror(ret)); goto fail; } errno = 0; written = sss_atomic_write_s(STDOUT_FILENO, resp->buf, resp->size); if (written == -1) { ret = errno; DEBUG(SSSDBG_CRIT_FAILURE, "write failed [%d][%s].\n", ret, strerror(ret)); goto fail; } if (written != resp->size) { DEBUG(SSSDBG_CRIT_FAILURE, "Expected to write %zu bytes, wrote %zu\n", resp->size, written); goto fail; } DEBUG(SSSDBG_TRACE_FUNC, "gpo_child completed successfully\n"); close(STDOUT_FILENO); talloc_free(main_ctx); return EXIT_SUCCESS; fail: DEBUG(SSSDBG_CRIT_FAILURE, "gpo_child failed!\n"); close(STDOUT_FILENO); talloc_free(main_ctx); return EXIT_FAILURE; }
int main(int argc, const char *argv[]) { int ret; int kerr; int opt; int debug_fd = -1; poptContext pc; TALLOC_CTX *main_ctx = NULL; uint8_t *buf = NULL; ssize_t len = 0; const char *ccname = NULL; time_t expire_time = 0; struct input_buffer *ibuf = NULL; struct response *resp = NULL; size_t written; struct poptOption long_options[] = { POPT_AUTOHELP {"debug-level", 'd', POPT_ARG_INT, &debug_level, 0, _("Debug level"), NULL}, {"debug-timestamps", 0, POPT_ARG_INT, &debug_timestamps, 0, _("Add debug timestamps"), NULL}, {"debug-microseconds", 0, POPT_ARG_INT, &debug_microseconds, 0, _("Show timestamps with microseconds"), NULL}, {"debug-fd", 0, POPT_ARG_INT, &debug_fd, 0, _("An open file descriptor for the debug logs"), NULL}, POPT_TABLEEND }; /* Set debug level to invalid value so we can decide if -d 0 was used. */ debug_level = SSSDBG_INVALID; pc = poptGetContext(argv[0], argc, argv, long_options, 0); while((opt = poptGetNextOpt(pc)) != -1) { switch(opt) { default: fprintf(stderr, "\nInvalid option %s: %s\n\n", poptBadOption(pc, 0), poptStrerror(opt)); poptPrintUsage(pc, stderr, 0); _exit(-1); } } poptFreeContext(pc); DEBUG_INIT(debug_level); debug_prg_name = talloc_asprintf(NULL, "[sssd[ldap_child[%d]]]", getpid()); if (!debug_prg_name) { DEBUG(SSSDBG_CRIT_FAILURE, ("talloc_asprintf failed.\n")); goto fail; } if (debug_fd != -1) { ret = set_debug_file_from_fd(debug_fd); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, ("set_debug_file_from_fd failed.\n")); } } DEBUG(SSSDBG_TRACE_FUNC, ("ldap_child started.\n")); main_ctx = talloc_new(NULL); if (main_ctx == NULL) { DEBUG(SSSDBG_CRIT_FAILURE, ("talloc_new failed.\n")); talloc_free(discard_const(debug_prg_name)); goto fail; } talloc_steal(main_ctx, debug_prg_name); buf = talloc_size(main_ctx, sizeof(uint8_t)*IN_BUF_SIZE); if (buf == NULL) { DEBUG(1, ("talloc_size failed.\n")); goto fail; } ibuf = talloc_zero(main_ctx, struct input_buffer); if (ibuf == NULL) { DEBUG(1, ("talloc_size failed.\n")); goto fail; } DEBUG(SSSDBG_TRACE_INTERNAL, ("context initialized\n")); errno = 0; len = sss_atomic_read_s(STDIN_FILENO, buf, IN_BUF_SIZE); if (len == -1) { ret = errno; DEBUG(SSSDBG_CRIT_FAILURE, ("read failed [%d][%s].\n", ret, strerror(ret))); goto fail; } close(STDIN_FILENO); ret = unpack_buffer(buf, len, ibuf); if (ret != EOK) { DEBUG(1, ("unpack_buffer failed.[%d][%s].\n", ret, strerror(ret))); goto fail; } DEBUG(SSSDBG_TRACE_INTERNAL, ("getting TGT sync\n")); kerr = ldap_child_get_tgt_sync(main_ctx, ibuf->realm_str, ibuf->princ_str, ibuf->keytab_name, ibuf->lifetime, &ccname, &expire_time); if (kerr != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, ("ldap_child_get_tgt_sync failed.\n")); /* Do not return, must report failure */ } ret = prepare_response(main_ctx, ccname, expire_time, kerr, &resp); if (ret != EOK) { DEBUG(SSSDBG_CRIT_FAILURE, ("prepare_response failed. [%d][%s].\n", ret, strerror(ret))); goto fail; } errno = 0; written = sss_atomic_write_s(STDOUT_FILENO, resp->buf, resp->size); if (written == -1) { ret = errno; DEBUG(SSSDBG_CRIT_FAILURE, ("write failed [%d][%s].\n", ret, strerror(ret))); goto fail; } if (written != resp->size) { DEBUG(SSSDBG_CRIT_FAILURE, ("Expected to write %d bytes, wrote %d\n", resp->size, written)); goto fail; } DEBUG(SSSDBG_TRACE_FUNC, ("ldap_child completed successfully\n")); close(STDOUT_FILENO); talloc_free(main_ctx); _exit(0); fail: DEBUG(SSSDBG_CRIT_FAILURE, ("ldap_child failed!\n")); close(STDOUT_FILENO); talloc_free(main_ctx); _exit(-1); }
void CThreadLoader::Start(CThreadUnit** pTasks, int countTasks) { m_pprimblock=new OSMPBF::PrimitiveBlock; if(m_nCount >0) { assert(false); } boost::timer::nanosecond_type const secunda(1000000000LL); boost::timer::cpu_timer uTimer; //unsigned uTime=0; if(m_nThredNumber==0) { uTimer.start(); } std::vector<char> buffer_blob_header; std::vector<char> buffer(OSMPBF::max_uncompressed_blob_size); std::vector<unsigned char> unpack_buffer(OSMPBF::max_uncompressed_blob_size); const char* pbWork; //for(int l=0;l<20;++l) for(;;) { // storage of size, used multiple times __int32 sz; OSMPBF::BlobHeader blobheader; { //Начало паралельной работы boost::lock_guard<boost::mutex> l(*m_pcsFile); if(feof(m_fp)) break; // read the first 4 bytes of the file, this is the size of the blob-header if(fread(&sz, sizeof(sz), 1, m_fp) != 1) break; // end of file reached // convert the size from network byte-order to host byte-order sz = ntohl(sz); // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE if(sz > OSMPBF::max_blob_header_size) err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size); buffer_blob_header.resize(sz); // read the blob-header from the file if(fread(&buffer_blob_header[0], sz, 1, m_fp) != 1) err("unable to read blob-header from file"); // parse the blob-header from the read-buffer if(!blobheader.ParseFromArray(&buffer_blob_header[0], sz)) err("unable to parse blob header"); // size of the following blob sz = blobheader.m_datasize.m_val; // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > OSMPBF::max_uncompressed_blob_size) err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size); // read the blob from the file if(fread(&buffer[0], sz, 1, m_fp) != 1) err("unable to read blob from file"); //Отсюда можно работать паралельно } // ++m_nCount; continue; // parse the blob from the read-buffer OSMPBF::Blob blob; if(!blob.ParseFromArray(&buffer[0], sz)) err("unable to parse blob"); // set when we find at least one data stream bool found_data = false; // if the blob has uncompressed data if(!blob.m_raw.empty()) { // we have at least one datastream found_data = true; // size of the blob-data sz = blob.m_raw.size(); // check that raw_size is set correctly if(sz != blob.m_raw_size.m_val) warn(" reports wrong raw_size: %u bytes", blob.m_raw_size.m_val); // copy the uncompressed data over to the unpack_buffer //memcpy(&unpack_buffer[0], &buffer[0], sz); pbWork=&buffer[0]; } // if the blob has zlib-compressed data if(!blob.m_zlib_data.empty()) { // issue a warning if there is more than one data steam, a blob may only contain one data stream if(found_data) warn(" contains several data streams"); // we have at least one datastream found_data = true; // the size of the compressesd data sz = blob.m_zlib_data.size(); // zlib information z_stream z; // next byte to decompress z.next_in = (z_const Bytef *) blob.m_zlib_data.m_pBegin; // number of bytes to decompress z.avail_in = sz; // place of next decompressed byte z.next_out = &unpack_buffer[0]; pbWork= (const char*) &unpack_buffer[0]; // space for decompressed data z.avail_out = blob.m_raw_size.m_val; // misc z.zalloc = Z_NULL; z.zfree = Z_NULL; z.opaque = Z_NULL; if(inflateInit(&z) != Z_OK) { err(" failed to init zlib stream"); } if(inflate(&z, Z_FINISH) != Z_STREAM_END) { err(" failed to inflate zlib stream"); } if(inflateEnd(&z) != Z_OK) { err(" failed to deinit zlib stream"); } // unpacked size sz = z.total_out; } // if the blob has lzma-compressed data if(!blob.m_lzma_data.empty()) { // issue a warning if there is more than one data steam, a blob may only contain one data stream if(found_data) warn(" contains several data streams"); // we have at least one datastream found_data = true; // issue a warning, lzma compression is not yet supported err(" lzma-decompression is not supported"); } // check we have at least one data-stream if(!found_data) err(" does not contain any known data stream"); // switch between different blob-types if(blobheader.m_type.compareString("OSMHeader")==0) { OSMPBF::Headerblock headerblock ; // parse the HeaderBlock from the blob if(!headerblock.ParseFromArray(pbWork, sz)) err("unable to parse header block"); } else if(blobheader.m_type.compareString("OSMData")==0) { m_pprimblock->Clear(); // parse the PrimitiveBlock from the blob if(!m_pprimblock->ParseFromArray(pbWork, sz)) err("unable to parse primitive block"); // iterate over all PrimitiveGroups for(int i = 0, l = m_pprimblock->m_primitivegroup.size(); i < l; i++) { // one PrimitiveGroup from the the Block OSMPBF::PrimitiveGroup& pg = m_pprimblock->m_primitivegroup[i]; bool found_items=false; // tell about nodes if(pg.m_nodes.size() > 0) { found_items = true; for(size_t i=0;i<pg.m_nodes.size();++i) AddNode(pg.m_nodes[i]); } // tell about dense nodes if(!pg.m_dense.empty()) { found_items = true; AddDense(pg.m_dense); } // tell about ways if(pg.m_ways.size() > 0) { found_items = true; for(size_t i=0;i<pg.m_ways.size();++i) AddWay(pg.m_ways[i]); } // tell about relations if(pg.m_relations.size() > 0) { found_items = true; for(size_t i=0;i<pg.m_relations.size();++i) AddRelations(pg.m_relations[i]); } if(!found_items) warn(" contains no items"); } } else { // unknown blob type warn(" unknown blob type: %s", blobheader.m_type.toString().c_str()); } ////////////////////////////////////////////////////////////////////////// LONG l =INTERLOCKED_INCREMENT(m_nCount); if(m_nThredNumber==0) { boost::timer::cpu_times const elapsed_times(uTimer.elapsed()); boost::timer::nanosecond_type elapsed(elapsed_times.wall); if(elapsed >= secunda) { info("Bloks=%d",l); uTimer.start(); } } } m_tabNode_Cash.Save(); m_tabNode_NotVisible_Cash.Save(); m_tabNI_Cash.Save(); m_tabNI_NotVisible_Cash.Save(); m_tabWI_Cash.Save(); m_tabWI_NotVisible_Cash.Save(); m_tabRI_Cash.Save(); m_tabRI_NotVisible_Cash.Save(); m_tabkvNode_cash.Save(); m_tabkvWay_cash.Save(); m_tabkvRelation_cash.Save(); delete m_pprimblock; m_pprimblock=NULL; }
int main(int argc, const char *argv[]) { int ret; int kerr; int opt; int debug_fd = -1; poptContext pc; TALLOC_CTX *main_ctx; uint8_t *buf = NULL; ssize_t len = 0; const char *ccname = NULL; time_t expire_time = 0; struct input_buffer *ibuf = NULL; struct response *resp = NULL; size_t written; struct poptOption long_options[] = { POPT_AUTOHELP {"debug-level", 'd', POPT_ARG_INT, &debug_level, 0, _("Debug level"), NULL}, {"debug-timestamps", 0, POPT_ARG_INT, &debug_timestamps, 0, _("Add debug timestamps"), NULL}, {"debug-microseconds", 0, POPT_ARG_INT, &debug_microseconds, 0, _("Show timestamps with microseconds"), NULL}, {"debug-fd", 0, POPT_ARG_INT, &debug_fd, 0, _("An open file descriptor for the debug logs"), NULL}, POPT_TABLEEND }; /* Set debug level to invalid value so we can deside if -d 0 was used. */ debug_level = SSSDBG_INVALID; pc = poptGetContext(argv[0], argc, argv, long_options, 0); while((opt = poptGetNextOpt(pc)) != -1) { switch(opt) { default: fprintf(stderr, "\nInvalid option %s: %s\n\n", poptBadOption(pc, 0), poptStrerror(opt)); poptPrintUsage(pc, stderr, 0); _exit(-1); } } poptFreeContext(pc); CONVERT_AND_SET_DEBUG_LEVEL(debug_level); DEBUG(7, ("ldap_child started.\n")); main_ctx = talloc_new(NULL); if (main_ctx == NULL) { DEBUG(1, ("talloc_new failed.\n")); _exit(-1); } debug_prg_name = talloc_asprintf(main_ctx, "[sssd[ldap_child[%d]]]", getpid()); if (debug_fd != -1) { ret = set_debug_file_from_fd(debug_fd); if (ret != EOK) { DEBUG(1, ("set_debug_file_from_fd failed.\n")); } } buf = talloc_size(main_ctx, sizeof(uint8_t)*IN_BUF_SIZE); if (buf == NULL) { DEBUG(1, ("talloc_size failed.\n")); goto fail; } ibuf = talloc_zero(main_ctx, struct input_buffer); if (ibuf == NULL) { DEBUG(1, ("talloc_size failed.\n")); goto fail; } while ((ret = read(STDIN_FILENO, buf + len, IN_BUF_SIZE - len)) != 0) { if (ret == -1) { if (errno == EINTR || errno == EAGAIN) { continue; } DEBUG(1, ("read failed [%d][%s].\n", errno, strerror(errno))); goto fail; } else if (ret > 0) { len += ret; if (len > IN_BUF_SIZE) { DEBUG(1, ("read too much, this should never happen.\n")); goto fail; } continue; } else { DEBUG(1, ("unexpected return code of read [%d].\n", ret)); goto fail; } } close(STDIN_FILENO); ret = unpack_buffer(buf, len, ibuf); if (ret != EOK) { DEBUG(1, ("unpack_buffer failed.[%d][%s].\n", ret, strerror(ret))); goto fail; } kerr = ldap_child_get_tgt_sync(main_ctx, ibuf->realm_str, ibuf->princ_str, ibuf->keytab_name, ibuf->lifetime, &ccname, &expire_time); if (kerr != EOK) { DEBUG(1, ("ldap_child_get_tgt_sync failed.\n")); /* Do not return, must report failure */ } ret = prepare_response(main_ctx, ccname, expire_time, kerr, &resp); if (ret != EOK) { DEBUG(1, ("prepare_response failed. [%d][%s].\n", ret, strerror(ret))); return ENOMEM; } written = 0; while (written < resp->size) { ret = write(STDOUT_FILENO, resp->buf + written, resp->size - written); if (ret == -1) { if (errno == EAGAIN || errno == EINTR) { continue; } ret = errno; DEBUG(1, ("write failed [%d][%s].\n", ret, strerror(ret))); return ret; } written += ret; } close(STDOUT_FILENO); talloc_free(main_ctx); _exit(0); fail: close(STDOUT_FILENO); talloc_free(main_ctx); _exit(-1); }