std::uint32_t split_into_chunks(const std::string& filename, std::uint64_t chunk_size) { std::ifstream input_file(filename, std::ios::binary); uint32_t chunks_count = 0; char* buffer = new char[chunk_size]; while (!input_file.eof()) { input_file.read(buffer, chunk_size); auto read_length = input_file.gcount(); if (read_length > 0) { auto chunk_filename = get_chunk_filename(chunks_count++); std::ofstream chunk_file(chunk_filename, std::ios::binary | std::ios::trunc); chunk_file.write(buffer, read_length); chunk_file.close(); } } delete [] buffer; return chunks_count; }
void chunk_directory(const char *dpath) { DIR *dirp; if ((dirp = opendir (dpath)) == 0) return; struct dirent *de = NULL; while ((de = readdir (dirp))) { if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; char path[PATH_MAX]; sprintf(path, "%s/%s", dpath, de->d_name); struct stat sb; lstat(path, &sb); if (S_ISDIR(sb.st_mode)) chunk_directory(path); else if (S_ISREG(sb.st_mode) && !S_ISLNK(sb.st_mode)){ //fprintf(stderr, "\n\n------------------------------new file starts %s -------------------------------------\n\n",path); chunk_file(path); } } closedir(dirp); }
void ASCHandler::chunking(std::vector<std::string> path) { total_file_num = path.size(); // SysOperation::remove_file("/tmp/asc_handler"); get_seg_size(Chunker::min_chunk_size, Chunker::avg_chunk_size, Chunker::max_chunk_size); Chunker::keep_buf = 1; //classify path_vec.clear(); for(int i = 0; i < path.size(); i++) { std::string m_type = SysOperation::get_type(path[i]); int m_size = SysOperation::get_size(path[i]); if(m_size) { total_origin_size += m_size; max_type_len = Util::max(max_type_len, m_type.size()); origin_size.insert(std::make_pair(m_type, 0)).first->second += m_size; if(path_vec.find(m_type) == path_vec.end()) { std::vector<std::string> vec; path_vec.insert(make_pair(m_type, vec)); seg_exp_size.insert(std::make_pair(m_type, Chunker::avg_chunk_size)); } path_vec[m_type].push_back(path[i]); } } //find exp size int done = 0; for(std::map<std::string, std::vector<std::string> >::iterator it = path_vec.begin(); it != path_vec.end(); ++it) { m_der_rate.insert(make_pair(it->first, 1.0)); bool div_two = false; while((Chunker::avg_chunk_size = seg_exp_size[it->first]) >= 512) { //init std::string path = "/tmp/asc_handler_" + it->first; SysOperation::remove_file(path); type.insert(std::make_pair(it->first, 0)).first->second = 0; //chunk for(int i = 0; i < it->second.size(); i++) { chunk_file(it->second[i]); } //update double der_rate = get_der_rate(it->first); // std::cout << it->first << "\t" << Chunker::avg_chunk_size << "\t" << der_rate << std::endl; if(der_rate > m_der_rate[it->first]) { m_der_rate[it->first] = der_rate; seg_exp_size[it->first] /= 2; div_two = true; } else break; } if(div_two) { seg_exp_size[it->first] *= 2; Chunker::avg_chunk_size = seg_exp_size[it->first]; std::string path = "/tmp/asc_handler_" + it->first; SysOperation::remove_file(path); type[it->first] = 0; //chunk for(int i = 0; i < it->second.size(); i++) { chunk_file(it->second[i]); } } total_seg_num += type[it->first]; done += it->second.size(); // std::cout << done << std::endl; loadbar(done, path.size()); // std::cout << "end" << std::endl; // std::cout << Util::currentTimeinString() << // ", Type: " << it->first << // ", exp size: " << seg_exp_size[it->first] << // ", ASC Der Rate: " << m_der_rate[it->first] << std::endl; } std::cout << std::endl; }
/** * Main routine, start of the program execution. * \param argc the number of arguments * \param argv pointer to the arguments array * \return don't return on sucess, -1 on error * \see main_loop */ int main(int argc, char** argv) { /* configure by default logging to syslog */ int cfg_log_stderr = 0; FILE* cfg_stream; int c,r; char *tmp; int tmp_len; int port; int proto; char *options; int ret; unsigned int seed; int rfd; /*init*/ ret=-1; my_argc=argc; my_argv=argv; /*init pkg mallocs (before parsing cfg or cmd line !)*/ if (init_pkg_mallocs()==-1) goto error; init_route_lists(); /* process command line (get port no, cfg. file path etc) */ opterr=0; options="f:cCm:b:l:n:N:rRvdDETSVhw:t:u:g:P:G:W:"; while((c=getopt(argc,argv,options))!=-1){ switch(c){ case 'f': cfg_file=optarg; break; case 'C': config_check |= 2; case 'c': if (config_check==3) break; config_check |= 1; cfg_log_stderr=1; /* force stderr logging */ break; case 'm': shm_mem_size=strtol(optarg, &tmp, 10) * 1024 * 1024; if (tmp &&(*tmp)){ LM_ERR("bad shmem size number: -m %s\n", optarg); goto error; }; break; case 'b': maxbuffer=strtol(optarg, &tmp, 10); if (tmp &&(*tmp)){ LM_ERR("bad max buffer size number: -b %s\n", optarg); goto error; } break; case 'l': if (parse_phostport(optarg, strlen(optarg), &tmp, &tmp_len, &port, &proto)<0){ LM_ERR("bad -l address specifier: %s\n", optarg); goto error; } tmp[tmp_len]=0; /* null terminate the host */ /* add a new addr. to our address list */ if (add_listen_iface(tmp, port, proto, 0)!=0){ LM_ERR("failed to add new listen address\n"); goto error; } break; case 'n': children_no=strtol(optarg, &tmp, 10); if ((tmp==0) ||(*tmp)){ LM_ERR("bad process number: -n %s\n", optarg); goto error; } break; case 'v': check_via=1; break; case 'r': received_dns|=DO_DNS; break; case 'R': received_dns|=DO_REV_DNS; case 'd': #ifdef CHANGEABLE_DEBUG_LEVEL (*debug)++; #else debug++; #endif break; case 'D': dont_fork=1; break; case 'E': cfg_log_stderr=1; break; case 'T': #ifdef USE_TCP tcp_disable=1; #else LM_WARN("tcp support not compiled in\n"); #endif break; case 'S': #ifdef USE_SCTP sctp_disable=1; #else LM_WARN("sctp support not compiled in\n"); #endif break; case 'N': #ifdef USE_TCP tcp_children_no=strtol(optarg, &tmp, 10); if ((tmp==0) ||(*tmp)){ LM_ERR("bad process number: -N %s\n", optarg); goto error; } #else LM_WARN("tcp support not compiled in\n"); #endif break; case 'W': #ifdef USE_TCP tcp_poll_method=get_poll_type(optarg); if (tcp_poll_method==POLL_NONE){ LM_ERR("bad poll method name: -W %s\ntry " "one of %s.\n", optarg, poll_support); goto error; } #else LM_WARN("tcp support not compiled in\n"); #endif break; case 'V': printf("version: %s\n", version); printf("flags: %s\n", flags ); print_ct_constants(); printf("%s\n",id); printf("%s compiled on %s with %s\n", __FILE__, compiled, COMPILER ); exit(0); break; case 'h': printf("version: %s\n", version); printf("%s",help_msg); exit(0); break; case 'w': working_dir=optarg; break; case 't': chroot_dir=optarg; break; case 'u': user=optarg; break; case 'g': group=optarg; break; case 'P': pid_file=optarg; break; case 'G': pgid_file=optarg; break; case '?': if (isprint(optopt)) LM_ERR("Unknown option `-%c`.\n", optopt); else LM_ERR("Unknown option character `\\x%x`.\n", optopt); goto error; case ':': LM_ERR("Option `-%c` requires an argument.\n", optopt); goto error; default: abort(); } } log_stderr = cfg_log_stderr; /* fill missing arguments with the default values*/ if (cfg_file==0) cfg_file=CFG_FILE; /* load config file or die */ cfg_stream=fopen (cfg_file, "r"); if (cfg_stream==0){ LM_ERR("loading config file(%s): %s\n", cfg_file, strerror(errno)); goto error; } /* seed the prng, try to use /dev/urandom if possible */ /* no debugging information is logged, because the standard log level prior the config file parsing is L_NOTICE */ seed=0; if ((rfd=open("/dev/urandom", O_RDONLY))!=-1){ try_again: if (read(rfd, (void*)&seed, sizeof(seed))==-1){ if (errno==EINTR) goto try_again; /* interrupted by signal */ LM_WARN("could not read from /dev/urandom (%d)\n", errno); } LM_DBG("initialize the pseudo random generator from " "/dev/urandom\n"); LM_DBG("read %u from /dev/urandom\n", seed); close(rfd); }else{ LM_WARN("could not open /dev/urandom (%d)\n", errno); LM_WARN("using a unsafe seed for the pseudo random number generator"); } seed+=getpid()+time(0); LM_DBG("seeding PRNG with %u\n", seed); srand(seed); LM_DBG("test random number %u\n", rand()); /*register builtin modules*/ register_builtin_modules(); #ifdef USE_TLS /* initialize default TLS domains, must be done before reading the config */ if (pre_init_tls()<0){ LM_CRIT("could not pre_init_tls, exiting...\n"); goto error; } #endif /* USE_TLS */ if (preinit_black_lists()!=0) { LM_CRIT("failed to alloc black list's anchor\n"); goto error; } /* parse the config file, prior to this only default values e.g. for debugging settings will be used */ yyin=cfg_stream; if ((yyparse()!=0)||(cfg_errors)){ LM_ERR("bad config file (%d errors)\n", cfg_errors); goto error; } if (config_check>1 && check_rls()!=0) { LM_ERR("bad function call in config file\n"); return ret; } #ifdef EXTRA_DEBUG print_rl(); #endif #ifdef SUPER_DEBUG chunked_file.size=ftell(cfg_stream); chunked_file.data=malloc(chunked_file.size); rewind(cfg_stream); fread(chunked_file.data,1,chunked_file.size,cfg_stream); chunk_file(&chunked_file); #endif /* init the resolver, before fixing the config */ resolv_init(); /* fix parameters */ if (port_no<=0) port_no=SIP_PORT; #ifdef USE_TLS if (tls_port_no<=0) tls_port_no=SIPS_PORT; #endif if (children_no<=0) children_no=CHILD_NO; #ifdef USE_TCP if (!tcp_disable){ if (tcp_children_no<=0) tcp_children_no=children_no; } #endif if (working_dir==0) working_dir="/"; /* get uid/gid */ if (user){ if (user2uid(&uid, &gid, user)<0){ LM_ERR("bad user name/uid number: -u %s\n", user); goto error; } } if (group){ if (group2gid(&gid, group)<0){ LM_ERR("bad group name/gid number: -u %s\n", group); goto error; } } if (fix_all_socket_lists()!=0){ LM_ERR("failed to initialize list addresses\n"); goto error; } /* print all the listen addresses */ printf("Listening on \n"); print_all_socket_lists(); printf("Aliases: \n"); /*print_aliases();*/ print_aliases(); printf("\n"); if (dont_fork){ LM_WARN("no fork mode %s\n", (udp_listen)?( (udp_listen->next)?" and more than one listen address found" "(will use only the first one)":"" ):"and no udp listen address found" ); } if (config_check){ LM_NOTICE("config file ok, exiting...\n"); return 0; } /*init shm mallocs * this must be here * -to allow setting shm mem size from the command line * => if shm_mem should be settable from the cfg file move * everything after * -it must be also before init_timer and init_tcp * -it must be after we know uid (so that in the SYSV sems case, * the sems will have the correct euid) * --andrei */ if (init_shm_mallocs()==-1) goto error; /*init timer, before parsing the cfg!*/ if (init_timer()<0){ LM_CRIT("could not initialize timer, exiting...\n"); goto error; } #ifdef USE_TCP if (!tcp_disable){ /*init tcp*/ if (init_tcp()<0){ LM_CRIT("could not initialize tcp, exiting...\n"); goto error; } } #ifdef USE_TLS if (!tls_disable){ /* init tls*/ if (init_tls()<0){ LM_CRIT("could not initialize tls, exiting...\n"); goto error; } } #endif /* USE_TLS */ #endif /* USE_TCP */ /* init_daemon? */ if (!dont_fork){ if ( daemonize((log_name==0)?argv[0]:log_name) <0 ) goto error; } /* install signal handlers */ if (install_sigs() != 0){ LM_ERR("could not install the signal handlers\n"); goto error; } #ifdef CHANGEABLE_DEBUG_LEVEL #ifdef SHM_MEM debug=shm_malloc(sizeof(int)); if (debug==0) { LM_ERR("ERROR: out of memory\n"); goto error; } *debug = debug_init; #else LM_WARN("no shm mem support compiled -> changeable debug " "level turned off\n"); #endif #endif if (disable_core_dump) set_core_dump(0, 0); else set_core_dump(1, shm_mem_size+PKG_MEM_POOL_SIZE+4*1024*1024); if (open_files_limit>0){ if(increase_open_fds(open_files_limit)<0){ LM_ERR("ERROR: error could not increase file limits\n"); goto error; } } /* print OpenSER version to log for history tracking */ LM_NOTICE("version: %s\n", version); /* print some data about the configuration */ #ifdef SHM_MEM LM_INFO("using %ld Mb shared memory\n", ((shm_mem_size/1024)/1024)); #endif LM_INFO("using %i Mb private memory per process\n", ((PKG_MEM_POOL_SIZE/1024)/1024)); /* init serial forking engine */ if (init_serialization()!=0) { LM_ERR("failed to initialize serialization\n"); goto error; } /* Init statistics */ if (init_stats_collector()<0) { LM_ERR("failed to initialize statistics\n"); goto error; } /* Init MI */ if (init_mi_core()<0) { LM_ERR("failed to initialize MI core\n"); goto error; } /* init black list engine */ if (init_black_lists()!=0) { LM_CRIT("failed to init black lists\n"); goto error; } /* init resolver's blacklist */ if (resolv_blacklist_init()!=0) { LM_CRIT("failed to create DNS blacklist\n"); goto error; } /* init modules */ if (init_modules() != 0) { LM_ERR("error while initializing modules\n"); goto error; } /* init multi processes support */ if (init_multi_proc_support()!=0) { LM_ERR("failed to init multi-proc support\n"); goto error; } /* fix routing lists */ if ( (r=fix_rls())!=0){ LM_ERR("failed to fix configuration with err code %d\n", r); goto error; }; ret=main_loop(); error: /*kill everything*/ kill_all_children(SIGTERM); /*clean-up*/ cleanup(0); return ret; }