void merge_iindex(char **p) {
while (*p != NULL && *(p+1) != NULL && *(p+2) != NULL && *(p+3) != NULL) {
p = merge_files(p);
if (p == NULL) {
printf("Merge Failed!\n");
return;
}
}
char filename[256] = {'\0'};
bzero(filename,256);
if (*(p+1) != NULL) {
snprintf(filename, 256, "%s%s%s%s", get_basedir(), "output/", "final", ".git");
rename(*(p+1), filename);
}
bzero(filename,256);
if (*(p+2) != NULL) {
snprintf(filename, 256, "%s%s%s%s", get_basedir(), "output/", "final", ".mit");
rename(*(p+2), filename);
}
compress_docid();
return;
}
int main(int argc, char *argv[]) {
int myrank;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
printf("%2d - started\n", myrank);
if (myrank == 0) { //Master, i.e. distributor of tasks
MPI_Comm_size(MPI_COMM_WORLD, &ITEMS_TO_BE_SENT);
int number_of_processes = ITEMS_TO_BE_SENT;
ITEMS_TO_BE_SENT--; //number of processes that are supposed to receive a task
if(argc == 1)
ftw(".", parse, 1);
else
ftw(argv[1], parse, 1);
kill_slaves();
sleep(1); //just to make sure that files are closed properly...
merge_files(number_of_processes);
fprintf(stderr, "Finished merging\n");
} else {
run_slave(myrank);
}
MPI_Finalize();
return 0;
}
static file_merger_error_t merge_view_files(const char *source_files[],
unsigned num_source_files,
const char *dest_path,
view_file_merge_ctx_t *ctx)
{
return merge_files(source_files, num_source_files, dest_path,
read_view_record, write_view_record, compare_view_records,
free_view_record, ctx);
}
// Merge 2 PBM files and create an output file
void merge(FILE *in1, FILE *in2, char* fname) {
int width;
int height;
char *temp_buf = (char*)malloc(10);
char magic_num[3];
// Make sure the magic number (P4) of both PBM files is correct
fgets(magic_num, 4, in1);
if(strcmp(magic_num, "P4\n") != 0) {
// Incorrect magic number
fprintf(stderr, "ERROR: invalid PBM file1 for input.\n");
return;
}
fgets(magic_num, 4, in2);
if(strcmp(magic_num, "P4\n") != 0) {
// Incorrect magic number
fprintf(stderr, "ERROR: invalid PBM file2 for input.\n");
return;
}
// Read height/width values from PBM file
fgets(temp_buf, 10, in1);
width = (int)strtol(temp_buf, &temp_buf, 10);
height = (int)strtol(temp_buf, NULL, 10);
fgets(temp_buf, 10, in2);
if(width != (int)strtol(temp_buf, &temp_buf, 10) || height != (int)strtol(temp_buf, NULL, 10)) {
// Height/width don't match
fprintf(stderr, "ERROR: dimensions in file1 and file2 do not match.\n");
return;
}
// Print Headers
printf("P4\n");
printf("%d %d\n", width, height);
// Merge all the bits together and write to fout
merge_files(in1, in2, width, height);
// PSEUDO-CODE
// - Read in 2 files and verify their headers
// - Read each value in, byte by byte
// - Bitwise OR each value and print it to the screen
// * TO TEST THIS, DIRECT OUTPUT TO FILE
// e.g. ./hw2 merge money.1.pbm money.2.pbm > money-merged.pbm
}
VOID merge_rsc(VOID)
{
BYTE sv_rfile[80], sv_dfile[80];
strcpy(&sv_rfile[0], rcs_rfile);
strcpy(&sv_dfile[0], rcs_dfile);
mouse_form(HGLASS);
if (open_files(FALSE))
if (merge_files())
redo_trees();
if (rcs_state == FILE_STATE)
{
strcpy(rcs_rfile, &sv_rfile[0]);
strcpy(rcs_dfile, &sv_dfile[0]);
}
mouse_form(ARROW);
}
int main(int c, const char** v) {
if (c <= 2) {
fprintf(stderr, "merge_files file1 fil2 file3 ...\n");
exit(1);
}
for (int i = 1; i < c; ++i) {
if (!system::file_exists(v[i])) {
fprintf(stderr, "file %s does not exists!\n", v[i]);
exit(1);
}
input_index_file *iif = new input_index_file(v[i]);
files_to_merge.push_back(iif);
term_count += iif->get_term_count();
collection_size += iif->get_collection_size();
}
std::string merged_file_name = get_merged_file_name(c, v);
merge_files(merged_file_name, output_index_file::default_block_size, collection_size);
return 0;
}
void operator()(List const &filenames, std::string const &dest)
{
std::copy(filenames.begin(), filenames.end(), std::back_inserter(files));
std::copy(filenames.begin(), filenames.end(), std::back_inserter(delete_files));
outfile.open(dest.c_str(), std::ios_base::out | std::ios_base::binary);
while (files.size() > 0)
{
open_files();
merge_files(dest);
assert(file_lines.size() == 0);
// subsequent merges need to merge with outfile from previous
// iteration. to do this, rename the output file to a temporary
// filename and add it to the list of files to merge, then re-open
// the destination file
if (files.size() > 0)
rename_result_for_iterative_merge(dest);
}
}
int order(char * entidad, int (*fp)(struct record_t*, struct record_t*)){
FILE * block_file;
FILE * tmp_file;
struct record_t *record;
struct block_t * current_block;
struct file_header * file_header;
int res, j = 0;
int current_pos = 0;
int current_pos_freeze = 0;
int registros_leidos = 0, bloques_leidos = 0;
int todo_leido = 1, totales_leidos = 0;
int registro_en_memoria = 1, registro_procesado = 0;
int tmp_files = 1;
char path[30];
struct record_t buffer[TMP_BUFFER];
struct record_t freeze_buffer[TMP_BUFFER];
block_file = open_block_file(entidad, 2);
if(!block_file){
return BAD_NAME_FILE;
}
/* Allocate buffer and record */
current_block = (struct block_t *) malloc(sizeof(struct block_t));
file_header = (struct file_header*) malloc(sizeof(struct file_header));
record = (struct record_t*) malloc(sizeof(struct record_t));
if(!current_block || !file_header || !record)
return ALLOCATE_FAIL;
initialize_block(current_block,1);
read_header(block_file, file_header);
initialize_record(record);
res = read_from_file(block_file, current_block, file_header, 1);
bloques_leidos++;
if(res == RES_OK){
/* Inicializo el buffer con registros */
while(j<TMP_BUFFER && (todo_leido == 1)){
if(registros_leidos < current_block->total_reg){
current_pos += read_block(current_block, record, current_pos);
memcpy(&buffer[j],record,sizeof(struct record_t));
registros_leidos++;
totales_leidos++;
j++;
}else{
if((bloques_leidos < file_header->total_blocks+1) && (file_header->total_records > totales_leidos)){
res = read_from_file(block_file, current_block, file_header, current_block->number+1);
bloques_leidos++;
registros_leidos = 0;
current_pos = 0;
}else{
todo_leido = 0;
}
}
}
/* Creo el primer sort file */
sprintf(path,"tmp/sort%d.txt",tmp_files);
tmp_file = fopen(path, "w");
if(file_header->total_records == totales_leidos)
heapify(buffer,totales_leidos, fp);
else
heapify(buffer,TMP_BUFFER, fp);
while((todo_leido == 1) && tmp_file){
if(registros_leidos < current_block->total_reg){
current_pos += read_block(current_block, record, current_pos);
registros_leidos++;
registro_en_memoria = 0;
registro_procesado = 1;
}else{
if(bloques_leidos < file_header->total_blocks){
res = read_from_file(block_file, current_block, file_header, current_block->number+1);
bloques_leidos++;
current_pos = 0;
registros_leidos = 0;
registro_procesado = 1;
registro_en_memoria = 1;
}else{
todo_leido = 0;
}
}
while((registro_procesado == 1) && (registro_en_memoria == 0)){
if(fp(record,&buffer[0]) < 0){
if(current_pos_freeze<TMP_BUFFER){
memcpy(&freeze_buffer[current_pos_freeze],record,sizeof(struct record_t));
current_pos_freeze++;
registro_procesado = 0;
registro_en_memoria = 1;
}
else {
/* vacio el buffer en el archivo */
print_buffer_to_file(buffer, TMP_BUFFER, tmp_file, fp);
/* cierro archivo y abro el siguiente */
tmp_files++;
fclose(tmp_file);
sprintf(path,"tmp/sort%d.txt",tmp_files);
tmp_file = fopen(path, "w");
/* copio el freeze_buffer al buffer y creo el heap*/
heapify(freeze_buffer,TMP_BUFFER, fp);
memcpy(buffer,freeze_buffer,sizeof(struct record_t) * TMP_BUFFER);
current_pos_freeze = 0;
registro_procesado = 1;
}
}
else{
/* Guardo en el archivo, libero memoria */
print_record_to_file(buffer, tmp_file);
/* Guardo en el buffer el record leido */
memcpy(&buffer[0],&buffer[TMP_BUFFER-1],sizeof(struct record_t));
memcpy(&buffer[TMP_BUFFER-1],record,sizeof(struct record_t));
registro_procesado = 0;
registro_en_memoria = 1;
update(buffer, TMP_BUFFER, fp);
}
}
}
/* Guardo en el archivo*/
if((file_header->total_records == totales_leidos) && (current_block->number == 1))
print_buffer_to_file(buffer, totales_leidos, tmp_file, fp);
else
print_buffer_to_file(buffer, TMP_BUFFER, tmp_file, fp);
fclose(tmp_file);
/* Guardo lo que tenia en el freeze_buffer en otro archivo */
if(current_pos_freeze > 1){
tmp_files++;
sprintf(path,"tmp/sort%d.txt",tmp_files);
tmp_file = fopen(path, "w");
print_buffer_to_file(freeze_buffer, current_pos_freeze, tmp_file, fp);
fclose(tmp_file);
}
}
free(record);
free(current_block);
free(file_header);
fclose(block_file);
/* Merge de los archivos generados */
res = merge_files(tmp_files, fp);
printf("\n");
return res;
}
/**
* Create a new ndnr instance
* @param progname - name of program binary, used for locating helpers
* @param logger - logger function
* @param loggerdata - data to pass to logger function
*/
PUBLIC struct ndnr_handle *
r_init_create(const char *progname, ndnr_logger logger, void *loggerdata)
{
char *sockname = NULL;
const char *portstr = NULL;
const char *listen_on = NULL;
const char *d = NULL;
struct ndnr_handle *h = NULL;
struct hashtb_param param = {0};
struct ndn_charbuf *config = NULL;
int res;
h = calloc(1, sizeof(*h));
if (h == NULL)
return(h);
h->notify_after = 0; //NDNR_MAX_ACCESSION;
h->logger = logger;
h->loggerdata = loggerdata;
h->logpid = (int)getpid();
h->progname = progname;
h->debug = -1;
config = r_init_read_config(h);
if (config == NULL)
goto Bail;
r_init_parse_config(h, config, 0); /* silent pass to pick up NDNR_DEBUG */
h->debug = 1; /* so that we see any complaints */
h->debug = r_init_debug_getenv(h, "NDNR_DEBUG");
res = r_init_parse_config(h, config, 1);
if (res < 0) {
h->running = -1;
goto Bail;
}
r_init_parse_config(h, config, 2);
sockname = r_net_get_local_sockname();
h->skiplinks = ndn_indexbuf_create();
h->face_limit = 10; /* soft limit */
h->fdholder_by_fd = calloc(h->face_limit, sizeof(h->fdholder_by_fd[0]));
param.finalize_data = h;
param.finalize = &r_fwd_finalize_nameprefix;
h->nameprefix_tab = hashtb_create(sizeof(struct nameprefix_entry), ¶m);
param.finalize = 0; // PRUNED &r_fwd_finalize_propagating;
h->propagating_tab = hashtb_create(sizeof(struct propagating_entry), ¶m);
param.finalize = &r_proto_finalize_enum_state;
h->enum_state_tab = hashtb_create(sizeof(struct enum_state), ¶m);
h->min_stale = ~0;
h->max_stale = 0;
h->unsol = ndn_indexbuf_create();
h->ticktock.descr[0] = 'C';
h->ticktock.micros_per_base = 1000000;
h->ticktock.gettime = &r_util_gettime;
h->ticktock.data = h;
h->sched = ndn_schedule_create(h, &h->ticktock);
h->starttime = h->sec;
h->starttime_usec = h->usec;
h->oldformatcontentgrumble = 1;
h->oldformatinterestgrumble = 1;
h->cob_limit = 4201;
h->start_write_scope_limit = r_init_confval(h, "NDNR_START_WRITE_SCOPE_LIMIT", 0, 3, 3);
h->debug = 1; /* so that we see any complaints */
h->debug = r_init_debug_getenv(h, "NDNR_DEBUG");
h->syncdebug = r_init_debug_getenv(h, "NDNS_DEBUG");
portstr = getenv("NDNR_STATUS_PORT");
if (portstr == NULL || portstr[0] == 0 || strlen(portstr) > 10)
portstr = "";
h->portstr = portstr;
ndnr_msg(h, "NDNR_DEBUG=%d NDNR_DIRECTORY=%s NDNR_STATUS_PORT=%s", h->debug, h->directory, h->portstr);
listen_on = getenv("NDNR_LISTEN_ON");
if (listen_on != NULL && listen_on[0] != 0)
ndnr_msg(h, "NDNR_LISTEN_ON=%s", listen_on);
if (ndnr_init_repo_keystore(h, NULL) < 0) {
h->running = -1;
goto Bail;
}
r_util_reseed(h);
r_store_init(h);
if (h->running == -1) goto Bail;
while (h->active_in_fd >= 0) {
r_dispatch_process_input(h, h->active_in_fd);
r_store_trim(h, h->cob_limit);
ndn_schedule_run(h->sched);
}
ndnr_msg(h, "Repository file is indexed");
if (h->face0 == NULL) {
struct fdholder *fdholder;
fdholder = calloc(1, sizeof(*fdholder));
if (dup2(open("/dev/null", O_RDONLY), 0) == -1)
ndnr_msg(h, "stdin: %s", strerror(errno));
fdholder->filedesc = 0;
fdholder->flags = (NDNR_FACE_GG | NDNR_FACE_NORECV);
r_io_enroll_face(h, fdholder);
}
ndnr_direct_client_start(h);
d = getenv("NDNR_SKIP_VERIFY");
#if (NDN_API_VERSION >= 4004)
if (d != NULL && strcmp(d, "1") == 0) {
ndnr_msg(h, "NDNR_SKIP_VERIFY=%s", d);
ndn_defer_verification(h->direct_client, 1);
}
#endif
if (ndn_connect(h->direct_client, NULL) != -1) {
int af = 0;
int bufsize;
int flags;
int fd;
struct fdholder *fdholder;
fd = ndn_get_connection_fd(h->direct_client);
// Play a dirty trick here - if this wins, we can fix it right in the c lib later on...
af = try_tcp_instead(fd);
flags = NDNR_FACE_NDND;
if (af == AF_INET)
flags |= NDNR_FACE_INET;
else if (af == AF_INET6)
flags |= NDNR_FACE_INET6;
else
flags |= NDNR_FACE_LOCAL;
fdholder = r_io_record_fd(h, fd, "NDND", 5, flags);
if (fdholder == NULL) abort();
ndnr_uri_listen(h, h->direct_client, "ndn:/%C1.M.S.localhost/%C1.M.SRV/repository",
&ndnr_answer_req, OP_SERVICE);
ndnr_uri_listen(h, h->direct_client, "ndn:/%C1.M.S.neighborhood/%C1.M.SRV/repository",
&ndnr_answer_req, OP_SERVICE);
bufsize = r_init_confval(h, "NDNR_MIN_SEND_BUFSIZE", 1, 2097152, 16384);
establish_min_send_bufsize(h, fd, bufsize);
}
else
ndn_disconnect(h->direct_client); // Apparently ndn_connect error case needs work.
if (1 == r_init_confval(h, "NDNS_ENABLE", 0, 1, 1)) {
h->sync_plumbing = calloc(1, sizeof(struct sync_plumbing));
h->sync_plumbing->ndn = h->direct_client;
h->sync_plumbing->sched = h->sched;
h->sync_plumbing->client_methods = &sync_client_methods;
h->sync_plumbing->client_data = h;
h->sync_base = SyncNewBaseForActions(h->sync_plumbing);
}
if (-1 == load_policy(h))
goto Bail;
r_net_listen_on(h, listen_on);
ndnr_internal_client_start(h);
r_proto_init(h);
r_proto_activate_policy(h, h->parsed_policy);
if (merge_files(h) == -1)
r_init_fail(h, __LINE__, "Unable to merge additional repository data files.", errno);
if (h->running == -1) goto Bail;
if (h->sync_plumbing) {
// Start sync running
// returns < 0 if a failure occurred
// returns 0 if the name updates should fully restart
// returns > 0 if the name updates should restart at last fence
res = h->sync_plumbing->sync_methods->sync_start(h->sync_plumbing, NULL);
if (res < 0) {
r_init_fail(h, __LINE__, "starting sync", res);
abort();
}
else if (res > 0) {
// XXX: need to work out details of starting from last fence.
// By examination of code, SyncActions won't take this path
}
}
Bail:
if (sockname)
free(sockname);
sockname = NULL;
ndn_charbuf_destroy(&config);
if (h->running == -1)
r_init_destroy(&h);
return(h);
}
/** @brief opens all the files necessary, sets a default
* delimiter if none was specified, and calls the
* merge_files() function.
*
* @param args contains the parsed cmd-line options & arguments.
* @param argc number of cmd-line arguments.
* @param argv list of cmd-line arguments
* @param optind index of the first non-option cmd-line argument.
*
* @return exit status for main() to return.
*/
int deltaforce(struct cmdargs *args, int argc, char *argv[], int optind) {
char default_delimiter[] = { 0xfe, 0x00 };
FILE *left, *right, *out; /* the two inputs & the output file ptrs */
dbfr_t *left_reader, *right_reader;
int fd_tmp, retval; /* file descriptor and return value */
int i;
if (argc - optind != 2) {
fprintf(stderr,
"%s: missing file arguments. see %s -h for usage information.\n",
argv[0], argv[0]);
return EXIT_HELP;
}
if (str_eq(argv[optind], argv[optind + 1])) {
fprintf(stderr, "%s: 2 input files cannot be the same\n", argv[0]);
return EXIT_HELP;
}
if (str_eq(argv[optind], "-")) {
left = stdin;
} else {
if ((fd_tmp = open64(argv[optind], O_RDONLY)) < 0) {
perror(argv[optind]);
return EXIT_FILE_ERR;
}
if ((left = fdopen(fd_tmp, "r")) == NULL) {
perror(argv[optind]);
return EXIT_FILE_ERR;
}
}
left_reader = dbfr_init(left);
if (str_eq(argv[optind + 1], "-")) {
right = stdin;
} else {
if ((fd_tmp = open64(argv[optind + 1], O_RDONLY)) < 0) {
perror(argv[optind + 1]);
return EXIT_FILE_ERR;
}
if ((right = fdopen(fd_tmp, "r")) == NULL) {
perror(argv[optind + 1]);
return EXIT_FILE_ERR;
}
}
right_reader = dbfr_init(right);
if (!args->outfile) {
out = stdout;
} else {
if ((fd_tmp = open64(args->outfile, O_WRONLY | O_CREAT | O_TRUNC)) < 0) {
perror(args->outfile);
return EXIT_FILE_ERR;
}
fchmod(fd_tmp, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
if ((out = fdopen(fd_tmp, "w")) == NULL) {
perror(args->outfile);
return EXIT_FILE_ERR;
}
}
if (!args->delim) {
args->delim = getenv("DELIMITER");
if (!args->delim)
args->delim = default_delimiter;
}
expand_chars(args->delim);
delim = args->delim;
if (args->keys) {
nkeys = expand_nums(args->keys, &keyfields, &keyfields_sz);
} else if (args->key_labels) {
nkeys = expand_label_list(args->key_labels, left_reader->next_line, delim,
&keyfields, &keyfields_sz);
} else {
keyfields = xmalloc(sizeof(int));
keyfields[0] = 1;
keyfields_sz = sizeof(int);
nkeys = 1;
}
if (nkeys < 1) {
fprintf(stderr, "%s: bad key specified: \"%s\"\n",
getenv("_"), args->keys ? args->keys : args->key_labels);
return EXIT_HELP;
}
for (i = 0; i < nkeys; i++)
keyfields[i]--;
/* set locale with values from the environment so strcoll()
will work correctly. */
setlocale(LC_ALL, "");
setlocale(LC_COLLATE, "");
retval = merge_files(left_reader, right_reader, out, args);
fclose(left);
fclose(right);
fclose(out);
return retval;
}
int main(void)
{
const char *source_files[N_FILES] = {
"merger_sorted_file_1.tmp",
"merger_sorted_file_2.tmp",
"merger_sorted_file_3.tmp",
"merger_sorted_file_4.tmp"
};
const char *dest_file = "merged_file.tmp";
const int batches[N_FILES][MAX_RECORDS_PER_FILE] = {
{3, 5, 6, 14, 18, 19, 29, 30, 35, 38, 44, 45, 46, 51, 54, 57, 62, 65,
75, 76, 81, 83, 91, 92, 95, 104, 105, 107},
{1, 2, 4, 9, 17, 23, 25, 32, 33, 37, 41, 49, 55, 58, 61, 68, 70, 71, 72,
77, 80, 87, 89, 94, 98, 100, 111, 112, 113, 114, 115, 116, 117, 119},
{10, 12, 15, 20, 21, 22, 27, 34, 36, 39, 42, 47, 52, 53, 56, 63, 64, 74,
78, 79, 86, 88, 93, 99, 103, 106, 108, 109, 121, 122, 123},
{7, 8, 11, 13, 16, 24, 26, 28, 31, 40, 43, 48, 50, 59, 60, 66, 69, 73,
82, 84, 85, 90, 96, 97, 101, 102, 110, 118, 120}
};
unsigned i, j;
unsigned num_records = 0;
file_merger_error_t ret;
fprintf(stderr, "\nRunning file merger tests...\n");
for (i = 0; i < N_FILES; ++i) {
FILE *f;
remove(source_files[i]);
f = fopen(source_files[i], "ab");
cb_assert(f != NULL);
for (j = 0; j < MAX_RECORDS_PER_FILE; ++j) {
if (batches[i][j] == 0) {
break;
}
if (j > 0) {
cb_assert(batches[i][j] > batches[i][j - 1]);
}
cb_assert(fwrite(&batches[i][j], sizeof(batches[i][j]), 1, f) == 1);
num_records += 1;
}
fclose(f);
}
remove(dest_file);
ret = merge_files(source_files, N_FILES,
dest_file,
read_record, write_record, NULL, compare_records,
NULL, free_record, 0, NULL);
cb_assert(ret == FILE_MERGER_SUCCESS);
cb_assert(check_file_sorted(dest_file) == num_records);
for (i = 0; i < N_FILES; ++i) {
remove(source_files[i]);
}
remove(dest_file);
fprintf(stderr, "Running file merger tests passed\n\n");
return 0;
}
static void *dispatch_part_download(d_task_t *d_task, file_info_t *file,
int per_part_len, int last_part_len, int parts)
{
int i;
int offset = 0;
// allocate on stack
task_desc descs[MAX_PART_NUMBER];
part_info_t parts_info[MAX_PART_NUMBER];
char tmp_files_name[MAX_PART_NUMBER][PATH_MAX];
int part_lens[MAX_PART_NUMBER];
int part_downloaded_len = 0;
struct stat sb;
int ret;
char sql_buf[256];
for (i = 0; i < parts; i++)
{
descs[i].arg = &parts_info[i];
descs[i].fire_task_over = NULL;
parts_info[i].beg_pos = offset;
if (i == parts - 1 && last_part_len > 0)
offset += last_part_len;
else
offset += per_part_len;
parts_info[i].end_pos = offset - 1;
parts_info[i].d_task = d_task;
parts_info[i].file = file;
parts_info[i].id = i;
part_lens[i] = parts_info[i].end_pos - parts_info[i].beg_pos + 1;
snprintf(tmp_files_name[i], PATH_MAX, d_task->tmp_file_name_fmt, i);
ret = stat(tmp_files_name[i], &sb);
if (ret != 0 && errno != ENOENT)
{
perror("stat file size failed:");
return;
}
if (ret == 0)
{
parts_info[i].beg_pos += sb.st_size;
d_task->len_downloaded += sb.st_size;
}
if (parts_info[i].end_pos - parts_info[i].beg_pos + 1 > 0)
easy_thread_pool_add_task(d_task->dm->tp, download_part_entry, &descs[i]);
}
pthread_mutex_lock(d_task->part_mutex);
while (d_task->parts_exit < parts)
pthread_cond_wait(d_task->part_cond, d_task->part_mutex);
pthread_mutex_unlock(d_task->part_mutex);
if (d_task->len_downloaded < file->length)
{
for (i = 0; i < parts; i++)
{
if (!parts_info[i].finished)
d_task->request_part_file(&parts_info[i]);
}
}
if (d_task->len_downloaded < file->length)
fprintf(stderr, "download failed\n");
else // save downloaded file
{
char file_full_path[PATH_MAX];
char *p;
snprintf(file_full_path, PATH_MAX, "%s/%s", d_task->file_saved_path, file->filename);
if(merge_files(file_full_path, file->length, tmp_files_name, part_lens, parts) < 0)
{
fprintf(stderr, "merge files failed\n");
unlink(file_full_path);
}
if(p = strrchr(tmp_files_name[0], '/'))
{
*++p = 0;
rmdir(tmp_files_name[0]);
}
// delete file record from db
snprintf(sql_buf, sizeof(sql_buf), SQL_DEL_FILE, file->filename, d_task->file_saved_path);
db_execute(d_task->dm->db_key, sql_buf, NULL);
}
}
int main (int argc, char **argv) {
char *filename,
*usersize;
int ret;
unsigned long int size;
size_t len;
if (argc < 3) {
displayHelpOptions(argv[0]);
exit (1);
}
if (!strcmp(argv[1], "-s")) {
if (argc != 4) {
displayHelpOptions(argv[0]);
exit (1);
}
if (argv[3][0] == '-') {
printf("\n\tTransEase 1.0\n\n");
printf("\t\tSplit size cannot be a negative number!\n\n");
exit (1);
}
len = strlen(argv[3]);
usersize = (char *)malloc((len + 1) * sizeof(char));
strncpy(usersize, argv[3], len);
if ((argv[3][len - 1] == 'M') || (argv[3][len - 1] == 'm')) {
usersize[len - 1] = '\0';
size = strtoul(usersize, NULL, 10);
size *= 1024 * 1024;
} else if ((argv[3][len - 1] == 'K') || (argv[3][len - 1] == 'k')) {
usersize[len - 1] = '\0';
size = strtoul(usersize, NULL, 10);
size *= 1024;
} else if ((argv[3][len - 1] == 'B' || argv[3][len - 1] == 'b')) {
usersize[len - 1] = '\0';
size = strtoul(usersize, NULL, 10);
}else if (isdigit(argv[3][len - 1])) {
size = strtoul(usersize, NULL, 10);
} else {
printf("\n\tTransEase 1.0\n\n");
printf("\t\tPlease specify the size correctly!\n\n");
displayHelpOptions(argv[0]);
exit (1);
}
if (size == 0) {
printf("\n\tTransEase 1.0\n\n");
printf("\t\tInvalid split size (0) \n\n");
exit (1);
}
if (size > MAX_SPLITSIZE) { /*MAX_SPLITSIZE is defined in configuration.h */
printf("\n\tTransEase 1.0\n\n");
printf("\t\tSplit size can be %lu bytes at maximum!\n\n", (uoff_t)MAX_SPLITSIZE);
exit (1);
}
filename = strdup(argv[2]);
ret = split_file(filename, size);
exit (ret);
} else if (!strcmp(argv[1], "-j")) {
if (argc != 3) {
displayHelpOptions(argv[0]);
exit (1);
}
filename = strdup(argv[2]);
ret = merge_files(filename);
exit (ret);
} else{
printf("\n\tTransEase 1.0\n\n");
printf("\t\tWhatch Wanna Do ?\n");
displayHelpOptions(argv[0]);
exit (1);
}
exit (0);
}
// thread entrypoint for code to handle a single proxy connection
void* proxy_thread(void* args) {
// cast parameters back to native data type, copy the params and free up mem
thread_params_t* params = (thread_params_t*) args;
int socket_id = params->socket_id;
struct cl_engine* cl_engine = params->cl_engine;
free(params);
args = NULL;
params = NULL;
printf("connection accepted on socket %d.\n", socket_id);
// detach the thread so the parent doesn't have to wait on it
pthread_detach(pthread_self());
// this sample code simply reads out what was received on the socket.
// when you start the proxy and run firefox, you'll see the HTTP request
// printed on screen.
//
// note that because the proxy is multithreaded, you may see an interleaving
// of printf() outputs from different threads.
char buf[BUF_SIZE];
int bytes_received;
while ((bytes_received = recv(socket_id, buf, sizeof(buf)-1, 0)) > 0) {
buf[bytes_received] = 0;
printf("--- RECEVIED FROM CLIENT: --- \n%s\n", buf);
// ------------------------------------ //
// --- PARSE FILE NAME FROM REQUEST --- //
// ------------------------------------ //
char* hostname;
char get_request[5] = "GET";
char post_request[5] = "POST";
char host_position[5] = "Host:";
char* tokens;
char* rest = buf;
// Ensure is GET or POST
if (strncmp(buf, get_request, 3) != 0 && strncmp(buf, post_request, 4) != 0) {
printf("ERROR: Neither a GET nor POST request");
break;
}
char buf_copy[BUF_SIZE];
memcpy(buf_copy, buf, sizeof(buf));
// If encrypted, tokens will hold the full url
tokens = strtok_r(buf, " ", &rest);
tokens = strtok_r(NULL, " ", &rest);
if (strncmp(tokens, "http", 4) == 0) {
tokens = strtok_r(tokens, "//", &rest); //do we need to add this to non encrypted too?
tokens = strtok_r(NULL, "/", &rest);
hostname = tokens;
} else {
printf("ERROR: GET/POST request in nonstandard format");
break;
}
// If not encrypted, get hostname
if (hostname == NULL) {
do {
if (strncmp(tokens, host_position, 5) == 0) {
hostname = strtok_r(NULL, " ", &rest);
printf("%s\n", hostname);
break;
}
tokens = strtok_r(NULL, " ", &rest);
} while (tokens != NULL);
}
// ------------------------------ //
// --- CONNECT TO HOST SOCKET --- //
// ------------------------------ //
struct hostent* he;
struct sockaddr_in server;
/* resolve hostname */
if ((he = gethostbyname((const char*)hostname) ) == NULL ) {
printf("hostname null"); /* error */
}
/* copy the network address to sockaddr_in structure */
memcpy(&server.sin_addr, he->h_addr_list[0], he->h_length);
server.sin_family = AF_INET;
server.sin_port = htons(80);
int ws = socket(AF_INET, SOCK_STREAM, 0); //ws = web socket
if (ws == -1) {
printf("could not create web socket. reason: %s\n", strerror(errno));
break;
}
//connect to web socket
if ( connect(ws, (struct sockaddr *)&server, sizeof(server) ) < 0) {
printf("could not connect to host socket. reason: %s\n", strerror(errno));
break;
}
printf("successfully connected to host on socket %d\n", ws);
// ---------------------------- //
// --- SEND REQUEST TO HOST --- //
// ---------------------------- //
int bytes_sent = 0;
if ((bytes_sent = send(ws, buf_copy, strlen(buf_copy), 0)) < 0) {
printf("could not send to web socket. reason: %s\n", strerror(errno));
break;
}
// -------------------------------------------- //
// --- SET UP FILE SYSTEM FOR HOST RESPONSE --- //
// -------------------------------------------- //
char file_name[] = "temp_XXXXXX";
FILE* web_data;
mkdtemp(file_name);
if (file_name == NULL){
printf("error creating temp directory");
}
//because C hates string concatenation
char* raw_filename = malloc(strlen(file_name)+6);
strcpy(raw_filename, file_name);
strcat(raw_filename, "/data");
char* unzip_filename = malloc(strlen(file_name)+12);
strcpy(unzip_filename, file_name);
strcat(unzip_filename, "/unzip_data");
char* zip_filename = malloc(strlen(file_name)+10);
strcpy(zip_filename, file_name);
strcat(zip_filename, "/zip_data");
web_data = fopen(raw_filename, "wb+");
fclose(web_data);
web_data = fopen(unzip_filename, "wb+");
fclose(web_data);
web_data = fopen(zip_filename, "wb+");
fclose(web_data);
// ------------------------------------------ //
// ------- RECEIVE RESPONSE FROM HOST ------- //
// --- HANDLES CHUNKED AND G-ZIPPED FILES --- //
// ------------------------------------------ //
char* line;
int chunk_size = 0;
int chunk_bytes_rec = 0;
int chunked = 0;
int decomp = 0;
long length = 0;
char* con_len;
//reads through header if chunked, otherwise reads everything
while ((line = getLine(ws)) != NULL) {
web_data = fopen(raw_filename, "ab");
fwrite(line, sizeof(char), strlen(line), web_data);
fclose(web_data);
printf("%s", line);
if (strcasecmp(line, "transfer-encoding: chunked\r\n") == 0) {
chunked = 1;
}
if (strcasecmp(line, "transfer-encoding: chunked, gzip\r\n") == 0){
chunked = 1;
decomp = 1;
}
if (strcasecmp(line, "transfer-encoding: gzip, chunked\r\n") == 0){
chunked = 1;
decomp = 1;
}
if (strncasecmp(line, "Content-Encoding: gzip", 22) == 0) {
decomp = 1;
}
if (strncasecmp(line, "Content-Length:", 15) == 0) {
char* tokens;
tokens = strtok_r(line, ": ", &con_len);
tokens = strtok_r(NULL, ": ", &con_len);
length = strtol(tokens, &con_len, 10);
}
if (!strcmp(line, "\r\n")) {
break;
}
free(line);
}
// PROCESS CHUNKED DATA
while (chunked == 1) { //set chunked from header parsing
line = getLine(ws); //line with chunk size, don't want to write that to file
if (line == NULL) {
break;
}
chunk_size = strtol(line, NULL, 16);
if (chunk_size > 0) {
char* chunk_buf = malloc(chunk_size);
bzero(chunk_buf, chunk_size);
if ((chunk_bytes_rec = recv(ws, chunk_buf, chunk_size, 0)) > 0) {
if (decomp == 1){
web_data = fopen(zip_filename, "ab");
}
else {
web_data = fopen(raw_filename, "ab");
}
fwrite(chunk_buf, sizeof(char), chunk_bytes_rec, web_data);
fclose(web_data);
}
}
else if (chunk_size == 0) { //last chunk in msg
while ((line = getLine(ws)) != NULL) { //while lines left in msg
if (decomp == 1){
web_data = fopen(zip_filename, "ab");
}
else {
web_data = fopen(raw_filename, "ab");
}
fwrite(line, sizeof(char), strlen(line), web_data);
fclose(web_data);
}
}
else {
printf("error in chunking\n");
}
}
//PROCESS NON CHUNKED DATA
if (chunked == 0){
int size = (int) length; //replace iwth content length
char* chunk_buf = malloc(size+1);
bzero(chunk_buf, size);
int msg_bytes_rec = 0;
while (msg_bytes_rec < size) {
if ((chunk_bytes_rec = recv(ws, chunk_buf, size, 0)) > 0) {
if (decomp == 1){
web_data = fopen(zip_filename, "ab");
}
else {
web_data = fopen(raw_filename, "ab");
}
fwrite(chunk_buf, sizeof(char), chunk_bytes_rec, web_data);
fclose(web_data);
}
msg_bytes_rec += chunk_bytes_rec;
}
}
// ------------------------------- //
// --- DECOMPRESS IF NECESSARY --- //
// ------------------------------- //
if (decomp == 1) {
decompress(zip_filename, unzip_filename);
}
// ----------------- //
// --- SCAN FILE --- //
// ----------------- //
int clean = 0;
if (decomp == 1) {
clean = scan(unzip_filename, cl_engine);
}
else {
clean = scan(raw_filename, cl_engine);
}
// ---------------------- //
// --- SEND TO CLIENT --- //
// -----------------------//
if (clean == 0) {
if (decomp == 1) {
merge_files(raw_filename, zip_filename);
}
web_data = fopen(raw_filename, "rb+");
}
else {
web_data = fopen("malware_detected.html", "r");
}
int fd = fileno(web_data);
struct stat sb;
fstat(fd, &sb);
off_t offset = 0;
bytes_sent = sendfile(socket_id, fd, &offset, (int)sb.st_size);
fclose(web_data);
// ---------------------------- //
// --- CLEAN ALL THE THINGS --- //
// ---------------------------- //
remove((const char*)raw_filename);
remove((const char*)unzip_filename);
free(raw_filename);
free(unzip_filename);
rmdir(file_name);
close(ws);
close(socket_id);
}
// if control got here, the socket was closed or had an error. either way,
// execution cannot continue so close the socket and terminate the thread.
close(socket_id);
printf("connection closed on socket %d\n", socket_id);
return NULL;
}
