/** * Process the file that is not already in our local cache * @param main_struct : main structure of the program * @param meta is the meta data of the file to be processed (it does * not contain any hashs at that point). */ static void process_small_file_not_in_cache(main_struct_t *main_struct, meta_data_t *meta) { GFile *a_file = NULL; gchar *answer = NULL; gint success = 0; /** success returns a CURL Error status such as CURLE_OK for instance */ a_clock_t *mesure_time = NULL; if (main_struct != NULL && main_struct->opt != NULL && meta != NULL) { print_debug(_("Processing small file: %s\n"), meta->name); if (meta->file_type == G_FILE_TYPE_REGULAR) { mesure_time = new_clock_t(); /* Calculates hashs and takes care of data */ a_file = g_file_new_for_path(meta->name); meta->hash_data_list = calculate_hash_data_list_for_file(a_file, meta->blocksize); a_file = free_object(a_file); end_clock(mesure_time, "calculate_hash_data_list"); } mesure_time = new_clock_t(); answer = send_meta_data_to_server(main_struct, meta, FALSE); end_clock(mesure_time, "send_meta_data_to_server"); mesure_time = new_clock_t(); if (meta->size < meta->blocksize) { /* Only one block to send (size is less than blocksize's value) */ meta->hash_data_list = send_data_to_server(main_struct, meta->hash_data_list, answer); } else { /* A least 2 blocks to send */ meta->hash_data_list = send_all_data_to_server(main_struct, meta->hash_data_list, answer); } end_clock(mesure_time, "send_(all)_data_to_server"); free_variable(answer); /* Not used by now */ if (success == CURLE_OK) { /* Everything has been transmitted so we can save meta data into the local db cache */ /* This is usefull for file carving to avoid sending too much things to the server */ mesure_time = new_clock_t(); db_save_meta_data(main_struct->database, meta, TRUE); end_clock(mesure_time, "db_save_meta_data"); } } }
END_TEST START_TEST(test_sset_xor) { long long int i, n = 10000; struct item *t, *k, *l; k = malloc(sizeof(struct item) * n); l = malloc(sizeof(struct item) * n); for (t = k, i = 0; i<n; i+=2, t++) { t->key = i; t->value = i; Sset_add(V, (void *)t); } for (t = l, i = 1; i<=n; i+=2, t++) { t->key = i; t->value = i; Sset_add(W, (void *)t); } start_clock(); Sset_T p = Sset_xor(V, W); end_clock("Sset_xor: "); fail_unless(Sset_len(p) == n, "Sset_xor failed"); Sset_free(&p); free(k); free(l); }
END_TEST START_TEST(test_sset_del) { int i, n = 10000; struct item p, *t, *k; t = k = malloc(sizeof(struct item) * n); for (i = 0; i<n; i++, t++) { t->key = i; t->value = rand(); Sset_add(V, (void *)t); } fail_unless(Sset_len(V) == n); memset(&p, 0, sizeof(struct item)); start_clock(); for (i = 0; i<n; i++) { p.key = i; Sset_del(V, (void *)&p); } end_clock("Sset_del: "); fail_unless(Sset_len(V) == 0, "sset still holds: [%d]:", Sset_len(V)); free(k); }
/** * This function gets meta data and data from a file and sends them * to the server in order to save the file located in the directory * 'directory' and represented by 'fileinfo' variable. * @param main_struct : main structure of the program * @param directory is the directory we are iterating over * @param fileinfo is a glib structure that contains all meta data and * more for a file. * @note This function is not threadable as is. One may have problems * when writing to the database for instance. */ void save_one_file(main_struct_t *main_struct, file_event_t *file_event) { meta_data_t *meta = NULL; a_clock_t *my_clock = NULL; gchar *message = NULL; gchar *another_dir = NULL; filter_file_t *filter = NULL; if (main_struct != NULL && file_event != NULL) { my_clock = new_clock_t(); /* Get data and meta_data for a file. */ filter = new_filter_t(main_struct->database, main_struct->regex_exclude_list, FALSE); meta = get_meta_data_from_fileinfo(file_event, filter, main_struct->opt); /* We want to save all files that are not excluded ie filter->excluded not TRUE */ if (meta != NULL && filter != NULL && filter->excluded == FALSE) { if (meta->in_cache == FALSE) { /* File is not in cache thus unknown thus we need to save it */ if (meta->size < CLIENT_SMALL_FILE_SIZE) { process_small_file_not_in_cache(main_struct, meta); } else { process_big_file_not_in_cache(main_struct, meta); } } if (meta->file_type == G_FILE_TYPE_DIRECTORY) { /* This is a recursive call */ another_dir = g_strdup(meta->name); g_async_queue_push(main_struct->dir_queue, another_dir); } message = g_strdup_printf(_("processing file %s"), meta->name); free_meta_data_t(meta, FALSE); free_filter_file_t(filter); } else if (meta != NULL && filter != NULL && filter->excluded == TRUE) { message = g_strdup_printf(_("processing excluded file %s"), meta->name); free_meta_data_t(meta, FALSE); free_filter_file_t(filter); } else { message = g_strdup_printf(_("Error with meta (%p) or filter (%p) structures\n"), meta, filter); print_error(__FILE__, __LINE__, message); } end_clock(my_clock, message); free_variable(message); } }
//****************************************************************************// int main(int argc, char *argv[]) { MPI_Init(&argc,&argv); MPI_Comm_size(MPI_COMM_WORLD, &numtasks); numworkers = numtasks - 1; MPI_Comm_rank(MPI_COMM_WORLD, &taskid); allocate_rows(); allocate_memory(); if(taskid == MASTER) { // This code fragment runs in the master porcesses start_clock(); printf("This is the Master.\n"); //***********main solver loop**********************************************// for (t = 1; t <= tsteps; t++) { //**********File output routine********************************************// if(t%savet == 0) { //receive from worker and print } printf("iteration no. %d \n",t); } end_clock(); } else { // This code fragment runs in the worker processes printf("This is worker no. :%d with rows:%d\n", taskid, rows); //************main solver loop*********************************************// for (t = 1; t <= tsteps; t++) { // phi_solver(); //**********File output routine**************************************************// if(t%savet == 0) { //send to master } } } free_mem(); MPI_Finalize(); return(0); }
void benchmark_key_calc_2gb (const int kht_depth, const int branching_factor) { int i; int fd; char test_block_data[HORUS_BLOCK_SIZE]; size_t test_file_size = 2*1024*1024*1024L; int *kht_branching_factor; // generate branching factor arrays on-the-fly from branching_factor kht_branching_factor = malloc (sizeof (int) * (kht_depth-1)); if (kht_branching_factor == NULL) { perror (__func__); abort (); } for (i = 0; i < kht_depth-1; ++i) { kht_branching_factor[i] = branching_factor; } // init, generate some data for test_block_data for (i = 0; i < HORUS_BLOCK_SIZE; ++i) test_block_data[i] = (char)('0' + i % 10); // open the file, set a master key unlink (test_file); fd = open (test_file, O_CREAT | O_RDWR, 0644); if (fd < 0) abort (); horus_set_kht (fd, kht_depth, HORUS_BLOCK_SIZE, kht_branching_factor); horus_add_key (fd, master_key, strnlen (master_key, HORUS_KEY_LEN), 0, 0); start_clock (); // write 2GB data for (i = 0; i < test_file_size / HORUS_BLOCK_SIZE; ++i) { if (unlikely (HORUS_BLOCK_SIZE != horus_write (fd, (void*)test_block_data, HORUS_BLOCK_SIZE))) { perror (__func__); abort (); } } end_clock (); printf ("write_2gb,%d,%d,", kht_depth, branching_factor); print_last_clock_diff (); printf ("\n"); close (fd); // Now read it back fd = open (test_file, O_CREAT | O_RDWR, 0644); if (fd < 0) abort (); horus_set_kht (fd, kht_depth, HORUS_BLOCK_SIZE, kht_branching_factor); horus_add_key (fd, master_key, strnlen (master_key, HORUS_KEY_LEN), 0, 0); start_clock (); // run it again for (i = 0; i < test_file_size / HORUS_BLOCK_SIZE; ++i) { if (unlikely (HORUS_BLOCK_SIZE != horus_read (fd, (void*)test_block_data, HORUS_BLOCK_SIZE))) { perror (__func__); abort (); } } end_clock (); printf ("read_2gb,%d,%d,", kht_depth, branching_factor); print_last_clock_diff (); printf ("\n"); }
/** * Process the file that is not already in our local cache * @param main_struct : main structure of the program * @param meta is the meta data of the file to be processed (it does * not contain any hashs at that point). */ static void process_big_file_not_in_cache(main_struct_t *main_struct, meta_data_t *meta) { GFile *a_file = NULL; gchar *answer = NULL; GFileInputStream *stream = NULL; GError *error = NULL; GList *hash_data_list = NULL; GList *hdl_copy = NULL; GList *saved_list = NULL; hash_data_t *hash_data = NULL; gssize read = 0; guchar *buffer = NULL; GChecksum *checksum = NULL; guint8 *a_hash = NULL; gsize digest_len = HASH_LEN; gsize read_bytes = 0; a_clock_t *elapsed = NULL; if (main_struct != NULL && main_struct->opt != NULL && meta != NULL) { a_file = g_file_new_for_path(meta->name); print_debug(_("Processing file: %s\n"), meta->name); if (a_file != NULL) { stream = g_file_read(a_file, NULL, &error); if (stream != NULL && error == NULL) { checksum = g_checksum_new(G_CHECKSUM_SHA256); buffer = (guchar *) g_malloc(meta->blocksize); a_hash = (guint8 *) g_malloc(digest_len); read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error); read_bytes = read_bytes + read; while (read != 0 && error == NULL) { g_checksum_update(checksum, buffer, read); g_checksum_get_digest(checksum, a_hash, &digest_len); /* Need to save 'data', 'read' and digest hash in an hash_data_t structure */ hash_data = new_hash_data_t(buffer, read, a_hash); hash_data_list = g_list_prepend(hash_data_list, hash_data); g_checksum_reset(checksum); digest_len = HASH_LEN; if (read_bytes >= main_struct->opt->buffersize) { elapsed = new_clock_t(); print_debug(_("Sending data: %d bytes\n"), read_bytes); /* 0. Save the list in order to keep hashs for meta-data */ hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL); saved_list = g_list_concat(hdl_copy, saved_list); /* 1. Send an array of hashs to Hash_Array.json server url */ answer = send_hash_array_to_server(main_struct->comm, hash_data_list); /* 2. Keep only hashs that are needed (answer from the server) */ hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer); /* 3. free memory of this list if any is left */ g_list_free_full(hash_data_list, free_hdt_struct); hash_data_list = NULL; read_bytes = 0; end_clock(elapsed, "process_big_file_not_in_cache"); } buffer = (guchar *) g_malloc(meta->blocksize); a_hash = (guint8 *) g_malloc(digest_len); read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error); read_bytes = read_bytes + read; } if (error != NULL) { print_error(__FILE__, __LINE__, _("Error while reading file: %s\n"), error->message); error = free_error(error); g_list_free_full(hash_data_list, free_hdt_struct); hash_data_list = NULL; } else { if (read_bytes > 0) { elapsed = new_clock_t(); print_debug(_("Sending data: %d bytes\n"), read_bytes); /* 0. Save the list in order to keep hashs for meta-data */ hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL); saved_list = g_list_concat(hdl_copy, saved_list); /* 1. Send an array of hashs to Hash_Array.json server url */ answer = send_hash_array_to_server(main_struct->comm, hash_data_list); /* 2. Keep only hashs that are needed (answer from the server) */ hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer); /* 3. free memory of this list if any is left */ g_list_free_full(hash_data_list, free_hdt_struct); hash_data_list = NULL; read_bytes = 0; end_clock(elapsed, "process_big_file_not_in_cache"); } /* get the list in correct order (because we prepended the hashs to get speed when inserting hashs in the list) */ saved_list = g_list_reverse(saved_list); } free_variable(buffer); free_variable(a_hash); g_checksum_free(checksum); g_input_stream_close((GInputStream *) stream, NULL, NULL); free_object(stream); } else { print_error(__FILE__, __LINE__, _("Unable to open file for reading: %s\n"), error->message); error = free_error(error); } meta->hash_data_list = saved_list; answer = send_meta_data_to_server(main_struct, meta, TRUE); if (answer != NULL) { /** @todo may be we should check that answer is something that tells that everything went Ok. */ /* Everything has been transmitted so we can save meta data into the local db cache */ /* This is usefull for file carving to avoid sending too much things to the server */ elapsed = new_clock_t(); db_save_meta_data(main_struct->database, meta, TRUE); end_clock(elapsed, "db_save_meta_data"); } a_file = free_object(a_file); } } }
/** * Sends data as requested by the server 'cdpfglserver' in a buffered way. * @param main_struct : main structure of the program. * @param hash_data_list : list of hash_data_t * pointers containing * all the data to be saved. * @param answer is the request sent back by server when we had send * meta data. * @note using directly main_struct->comm->buffer -> not threadable as is. */ static GList *send_all_data_to_server(main_struct_t *main_struct, GList *hash_data_list, gchar *answer) { json_t *root = NULL; json_t *array = NULL; GList *hash_list = NULL; /** hash_list is local to this function and contains the needed hashs as answer by server */ GList *head = NULL; GList *iter = NULL; hash_data_t *found = NULL; hash_data_t *hash_data = NULL; gint bytes = 0; json_t *to_insert = NULL; gint64 limit = 0; a_clock_t *elapsed = NULL; if (answer != NULL && hash_data_list != NULL && main_struct != NULL && main_struct->opt != NULL) { root = load_json(answer); limit = main_struct->opt->buffersize; if (root != NULL) { /* This hash_list is the needed hashs from server */ hash_list = extract_glist_from_array(root, "hash_list", TRUE); json_decref(root); array = json_array(); head = hash_list; while (hash_list != NULL) { hash_data = hash_list->data; /* hash_data_list contains all hashs and their associated data for the file * being processed */ iter = find_hash_in_list(hash_data_list, hash_data->hash); found = iter->data; to_insert = convert_hash_data_t_to_json(found); json_array_append_new(array, to_insert); bytes = bytes + found->read; hash_data_list = g_list_remove_link(hash_data_list, iter); /* iter is now a single element list and we can delete * data in this element and then remove this single element list */ g_list_free_full(iter, free_hdt_struct); if (bytes >= limit) { /* when we've got opt->buffersize bytes of data send them ! */ elapsed = new_clock_t(); insert_array_in_root_and_send(main_struct, array); array = json_array(); bytes = 0; end_clock(elapsed, "insert_array_in_root_and_send"); } hash_list = g_list_next(hash_list); } if (bytes > 0) { /* Send the rest of the data (less than opt->buffersize bytes) */ elapsed = new_clock_t(); insert_array_in_root_and_send(main_struct, array); end_clock(elapsed, "insert_array_in_root_and_send"); } else { json_decref(array); } if (head != NULL) { g_list_free_full(head, free_hdt_struct); } } else { print_error(__FILE__, __LINE__, _("Error while loading JSON answer from server\n")); } } return hash_data_list; }
/** * Function that process the received data from the POST command and * answers to the client. * Here we may do something with this data (we may want to store it * somewhere). * * @param server_struct is the main structure for the server. * @param connection is the connection in MHD * @param url is the requested url * @param received_data is a gchar * string to the data that was received * by the POST request. */ static int process_received_data(server_struct_t *server_struct, struct MHD_Connection *connection, const char *url, gchar *received_data) { gchar *answer = NULL; /** gchar *answer : Do not free answer variable as MHD will do it for us ! */ int success = MHD_NO; gboolean debug = FALSE; hash_data_t *hash_data = NULL; json_t *root = NULL; GList *hash_data_list = NULL; GList *head = NULL; a_clock_t *elapsed = NULL; if (g_strcmp0(url, "/Meta.json") == 0 && received_data != NULL) { success = answer_meta_json_post_request(server_struct, connection, received_data); } else if (g_strcmp0(url, "/Hash_Array.json") == 0 && received_data != NULL) { success = answer_hash_array_post_request(server_struct, connection, received_data); /* Here we will try to answer which hashs are needed and then * send thoses hashs back in the answer */ } else if (g_strcmp0(url, "/Data.json") == 0 && received_data != NULL) { hash_data = convert_string_to_hash_data(received_data); if (get_debug_mode() == TRUE) { print_received_data_for_hash(hash_data->hash, hash_data->read); } /** * Sending received_data into the queue in order to be treated by * the corresponding thread. hash_data is freed by data_thread * and should not be used after this "call" here. */ g_async_queue_push(server_struct->data_queue, hash_data); /** * creating an answer for the client to say that everything went Ok! */ answer = g_strdup_printf(_("Ok!")); success = create_MHD_response(connection, answer); } else if (g_strcmp0(url, "/Data_Array.json") == 0 && received_data != NULL) { /* print_debug("/Data_Array.json: %s\n", received_data); */ elapsed = new_clock_t(); root = load_json(received_data); end_clock(elapsed, "load_json"); hash_data_list = extract_glist_from_array(root, "data_array", FALSE); head = hash_data_list; json_decref(root); debug = get_debug_mode(); while (hash_data_list != NULL) { hash_data = hash_data_list->data; if (debug == TRUE) { /* Only for debbugging ! */ print_received_data_for_hash(hash_data->hash, hash_data->read); } /** Sending hash_data into the queue. */ g_async_queue_push(server_struct->data_queue, hash_data); hash_data_list = g_list_next(hash_data_list); } g_list_free(head); /** * creating an answer for the client to say that everything went Ok! */ answer = g_strdup_printf(_("Ok!")); success = create_MHD_response(connection, answer); } else { /* The url is unknown to the server and we can not process the request ! */ print_error(__FILE__, __LINE__, "Error: invalid url: %s\n", url); answer = g_strdup_printf(_("Error: invalid url!\n")); success = create_MHD_response(connection, answer); } return success; }