/**
* Process the file that is not already in our local cache
* @param main_struct : main structure of the program
* @param meta is the meta data of the file to be processed (it does
* not contain any hashs at that point).
*/
static void process_small_file_not_in_cache(main_struct_t *main_struct, meta_data_t *meta)
{
GFile *a_file = NULL;
gchar *answer = NULL;
gint success = 0; /** success returns a CURL Error status such as CURLE_OK for instance */
a_clock_t *mesure_time = NULL;
if (main_struct != NULL && main_struct->opt != NULL && meta != NULL)
{
print_debug(_("Processing small file: %s\n"), meta->name);
if (meta->file_type == G_FILE_TYPE_REGULAR)
{
mesure_time = new_clock_t();
/* Calculates hashs and takes care of data */
a_file = g_file_new_for_path(meta->name);
meta->hash_data_list = calculate_hash_data_list_for_file(a_file, meta->blocksize);
a_file = free_object(a_file);
end_clock(mesure_time, "calculate_hash_data_list");
}
mesure_time = new_clock_t();
answer = send_meta_data_to_server(main_struct, meta, FALSE);
end_clock(mesure_time, "send_meta_data_to_server");
mesure_time = new_clock_t();
if (meta->size < meta->blocksize)
{
/* Only one block to send (size is less than blocksize's value) */
meta->hash_data_list = send_data_to_server(main_struct, meta->hash_data_list, answer);
}
else
{
/* A least 2 blocks to send */
meta->hash_data_list = send_all_data_to_server(main_struct, meta->hash_data_list, answer);
}
end_clock(mesure_time, "send_(all)_data_to_server");
free_variable(answer); /* Not used by now */
if (success == CURLE_OK)
{
/* Everything has been transmitted so we can save meta data into the local db cache */
/* This is usefull for file carving to avoid sending too much things to the server */
mesure_time = new_clock_t();
db_save_meta_data(main_struct->database, meta, TRUE);
end_clock(mesure_time, "db_save_meta_data");
}
}
}
END_TEST
START_TEST(test_sset_xor)
{
long long int i, n = 10000;
struct item *t, *k, *l;
k = malloc(sizeof(struct item) * n);
l = malloc(sizeof(struct item) * n);
for (t = k, i = 0; i<n; i+=2, t++) {
t->key = i;
t->value = i;
Sset_add(V, (void *)t);
}
for (t = l, i = 1; i<=n; i+=2, t++) {
t->key = i;
t->value = i;
Sset_add(W, (void *)t);
}
start_clock();
Sset_T p = Sset_xor(V, W);
end_clock("Sset_xor: ");
fail_unless(Sset_len(p) == n, "Sset_xor failed");
Sset_free(&p);
free(k);
free(l);
}
END_TEST
START_TEST(test_sset_del)
{
int i, n = 10000;
struct item p, *t, *k;
t = k = malloc(sizeof(struct item) * n);
for (i = 0; i<n; i++, t++) {
t->key = i;
t->value = rand();
Sset_add(V, (void *)t);
}
fail_unless(Sset_len(V) == n);
memset(&p, 0, sizeof(struct item));
start_clock();
for (i = 0; i<n; i++) {
p.key = i;
Sset_del(V, (void *)&p);
}
end_clock("Sset_del: ");
fail_unless(Sset_len(V) == 0, "sset still holds: [%d]:", Sset_len(V));
free(k);
}
/**
* This function gets meta data and data from a file and sends them
* to the server in order to save the file located in the directory
* 'directory' and represented by 'fileinfo' variable.
* @param main_struct : main structure of the program
* @param directory is the directory we are iterating over
* @param fileinfo is a glib structure that contains all meta data and
* more for a file.
* @note This function is not threadable as is. One may have problems
* when writing to the database for instance.
*/
void save_one_file(main_struct_t *main_struct, file_event_t *file_event)
{
meta_data_t *meta = NULL;
a_clock_t *my_clock = NULL;
gchar *message = NULL;
gchar *another_dir = NULL;
filter_file_t *filter = NULL;
if (main_struct != NULL && file_event != NULL)
{
my_clock = new_clock_t();
/* Get data and meta_data for a file. */
filter = new_filter_t(main_struct->database, main_struct->regex_exclude_list, FALSE);
meta = get_meta_data_from_fileinfo(file_event, filter, main_struct->opt);
/* We want to save all files that are not excluded ie filter->excluded not TRUE */
if (meta != NULL && filter != NULL && filter->excluded == FALSE)
{
if (meta->in_cache == FALSE)
{
/* File is not in cache thus unknown thus we need to save it */
if (meta->size < CLIENT_SMALL_FILE_SIZE)
{
process_small_file_not_in_cache(main_struct, meta);
}
else
{
process_big_file_not_in_cache(main_struct, meta);
}
}
if (meta->file_type == G_FILE_TYPE_DIRECTORY)
{
/* This is a recursive call */
another_dir = g_strdup(meta->name);
g_async_queue_push(main_struct->dir_queue, another_dir);
}
message = g_strdup_printf(_("processing file %s"), meta->name);
free_meta_data_t(meta, FALSE);
free_filter_file_t(filter);
}
else if (meta != NULL && filter != NULL && filter->excluded == TRUE)
{
message = g_strdup_printf(_("processing excluded file %s"), meta->name);
free_meta_data_t(meta, FALSE);
free_filter_file_t(filter);
}
else
{
message = g_strdup_printf(_("Error with meta (%p) or filter (%p) structures\n"), meta, filter);
print_error(__FILE__, __LINE__, message);
}
end_clock(my_clock, message);
free_variable(message);
}
}
//****************************************************************************//
int main(int argc, char *argv[]) {
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &numtasks);
numworkers = numtasks - 1;
MPI_Comm_rank(MPI_COMM_WORLD, &taskid);
allocate_rows();
allocate_memory();
if(taskid == MASTER) { // This code fragment runs in the master porcesses
start_clock();
printf("This is the Master.\n");
//***********main solver loop**********************************************//
for (t = 1; t <= tsteps; t++) {
//**********File output routine********************************************//
if(t%savet == 0) {
//receive from worker and print
}
printf("iteration no. %d \n",t);
}
end_clock();
} else { // This code fragment runs in the worker processes
printf("This is worker no. :%d with rows:%d\n", taskid, rows);
//************main solver loop*********************************************//
for (t = 1; t <= tsteps; t++) {
// phi_solver();
//**********File output routine**************************************************//
if(t%savet == 0) {
//send to master
}
}
}
free_mem();
MPI_Finalize();
return(0);
}
void benchmark_key_calc_2gb (const int kht_depth,
const int branching_factor)
{
int i;
int fd;
char test_block_data[HORUS_BLOCK_SIZE];
size_t test_file_size = 2*1024*1024*1024L;
int *kht_branching_factor;
// generate branching factor arrays on-the-fly from branching_factor
kht_branching_factor = malloc (sizeof (int) * (kht_depth-1));
if (kht_branching_factor == NULL)
{
perror (__func__);
abort ();
}
for (i = 0; i < kht_depth-1; ++i)
{
kht_branching_factor[i] = branching_factor;
}
// init, generate some data for test_block_data
for (i = 0; i < HORUS_BLOCK_SIZE; ++i)
test_block_data[i] = (char)('0' + i % 10);
// open the file, set a master key
unlink (test_file);
fd = open (test_file, O_CREAT | O_RDWR, 0644);
if (fd < 0)
abort ();
horus_set_kht (fd, kht_depth, HORUS_BLOCK_SIZE, kht_branching_factor);
horus_add_key (fd, master_key, strnlen (master_key, HORUS_KEY_LEN), 0, 0);
start_clock ();
// write 2GB data
for (i = 0; i < test_file_size / HORUS_BLOCK_SIZE; ++i)
{
if (unlikely (HORUS_BLOCK_SIZE !=
horus_write (fd, (void*)test_block_data, HORUS_BLOCK_SIZE)))
{
perror (__func__);
abort ();
}
}
end_clock ();
printf ("write_2gb,%d,%d,", kht_depth, branching_factor);
print_last_clock_diff ();
printf ("\n");
close (fd);
// Now read it back
fd = open (test_file, O_CREAT | O_RDWR, 0644);
if (fd < 0)
abort ();
horus_set_kht (fd, kht_depth, HORUS_BLOCK_SIZE, kht_branching_factor);
horus_add_key (fd, master_key, strnlen (master_key, HORUS_KEY_LEN), 0, 0);
start_clock ();
// run it again
for (i = 0; i < test_file_size / HORUS_BLOCK_SIZE; ++i)
{
if (unlikely (HORUS_BLOCK_SIZE !=
horus_read (fd, (void*)test_block_data, HORUS_BLOCK_SIZE)))
{
perror (__func__);
abort ();
}
}
end_clock ();
printf ("read_2gb,%d,%d,", kht_depth, branching_factor);
print_last_clock_diff ();
printf ("\n");
}
/**
* Process the file that is not already in our local cache
* @param main_struct : main structure of the program
* @param meta is the meta data of the file to be processed (it does
* not contain any hashs at that point).
*/
static void process_big_file_not_in_cache(main_struct_t *main_struct, meta_data_t *meta)
{
GFile *a_file = NULL;
gchar *answer = NULL;
GFileInputStream *stream = NULL;
GError *error = NULL;
GList *hash_data_list = NULL;
GList *hdl_copy = NULL;
GList *saved_list = NULL;
hash_data_t *hash_data = NULL;
gssize read = 0;
guchar *buffer = NULL;
GChecksum *checksum = NULL;
guint8 *a_hash = NULL;
gsize digest_len = HASH_LEN;
gsize read_bytes = 0;
a_clock_t *elapsed = NULL;
if (main_struct != NULL && main_struct->opt != NULL && meta != NULL)
{
a_file = g_file_new_for_path(meta->name);
print_debug(_("Processing file: %s\n"), meta->name);
if (a_file != NULL)
{
stream = g_file_read(a_file, NULL, &error);
if (stream != NULL && error == NULL)
{
checksum = g_checksum_new(G_CHECKSUM_SHA256);
buffer = (guchar *) g_malloc(meta->blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error);
read_bytes = read_bytes + read;
while (read != 0 && error == NULL)
{
g_checksum_update(checksum, buffer, read);
g_checksum_get_digest(checksum, a_hash, &digest_len);
/* Need to save 'data', 'read' and digest hash in an hash_data_t structure */
hash_data = new_hash_data_t(buffer, read, a_hash);
hash_data_list = g_list_prepend(hash_data_list, hash_data);
g_checksum_reset(checksum);
digest_len = HASH_LEN;
if (read_bytes >= main_struct->opt->buffersize)
{
elapsed = new_clock_t();
print_debug(_("Sending data: %d bytes\n"), read_bytes);
/* 0. Save the list in order to keep hashs for meta-data */
hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL);
saved_list = g_list_concat(hdl_copy, saved_list);
/* 1. Send an array of hashs to Hash_Array.json server url */
answer = send_hash_array_to_server(main_struct->comm, hash_data_list);
/* 2. Keep only hashs that are needed (answer from the server) */
hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer);
/* 3. free memory of this list if any is left */
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
read_bytes = 0;
end_clock(elapsed, "process_big_file_not_in_cache");
}
buffer = (guchar *) g_malloc(meta->blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error);
read_bytes = read_bytes + read;
}
if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Error while reading file: %s\n"), error->message);
error = free_error(error);
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
}
else
{
if (read_bytes > 0)
{
elapsed = new_clock_t();
print_debug(_("Sending data: %d bytes\n"), read_bytes);
/* 0. Save the list in order to keep hashs for meta-data */
hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL);
saved_list = g_list_concat(hdl_copy, saved_list);
/* 1. Send an array of hashs to Hash_Array.json server url */
answer = send_hash_array_to_server(main_struct->comm, hash_data_list);
/* 2. Keep only hashs that are needed (answer from the server) */
hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer);
/* 3. free memory of this list if any is left */
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
read_bytes = 0;
end_clock(elapsed, "process_big_file_not_in_cache");
}
/* get the list in correct order (because we prepended the hashs to get speed when inserting hashs in the list) */
saved_list = g_list_reverse(saved_list);
}
free_variable(buffer);
free_variable(a_hash);
g_checksum_free(checksum);
g_input_stream_close((GInputStream *) stream, NULL, NULL);
free_object(stream);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to open file for reading: %s\n"), error->message);
error = free_error(error);
}
meta->hash_data_list = saved_list;
answer = send_meta_data_to_server(main_struct, meta, TRUE);
if (answer != NULL)
{ /** @todo may be we should check that answer is something that tells that everything went Ok. */
/* Everything has been transmitted so we can save meta data into the local db cache */
/* This is usefull for file carving to avoid sending too much things to the server */
elapsed = new_clock_t();
db_save_meta_data(main_struct->database, meta, TRUE);
end_clock(elapsed, "db_save_meta_data");
}
a_file = free_object(a_file);
}
}
}
/**
* Sends data as requested by the server 'cdpfglserver' in a buffered way.
* @param main_struct : main structure of the program.
* @param hash_data_list : list of hash_data_t * pointers containing
* all the data to be saved.
* @param answer is the request sent back by server when we had send
* meta data.
* @note using directly main_struct->comm->buffer -> not threadable as is.
*/
static GList *send_all_data_to_server(main_struct_t *main_struct, GList *hash_data_list, gchar *answer)
{
json_t *root = NULL;
json_t *array = NULL;
GList *hash_list = NULL; /** hash_list is local to this function and contains the needed hashs as answer by server */
GList *head = NULL;
GList *iter = NULL;
hash_data_t *found = NULL;
hash_data_t *hash_data = NULL;
gint bytes = 0;
json_t *to_insert = NULL;
gint64 limit = 0;
a_clock_t *elapsed = NULL;
if (answer != NULL && hash_data_list != NULL && main_struct != NULL && main_struct->opt != NULL)
{
root = load_json(answer);
limit = main_struct->opt->buffersize;
if (root != NULL)
{
/* This hash_list is the needed hashs from server */
hash_list = extract_glist_from_array(root, "hash_list", TRUE);
json_decref(root);
array = json_array();
head = hash_list;
while (hash_list != NULL)
{
hash_data = hash_list->data;
/* hash_data_list contains all hashs and their associated data for the file
* being processed */
iter = find_hash_in_list(hash_data_list, hash_data->hash);
found = iter->data;
to_insert = convert_hash_data_t_to_json(found);
json_array_append_new(array, to_insert);
bytes = bytes + found->read;
hash_data_list = g_list_remove_link(hash_data_list, iter);
/* iter is now a single element list and we can delete
* data in this element and then remove this single element list
*/
g_list_free_full(iter, free_hdt_struct);
if (bytes >= limit)
{
/* when we've got opt->buffersize bytes of data send them ! */
elapsed = new_clock_t();
insert_array_in_root_and_send(main_struct, array);
array = json_array();
bytes = 0;
end_clock(elapsed, "insert_array_in_root_and_send");
}
hash_list = g_list_next(hash_list);
}
if (bytes > 0)
{
/* Send the rest of the data (less than opt->buffersize bytes) */
elapsed = new_clock_t();
insert_array_in_root_and_send(main_struct, array);
end_clock(elapsed, "insert_array_in_root_and_send");
}
else
{
json_decref(array);
}
if (head != NULL)
{
g_list_free_full(head, free_hdt_struct);
}
}
else
{
print_error(__FILE__, __LINE__, _("Error while loading JSON answer from server\n"));
}
}
return hash_data_list;
}
/**
* Function that process the received data from the POST command and
* answers to the client.
* Here we may do something with this data (we may want to store it
* somewhere).
*
* @param server_struct is the main structure for the server.
* @param connection is the connection in MHD
* @param url is the requested url
* @param received_data is a gchar * string to the data that was received
* by the POST request.
*/
static int process_received_data(server_struct_t *server_struct, struct MHD_Connection *connection, const char *url, gchar *received_data)
{
gchar *answer = NULL; /** gchar *answer : Do not free answer variable as MHD will do it for us ! */
int success = MHD_NO;
gboolean debug = FALSE;
hash_data_t *hash_data = NULL;
json_t *root = NULL;
GList *hash_data_list = NULL;
GList *head = NULL;
a_clock_t *elapsed = NULL;
if (g_strcmp0(url, "/Meta.json") == 0 && received_data != NULL)
{
success = answer_meta_json_post_request(server_struct, connection, received_data);
}
else if (g_strcmp0(url, "/Hash_Array.json") == 0 && received_data != NULL)
{
success = answer_hash_array_post_request(server_struct, connection, received_data);
/* Here we will try to answer which hashs are needed and then
* send thoses hashs back in the answer
*/
}
else if (g_strcmp0(url, "/Data.json") == 0 && received_data != NULL)
{
hash_data = convert_string_to_hash_data(received_data);
if (get_debug_mode() == TRUE)
{
print_received_data_for_hash(hash_data->hash, hash_data->read);
}
/**
* Sending received_data into the queue in order to be treated by
* the corresponding thread. hash_data is freed by data_thread
* and should not be used after this "call" here.
*/
g_async_queue_push(server_struct->data_queue, hash_data);
/**
* creating an answer for the client to say that everything went Ok!
*/
answer = g_strdup_printf(_("Ok!"));
success = create_MHD_response(connection, answer);
}
else if (g_strcmp0(url, "/Data_Array.json") == 0 && received_data != NULL)
{
/* print_debug("/Data_Array.json: %s\n", received_data); */
elapsed = new_clock_t();
root = load_json(received_data);
end_clock(elapsed, "load_json");
hash_data_list = extract_glist_from_array(root, "data_array", FALSE);
head = hash_data_list;
json_decref(root);
debug = get_debug_mode();
while (hash_data_list != NULL)
{
hash_data = hash_data_list->data;
if (debug == TRUE)
{
/* Only for debbugging ! */
print_received_data_for_hash(hash_data->hash, hash_data->read);
}
/** Sending hash_data into the queue. */
g_async_queue_push(server_struct->data_queue, hash_data);
hash_data_list = g_list_next(hash_data_list);
}
g_list_free(head);
/**
* creating an answer for the client to say that everything went Ok!
*/
answer = g_strdup_printf(_("Ok!"));
success = create_MHD_response(connection, answer);
}
else
{
/* The url is unknown to the server and we can not process the request ! */
print_error(__FILE__, __LINE__, "Error: invalid url: %s\n", url);
answer = g_strdup_printf(_("Error: invalid url!\n"));
success = create_MHD_response(connection, answer);
}
return success;
}
