/**
* Sets file attributes
* @param file is a GFile pointer and must not be null
* @param meta is the structure that contains all meta data for the
* file that we want to set.
*/
void set_file_attributes(GFile *file, meta_data_t *meta)
{
GError *error = NULL;
GFileInfo *fileinfo = NULL;
if (file != NULL && meta != NULL)
{
fileinfo = g_file_query_info(file, "*", G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, &error);
if (fileinfo == NULL || error != NULL)
{
print_error(__FILE__, __LINE__, _("Error while getting file information: %s\n"), error->message);
error = free_error(error);
}
else
{
set_file_mode_to_gfile(fileinfo, meta);
set_dates_to_gfile(fileinfo, meta);
if (g_file_set_attributes_from_info(file, fileinfo, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, &error) == FALSE && error != NULL)
{
print_error(__FILE__, __LINE__, _("Error or warning for file (%s): %s\n"), meta->name, error->message);
free_error(error);
}
free_object(fileinfo);
}
}
else
{
/* To translators : do not translate this ! */
print_error(__FILE__, __LINE__, "set_file_attribute(file = %p, meta = %p)\n", file, meta);
}
}
char *sm_eval(jaegermonkey_vm *vm, const char *filename, const char *code, int handle_retval) {
char *retval = NULL;
JSScript *script;
jsval result;
begin_request(vm);
script = JS_CompileScript(vm->context,
vm->global,
code, strlen(code),
filename, 1);
jaegermonkey_error *error = (jaegermonkey_error *) JS_GetContextPrivate(vm->context);
if (error == NULL) {
JS_ClearPendingException(vm->context);
JS_ExecuteScript(vm->context, vm->global, script, &result);
vm->invoke_count++;
error = (jaegermonkey_error *) JS_GetContextPrivate(vm->context);
if (error == NULL) {
if (handle_retval) {
if (JSVAL_IS_STRING(result)) {
JSString *str = JS_ValueToString(vm->context, result);
retval = copy_jsstring(str);
}
else if(strcmp(JS_GetStringBytes(JS_ValueToString(vm->context, result)), "undefined") == 0) {
retval = copy_string("{\"error\": \"Expression returned undefined\", \"lineno\": 0, \"source\": \"unknown\"}");
}
else {
retval = copy_string("{\"error\": \"non-JSON return value\", \"lineno\": 0, \"source\": \"unknown\"}");
}
}
JS_DestroyScript(vm->context, script);
}
else {
retval = error_to_json(error);
free_error(error);
JS_SetContextPrivate(vm->context, NULL);
}
}
else {
retval = error_to_json(error);
free_error(error);
JS_SetContextPrivate(vm->context, NULL);
}
if (vm->invoke_count > 200) {
JS_GC(vm->context);
vm->invoke_count = 0;
}
else {
JS_MaybeGC(vm->context);
}
end_request(vm);
return retval;
}
char *sm_eval(spidermonkey_vm *vm, const char *filename, const char *code, int handle_retval) {
char *retval = NULL;
JSObject *script;
jsval result;
if (code == NULL) {
return NULL;
}
begin_request(vm);
script = JS_CompileScript(vm->context,
vm->global,
code, strlen(code),
filename, 1);
spidermonkey_state *state = (spidermonkey_state *) JS_GetContextPrivate(vm->context);
if (state->error == NULL) {
JS_ClearPendingException(vm->context);
JS_ExecuteScript(vm->context, vm->global, script, &result);
state = (spidermonkey_state *) JS_GetContextPrivate(vm->context);
if (state->error == NULL) {
if (handle_retval) {
if (JSVAL_IS_STRING(result)) {
JSString *str = JS_ValueToString(vm->context, result);
retval = copy_jsstring(vm->context, str);
}
else {
char *tmp = JS_EncodeString(vm->context, JS_ValueToString(vm->context, result));
if(strcmp(tmp, "undefined") == 0) {
retval = copy_string("{\"error\": \"Expression returned undefined\", \"lineno\": 0, \"source\": \"unknown\"}");
}
else {
retval = copy_string("{\"error\": \"non-JSON return value\", \"lineno\": 0, \"source\": \"unknown\"}");
}
JS_free(vm->context, tmp);
}
}
}
else {
retval = error_to_json(state->error);
free_error(state);
JS_SetContextPrivate(vm->context, state);
}
}
else {
retval = error_to_json(state->error);
free_error(state);
JS_SetContextPrivate(vm->context, state);
}
end_request(vm);
return retval;
}
void
close_query (query *q)
{
if (q->error) {
free_error(q->error);
}
if (q->c) {
// free the columns
free_value_array(q->c);
}
if (q->where_condition) {
free_condition(q->where_condition);
}
if (q->group_by) {
free_value_array(q->group_by);
}
if (q->from_clause) {
// free the from clause
free_from_clause(q->from_clause);
}
}
/**
* Reads from the configuration file "filename" and fills the options_t *
* opt structure.
* @param[in,out] opt : options_t * structure to store options read from
* the configuration file "filename"
* @param filename : the filename of the configuration file to read from
*/
static void read_from_configuration_file(options_t *opt, gchar *filename)
{
GKeyFile *keyfile = NULL; /** Configuration file parser */
GError *error = NULL; /** Glib error handling */
if (filename != NULL)
{
print_debug(_("Reading configuration from file %s\n"), filename);
keyfile = g_key_file_new();
if (g_key_file_load_from_file(keyfile, filename, G_KEY_FILE_KEEP_COMMENTS, &error))
{
opt->configfile = manage_opt_configfile(opt->configfile, filename);
read_from_group_client(opt, keyfile, filename);
read_debug_mode_from_file(keyfile, filename);
opt->srv_conf = manage_opt_srv_conf(opt->srv_conf, keyfile, filename);
}
else if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Failed to open %s configuration file: %s\n"), filename, error->message);
free_error(error);
}
g_key_file_free(keyfile);
}
}
/**
* Call back for the g_slist_foreach function that carves one directory
* and sub directories in a recursive way.
* @param data is an element of opt->list ie: a gchar * that represents
* a directory name
* @param user_data is the main_struct_t * pointer to the main structure.
*/
static void carve_one_directory(gpointer data, gpointer user_data)
{
gchar *directory = (gchar *) data;
main_struct_t *main_struct = (main_struct_t *) user_data;
GFile *a_dir = NULL;
GFileEnumerator *file_enum = NULL;
GError *error = NULL;
if (directory != NULL && main_struct != NULL)
{
a_dir = g_file_new_for_path(directory);
file_enum = g_file_enumerate_children(a_dir, "*", G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, &error);
if (error == NULL && file_enum != NULL)
{
iterate_over_enum(main_struct, directory, file_enum);
g_file_enumerator_close(file_enum, NULL, NULL);
file_enum = free_object(file_enum);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to enumerate directory %s: %s\n"), directory, error->message);
error = free_error(error);
}
a_dir = free_object(a_dir);
}
}
/**
* Reads from the configuration file "filename" and fills the options_t *
* opt structure.
* @param[in,out] opt : options_t * structure to store options read from
* the configuration file "filename"
* @param filename : the filename of the configuration file to read from
*/
static void read_from_configuration_file(options_t *opt, gchar *filename)
{
GKeyFile *keyfile = NULL; /** Configuration file parser */
GError *error = NULL; /** Glib error handling */
if (filename != NULL)
{
if (opt->configfile != NULL)
{
free_variable(opt->configfile);
}
opt->configfile = g_strdup(filename);
print_debug(_("Reading configuration from file %s\n"), filename);
keyfile = g_key_file_new();
if (g_key_file_load_from_file(keyfile, filename, G_KEY_FILE_KEEP_COMMENTS, &error))
{
read_from_group_all(keyfile, filename);
read_from_group_server(opt, keyfile, filename);
}
else if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Failed to open %s configuration file: %s\n"), filename, error->message);
error = free_error(error);
}
g_key_file_free(keyfile);
}
}
void
close_query (query *q)
{
if (q->error) {
free_error(q->error);
}
if (q->select_query) {
free_select_query(q->select_query);
free(q->select_query);
}
if (q->drop_series_query) {
free_drop_series_query(q->drop_series_query);
free(q->drop_series_query);
}
if (q->drop_query) {
free(q->drop_query);
}
if (q->delete_query) {
free_delete_query(q->delete_query);
free(q->delete_query);
}
}
void sm_stop(spidermonkey_vm *vm) {
begin_request(vm);
spidermonkey_state *state = (spidermonkey_state *) JS_GetContextPrivate(vm->context);
state->terminate = 1;
JS_SetContextPrivate(vm->context, state);
//Wait for any executing function to stop
//before beginning to free up any memory.
while (JS_IsRunning(vm->context)) {
sleep(1);
}
end_request(vm);
//Now we should be free to proceed with
//freeing up memory without worrying about
//crashing the VM.
if (state != NULL) {
if (state->error != NULL) {
free_error(state);
}
driver_free(state);
}
JS_SetContextPrivate(vm->context, NULL);
JS_DestroyContext(vm->context);
JS_DestroyRuntime(vm->runtime);
driver_free(vm);
}
/**
* Prepares everything in order to call save_one_file function that does
* everything to save one file !
* @param main_struct : main structure of the program
* @param path is the entire path and name of the considered file.
*/
static void prepare_before_saving(main_struct_t *main_struct, gchar *path)
{
gchar *directory = NULL;
GFileInfo *fileinfo = NULL;
GFile *file = NULL;
GError *error = NULL;
file_event_t *file_event = NULL;
if (main_struct != NULL && path != NULL)
{
directory = g_path_get_dirname(path);
file = g_file_new_for_path(path);
fileinfo = g_file_query_info(file, "*", G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, &error);
if (error == NULL && fileinfo != NULL)
{
/* file_event is used and freed in the thread
* save_one_file_threaded where the queue save_queue
* is used
*/
file_event = new_file_event_t(directory, fileinfo);
g_async_queue_push(main_struct->save_queue, file_event);
fileinfo = free_object(fileinfo);
file = free_object(file);
free_variable(directory);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to get meta data for file %s: %s\n"), path, error->message);
error = free_error(error);
}
}
}
/**
* Sets the LibVLC error status and message for the current thread.
* Any previous error is overridden.
* @return a nul terminated string (always)
*/
const char *libvlc_vprinterr (const char *fmt, va_list ap)
{
char *msg;
assert (fmt != NULL);
if (vasprintf (&msg, fmt, ap) == -1)
msg = (char *)oom;
free_error ();
vlc_threadvar_set (context, msg);
return msg;
}
void
close_queries (queries *q)
{
if (q->error) {
free_error(q->error);
}
while (q->size > 0) {
query *query = q->qs[--q->size];
close_query(query);
free(query);
}
free(q->qs);
q->qs = NULL;
}
/**
* Process the file that is not already in our local cache
* @param main_struct : main structure of the program
* @param meta is the meta data of the file to be processed (it does
* not contain any hashs at that point).
*/
static void process_big_file_not_in_cache(main_struct_t *main_struct, meta_data_t *meta)
{
GFile *a_file = NULL;
gchar *answer = NULL;
GFileInputStream *stream = NULL;
GError *error = NULL;
GList *hash_data_list = NULL;
GList *hdl_copy = NULL;
GList *saved_list = NULL;
hash_data_t *hash_data = NULL;
gssize read = 0;
guchar *buffer = NULL;
GChecksum *checksum = NULL;
guint8 *a_hash = NULL;
gsize digest_len = HASH_LEN;
gsize read_bytes = 0;
a_clock_t *elapsed = NULL;
if (main_struct != NULL && main_struct->opt != NULL && meta != NULL)
{
a_file = g_file_new_for_path(meta->name);
print_debug(_("Processing file: %s\n"), meta->name);
if (a_file != NULL)
{
stream = g_file_read(a_file, NULL, &error);
if (stream != NULL && error == NULL)
{
checksum = g_checksum_new(G_CHECKSUM_SHA256);
buffer = (guchar *) g_malloc(meta->blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error);
read_bytes = read_bytes + read;
while (read != 0 && error == NULL)
{
g_checksum_update(checksum, buffer, read);
g_checksum_get_digest(checksum, a_hash, &digest_len);
/* Need to save 'data', 'read' and digest hash in an hash_data_t structure */
hash_data = new_hash_data_t(buffer, read, a_hash);
hash_data_list = g_list_prepend(hash_data_list, hash_data);
g_checksum_reset(checksum);
digest_len = HASH_LEN;
if (read_bytes >= main_struct->opt->buffersize)
{
elapsed = new_clock_t();
print_debug(_("Sending data: %d bytes\n"), read_bytes);
/* 0. Save the list in order to keep hashs for meta-data */
hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL);
saved_list = g_list_concat(hdl_copy, saved_list);
/* 1. Send an array of hashs to Hash_Array.json server url */
answer = send_hash_array_to_server(main_struct->comm, hash_data_list);
/* 2. Keep only hashs that are needed (answer from the server) */
hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer);
/* 3. free memory of this list if any is left */
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
read_bytes = 0;
end_clock(elapsed, "process_big_file_not_in_cache");
}
buffer = (guchar *) g_malloc(meta->blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, meta->blocksize, NULL, &error);
read_bytes = read_bytes + read;
}
if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Error while reading file: %s\n"), error->message);
error = free_error(error);
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
}
else
{
if (read_bytes > 0)
{
elapsed = new_clock_t();
print_debug(_("Sending data: %d bytes\n"), read_bytes);
/* 0. Save the list in order to keep hashs for meta-data */
hdl_copy = g_list_copy_deep(hash_data_list, copy_only_hash, NULL);
saved_list = g_list_concat(hdl_copy, saved_list);
/* 1. Send an array of hashs to Hash_Array.json server url */
answer = send_hash_array_to_server(main_struct->comm, hash_data_list);
/* 2. Keep only hashs that are needed (answer from the server) */
hash_data_list = send_all_data_to_server(main_struct, hash_data_list, answer);
/* 3. free memory of this list if any is left */
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
read_bytes = 0;
end_clock(elapsed, "process_big_file_not_in_cache");
}
/* get the list in correct order (because we prepended the hashs to get speed when inserting hashs in the list) */
saved_list = g_list_reverse(saved_list);
}
free_variable(buffer);
free_variable(a_hash);
g_checksum_free(checksum);
g_input_stream_close((GInputStream *) stream, NULL, NULL);
free_object(stream);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to open file for reading: %s\n"), error->message);
error = free_error(error);
}
meta->hash_data_list = saved_list;
answer = send_meta_data_to_server(main_struct, meta, TRUE);
if (answer != NULL)
{ /** @todo may be we should check that answer is something that tells that everything went Ok. */
/* Everything has been transmitted so we can save meta data into the local db cache */
/* This is usefull for file carving to avoid sending too much things to the server */
elapsed = new_clock_t();
db_save_meta_data(main_struct->database, meta, TRUE);
end_clock(elapsed, "db_save_meta_data");
}
a_file = free_object(a_file);
}
}
}
/**
* Calculates hashs for each block of blocksize bytes long on the file
* and returns a list of all hashs in correct order stored in a binary
* form to save space.
* @note This technique has some limits in term of memory footprint
* because one file is entirely in memory at a time. Saving huge
* files may not be possible with this, depending on the size of
* the file and the size of the memory.
* @todo Imagine a new way to checksum huge files because of limitations.
* May be with the local sqlite database ?
* @param a_file is the file from which we want the hashs.
* @param blocksize is the blocksize to be used to calculate hashs upon.
* @returns a GSList * list of hashs stored in a binary form.
*/
static GList *calculate_hash_data_list_for_file(GFile *a_file, gint64 blocksize)
{
GFileInputStream *stream = NULL;
GError *error = NULL;
GList *hash_data_list = NULL;
hash_data_t *hash_data = NULL;
gssize read = 0;
guchar *buffer = NULL;
GChecksum *checksum = NULL;
guint8 *a_hash = NULL;
gsize digest_len = HASH_LEN;
if (a_file != NULL)
{
stream = g_file_read(a_file, NULL, &error);
if (stream != NULL && error == NULL)
{
checksum = g_checksum_new(G_CHECKSUM_SHA256);
buffer = (guchar *) g_malloc(blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, blocksize, NULL, &error);
while (read != 0 && error == NULL)
{
g_checksum_update(checksum, buffer, read);
g_checksum_get_digest(checksum, a_hash, &digest_len);
/* Need to save data and read in hash_data_t structure */
hash_data = new_hash_data_t(buffer, read, a_hash);
hash_data_list = g_list_prepend(hash_data_list, hash_data);
g_checksum_reset(checksum);
digest_len = HASH_LEN;
buffer = (guchar *) g_malloc(blocksize);
a_hash = (guint8 *) g_malloc(digest_len);
read = g_input_stream_read((GInputStream *) stream, buffer, blocksize, NULL, &error);
}
if (error != NULL)
{
print_error(__FILE__, __LINE__, _("Error while reading file: %s\n"), error->message);
error = free_error(error);
g_list_free_full(hash_data_list, free_hdt_struct);
hash_data_list = NULL;
}
else
{
/* get the list in correct order (because we prepended the hashs to get speed when inserting hashs in the list) */
hash_data_list = g_list_reverse(hash_data_list);
}
free_variable(buffer);
free_variable(a_hash);
g_checksum_free(checksum);
g_input_stream_close((GInputStream *) stream, NULL, NULL);
free_object(stream);
}
else
{
print_error(__FILE__, __LINE__, _("Unable to open file for reading: %s\n"), error->message);
error = free_error(error);
}
}
return hash_data_list;
}
/**
* Clears the LibVLC error status for the current thread. This is optional.
* By default, the error status is automatically overridden when a new error
* occurs, and destroyed when the thread exits.
*/
void libvlc_clearerr (void)
{
free_error ();
vlc_threadvar_set (context, NULL);
}
