static void
tracker_extract_finalize (GObject *object)
{
TrackerExtractPrivate *priv;
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
/* FIXME: Shutdown modules? */
g_hash_table_destroy (priv->single_thread_extractors);
g_thread_pool_free (priv->thread_pool, TRUE, FALSE);
if (!priv->disable_summary_on_finalize) {
report_statistics (object);
}
#ifdef HAVE_LIBSTREAMANALYZER
tracker_topanalyzer_shutdown ();
#endif /* HAVE_STREAMANALYZER */
g_hash_table_destroy (priv->statistics_data);
g_mutex_clear (&priv->task_mutex);
G_OBJECT_CLASS (tracker_extract_parent_class)->finalize (object);
}
static void
tracker_extract_init (TrackerExtract *object)
{
TrackerExtractPrivate *priv;
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
priv->statistics_data = g_hash_table_new_full (NULL, NULL, NULL,
(GDestroyNotify) statistics_data_free);
priv->single_thread_extractors = g_hash_table_new (NULL, NULL);
priv->thread_pool = g_thread_pool_new ((GFunc) get_metadata,
NULL, 10, TRUE, NULL);
g_mutex_init (&priv->task_mutex);
}
/* This function can be called in any thread */
void
tracker_extract_file (TrackerExtract *extract,
const gchar *file,
const gchar *mimetype,
const gchar *graph,
GCancellable *cancellable,
GAsyncReadyCallback cb,
gpointer user_data)
{
GSimpleAsyncResult *res;
GError *error = NULL;
TrackerExtractTask *task;
g_return_if_fail (TRACKER_IS_EXTRACT (extract));
g_return_if_fail (file != NULL);
g_return_if_fail (cb != NULL);
#ifdef THREAD_ENABLE_TRACE
g_debug ("Thread:%p <-- '%s': Processing file\n",
g_thread_self (),
file);
#endif /* THREAD_ENABLE_TRACE */
res = g_simple_async_result_new (G_OBJECT (extract), cb, user_data, NULL);
task = extract_task_new (extract, file, mimetype, graph,
cancellable, G_ASYNC_RESULT (res), &error);
if (error) {
g_warning ("Could not get mimetype, %s", error->message);
g_simple_async_result_set_from_error (res, error);
g_simple_async_result_complete_in_idle (res);
g_error_free (error);
} else {
TrackerExtractPrivate *priv;
priv = TRACKER_EXTRACT_GET_PRIVATE (task->extract);
g_mutex_lock (&priv->task_mutex);
priv->running_tasks = g_list_prepend (priv->running_tasks, task);
g_mutex_unlock (&priv->task_mutex);
g_idle_add ((GSourceFunc) dispatch_task_cb, task);
}
/* Task takes a ref and if this fails, we want to unref anyway */
g_object_unref (res);
}
static void
tracker_extract_init (TrackerExtract *object)
{
TrackerExtractPrivate *priv;
#ifdef HAVE_LIBSTREAMANALYZER
tracker_topanalyzer_init ();
#endif /* HAVE_STREAMANALYZER */
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
priv->statistics_data = g_hash_table_new_full (NULL, NULL, NULL,
(GDestroyNotify) statistics_data_free);
priv->single_thread_extractors = g_hash_table_new (NULL, NULL);
priv->thread_pool = g_thread_pool_new ((GFunc) get_metadata,
NULL, 10, TRUE, NULL);
g_mutex_init (&priv->task_mutex);
}
static void
report_statistics (GObject *object)
{
TrackerExtractPrivate *priv;
GHashTableIter iter;
gpointer key, value;
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
g_mutex_lock (&priv->task_mutex);
g_message ("--------------------------------------------------");
g_message ("Statistics:");
g_hash_table_iter_init (&iter, priv->statistics_data);
while (g_hash_table_iter_next (&iter, &key, &value)) {
GModule *module = key;
StatisticsData *data = value;
if (data->extracted_count > 0 || data->failed_count > 0) {
const gchar *name, *name_without_path;
name = g_module_name (module);
name_without_path = strrchr (name, G_DIR_SEPARATOR) + 1;
g_message (" Module:'%s', extracted:%d, failures:%d",
name_without_path,
data->extracted_count,
data->failed_count);
}
}
g_message ("Unhandled files: %d", priv->unhandled_count);
if (priv->unhandled_count == 0 &&
g_hash_table_size (priv->statistics_data) < 1) {
g_message (" No files handled");
}
g_message ("--------------------------------------------------");
g_mutex_unlock (&priv->task_mutex);
}
static gboolean
filter_module (TrackerExtract *extract,
GModule *module)
{
TrackerExtractPrivate *priv;
gchar *module_basename, *filter_name;
gboolean filter;
priv = TRACKER_EXTRACT_GET_PRIVATE (extract);
if (!priv->force_module) {
return FALSE;
}
/* Module name is the full path to it */
module_basename = g_path_get_basename (g_module_name (module));
if (g_str_has_prefix (priv->force_module, "lib") &&
g_str_has_suffix (priv->force_module, "." G_MODULE_SUFFIX)) {
filter_name = g_strdup (priv->force_module);
} else {
filter_name = g_strdup_printf ("libextract-%s.so",
priv->force_module);
}
filter = strcmp (module_basename, filter_name) != 0;
if (filter) {
g_debug ("Module filtered out '%s' (due to --force-module='%s')",
module_basename,
filter_name);
} else {
g_debug ("Module used '%s' (due to --force-module='%s')",
module_basename,
filter_name);
}
g_free (module_basename);
g_free (filter_name);
return filter;
}
/* This function is called on the thread calling g_cancellable_cancel() */
static void
task_cancellable_cancelled_cb (GCancellable *cancellable,
TrackerExtractTask *task)
{
TrackerExtractPrivate *priv;
TrackerExtract *extract;
extract = task->extract;
priv = TRACKER_EXTRACT_GET_PRIVATE (extract);
g_mutex_lock (&priv->task_mutex);
if (g_list_find (priv->running_tasks, task)) {
g_message ("Cancelled task for '%s' was currently being "
"processed, _exit()ing immediately",
task->file);
_exit (0);
}
g_mutex_unlock (&priv->task_mutex);
}
TrackerExtract *
tracker_extract_new (gboolean disable_shutdown,
const gchar *force_module)
{
TrackerExtract *object;
TrackerExtractPrivate *priv;
if (!tracker_extract_module_manager_init ()) {
return NULL;
}
/* Set extractors */
object = g_object_new (TRACKER_TYPE_EXTRACT, NULL);
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
priv->disable_shutdown = disable_shutdown;
priv->force_module = g_strdup (force_module);
return object;
}
static void
notify_task_finish (TrackerExtractTask *task,
gboolean success)
{
TrackerExtract *extract;
TrackerExtractPrivate *priv;
StatisticsData *stats_data;
extract = task->extract;
priv = TRACKER_EXTRACT_GET_PRIVATE (extract);
/* Reports and ongoing tasks may be
* accessed from other threads.
*/
g_mutex_lock (&priv->task_mutex);
if (task->cur_module) {
stats_data = g_hash_table_lookup (priv->statistics_data,
task->cur_module);
if (!stats_data) {
stats_data = g_slice_new0 (StatisticsData);
g_hash_table_insert (priv->statistics_data,
task->cur_module,
stats_data);
}
stats_data->extracted_count++;
if (!success) {
stats_data->failed_count++;
}
} else {
priv->unhandled_count++;
}
priv->running_tasks = g_list_remove (priv->running_tasks, task);
g_mutex_unlock (&priv->task_mutex);
}
void
tracker_extract_get_metadata_by_cmdline (TrackerExtract *object,
const gchar *uri,
const gchar *mime)
{
GError *error = NULL;
TrackerExtractPrivate *priv;
TrackerExtractTask *task;
TrackerExtractInfo *info;
gboolean no_modules = TRUE;
priv = TRACKER_EXTRACT_GET_PRIVATE (object);
priv->disable_summary_on_finalize = TRUE;
g_return_if_fail (uri != NULL);
task = extract_task_new (object, uri, mime, NULL, NULL, NULL, &error);
if (error) {
g_printerr ("Extraction failed, %s\n", error->message);
g_error_free (error);
return;
}
task->mimetype_handlers = tracker_extract_module_manager_get_mimetype_handlers (task->mimetype);
task->cur_module = tracker_mimetype_info_get_module (task->mimetype_handlers, &task->cur_func, NULL);
while (task->cur_module && task->cur_func) {
if (!filter_module (object, task->cur_module) &&
get_file_metadata (task, &info)) {
const gchar *preupdate_str, *postupdate_str, *statements_str, *where;
TrackerSparqlBuilder *builder;
no_modules = FALSE;
preupdate_str = statements_str = postupdate_str = NULL;
builder = tracker_extract_info_get_metadata_builder (info);
if (tracker_sparql_builder_get_length (builder) > 0) {
statements_str = tracker_sparql_builder_get_result (builder);
}
builder = tracker_extract_info_get_preupdate_builder (info);
if (tracker_sparql_builder_get_length (builder) > 0) {
preupdate_str = tracker_sparql_builder_get_result (builder);
}
builder = tracker_extract_info_get_postupdate_builder (info);
if (tracker_sparql_builder_get_length (builder) > 0) {
postupdate_str = tracker_sparql_builder_get_result (builder);
}
where = tracker_extract_info_get_where_clause (info);
g_print ("\n");
g_print ("SPARQL pre-update:\n--\n%s--\n\n",
preupdate_str ? preupdate_str : "");
g_print ("SPARQL item:\n--\n%s--\n\n",
statements_str ? statements_str : "");
g_print ("SPARQL where clause:\n--\n%s--\n\n",
where ? where : "");
g_print ("SPARQL post-update:\n--\n%s--\n\n",
postupdate_str ? postupdate_str : "");
tracker_extract_info_unref (info);
break;
} else {
if (!tracker_mimetype_info_iter_next (task->mimetype_handlers)) {
break;
}
task->cur_module = tracker_mimetype_info_get_module (task->mimetype_handlers,
&task->cur_func,
NULL);
}
}
if (no_modules) {
g_print ("No modules found to handle metadata extraction\n\n");
}
extract_task_free (task);
}
/* This function is executed in the main thread, decides the
* module that's going to be run for a given task, and dispatches
* the task according to the threading strategy of that module.
*/
static gboolean
dispatch_task_cb (TrackerExtractTask *task)
{
TrackerModuleThreadAwareness thread_awareness;
TrackerExtractPrivate *priv;
GError *error = NULL;
GModule *module;
#ifdef THREAD_ENABLE_TRACE
g_debug ("Thread:%p (Main) <-- File:'%s' - Dispatching\n",
g_thread_self (),
task->file);
#endif /* THREAD_ENABLE_TRACE */
priv = TRACKER_EXTRACT_GET_PRIVATE (task->extract);
if (!task->mimetype) {
error = g_error_new (TRACKER_DBUS_ERROR, 0,
"No mimetype for '%s'",
task->file);
} else {
if (!task->mimetype_handlers) {
/* First iteration for task, get the mimetype handlers */
task->mimetype_handlers = tracker_extract_module_manager_get_mimetype_handlers (task->mimetype);
if (!task->mimetype_handlers) {
error = g_error_new (TRACKER_DBUS_ERROR, 0,
"No mimetype extractor handlers for uri:'%s' and mime:'%s'",
task->file, task->mimetype);
}
} else {
/* Any further iteration, should happen rarely if
* most specific handlers know nothing about the file
*/
g_message ("Trying next extractor for '%s'", task->file);
if (!tracker_mimetype_info_iter_next (task->mimetype_handlers)) {
g_message (" There's no next extractor");
error = g_error_new (TRACKER_DBUS_ERROR, 0,
"Could not get any metadata for uri:'%s' and mime:'%s'",
task->file, task->mimetype);
}
}
}
if (error) {
g_simple_async_result_set_from_error ((GSimpleAsyncResult *) task->res, error);
g_simple_async_result_complete_in_idle ((GSimpleAsyncResult *) task->res);
extract_task_free (task);
g_error_free (error);
return FALSE;
}
task->cur_module = module = tracker_mimetype_info_get_module (task->mimetype_handlers, &task->cur_func, &thread_awareness);
if (!module || !task->cur_func) {
g_warning ("Discarding task with no module '%s'", task->file);
priv->unhandled_count++;
return FALSE;
}
g_mutex_lock (&priv->task_mutex);
priv->running_tasks = g_list_prepend (priv->running_tasks, task);
g_mutex_unlock (&priv->task_mutex);
switch (thread_awareness) {
case TRACKER_MODULE_NONE:
/* Error out */
g_simple_async_result_set_error ((GSimpleAsyncResult *) task->res,
TRACKER_DBUS_ERROR, 0,
"Module '%s' initialization failed",
g_module_name (module));
g_simple_async_result_complete_in_idle ((GSimpleAsyncResult *) task->res);
extract_task_free (task);
break;
case TRACKER_MODULE_MAIN_THREAD:
/* Dispatch the task right away in this thread */
g_message ("Dispatching '%s' in main thread", task->file);
get_metadata (task);
break;
case TRACKER_MODULE_SINGLE_THREAD: {
GAsyncQueue *async_queue;
async_queue = g_hash_table_lookup (priv->single_thread_extractors, module);
if (!async_queue) {
GThread *thread;
/* No thread created yet for this module, create it
* together with the async queue used to pass data to it
*/
async_queue = g_async_queue_new ();
thread = g_thread_try_new ("single",
(GThreadFunc) single_thread_get_metadata,
g_async_queue_ref (async_queue),
&error);
if (!thread) {
g_simple_async_result_take_error ((GSimpleAsyncResult *) task->res, error);
g_simple_async_result_complete_in_idle ((GSimpleAsyncResult *) task->res);
extract_task_free (task);
return FALSE;
}
/* We won't join the thread, so just unref it here */
g_object_unref (thread);
g_hash_table_insert (priv->single_thread_extractors, module, async_queue);
}
g_async_queue_push (async_queue, task);
break;
}
case TRACKER_MODULE_MULTI_THREAD:
/* Put task in thread pool */
g_message ("Dispatching '%s' in thread pool", task->file);
g_thread_pool_push (priv->thread_pool, task, &error);
if (error) {
g_simple_async_result_set_from_error ((GSimpleAsyncResult *) task->res, error);
g_simple_async_result_complete_in_idle ((GSimpleAsyncResult *) task->res);
extract_task_free (task);
g_error_free (error);
return FALSE;
}
break;
}
return FALSE;
}
