static gboolean maybe_prune_loose_object (OtPruneData *data, OstreeRepoPruneFlags flags, const char *checksum, OstreeObjectType objtype, GCancellable *cancellable, GError **error) { g_autoptr(GVariant) key = NULL; key = ostree_object_name_serialize (checksum, objtype); if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) { g_debug ("Pruning unneeded object %s.%s", checksum, ostree_object_type_to_string (objtype)); if (!(flags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)) { guint64 storage_size = 0; if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!prune_commitpartial_file (data->repo, checksum, cancellable, error)) return FALSE; } if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum, &storage_size, cancellable, error)) return FALSE; if (!ostree_repo_delete_object (data->repo, objtype, checksum, cancellable, error)) return FALSE; data->freed_bytes += storage_size; } if (OSTREE_OBJECT_TYPE_IS_META (objtype)) data->n_unreachable_meta++; else data->n_unreachable_content++; } else { g_debug ("Keeping needed object %s.%s", checksum, ostree_object_type_to_string (objtype)); if (OSTREE_OBJECT_TYPE_IS_META (objtype)) data->n_reachable_meta++; else data->n_reachable_content++; } return TRUE; }
static gboolean scan_one_metadata_object (OtPullData *pull_data, const guchar *csum, OstreeObjectType objtype, guint recursion_depth, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; gs_unref_variant GVariant *object = NULL; gs_free char *tmp_checksum = NULL; gboolean is_requested; gboolean is_stored; tmp_checksum = ostree_checksum_from_bytes (csum); object = ostree_object_name_serialize (tmp_checksum, objtype); if (g_hash_table_lookup (pull_data->scanned_metadata, object)) return TRUE; is_requested = g_hash_table_lookup (pull_data->requested_metadata, tmp_checksum) != NULL; if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored, cancellable, error)) goto out; if (!is_stored && !is_requested) { char *duped_checksum = g_strdup (tmp_checksum); g_hash_table_insert (pull_data->requested_metadata, duped_checksum, duped_checksum); if (objtype == OSTREE_OBJECT_TYPE_COMMIT) ot_waitable_queue_push (pull_data->metadata_objects_to_fetch, pull_worker_message_new (PULL_MSG_FETCH_DETACHED_METADATA, g_variant_ref (object))); else ot_waitable_queue_push (pull_data->metadata_objects_to_fetch, pull_worker_message_new (PULL_MSG_FETCH, g_variant_ref (object))); } else if (is_stored) { if (pull_data->transaction_resuming || is_requested) { switch (objtype) { case OSTREE_OBJECT_TYPE_COMMIT: if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, pull_data->cancellable, error)) goto out; break; case OSTREE_OBJECT_TYPE_DIR_META: break; case OSTREE_OBJECT_TYPE_DIR_TREE: if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth, pull_data->cancellable, error)) goto out; break; case OSTREE_OBJECT_TYPE_FILE: g_assert_not_reached (); break; } } g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object); g_atomic_int_inc (&pull_data->n_scanned_metadata); } ret = TRUE; out: return ret; }
static gboolean scan_dirtree_object (OtPullData *pull_data, const char *checksum, int recursion_depth, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; int i, n; gs_unref_variant GVariant *tree = NULL; gs_unref_variant GVariant *files_variant = NULL; gs_unref_variant GVariant *dirs_variant = NULL; if (recursion_depth > OSTREE_MAX_RECURSION) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Exceeded maximum recursion"); goto out; } if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_DIR_TREE, checksum, &tree, error)) goto out; /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ files_variant = g_variant_get_child_value (tree, 0); dirs_variant = g_variant_get_child_value (tree, 1); n = g_variant_n_children (files_variant); for (i = 0; i < n; i++) { const char *filename; gboolean file_is_stored; gs_unref_variant GVariant *csum = NULL; gs_free char *file_checksum = NULL; g_variant_get_child (files_variant, i, "(&s@ay)", &filename, &csum); if (!ot_util_filename_validate (filename, error)) goto out; file_checksum = ostree_checksum_from_bytes_v (csum); if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, file_checksum, &file_is_stored, cancellable, error)) goto out; if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum)) { g_hash_table_insert (pull_data->requested_content, file_checksum, file_checksum); ot_waitable_queue_push (pull_data->metadata_objects_to_fetch, pull_worker_message_new (PULL_MSG_FETCH, ostree_object_name_serialize (file_checksum, OSTREE_OBJECT_TYPE_FILE))); file_checksum = NULL; /* Transfer ownership to hash */ } } n = g_variant_n_children (dirs_variant); for (i = 0; i < n; i++) { const char *dirname; gs_unref_variant GVariant *tree_csum = NULL; gs_unref_variant GVariant *meta_csum = NULL; g_variant_get_child (dirs_variant, i, "(&s@ay@ay)", &dirname, &tree_csum, &meta_csum); if (!ot_util_filename_validate (dirname, error)) goto out; if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_csum), OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1, cancellable, error)) goto out; if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (meta_csum), OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1, cancellable, error)) goto out; } ret = TRUE; out: return ret; }
gboolean ostree_repo_pull (OstreeRepo *self, const char *remote_name, char **refs_to_fetch, OstreeRepoPullFlags flags, OstreeAsyncProgress *progress, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; GHashTableIter hash_iter; gpointer key, value; gboolean tls_permissive = FALSE; OstreeFetcherConfigFlags fetcher_flags = 0; gs_free char *remote_key = NULL; gs_free char *path = NULL; gs_free char *baseurl = NULL; gs_free char *summary_data = NULL; gs_unref_hashtable GHashTable *requested_refs_to_fetch = NULL; gs_unref_hashtable GHashTable *updated_refs = NULL; gs_unref_hashtable GHashTable *commits_to_fetch = NULL; gs_free char *remote_mode_str = NULL; GSource *queue_src = NULL; OtPullData pull_data_real = { 0, }; OtPullData *pull_data = &pull_data_real; SoupURI *summary_uri = NULL; GKeyFile *config = NULL; GKeyFile *remote_config = NULL; char **configured_branches = NULL; guint64 bytes_transferred; guint64 start_time; guint64 end_time; pull_data->async_error = error; pull_data->main_context = g_main_context_ref_thread_default (); pull_data->loop = g_main_loop_new (pull_data->main_context, FALSE); pull_data->flags = flags; pull_data->repo = self; pull_data->progress = progress; pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, NULL); pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); start_time = g_get_monotonic_time (); pull_data->remote_name = g_strdup (remote_name); config = ostree_repo_get_config (self); remote_key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name); if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error)) goto out; pull_data->base_uri = soup_uri_new (baseurl); #ifdef HAVE_GPGME if (!ot_keyfile_get_boolean_with_default (config, remote_key, "gpg-verify", TRUE, &pull_data->gpg_verify, error)) goto out; #else pull_data->gpg_verify = FALSE; #endif if (!ot_keyfile_get_boolean_with_default (config, remote_key, "tls-permissive", FALSE, &tls_permissive, error)) goto out; if (tls_permissive) fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE; pull_data->fetcher = ostree_fetcher_new (pull_data->repo->tmp_dir, fetcher_flags); if (!pull_data->base_uri) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Failed to parse url '%s'", baseurl); goto out; } if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) goto out; if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare", &remote_mode_str, error)) goto out; if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) goto out; if (pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE_Z2) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't pull from archives with mode \"%s\"", remote_mode_str); goto out; } requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); updated_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (refs_to_fetch != NULL) { char **strviter; for (strviter = refs_to_fetch; *strviter; strviter++) { const char *branch = *strviter; char *contents; if (ostree_validate_checksum_string (branch, NULL)) { char *key = g_strdup (branch); g_hash_table_insert (commits_to_fetch, key, key); } else { if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) goto out; /* Transfer ownership of contents */ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } } } else { GError *temp_error = NULL; gboolean fetch_all_refs; configured_branches = g_key_file_get_string_list (config, remote_key, "branches", NULL, &temp_error); if (configured_branches == NULL && temp_error != NULL) { if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) { g_clear_error (&temp_error); fetch_all_refs = TRUE; } else { g_propagate_error (error, temp_error); goto out; } } else fetch_all_refs = FALSE; if (fetch_all_refs) { summary_uri = soup_uri_copy (pull_data->base_uri); path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL); soup_uri_set_path (summary_uri, path); if (!fetch_uri_contents_utf8_sync (pull_data, summary_uri, &summary_data, cancellable, error)) goto out; if (!parse_ref_summary (summary_data, &requested_refs_to_fetch, error)) goto out; } else { char **branches_iter = configured_branches; if (!(branches_iter && *branches_iter)) g_print ("No configured branches for remote %s\n", pull_data->remote_name); for (;branches_iter && *branches_iter; branches_iter++) { const char *branch = *branches_iter; char *contents; if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) goto out; /* Transfer ownership of contents */ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } } } if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming, cancellable, error)) goto out; pull_data->metadata_objects_to_fetch = ot_waitable_queue_new (); pull_data->metadata_objects_to_scan = ot_waitable_queue_new (); pull_data->metadata_thread = g_thread_new ("metadatascan", metadata_thread_main, pull_data); g_hash_table_iter_init (&hash_iter, commits_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *commit = value; ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_SCAN, ostree_object_name_serialize (commit, OSTREE_OBJECT_TYPE_COMMIT))); } g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *ref = key; const char *sha256 = value; ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_SCAN, ostree_object_name_serialize (sha256, OSTREE_OBJECT_TYPE_COMMIT))); g_hash_table_insert (updated_refs, g_strdup (ref), g_strdup (sha256)); } { queue_src = ot_waitable_queue_create_source (pull_data->metadata_objects_to_fetch); g_source_set_callback (queue_src, (GSourceFunc)on_metadata_objects_to_fetch_ready, pull_data, NULL); g_source_attach (queue_src, pull_data->main_context); g_source_unref (queue_src); } /* Prime the message queue */ pull_data->idle_serial++; ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_MAIN_IDLE, GUINT_TO_POINTER (pull_data->idle_serial))); /* Now await work completion */ if (!run_mainloop_monitor_fetcher (pull_data)) goto out; g_hash_table_iter_init (&hash_iter, updated_refs); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *ref = key; const char *checksum = value; gs_free char *remote_ref = NULL; gs_free char *original_rev = NULL; remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, ref); if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error)) goto out; if (original_rev && strcmp (checksum, original_rev) == 0) { g_print ("remote %s is unchanged from %s\n", remote_ref, original_rev); } else { ostree_repo_transaction_set_ref (pull_data->repo, pull_data->remote_name, ref, checksum); g_print ("remote %s is now %s\n", remote_ref, checksum); } } if (!ostree_repo_commit_transaction (pull_data->repo, NULL, cancellable, error)) goto out; end_time = g_get_monotonic_time (); bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher); if (bytes_transferred > 0) { guint shift; if (bytes_transferred < 1024) shift = 1; else shift = 1024; g_print ("%u metadata, %u content objects fetched; %" G_GUINT64_FORMAT " %s transferred in %u seconds\n", pull_data->n_fetched_metadata, pull_data->n_fetched_content, (guint64)(bytes_transferred / shift), shift == 1 ? "B" : "KiB", (guint) ((end_time - start_time) / G_USEC_PER_SEC)); } ret = TRUE; out: if (pull_data->main_context) g_main_context_unref (pull_data->main_context); if (pull_data->loop) g_main_loop_unref (pull_data->loop); g_strfreev (configured_branches); g_clear_object (&pull_data->fetcher); g_free (pull_data->remote_name); if (pull_data->base_uri) soup_uri_free (pull_data->base_uri); if (queue_src) g_source_destroy (queue_src); if (pull_data->metadata_thread) { ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_QUIT, NULL)); g_thread_join (pull_data->metadata_thread); } g_clear_pointer (&pull_data->metadata_objects_to_scan, (GDestroyNotify) ot_waitable_queue_unref); g_clear_pointer (&pull_data->metadata_objects_to_fetch, (GDestroyNotify) ot_waitable_queue_unref); g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); if (summary_uri) soup_uri_free (summary_uri); return ret; }
/** * ostree_repo_traverse_commit_union: (skip) * @repo: Repo * @commit_checksum: ASCII SHA256 checksum * @maxdepth: Traverse this many parent commits, -1 for unlimited * @inout_reachable: Set of reachable objects * @cancellable: Cancellable * @error: Error * * Update the set @inout_reachable containing all objects reachable * from @commit_checksum, traversing @maxdepth parent commits. */ gboolean ostree_repo_traverse_commit_union (OstreeRepo *repo, const char *commit_checksum, int maxdepth, GHashTable *inout_reachable, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; g_autofree char *tmp_checksum = NULL; while (TRUE) { gboolean recurse = FALSE; g_autoptr(GVariant) key = NULL; g_autoptr(GVariant) commit = NULL; ostree_cleanup_repo_commit_traverse_iter OstreeRepoCommitTraverseIter iter = { 0, }; key = ostree_object_name_serialize (commit_checksum, OSTREE_OBJECT_TYPE_COMMIT); if (g_hash_table_contains (inout_reachable, key)) break; if (!ostree_repo_load_variant_if_exists (repo, OSTREE_OBJECT_TYPE_COMMIT, commit_checksum, &commit, error)) goto out; /* Just return if the parent isn't found; we do expect most * people to have partial repositories. */ if (!commit) break; g_hash_table_add (inout_reachable, key); key = NULL; if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit, OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, error)) goto out; if (!traverse_iter (repo, &iter, inout_reachable, cancellable, error)) goto out; if (maxdepth == -1 || maxdepth > 0) { g_free (tmp_checksum); tmp_checksum = ostree_commit_get_parent (commit); if (tmp_checksum) { commit_checksum = tmp_checksum; if (maxdepth > 0) maxdepth -= 1; recurse = TRUE; } } if (!recurse) break; } ret = TRUE; out: return ret; }
static gboolean traverse_iter (OstreeRepo *repo, OstreeRepoCommitTraverseIter *iter, GHashTable *inout_reachable, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; while (TRUE) { g_autoptr(GVariant) key = NULL; OstreeRepoCommitIterResult iterres = ostree_repo_commit_traverse_iter_next (iter, cancellable, error); if (iterres == OSTREE_REPO_COMMIT_ITER_RESULT_ERROR) goto out; else if (iterres == OSTREE_REPO_COMMIT_ITER_RESULT_END) break; else if (iterres == OSTREE_REPO_COMMIT_ITER_RESULT_FILE) { char *name; char *checksum; ostree_repo_commit_traverse_iter_get_file (iter, &name, &checksum); key = ostree_object_name_serialize (checksum, OSTREE_OBJECT_TYPE_FILE); g_hash_table_replace (inout_reachable, key, key); key = NULL; } else if (iterres == OSTREE_REPO_COMMIT_ITER_RESULT_DIR) { char *name; char *content_checksum; char *meta_checksum; ostree_repo_commit_traverse_iter_get_dir (iter, &name, &content_checksum, &meta_checksum); key = ostree_object_name_serialize (meta_checksum, OSTREE_OBJECT_TYPE_DIR_META); g_hash_table_replace (inout_reachable, key, key); key = NULL; key = ostree_object_name_serialize (content_checksum, OSTREE_OBJECT_TYPE_DIR_TREE); if (!g_hash_table_lookup (inout_reachable, key)) { g_hash_table_replace (inout_reachable, key, key); key = NULL; if (!traverse_dirtree (repo, content_checksum, inout_reachable, cancellable, error)) goto out; } } else g_assert_not_reached (); } ret = TRUE; out: return ret; }
static gboolean maybe_prune_loose_object (OtPruneData *data, OstreeRepoPruneFlags flags, const char *checksum, OstreeObjectType objtype, GCancellable *cancellable, GError **error) { gboolean reachable = FALSE; g_autoptr(GVariant) key = NULL; key = ostree_object_name_serialize (checksum, objtype); if (g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) reachable = TRUE; else { guint64 storage_size = 0; g_debug ("Pruning unneeded object %s.%s", checksum, ostree_object_type_to_string (objtype)); if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum, &storage_size, cancellable, error)) return FALSE; data->freed_bytes += storage_size; if (!(flags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)) { if (objtype == OSTREE_OBJECT_TYPE_PAYLOAD_LINK) { ssize_t size; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; char target_checksum[OSTREE_SHA256_STRING_LEN+1]; char target_buf[_OSTREE_LOOSE_PATH_MAX + _OSTREE_PAYLOAD_LINK_PREFIX_LEN]; _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_PAYLOAD_LINK, data->repo->mode); size = readlinkat (data->repo->objects_dir_fd, loose_path_buf, target_buf, sizeof (target_buf)); if (size < 0) return glnx_throw_errno_prefix (error, "readlinkat"); if (size < OSTREE_SHA256_STRING_LEN + _OSTREE_PAYLOAD_LINK_PREFIX_LEN) return glnx_throw (error, "invalid data size for %s", loose_path_buf); sprintf (target_checksum, "%.2s%.62s", target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN, target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN + 3); g_autoptr(GVariant) target_key = ostree_object_name_serialize (target_checksum, OSTREE_OBJECT_TYPE_FILE); if (g_hash_table_lookup_extended (data->reachable, target_key, NULL, NULL)) { guint64 target_storage_size = 0; if (!ostree_repo_query_object_storage_size (data->repo, OSTREE_OBJECT_TYPE_FILE, target_checksum, &target_storage_size, cancellable, error)) return FALSE; reachable = target_storage_size >= data->repo->payload_link_threshold; if (reachable) goto exit; } } else if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!ostree_repo_mark_commit_partial (data->repo, checksum, FALSE, error)) return FALSE; } if (!ostree_repo_delete_object (data->repo, objtype, checksum, cancellable, error)) return FALSE; } if (OSTREE_OBJECT_TYPE_IS_META (objtype)) data->n_unreachable_meta++; else data->n_unreachable_content++; } exit: if (reachable) { g_debug ("Keeping needed object %s.%s", checksum, ostree_object_type_to_string (objtype)); if (OSTREE_OBJECT_TYPE_IS_META (objtype)) data->n_reachable_meta++; else data->n_reachable_content++; } return TRUE; }