gboolean ostree_repo_pull (OstreeRepo *self, const char *remote_name, char **refs_to_fetch, OstreeRepoPullFlags flags, OstreeAsyncProgress *progress, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; GHashTableIter hash_iter; gpointer key, value; gboolean tls_permissive = FALSE; OstreeFetcherConfigFlags fetcher_flags = 0; gs_free char *remote_key = NULL; gs_free char *path = NULL; gs_free char *baseurl = NULL; gs_free char *summary_data = NULL; gs_unref_hashtable GHashTable *requested_refs_to_fetch = NULL; gs_unref_hashtable GHashTable *updated_refs = NULL; gs_unref_hashtable GHashTable *commits_to_fetch = NULL; gs_free char *remote_mode_str = NULL; GSource *queue_src = NULL; OtPullData pull_data_real = { 0, }; OtPullData *pull_data = &pull_data_real; SoupURI *summary_uri = NULL; GKeyFile *config = NULL; GKeyFile *remote_config = NULL; char **configured_branches = NULL; guint64 bytes_transferred; guint64 start_time; guint64 end_time; pull_data->async_error = error; pull_data->main_context = g_main_context_ref_thread_default (); pull_data->loop = g_main_loop_new (pull_data->main_context, FALSE); pull_data->flags = flags; pull_data->repo = self; pull_data->progress = progress; pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, NULL); pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); start_time = g_get_monotonic_time (); pull_data->remote_name = g_strdup (remote_name); config = ostree_repo_get_config (self); remote_key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name); if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error)) goto out; pull_data->base_uri = soup_uri_new (baseurl); #ifdef HAVE_GPGME if (!ot_keyfile_get_boolean_with_default (config, remote_key, "gpg-verify", TRUE, &pull_data->gpg_verify, error)) goto out; #else pull_data->gpg_verify = FALSE; #endif if (!ot_keyfile_get_boolean_with_default (config, remote_key, "tls-permissive", FALSE, &tls_permissive, error)) goto out; if (tls_permissive) fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE; pull_data->fetcher = ostree_fetcher_new (pull_data->repo->tmp_dir, fetcher_flags); if (!pull_data->base_uri) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Failed to parse url '%s'", baseurl); goto out; } if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) goto out; if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare", &remote_mode_str, error)) goto out; if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) goto out; if (pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE_Z2) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't pull from archives with mode \"%s\"", remote_mode_str); goto out; } requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); updated_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (refs_to_fetch != NULL) { char **strviter; for (strviter = refs_to_fetch; *strviter; strviter++) { const char *branch = *strviter; char *contents; if (ostree_validate_checksum_string (branch, NULL)) { char *key = g_strdup (branch); g_hash_table_insert (commits_to_fetch, key, key); } else { if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) goto out; /* Transfer ownership of contents */ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } } } else { GError *temp_error = NULL; gboolean fetch_all_refs; configured_branches = g_key_file_get_string_list (config, remote_key, "branches", NULL, &temp_error); if (configured_branches == NULL && temp_error != NULL) { if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) { g_clear_error (&temp_error); fetch_all_refs = TRUE; } else { g_propagate_error (error, temp_error); goto out; } } else fetch_all_refs = FALSE; if (fetch_all_refs) { summary_uri = soup_uri_copy (pull_data->base_uri); path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL); soup_uri_set_path (summary_uri, path); if (!fetch_uri_contents_utf8_sync (pull_data, summary_uri, &summary_data, cancellable, error)) goto out; if (!parse_ref_summary (summary_data, &requested_refs_to_fetch, error)) goto out; } else { char **branches_iter = configured_branches; if (!(branches_iter && *branches_iter)) g_print ("No configured branches for remote %s\n", pull_data->remote_name); for (;branches_iter && *branches_iter; branches_iter++) { const char *branch = *branches_iter; char *contents; if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) goto out; /* Transfer ownership of contents */ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } } } if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming, cancellable, error)) goto out; pull_data->metadata_objects_to_fetch = ot_waitable_queue_new (); pull_data->metadata_objects_to_scan = ot_waitable_queue_new (); pull_data->metadata_thread = g_thread_new ("metadatascan", metadata_thread_main, pull_data); g_hash_table_iter_init (&hash_iter, commits_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *commit = value; ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_SCAN, ostree_object_name_serialize (commit, OSTREE_OBJECT_TYPE_COMMIT))); } g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *ref = key; const char *sha256 = value; ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_SCAN, ostree_object_name_serialize (sha256, OSTREE_OBJECT_TYPE_COMMIT))); g_hash_table_insert (updated_refs, g_strdup (ref), g_strdup (sha256)); } { queue_src = ot_waitable_queue_create_source (pull_data->metadata_objects_to_fetch); g_source_set_callback (queue_src, (GSourceFunc)on_metadata_objects_to_fetch_ready, pull_data, NULL); g_source_attach (queue_src, pull_data->main_context); g_source_unref (queue_src); } /* Prime the message queue */ pull_data->idle_serial++; ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_MAIN_IDLE, GUINT_TO_POINTER (pull_data->idle_serial))); /* Now await work completion */ if (!run_mainloop_monitor_fetcher (pull_data)) goto out; g_hash_table_iter_init (&hash_iter, updated_refs); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *ref = key; const char *checksum = value; gs_free char *remote_ref = NULL; gs_free char *original_rev = NULL; remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, ref); if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error)) goto out; if (original_rev && strcmp (checksum, original_rev) == 0) { g_print ("remote %s is unchanged from %s\n", remote_ref, original_rev); } else { ostree_repo_transaction_set_ref (pull_data->repo, pull_data->remote_name, ref, checksum); g_print ("remote %s is now %s\n", remote_ref, checksum); } } if (!ostree_repo_commit_transaction (pull_data->repo, NULL, cancellable, error)) goto out; end_time = g_get_monotonic_time (); bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher); if (bytes_transferred > 0) { guint shift; if (bytes_transferred < 1024) shift = 1; else shift = 1024; g_print ("%u metadata, %u content objects fetched; %" G_GUINT64_FORMAT " %s transferred in %u seconds\n", pull_data->n_fetched_metadata, pull_data->n_fetched_content, (guint64)(bytes_transferred / shift), shift == 1 ? "B" : "KiB", (guint) ((end_time - start_time) / G_USEC_PER_SEC)); } ret = TRUE; out: if (pull_data->main_context) g_main_context_unref (pull_data->main_context); if (pull_data->loop) g_main_loop_unref (pull_data->loop); g_strfreev (configured_branches); g_clear_object (&pull_data->fetcher); g_free (pull_data->remote_name); if (pull_data->base_uri) soup_uri_free (pull_data->base_uri); if (queue_src) g_source_destroy (queue_src); if (pull_data->metadata_thread) { ot_waitable_queue_push (pull_data->metadata_objects_to_scan, pull_worker_message_new (PULL_MSG_QUIT, NULL)); g_thread_join (pull_data->metadata_thread); } g_clear_pointer (&pull_data->metadata_objects_to_scan, (GDestroyNotify) ot_waitable_queue_unref); g_clear_pointer (&pull_data->metadata_objects_to_fetch, (GDestroyNotify) ot_waitable_queue_unref); g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref); if (summary_uri) soup_uri_free (summary_uri); return ret; }
gboolean ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError **error) { GOptionContext *context; glnx_unref_object OstreeRepo *repo = NULL; gboolean ret = FALSE; gboolean skip_commit = FALSE; g_autoptr(GFile) object_to_commit = NULL; g_autofree char *parent = NULL; g_autofree char *commit_checksum = NULL; g_autoptr(GFile) root = NULL; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) detached_metadata = NULL; glnx_unref_object OstreeMutableTree *mtree = NULL; g_autofree char *tree_type = NULL; g_autoptr(GHashTable) mode_adds = NULL; OstreeRepoCommitModifierFlags flags = 0; OstreeRepoCommitModifier *modifier = NULL; OstreeRepoTransactionStats stats; context = g_option_context_new ("[PATH] - Commit a new revision"); if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) goto out; if (!ostree_ensure_repo_writable (repo, error)) goto out; if (opt_statoverride_file) { if (!parse_statoverride_file (&mode_adds, cancellable, error)) goto out; } if (opt_metadata_strings) { if (!parse_keyvalue_strings (opt_metadata_strings, &metadata, error)) goto out; } if (opt_detached_metadata_strings) { if (!parse_keyvalue_strings (opt_detached_metadata_strings, &detached_metadata, error)) goto out; } if (!opt_branch) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "A branch must be specified with --branch"); goto out; } if (opt_no_xattrs) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS; if (opt_generate_sizes) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES; if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); if (flags != 0 || opt_owner_uid >= 0 || opt_owner_gid >= 0 || opt_statoverride_file != NULL || opt_no_xattrs) { modifier = ostree_repo_commit_modifier_new (flags, commit_filter, mode_adds, NULL); } if (!ostree_repo_resolve_rev (repo, opt_branch, TRUE, &parent, error)) goto out; if (!opt_subject && !opt_body) { if (!commit_editor (repo, opt_branch, &opt_subject, &opt_body, cancellable, error)) goto out; } if (!opt_subject) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "A subject must be specified with --subject"); goto out; } if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error)) goto out; if (opt_link_checkout_speedup && !ostree_repo_scan_hardlinks (repo, cancellable, error)) goto out; mtree = ostree_mutable_tree_new (); if (argc <= 1 && (opt_trees == NULL || opt_trees[0] == NULL)) { char *current_dir = g_get_current_dir (); object_to_commit = g_file_new_for_path (current_dir); g_free (current_dir); if (!ostree_repo_write_directory_to_mtree (repo, object_to_commit, mtree, modifier, cancellable, error)) goto out; } else if (opt_trees != NULL) { const char *const*tree_iter; const char *tree; const char *eq; for (tree_iter = (const char *const*)opt_trees; *tree_iter; tree_iter++) { tree = *tree_iter; eq = strchr (tree, '='); if (!eq) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Missing type in tree specification '%s'", tree); goto out; } g_free (tree_type); tree_type = g_strndup (tree, eq - tree); tree = eq + 1; g_clear_object (&object_to_commit); if (strcmp (tree_type, "dir") == 0) { object_to_commit = g_file_new_for_path (tree); if (!ostree_repo_write_directory_to_mtree (repo, object_to_commit, mtree, modifier, cancellable, error)) goto out; } else if (strcmp (tree_type, "tar") == 0) { object_to_commit = g_file_new_for_path (tree); if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier, opt_tar_autocreate_parents, cancellable, error)) goto out; } else if (strcmp (tree_type, "ref") == 0) { if (!ostree_repo_read_commit (repo, tree, &object_to_commit, NULL, cancellable, error)) goto out; if (!ostree_repo_write_directory_to_mtree (repo, object_to_commit, mtree, modifier, cancellable, error)) goto out; } else { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid tree type specification '%s'", tree_type); goto out; } } } else { g_assert (argc > 1); object_to_commit = g_file_new_for_path (argv[1]); if (!ostree_repo_write_directory_to_mtree (repo, object_to_commit, mtree, modifier, cancellable, error)) goto out; } if (mode_adds && g_hash_table_size (mode_adds) > 0) { GHashTableIter hash_iter; gpointer key, value; g_hash_table_iter_init (&hash_iter, mode_adds); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { g_printerr ("Unmatched statoverride path: %s\n", (char*)key); } g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Unmatched statoverride paths"); goto out; } if (!ostree_repo_write_mtree (repo, mtree, &root, cancellable, error)) goto out; if (opt_skip_if_unchanged && parent) { g_autoptr(GFile) parent_root; if (!ostree_repo_read_commit (repo, parent, &parent_root, NULL, cancellable, error)) goto out; if (g_file_equal (root, parent_root)) skip_commit = TRUE; } if (!skip_commit) { gboolean update_summary; if (!ostree_repo_write_commit (repo, parent, opt_subject, opt_body, metadata, OSTREE_REPO_FILE (root), &commit_checksum, cancellable, error)) goto out; if (detached_metadata) { if (!ostree_repo_write_commit_detached_metadata (repo, commit_checksum, detached_metadata, cancellable, error)) goto out; } if (opt_key_ids) { char **iter; for (iter = opt_key_ids; iter && *iter; iter++) { const char *keyid = *iter; if (!ostree_repo_sign_commit (repo, commit_checksum, keyid, opt_gpg_homedir, cancellable, error)) goto out; } } ostree_repo_transaction_set_ref (repo, NULL, opt_branch, commit_checksum); if (!ostree_repo_commit_transaction (repo, &stats, cancellable, error)) goto out; /* The default for this option is FALSE, even for archive-z2 repos, * because ostree supports multiple processes committing to the same * repo (but different refs) concurrently, and in fact gnome-continuous * actually does this. In that context it's best to update the summary * explicitly instead of automatically here. */ if (!ot_keyfile_get_boolean_with_default (ostree_repo_get_config (repo), "core", "commit-update-summary", FALSE, &update_summary, error)) goto out; if (update_summary && !ostree_repo_regenerate_summary (repo, NULL, cancellable, error)) goto out; } else { commit_checksum = g_strdup (parent); } if (opt_table_output) { g_print ("Commit: %s\n", commit_checksum); g_print ("Metadata Total: %u\n", stats.metadata_objects_total); g_print ("Metadata Written: %u\n", stats.metadata_objects_written); g_print ("Content Total: %u\n", stats.content_objects_total); g_print ("Content Written: %u\n", stats.content_objects_written); g_print ("Content Bytes Written: %" G_GUINT64_FORMAT "\n", stats.content_bytes_written); } else { g_print ("%s\n", commit_checksum); } ret = TRUE; out: if (repo) ostree_repo_abort_transaction (repo, cancellable, NULL); if (context) g_option_context_free (context); if (modifier) ostree_repo_commit_modifier_unref (modifier); return ret; }