void QMLManager::saveChangesCloud()
{
git_storage_update_progress(true, "start save change to cloud");
if (!loadFromCloud()) {
appendTextToLog("Don't save dives without loading from the cloud, first.");
return;
}
bool glo = prefs.git_local_only;
bool cbs = prefs.cloud_background_sync;
// first we need to store any unsaved changes to the local repo
saveChangesLocal();
if (alreadySaving) {
appendTextToLog("save operation in progress already, can't sync with server");
return;
}
prefs.git_local_only = false;
alreadySaving = true;
loadDivesWithValidCredentials();
alreadySaving = false;
git_storage_update_progress(false, "finished syncing dive list to cloud server");
setAccessingCloud(-1);
prefs.git_local_only = glo;
prefs.cloud_background_sync = cbs;
alreadySaving = false;
}
void QMLManager::saveChangesLocal()
{
if (unsaved_changes()) {
git_storage_update_progress(true, "saving dives locally"); // reset the timers
if (!loadFromCloud()) {
// this seems silly, but you need a common ancestor in the repository in
// order to be able to merge che changes later
appendTextToLog("Don't save dives without loading from the cloud, first.");
return;
}
if (alreadySaving) {
appendTextToLog("save operation already in progress, can't save locally");
return;
}
alreadySaving = true;
bool glo = prefs.git_local_only;
prefs.git_local_only = true;
if (save_dives(existing_filename)) {
appendTextToLog(get_error_string());
setAccessingCloud(-1);
prefs.git_local_only = glo;
alreadySaving = false;
return;
}
prefs.git_local_only = glo;
mark_divelist_changed(false);
git_storage_update_progress(false, "done with local save");
alreadySaving = false;
} else {
appendTextToLog("local save requested with no unsaved changes");
}
}
void QMLManager::saveChangesCloud(bool forceRemoteSync)
{
if (!unsaved_changes() && !forceRemoteSync) {
appendTextToLog("asked to save changes but no unsaved changes");
return;
}
if (alreadySaving) {
appendTextToLog("save operation in progress already");
return;
}
// first we need to store any unsaved changes to the local repo
saveChangesLocal();
// if the user asked not to push to the cloud we are done
if (prefs.git_local_only && !forceRemoteSync)
return;
if (!loadFromCloud()) {
appendTextToLog("Don't save dives without loading from the cloud, first.");
return;
}
bool glo = prefs.git_local_only;
git_storage_update_progress(false, "start save change to cloud");
prefs.git_local_only = false;
alreadySaving = true;
loadDivesWithValidCredentials();
alreadySaving = false;
git_storage_update_progress(false, "finished syncing dive list to cloud server");
setAccessingCloud(-1);
prefs.git_local_only = glo;
}
bool CheckCloudConnection::checkServer()
{
if (verbose)
fprintf(stderr, "Checking cloud connection...\n");
QTimer timer;
timer.setSingleShot(true);
QEventLoop loop;
QNetworkRequest request;
request.setRawHeader("Accept", "text/plain");
request.setRawHeader("User-Agent", getUserAgent().toUtf8());
request.setRawHeader("Client-Id", getUUID().toUtf8());
request.setUrl(QString(prefs.cloud_base_url) + TEAPOT);
QNetworkAccessManager *mgr = new QNetworkAccessManager();
reply = mgr->get(request);
connect(&timer, &QTimer::timeout, &loop, &QEventLoop::quit);
connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
connect(reply, &QNetworkReply::sslErrors, this, &CheckCloudConnection::sslErrors);
for (int seconds = 1; seconds <= prefs.cloud_timeout; seconds++) {
timer.start(1000); // wait the given number of seconds (default 5)
loop.exec();
if (timer.isActive()) {
// didn't time out, did we get the right response?
timer.stop();
if (reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == HTTP_I_AM_A_TEAPOT &&
reply->readAll() == QByteArray(MILK)) {
reply->deleteLater();
mgr->deleteLater();
if (verbose > 1)
qWarning() << "Cloud storage: successfully checked connection to cloud server";
git_storage_update_progress(false, "successfully checked cloud connection");
return true;
}
} else if (seconds < prefs.cloud_timeout) {
QString text = QString("waited %1 sec for cloud connetion").arg(seconds);
git_storage_update_progress(false, qPrintable(text));
} else {
disconnect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
reply->abort();
}
}
git_storage_update_progress(false, "cloud connection failed");
prefs.git_local_only = true;
if (verbose)
qDebug() << "connection test to cloud server failed" <<
reply->error() << reply->errorString() <<
reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() <<
reply->readAll();
reply->deleteLater();
mgr->deleteLater();
if (verbose)
qWarning() << "Cloud storage: unable to connect to cloud server";
return false;
}
static struct git_repository *get_remote_repo(const char *localdir, const char *remote, const char *branch)
{
struct stat st;
enum remote_transport rt;
/* figure out the remote transport */
if (strncmp(remote, "ssh://", 6) == 0)
rt = RT_SSH;
else if (strncmp(remote, "https://", 8) == 0)
rt = RT_HTTPS;
else
rt = RT_OTHER;
if (verbose > 1) {
fprintf(stderr, "git_remote_repo: accessing %s\n", remote);
}
git_storage_update_progress(false, "start git interaction");
/* Do we already have a local cache? */
if (!stat(localdir, &st)) {
if (!S_ISDIR(st.st_mode)) {
if (is_subsurface_cloud)
(void)cleanup_local_cache(remote, branch);
else
report_error("local git cache at '%s' is corrupt");
return NULL;
}
return update_local_repo(localdir, remote, branch, rt);
}
if (!prefs.git_local_only)
return create_local_repo(localdir, remote, branch, rt);
else
return 0;
}
static struct git_repository *get_remote_repo(const char *localdir, const char *remote, const char *branch)
{
struct stat st;
enum remote_transport rt = url_to_remote_transport(remote);
if (verbose > 1) {
fprintf(stderr, "git_remote_repo: accessing %s\n", remote);
}
git_storage_update_progress(translate("gettextFromC", "Synchronising data file"));
/* Do we already have a local cache? */
if (!subsurface_stat(localdir, &st)) {
if (!S_ISDIR(st.st_mode)) {
if (is_subsurface_cloud)
(void)cleanup_local_cache(remote, branch);
else
report_error("local git cache at '%s' is corrupt", localdir);
return NULL;
}
return update_local_repo(localdir, remote, branch, rt);
} else {
/* We have no local cache yet.
* Take us temporarly online to create a local and
* remote cloud repo.
*/
git_repository *ret;
bool glo = git_local_only;
git_local_only = false;
ret = create_local_repo(localdir, remote, branch, rt);
git_local_only = glo;
return ret;
}
/* all normal cases are handled above */
return 0;
}
// this randomly assumes that 80% of the time is spent on the objects and 20% on the deltas
// map the git progress to 20% of overall progress
// if the user cancels the dialog this is passed back to libgit2
static int transfer_progress_cb(const git_transfer_progress *stats, void *payload)
{
UNUSED(payload);
static int last_done = -1;
char buf[80];
int done = 0;
int total = 0;
if (stats->total_objects) {
total = 60;
done = 60 * stats->received_objects / stats->total_objects;
}
if (stats->total_deltas) {
total += 20;
done += 20 * stats->indexed_deltas / stats->total_deltas;
}
/* for debugging this is useful
char buf[100];
snprintf(buf, 100, "transfer cb rec_obj %d tot_obj %d idx_delta %d total_delta %d local obj %d", stats->received_objects, stats->total_objects, stats->indexed_deltas, stats->total_deltas, stats->local_objects);
return git_storage_update_progress(buf);
*/
if (done > last_done) {
last_done = done;
snprintf(buf, sizeof(buf), translate("gettextFromC", "Transfer from storage (%d/%d)"), done, total);
return git_storage_update_progress(buf);
}
return 0;
}
void QMLManager::retrieveUserid()
{
if (reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) != 302) {
appendTextToLog(QStringLiteral("Cloud storage connection not working correctly: (%1) %2")
.arg(reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt())
.arg(QString(reply->readAll())));
setStartPageText(RED_FONT + tr("Cannot connect to cloud storage") + END_FONT);
revertToNoCloudIfNeeded();
return;
}
setCredentialStatus(VALID);
QString userid(prefs.userid);
if (userid.isEmpty()) {
if (same_string(prefs.cloud_storage_email, "") || same_string(prefs.cloud_storage_password, "")) {
appendTextToLog("cloud user name or password are empty, can't retrieve web user id");
revertToNoCloudIfNeeded();
return;
}
appendTextToLog(QStringLiteral("calling getUserid with user %1").arg(prefs.cloud_storage_email));
userid = locationProvider->getUserid(prefs.cloud_storage_email, prefs.cloud_storage_password);
}
if (!userid.isEmpty()) {
// overwrite the existing userid
free(prefs.userid);
prefs.userid = strdup(qPrintable(userid));
QSettings s;
s.setValue("subsurface_webservice_uid", prefs.userid);
s.sync();
}
setCredentialStatus(VALID);
setStartPageText("Cloud credentials valid, loading dives...");
git_storage_update_progress(true, "load dives with valid credentials");
// this only gets called with "alreadySaving" already locked
loadDivesWithValidCredentials();
}
// the initial push to sync the repos is mapped to 10% of overall progress
static int push_transfer_progress_cb(unsigned int current, unsigned int total, size_t bytes, void *payload)
{
UNUSED(bytes);
UNUSED(payload);
char buf[80];
snprintf(buf, sizeof(buf), translate("gettextFromC", "Transfer to storage (%d/%d)"), current, total);
return git_storage_update_progress(buf);
}
// the checkout_progress_cb doesn't allow canceling of the operation
// map the git progress to 20% of overall progress
static void progress_cb(const char *path, size_t completed_steps, size_t total_steps, void *payload)
{
UNUSED(path);
UNUSED(payload);
char buf[80];
snprintf(buf, sizeof(buf), translate("gettextFromC", "Checkout from storage (%lu/%lu)"), completed_steps, total_steps);
(void)git_storage_update_progress(buf);
}
void QMLManager::saveChangesLocal()
{
if (unsaved_changes()) {
git_storage_update_progress(true, "saving dives locally"); // reset the timers
if (credentialStatus() == NOCLOUD) {
if (same_string(existing_filename, "")) {
char *filename = NOCLOUD_LOCALSTORAGE;
if (git_create_local_repo(filename))
appendTextToLog(get_error_string());
set_filename(filename, true);
GeneralSettingsObjectWrapper s(this);
s.setDefaultFilename(filename);
s.setDefaultFileBehavior(LOCAL_DEFAULT_FILE);
qDebug() << "setting default file to" << filename;
}
} else if (!loadFromCloud()) {
// this seems silly, but you need a common ancestor in the repository in
// order to be able to merge che changes later
appendTextToLog("Don't save dives without loading from the cloud, first.");
return;
}
if (alreadySaving) {
appendTextToLog("save operation already in progress, can't save locally");
return;
}
alreadySaving = true;
bool glo = prefs.git_local_only;
prefs.git_local_only = true;
if (save_dives(existing_filename)) {
appendTextToLog(get_error_string());
set_filename(NULL, true);
setAccessingCloud(-1);
prefs.git_local_only = glo;
alreadySaving = false;
return;
}
prefs.git_local_only = glo;
mark_divelist_changed(false);
git_storage_update_progress(false, "done with local save");
alreadySaving = false;
} else {
appendTextToLog("local save requested with no unsaved changes");
}
}
// the checkout_progress_cb doesn't allow canceling of the operation
// map the git progress to 20% of overall progress
static void progress_cb(const char *path, size_t completed_steps, size_t total_steps, void *payload)
{
(void) path;
(void) payload;
static size_t last_percent = -1;
if (total_steps && 20 * completed_steps / total_steps > last_percent) {
(void)git_storage_update_progress(false, "checkout_progress_cb");
last_percent = 20 * completed_steps / total_steps;
}
}
static int do_git_load(git_repository *repo, const char *branch)
{
int ret;
git_commit *commit;
git_tree *tree;
git_storage_update_progress(false, "do_git_load, find the commit");
ret = find_commit(repo, branch, &commit);
if (ret)
return ret;
git_storage_update_progress(false, "git commit tree");
if (git_commit_tree(&tree, commit))
return report_error("Could not look up tree of commit in branch '%s'", branch);
git_storage_update_progress(false, "load dives from tree");
ret = load_dives_from_tree(repo, tree);
if (!ret)
set_git_id(git_commit_id(commit));
git_object_free((git_object *)tree);
git_storage_update_progress(false, "done do_git_load");
return ret;
}
static int check_remote_status(git_repository *repo, git_remote *origin, const char *remote, const char *branch, enum remote_transport rt)
{
int error = 0;
git_reference *local_ref, *remote_ref;
if (verbose)
fprintf(stderr, "git storage: check remote status\n");
git_storage_update_progress(false, "git check remote status");
if (git_branch_lookup(&local_ref, repo, branch, GIT_BRANCH_LOCAL)) {
if (is_subsurface_cloud)
return cleanup_local_cache(remote, branch);
else
return report_error("Git cache branch %s no longer exists", branch);
}
if (git_branch_upstream(&remote_ref, local_ref)) {
/* so there is no upstream branch for our branch; that's a problem.
* let's push our branch */
git_strarray refspec;
git_reference_list(&refspec, repo);
git_push_options opts = GIT_PUSH_OPTIONS_INIT;
opts.callbacks.transfer_progress = &transfer_progress_cb;
if (rt == RT_SSH)
opts.callbacks.credentials = credential_ssh_cb;
else if (rt == RT_HTTPS)
opts.callbacks.credentials = credential_https_cb;
opts.callbacks.certificate_check = certificate_check_cb;
git_storage_update_progress(false, "git remote push (no remote existed)");
error = git_remote_push(origin, &refspec, &opts);
} else {
error = try_to_update(repo, origin, local_ref, remote_ref, remote, branch, rt);
git_reference_free(remote_ref);
}
git_reference_free(local_ref);
return error;
}
// the initial push to sync the repos is mapped to 10% of overall progress
static int push_transfer_progress_cb(unsigned int current, unsigned int total, size_t bytes, void *payload)
{
(void) bytes;
(void) payload;
static int last_percent = -1;
int percent = 0;
if (total != 0)
percent = 5 * current / total;
if (percent > last_percent) {
last_percent = percent;
return git_storage_update_progress(false, "push trasfer cb");
}
return 0;
}
// this randomly assumes that 80% of the time is spent on the objects and 20% on the deltas
// map the git progress to 20% of overall progress
// if the user cancels the dialog this is passed back to libgit2
static int transfer_progress_cb(const git_transfer_progress *stats, void *payload)
{
(void) payload;
static int last_percent = -1;
int percent = 0;
if (stats->total_objects)
percent = 16 * stats->received_objects / stats->total_objects;
if (stats->total_deltas)
percent += 4 * stats->indexed_deltas / stats->total_deltas;
/* for debugging this is useful
char buf[100];
snprintf(buf, 100, "transfer cb rec_obj %d tot_obj %d idx_delta %d total_delta %d local obj %d", stats->received_objects, stats->total_objects, stats->indexed_deltas, stats->total_deltas, stats->local_objects);
return git_storage_update_progress(false, buf);
*/
if (percent > last_percent) {
last_percent = percent;
return git_storage_update_progress(false, "transfer cb");
}
return 0;
}
void QMLManager::loadDivesWithValidCredentials()
{
QString url;
timestamp_t currentDiveTimestamp = selectedDiveTimestamp();
if (getCloudURL(url)) {
QString errorString(get_error_string());
appendTextToLog(errorString);
setStartPageText(RED_FONT + tr("Cloud storage error: %1").arg(errorString) + END_FONT);
revertToNoCloudIfNeeded();
return;
}
QByteArray fileNamePrt = QFile::encodeName(url);
git_repository *git;
const char *branch;
int error;
if (check_git_sha(fileNamePrt.data(), &git, &branch) == 0) {
qDebug() << "local cache was current, no need to modify dive list";
appendTextToLog("Cloud sync shows local cache was current");
goto successful_exit;
}
appendTextToLog("Cloud sync brought newer data, reloading the dive list");
clear_dive_file_data();
if (git != dummy_git_repository) {
appendTextToLog(QString("have repository and branch %1").arg(branch));
error = git_load_dives(git, branch);
} else {
appendTextToLog(QString("didn't receive valid git repo, try again"));
error = parse_file(fileNamePrt.data());
}
setAccessingCloud(-1);
if (!error) {
report_error("filename is now %s", fileNamePrt.data());
const char *error_string = get_error_string();
appendTextToLog(error_string);
set_filename(fileNamePrt.data(), true);
} else {
report_error("failed to open file %s", fileNamePrt.data());
QString errorString(get_error_string());
appendTextToLog(errorString);
revertToNoCloudIfNeeded();
return;
}
consumeFinishedLoad(currentDiveTimestamp);
successful_exit:
alreadySaving = false;
setLoadFromCloud(true);
// if we came from local storage mode, let's merge the local data into the local cache
// for the remote data - which then later gets merged with the remote data if necessary
if (oldStatus() == NOCLOUD) {
git_storage_update_progress(false, "import dives from nocloud local storage");
dive_table.preexisting = dive_table.nr;
mergeLocalRepo();
DiveListModel::instance()->clear();
DiveListModel::instance()->addAllDives();
appendTextToLog(QStringLiteral("%1 dives loaded after importing nocloud local storage").arg(dive_table.nr));
saveChangesLocal();
if (syncToCloud() == false) {
appendTextToLog(QStringLiteral("taking things back offline now that storage is synced"));
prefs.git_local_only = syncToCloud();
}
}
setAccessingCloud(-1);
// if we got here just for an initial connection to the cloud, reset to offline
if (currentGitLocalOnly) {
currentGitLocalOnly = false;
prefs.git_local_only = true;
}
return;
}
int sync_with_remote(git_repository *repo, const char *remote, const char *branch, enum remote_transport rt)
{
int error;
git_remote *origin;
char *proxy_string;
git_config *conf;
if (git_local_only) {
if (verbose)
fprintf(stderr, "don't sync with remote - read from cache only\n");
return 0;
}
if (verbose)
fprintf(stderr, "sync with remote %s[%s]\n", remote, branch);
git_storage_update_progress(translate("gettextFromC", "Sync with cloud storage"));
git_repository_config(&conf, repo);
if (rt == RT_HTTPS && getProxyString(&proxy_string)) {
if (verbose)
fprintf(stderr, "set proxy to \"%s\"\n", proxy_string);
git_config_set_string(conf, "http.proxy", proxy_string);
free(proxy_string);
} else {
if (verbose)
fprintf(stderr, "delete proxy setting\n");
git_config_delete_entry(conf, "http.proxy");
}
/*
* NOTE! Remote errors are reported, but are nonfatal:
* we still successfully return the local repository.
*/
error = git_remote_lookup(&origin, repo, "origin");
if (error) {
if (!is_subsurface_cloud)
report_error("Repository '%s' origin lookup failed (%s)", remote, giterr_last()->message);
return 0;
}
if (is_subsurface_cloud && !canReachCloudServer()) {
// this is not an error, just a warning message, so return 0
report_error("Cannot connect to cloud server, working with local copy");
git_storage_update_progress(translate("gettextFromC", "Can't reach cloud server, working with local data"));
return 0;
}
if (verbose)
fprintf(stderr, "git storage: fetch remote\n");
git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
opts.callbacks.transfer_progress = &transfer_progress_cb;
auth_attempt = 0;
if (rt == RT_SSH)
opts.callbacks.credentials = credential_ssh_cb;
else if (rt == RT_HTTPS)
opts.callbacks.credentials = credential_https_cb;
opts.callbacks.certificate_check = certificate_check_cb;
git_storage_update_progress(translate("gettextFromC", "Successful cloud connection, fetch remote"));
error = git_remote_fetch(origin, NULL, &opts, NULL);
// NOTE! A fetch error is not fatal, we just report it
if (error) {
if (is_subsurface_cloud)
report_error("Cannot sync with cloud server, working with offline copy");
else
report_error("Unable to fetch remote '%s'", remote);
if (verbose)
// If we returned GIT_EUSER during authentication, giterr_last() returns NULL
fprintf(stderr, "remote fetch failed (%s)\n",
giterr_last() ? giterr_last()->message : "authentication failed");
// Since we failed to sync with online repository, enter offline mode
git_local_only = true;
error = 0;
} else {
error = check_remote_status(repo, origin, remote, branch, rt);
}
git_remote_free(origin);
git_storage_update_progress(translate("gettextFromC", "Done syncing with cloud storage"));
return error;
}
static int try_to_update(git_repository *repo, git_remote *origin, git_reference *local, git_reference *remote,
const char *remote_url, const char *branch, enum remote_transport rt)
{
git_oid base;
const git_oid *local_id, *remote_id;
int ret = 0;
if (verbose)
fprintf(stderr, "git storage: try to update\n");
if (!git_reference_cmp(local, remote))
return 0;
// Dirty modified state in the working tree? We're not going
// to update either way
if (git_status_foreach(repo, check_clean, NULL)) {
if (is_subsurface_cloud)
goto cloud_data_error;
else
return report_error("local cached copy is dirty, skipping update");
}
local_id = git_reference_target(local);
remote_id = git_reference_target(remote);
if (!local_id || !remote_id) {
if (is_subsurface_cloud)
goto cloud_data_error;
else
return report_error("Unable to get local or remote SHA1");
}
if (git_merge_base(&base, repo, local_id, remote_id)) {
// TODO:
// if they have no merge base, they actually are different repos
// so instead merge this as merging a commit into a repo - git_merge() appears to do that
// but needs testing and cleanup afterwards
//
if (is_subsurface_cloud)
goto cloud_data_error;
else
return report_error("Unable to find common commit of local and remote branches");
}
/* Is the remote strictly newer? Use it */
if (git_oid_equal(&base, local_id)) {
git_storage_update_progress(translate("gettextFromC", "Update local storage to match cloud storage"));
return reset_to_remote(repo, local, remote_id);
}
/* Is the local repo the more recent one? See if we can update upstream */
if (git_oid_equal(&base, remote_id)) {
if (verbose)
fprintf(stderr, "local is newer than remote, update remote\n");
git_storage_update_progress(translate("gettextFromC", "Push local changes to cloud storage"));
return update_remote(repo, origin, local, remote, rt);
}
/* Merging a bare repository always needs user action */
if (git_repository_is_bare(repo)) {
if (is_subsurface_cloud)
goto cloud_data_error;
else
return report_error("Local and remote have diverged, merge of bare branch needed");
}
/* Merging will definitely need the head branch too */
if (git_branch_is_head(local) != 1) {
if (is_subsurface_cloud)
goto cloud_data_error;
else
return report_error("Local and remote do not match, local branch not HEAD - cannot update");
}
/* Ok, let's try to merge these */
git_storage_update_progress(translate("gettextFromC", "Try to merge local changes into cloud storage"));
ret = try_to_git_merge(repo, &local, remote, &base, local_id, remote_id);
if (ret == 0)
return update_remote(repo, origin, local, remote, rt);
else
return ret;
cloud_data_error:
// since we are working with Subsurface cloud storage we want to make the user interaction
// as painless as possible. So if something went wrong with the local cache, tell the user
// about it an move it away
return cleanup_local_cache(remote_url, branch);
}
