static int pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset, *length; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0; long num_offset, num_length; time_t local_t = *mtime; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; time_t now, last; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite)) != EPKG_OK) { return (EPKG_FATAL); } if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) { goto cleanup; } if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest); pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0); } if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, use default settings", repo->name); fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) goto cleanup; digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) goto cleanup; packagesite_t = digest_t; *mtime = packagesite_t > digest_t ? packagesite_t : digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); assert(0); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); removed = HASH_COUNT(ldel); hash_it = 0; last = 0; HASH_ITER(hh, ldel, item, tmp_item) { now = time(NULL); if (++hash_it == removed || now > last) { pkg_emit_update_remove(removed, hash_it); last = now; } if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); }
static int pkg_repo_binary_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime, bool force) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; struct pkg *pkg = NULL; int rc = EPKG_FATAL; sqlite3 *sqlite = NULL; sqlite3_stmt *stmt; const char *origin, *digest, *offset, *length, *checksum; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0; long num_offset, num_length; time_t local_t; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; bool in_trans = false, legacy_repo = false; /* Required for making iterator */ struct pkgdb_it *it = NULL; struct pkgdb fakedb; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); /* In forced mode, ignore mtime */ if (force) *mtime = 0; /* Fetch meta */ local_t = *mtime; if (pkg_repo_fetch_meta(repo, &local_t) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, using " "default settings", repo->name); /* Fetch digests */ local_t = *mtime; fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) goto cleanup; /* Fetch packagesite */ digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) goto cleanup; packagesite_t = local_t; *mtime = digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); /* Detect whether we have legacy repo */ if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); if (digest == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } if (!pkg_checksum_is_valid(digest, strlen(digest))) { legacy_repo = true; pkg_debug(1, "repository '%s' has a legacy digests format", repo->name); } } fseek(fdigests, 0, SEEK_SET); /* Load local repository data */ rc = pkg_repo_binary_init_update(repo, name, force); if (rc == EPKG_END) { /* Need to perform forced update */ repo->ops->close(repo, false); return (pkg_repo_binary_update_incremental(name, repo, mtime, true)); } if (rc != EPKG_OK) { rc = EPKG_FATAL; goto cleanup; } /* Here sqlite is initialized */ sqlite = PRIV_GET(repo); stmt = pkg_repo_binary_get_origins(sqlite); if (stmt == NULL) { rc = EPKG_FATAL; goto cleanup; } fakedb.sqlite = sqlite; it = pkgdb_it_new_sqlite(&fakedb, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE); if (it != NULL) { while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST, &digest, PKG_CKSUM, &checksum); pkg_repo_binary_update_item_new(&ldel, origin, digest, 0, 0, checksum); } pkgdb_it_free(it); } else { sqlite3_finalize(stmt); } pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = p ? strsep(&p, ":\n") : NULL; checksum = p ? strsep(&p, ":\n") : NULL; if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_binary_update_item_new(&ladd, origin, digest, num_offset, num_length, checksum); } else { HASH_DEL(ldel, item); if (checksum == NULL || item->checksum == NULL) { pkg_repo_binary_update_item_new(&ladd, origin, digest, num_offset, num_length, checksum); updated++; } else if (strcmp(checksum, item->checksum) != 0) { /* Allow checksum to be used as unique mark */ pkg_repo_binary_update_item_new(&ladd, origin, digest, num_offset, num_length, checksum); updated++; } pkg_repo_binary_update_item_free(item); } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); rc = pkgdb_transaction_begin(sqlite, "REPO"); if (rc != EPKG_OK) goto cleanup; in_trans = true; removed = HASH_COUNT(ldel); hash_it = 0; if (removed > 0) pkg_emit_progress_start("Removing expired repository entries"); HASH_ITER(hh, ldel, item, tmp_item) { pkg_emit_progress_tick(++hash_it, removed); if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } else { pkg_emit_progress_tick(removed, removed); } HASH_DEL(ldel, item); pkg_repo_binary_update_item_free(item); }
static int pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset, *length; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0; long num_offset, num_length; time_t local_t = *mtime; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo; if (access(name, R_OK) != -1) new_repo = false; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) { return (EPKG_FATAL); } if (!reuse_repo) { pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name); local_t = 0; *mtime = 0; } if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) { goto cleanup; } if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, using " "default settings", repo->name); fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) { if (rc == EPKG_FATAL) /* Destroy repo completely */ if (new_repo) unlink(name); goto cleanup; } digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) { if (rc == EPKG_FATAL) /* Destroy repo completely */ if (new_repo) unlink(name); goto cleanup; } packagesite_t = digest_t; *mtime = packagesite_t > digest_t ? packagesite_t : digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); /* Detect whether we have legacy repo */ if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); if (digest == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } if (!pkg_checksum_is_valid(digest, strlen(digest))) { legacy_repo = true; pkg_debug(1, "repository '%s' has a legacy digests format", repo->name); } } fseek(fdigests, 0, SEEK_SET); /* Load local repo data */ while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST, &digest); pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0); } pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);"); in_trans = true; rc = pkgdb_transaction_begin(sqlite, "REPO"); if (rc != EPKG_OK) goto cleanup; removed = HASH_COUNT(ldel); hash_it = 0; pkg_emit_progress_start("Removing expired entries"); HASH_ITER(hh, ldel, item, tmp_item) { pkg_emit_progress_tick(++hash_it, removed); if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); }