static int pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset, *length; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0; long num_offset, num_length; time_t local_t = *mtime; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; time_t now, last; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite)) != EPKG_OK) { return (EPKG_FATAL); } if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) { goto cleanup; } if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest); pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0); } if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, use default settings", repo->name); fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) goto cleanup; digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) goto cleanup; packagesite_t = digest_t; *mtime = packagesite_t > digest_t ? packagesite_t : digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); assert(0); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); removed = HASH_COUNT(ldel); hash_it = 0; last = 0; HASH_ITER(hh, ldel, item, tmp_item) { now = time(NULL); if (++hash_it == removed || now > last) { pkg_emit_update_remove(removed, hash_it); last = now; } if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); }
static int pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset, *length; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0; long num_offset, num_length; time_t local_t = *mtime; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo; if (access(name, R_OK) != -1) new_repo = false; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) { return (EPKG_FATAL); } if (!reuse_repo) { pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name); local_t = 0; *mtime = 0; } if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) { goto cleanup; } if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, using " "default settings", repo->name); fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) { if (rc == EPKG_FATAL) /* Destroy repo completely */ if (new_repo) unlink(name); goto cleanup; } digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) { if (rc == EPKG_FATAL) /* Destroy repo completely */ if (new_repo) unlink(name); goto cleanup; } packagesite_t = digest_t; *mtime = packagesite_t > digest_t ? packagesite_t : digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); /* Detect whether we have legacy repo */ if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); if (digest == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } if (!pkg_checksum_is_valid(digest, strlen(digest))) { legacy_repo = true; pkg_debug(1, "repository '%s' has a legacy digests format", repo->name); } } fseek(fdigests, 0, SEEK_SET); /* Load local repo data */ while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST, &digest); pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0); } pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);"); in_trans = true; rc = pkgdb_transaction_begin(sqlite, "REPO"); if (rc != EPKG_OK) goto cleanup; removed = HASH_COUNT(ldel); hash_it = 0; pkg_emit_progress_start("Removing expired entries"); HASH_ITER(hh, ldel, item, tmp_item) { pkg_emit_progress_tick(++hash_it, removed); if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); }