static int pkg_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0; long num_offset; time_t local_t = *mtime; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; const char *myarch; struct pkg_manifest_parser *parser = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite, false)) != EPKG_OK) { return (EPKG_FATAL); } if ((rc = pkgdb_repo_init(sqlite, false)) != EPKG_OK) goto cleanup; if ((rc = pkg_register_repo(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest); pkg_update_increment_item_new(&ldel, origin, digest, 4); } fdigests = repo_fetch_remote_extract_tmp(repo, repo_digests_archive, "txz", &local_t, &rc, repo_digests_file); if (fdigests == NULL) goto cleanup; local_t = *mtime; fmanifest = repo_fetch_remote_extract_tmp(repo, repo_packagesite_archive, "txz", &local_t, &rc, repo_packagesite_file); if (fmanifest == NULL) goto cleanup; *mtime = local_t; fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); assert(0); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } processed++; HASH_FIND_STR(ldel, __DECONST(char *, origin), item); if (item == NULL) { added++; pkg_update_increment_item_new(&ladd, origin, digest, num_offset); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_update_increment_item_new(&ladd, origin, digest, num_offset); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); removed = HASH_COUNT(ldel) - updated; HASH_ITER(hh, ldel, item, tmp_item) { if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); } pkg_config_string(PKG_CONFIG_ABI, &myarch); pkg_debug(1, "Pkgrepo, pushing new entries for '%s'", name); pkg = NULL; if (len > 0 && len < SSIZE_MAX) { map = mmap(NULL, len, PROT_READ, MAP_SHARED, fileno(fmanifest), 0); fclose(fmanifest); } HASH_ITER(hh, ladd, item, tmp_item) { if (rc == EPKG_OK) { if (map != MAP_FAILED) { rc = pkg_add_from_manifest(NULL, map + item->offset, item->origin, len - item->offset, item->digest, myarch, sqlite, &parser, &pkg); } else { rc = pkg_add_from_manifest(fmanifest, NULL, item->origin, item->offset, item->digest, myarch, sqlite, &parser, &pkg); } } free(item->origin); free(item->digest); HASH_DEL(ladd, item); free(item); } pkg_manifest_parser_free(parser); pkg_emit_incremental_update(updated, removed, added, processed); cleanup: if (pkg != NULL) pkg_free(pkg); if (it != NULL) pkgdb_it_free(it); if (pkgdb_repo_close(sqlite, rc == EPKG_OK) != EPKG_OK) rc = EPKG_FATAL; if (map == MAP_FAILED && fmanifest) fclose(fmanifest); if (fdigests) fclose(fdigests); if (map != MAP_FAILED) munmap(map, len); return (rc); }
static int pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset, *length; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0; long num_offset, num_length; time_t local_t = *mtime; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; time_t now, last; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite)) != EPKG_OK) { return (EPKG_FATAL); } if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) { goto cleanup; } if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest); pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0); } if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, use default settings", repo->name); fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) goto cleanup; digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) goto cleanup; packagesite_t = digest_t; *mtime = packagesite_t > digest_t ? packagesite_t : digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); assert(0); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); removed = HASH_COUNT(ldel); hash_it = 0; last = 0; HASH_ITER(hh, ldel, item, tmp_item) { now = time(NULL); if (++hash_it == removed || now > last) { pkg_emit_update_remove(removed, hash_it); last = now; } if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); }
static int pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset, *length; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0; long num_offset, num_length; time_t local_t = *mtime; time_t digest_t; time_t packagesite_t; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; struct pkg_manifest_key *keys = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; int hash_it = 0; bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo; if (access(name, R_OK) != -1) new_repo = false; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) { return (EPKG_FATAL); } if (!reuse_repo) { pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name); local_t = 0; *mtime = 0; } if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) { goto cleanup; } if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL) pkg_emit_notice("repository %s has no meta file, using " "default settings", repo->name); fdigests = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->digests, &local_t, &rc); if (fdigests == NULL) { if (rc == EPKG_FATAL) /* Destroy repo completely */ if (new_repo) unlink(name); goto cleanup; } digest_t = local_t; local_t = *mtime; fmanifest = pkg_repo_fetch_remote_extract_tmp(repo, repo->meta->manifests, &local_t, &rc); if (fmanifest == NULL) { if (rc == EPKG_FATAL) /* Destroy repo completely */ if (new_repo) unlink(name); goto cleanup; } packagesite_t = digest_t; *mtime = packagesite_t > digest_t ? packagesite_t : digest_t; /*fconflicts = repo_fetch_remote_extract_tmp(repo, repo_conflicts_archive, "txz", &local_t, &rc, repo_conflicts_file);*/ fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); /* Detect whether we have legacy repo */ if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); if (digest == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } if (!pkg_checksum_is_valid(digest, strlen(digest))) { legacy_repo = true; pkg_debug(1, "repository '%s' has a legacy digests format", repo->name); } } fseek(fdigests, 0, SEEK_SET); /* Load local repo data */ while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST, &digest); pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0); } pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); /* files offset */ strsep(&p, ":"); length = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } if (length != NULL) { errno = 0; num_length = (long)strtoul(length, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } } else { num_length = 0; } processed++; HASH_FIND_STR(ldel, origin, item); if (item == NULL) { added++; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset, num_length); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);"); in_trans = true; rc = pkgdb_transaction_begin(sqlite, "REPO"); if (rc != EPKG_OK) goto cleanup; removed = HASH_COUNT(ldel); hash_it = 0; pkg_emit_progress_start("Removing expired entries"); HASH_ITER(hh, ldel, item, tmp_item) { pkg_emit_progress_tick(++hash_it, removed); if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); }