int
pkg_repo_binary_open(struct pkg_repo *repo, unsigned mode)
{
char filepath[MAXPATHLEN];
struct statfs stfs;
const char *dbdir = NULL;
sqlite3 *sqlite = NULL;
int flags;
int64_t res;
struct pkg_repo_it *it;
struct pkg *pkg = NULL;
const char *digest;
sqlite3_initialize();
dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));
/*
* Fall back on unix-dotfile locking strategy if on a network filesystem
*/
if (statfs(dbdir, &stfs) == 0) {
if ((stfs.f_flags & MNT_LOCAL) != MNT_LOCAL)
sqlite3_vfs_register(sqlite3_vfs_find("unix-dotfile"), 1);
}
snprintf(filepath, sizeof(filepath), "%s/%s.meta",
dbdir, pkg_repo_name(repo));
/* Open metafile */
if (access(filepath, R_OK) != -1) {
if (pkg_repo_meta_load(filepath, &repo->meta) != EPKG_OK)
return (EPKG_FATAL);
}
snprintf(filepath, sizeof(filepath), "%s/%s",
dbdir, pkg_repo_binary_get_filename(pkg_repo_name(repo)));
/* Always want read mode here */
if (access(filepath, R_OK | mode) != 0)
return (EPKG_ENOACCESS);
flags = (mode & W_OK) != 0 ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY;
if (sqlite3_open_v2(filepath, &sqlite, flags, NULL) != SQLITE_OK)
return (EPKG_FATAL);
/* Sanitise sqlite database */
if (get_pragma(sqlite, "SELECT count(name) FROM sqlite_master "
"WHERE type='table' AND name='repodata';", &res, false) != EPKG_OK) {
pkg_emit_error("Unable to query repository");
sqlite3_close(sqlite);
return (EPKG_FATAL);
}
if (res != 1) {
pkg_emit_notice("Repository %s contains no repodata table, "
"need to re-create database", repo->name);
sqlite3_close(sqlite);
return (EPKG_FATAL);
}
/* Check package site */
char *req = sqlite3_mprintf("select count(key) from repodata "
"WHERE key = \"packagesite\" and value = '%q'", pkg_repo_url(repo));
res = 0;
get_pragma(sqlite, req, &res, true);
sqlite3_free(req);
if (res != 1) {
pkg_emit_notice("Repository %s has a wrong packagesite, need to "
"re-create database", repo->name);
sqlite3_close(sqlite);
return (EPKG_FATAL);
}
/* Check version */
if (pkg_repo_binary_check_version(repo, sqlite) != EPKG_OK) {
pkg_emit_error("need to re-create repo %s to upgrade schema version",
repo->name);
sqlite3_close(sqlite);
if (mode & W_OK)
unlink(filepath);
return (EPKG_REPOSCHEMA);
}
repo->priv = sqlite;
/* Check digests format */
if ((it = pkg_repo_binary_query(repo, NULL, MATCH_ALL)) == NULL)
return (EPKG_OK);
if (it->ops->next(it, &pkg, PKG_LOAD_BASIC) != EPKG_OK) {
it->ops->free(it);
return (EPKG_OK);
}
it->ops->free(it);
pkg_get(pkg, PKG_DIGEST, &digest);
if (digest == NULL || !pkg_checksum_is_valid(digest, strlen(digest))) {
pkg_emit_notice("Repository %s has incompatible checksum format, need to "
"re-create database", repo->name);
pkg_free(pkg);
sqlite3_close(sqlite);
repo->priv = NULL;
return (EPKG_FATAL);
}
pkg_free(pkg);
return (EPKG_OK);
}
static int
pkg_repo_binary_update_incremental(const char *name, struct pkg_repo *repo,
time_t *mtime, bool force)
{
FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
sqlite3 *sqlite = NULL;
sqlite3_stmt *stmt;
const char *origin, *digest, *offset, *length, *checksum;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
long num_offset, num_length;
time_t local_t;
time_t digest_t;
time_t packagesite_t;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
struct pkg_manifest_key *keys = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
int hash_it = 0;
bool in_trans = false, legacy_repo = false;
/* Required for making iterator */
struct pkgdb_it *it = NULL;
struct pkgdb fakedb;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
/* In forced mode, ignore mtime */
if (force)
*mtime = 0;
/* Fetch meta */
local_t = *mtime;
if (pkg_repo_fetch_meta(repo, &local_t) == EPKG_FATAL)
pkg_emit_notice("repository %s has no meta file, using "
"default settings", repo->name);
/* Fetch digests */
local_t = *mtime;
fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->digests, &local_t, &rc);
if (fdigests == NULL)
goto cleanup;
/* Fetch packagesite */
digest_t = local_t;
local_t = *mtime;
fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->manifests, &local_t, &rc);
if (fmanifest == NULL)
goto cleanup;
packagesite_t = local_t;
*mtime = digest_t;
/*fconflicts = repo_fetch_remote_extract_tmp(repo,
repo_conflicts_archive, "txz", &local_t,
&rc, repo_conflicts_file);*/
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
/* Detect whether we have legacy repo */
if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
if (digest == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
if (!pkg_checksum_is_valid(digest, strlen(digest))) {
legacy_repo = true;
pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
}
}
fseek(fdigests, 0, SEEK_SET);
/* Load local repository data */
rc = pkg_repo_binary_init_update(repo, name, force);
if (rc == EPKG_END) {
/* Need to perform forced update */
repo->ops->close(repo, false);
return (pkg_repo_binary_update_incremental(name, repo, mtime, true));
}
if (rc != EPKG_OK) {
rc = EPKG_FATAL;
goto cleanup;
}
/* Here sqlite is initialized */
sqlite = PRIV_GET(repo);
stmt = pkg_repo_binary_get_origins(sqlite);
if (stmt == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
fakedb.sqlite = sqlite;
it = pkgdb_it_new_sqlite(&fakedb, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE);
if (it != NULL) {
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
&digest, PKG_CKSUM, &checksum);
pkg_repo_binary_update_item_new(&ldel, origin, digest, 0, 0, checksum);
}
pkgdb_it_free(it);
}
else {
sqlite3_finalize(stmt);
}
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
/* files offset */
strsep(&p, ":");
length = p ? strsep(&p, ":\n") : NULL;
checksum = p ? strsep(&p, ":\n") : NULL;
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
if (length != NULL) {
errno = 0;
num_length = (long)strtoul(length, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
}
else {
num_length = 0;
}
processed++;
HASH_FIND_STR(ldel, origin, item);
if (item == NULL) {
added++;
pkg_repo_binary_update_item_new(&ladd, origin, digest, num_offset,
num_length, checksum);
} else {
HASH_DEL(ldel, item);
if (checksum == NULL || item->checksum == NULL) {
pkg_repo_binary_update_item_new(&ladd, origin, digest,
num_offset, num_length, checksum);
updated++;
}
else if (strcmp(checksum, item->checksum) != 0) {
/* Allow checksum to be used as unique mark */
pkg_repo_binary_update_item_new(&ladd, origin, digest,
num_offset, num_length, checksum);
updated++;
}
pkg_repo_binary_update_item_free(item);
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
rc = pkgdb_transaction_begin(sqlite, "REPO");
if (rc != EPKG_OK)
goto cleanup;
in_trans = true;
removed = HASH_COUNT(ldel);
hash_it = 0;
if (removed > 0)
pkg_emit_progress_start("Removing expired repository entries");
HASH_ITER(hh, ldel, item, tmp_item) {
pkg_emit_progress_tick(++hash_it, removed);
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
else {
pkg_emit_progress_tick(removed, removed);
}
HASH_DEL(ldel, item);
pkg_repo_binary_update_item_free(item);
}
static int
pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime)
{
FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
sqlite3 *sqlite = NULL;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
const char *origin, *digest, *offset, *length;
struct pkgdb_it *it = NULL;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
long num_offset, num_length;
time_t local_t = *mtime;
time_t digest_t;
time_t packagesite_t;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
struct pkg_manifest_key *keys = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
int hash_it = 0;
bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo;
if (access(name, R_OK) != -1)
new_repo = false;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) {
return (EPKG_FATAL);
}
if (!reuse_repo) {
pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name);
local_t = 0;
*mtime = 0;
}
if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) {
goto cleanup;
}
if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK)
goto cleanup;
it = pkgdb_repo_origins(sqlite);
if (it == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)
pkg_emit_notice("repository %s has no meta file, using "
"default settings", repo->name);
fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->digests, &local_t, &rc);
if (fdigests == NULL) {
if (rc == EPKG_FATAL)
/* Destroy repo completely */
if (new_repo)
unlink(name);
goto cleanup;
}
digest_t = local_t;
local_t = *mtime;
fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->manifests, &local_t, &rc);
if (fmanifest == NULL) {
if (rc == EPKG_FATAL)
/* Destroy repo completely */
if (new_repo)
unlink(name);
goto cleanup;
}
packagesite_t = digest_t;
*mtime = packagesite_t > digest_t ? packagesite_t : digest_t;
/*fconflicts = repo_fetch_remote_extract_tmp(repo,
repo_conflicts_archive, "txz", &local_t,
&rc, repo_conflicts_file);*/
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
/* Detect whether we have legacy repo */
if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
if (digest == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
if (!pkg_checksum_is_valid(digest, strlen(digest))) {
legacy_repo = true;
pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
}
}
fseek(fdigests, 0, SEEK_SET);
/* Load local repo data */
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
&digest);
pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
}
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
/* files offset */
strsep(&p, ":");
length = strsep(&p, ":");
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
if (length != NULL) {
errno = 0;
num_length = (long)strtoul(length, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
}
else {
num_length = 0;
}
processed++;
HASH_FIND_STR(ldel, origin, item);
if (item == NULL) {
added++;
pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset,
num_length);
} else {
if (strcmp(digest, item->digest) == 0) {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
} else {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
pkg_repo_update_increment_item_new(&ladd, origin, digest,
num_offset, num_length);
updated++;
}
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);");
in_trans = true;
rc = pkgdb_transaction_begin(sqlite, "REPO");
if (rc != EPKG_OK)
goto cleanup;
removed = HASH_COUNT(ldel);
hash_it = 0;
pkg_emit_progress_start("Removing expired entries");
HASH_ITER(hh, ldel, item, tmp_item) {
pkg_emit_progress_tick(++hash_it, removed);
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
}