int
pkg_delete_files(struct pkg *pkg, unsigned force)
/* force: 0 ... be careful and vocal about it.
* 1 ... remove files without bothering about checksums.
* 2 ... like 1, but remain silent if removal fails.
*/
{
struct pkg_file *file = NULL;
int nfiles, cur_file = 0;
nfiles = HASH_COUNT(pkg->files);
if (nfiles == 0)
return (EPKG_OK);
pkg_emit_delete_files_begin(pkg);
pkg_emit_progress_start(NULL);
while (pkg_files(pkg, &file) == EPKG_OK) {
pkg_emit_progress_tick(cur_file++, nfiles);
if (file->keep == 1)
continue;
pkg_delete_file(pkg, file, force);
}
pkg_emit_progress_tick(nfiles, nfiles);
pkg_emit_delete_files_finished(pkg);
return (EPKG_OK);
}
static void
counter_init(const char *count_what, int64_t max)
{
count = 0;
what = count_what;
maxcount = max;
pkg_emit_progress_start("%-20s%*s[%ld]", what,
6 - magnitude(maxcount), " ", maxcount);
return;
}
static int
pkg_repo_binary_update_incremental(const char *name, struct pkg_repo *repo,
time_t *mtime, bool force)
{
FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
sqlite3 *sqlite = NULL;
sqlite3_stmt *stmt;
const char *origin, *digest, *offset, *length, *checksum;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
long num_offset, num_length;
time_t local_t;
time_t digest_t;
time_t packagesite_t;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
struct pkg_manifest_key *keys = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
int hash_it = 0;
bool in_trans = false, legacy_repo = false;
/* Required for making iterator */
struct pkgdb_it *it = NULL;
struct pkgdb fakedb;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
/* In forced mode, ignore mtime */
if (force)
*mtime = 0;
/* Fetch meta */
local_t = *mtime;
if (pkg_repo_fetch_meta(repo, &local_t) == EPKG_FATAL)
pkg_emit_notice("repository %s has no meta file, using "
"default settings", repo->name);
/* Fetch digests */
local_t = *mtime;
fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->digests, &local_t, &rc);
if (fdigests == NULL)
goto cleanup;
/* Fetch packagesite */
digest_t = local_t;
local_t = *mtime;
fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->manifests, &local_t, &rc);
if (fmanifest == NULL)
goto cleanup;
packagesite_t = local_t;
*mtime = digest_t;
/*fconflicts = repo_fetch_remote_extract_tmp(repo,
repo_conflicts_archive, "txz", &local_t,
&rc, repo_conflicts_file);*/
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
/* Detect whether we have legacy repo */
if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
if (digest == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
if (!pkg_checksum_is_valid(digest, strlen(digest))) {
legacy_repo = true;
pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
}
}
fseek(fdigests, 0, SEEK_SET);
/* Load local repository data */
rc = pkg_repo_binary_init_update(repo, name, force);
if (rc == EPKG_END) {
/* Need to perform forced update */
repo->ops->close(repo, false);
return (pkg_repo_binary_update_incremental(name, repo, mtime, true));
}
if (rc != EPKG_OK) {
rc = EPKG_FATAL;
goto cleanup;
}
/* Here sqlite is initialized */
sqlite = PRIV_GET(repo);
stmt = pkg_repo_binary_get_origins(sqlite);
if (stmt == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
fakedb.sqlite = sqlite;
it = pkgdb_it_new_sqlite(&fakedb, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE);
if (it != NULL) {
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
&digest, PKG_CKSUM, &checksum);
pkg_repo_binary_update_item_new(&ldel, origin, digest, 0, 0, checksum);
}
pkgdb_it_free(it);
}
else {
sqlite3_finalize(stmt);
}
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
/* files offset */
strsep(&p, ":");
length = p ? strsep(&p, ":\n") : NULL;
checksum = p ? strsep(&p, ":\n") : NULL;
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
if (length != NULL) {
errno = 0;
num_length = (long)strtoul(length, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
}
else {
num_length = 0;
}
processed++;
HASH_FIND_STR(ldel, origin, item);
if (item == NULL) {
added++;
pkg_repo_binary_update_item_new(&ladd, origin, digest, num_offset,
num_length, checksum);
} else {
HASH_DEL(ldel, item);
if (checksum == NULL || item->checksum == NULL) {
pkg_repo_binary_update_item_new(&ladd, origin, digest,
num_offset, num_length, checksum);
updated++;
}
else if (strcmp(checksum, item->checksum) != 0) {
/* Allow checksum to be used as unique mark */
pkg_repo_binary_update_item_new(&ladd, origin, digest,
num_offset, num_length, checksum);
updated++;
}
pkg_repo_binary_update_item_free(item);
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
rc = pkgdb_transaction_begin(sqlite, "REPO");
if (rc != EPKG_OK)
goto cleanup;
in_trans = true;
removed = HASH_COUNT(ldel);
hash_it = 0;
if (removed > 0)
pkg_emit_progress_start("Removing expired repository entries");
HASH_ITER(hh, ldel, item, tmp_item) {
pkg_emit_progress_tick(++hash_it, removed);
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
else {
pkg_emit_progress_tick(removed, removed);
}
HASH_DEL(ldel, item);
pkg_repo_binary_update_item_free(item);
}
static int
do_extract(struct archive *a, struct archive_entry *ae, const char *location,
int nfiles, struct pkg *pkg, struct pkg *local)
{
int retcode = EPKG_OK;
int ret = 0, cur_file = 0;
char path[MAXPATHLEN], pathname[MAXPATHLEN], rpath[MAXPATHLEN];
struct stat st;
const struct stat *aest;
bool renamed = false;
const struct pkg_file *rf;
struct pkg_config_file *rcf;
struct sbuf *newconf;
bool automerge = pkg_object_bool(pkg_config_get("AUTOMERGE"));
unsigned long set, clear;
#ifndef HAVE_ARC4RANDOM
srand(time(NULL));
#endif
if (nfiles == 0)
return (EPKG_OK);
pkg_emit_extract_begin(pkg);
pkg_emit_progress_start(NULL);
newconf = sbuf_new_auto();
do {
ret = ARCHIVE_OK;
sbuf_clear(newconf);
rf = NULL;
rcf = NULL;
pkg_absolutepath(archive_entry_pathname(ae), path, sizeof(path));
snprintf(pathname, sizeof(pathname), "%s%s%s",
location ? location : "", *path == '/' ? "" : "/",
path
);
strlcpy(rpath, pathname, sizeof(rpath));
aest = archive_entry_stat(ae);
archive_entry_fflags(ae, &set, &clear);
if (lstat(rpath, &st) != -1) {
/*
* We have an existing file on the path, so handle it
*/
if (!S_ISDIR(aest->st_mode)) {
pkg_debug(2, "Old version found, renaming");
pkg_add_file_random_suffix(rpath, sizeof(rpath), 12);
renamed = true;
}
if (!S_ISDIR(st.st_mode) && S_ISDIR(aest->st_mode)) {
if (S_ISLNK(st.st_mode)) {
if (stat(rpath, &st) == -1) {
pkg_emit_error("Dead symlink %s", rpath);
} else {
pkg_debug(2, "Directory is a symlink, use it");
pkg_emit_progress_tick(cur_file++, nfiles);
continue;
}
}
}
}
archive_entry_set_pathname(ae, rpath);
/* load in memory the content of config files */
if (pkg_is_config_file(pkg, path, &rf, &rcf)) {
pkg_debug(1, "Populating config_file %s", pathname);
size_t len = archive_entry_size(ae);
rcf->content = malloc(len);
archive_read_data(a, rcf->content, len);
if (renamed && (!automerge || local == NULL))
strlcat(pathname, ".pkgnew", sizeof(pathname));
}
/*
* check if the file is already provided by previous package
*/
if (!automerge)
attempt_to_merge(renamed, rcf, local, pathname, path, newconf);
if (sbuf_len(newconf) == 0 && (rcf == NULL || rcf->content == NULL)) {
pkg_debug(1, "Extracting: %s", archive_entry_pathname(ae));
int install_as_user = (getenv("INSTALL_AS_USER") != NULL);
int extract_flags = EXTRACT_ARCHIVE_FLAGS;
if (install_as_user) {
/* when installing as user don't try to set file ownership */
extract_flags &= ~ARCHIVE_EXTRACT_OWNER;
}
ret = archive_read_extract(a, ae, extract_flags);
} else {
if (sbuf_len(newconf) == 0) {
sbuf_cat(newconf, rcf->content);
sbuf_finish(newconf);
}
pkg_debug(2, "Writing conf in %s", pathname);
unlink(rpath);
FILE *f = fopen(rpath, "w+");
fprintf(f, "%s", sbuf_data(newconf));
fclose(f);
}
if (ret != ARCHIVE_OK) {
/*
* show error except when the failure is during
* extracting a directory and that the directory already
* exists.
* this allow to install packages linux_base from
* package for example
*/
if (archive_entry_filetype(ae) != AE_IFDIR ||
!is_dir(pathname)) {
pkg_emit_error("archive_read_extract(): %s",
archive_error_string(a));
retcode = EPKG_FATAL;
goto cleanup;
}
}
/* Reapply modes to the directories to work around a problem on FreeBSD 9 */
if (archive_entry_filetype(ae) == AE_IFDIR)
chmod(pathname, aest->st_mode);
pkg_emit_progress_tick(cur_file++, nfiles);
/* Rename old file */
if (renamed) {
pkg_debug(1, "Renaming %s -> %s", rpath, pathname);
#ifdef HAVE_CHFLAGS
bool old = false;
if (set & NOCHANGESFLAGS)
chflags(rpath, 0);
if (lstat(pathname, &st) != -1) {
old = true;
if (st.st_flags & NOCHANGESFLAGS)
chflags(pathname, 0);
}
#endif
if (rename(rpath, pathname) == -1) {
#ifdef HAVE_CHFLAGS
/* restore flags */
if (old)
chflags(pathname, st.st_flags);
#endif
pkg_emit_error("cannot rename %s to %s: %s", rpath, pathname,
strerror(errno));
retcode = EPKG_FATAL;
goto cleanup;
}
#ifdef HAVE_CHFLAGS
/* Restore flags */
chflags(pathname, set);
#endif
}
if (string_end_with(pathname, ".pkgnew"))
pkg_emit_notice("New configuration file: %s", pathname);
renamed = false;
} while ((ret = archive_read_next_header(a, &ae)) == ARCHIVE_OK);
if (ret != ARCHIVE_EOF) {
pkg_emit_error("archive_read_next_header(): %s",
archive_error_string(a));
retcode = EPKG_FATAL;
}
cleanup:
pkg_emit_progress_tick(nfiles, nfiles);
pkg_emit_extract_finished(pkg);
if (renamed && retcode == EPKG_FATAL) {
#ifdef HAVE_CHFLAGS
if (set & NOCHANGESFLAGS)
chflags(rpath, set & ~NOCHANGESFLAGS);
#endif
unlink(rpath);
}
return (retcode);
}
static int
copy_database(sqlite3 *src, sqlite3 *dst, const char *name)
{
sqlite3_backup *b;
char *errmsg;
off_t total;
off_t done;
off_t page_size;
int ret;
assert(src != NULL);
assert(dst != NULL);
ret = sqlite3_exec(dst, "PRAGMA main.locking_mode=EXCLUSIVE;"
"BEGIN IMMEDIATE;COMMIT;", NULL, NULL, &errmsg);
if (ret != SQLITE_OK) {
pkg_emit_error("sqlite error -- %s", errmsg);
sqlite3_free(errmsg);
return (EPKG_FATAL);
}
ret = sqlite3_exec(dst, "PRAGMA page_size", ps_cb, &page_size, &errmsg);
if (ret != SQLITE_OK) {
pkg_emit_error("sqlite error -- %s", errmsg);
sqlite3_free(errmsg);
return (EPKG_FATAL);
}
b = sqlite3_backup_init(dst, "main", src, "main");
done = total = 0;
pkg_emit_progress_start(NULL);
do {
ret = sqlite3_backup_step(b, NPAGES);
total = sqlite3_backup_pagecount(b);
done = total - sqlite3_backup_remaining(b);
pkg_emit_progress_tick(done, total);
if (ret != SQLITE_OK && ret != SQLITE_DONE ) {
if (ret == SQLITE_BUSY) {
sqlite3_sleep(250);
} else {
ERROR_SQLITE(dst, "backup step");
break;
}
}
} while(done < total);
ret = sqlite3_backup_finish(b);
pkg_emit_progress_tick(total, total);
sqlite3_exec(dst, "PRAGMA main.locking_mode=NORMAL;"
"BEGIN IMMEDIATE;COMMIT;", NULL, NULL, &errmsg);
if (ret != SQLITE_OK) {
pkg_emit_error("sqlite error -- %s", errmsg);
sqlite3_free(errmsg);
return (EPKG_FATAL);
}
return ret;
}
static int
pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime)
{
FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
sqlite3 *sqlite = NULL;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
const char *origin, *digest, *offset, *length;
struct pkgdb_it *it = NULL;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
long num_offset, num_length;
time_t local_t = *mtime;
time_t digest_t;
time_t packagesite_t;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
struct pkg_manifest_key *keys = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
int hash_it = 0;
bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo;
if (access(name, R_OK) != -1)
new_repo = false;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) {
return (EPKG_FATAL);
}
if (!reuse_repo) {
pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name);
local_t = 0;
*mtime = 0;
}
if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) {
goto cleanup;
}
if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK)
goto cleanup;
it = pkgdb_repo_origins(sqlite);
if (it == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)
pkg_emit_notice("repository %s has no meta file, using "
"default settings", repo->name);
fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->digests, &local_t, &rc);
if (fdigests == NULL) {
if (rc == EPKG_FATAL)
/* Destroy repo completely */
if (new_repo)
unlink(name);
goto cleanup;
}
digest_t = local_t;
local_t = *mtime;
fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->manifests, &local_t, &rc);
if (fmanifest == NULL) {
if (rc == EPKG_FATAL)
/* Destroy repo completely */
if (new_repo)
unlink(name);
goto cleanup;
}
packagesite_t = digest_t;
*mtime = packagesite_t > digest_t ? packagesite_t : digest_t;
/*fconflicts = repo_fetch_remote_extract_tmp(repo,
repo_conflicts_archive, "txz", &local_t,
&rc, repo_conflicts_file);*/
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
/* Detect whether we have legacy repo */
if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
if (digest == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
if (!pkg_checksum_is_valid(digest, strlen(digest))) {
legacy_repo = true;
pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
}
}
fseek(fdigests, 0, SEEK_SET);
/* Load local repo data */
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
&digest);
pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
}
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
/* files offset */
strsep(&p, ":");
length = strsep(&p, ":");
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
if (length != NULL) {
errno = 0;
num_length = (long)strtoul(length, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
}
else {
num_length = 0;
}
processed++;
HASH_FIND_STR(ldel, origin, item);
if (item == NULL) {
added++;
pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset,
num_length);
} else {
if (strcmp(digest, item->digest) == 0) {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
} else {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
pkg_repo_update_increment_item_new(&ladd, origin, digest,
num_offset, num_length);
updated++;
}
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);");
in_trans = true;
rc = pkgdb_transaction_begin(sqlite, "REPO");
if (rc != EPKG_OK)
goto cleanup;
removed = HASH_COUNT(ldel);
hash_it = 0;
pkg_emit_progress_start("Removing expired entries");
HASH_ITER(hh, ldel, item, tmp_item) {
pkg_emit_progress_tick(++hash_it, removed);
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
}
