static int
pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime)
{
FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
sqlite3 *sqlite = NULL;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
const char *origin, *digest, *offset, *length;
struct pkgdb_it *it = NULL;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0;
long num_offset, num_length;
time_t local_t = *mtime;
time_t digest_t;
time_t packagesite_t;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
struct pkg_manifest_key *keys = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
int hash_it = 0;
time_t now, last;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
if ((rc = pkgdb_repo_open(name, false, &sqlite)) != EPKG_OK) {
return (EPKG_FATAL);
}
if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) {
goto cleanup;
}
if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK)
goto cleanup;
it = pkgdb_repo_origins(sqlite);
if (it == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest);
pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
}
if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)
pkg_emit_notice("repository %s has no meta file, use default settings",
repo->name);
fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->digests, &local_t, &rc);
if (fdigests == NULL)
goto cleanup;
digest_t = local_t;
local_t = *mtime;
fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->manifests, &local_t, &rc);
if (fmanifest == NULL)
goto cleanup;
packagesite_t = digest_t;
*mtime = packagesite_t > digest_t ? packagesite_t : digest_t;
/*fconflicts = repo_fetch_remote_extract_tmp(repo,
repo_conflicts_archive, "txz", &local_t,
&rc, repo_conflicts_file);*/
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
/* files offset */
strsep(&p, ":");
length = strsep(&p, ":");
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
assert(0);
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
if (length != NULL) {
errno = 0;
num_length = (long)strtoul(length, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
}
else {
num_length = 0;
}
processed++;
HASH_FIND_STR(ldel, origin, item);
if (item == NULL) {
added++;
pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset,
num_length);
} else {
if (strcmp(digest, item->digest) == 0) {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
} else {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
pkg_repo_update_increment_item_new(&ladd, origin, digest,
num_offset, num_length);
updated++;
}
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
removed = HASH_COUNT(ldel);
hash_it = 0;
last = 0;
HASH_ITER(hh, ldel, item, tmp_item) {
now = time(NULL);
if (++hash_it == removed || now > last) {
pkg_emit_update_remove(removed, hash_it);
last = now;
}
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
}
static int
pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime)
{
FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
sqlite3 *sqlite = NULL;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
const char *origin, *digest, *offset, *length;
struct pkgdb_it *it = NULL;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
long num_offset, num_length;
time_t local_t = *mtime;
time_t digest_t;
time_t packagesite_t;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
struct pkg_manifest_key *keys = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
int hash_it = 0;
bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo;
if (access(name, R_OK) != -1)
new_repo = false;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) {
return (EPKG_FATAL);
}
if (!reuse_repo) {
pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name);
local_t = 0;
*mtime = 0;
}
if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) {
goto cleanup;
}
if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK)
goto cleanup;
it = pkgdb_repo_origins(sqlite);
if (it == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)
pkg_emit_notice("repository %s has no meta file, using "
"default settings", repo->name);
fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->digests, &local_t, &rc);
if (fdigests == NULL) {
if (rc == EPKG_FATAL)
/* Destroy repo completely */
if (new_repo)
unlink(name);
goto cleanup;
}
digest_t = local_t;
local_t = *mtime;
fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
repo->meta->manifests, &local_t, &rc);
if (fmanifest == NULL) {
if (rc == EPKG_FATAL)
/* Destroy repo completely */
if (new_repo)
unlink(name);
goto cleanup;
}
packagesite_t = digest_t;
*mtime = packagesite_t > digest_t ? packagesite_t : digest_t;
/*fconflicts = repo_fetch_remote_extract_tmp(repo,
repo_conflicts_archive, "txz", &local_t,
&rc, repo_conflicts_file);*/
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
/* Detect whether we have legacy repo */
if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
if (digest == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
if (!pkg_checksum_is_valid(digest, strlen(digest))) {
legacy_repo = true;
pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
}
}
fseek(fdigests, 0, SEEK_SET);
/* Load local repo data */
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
&digest);
pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
}
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
/* files offset */
strsep(&p, ":");
length = strsep(&p, ":");
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
if (length != NULL) {
errno = 0;
num_length = (long)strtoul(length, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
}
else {
num_length = 0;
}
processed++;
HASH_FIND_STR(ldel, origin, item);
if (item == NULL) {
added++;
pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset,
num_length);
} else {
if (strcmp(digest, item->digest) == 0) {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
} else {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
pkg_repo_update_increment_item_new(&ladd, origin, digest,
num_offset, num_length);
updated++;
}
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);");
in_trans = true;
rc = pkgdb_transaction_begin(sqlite, "REPO");
if (rc != EPKG_OK)
goto cleanup;
removed = HASH_COUNT(ldel);
hash_it = 0;
pkg_emit_progress_start("Removing expired entries");
HASH_ITER(hh, ldel, item, tmp_item) {
pkg_emit_progress_tick(++hash_it, removed);
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
}
static int
pkg_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime)
{
FILE *fmanifest = NULL, *fdigests = NULL;
sqlite3 *sqlite = NULL;
struct pkg *pkg = NULL;
int rc = EPKG_FATAL;
const char *origin, *digest, *offset;
struct pkgdb_it *it = NULL;
char *linebuf = NULL, *p;
int updated = 0, removed = 0, added = 0, processed = 0;
long num_offset;
time_t local_t = *mtime;
struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
*item, *tmp_item;
const char *myarch;
struct pkg_manifest_parser *parser = NULL;
size_t linecap = 0;
ssize_t linelen;
char *map = MAP_FAILED;
size_t len = 0;
pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
if ((rc = pkgdb_repo_open(name, false, &sqlite, false)) != EPKG_OK) {
return (EPKG_FATAL);
}
if ((rc = pkgdb_repo_init(sqlite, false)) != EPKG_OK)
goto cleanup;
if ((rc = pkg_register_repo(repo, sqlite)) != EPKG_OK)
goto cleanup;
it = pkgdb_repo_origins(sqlite);
if (it == NULL) {
rc = EPKG_FATAL;
goto cleanup;
}
while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest);
pkg_update_increment_item_new(&ldel, origin, digest, 4);
}
fdigests = repo_fetch_remote_extract_tmp(repo,
repo_digests_archive, "txz", &local_t,
&rc, repo_digests_file);
if (fdigests == NULL)
goto cleanup;
local_t = *mtime;
fmanifest = repo_fetch_remote_extract_tmp(repo,
repo_packagesite_archive, "txz", &local_t,
&rc, repo_packagesite_file);
if (fmanifest == NULL)
goto cleanup;
*mtime = local_t;
fseek(fmanifest, 0, SEEK_END);
len = ftell(fmanifest);
pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
/* load the while digests */
while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
p = linebuf;
origin = strsep(&p, ":");
digest = strsep(&p, ":");
offset = strsep(&p, ":");
if (origin == NULL || digest == NULL ||
offset == NULL) {
pkg_emit_error("invalid digest file format");
assert(0);
rc = EPKG_FATAL;
goto cleanup;
}
errno = 0;
num_offset = (long)strtoul(offset, NULL, 10);
if (errno != 0) {
pkg_emit_errno("strtoul", "digest format error");
rc = EPKG_FATAL;
goto cleanup;
}
processed++;
HASH_FIND_STR(ldel, __DECONST(char *, origin), item);
if (item == NULL) {
added++;
pkg_update_increment_item_new(&ladd, origin, digest, num_offset);
} else {
if (strcmp(digest, item->digest) == 0) {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
} else {
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
item = NULL;
pkg_update_increment_item_new(&ladd, origin, digest, num_offset);
updated++;
}
}
}
rc = EPKG_OK;
pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
removed = HASH_COUNT(ldel) - updated;
HASH_ITER(hh, ldel, item, tmp_item) {
if (rc == EPKG_OK) {
rc = pkgdb_repo_remove_package(item->origin);
}
free(item->origin);
free(item->digest);
HASH_DEL(ldel, item);
free(item);
}
pkg_config_string(PKG_CONFIG_ABI, &myarch);
pkg_debug(1, "Pkgrepo, pushing new entries for '%s'", name);
pkg = NULL;
if (len > 0 && len < SSIZE_MAX) {
map = mmap(NULL, len, PROT_READ, MAP_SHARED, fileno(fmanifest), 0);
fclose(fmanifest);
}
HASH_ITER(hh, ladd, item, tmp_item) {
if (rc == EPKG_OK) {
if (map != MAP_FAILED) {
rc = pkg_add_from_manifest(NULL, map + item->offset, item->origin,
len - item->offset, item->digest, myarch, sqlite, &parser, &pkg);
} else {
rc = pkg_add_from_manifest(fmanifest, NULL, item->origin,
item->offset, item->digest, myarch, sqlite, &parser, &pkg);
}
}
free(item->origin);
free(item->digest);
HASH_DEL(ladd, item);
free(item);
}
pkg_manifest_parser_free(parser);
pkg_emit_incremental_update(updated, removed, added, processed);
cleanup:
if (pkg != NULL)
pkg_free(pkg);
if (it != NULL)
pkgdb_it_free(it);
if (pkgdb_repo_close(sqlite, rc == EPKG_OK) != EPKG_OK)
rc = EPKG_FATAL;
if (map == MAP_FAILED && fmanifest)
fclose(fmanifest);
if (fdigests)
fclose(fdigests);
if (map != MAP_FAILED)
munmap(map, len);
return (rc);
}
int
pkg_create_repo(char *path, bool force, bool filelist,
void (progress)(struct pkg *pkg, void *data), void *data)
{
FTS *fts = NULL;
struct thd_data thd_data;
int num_workers;
size_t len;
pthread_t *tids = NULL;
struct digest_list_entry *dlist = NULL, *cur_dig, *dtmp;
sqlite3 *sqlite = NULL;
char *errmsg = NULL;
int retcode = EPKG_OK;
char *repopath[2];
char repodb[MAXPATHLEN + 1];
char repopack[MAXPATHLEN + 1];
char *manifest_digest;
FILE *psyml, *fsyml, *mandigests;
psyml = fsyml = mandigests = NULL;
if (!is_dir(path)) {
pkg_emit_error("%s is not a directory", path);
return (EPKG_FATAL);
}
repopath[0] = path;
repopath[1] = NULL;
len = sizeof(num_workers);
if (sysctlbyname("hw.ncpu", &num_workers, &len, NULL, 0) == -1)
num_workers = 6;
if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, NULL)) == NULL) {
pkg_emit_errno("fts_open", path);
retcode = EPKG_FATAL;
goto cleanup;
}
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_packagesite_file);
if ((psyml = fopen(repodb, "w")) == NULL) {
retcode = EPKG_FATAL;
goto cleanup;
}
if (filelist) {
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_filesite_file);
if ((fsyml = fopen(repodb, "w")) == NULL) {
retcode = EPKG_FATAL;
goto cleanup;
}
}
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_digests_file);
if ((mandigests = fopen(repodb, "w")) == NULL) {
retcode = EPKG_FATAL;
goto cleanup;
}
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_db_file);
snprintf(repopack, sizeof(repopack), "%s/repo.txz", path);
pack_extract(repopack, repo_db_file, repodb);
if ((retcode = pkgdb_repo_open(repodb, force, &sqlite, true)) != EPKG_OK)
goto cleanup;
if ((retcode = pkgdb_repo_init(sqlite, true)) != EPKG_OK)
goto cleanup;
thd_data.root_path = path;
thd_data.max_results = num_workers;
thd_data.num_results = 0;
thd_data.stop = false;
thd_data.fts = fts;
thd_data.read_files = filelist;
pthread_mutex_init(&thd_data.fts_m, NULL);
thd_data.results = NULL;
thd_data.thd_finished = 0;
pthread_mutex_init(&thd_data.results_m, NULL);
pthread_cond_init(&thd_data.has_result, NULL);
pthread_cond_init(&thd_data.has_room, NULL);
/* Launch workers */
tids = calloc(num_workers, sizeof(pthread_t));
for (int i = 0; i < num_workers; i++) {
pthread_create(&tids[i], NULL, (void *)&read_pkg_file, &thd_data);
}
for (;;) {
struct pkg_result *r;
const char *origin;
long manifest_pos, files_pos;
pthread_mutex_lock(&thd_data.results_m);
while ((r = thd_data.results) == NULL) {
if (thd_data.thd_finished == num_workers) {
break;
}
pthread_cond_wait(&thd_data.has_result, &thd_data.results_m);
}
if (r != NULL) {
LL_DELETE(thd_data.results, thd_data.results);
thd_data.num_results--;
pthread_cond_signal(&thd_data.has_room);
}
pthread_mutex_unlock(&thd_data.results_m);
if (r == NULL) {
break;
}
if (r->retcode != EPKG_OK) {
continue;
}
/* do not add if package if already in repodb
(possibly at a different pkg_path) */
retcode = pkgdb_repo_cksum_exists(sqlite, r->cksum);
if (retcode == EPKG_FATAL) {
goto cleanup;
}
else if (retcode == EPKG_OK) {
continue;
}
if (progress != NULL)
progress(r->pkg, data);
manifest_pos = ftell(psyml);
pkg_emit_manifest_file(r->pkg, psyml, PKG_MANIFEST_EMIT_COMPACT, &manifest_digest);
if (filelist) {
files_pos = ftell(fsyml);
pkg_emit_filelist(r->pkg, fsyml);
} else {
files_pos = 0;
}
pkg_get(r->pkg, PKG_ORIGIN, &origin);
cur_dig = malloc(sizeof (struct digest_list_entry));
cur_dig->origin = strdup(origin);
cur_dig->digest = manifest_digest;
cur_dig->manifest_pos = manifest_pos;
cur_dig->files_pos = files_pos;
LL_PREPEND(dlist, cur_dig);
retcode = pkgdb_repo_add_package(r->pkg, r->path, sqlite,
manifest_digest, false, true);
if (retcode == EPKG_END) {
continue;
}
else if (retcode != EPKG_OK) {
goto cleanup;
}
pkg_free(r->pkg);
free(r);
}
/* Now sort all digests */
LL_SORT(dlist, digest_sort_compare_func);
cleanup:
if (pkgdb_repo_close(sqlite, retcode == EPKG_OK) != EPKG_OK) {
retcode = EPKG_FATAL;
}
LL_FOREACH_SAFE(dlist, cur_dig, dtmp) {
if (retcode == EPKG_OK) {
fprintf(mandigests, "%s:%s:%ld:%ld\n", cur_dig->origin,
cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos);
}
free(cur_dig->digest);
free(cur_dig->origin);
free(cur_dig);
}
if (tids != NULL) {
// Cancel running threads
if (retcode != EPKG_OK) {
pthread_mutex_lock(&thd_data.fts_m);
thd_data.stop = true;
pthread_mutex_unlock(&thd_data.fts_m);
}
// Join on threads to release thread IDs
for (int i = 0; i < num_workers; i++) {
pthread_join(tids[i], NULL);
}
free(tids);
}
if (fts != NULL)
fts_close(fts);
if (fsyml != NULL)
fclose(fsyml);
if (psyml != NULL)
fclose(psyml);
if (mandigests != NULL)
fclose(mandigests);
if (sqlite != NULL)
sqlite3_close(sqlite);
if (errmsg != NULL)
sqlite3_free(errmsg);
sqlite3_shutdown();
return (retcode);
}
