static int pkg_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime) { FILE *fmanifest = NULL, *fdigests = NULL; sqlite3 *sqlite = NULL; struct pkg *pkg = NULL; int rc = EPKG_FATAL; const char *origin, *digest, *offset; struct pkgdb_it *it = NULL; char *linebuf = NULL, *p; int updated = 0, removed = 0, added = 0, processed = 0; long num_offset; time_t local_t = *mtime; struct pkg_increment_task_item *ldel = NULL, *ladd = NULL, *item, *tmp_item; const char *myarch; struct pkg_manifest_parser *parser = NULL; size_t linecap = 0; ssize_t linelen; char *map = MAP_FAILED; size_t len = 0; pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name); if ((rc = pkgdb_repo_open(name, false, &sqlite, false)) != EPKG_OK) { return (EPKG_FATAL); } if ((rc = pkgdb_repo_init(sqlite, false)) != EPKG_OK) goto cleanup; if ((rc = pkg_register_repo(repo, sqlite)) != EPKG_OK) goto cleanup; it = pkgdb_repo_origins(sqlite); if (it == NULL) { rc = EPKG_FATAL; goto cleanup; } while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) { pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest); pkg_update_increment_item_new(&ldel, origin, digest, 4); } fdigests = repo_fetch_remote_extract_tmp(repo, repo_digests_archive, "txz", &local_t, &rc, repo_digests_file); if (fdigests == NULL) goto cleanup; local_t = *mtime; fmanifest = repo_fetch_remote_extract_tmp(repo, repo_packagesite_archive, "txz", &local_t, &rc, repo_packagesite_file); if (fmanifest == NULL) goto cleanup; *mtime = local_t; fseek(fmanifest, 0, SEEK_END); len = ftell(fmanifest); pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name); /* load the while digests */ while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) { p = linebuf; origin = strsep(&p, ":"); digest = strsep(&p, ":"); offset = strsep(&p, ":"); if (origin == NULL || digest == NULL || offset == NULL) { pkg_emit_error("invalid digest file format"); assert(0); rc = EPKG_FATAL; goto cleanup; } errno = 0; num_offset = (long)strtoul(offset, NULL, 10); if (errno != 0) { pkg_emit_errno("strtoul", "digest format error"); rc = EPKG_FATAL; goto cleanup; } processed++; HASH_FIND_STR(ldel, __DECONST(char *, origin), item); if (item == NULL) { added++; pkg_update_increment_item_new(&ladd, origin, digest, num_offset); } else { if (strcmp(digest, item->digest) == 0) { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; } else { free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); item = NULL; pkg_update_increment_item_new(&ladd, origin, digest, num_offset); updated++; } } } rc = EPKG_OK; pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name); removed = HASH_COUNT(ldel) - updated; HASH_ITER(hh, ldel, item, tmp_item) { if (rc == EPKG_OK) { rc = pkgdb_repo_remove_package(item->origin); } free(item->origin); free(item->digest); HASH_DEL(ldel, item); free(item); } pkg_config_string(PKG_CONFIG_ABI, &myarch); pkg_debug(1, "Pkgrepo, pushing new entries for '%s'", name); pkg = NULL; if (len > 0 && len < SSIZE_MAX) { map = mmap(NULL, len, PROT_READ, MAP_SHARED, fileno(fmanifest), 0); fclose(fmanifest); } HASH_ITER(hh, ladd, item, tmp_item) { if (rc == EPKG_OK) { if (map != MAP_FAILED) { rc = pkg_add_from_manifest(NULL, map + item->offset, item->origin, len - item->offset, item->digest, myarch, sqlite, &parser, &pkg); } else { rc = pkg_add_from_manifest(fmanifest, NULL, item->origin, item->offset, item->digest, myarch, sqlite, &parser, &pkg); } } free(item->origin); free(item->digest); HASH_DEL(ladd, item); free(item); } pkg_manifest_parser_free(parser); pkg_emit_incremental_update(updated, removed, added, processed); cleanup: if (pkg != NULL) pkg_free(pkg); if (it != NULL) pkgdb_it_free(it); if (pkgdb_repo_close(sqlite, rc == EPKG_OK) != EPKG_OK) rc = EPKG_FATAL; if (map == MAP_FAILED && fmanifest) fclose(fmanifest); if (fdigests) fclose(fdigests); if (map != MAP_FAILED) munmap(map, len); return (rc); }
int pkg_create_repo(char *path, bool force, bool filelist, void (progress)(struct pkg *pkg, void *data), void *data) { FTS *fts = NULL; struct thd_data thd_data; int num_workers; size_t len; pthread_t *tids = NULL; struct digest_list_entry *dlist = NULL, *cur_dig, *dtmp; sqlite3 *sqlite = NULL; char *errmsg = NULL; int retcode = EPKG_OK; char *repopath[2]; char repodb[MAXPATHLEN + 1]; char repopack[MAXPATHLEN + 1]; char *manifest_digest; FILE *psyml, *fsyml, *mandigests; psyml = fsyml = mandigests = NULL; if (!is_dir(path)) { pkg_emit_error("%s is not a directory", path); return (EPKG_FATAL); } repopath[0] = path; repopath[1] = NULL; len = sizeof(num_workers); if (sysctlbyname("hw.ncpu", &num_workers, &len, NULL, 0) == -1) num_workers = 6; if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, NULL)) == NULL) { pkg_emit_errno("fts_open", path); retcode = EPKG_FATAL; goto cleanup; } snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_packagesite_file); if ((psyml = fopen(repodb, "w")) == NULL) { retcode = EPKG_FATAL; goto cleanup; } if (filelist) { snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_filesite_file); if ((fsyml = fopen(repodb, "w")) == NULL) { retcode = EPKG_FATAL; goto cleanup; } } snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_digests_file); if ((mandigests = fopen(repodb, "w")) == NULL) { retcode = EPKG_FATAL; goto cleanup; } snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_db_file); snprintf(repopack, sizeof(repopack), "%s/repo.txz", path); pack_extract(repopack, repo_db_file, repodb); if ((retcode = pkgdb_repo_open(repodb, force, &sqlite, true)) != EPKG_OK) goto cleanup; if ((retcode = pkgdb_repo_init(sqlite, true)) != EPKG_OK) goto cleanup; thd_data.root_path = path; thd_data.max_results = num_workers; thd_data.num_results = 0; thd_data.stop = false; thd_data.fts = fts; thd_data.read_files = filelist; pthread_mutex_init(&thd_data.fts_m, NULL); thd_data.results = NULL; thd_data.thd_finished = 0; pthread_mutex_init(&thd_data.results_m, NULL); pthread_cond_init(&thd_data.has_result, NULL); pthread_cond_init(&thd_data.has_room, NULL); /* Launch workers */ tids = calloc(num_workers, sizeof(pthread_t)); for (int i = 0; i < num_workers; i++) { pthread_create(&tids[i], NULL, (void *)&read_pkg_file, &thd_data); } for (;;) { struct pkg_result *r; const char *origin; long manifest_pos, files_pos; pthread_mutex_lock(&thd_data.results_m); while ((r = thd_data.results) == NULL) { if (thd_data.thd_finished == num_workers) { break; } pthread_cond_wait(&thd_data.has_result, &thd_data.results_m); } if (r != NULL) { LL_DELETE(thd_data.results, thd_data.results); thd_data.num_results--; pthread_cond_signal(&thd_data.has_room); } pthread_mutex_unlock(&thd_data.results_m); if (r == NULL) { break; } if (r->retcode != EPKG_OK) { continue; } /* do not add if package if already in repodb (possibly at a different pkg_path) */ retcode = pkgdb_repo_cksum_exists(sqlite, r->cksum); if (retcode == EPKG_FATAL) { goto cleanup; } else if (retcode == EPKG_OK) { continue; } if (progress != NULL) progress(r->pkg, data); manifest_pos = ftell(psyml); pkg_emit_manifest_file(r->pkg, psyml, PKG_MANIFEST_EMIT_COMPACT, &manifest_digest); if (filelist) { files_pos = ftell(fsyml); pkg_emit_filelist(r->pkg, fsyml); } else { files_pos = 0; } pkg_get(r->pkg, PKG_ORIGIN, &origin); cur_dig = malloc(sizeof (struct digest_list_entry)); cur_dig->origin = strdup(origin); cur_dig->digest = manifest_digest; cur_dig->manifest_pos = manifest_pos; cur_dig->files_pos = files_pos; LL_PREPEND(dlist, cur_dig); retcode = pkgdb_repo_add_package(r->pkg, r->path, sqlite, manifest_digest, false, true); if (retcode == EPKG_END) { continue; } else if (retcode != EPKG_OK) { goto cleanup; } pkg_free(r->pkg); free(r); } /* Now sort all digests */ LL_SORT(dlist, digest_sort_compare_func); cleanup: if (pkgdb_repo_close(sqlite, retcode == EPKG_OK) != EPKG_OK) { retcode = EPKG_FATAL; } LL_FOREACH_SAFE(dlist, cur_dig, dtmp) { if (retcode == EPKG_OK) { fprintf(mandigests, "%s:%s:%ld:%ld\n", cur_dig->origin, cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos); } free(cur_dig->digest); free(cur_dig->origin); free(cur_dig); } if (tids != NULL) { // Cancel running threads if (retcode != EPKG_OK) { pthread_mutex_lock(&thd_data.fts_m); thd_data.stop = true; pthread_mutex_unlock(&thd_data.fts_m); } // Join on threads to release thread IDs for (int i = 0; i < num_workers; i++) { pthread_join(tids[i], NULL); } free(tids); } if (fts != NULL) fts_close(fts); if (fsyml != NULL) fclose(fsyml); if (psyml != NULL) fclose(psyml); if (mandigests != NULL) fclose(mandigests); if (sqlite != NULL) sqlite3_close(sqlite); if (errmsg != NULL) sqlite3_free(errmsg); sqlite3_shutdown(); return (retcode); }