static int pkg_repo_add_from_manifest(char *buf, const char *origin, const char *digest, long offset, sqlite3 *sqlite, struct pkg_manifest_key **keys, struct pkg **p, bool is_legacy, struct pkg_repo *repo) { int rc = EPKG_OK; struct pkg *pkg; const char *local_origin, *pkg_arch; if (*p == NULL) { rc = pkg_new(p, PKG_REMOTE); if (rc != EPKG_OK) return (EPKG_FATAL); } else { pkg_reset(*p, PKG_REMOTE); } pkg = *p; pkg_manifest_keys_new(keys); rc = pkg_parse_manifest(pkg, buf, offset, *keys); if (rc != EPKG_OK) { goto cleanup; } rc = pkg_is_valid(pkg); if (rc != EPKG_OK) { goto cleanup; } /* Ensure that we have a proper origin and arch*/ pkg_get(pkg, PKG_ORIGIN, &local_origin, PKG_ARCH, &pkg_arch); if (local_origin == NULL || strcmp(local_origin, origin) != 0) { pkg_emit_error("manifest contains origin %s while we wanted to add origin %s", local_origin ? local_origin : "NULL", origin); rc = EPKG_FATAL; goto cleanup; } if (pkg_arch == NULL || !is_valid_abi(pkg_arch, true)) { rc = EPKG_FATAL; goto cleanup; } pkg_set(pkg, PKG_REPONAME, repo->name); if (is_legacy) { pkg_set(pkg, PKG_OLD_DIGEST, digest); pkg_checksum_calculate(pkg, NULL); } else { pkg_set(pkg, PKG_DIGEST, digest); } rc = pkgdb_repo_add_package(pkg, NULL, sqlite, true); cleanup: return (rc); }
static int pkg_add_from_manifest(FILE *f, char *buf, const char *origin, long offset, const char *manifest_digest, const char *local_arch, sqlite3 *sqlite, struct pkg_manifest_parser **parser, struct pkg **p) { int rc = EPKG_OK; struct pkg *pkg; const char *local_origin, *pkg_arch; if (buf == NULL && fseek(f, offset, SEEK_SET) == -1) { pkg_emit_errno("fseek", "invalid manifest offset"); return (EPKG_FATAL); } if (*p == NULL) { rc = pkg_new(p, PKG_REMOTE); if (rc != EPKG_OK) return (EPKG_FATAL); } else { pkg_reset(*p, PKG_REMOTE); } pkg = *p; pkg_manifest_parser_new(parser); if (buf == NULL) { rc = pkg_parse_manifest_file_ev(pkg, f, *parser); } else { rc = pkg_parse_manifest_ev(pkg, buf, offset, *parser); } if (rc != EPKG_OK) { goto cleanup; } rc = pkg_is_valid(pkg); if (rc != EPKG_OK) { goto cleanup; } /* Ensure that we have a proper origin and arch*/ pkg_get(pkg, PKG_ORIGIN, &local_origin, PKG_ARCH, &pkg_arch); if (local_origin == NULL || strcmp(local_origin, origin) != 0) { pkg_emit_error("manifest contains origin %s while we wanted to add origin %s", local_origin ? local_origin : "NULL", origin); rc = EPKG_FATAL; goto cleanup; } if (pkg_arch == NULL || strcmp(pkg_arch, local_arch) != 0) { pkg_emit_error("package %s is built for %s arch, and local arch is %s", origin, pkg_arch ? pkg_arch : "NULL", local_arch); rc = EPKG_FATAL; goto cleanup; } rc = pkgdb_repo_add_package(pkg, NULL, sqlite, manifest_digest, true, false); cleanup: return (rc); }
int pkg_create_repo(char *path, bool force, bool filelist, void (progress)(struct pkg *pkg, void *data), void *data) { FTS *fts = NULL; struct thd_data thd_data; int num_workers; size_t len; pthread_t *tids = NULL; struct digest_list_entry *dlist = NULL, *cur_dig, *dtmp; sqlite3 *sqlite = NULL; char *errmsg = NULL; int retcode = EPKG_OK; char *repopath[2]; char repodb[MAXPATHLEN + 1]; char repopack[MAXPATHLEN + 1]; char *manifest_digest; FILE *psyml, *fsyml, *mandigests; psyml = fsyml = mandigests = NULL; if (!is_dir(path)) { pkg_emit_error("%s is not a directory", path); return (EPKG_FATAL); } repopath[0] = path; repopath[1] = NULL; len = sizeof(num_workers); if (sysctlbyname("hw.ncpu", &num_workers, &len, NULL, 0) == -1) num_workers = 6; if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, NULL)) == NULL) { pkg_emit_errno("fts_open", path); retcode = EPKG_FATAL; goto cleanup; } snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_packagesite_file); if ((psyml = fopen(repodb, "w")) == NULL) { retcode = EPKG_FATAL; goto cleanup; } if (filelist) { snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_filesite_file); if ((fsyml = fopen(repodb, "w")) == NULL) { retcode = EPKG_FATAL; goto cleanup; } } snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_digests_file); if ((mandigests = fopen(repodb, "w")) == NULL) { retcode = EPKG_FATAL; goto cleanup; } snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_db_file); snprintf(repopack, sizeof(repopack), "%s/repo.txz", path); pack_extract(repopack, repo_db_file, repodb); if ((retcode = pkgdb_repo_open(repodb, force, &sqlite, true)) != EPKG_OK) goto cleanup; if ((retcode = pkgdb_repo_init(sqlite, true)) != EPKG_OK) goto cleanup; thd_data.root_path = path; thd_data.max_results = num_workers; thd_data.num_results = 0; thd_data.stop = false; thd_data.fts = fts; thd_data.read_files = filelist; pthread_mutex_init(&thd_data.fts_m, NULL); thd_data.results = NULL; thd_data.thd_finished = 0; pthread_mutex_init(&thd_data.results_m, NULL); pthread_cond_init(&thd_data.has_result, NULL); pthread_cond_init(&thd_data.has_room, NULL); /* Launch workers */ tids = calloc(num_workers, sizeof(pthread_t)); for (int i = 0; i < num_workers; i++) { pthread_create(&tids[i], NULL, (void *)&read_pkg_file, &thd_data); } for (;;) { struct pkg_result *r; const char *origin; long manifest_pos, files_pos; pthread_mutex_lock(&thd_data.results_m); while ((r = thd_data.results) == NULL) { if (thd_data.thd_finished == num_workers) { break; } pthread_cond_wait(&thd_data.has_result, &thd_data.results_m); } if (r != NULL) { LL_DELETE(thd_data.results, thd_data.results); thd_data.num_results--; pthread_cond_signal(&thd_data.has_room); } pthread_mutex_unlock(&thd_data.results_m); if (r == NULL) { break; } if (r->retcode != EPKG_OK) { continue; } /* do not add if package if already in repodb (possibly at a different pkg_path) */ retcode = pkgdb_repo_cksum_exists(sqlite, r->cksum); if (retcode == EPKG_FATAL) { goto cleanup; } else if (retcode == EPKG_OK) { continue; } if (progress != NULL) progress(r->pkg, data); manifest_pos = ftell(psyml); pkg_emit_manifest_file(r->pkg, psyml, PKG_MANIFEST_EMIT_COMPACT, &manifest_digest); if (filelist) { files_pos = ftell(fsyml); pkg_emit_filelist(r->pkg, fsyml); } else { files_pos = 0; } pkg_get(r->pkg, PKG_ORIGIN, &origin); cur_dig = malloc(sizeof (struct digest_list_entry)); cur_dig->origin = strdup(origin); cur_dig->digest = manifest_digest; cur_dig->manifest_pos = manifest_pos; cur_dig->files_pos = files_pos; LL_PREPEND(dlist, cur_dig); retcode = pkgdb_repo_add_package(r->pkg, r->path, sqlite, manifest_digest, false, true); if (retcode == EPKG_END) { continue; } else if (retcode != EPKG_OK) { goto cleanup; } pkg_free(r->pkg); free(r); } /* Now sort all digests */ LL_SORT(dlist, digest_sort_compare_func); cleanup: if (pkgdb_repo_close(sqlite, retcode == EPKG_OK) != EPKG_OK) { retcode = EPKG_FATAL; } LL_FOREACH_SAFE(dlist, cur_dig, dtmp) { if (retcode == EPKG_OK) { fprintf(mandigests, "%s:%s:%ld:%ld\n", cur_dig->origin, cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos); } free(cur_dig->digest); free(cur_dig->origin); free(cur_dig); } if (tids != NULL) { // Cancel running threads if (retcode != EPKG_OK) { pthread_mutex_lock(&thd_data.fts_m); thd_data.stop = true; pthread_mutex_unlock(&thd_data.fts_m); } // Join on threads to release thread IDs for (int i = 0; i < num_workers; i++) { pthread_join(tids[i], NULL); } free(tids); } if (fts != NULL) fts_close(fts); if (fsyml != NULL) fclose(fsyml); if (psyml != NULL) fclose(psyml); if (mandigests != NULL) fclose(mandigests); if (sqlite != NULL) sqlite3_close(sqlite); if (errmsg != NULL) sqlite3_free(errmsg); sqlite3_shutdown(); return (retcode); }