int pkgdb_repo_init(sqlite3 *sqlite) { int retcode = EPKG_OK; retcode = sql_exec(sqlite, "PRAGMA synchronous=off"); if (retcode != EPKG_OK) return (retcode); retcode = sql_exec(sqlite, "PRAGMA journal_mode=memory"); if (retcode != EPKG_OK) return (retcode); retcode = sql_exec(sqlite, "PRAGMA foreign_keys=on"); if (retcode != EPKG_OK) return (retcode); retcode = initialize_prepared_statements(sqlite); if (retcode != EPKG_OK) return (retcode); retcode = pkgdb_transaction_begin(sqlite, NULL); if (retcode != EPKG_OK) return (retcode); return (EPKG_OK); }
int pkgdb_repo_init(sqlite3 *sqlite) { int retcode = EPKG_OK; retcode = sql_exec(sqlite, "PRAGMA synchronous=default"); if (retcode != EPKG_OK) return (retcode); retcode = sql_exec(sqlite, "PRAGMA foreign_keys=on"); if (retcode != EPKG_OK) return (retcode); retcode = initialize_prepared_statements(sqlite); if (retcode != EPKG_OK) return (retcode); return (EPKG_OK); }
int pkg_create_repo(char *path, bool force, void (progress)(struct pkg *pkg, void *data), void *data) { FTS *fts = NULL; struct thd_data thd_data; int num_workers; size_t len; pthread_t *tids = NULL; struct pkg_dep *dep = NULL; struct pkg_category *category = NULL; struct pkg_license *license = NULL; struct pkg_option *option = NULL; struct pkg_shlib *shlib = NULL; sqlite3 *sqlite = NULL; int64_t package_id; char *errmsg = NULL; int retcode = EPKG_OK; int ret; char *repopath[2]; char repodb[MAXPATHLEN + 1]; char repopack[MAXPATHLEN + 1]; struct archive *a = NULL; struct archive_entry *ae = NULL; if (!is_dir(path)) { pkg_emit_error("%s is not a directory", path); return (EPKG_FATAL); } repopath[0] = path; repopath[1] = NULL; len = sizeof(num_workers); if (sysctlbyname("hw.ncpu", &num_workers, &len, NULL, 0) == -1) num_workers = 6; if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, NULL)) == NULL) { pkg_emit_errno("fts_open", path); retcode = EPKG_FATAL; goto cleanup; } snprintf(repodb, sizeof(repodb), "%s/repo.sqlite", path); snprintf(repopack, sizeof(repopack), "%s/repo.txz", path); if (access(repopack, F_OK) == 0) { a = archive_read_new(); archive_read_support_compression_all(a); archive_read_support_format_tar(a); ret = archive_read_open_filename(a, repopack, 4096); if (ret != ARCHIVE_OK) { /* if we can't unpack it it won't be useful for us */ unlink(repopack); } else { while (archive_read_next_header(a, &ae) == ARCHIVE_OK) { if (!strcmp(archive_entry_pathname(ae), "repo.sqlite")) { archive_entry_set_pathname(ae, repodb); archive_read_extract(a, ae, EXTRACT_ARCHIVE_FLAGS); break; } } } if (a != NULL) archive_read_finish(a); } if ((retcode = initialize_repo(repodb, force, &sqlite)) != EPKG_OK) goto cleanup; if ((retcode = initialize_prepared_statements(sqlite)) != EPKG_OK) goto cleanup; thd_data.root_path = path; thd_data.max_results = num_workers; thd_data.num_results = 0; thd_data.stop = false; thd_data.fts = fts; pthread_mutex_init(&thd_data.fts_m, NULL); STAILQ_INIT(&thd_data.results); thd_data.thd_finished = 0; pthread_mutex_init(&thd_data.results_m, NULL); pthread_cond_init(&thd_data.has_result, NULL); pthread_cond_init(&thd_data.has_room, NULL); /* Launch workers */ tids = calloc(num_workers, sizeof(pthread_t)); for (int i = 0; i < num_workers; i++) { pthread_create(&tids[i], NULL, (void *)&read_pkg_file, &thd_data); } for (;;) { struct pkg_result *r; const char *name, *version, *origin, *comment, *desc; const char *arch, *maintainer, *www, *prefix; int64_t flatsize; lic_t licenselogic; pthread_mutex_lock(&thd_data.results_m); while ((r = STAILQ_FIRST(&thd_data.results)) == NULL) { if (thd_data.thd_finished == num_workers) { break; } pthread_cond_wait(&thd_data.has_result, &thd_data.results_m); } if (r != NULL) { STAILQ_REMOVE_HEAD(&thd_data.results, next); thd_data.num_results--; pthread_cond_signal(&thd_data.has_room); } pthread_mutex_unlock(&thd_data.results_m); if (r == NULL) { break; } if (r->retcode != EPKG_OK) { continue; } /* do not add if package if already in repodb (possibly at a different pkg_path) */ if (run_prepared_statement(EXISTS, r->cksum) != SQLITE_ROW) { ERROR_SQLITE(sqlite); goto cleanup; } if (sqlite3_column_int(STMT(EXISTS), 0) > 0) { continue; } if (progress != NULL) progress(r->pkg, data); pkg_get(r->pkg, PKG_ORIGIN, &origin, PKG_NAME, &name, PKG_VERSION, &version, PKG_COMMENT, &comment, PKG_DESC, &desc, PKG_ARCH, &arch, PKG_MAINTAINER, &maintainer, PKG_WWW, &www, PKG_PREFIX, &prefix, PKG_FLATSIZE, &flatsize, PKG_LICENSE_LOGIC, &licenselogic); try_again: if ((ret = run_prepared_statement(PKG, origin, name, version, comment, desc, arch, maintainer, www, prefix, r->size, flatsize, (int64_t)licenselogic, r->cksum, r->path)) != SQLITE_DONE) { if (ret == SQLITE_CONSTRAINT) { switch(maybe_delete_conflicting(origin, version, r->path)) { case EPKG_FATAL: /* sqlite error */ ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; break; case EPKG_END: /* repo already has newer */ continue; break; default: /* conflict cleared, try again */ goto try_again; break; } } else { ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; } } package_id = sqlite3_last_insert_rowid(sqlite); dep = NULL; while (pkg_deps(r->pkg, &dep) == EPKG_OK) { if (run_prepared_statement(DEPS, pkg_dep_origin(dep), pkg_dep_name(dep), pkg_dep_version(dep), package_id) != SQLITE_DONE) { ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; } } category = NULL; while (pkg_categories(r->pkg, &category) == EPKG_OK) { const char *cat_name = pkg_category_name(category); ret = run_prepared_statement(CAT1, cat_name); if (ret == SQLITE_DONE) ret = run_prepared_statement(CAT2, package_id, cat_name); if (ret != SQLITE_DONE) { ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; } } license = NULL; while (pkg_licenses(r->pkg, &license) == EPKG_OK) { const char *lic_name = pkg_license_name(license); ret = run_prepared_statement(LIC1, lic_name); if (ret == SQLITE_DONE) ret = run_prepared_statement(LIC2, package_id, lic_name); if (ret != SQLITE_DONE) { ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; } } option = NULL; while (pkg_options(r->pkg, &option) == EPKG_OK) { if (run_prepared_statement(OPTS, pkg_option_opt(option), pkg_option_value(option), package_id) != SQLITE_DONE) { ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; } } shlib = NULL; while (pkg_shlibs(r->pkg, &shlib) == EPKG_OK) { const char *shlib_name = pkg_shlib_name(shlib); ret = run_prepared_statement(SHLIB1, shlib_name); if (ret == SQLITE_DONE) ret = run_prepared_statement(SHLIB2, package_id, shlib_name); if (ret != SQLITE_DONE) { ERROR_SQLITE(sqlite); retcode = EPKG_FATAL; goto cleanup; } } pkg_free(r->pkg); free(r); } if (pkgdb_transaction_commit(sqlite, NULL) != SQLITE_OK) retcode = EPKG_FATAL; cleanup: if (tids != NULL) { // Cancel running threads if (retcode != EPKG_OK) { pthread_mutex_lock(&thd_data.fts_m); thd_data.stop = true; pthread_mutex_unlock(&thd_data.fts_m); } // Join on threads to release thread IDs for (int i = 0; i < num_workers; i++) { pthread_join(tids[i], NULL); } free(tids); } if (fts != NULL) fts_close(fts); finalize_prepared_statements(); if (sqlite != NULL) sqlite3_close(sqlite); if (errmsg != NULL) sqlite3_free(errmsg); sqlite3_shutdown(); return (retcode); }