/*
* prints the indexed words and counts to the given output file
*/
void printIndexToFile(char *outputFile, int upDir) {
if(upDir) {
chdir("..");
}
FILE *fp;
fp = fopen( outputFile, "w+" );
HASH_SORT( words, name_sort );
int isFirstIter = 1;
TokenPtr tmp, currentWord;
HASH_ITER(hh, words, currentWord, tmp) {
if(isFirstIter){
isFirstIter = 0;
} else {
fprintf(fp, "\n");
}
fprintf(fp, "<list> ");
fprintf(fp, "%s\n", currentWord->key);
LL_SORT(currentWord->fileHead, count_sort);
FileNodePtr tmp = currentWord->fileHead;
fprintf(fp, "%s %d", tmp->filename, tmp->tokenCount);
while(tmp->next != NULL) {
tmp = tmp->next;
// printf("file: %s \t count: %d\n", tmp->filename, tmp->tokenCount);
fprintf(fp, " %s %d", tmp->filename, tmp->tokenCount);
}
// if(currentWord->fileHead == NULL)
fprintf(fp, "\n</list>");
}
fclose(fp);
}
int main(int argc, char *argv[]) {
el *name, *tmp;
el *head = NULL;
char linebuf[BUFLEN];
FILE *file;
file = fopen( "test11.dat", "r" );
if (file == NULL) {
perror("can't open: ");
exit(-1);
}
while (fgets(linebuf,BUFLEN,file) != NULL) {
name = (el*)malloc(sizeof(el));
if (name == NULL) exit(-1);
strncpy(name->bname,linebuf,sizeof(name->bname));
LL_PREPEND(head, name);
}
LL_SORT(head, namecmp);
LL_FOREACH(head,tmp) printf("%s", tmp->bname);
fclose(file);
return 0;
}
int
pkg_create_repo(char *path, bool force, bool filelist,
void (progress)(struct pkg *pkg, void *data), void *data)
{
FTS *fts = NULL;
struct thd_data thd_data;
int num_workers;
size_t len;
pthread_t *tids = NULL;
struct digest_list_entry *dlist = NULL, *cur_dig, *dtmp;
sqlite3 *sqlite = NULL;
char *errmsg = NULL;
int retcode = EPKG_OK;
char *repopath[2];
char repodb[MAXPATHLEN + 1];
char repopack[MAXPATHLEN + 1];
char *manifest_digest;
FILE *psyml, *fsyml, *mandigests;
psyml = fsyml = mandigests = NULL;
if (!is_dir(path)) {
pkg_emit_error("%s is not a directory", path);
return (EPKG_FATAL);
}
repopath[0] = path;
repopath[1] = NULL;
len = sizeof(num_workers);
if (sysctlbyname("hw.ncpu", &num_workers, &len, NULL, 0) == -1)
num_workers = 6;
if ((fts = fts_open(repopath, FTS_PHYSICAL|FTS_NOCHDIR, NULL)) == NULL) {
pkg_emit_errno("fts_open", path);
retcode = EPKG_FATAL;
goto cleanup;
}
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_packagesite_file);
if ((psyml = fopen(repodb, "w")) == NULL) {
retcode = EPKG_FATAL;
goto cleanup;
}
if (filelist) {
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_filesite_file);
if ((fsyml = fopen(repodb, "w")) == NULL) {
retcode = EPKG_FATAL;
goto cleanup;
}
}
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_digests_file);
if ((mandigests = fopen(repodb, "w")) == NULL) {
retcode = EPKG_FATAL;
goto cleanup;
}
snprintf(repodb, sizeof(repodb), "%s/%s", path, repo_db_file);
snprintf(repopack, sizeof(repopack), "%s/repo.txz", path);
pack_extract(repopack, repo_db_file, repodb);
if ((retcode = pkgdb_repo_open(repodb, force, &sqlite, true)) != EPKG_OK)
goto cleanup;
if ((retcode = pkgdb_repo_init(sqlite, true)) != EPKG_OK)
goto cleanup;
thd_data.root_path = path;
thd_data.max_results = num_workers;
thd_data.num_results = 0;
thd_data.stop = false;
thd_data.fts = fts;
thd_data.read_files = filelist;
pthread_mutex_init(&thd_data.fts_m, NULL);
thd_data.results = NULL;
thd_data.thd_finished = 0;
pthread_mutex_init(&thd_data.results_m, NULL);
pthread_cond_init(&thd_data.has_result, NULL);
pthread_cond_init(&thd_data.has_room, NULL);
/* Launch workers */
tids = calloc(num_workers, sizeof(pthread_t));
for (int i = 0; i < num_workers; i++) {
pthread_create(&tids[i], NULL, (void *)&read_pkg_file, &thd_data);
}
for (;;) {
struct pkg_result *r;
const char *origin;
long manifest_pos, files_pos;
pthread_mutex_lock(&thd_data.results_m);
while ((r = thd_data.results) == NULL) {
if (thd_data.thd_finished == num_workers) {
break;
}
pthread_cond_wait(&thd_data.has_result, &thd_data.results_m);
}
if (r != NULL) {
LL_DELETE(thd_data.results, thd_data.results);
thd_data.num_results--;
pthread_cond_signal(&thd_data.has_room);
}
pthread_mutex_unlock(&thd_data.results_m);
if (r == NULL) {
break;
}
if (r->retcode != EPKG_OK) {
continue;
}
/* do not add if package if already in repodb
(possibly at a different pkg_path) */
retcode = pkgdb_repo_cksum_exists(sqlite, r->cksum);
if (retcode == EPKG_FATAL) {
goto cleanup;
}
else if (retcode == EPKG_OK) {
continue;
}
if (progress != NULL)
progress(r->pkg, data);
manifest_pos = ftell(psyml);
pkg_emit_manifest_file(r->pkg, psyml, PKG_MANIFEST_EMIT_COMPACT, &manifest_digest);
if (filelist) {
files_pos = ftell(fsyml);
pkg_emit_filelist(r->pkg, fsyml);
} else {
files_pos = 0;
}
pkg_get(r->pkg, PKG_ORIGIN, &origin);
cur_dig = malloc(sizeof (struct digest_list_entry));
cur_dig->origin = strdup(origin);
cur_dig->digest = manifest_digest;
cur_dig->manifest_pos = manifest_pos;
cur_dig->files_pos = files_pos;
LL_PREPEND(dlist, cur_dig);
retcode = pkgdb_repo_add_package(r->pkg, r->path, sqlite,
manifest_digest, false, true);
if (retcode == EPKG_END) {
continue;
}
else if (retcode != EPKG_OK) {
goto cleanup;
}
pkg_free(r->pkg);
free(r);
}
/* Now sort all digests */
LL_SORT(dlist, digest_sort_compare_func);
cleanup:
if (pkgdb_repo_close(sqlite, retcode == EPKG_OK) != EPKG_OK) {
retcode = EPKG_FATAL;
}
LL_FOREACH_SAFE(dlist, cur_dig, dtmp) {
if (retcode == EPKG_OK) {
fprintf(mandigests, "%s:%s:%ld:%ld\n", cur_dig->origin,
cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos);
}
free(cur_dig->digest);
free(cur_dig->origin);
free(cur_dig);
}
if (tids != NULL) {
// Cancel running threads
if (retcode != EPKG_OK) {
pthread_mutex_lock(&thd_data.fts_m);
thd_data.stop = true;
pthread_mutex_unlock(&thd_data.fts_m);
}
// Join on threads to release thread IDs
for (int i = 0; i < num_workers; i++) {
pthread_join(tids[i], NULL);
}
free(tids);
}
if (fts != NULL)
fts_close(fts);
if (fsyml != NULL)
fclose(fsyml);
if (psyml != NULL)
fclose(psyml);
if (mandigests != NULL)
fclose(mandigests);
if (sqlite != NULL)
sqlite3_close(sqlite);
if (errmsg != NULL)
sqlite3_free(errmsg);
sqlite3_shutdown();
return (retcode);
}
