void pool_addfileprovides_queue(Pool *pool, Queue *idq, Queue *idqinst) { Solvable *s; Repo *installed, *repo; struct searchfiles sf, isf, *isfp; struct addfileprovides_cbdata cbd; int i; unsigned int now; installed = pool->installed; now = solv_timems(0); memset(&cbd, 0, sizeof(cbd)); memset(&sf, 0, sizeof(sf)); map_init(&sf.seen, pool->ss.nstrings + pool->nrels); memset(&isf, 0, sizeof(isf)); map_init(&isf.seen, pool->ss.nstrings + pool->nrels); pool->addedfileprovides = pool->addfileprovidesfiltered ? 1 : 2; if (idq) queue_empty(idq); if (idqinst) queue_empty(idqinst); isfp = installed ? &isf : 0; for (i = 1, s = pool->solvables + i; i < pool->nsolvables; i++, s++) { repo = s->repo; if (!repo) continue; if (s->obsoletes) pool_addfileprovides_dep(pool, repo->idarraydata + s->obsoletes, &sf, isfp); if (s->conflicts) pool_addfileprovides_dep(pool, repo->idarraydata + s->conflicts, &sf, isfp); if (s->requires) pool_addfileprovides_dep(pool, repo->idarraydata + s->requires, &sf, isfp); if (s->recommends) pool_addfileprovides_dep(pool, repo->idarraydata + s->recommends, &sf, isfp); if (s->suggests) pool_addfileprovides_dep(pool, repo->idarraydata + s->suggests, &sf, isfp); if (s->supplements) pool_addfileprovides_dep(pool, repo->idarraydata + s->supplements, &sf, isfp); if (s->enhances) pool_addfileprovides_dep(pool, repo->idarraydata + s->enhances, &sf, isfp); } map_free(&sf.seen); map_free(&isf.seen); POOL_DEBUG(SOLV_DEBUG_STATS, "found %d file dependencies, %d installed file dependencies\n", sf.nfiles, isf.nfiles); if (sf.nfiles) { #if 0 for (i = 0; i < sf.nfiles; i++) POOL_DEBUG(SOLV_DEBUG_STATS, "looking up %s in filelist\n", pool_id2str(pool, sf.ids[i])); #endif FOR_REPOS(i, repo) repo_addfileprovides_search(repo, &cbd, &sf); if (idq) queue_insertn(idq, idq->count, sf.nfiles, sf.ids); if (idqinst) queue_insertn(idqinst, idqinst->count, sf.nfiles, sf.ids); solv_free(sf.ids); } if (isf.nfiles) { #if 0 for (i = 0; i < isf.nfiles; i++) POOL_DEBUG(SOLV_DEBUG_STATS, "looking up %s in installed filelist\n", pool_id2str(pool, isf.ids[i])); #endif if (installed) repo_addfileprovides_search(installed, &cbd, &isf); if (installed && idqinst) for (i = 0; i < isf.nfiles; i++) queue_pushunique(idqinst, isf.ids[i]); solv_free(isf.ids); } free_dirs_names_array(&cbd); solv_free(cbd.dids); pool_freewhatprovides(pool); /* as we have added provides */ POOL_DEBUG(SOLV_DEBUG_STATS, "addfileprovides took %d ms\n", solv_timems(now)); }
int repo_add_helix(Repo *repo, FILE *fp, int flags) { Pool *pool = repo->pool; Parsedata pd; Repodata *data; char buf[BUFF_SIZE]; int i, l; struct stateswitch *sw; unsigned int now; XML_Parser parser; now = solv_timems(0); data = repo_add_repodata(repo, flags); /* prepare parsedata */ memset(&pd, 0, sizeof(pd)); for (i = 0, sw = stateswitches; sw->from != NUMSTATES; i++, sw++) { if (!pd.swtab[sw->from]) pd.swtab[sw->from] = sw; pd.sbtab[sw->to] = sw->from; } pd.pool = pool; pd.repo = repo; pd.content = (char *)malloc(256); /* must hold all solvable kinds! */ pd.acontent = 256; pd.lcontent = 0; pd.evrspace = (char *)malloc(256); pd.aevrspace= 256; pd.levrspace = 1; pd.data = data; /* set up XML parser */ parser = XML_ParserCreate(NULL); XML_SetUserData(parser, &pd); /* make parserdata available to XML callbacks */ XML_SetElementHandler(parser, startElement, endElement); XML_SetCharacterDataHandler(parser, characterData); /* read/parse XML file */ for (;;) { l = fread(buf, 1, sizeof(buf), fp); if (XML_Parse(parser, buf, l, l == 0) == XML_STATUS_ERROR) { pd.ret = pool_error(pool, -1, "%s at line %u", XML_ErrorString(XML_GetErrorCode(parser)), (unsigned int)XML_GetCurrentLineNumber(parser)); break; } if (l == 0) break; } XML_ParserFree(parser); free(pd.content); free(pd.evrspace); if (!(flags & REPO_NO_INTERNALIZE)) repodata_internalize(data); POOL_DEBUG(SOLV_DEBUG_STATS, "repo_add_helix took %d ms\n", solv_timems(now)); POOL_DEBUG(SOLV_DEBUG_STATS, "repo size: %d solvables\n", repo->nsolvables); POOL_DEBUG(SOLV_DEBUG_STATS, "repo memory used: %d K incore, %d K idarray\n", repodata_memused(data)/1024, repo->idarraysize / (int)(1024/sizeof(Id))); return pd.ret; }
int repo_add_rpmmd(Repo *repo, FILE *fp, const char *language, int flags) { Pool *pool = repo->pool; struct parsedata pd; char buf[BUFF_SIZE]; int i, l; struct stateswitch *sw; Repodata *data; unsigned int now; XML_Parser parser; now = solv_timems(0); data = repo_add_repodata(repo, flags); memset(&pd, 0, sizeof(pd)); for (i = 0, sw = stateswitches; sw->from != NUMSTATES; i++, sw++) { if (!pd.swtab[sw->from]) pd.swtab[sw->from] = sw; pd.sbtab[sw->to] = sw->from; } pd.common.pool = pool; pd.common.repo = repo; pd.data = data; pd.content = solv_malloc(256); pd.acontent = 256; pd.lcontent = 0; pd.common.tmp = 0; pd.common.tmpl = 0; pd.kind = 0; pd.language = language; /* initialize the string pool where we will store the package checksums we know about, to get an Id we can use in a cache */ stringpool_init_empty(&pd.cspool); if ((flags & REPO_EXTEND_SOLVABLES) != 0) { /* setup join data */ Dataiterator di; dataiterator_init(&di, pool, repo, 0, SOLVABLE_CHECKSUM, 0, 0); while (dataiterator_step(&di)) { const char *str; int index; if (!solv_chksum_len(di.key->type)) continue; str = repodata_chk2str(di.data, di.key->type, (const unsigned char *)di.kv.str); index = stringpool_str2id(&pd.cspool, str, 1); if (index >= pd.ncscache) { pd.cscache = solv_zextend(pd.cscache, pd.ncscache, index + 1 - pd.ncscache, sizeof(Id), 255); pd.ncscache = index + 1; } pd.cscache[index] = di.solvid; } dataiterator_free(&di); } parser = XML_ParserCreate(NULL); XML_SetUserData(parser, &pd); pd.parser = &parser; XML_SetElementHandler(parser, startElement, endElement); XML_SetCharacterDataHandler(parser, characterData); for (;;) { l = fread(buf, 1, sizeof(buf), fp); if (XML_Parse(parser, buf, l, l == 0) == XML_STATUS_ERROR) { pool_debug(pool, SOLV_FATAL, "repo_rpmmd: %s at line %u:%u\n", XML_ErrorString(XML_GetErrorCode(parser)), (unsigned int)XML_GetCurrentLineNumber(parser), (unsigned int)XML_GetCurrentColumnNumber(parser)); exit(1); } if (l == 0) break; } XML_ParserFree(parser); solv_free(pd.content); solv_free(pd.lastdirstr); join_freemem(); stringpool_free(&pd.cspool); solv_free(pd.cscache); if (!(flags & REPO_NO_INTERNALIZE)) repodata_internalize(data); POOL_DEBUG(SOLV_DEBUG_STATS, "repo_add_rpmmd took %d ms\n", solv_timems(now)); POOL_DEBUG(SOLV_DEBUG_STATS, "repo size: %d solvables\n", repo->nsolvables); POOL_DEBUG(SOLV_DEBUG_STATS, "repo memory used: %d K incore, %d K idarray\n", data->incoredatalen/1024, repo->idarraysize / (int)(1024/sizeof(Id))); return 0; }
int pool_findfileconflicts(Pool *pool, Queue *pkgs, int cutoff, Queue *conflicts, int flags, void *(*handle_cb)(Pool *, Id, void *) , void *handle_cbdata) { int i, j, cflmapn, idxmapset; struct cbdata cbdata; unsigned int now, start; void *handle; Repo *installed = pool->installed; Id p; int obsoleteusescolors = pool_get_flag(pool, POOL_FLAG_OBSOLETEUSESCOLORS); int hdrfetches; queue_empty(conflicts); if (!pkgs->count) return 0; now = start = solv_timems(0); POOL_DEBUG(SOLV_DEBUG_STATS, "searching for file conflicts\n"); POOL_DEBUG(SOLV_DEBUG_STATS, "packages: %d, cutoff %d\n", pkgs->count, cutoff); memset(&cbdata, 0, sizeof(cbdata)); cbdata.aliases = flags & FINDFILECONFLICTS_CHECK_DIRALIASING; cbdata.pool = pool; if (cbdata.aliases && (flags & FINDFILECONFLICTS_USE_ROOTDIR) != 0) { cbdata.rootdir = pool_get_rootdir(pool); if (cbdata.rootdir && !strcmp(cbdata.rootdir, "/")) cbdata.rootdir = 0; if (cbdata.rootdir) cbdata.rootdirl = strlen(cbdata.rootdir); if (!cbdata.rootdir) cbdata.usestat = 1; } queue_init(&cbdata.lookat); queue_init(&cbdata.lookat_dir); map_init(&cbdata.idxmap, pkgs->count); if (cutoff <= 0) cutoff = pkgs->count; /* avarage file list size: 200 files per package */ /* avarage dir count: 20 dirs per package */ /* first pass: scan dirs */ if (!cbdata.aliases) { hdrfetches = 0; cflmapn = (cutoff + 3) * 64; while ((cflmapn & (cflmapn - 1)) != 0) cflmapn = cflmapn & (cflmapn - 1); cbdata.dirmap = solv_calloc(cflmapn, 2 * sizeof(Id)); cbdata.dirmapn = cflmapn - 1; /* make it a mask */ cbdata.create = 1; idxmapset = 0; for (i = 0; i < pkgs->count; i++) { if (i == cutoff) cbdata.create = 0; cbdata.idx = i; p = pkgs->elements[i]; if ((flags & FINDFILECONFLICTS_USE_SOLVABLEFILELIST) != 0 && installed) { if (p >= installed->start && p < installed->end && pool->solvables[p].repo == installed) { iterate_solvable_dirs(pool, p, finddirs_cb, &cbdata); if (MAPTST(&cbdata.idxmap, i)) idxmapset++; continue; } } handle = (*handle_cb)(pool, p, handle_cbdata); if (!handle) continue; hdrfetches++; rpm_iterate_filelist(handle, RPM_ITERATE_FILELIST_ONLYDIRS, finddirs_cb, &cbdata); if (MAPTST(&cbdata.idxmap, i)) idxmapset++; } POOL_DEBUG(SOLV_DEBUG_STATS, "dirmap size: %d, used %d\n", cbdata.dirmapn + 1, cbdata.dirmapused); POOL_DEBUG(SOLV_DEBUG_STATS, "dirmap memory usage: %d K\n", (cbdata.dirmapn + 1) * 2 * (int)sizeof(Id) / 1024); POOL_DEBUG(SOLV_DEBUG_STATS, "header fetches: %d\n", hdrfetches); POOL_DEBUG(SOLV_DEBUG_STATS, "dirmap creation took %d ms\n", solv_timems(now)); POOL_DEBUG(SOLV_DEBUG_STATS, "dir conflicts found: %d, idxmap %d of %d\n", cbdata.dirconflicts, idxmapset, pkgs->count); } /* second pass: scan files */ now = solv_timems(0); cflmapn = (cutoff + 3) * 128; while ((cflmapn & (cflmapn - 1)) != 0) cflmapn = cflmapn & (cflmapn - 1); cbdata.cflmap = solv_calloc(cflmapn, 2 * sizeof(Id)); cbdata.cflmapn = cflmapn - 1; /* make it a mask */ cbdata.create = 1; hdrfetches = 0; for (i = 0; i < pkgs->count; i++) { if (i == cutoff) cbdata.create = 0; if (!cbdata.aliases && !MAPTST(&cbdata.idxmap, i)) continue; cbdata.idx = i; p = pkgs->elements[i]; if (!cbdata.create && (flags & FINDFILECONFLICTS_USE_SOLVABLEFILELIST) != 0 && installed) { if (p >= installed->start && p < installed->end && pool->solvables[p].repo == installed) if (!precheck_solvable_files(&cbdata, pool, p)) continue; } /* can't use FINDFILECONFLICTS_USE_SOLVABLEFILELIST because we have to know if * the file is a directory or not */ handle = (*handle_cb)(pool, p, handle_cbdata); if (!handle) continue; hdrfetches++; cbdata.lastdiridx = -1; rpm_iterate_filelist(handle, RPM_ITERATE_FILELIST_NOGHOSTS, cbdata.aliases ? findfileconflicts_basename_cb : findfileconflicts_cb, &cbdata); } POOL_DEBUG(SOLV_DEBUG_STATS, "filemap size: %d, used %d\n", cbdata.cflmapn + 1, cbdata.cflmapused); POOL_DEBUG(SOLV_DEBUG_STATS, "filemap memory usage: %d K\n", (cbdata.cflmapn + 1) * 2 * (int)sizeof(Id) / 1024); POOL_DEBUG(SOLV_DEBUG_STATS, "header fetches: %d\n", hdrfetches); POOL_DEBUG(SOLV_DEBUG_STATS, "filemap creation took %d ms\n", solv_timems(now)); POOL_DEBUG(SOLV_DEBUG_STATS, "lookat_dir size: %d\n", cbdata.lookat_dir.count); queue_free(&cbdata.lookat_dir); /* we need another pass for aliases */ if (cbdata.aliases) { now = solv_timems(0); /* make sure the first offset is not zero */ addfilesspace(&cbdata, 1); cflmapn = (cutoff + 3) * 16; while ((cflmapn & (cflmapn - 1)) != 0) cflmapn = cflmapn & (cflmapn - 1); cbdata.normap = solv_calloc(cflmapn, 2 * sizeof(Id)); cbdata.normapn = cflmapn - 1; /* make it a mask */ if (cbdata.usestat) { cbdata.statmap = solv_calloc(cflmapn, 2 * sizeof(Id)); cbdata.statmapn = cflmapn - 1; /* make it a mask */ } cbdata.create = 0; hdrfetches = 0; for (i = 0; i < pkgs->count; i++) { if (!MAPTST(&cbdata.idxmap, i)) continue; p = pkgs->elements[i]; cbdata.idx = i; /* can't use FINDFILECONFLICTS_USE_SOLVABLEFILELIST because we have to know if * the file is a directory or not */ handle = (*handle_cb)(pool, p, handle_cbdata); if (!handle) continue; hdrfetches++; cbdata.lastdiridx = -1; rpm_iterate_filelist(handle, RPM_ITERATE_FILELIST_NOGHOSTS, findfileconflicts_alias_cb, &cbdata); } POOL_DEBUG(SOLV_DEBUG_STATS, "normap size: %d, used %d\n", cbdata.normapn + 1, cbdata.normapused); POOL_DEBUG(SOLV_DEBUG_STATS, "normap memory usage: %d K\n", (cbdata.normapn + 1) * 2 * (int)sizeof(Id) / 1024); POOL_DEBUG(SOLV_DEBUG_STATS, "header fetches: %d\n", hdrfetches); POOL_DEBUG(SOLV_DEBUG_STATS, "stats made: %d\n", cbdata.statsmade); if (cbdata.usestat) { POOL_DEBUG(SOLV_DEBUG_STATS, "statmap size: %d, used %d\n", cbdata.statmapn + 1, cbdata.statmapused); POOL_DEBUG(SOLV_DEBUG_STATS, "statmap memory usage: %d K\n", (cbdata.statmapn + 1) * 2 * (int)sizeof(Id) / 1024); } cbdata.statmap = solv_free(cbdata.statmap); cbdata.statmapn = 0; cbdata.canonspace = solv_free(cbdata.canonspace); cbdata.canonspacen = 0; POOL_DEBUG(SOLV_DEBUG_STATS, "alias processing took %d ms\n", solv_timems(now)); } cbdata.dirmap = solv_free(cbdata.dirmap); cbdata.dirmapn = 0; cbdata.dirmapused = 0; cbdata.cflmap = solv_free(cbdata.cflmap); cbdata.cflmapn = 0; cbdata.cflmapused = 0; map_free(&cbdata.idxmap); /* sort and unify/prune */ now = solv_timems(0); POOL_DEBUG(SOLV_DEBUG_STATS, "raw candidates: %d, pruning\n", cbdata.lookat.count / 4); solv_sort(cbdata.lookat.elements, cbdata.lookat.count / 4, sizeof(Id) * 4, &lookat_hx_cmp, pool); for (i = j = 0; i < cbdata.lookat.count; ) { int first = 1; Id hx = cbdata.lookat.elements[i]; Id idx = cbdata.lookat.elements[i + 1]; Id dhx = cbdata.lookat.elements[i + 2]; Id dirid = cbdata.lookat.elements[i + 3]; i += 4; for (; i < cbdata.lookat.count && hx == cbdata.lookat.elements[i] && (dirid == cbdata.lookat.elements[i + 3] || dirid == -cbdata.lookat.elements[i + 3]); i += 4) { if (idx == cbdata.lookat.elements[i + 1] && dhx == cbdata.lookat.elements[i + 2]) continue; /* ignore duplicates */ if (first) { if (dirid < 0) continue; /* all have a neg dirid */ cbdata.lookat.elements[j++] = hx; cbdata.lookat.elements[j++] = idx; cbdata.lookat.elements[j++] = dhx; cbdata.lookat.elements[j++] = dirid; first = 0; } idx = cbdata.lookat.elements[i + 1]; dhx = cbdata.lookat.elements[i + 2]; cbdata.lookat.elements[j++] = hx; cbdata.lookat.elements[j++] = idx; cbdata.lookat.elements[j++] = dhx; cbdata.lookat.elements[j++] = dirid; } } queue_truncate(&cbdata.lookat, j); POOL_DEBUG(SOLV_DEBUG_STATS, "candidates now: %d\n", cbdata.lookat.count / 4); POOL_DEBUG(SOLV_DEBUG_STATS, "pruning took %d ms\n", solv_timems(now)); /* third pass: collect file info for all files that match a hx */ now = solv_timems(0); solv_sort(cbdata.lookat.elements, cbdata.lookat.count / 4, sizeof(Id) * 4, &lookat_idx_cmp, pool); queue_init(&cbdata.files); hdrfetches = 0; for (i = 0; i < cbdata.lookat.count; i += 4) { Id idx = cbdata.lookat.elements[i + 1]; int iterflags = RPM_ITERATE_FILELIST_WITHMD5 | RPM_ITERATE_FILELIST_NOGHOSTS; if (obsoleteusescolors) iterflags |= RPM_ITERATE_FILELIST_WITHCOL; p = pkgs->elements[idx]; handle = (*handle_cb)(pool, p, handle_cbdata); if (handle) hdrfetches++; for (;; i += 4) { int fstart = cbdata.files.count; queue_push(&cbdata.files, idx); queue_push(&cbdata.files, 0); cbdata.idx = idx; cbdata.hx = cbdata.lookat.elements[i]; cbdata.dirhash = cbdata.lookat.elements[i + 2]; cbdata.dirid = cbdata.lookat.elements[i + 3]; cbdata.lastdiridx = -1; if (handle) rpm_iterate_filelist(handle, iterflags, findfileconflicts2_cb, &cbdata); cbdata.files.elements[fstart + 1] = cbdata.files.count; cbdata.lookat.elements[i + 1] = fstart; if (i + 4 >= cbdata.lookat.count || cbdata.lookat.elements[i + 4 + 1] != idx) break; } } POOL_DEBUG(SOLV_DEBUG_STATS, "header fetches: %d\n", hdrfetches); POOL_DEBUG(SOLV_DEBUG_STATS, "file info fetching took %d ms\n", solv_timems(now)); cbdata.normap = solv_free(cbdata.normap); cbdata.normapn = 0; /* forth pass: for each hx we have, compare all matching files against all other matching files */ now = solv_timems(0); solv_sort(cbdata.lookat.elements, cbdata.lookat.count / 4, sizeof(Id) * 4, &lookat_hx_cmp, pool); for (i = 0; i < cbdata.lookat.count - 4; i += 4) { Id hx = cbdata.lookat.elements[i]; Id pstart = cbdata.lookat.elements[i + 1]; Id dirid = cbdata.lookat.elements[i + 3]; Id pidx = cbdata.files.elements[pstart]; Id pend = cbdata.files.elements[pstart + 1]; if (cbdata.lookat.elements[i + 4] != hx) continue; /* no package left with that hx */ for (j = i + 4; j < cbdata.lookat.count && cbdata.lookat.elements[j] == hx && cbdata.lookat.elements[j + 3] == dirid; j += 4) { Id qstart = cbdata.lookat.elements[j + 1]; Id qidx = cbdata.files.elements[qstart]; Id qend = cbdata.files.elements[qstart + 1]; int ii, jj; if (pidx >= cutoff && qidx >= cutoff) continue; /* no conflicts between packages with idx >= cutoff */ for (ii = pstart + 2; ii < pend; ii++) for (jj = qstart + 2; jj < qend; jj++) { char *fsi = (char *)cbdata.filesspace + cbdata.files.elements[ii]; char *fsj = (char *)cbdata.filesspace + cbdata.files.elements[jj]; if (cbdata.aliases) { /* compare just the basenames, the dirs match because of the dirid */ char *bsi = strrchr(fsi + 34, '/'); char *bsj = strrchr(fsj + 34, '/'); if (!bsi || !bsj) continue; if (strcmp(bsi, bsj)) continue; /* different base names */ } else { if (strcmp(fsi + 34, fsj + 34)) continue; /* different file names */ } if (!strcmp(fsi, fsj)) continue; /* file digests match, no conflict */ if (obsoleteusescolors && fsi[33] && fsj[33] && (fsi[33] & fsj[33]) == 0) continue; /* colors do not conflict */ queue_push(conflicts, pool_str2id(pool, fsi + 34, 1)); queue_push(conflicts, pkgs->elements[pidx]); queue_push(conflicts, pool_str2id(pool, fsi, 1)); queue_push(conflicts, pool_str2id(pool, fsj + 34, 1)); queue_push(conflicts, pkgs->elements[qidx]); queue_push(conflicts, pool_str2id(pool, fsj, 1)); } } } POOL_DEBUG(SOLV_DEBUG_STATS, "filespace size: %d K\n", cbdata.filesspacen / 1024); POOL_DEBUG(SOLV_DEBUG_STATS, "candidate check took %d ms\n", solv_timems(now)); cbdata.filesspace = solv_free(cbdata.filesspace); cbdata.filesspacen = 0; queue_free(&cbdata.lookat); queue_free(&cbdata.files); if (conflicts->count > 6) solv_sort(conflicts->elements, conflicts->count / 6, 6 * sizeof(Id), conflicts_cmp, pool); POOL_DEBUG(SOLV_DEBUG_STATS, "found %d file conflicts\n", conflicts->count / 6); POOL_DEBUG(SOLV_DEBUG_STATS, "file conflict detection took %d ms\n", solv_timems(start)); return conflicts->count / 6; }