/* check if file is already there (RET_NOTHING) or could be added (RET_OK)
* or RET_ERROR_WRONG_MD5SUM if filekey already has different md5sum */
retvalue files_canadd(const char *filekey, const struct checksums *checksums) {
retvalue r;
struct checksums *indatabase;
bool improves;
r = files_get_checksums(filekey, &indatabase);
if (r == RET_NOTHING)
return RET_OK;
if (RET_WAS_ERROR(r))
return r;
if (!checksums_check(indatabase, checksums, &improves)) {
fprintf(stderr,
"File \"%s\" is already registered with different checksums!\n",
filekey);
checksums_printdifferences(stderr, indatabase, checksums);
checksums_free(indatabase);
return RET_ERROR_WRONG_MD5;
}
// TODO: sometimes the caller might want to have additional
// checksums from the database already, think about ways to
// make them available...
checksums_free(indatabase);
return RET_NOTHING;
}
void diffindex_free(struct diffindex *diffindex) {
int i;
if (diffindex == NULL)
return;
checksums_free(diffindex->destination);
for (i = 0 ; i < diffindex->patchcount ; i ++) {
checksums_free(diffindex->patches[i].frompackages);
free(diffindex->patches[i].name);
checksums_free(diffindex->patches[i].checksums);
}
free(diffindex);
}
retvalue files_collectnewchecksums(void) {
retvalue result, r;
struct cursor *cursor;
const char *filekey, *all;
size_t alllen;
struct checksums *expected;
char *fullfilename;
result = RET_NOTHING;
r = table_newglobalcursor(rdb_checksums, &cursor);
if (!RET_IS_OK(r))
return r;
while (cursor_nexttempdata(rdb_checksums, cursor,
&filekey, &all, &alllen)) {
r = checksums_setall(&expected, all, alllen);
if (!RET_IS_OK(r)) {
RET_UPDATE(result, r);
continue;
}
if (checksums_iscomplete(expected)) {
checksums_free(expected);
continue;
}
fullfilename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(fullfilename)) {
result = RET_ERROR_OOM;
checksums_free(expected);
break;
}
r = checksums_complete(&expected, fullfilename);
if (r == RET_NOTHING) {
fprintf(stderr, "Missing file '%s'!\n", fullfilename);
r = RET_ERROR_MISSING;
}
if (r == RET_ERROR_WRONG_MD5) {
fprintf(stderr,
"ERROR: Cannot collect missing checksums for '%s'\n"
"as the file in the pool does not match the already recorded checksums\n",
filekey);
}
free(fullfilename);
if (RET_IS_OK(r))
r = files_replace_checksums(filekey, expected);
checksums_free(expected);
RET_UPDATE(result, r);
}
r = cursor_close(rdb_checksums, cursor);
RET_ENDUPDATE(result, r);
return result;
}
static inline retvalue checkorimprove(const char *filekey, struct checksums **checksums_p) {
const struct checksums *checksums = *checksums_p;
struct checksums *indatabase;
bool improves;
retvalue r;
r = files_get_checksums(filekey, &indatabase);
if (r == RET_NOTHING) {
fprintf(stderr, "Missing file %s\n", filekey);
return RET_ERROR_MISSING;
}
if (RET_WAS_ERROR(r))
return r;
if (!checksums_check(checksums, indatabase, &improves)) {
fprintf(stderr,
"File \"%s\" is already registered with different checksums!\n",
filekey);
checksums_printdifferences(stderr, indatabase, checksums);
r = RET_ERROR_WRONG_MD5;
} else if (improves) {
r = checksums_combine(checksums_p, indatabase, NULL);
} else
r = RET_NOTHING;
checksums_free(indatabase);
return r;
}
retvalue files_checkpool(bool fast) {
retvalue result, r;
struct cursor *cursor;
const char *filekey, *combined;
size_t combinedlen;
struct checksums *expected;
char *fullfilename;
bool improveable = false;
result = RET_NOTHING;
r = table_newglobalcursor(rdb_checksums, &cursor);
if (!RET_IS_OK(r))
return r;
while (cursor_nexttempdata(rdb_checksums, cursor,
&filekey, &combined, &combinedlen)) {
r = checksums_setall(&expected, combined, combinedlen);
if (RET_WAS_ERROR(r)) {
RET_UPDATE(result, r);
continue;
}
fullfilename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(fullfilename)) {
result = RET_ERROR_OOM;
checksums_free(expected);
break;
}
if (fast)
r = checksums_cheaptest(fullfilename, expected, true);
else
r = checkpoolfile(fullfilename, expected, &improveable);
if (r == RET_NOTHING) {
fprintf(stderr, "Missing file '%s'!\n", fullfilename);
r = RET_ERROR_MISSING;
}
free(fullfilename);
checksums_free(expected);
RET_UPDATE(result, r);
}
r = cursor_close(rdb_checksums, cursor);
RET_ENDUPDATE(result, r);
if (improveable && verbose >= 0)
printf(
"There were files with only some of the checksums this version of reprepro\n"
"can compute recorded. To add those run reprepro collectnewchecksums.\n");
return result;
}
static inline retvalue add_history(const char *diffindexfile, struct diffindex *n, const struct strlist *history) {
int i, j;
for (i = 0 ; i < history->count ; i++) {
struct hashes hashes;
const char *patchname;
struct checksums *checksums;
retvalue r;
parse_sha1line(history->values[i], &hashes, &patchname);
if (hashes.hashes[cs_sha1sum].len == 0
|| hashes.hashes[cs_length].len == 0
|| *patchname == '\0') {
r = RET_ERROR;
} else
r = checksums_initialize(&checksums,
hashes.hashes);
ASSERT_NOT_NOTHING(r);
if (RET_WAS_ERROR(r)) {
fprintf(stderr,
"Error parsing SHA1-History line %d in '%s':!\n'%s'\n",
i, diffindexfile, history->values[i]);
return r;
}
j = 0;
while (j < n->patchcount && strcmp(n->patches[j].name,
patchname) != 0)
j++;
if (j >= n->patchcount) {
fprintf(stderr,
"'%s' lists '%s' in history but not in patches!\n",
diffindexfile, patchname);
checksums_free(checksums);
continue;
}
if (n->patches[j].frompackages != NULL) {
fprintf(stderr,
"Warning: '%s' lists multiple histories for '%s'!\nOnly using last one!\n",
diffindexfile, patchname);
checksums_free(n->patches[j].frompackages);
}
n->patches[j].frompackages = checksums;
}
return RET_OK;
}
off_t files_getsize(const char *filekey) {
retvalue r;
off_t s;
struct checksums *checksums;
r = files_get_checksums(filekey, &checksums);
if (!RET_IS_OK(r))
return -1;
s = checksums_getfilesize(checksums);
checksums_free(checksums);
return s;
}
/* check for file in the database and if not found there, if it can be detected */
retvalue files_expect(const char *filekey, const struct checksums *checksums, bool warnifadded) {
retvalue r;
char *filename;
struct checksums *improvedchecksums = NULL;
r = files_canadd(filekey, checksums);
if (r == RET_NOTHING)
return RET_OK;
if (RET_WAS_ERROR(r))
return r;
/* ready to add means missing, so have to look for the file itself: */
filename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(filename))
return RET_ERROR_OOM;
/* first check if a possible manually put (or left over from previous
* downloads attepts) file is there and is correct */
r = checksums_test(filename, checksums, &improvedchecksums);
if (r == RET_ERROR_WRONG_MD5) {
fprintf(stderr,
"Deleting unexpected file '%s'!\n"
"(not in database and wrong in pool)\n ",
filename);
if (unlink(filename) == 0)
r = RET_NOTHING;
else {
int e = errno;
fprintf(stderr,
"Error %d deleting '%s': %s!\n", e, filename, strerror(e));
}
}
free(filename);
if (!RET_IS_OK(r))
return r;
if (warnifadded)
fprintf(stderr,
"Warning: readded existing file '%s' mysteriously missing from the checksum database.\n",
filekey);
// TODO: some callers might want the updated checksum when
// improves is true, how to get them there?
/* add found file to database */
if (improvedchecksums != NULL) {
r = files_add_checksums(filekey, improvedchecksums);
checksums_free(improvedchecksums);
} else
r = files_add_checksums(filekey, checksums);
assert (r != RET_NOTHING);
return r;
}
static retvalue checkpoolfile(const char *fullfilename, const struct checksums *expected, bool *improveable) {
struct checksums *actual;
retvalue r;
bool improves;
r = checksums_read(fullfilename, &actual);
if (RET_IS_OK(r)) {
if (!checksums_check(expected, actual, &improves)) {
fprintf(stderr, "WRONG CHECKSUMS of '%s':\n",
fullfilename);
checksums_printdifferences(stderr, expected, actual);
r = RET_ERROR_WRONG_MD5;
} else if (improves)
*improveable = true;
checksums_free(actual);
}
return r;
}
retvalue files_detect(const char *filekey) {
struct checksums *checksums;
char *fullfilename;
retvalue r;
fullfilename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(fullfilename))
return RET_ERROR_OOM;
r = checksums_read(fullfilename, &checksums);
if (r == RET_NOTHING) {
fprintf(stderr, "Error opening '%s'!\n", fullfilename);
r = RET_ERROR_MISSING;
}
if (RET_WAS_ERROR(r)) {
free(fullfilename);
return r;
}
free(fullfilename);
r = files_add_checksums(filekey, checksums);
checksums_free(checksums);
return r;
}
retvalue files_checkincludefile(const char *sourcedir, const char *basefilename, const char *filekey, struct checksums **checksums_p) {
char *sourcefilename, *fullfilename;
struct checksums *checksums;
retvalue r;
bool improves;
assert (*checksums_p != NULL);
r = files_get_checksums(filekey, &checksums);
if (RET_WAS_ERROR(r))
return r;
if (RET_IS_OK(r)) {
/* there are three sources now:
* - the checksums from the database (may have some we
* do not even know about, and may miss some we can
* generate)
* - the checksums provided (typically only md5sum,
* as this comes from a .changes or .dsc)
* - the checksums of the file
*
* to make things more complicated, the file should only
* be read if needed, as this needs time.
* And it can happen the file got lost in the pool, then
* this is the best place to replace it.
*/
if (!checksums_check(checksums, *checksums_p, &improves)) {
fprintf(stderr,
"ERROR: '%s/%s' cannot be included as '%s'.\n"
"Already existing files can only be included again, if they are the same, but:\n",
sourcedir, basefilename, filekey);
checksums_printdifferences(stderr, checksums,
*checksums_p);
checksums_free(checksums);
return RET_ERROR_WRONG_MD5;
}
r = RET_NOTHING;
if (improves)
r = checksums_combine(&checksums, *checksums_p, NULL);
if (!RET_WAS_ERROR(r))
r = checkimproveorinclude(sourcedir,
basefilename, filekey, &checksums, &improves);
if (!RET_WAS_ERROR(r) && improves)
r = files_replace_checksums(filekey, checksums);
if (RET_IS_OK(r))
r = RET_NOTHING;
/* return the combined checksum */
checksums_free(*checksums_p);
*checksums_p = checksums;
return r;
}
assert (sourcedir != NULL);
sourcefilename = calc_dirconcat(sourcedir, basefilename);
if (FAILEDTOALLOC(sourcefilename))
return RET_ERROR_OOM;
fullfilename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(fullfilename)) {
free(sourcefilename);
return RET_ERROR_OOM;
}
(void)dirs_make_parent(fullfilename);
r = checksums_copyfile(fullfilename, sourcefilename, true, &checksums);
if (r == RET_NOTHING) {
fprintf(stderr, "Could not open '%s'!\n", sourcefilename);
r = RET_ERROR_MISSING;
}
if (RET_WAS_ERROR(r)) {
free(fullfilename);
free(sourcefilename);
return r;
}
if (!checksums_check(*checksums_p, checksums, &improves)) {
deletefile(fullfilename);
fprintf(stderr, "ERROR: Unexpected content of file '%s'!\n",
sourcefilename);
checksums_printdifferences(stderr, *checksums_p, checksums);
r = RET_ERROR_WRONG_MD5;
}
free(sourcefilename);
free(fullfilename);
if (RET_WAS_ERROR(r)) {
return r;
}
if (improves) {
r = checksums_combine(checksums_p, checksums, NULL);
checksums_free(checksums);
if (RET_WAS_ERROR(r))
return r;
} else
checksums_free(checksums);
return files_add_checksums(filekey, *checksums_p);
}
static retvalue checkimproveorinclude(const char *sourcedir, const char *basefilename, const char *filekey, struct checksums **checksums_p, bool *improving) {
retvalue r;
struct checksums *checksums = NULL;
bool improves, copied = false;
char *fullfilename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(fullfilename))
return RET_ERROR_OOM;
if (checksums_iscomplete(*checksums_p)) {
r = checksums_cheaptest(fullfilename, *checksums_p, true);
if (r != RET_NOTHING) {
free(fullfilename);
return r;
}
} else {
r = checksums_read(fullfilename, &checksums);
if (RET_WAS_ERROR(r)) {
free(fullfilename);
return r;
}
}
if (r == RET_NOTHING) {
char *sourcefilename = calc_dirconcat(sourcedir, basefilename);
if (FAILEDTOALLOC(sourcefilename)) {
free(fullfilename);
return RET_ERROR_OOM;
}
fprintf(stderr,
"WARNING: file %s was lost!\n"
"(i.e. found in the database, but not in the pool)\n"
"trying to compensate...\n",
filekey);
(void)dirs_make_parent(fullfilename);
r = checksums_copyfile(fullfilename, sourcefilename, false,
&checksums);
if (r == RET_ERROR_EXIST) {
fprintf(stderr,
"File '%s' seems to be missing and existing at the same time!\n"
"To confused to continue...\n",
fullfilename);
}
if (r == RET_NOTHING) {
fprintf(stderr, "Could not open '%s'!\n",
sourcefilename);
r = RET_ERROR_MISSING;
}
free(sourcefilename);
if (RET_WAS_ERROR(r)) {
free(fullfilename);
return r;
}
copied = true;
}
assert (checksums != NULL);
if (!checksums_check(*checksums_p, checksums, &improves)) {
if (copied) {
deletefile(fullfilename);
fprintf(stderr,
"ERROR: Unexpected content of file '%s/%s'!\n", sourcedir, basefilename);
} else
// TODO: if the database only listed some of the currently supported checksums,
// and the caller of checkincludefile supplied some (which none yet does), but
// not all (which needs at least three checksums, i.e. not applicaple before
// sha256 get added), then this might also be called if the file in the pool
// just has the same checksums as previously recorded (e.g. a md5sum collision)
// but the new file was listed with another secondary hash than the original.
// In that situation it might be a bit misleading...
fprintf(stderr,
"ERROR: file %s is damaged!\n"
"(i.e. found in the database, but with different checksums in the pool)\n",
filekey);
checksums_printdifferences(stderr, *checksums_p, checksums);
r = RET_ERROR_WRONG_MD5;
}
if (improves) {
r = checksums_combine(checksums_p, checksums, NULL);
if (RET_IS_OK(r))
*improving = true;
}
checksums_free(checksums);
free(fullfilename);
return r;
}
/* Include a yet unknown file into the pool */
retvalue files_preinclude(const char *sourcefilename, const char *filekey, struct checksums **checksums_p) {
retvalue r;
struct checksums *checksums, *realchecksums;
bool improves;
char *fullfilename;
r = files_get_checksums(filekey, &checksums);
if (RET_WAS_ERROR(r))
return r;
if (RET_IS_OK(r)) {
r = checksums_read(sourcefilename, &realchecksums);
if (r == RET_NOTHING)
r = RET_ERROR_MISSING;
if (RET_WAS_ERROR(r)) {
checksums_free(checksums);
return r;
}
if (!checksums_check(checksums, realchecksums, &improves)) {
fprintf(stderr,
"ERROR: '%s' cannot be included as '%s'.\n"
"Already existing files can only be included again, if they are the same, but:\n",
sourcefilename, filekey);
checksums_printdifferences(stderr, checksums,
realchecksums);
checksums_free(checksums);
checksums_free(realchecksums);
return RET_ERROR_WRONG_MD5;
}
if (improves) {
r = checksums_combine(&checksums, realchecksums, NULL);
if (RET_WAS_ERROR(r)) {
checksums_free(realchecksums);
checksums_free(checksums);
return r;
}
r = files_replace_checksums(filekey, checksums);
if (RET_WAS_ERROR(r)) {
checksums_free(realchecksums);
checksums_free(checksums);
return r;
}
}
checksums_free(realchecksums);
// args, this breaks retvalue semantics!
if (checksums_p != NULL)
*checksums_p = checksums;
else
checksums_free(checksums);
return RET_NOTHING;
}
assert (sourcefilename != NULL);
fullfilename = files_calcfullfilename(filekey);
if (FAILEDTOALLOC(fullfilename))
return RET_ERROR_OOM;
(void)dirs_make_parent(fullfilename);
r = checksums_copyfile(fullfilename, sourcefilename, true, &checksums);
if (r == RET_ERROR_EXIST) {
// TODO: deal with already existing files!
fprintf(stderr, "File '%s' does already exist!\n",
fullfilename);
}
if (r == RET_NOTHING) {
fprintf(stderr, "Could not open '%s'!\n", sourcefilename);
r = RET_ERROR_MISSING;
}
if (RET_WAS_ERROR(r)) {
free(fullfilename);
return r;
}
free(fullfilename);
r = files_add_checksums(filekey, checksums);
if (RET_WAS_ERROR(r)) {
checksums_free(checksums);
return r;
}
if (checksums_p != NULL)
*checksums_p = checksums;
else
checksums_free(checksums);
return RET_OK;
}
/* look where a received file has to go to: */
static retvalue uridone(struct aptmethod *method, const char *uri, const char *filename, /*@only@*//*@null@*/struct checksums *checksumsfromapt) {
struct tobedone *todo, *lasttodo;
retvalue r;
lasttodo = NULL; todo = method->tobedone;
while (todo != NULL) {
bool expectduplicates = false;
if (strcmp(todo->uri, uri) != 0) {
lasttodo = todo;
todo = todo->next;
continue;
}
if (todo->ignore) {
assert (method->old103);
assert (todo->redirected_filename != NULL);
r = RET_NOTHING;
} else {
/* This detection (by requesting the redirected file to
* another filename and comparing here is a ugly hack
* (assuming the method will process the old one before
* the new one and causing an unnecessary copy with the
* new behaviour...., but there is no way to destinguish
* those methods).*/
if (todo->redirected_filename != NULL &&
strcmp(filename, todo->filename) == 0) {
/* this looks like we met a method that gives 103
* but still downloads the file. remember this */
if (!method->old103)
fprintf(stderr,
"aptmethod '%s' seems to have a obsoleted redirect handling which causes\n"
"reprepro to request files multiple times. Work-around activated, but better\n"
"only use it for targets not redirecting (or upgrade to apt >= 0.9.4 if\n"
"that is the http method from apt)!\n", method->name);
method->old103 = true;
method->new103 = false;
expectduplicates = true;
} else if (!method->old103 &&
todo->redirected_filename != NULL &&
strcmp(filename, todo->redirected_filename) == 0) {
/* nothing hints for a old 103 handling, and the redirected
* file was gotten before any redirected was, so assume
* this is the new style */
method->new103 = true;
}
r = todo->callback(qa_got,
todo->privdata1, todo->privdata2,
todo->uri, filename, todo->filename,
checksumsfromapt, method->name);
}
checksums_free(checksumsfromapt);
if (todo->redirected_filename != NULL
&& strcmp(filename, todo->redirected_filename) == 0)
unlink(todo->redirected_filename);
if (expectduplicates) {
todo->ignore = true;
} else {
/* remove item: */
if (lasttodo == NULL)
method->tobedone = todo->next;
else
lasttodo->next = todo->next;
if (method->nexttosend == todo) {
/* just in case some method received
* files before we request them ;-) */
method->nexttosend = todo->next;
}
if (method->lasttobedone == todo) {
method->lasttobedone = todo->next;
}
todo_free(todo);
}
return r;
}
/* huh? */
fprintf(stderr,
"Method '%s' retrieved unexpected file '%s' at '%s'!\n",
method->name, uri, filename);
checksums_free(checksumsfromapt);
return RET_ERROR;
}
