/* * Open a pkg_summary and if newer than local return an open libfetch * connection to it. */ Sumfile * sum_open(char *str_url, time_t *db_mtime) { Sumfile *sum = NULL; fetchIO *f = NULL; struct url *url; struct url_stat st; url = fetchParseURL(str_url); if (url == NULL || (f = fetchXGet(url, &st, "")) == NULL) goto nofetch; if (st.size == -1) { /* could not obtain file size */ *db_mtime = 0; /* not -1, don't force update */ goto nofetch; } if (st.mtime <= *db_mtime) { /* * -1 used to identify return type, * local summary up-to-date */ *db_mtime = -1; goto nofetch; } *db_mtime = st.mtime; #ifndef _MINIX /* XXX: SSIZE_MAX fails under MINIX */ /* st.size is an off_t, it will be > SSIZE_MAX on 32 bits systems */ if (sizeof(st.size) == sizeof(SSIZE_MAX) && st.size > SSIZE_MAX - 1) err(EXIT_FAILURE, "file is too large"); #endif XMALLOC(sum, sizeof(Sumfile)); sum->fd = f; sum->url = url; sum->size = st.size; goto out; nofetch: if (url) fetchFreeURL(url); if (f) fetchIO_close(f); out: return sum; }
int main(int argc, char **argv) { struct db_connection dbc; struct utsname u; struct url *uo; char *url; if (dbc_open_fetchlist(&dbc)) exit(EXIT_FAILURE); if (uname(&u)) { perror("uname"); dbc_abort(&dbc); exit(EXIT_FAILURE); } /* Build patch directory URL. */ if (asprintf(&url, "%s%s/%s/%s", PATCHSITE_URL, u.machine, u.release, PATCHLIST_NAME) < 0) { perror("asprintf"); dbc_abort(&dbc); exit(EXIT_FAILURE); } /* Fetch the directory list. */ uo = fetchParseURL(url); if (!uo) { fprintf(stderr, "Invalid URL: %s", url); free(url); dbc_abort(&dbc); exit(EXIT_FAILURE); } fio = fetchGet(uo, ""); /* Reading fio is done by yacc. */ yyparse(); fetchIO_close(fio); fetchFreeURL(uo); free(url); dbc_done(&dbc); exit(EXIT_SUCCESS); }
/* * archive_read_open close callback. Stop the progress meter and close the * libfetch handler. */ int sum_close(struct archive *a, void *data) { Sumfile *sum = data; if (!parsable) stop_progress_meter(); else printf(MSG_DOWNLOAD_END); fetchIO_close(sum->fd); fetchFreeURL(sum->url); XFREE(sum); return ARCHIVE_OK; }
/* if db_mtime == NULL, we're downloading a package, pkg_summary otherwise */ Dlfile * download_file(char *str_url, time_t *db_mtime) { /* from pkg_install/files/admin/audit.c */ Dlfile *file; char *p; size_t buf_len, buf_fetched; ssize_t cur_fetched; off_t statsize; time_t begin_dl, now; struct url_stat st; struct url *url; fetchIO *f = NULL; url = fetchParseURL(str_url); if (url == NULL || (f = fetchXGet(url, &st, "")) == NULL) return NULL; if (st.size == -1) { /* could not obtain file size */ if (db_mtime != NULL) /* we're downloading pkg_summary */ *db_mtime = 0; /* not -1, don't force update */ return NULL; } if (db_mtime != NULL) { if (st.mtime <= *db_mtime) { /* -1 used to identify return type, local summary up-to-date */ *db_mtime = -1; fetchIO_close(f); return NULL; } *db_mtime = st.mtime; } if ((p = strrchr(str_url, '/')) != NULL) p++; else p = (char *)str_url; /* should not happen */ #ifndef _MINIX /* XXX: SSIZE_MAX fails under MINIX */ /* st.size is an off_t, it will be > SSIZE_MAX on 32 bits systems */ if (sizeof(st.size) == sizeof(SSIZE_MAX) && st.size > SSIZE_MAX - 1) err(EXIT_FAILURE, "file is too large"); #endif buf_len = st.size; XMALLOC(file, sizeof(Dlfile)); XMALLOC(file->buf, buf_len + 1); printf(MSG_DOWNLOADING, p); fflush(stdout); buf_fetched = 0; begin_dl = time(NULL); statsize = 0; start_progress_meter(p, buf_len, &statsize); while (buf_fetched < buf_len) { cur_fetched = fetchIO_read(f, file->buf + buf_fetched, fetch_buffer); if (cur_fetched == 0) errx(EXIT_FAILURE, "truncated file"); else if (cur_fetched == -1) errx(EXIT_FAILURE, "failure during fetch of file: %s", fetchLastErrString); buf_fetched += cur_fetched; statsize += cur_fetched; now = time(NULL); } stop_progress_meter(); file->buf[buf_len] = '\0'; file->size = buf_len; if (file->buf[0] == '\0') errx(EXIT_FAILURE, "empty download, exiting.\n"); fetchIO_close(f); return file; }
/* * Download a package to the local cache. */ ssize_t download_pkg(char *pkg_url, FILE *fp) { struct url_stat st; size_t size, wrote; ssize_t fetched, written = 0; off_t statsize = 0; struct url *url; fetchIO *f = NULL; char buf[4096]; char *pkg, *ptr; if ((url = fetchParseURL(pkg_url)) == NULL) errx(EXIT_FAILURE, "%s: parse failure", pkg_url); if ((f = fetchXGet(url, &st, "")) == NULL) errx(EXIT_FAILURE, "%s: %s", pkg_url, fetchLastErrString); /* Package not available */ if (st.size == -1) return st.size; if ((pkg = strrchr(pkg_url, '/')) != NULL) pkg++; else pkg = (char *)pkg_url; /* should not happen */ if (parsable) { printf(MSG_DOWNLOAD_START); } else { printf(MSG_DOWNLOADING, pkg); fflush(stdout); start_progress_meter(pkg, st.size, &statsize); } while (written < st.size) { if ((fetched = fetchIO_read(f, buf, sizeof(buf))) == 0) break; if (fetched == -1 && errno == EINTR) continue; if (fetched == -1) errx(EXIT_FAILURE, "fetch failure: %s", fetchLastErrString); statsize += fetched; size = fetched; for (ptr = buf; size > 0; ptr += wrote, size -= wrote) { if ((wrote = fwrite(ptr, 1, size, fp)) < size) { if (ferror(fp) && errno == EINTR) clearerr(fp); else break; } written += wrote; } } if (parsable) printf(MSG_DOWNLOAD_END); else stop_progress_meter(); fetchIO_close(f); fetchFreeURL(url); if (written != st.size) return -1; return written; }
/* * Fetch a file */ static int fetch(char *URL, const char *path) { struct url *url; struct url_stat us; struct stat sb, nsb; struct xferstat xs; FILE *of; fetchIO *f; size_t size, wr; ssize_t ssize; off_t count; char flags[8]; char *tmppath; int r; unsigned timeout; char *ptr; f = NULL; of = NULL; tmppath = NULL; timeout = 0; *flags = 0; count = 0; /* set verbosity level */ if (v_level > 1) strcat(flags, "v"); if (v_level > 2) fetchDebug = 1; /* parse URL */ if ((url = fetchParseURL(URL)) == NULL) { warnx("%s: parse error", URL); goto failure; } /* if no scheme was specified, take a guess */ if (!*url->scheme) { if (!*url->host) strcpy(url->scheme, SCHEME_FILE); else if (strncasecmp(url->host, "ftp.", 4) == 0) strcpy(url->scheme, SCHEME_FTP); else if (strncasecmp(url->host, "www.", 4) == 0) strcpy(url->scheme, SCHEME_HTTP); } /* common flags */ switch (family) { case PF_INET: strcat(flags, "4"); break; #ifndef __minix case PF_INET6: strcat(flags, "6"); break; #endif } /* Protocol independent flags */ if (i_flag) { if (stat(path, &sb) == 0) { url->last_modified = sb.st_mtime; strcat(flags, "i"); } else if (errno != ENOENT) { warn("%s: stat()", path); goto failure; } } /* FTP specific flags */ if (strcmp(url->scheme, SCHEME_FTP) == 0) { if (d_flag) strcat(flags, "d"); if (U_flag) strcat(flags, "l"); timeout = T_secs ? T_secs : ftp_timeout; } /* HTTP specific flags */ if (strcmp(url->scheme, SCHEME_HTTP) == 0) { if (d_flag) strcat(flags, "d"); if (A_flag) strcat(flags, "A"); timeout = T_secs ? T_secs : http_timeout; } /* set the protocol timeout. */ fetchTimeout = timeout; /* just print size */ if (s_flag) { if (timeout) alarm(timeout); r = fetchStat(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (r == -1) { warnx("%s", fetchLastErrString); goto failure; } if (us.size == -1) printf("Unknown\n"); else printf("%jd\n", (intmax_t)us.size); goto success; } /* * If the -r flag was specified, we have to compare the local * and remote files, so we should really do a fetchStat() * first, but I know of at least one HTTP server that only * sends the content size in response to GET requests, and * leaves it out of replies to HEAD requests. Also, in the * (frequent) case that the local and remote files match but * the local file is truncated, we have sufficient information * before the compare to issue a correct request. Therefore, * we always issue a GET request as if we were sure the local * file was a truncated copy of the remote file; we can drop * the connection later if we change our minds. */ sb.st_size = -1; if (!o_stdout) { r = stat(path, &sb); if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { url->offset = sb.st_size; } else if (r == -1 || !S_ISREG(sb.st_mode)) { /* * Whatever value sb.st_size has now is either * wrong (if stat(2) failed) or irrelevant (if the * path does not refer to a regular file) */ sb.st_size = -1; } if (r == -1 && errno != ENOENT) { warnx("%s: stat()", path); goto failure; } } /* start the transfer */ if (timeout) alarm(timeout); f = fetchXGet(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (f == NULL && i_flag && fetchLastErrCode == FETCH_UNCHANGED) { /* URL was not modified, return OK. */ printf("%s: not modified\n", URL); r = 0; goto done; } if (f == NULL) { warnx("%s: %s", URL, fetchLastErrString); goto failure; } if (sigint) goto signal; /* check that size is as expected */ if (S_size) { if (us.size == -1) { warnx("%s: size unknown", URL); } else if (us.size != S_size) { warnx("%s: size mismatch: expected %jd, actual %jd", URL, (intmax_t)S_size, (intmax_t)us.size); goto failure; } } /* symlink instead of copy */ if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { char *name = fetchUnquotePath(url); if (name == NULL) { warnx("Can't unquote URL"); goto failure; } if (symlink(name, path) == -1) { warn("%s: symlink()", path); free(name); goto failure; } free(name); goto success; } if (us.size == -1 && !o_stdout && v_level > 0) warnx("%s: size of remote file is not known", URL); if (v_level > 1) { if (sb.st_size != -1) fprintf(stderr, "local size / mtime: %jd / %ld\n", (intmax_t)sb.st_size, (long)sb.st_mtime); if (us.size != -1) fprintf(stderr, "remote size / mtime: %jd / %ld\n", (intmax_t)us.size, (long)us.mtime); } /* open output file */ if (o_stdout) { /* output to stdout */ of = stdout; } else if (r_flag && sb.st_size != -1) { /* resume mode, local file exists */ if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { /* no match! have to refetch */ fetchIO_close(f); /* if precious, warn the user and give up */ if (R_flag) { warnx("%s: local modification time " "does not match remote", path); goto failure_keep; } } else if (us.size != -1) { if (us.size == sb.st_size) /* nothing to do */ goto success; if (sb.st_size > us.size) { /* local file too long! */ warnx("%s: local file (%jd bytes) is longer " "than remote file (%jd bytes)", path, (intmax_t)sb.st_size, (intmax_t)us.size); goto failure; } /* we got it, open local file */ if ((of = fopen(path, "a")) == NULL) { warn("%s: fopen()", path); goto failure; } /* check that it didn't move under our feet */ if (fstat(fileno(of), &nsb) == -1) { /* can't happen! */ warn("%s: fstat()", path); goto failure; } if (nsb.st_dev != sb.st_dev || nsb.st_ino != nsb.st_ino || nsb.st_size != sb.st_size) { warnx("%s: file has changed", URL); fclose(of); of = NULL; sb = nsb; } } } else if (m_flag && sb.st_size != -1) { /* mirror mode, local file exists */ if (sb.st_size == us.size && sb.st_mtime == us.mtime) goto success; } if (of == NULL) { /* * We don't yet have an output file; either this is a * vanilla run with no special flags, or the local and * remote files didn't match. */ if (url->offset > 0) { /* * We tried to restart a transfer, but for * some reason gave up - so we have to restart * from scratch if we want the whole file */ url->offset = 0; if ((f = fetchXGet(url, &us, flags)) == NULL) { warnx("%s: %s", URL, fetchLastErrString); goto failure; } if (sigint) goto signal; } /* construct a temp file name */ if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { #ifndef __minix asprintf(&tmppath, "%s.fetch.XXXXXX", path); #else { int len; if((tmppath = malloc(sizeof(char)*MINBUFSIZE)) != NULL) { len = snprintf(tmppath, MINBUFSIZE, "%s.fetch.XXXXXX", path); if(len >= MINBUFSIZE) { free(tmppath); tmppath = NULL; } } } #endif if (tmppath != NULL) { int fd; fd = mkstemp(tmppath); if (fd == -1) { warn("%s: mkstemp failed", tmppath); goto failure; } fchown(fd, sb.st_uid, sb.st_gid); fchmod(fd, sb.st_mode & ALLPERMS); of = fdopen(fd, "w"); if (of == NULL) { close(fd); unlink(tmppath); free(tmppath); tmppath = NULL; } } } if (of == NULL) of = fopen(path, "w"); if (of == NULL) { warn("%s: open()", path); goto failure; } } count = url->offset; /* start the counter */ stat_start(&xs, path, us.size, count); sigalrm = sigint = 0; /* suck in the data */ #ifdef SIGINFO siginfo = 0; signal(SIGINFO, sig_handler); #endif while (!sigint) { if (us.size != -1 && us.size - count < B_size && us.size - count >= 0) size = us.size - count; else size = B_size; #ifdef SIGINFO if (siginfo) { stat_display(&xs, 1); siginfo = 0; } #else /* Constant info is better than none. */ if (v_level) { stat_display(&xs, 1); } #endif if ((ssize = fetchIO_read(f, buf, B_size)) == 0) break; if (ssize == -1 && errno == EINTR) continue; if (ssize == -1) break; size = ssize; stat_update(&xs, count += size); for (ptr = buf; size > 0; ptr += wr, size -= wr) { if ((wr = fwrite(ptr, 1, size, of)) < size) { if (ferror(of) && errno == EINTR && !sigint) clearerr(of); else break; } } if (size != 0) break; } if (!sigalrm) sigalrm = 0; #ifdef SIGINFO signal(SIGINFO, SIG_DFL); #endif stat_end(&xs); /* * If the transfer timed out or was interrupted, we still want to * set the mtime in case the file is not removed (-r or -R) and * the user later restarts the transfer. */ signal: /* set mtime of local file */ if (!n_flag && us.mtime && !o_stdout && of != NULL && (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { struct timeval tv[2]; fflush(of); tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); tv[1].tv_sec = (long)us.mtime; tv[0].tv_usec = tv[1].tv_usec = 0; if (utimes(tmppath ? tmppath : path, tv)) warn("%s: utimes()", tmppath ? tmppath : path); } /* timed out or interrupted? */ if (fetchLastErrCode == FETCH_TIMEOUT) sigalrm = 1; if (sigalrm) warnx("transfer timed out"); if (sigint) { warnx("transfer interrupted"); goto failure; } /* timeout / interrupt before connection completley established? */ if (f == NULL) goto failure; if (!sigalrm && ferror(of)) { /* check the status of our files */ warn("writing to %s failed", path); goto failure; } /* did the transfer complete normally? */ if (us.size != -1 && count < us.size) { warnx("%s appears to be truncated: %jd/%jd bytes", path, (intmax_t)count, (intmax_t)us.size); goto failure_keep; } /* * If the transfer timed out and we didn't know how much to * expect, assume the worst (i.e. we didn't get all of it) */ if (sigalrm && us.size == -1) { warnx("%s may be truncated", path); goto failure_keep; } success: r = 0; if (tmppath != NULL && rename(tmppath, path) == -1) { warn("%s: rename()", path); goto failure_keep; } goto done; failure: if (of && of != stdout && !R_flag && !r_flag) if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) unlink(tmppath ? tmppath : path); if (R_flag && tmppath != NULL && sb.st_size == -1) rename(tmppath, path); /* ignore errors here */ failure_keep: r = -1; goto done; done: if (f) fetchIO_close(f); if (of && of != stdout) fclose(of); if (url) fetchFreeURL(url); if (tmppath != NULL) free(tmppath); return (r); }