int fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const char *flags) { char *path; struct dirent *de; DIR *dir; if ((path = fetchUnquotePath(u)) == NULL) { fetch_syserr(); return -1; } dir = opendir(path); free(path); if (dir == NULL) { fetch_syserr(); return -1; } while ((de = readdir(dir)) != NULL) { if (pattern && fnmatch(pattern, de->d_name, 0) != 0) continue; fetch_add_entry(ue, u, de->d_name, 0); } closedir(dir); return 0; }
fetchIO * fetchXGetFile(struct url *u, struct url_stat *us, const char *flags) { char *path; fetchIO *f; struct url_stat local_us; int if_modified_since, fd, *cookie; if_modified_since = CHECK_FLAG('i'); if (if_modified_since && us == NULL) us = &local_us; if ((path = fetchUnquotePath(u)) == NULL) { fetch_syserr(); return NULL; } fd = open(path, O_RDONLY); free(path); if (fd == -1) { fetch_syserr(); return NULL; } if (us && fetch_stat_file(fd, us) == -1) { close(fd); fetch_syserr(); return NULL; } if (if_modified_since && u->last_modified > 0 && u->last_modified >= us->mtime) { close(fd); fetchLastErrCode = FETCH_UNCHANGED; snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged"); return NULL; } if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) { close(fd); fetch_syserr(); return NULL; } cookie = malloc(sizeof(int)); if (cookie == NULL) { close(fd); fetch_syserr(); return NULL; } *cookie = fd; f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close); if (f == NULL) { close(fd); free(cookie); } return f; }
/* * Extract the file name component of a URL. */ char * fetchUnquoteFilename(struct url *url) { char *unquoted, *filename; const char *last_slash; if ((unquoted = fetchUnquotePath(url)) == NULL) return NULL; if ((last_slash = strrchr(unquoted, '/')) == NULL) return unquoted; filename = strdup(last_slash + 1); free(unquoted); return filename; }
fetchIO * fetchPutFile(struct url *u, const char *flags) { char *path; fetchIO *f; int fd, *cookie; if ((path = fetchUnquotePath(u)) == NULL) { fetch_syserr(); return NULL; } if (CHECK_FLAG('a')) fd = open(path, O_WRONLY | O_APPEND); else fd = open(path, O_WRONLY); free(path); if (fd == -1) { fetch_syserr(); return NULL; } if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) { close(fd); fetch_syserr(); return NULL; } cookie = malloc(sizeof(int)); if (cookie == NULL) { close(fd); fetch_syserr(); return NULL; } *cookie = fd; f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close); if (f == NULL) { close(fd); free(cookie); } return f; }
int fetchStatFile(struct url *u, struct url_stat *us, const char *flags) { char *path; int fd, rv; if ((path = fetchUnquotePath(u)) == NULL) { fetch_syserr(); return -1; } fd = open(path, O_RDONLY); free(path); if (fd == -1) { fetch_syserr(); return -1; } rv = fetch_stat_file(fd, us); close(fd); return rv; }
/* * Fetch a file */ static int fetch(char *URL, const char *path) { struct url *url; struct url_stat us; struct stat sb, nsb; struct xferstat xs; FILE *of; fetchIO *f; size_t size, wr; ssize_t ssize; off_t count; char flags[8]; char *tmppath; int r; unsigned timeout; char *ptr; f = NULL; of = NULL; tmppath = NULL; timeout = 0; *flags = 0; count = 0; /* set verbosity level */ if (v_level > 1) strcat(flags, "v"); if (v_level > 2) fetchDebug = 1; /* parse URL */ if ((url = fetchParseURL(URL)) == NULL) { warnx("%s: parse error", URL); goto failure; } /* if no scheme was specified, take a guess */ if (!*url->scheme) { if (!*url->host) strcpy(url->scheme, SCHEME_FILE); else if (strncasecmp(url->host, "ftp.", 4) == 0) strcpy(url->scheme, SCHEME_FTP); else if (strncasecmp(url->host, "www.", 4) == 0) strcpy(url->scheme, SCHEME_HTTP); } /* common flags */ switch (family) { case PF_INET: strcat(flags, "4"); break; #ifndef __minix case PF_INET6: strcat(flags, "6"); break; #endif } /* Protocol independent flags */ if (i_flag) { if (stat(path, &sb) == 0) { url->last_modified = sb.st_mtime; strcat(flags, "i"); } else if (errno != ENOENT) { warn("%s: stat()", path); goto failure; } } /* FTP specific flags */ if (strcmp(url->scheme, SCHEME_FTP) == 0) { if (d_flag) strcat(flags, "d"); if (U_flag) strcat(flags, "l"); timeout = T_secs ? T_secs : ftp_timeout; } /* HTTP specific flags */ if (strcmp(url->scheme, SCHEME_HTTP) == 0) { if (d_flag) strcat(flags, "d"); if (A_flag) strcat(flags, "A"); timeout = T_secs ? T_secs : http_timeout; } /* set the protocol timeout. */ fetchTimeout = timeout; /* just print size */ if (s_flag) { if (timeout) alarm(timeout); r = fetchStat(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (r == -1) { warnx("%s", fetchLastErrString); goto failure; } if (us.size == -1) printf("Unknown\n"); else printf("%jd\n", (intmax_t)us.size); goto success; } /* * If the -r flag was specified, we have to compare the local * and remote files, so we should really do a fetchStat() * first, but I know of at least one HTTP server that only * sends the content size in response to GET requests, and * leaves it out of replies to HEAD requests. Also, in the * (frequent) case that the local and remote files match but * the local file is truncated, we have sufficient information * before the compare to issue a correct request. Therefore, * we always issue a GET request as if we were sure the local * file was a truncated copy of the remote file; we can drop * the connection later if we change our minds. */ sb.st_size = -1; if (!o_stdout) { r = stat(path, &sb); if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { url->offset = sb.st_size; } else if (r == -1 || !S_ISREG(sb.st_mode)) { /* * Whatever value sb.st_size has now is either * wrong (if stat(2) failed) or irrelevant (if the * path does not refer to a regular file) */ sb.st_size = -1; } if (r == -1 && errno != ENOENT) { warnx("%s: stat()", path); goto failure; } } /* start the transfer */ if (timeout) alarm(timeout); f = fetchXGet(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (f == NULL && i_flag && fetchLastErrCode == FETCH_UNCHANGED) { /* URL was not modified, return OK. */ printf("%s: not modified\n", URL); r = 0; goto done; } if (f == NULL) { warnx("%s: %s", URL, fetchLastErrString); goto failure; } if (sigint) goto signal; /* check that size is as expected */ if (S_size) { if (us.size == -1) { warnx("%s: size unknown", URL); } else if (us.size != S_size) { warnx("%s: size mismatch: expected %jd, actual %jd", URL, (intmax_t)S_size, (intmax_t)us.size); goto failure; } } /* symlink instead of copy */ if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { char *name = fetchUnquotePath(url); if (name == NULL) { warnx("Can't unquote URL"); goto failure; } if (symlink(name, path) == -1) { warn("%s: symlink()", path); free(name); goto failure; } free(name); goto success; } if (us.size == -1 && !o_stdout && v_level > 0) warnx("%s: size of remote file is not known", URL); if (v_level > 1) { if (sb.st_size != -1) fprintf(stderr, "local size / mtime: %jd / %ld\n", (intmax_t)sb.st_size, (long)sb.st_mtime); if (us.size != -1) fprintf(stderr, "remote size / mtime: %jd / %ld\n", (intmax_t)us.size, (long)us.mtime); } /* open output file */ if (o_stdout) { /* output to stdout */ of = stdout; } else if (r_flag && sb.st_size != -1) { /* resume mode, local file exists */ if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { /* no match! have to refetch */ fetchIO_close(f); /* if precious, warn the user and give up */ if (R_flag) { warnx("%s: local modification time " "does not match remote", path); goto failure_keep; } } else if (us.size != -1) { if (us.size == sb.st_size) /* nothing to do */ goto success; if (sb.st_size > us.size) { /* local file too long! */ warnx("%s: local file (%jd bytes) is longer " "than remote file (%jd bytes)", path, (intmax_t)sb.st_size, (intmax_t)us.size); goto failure; } /* we got it, open local file */ if ((of = fopen(path, "a")) == NULL) { warn("%s: fopen()", path); goto failure; } /* check that it didn't move under our feet */ if (fstat(fileno(of), &nsb) == -1) { /* can't happen! */ warn("%s: fstat()", path); goto failure; } if (nsb.st_dev != sb.st_dev || nsb.st_ino != nsb.st_ino || nsb.st_size != sb.st_size) { warnx("%s: file has changed", URL); fclose(of); of = NULL; sb = nsb; } } } else if (m_flag && sb.st_size != -1) { /* mirror mode, local file exists */ if (sb.st_size == us.size && sb.st_mtime == us.mtime) goto success; } if (of == NULL) { /* * We don't yet have an output file; either this is a * vanilla run with no special flags, or the local and * remote files didn't match. */ if (url->offset > 0) { /* * We tried to restart a transfer, but for * some reason gave up - so we have to restart * from scratch if we want the whole file */ url->offset = 0; if ((f = fetchXGet(url, &us, flags)) == NULL) { warnx("%s: %s", URL, fetchLastErrString); goto failure; } if (sigint) goto signal; } /* construct a temp file name */ if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { #ifndef __minix asprintf(&tmppath, "%s.fetch.XXXXXX", path); #else { int len; if((tmppath = malloc(sizeof(char)*MINBUFSIZE)) != NULL) { len = snprintf(tmppath, MINBUFSIZE, "%s.fetch.XXXXXX", path); if(len >= MINBUFSIZE) { free(tmppath); tmppath = NULL; } } } #endif if (tmppath != NULL) { int fd; fd = mkstemp(tmppath); if (fd == -1) { warn("%s: mkstemp failed", tmppath); goto failure; } fchown(fd, sb.st_uid, sb.st_gid); fchmod(fd, sb.st_mode & ALLPERMS); of = fdopen(fd, "w"); if (of == NULL) { close(fd); unlink(tmppath); free(tmppath); tmppath = NULL; } } } if (of == NULL) of = fopen(path, "w"); if (of == NULL) { warn("%s: open()", path); goto failure; } } count = url->offset; /* start the counter */ stat_start(&xs, path, us.size, count); sigalrm = sigint = 0; /* suck in the data */ #ifdef SIGINFO siginfo = 0; signal(SIGINFO, sig_handler); #endif while (!sigint) { if (us.size != -1 && us.size - count < B_size && us.size - count >= 0) size = us.size - count; else size = B_size; #ifdef SIGINFO if (siginfo) { stat_display(&xs, 1); siginfo = 0; } #else /* Constant info is better than none. */ if (v_level) { stat_display(&xs, 1); } #endif if ((ssize = fetchIO_read(f, buf, B_size)) == 0) break; if (ssize == -1 && errno == EINTR) continue; if (ssize == -1) break; size = ssize; stat_update(&xs, count += size); for (ptr = buf; size > 0; ptr += wr, size -= wr) { if ((wr = fwrite(ptr, 1, size, of)) < size) { if (ferror(of) && errno == EINTR && !sigint) clearerr(of); else break; } } if (size != 0) break; } if (!sigalrm) sigalrm = 0; #ifdef SIGINFO signal(SIGINFO, SIG_DFL); #endif stat_end(&xs); /* * If the transfer timed out or was interrupted, we still want to * set the mtime in case the file is not removed (-r or -R) and * the user later restarts the transfer. */ signal: /* set mtime of local file */ if (!n_flag && us.mtime && !o_stdout && of != NULL && (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { struct timeval tv[2]; fflush(of); tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); tv[1].tv_sec = (long)us.mtime; tv[0].tv_usec = tv[1].tv_usec = 0; if (utimes(tmppath ? tmppath : path, tv)) warn("%s: utimes()", tmppath ? tmppath : path); } /* timed out or interrupted? */ if (fetchLastErrCode == FETCH_TIMEOUT) sigalrm = 1; if (sigalrm) warnx("transfer timed out"); if (sigint) { warnx("transfer interrupted"); goto failure; } /* timeout / interrupt before connection completley established? */ if (f == NULL) goto failure; if (!sigalrm && ferror(of)) { /* check the status of our files */ warn("writing to %s failed", path); goto failure; } /* did the transfer complete normally? */ if (us.size != -1 && count < us.size) { warnx("%s appears to be truncated: %jd/%jd bytes", path, (intmax_t)count, (intmax_t)us.size); goto failure_keep; } /* * If the transfer timed out and we didn't know how much to * expect, assume the worst (i.e. we didn't get all of it) */ if (sigalrm && us.size == -1) { warnx("%s may be truncated", path); goto failure_keep; } success: r = 0; if (tmppath != NULL && rename(tmppath, path) == -1) { warn("%s: rename()", path); goto failure_keep; } goto done; failure: if (of && of != stdout && !R_flag && !r_flag) if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) unlink(tmppath ? tmppath : path); if (R_flag && tmppath != NULL && sb.st_size == -1) rename(tmppath, path); /* ignore errors here */ failure_keep: r = -1; goto done; done: if (f) fetchIO_close(f); if (of && of != stdout) fclose(of); if (url) fetchFreeURL(url); if (tmppath != NULL) free(tmppath); return (r); }