Пример #1
0
Файл: fetch.c Проект: flz/pkgng
int
pkg_fetch_buffer(const char *url, char **buffer, void *data, fetch_cb cb)
{
	FILE *remote = NULL;
	struct url_stat st;
	off_t done = 0;
	off_t r;
	int retry = 3;
	time_t begin_dl;
	time_t now;
	time_t last = 0;
	int retcode = EPKG_OK;

	while (remote == NULL) {
		remote = fetchXGetURL(url, &st, "");
		if (remote == NULL) {
			--retry;
			if (retry == 0) {
				pkg_error_set(EPKG_FATAL, "%s", fetchLastErrString);
				goto cleanup;
			}
			sleep(1);
		}
	}

	*buffer = malloc(st.size + 1);

	begin_dl = time(NULL);
	while (done < st.size) {
		if ((r = fread(*buffer + done, 1, 10240, remote)) < 1)
			break;

		done += r;

		now = time(NULL);
		/* Only call the callback every second */

		if (cb != NULL && (now > last || done == st.size)) {
			cb(data, url, st.size, done, (now - begin_dl));
			last = now;
		}
	}

	if (ferror(remote)) {
		retcode = pkg_error_set(EPKG_FATAL, "%s", fetchLastErrString);
		goto cleanup;
	}

	cleanup:

	if (remote != NULL)
		fclose(remote);

	return (retcode);
}
Пример #2
0
static int
fetch_files(char *urls, char *fout)
{
    FILE *fetch_out, *file_out;
    struct url_stat ustat;
    off_t total_bytes, current_bytes, fsize;
    uint8_t block[4096];
    size_t chunk;
    int progress, last_progress;
    int nsuccess = 0; /* Number of files successfully downloaded */
    int lprg=0;

	progress = 0;
	total_bytes = 0;
	if (fetchStatURL(urls, &ustat, "") == 0 && ustat.size > 0) total_bytes += ustat.size;

	fetchTimeout = 30;

	fetch_out=fetchXGetURL(urls,&ustat,"");
	if (fetch_out == NULL ) return 1;
	file_out=fopen(fout,"w+");

        printf("Size: %d Mb\n",((int)total_bytes / 1024 / 1024));
	current_bytes = 0;
		while ((chunk = fread(block, 1, sizeof(block), fetch_out)) > 0) {
			if (fwrite(block, 1, chunk, file_out) < chunk)
				break;

			current_bytes += chunk;
			fsize += chunk;
	
			if (total_bytes > 0) {
				last_progress = progress;
				progress = (current_bytes*100)/total_bytes; 
			}

		    if ((progress%10 == 0)&&(lprg!=progress)) {
			lprg=progress; printf("Progress: %d%% \n",progress); 
		    }
		}

	if (ustat.size > 0 && fsize < ustat.size) {
	    if (fetchLastErrCode == 0) { //small chunk
		fclose(fetch_out);
		fclose(file_out);
		return 0;
	    }
	} else nsuccess++;

	fclose(fetch_out);
	fclose(file_out);
    return (0);
}
Пример #3
0
int
raptor_www_libfetch_fetch(raptor_www *www) 
{
  FILE *stream;

  if(www->proxy) {
    setenv("HTTP_PROXY", www->proxy, 0);
    setenv("FTP_PROXY", www->proxy, 0);
  }

  if(www->user_agent)
    setenv("HTTP_USER_AGENT", www->user_agent, 0);

  stream = fetchXGetURL((const char*)raptor_uri_as_string(www->uri), NULL, NULL);
  if(!stream) {
    www->failed = 1;
    raptor_www_error(www, "%s", fetchLastErrString);
    return 1;
  }
  
  /* fetch does not give us access to this */
  www->status_code = 200;
  
  while(!feof(stream)) {
    size_t len = fread(www->buffer, 1, RAPTOR_WWW_BUFFER_SIZE, stream);
    
    www->total_bytes += len;

    if(www->write_bytes)
      www->write_bytes(www, www->write_bytes_userdata, www->buffer, len, 1);
    
    if(len < RAPTOR_WWW_BUFFER_SIZE)
      break;
  }
  fclose(stream);
  
  return www->failed;
}
Пример #4
0
/*
 * Attempt to parse the given URL; if successful, call fetchGet().
 */
fetchIO *
fetchGetURL(const char *URL, const char *flags)
{
	return (fetchXGetURL(URL, NULL, flags));
}
Пример #5
0
static int
bootstrap_pkg(void)
{
	FILE *remote;
	FILE *config;
	char *site;
	char url[MAXPATHLEN];
	char conf[MAXPATHLEN];
	char abi[BUFSIZ];
	char tmppkg[MAXPATHLEN];
	char buf[10240];
	char pkgstatic[MAXPATHLEN];
	int fd, retry, ret;
	struct url_stat st;
	off_t done, r;
	time_t now;
	time_t last;

	done = 0;
	last = 0;
	ret = -1;
	remote = NULL;
	config = NULL;

	printf("Bootstrapping pkg please wait\n");

	if (pkg_get_myabi(abi, MAXPATHLEN) != 0) {
		warnx("failed to determine the system ABI");
		return (-1);
	}

	if (getenv("PACKAGESITE") != NULL)
		snprintf(url, MAXPATHLEN, "%s/Latest/pkg.txz", getenv("PACKAGESITE"));
	else
		snprintf(url, MAXPATHLEN, "%s/%s/latest/Latest/pkg.txz",
		    getenv("PACKAGEROOT") ? getenv("PACKAGEROOT") : _PKGS_URL,
		    getenv("ABI") ? getenv("ABI") : abi);

	snprintf(tmppkg, MAXPATHLEN, "%s/pkg.txz.XXXXXX",
	    getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP);

	if ((fd = mkstemp(tmppkg)) == -1) {
		warn("mkstemp()");
		return (-1);
	}

	retry = 3;
	do {
		remote = fetchXGetURL(url, &st, "");
		if (remote == NULL)
			sleep(1);
	} while (remote == NULL && retry-- > 0);

	if (remote == NULL)
		goto fetchfail;

	while (done < st.size) {
		if ((r = fread(buf, 1, sizeof(buf), remote)) < 1)
			break;

		if (write(fd, buf, r) != r) {
			warn("write()");
			goto cleanup;
		}

		done += r;
		now = time(NULL);
		if (now > last || done == st.size)
			last = now;
	}

	if (ferror(remote))
		goto fetchfail;

	if ((ret = extract_pkg_static(fd, pkgstatic, MAXPATHLEN)) == 0)
		ret = install_pkg_static(pkgstatic, tmppkg);

	snprintf(conf, MAXPATHLEN, "%s/etc/pkg.conf",
	    getenv("LOCALBASE") ? getenv("LOCALBASE") : _LOCALBASE);

	if (access(conf, R_OK) == -1) {
		site = strrchr(url, '/');
		if (site == NULL)
			goto cleanup;
		site[0] = '\0';
		site = strrchr(url, '/');
		if (site == NULL)
			goto cleanup;
		site[0] = '\0';

		config = fopen(conf, "w+");
		if (config == NULL)
			goto cleanup;
		fprintf(config, "packagesite: %s", url);
		fclose(config);
	}

	goto cleanup;

fetchfail:
	warnx("Error fetching %s: %s", url, fetchLastErrString);

cleanup:
	if (remote != NULL)
		fclose(remote);
	close(fd);
	unlink(tmppkg);

	return (ret);
}
Пример #6
0
Файл: fetch.c Проект: flz/pkgng
int
pkg_fetch_file(const char *url, const char *dest, void *data, fetch_cb cb)
{
	int fd = -1;
	FILE *remote = NULL;
	struct url_stat st;
	off_t done = 0;
	off_t r;
	int retry = 3;
	time_t begin_dl;
	time_t now;
	time_t last = 0;
	char buf[10240];
	int retcode = EPKG_OK;

	if ((fd = open(dest, O_WRONLY|O_CREAT|O_TRUNC, 0644)) == -1) {
		retcode = pkg_error_set(EPKG_FATAL, "open(%s): %s", dest,
								strerror(errno));
		goto cleanup;
	}

	while (remote == NULL) {
		remote = fetchXGetURL(url, &st, "");
		if (remote == NULL) {
			--retry;
			if (retry == 0) {
				retcode = pkg_error_set(EPKG_FATAL, "%s", fetchLastErrString);
				goto cleanup;
			}
			sleep(1);
		}
	}

	begin_dl = time(NULL);
	while (done < st.size) {
		if ((r = fread(buf, 1, sizeof(buf), remote)) < 1)
			break;

		if (write(fd, buf, r) != r) {
			retcode = pkg_error_set(EPKG_FATAL, "write(%s): %s", dest,
									strerror(errno));
			goto cleanup;
		}

		done += r;
		now = time(NULL);
		/* Only call the callback every second */
		if (cb != NULL && (now > last || done == st.size)) {
			cb(data, url, st.size, done, (now - begin_dl));
			last = now;
		}
	}

	if (ferror(remote)) {
		retcode = pkg_error_set(EPKG_FATAL, "%s", fetchLastErrString);
		goto cleanup;
	}

	cleanup:

	if (fd > 0)
		close(fd);

	if (remote != NULL)
		fclose(remote);

	/* Remove local file if fetch failed */
	if (retcode != EPKG_OK)
		unlink(dest);

	return (retcode);
}