void HTMLPage::load(const CL_String &page_url, const CL_String &refererer_url) { pageurl = HTMLUrl(page_url, refererer_url); webpage = download_url(page_url, refererer_url); HTMLTokenizer tokenizer; tokenizer.append(webpage); HTMLToken token; while (true) { tokenizer.tokenize(token); if (token.type == HTMLToken::type_null) break; if (token.type == HTMLToken::type_style_tag) { pagecss += load_css(token.value, page_url); } if (token.type == HTMLToken::type_tag_begin || token.type == HTMLToken::type_tag_single) { if (token.name == "link") { CL_String rel, href, media; for (size_t i = 0; i < token.attributes.size(); i++) { if (token.attributes[i].name == "rel") rel = token.attributes[i].value; if (token.attributes[i].name == "href") href = token.attributes[i].value; if (token.attributes[i].name == "media") media = token.attributes[i].value; } if (rel == "stylesheet" && !href.empty() && (media.empty() || media == "screen")) { CL_String css = download_url(href, page_url); pagecss += load_css(css, HTMLUrl(href, page_url).to_string()); } } } } CL_File fhtml("htmlpage.html", CL_File::create_always, CL_File::access_write); fhtml.write(webpage.data(), webpage.length()); fhtml.close(); CL_File fcss("htmlpage.css", CL_File::create_always, CL_File::access_write); fcss.write(pagecss.data(), pagecss.length()); fcss.close(); }
int rss_to_file(char *url, char *path) { download_data_t *download_data = NULL; char *json_string = NULL; int error = 0; download_data = download_url(url); if(download_data == NULL) { return -1; } json_string = rss_to_json(download_data->data, download_data->length); if(json_string == NULL) { return -1; } if(io_write(path, json_string, strlen(json_string)) < 0) { error = 1; } free(json_string); if(error == 1) { return -1; } return 0; }
CL_String HTMLPage::load_css(const CL_String &csstext, const CL_String &base_url) { CL_String pagecss; CL_CSSTokenizer css_tokenizer(csstext); CL_CSSToken css_token; while (true) { css_tokenizer.read(css_token, true); if (css_token.type != CL_CSSToken::type_atkeyword || css_token.value != "import") break; css_tokenizer.read(css_token, true); if (css_token.type != CL_CSSToken::type_string) break; CL_String import_url = css_token.value; css_tokenizer.read(css_token, true); if (css_token.type != CL_CSSToken::type_semi_colon) break; CL_String css = download_url(import_url, base_url); pagecss += load_css(css, HTMLUrl(import_url, base_url).to_string()); } pagecss += cl_format("@-clan-base-uri \"%1\";\r\n", base_url); pagecss += csstext; return pagecss; }
int download_url_to_file(const char *url, const char *filename) { int len; unsigned char *data = download_url(url, FALSE, &len); FILE *fp = FOPEN(filename, "w"); ASF_FWRITE(data, sizeof(char), len, fp); FCLOSE(fp); return TRUE; }
void a_geonames_wikipedia_box(VikWindow *vw, VikTrwLayer *vtl, VikLayersPanel *vlp, struct LatLon maxmin[2]) { gchar *uri; gchar *tmpname; GList *wiki_places; GList *selected; GList *wp_runner; VikWaypoint *wiki_wp; found_geoname *wiki_geoname; /* encode doubles in a C locale */ gchar *north = a_coords_dtostr(maxmin[0].lat); gchar *south = a_coords_dtostr(maxmin[1].lat); gchar *east = a_coords_dtostr(maxmin[0].lon); gchar *west = a_coords_dtostr(maxmin[1].lon); uri = g_strdup_printf(GEONAMES_WIKIPEDIA_URL_FMT, north, south, east, west); g_free(north); north = NULL; g_free(south); south = NULL; g_free(east); east = NULL; g_free(west); west = NULL; tmpname = download_url(uri); if (!tmpname) { none_found(vw); return; } wiki_places = get_entries_from_file(tmpname); if (g_list_length(wiki_places) == 0) { none_found(vw); return; } selected = a_select_geoname_from_list(VIK_GTK_WINDOW_FROM_WIDGET(vw), wiki_places, TRUE, "Select articles", "Select the articles you want to add."); wp_runner = selected; while (wp_runner) { wiki_geoname = (found_geoname *)wp_runner->data; wiki_wp = vik_waypoint_new(); wiki_wp->visible = TRUE; vik_coord_load_from_latlon(&(wiki_wp->coord), vik_trw_layer_get_coord_mode ( vtl ), &(wiki_geoname->ll)); vik_waypoint_set_comment(wiki_wp, wiki_geoname->desc); vik_trw_layer_filein_add_waypoint ( vtl, wiki_geoname->name, wiki_wp ); wp_runner = g_list_next(wp_runner); } free_geoname_list(wiki_places); free_geoname_list(selected); g_free(uri); if (tmpname) { g_free(tmpname); } vik_layers_panel_emit_update(vlp); }
static int geonames_search_get_coord(VikWindow *vw, VikViewport *vvp, gchar *srch_str, VikCoord *coord) { gchar *uri; gchar *escaped_srch_str; int ret = 1; /* OK */ struct LatLon ll; gchar *tmpname; g_debug("%s: raw search: %s", __FUNCTION__, srch_str); escaped_srch_str = uri_escape(srch_str); g_debug("%s: escaped search: %s", __FUNCTION__, escaped_srch_str); //uri = g_strdup_printf(GEONAMES_SEARCH_URL_FMT, srch_str); uri = g_strdup_printf(GEONAMES_SEARCH_URL_FMT, escaped_srch_str); tmpname = download_url(uri); if (!tmpname) { ret = -1; goto done; } ret = parse_file_for_latlon(vw, tmpname, &ll); if (ret == 3) { goto done; } vik_coord_load_from_latlon ( coord, vik_viewport_get_coord_mode(vvp), &ll ); if (last_coord) g_free(last_coord); last_coord = g_malloc(sizeof(VikCoord)); *last_coord = *coord; if (last_successful_search_str) g_free(last_successful_search_str); last_successful_search_str = g_strdup(last_search_str); done: g_free(escaped_srch_str); g_free(uri); if (tmpname) { g_remove(tmpname); g_free(tmpname); } return ret; }
Link::Link (std::string& url, std::string root) : url{url} { name = make_name(url, root); download_url(); }
static HRESULT process_component(InstallEngine *This, ICifComponent *comp) { DWORD size_dl, size_install, phase; char display[MAX_DISPLAYNAME_LENGTH+1]; char id[MAX_ID_LENGTH+1]; HRESULT hr; int i; hr = ICifComponent_GetID(comp, id, sizeof(id)); if (FAILED(hr)) return hr; TRACE("processing component %s\n", debugstr_a(id)); hr = ICifComponent_GetDescription(comp, display, sizeof(display)); if (FAILED(hr)) return hr; size_dl = (This->thread.operation == OP_DOWNLOAD) ? ICifComponent_GetDownloadSize(comp) : 0; size_install = 0; /* (This->thread.operation == OP_INSTALL) ? ICifComponent_GetInstalledSize(comp) : 0; */ if (This->callback) { IInstallEngineCallback_OnStartComponent(This->callback, id, size_dl, size_install, display); IInstallEngineCallback_OnComponentProgress(This->callback, id, INSTALLSTATUS_INITIALIZING, display, NULL, 0, 0); phase = INSTALLSTATUS_INITIALIZING; } hr = process_component_dependencies(This, comp); if (FAILED(hr)) return hr; if (This->thread.operation == OP_DOWNLOAD) { for (i = 0;; i++) { DWORD flags; char *url; phase = INSTALLSTATUS_DOWNLOADING; hr = get_url(comp, i, &url, &flags); if (FAILED(hr)) goto done; if (!url) break; TRACE("processing url %s\n", debugstr_a(url)); hr = download_url(This, id, display, url, flags, size_dl); heap_free(url); if (FAILED(hr)) { DWORD retry = 0; if (This->callback) IInstallEngineCallback_OnEngineProblem(This->callback, ENGINEPROBLEM_DOWNLOADFAIL, &retry); if (!retry) goto done; i--; continue; } phase = INSTALLSTATUS_CHECKINGTRUST; /* FIXME: check trust */ IInstallEngineCallback_OnComponentProgress(This->callback, id, INSTALLSTATUS_CHECKINGTRUST, display, NULL, 0, 0); } component_set_downloaded(comp, TRUE); phase = INSTALLSTATUS_DOWNLOADFINISHED; } else FIXME("Installation not yet implemented\n"); done: IInstallEngineCallback_OnStopComponent(This->callback, id, hr, phase, display, 0); return hr; }
std::string file_download_cache::get_file(const std::string& url) { // first check if the file has been downloaded. // if it has, return the downloaded location lock.lock(); if (url_to_file.count(url)) { bool cache_dirty = false; if (boost::starts_with(url, "s3://")) { std::string last_modified = ""; try { last_modified = webstor::get_s3_file_last_modified(url); } catch (...) { lock.unlock(); throw; } if (last_modified != url_to_file[url].last_modified) { cache_dirty = true; } } if (!cache_dirty) { std::string ret = url_to_file[url].filename; lock.unlock(); return ret; } } lock.unlock(); // ok. we need to download the file if (boost::starts_with(url, "s3://")) { // if it is s3. std::string localfile = get_temp_name(); std::string message = webstor::download_from_s3(url, localfile, "").get(); size_t i = 0; // if message contains a permanentredirect error code, we need to try other endpoints. while (boost::algorithm::icontains(message, "PermanentRedirect") && i < webstor::S3_END_POINTS.size()) { message = webstor::download_from_s3(url, localfile, "", webstor::S3_END_POINTS[i]).get(); ++i; } if (!message.empty()) { // OK, we failed. Let's clean up anything that was downloaded // since the real file lives in S3. if(std::remove(localfile.c_str()) != 0) { logstream(LOG_WARNING) << "Could not delete failed cached file: " << localfile << std::endl; } log_and_throw_io_failure("Fail to download from " + webstor::sanitize_s3_url(url) + ". " + webstor::get_s3_error_code(message)); } lock.lock(); url_to_file[url].filename = localfile; try { url_to_file[url].last_modified = webstor::get_s3_file_last_modified(url); } catch (...) { lock.unlock(); throw; } lock.unlock(); return localfile; } else { // Ok, it is either local regular file, file:///, or remote urls http://. // For remote urls, download_url download it into to local file. // For local urls, download_url return as is. std::string localfile; int status; bool is_temp; std::tie(status, is_temp, localfile) = download_url(url); if (status) { log_and_throw_io_failure("Fail to download from " + url + ". " + get_curl_error_string(status)); } if (is_temp) { // if it is a remote file, we check the download status code lock.lock(); url_to_file[url].filename = localfile; url_to_file[url].last_modified = ""; lock.unlock(); return localfile; } else { // purely a local file. just return it return localfile; } } }