void HTMLPage::load(const CL_String &page_url, const CL_String &refererer_url)
{
pageurl = HTMLUrl(page_url, refererer_url);
webpage = download_url(page_url, refererer_url);
HTMLTokenizer tokenizer;
tokenizer.append(webpage);
HTMLToken token;
while (true)
{
tokenizer.tokenize(token);
if (token.type == HTMLToken::type_null)
break;
if (token.type == HTMLToken::type_style_tag)
{
pagecss += load_css(token.value, page_url);
}
if (token.type == HTMLToken::type_tag_begin || token.type == HTMLToken::type_tag_single)
{
if (token.name == "link")
{
CL_String rel, href, media;
for (size_t i = 0; i < token.attributes.size(); i++)
{
if (token.attributes[i].name == "rel")
rel = token.attributes[i].value;
if (token.attributes[i].name == "href")
href = token.attributes[i].value;
if (token.attributes[i].name == "media")
media = token.attributes[i].value;
}
if (rel == "stylesheet" && !href.empty() && (media.empty() || media == "screen"))
{
CL_String css = download_url(href, page_url);
pagecss += load_css(css, HTMLUrl(href, page_url).to_string());
}
}
}
}
CL_File fhtml("htmlpage.html", CL_File::create_always, CL_File::access_write);
fhtml.write(webpage.data(), webpage.length());
fhtml.close();
CL_File fcss("htmlpage.css", CL_File::create_always, CL_File::access_write);
fcss.write(pagecss.data(), pagecss.length());
fcss.close();
}
int rss_to_file(char *url, char *path)
{
download_data_t *download_data = NULL;
char *json_string = NULL;
int error = 0;
download_data = download_url(url);
if(download_data == NULL)
{
return -1;
}
json_string = rss_to_json(download_data->data, download_data->length);
if(json_string == NULL)
{
return -1;
}
if(io_write(path, json_string, strlen(json_string)) < 0)
{
error = 1;
}
free(json_string);
if(error == 1)
{
return -1;
}
return 0;
}
CL_String HTMLPage::load_css(const CL_String &csstext, const CL_String &base_url)
{
CL_String pagecss;
CL_CSSTokenizer css_tokenizer(csstext);
CL_CSSToken css_token;
while (true)
{
css_tokenizer.read(css_token, true);
if (css_token.type != CL_CSSToken::type_atkeyword || css_token.value != "import")
break;
css_tokenizer.read(css_token, true);
if (css_token.type != CL_CSSToken::type_string)
break;
CL_String import_url = css_token.value;
css_tokenizer.read(css_token, true);
if (css_token.type != CL_CSSToken::type_semi_colon)
break;
CL_String css = download_url(import_url, base_url);
pagecss += load_css(css, HTMLUrl(import_url, base_url).to_string());
}
pagecss += cl_format("@-clan-base-uri \"%1\";\r\n", base_url);
pagecss += csstext;
return pagecss;
}
int download_url_to_file(const char *url, const char *filename)
{
int len;
unsigned char *data = download_url(url, FALSE, &len);
FILE *fp = FOPEN(filename, "w");
ASF_FWRITE(data, sizeof(char), len, fp);
FCLOSE(fp);
return TRUE;
}
void a_geonames_wikipedia_box(VikWindow *vw, VikTrwLayer *vtl, VikLayersPanel *vlp, struct LatLon maxmin[2])
{
gchar *uri;
gchar *tmpname;
GList *wiki_places;
GList *selected;
GList *wp_runner;
VikWaypoint *wiki_wp;
found_geoname *wiki_geoname;
/* encode doubles in a C locale */
gchar *north = a_coords_dtostr(maxmin[0].lat);
gchar *south = a_coords_dtostr(maxmin[1].lat);
gchar *east = a_coords_dtostr(maxmin[0].lon);
gchar *west = a_coords_dtostr(maxmin[1].lon);
uri = g_strdup_printf(GEONAMES_WIKIPEDIA_URL_FMT, north, south, east, west);
g_free(north); north = NULL;
g_free(south); south = NULL;
g_free(east); east = NULL;
g_free(west); west = NULL;
tmpname = download_url(uri);
if (!tmpname) {
none_found(vw);
return;
}
wiki_places = get_entries_from_file(tmpname);
if (g_list_length(wiki_places) == 0) {
none_found(vw);
return;
}
selected = a_select_geoname_from_list(VIK_GTK_WINDOW_FROM_WIDGET(vw), wiki_places, TRUE, "Select articles", "Select the articles you want to add.");
wp_runner = selected;
while (wp_runner) {
wiki_geoname = (found_geoname *)wp_runner->data;
wiki_wp = vik_waypoint_new();
wiki_wp->visible = TRUE;
vik_coord_load_from_latlon(&(wiki_wp->coord), vik_trw_layer_get_coord_mode ( vtl ), &(wiki_geoname->ll));
vik_waypoint_set_comment(wiki_wp, wiki_geoname->desc);
vik_trw_layer_filein_add_waypoint ( vtl, wiki_geoname->name, wiki_wp );
wp_runner = g_list_next(wp_runner);
}
free_geoname_list(wiki_places);
free_geoname_list(selected);
g_free(uri);
if (tmpname) {
g_free(tmpname);
}
vik_layers_panel_emit_update(vlp);
}
static int geonames_search_get_coord(VikWindow *vw, VikViewport *vvp, gchar *srch_str, VikCoord *coord)
{
gchar *uri;
gchar *escaped_srch_str;
int ret = 1; /* OK */
struct LatLon ll;
gchar *tmpname;
g_debug("%s: raw search: %s", __FUNCTION__, srch_str);
escaped_srch_str = uri_escape(srch_str);
g_debug("%s: escaped search: %s", __FUNCTION__, escaped_srch_str);
//uri = g_strdup_printf(GEONAMES_SEARCH_URL_FMT, srch_str);
uri = g_strdup_printf(GEONAMES_SEARCH_URL_FMT, escaped_srch_str);
tmpname = download_url(uri);
if (!tmpname) {
ret = -1;
goto done;
}
ret = parse_file_for_latlon(vw, tmpname, &ll);
if (ret == 3) {
goto done;
}
vik_coord_load_from_latlon ( coord, vik_viewport_get_coord_mode(vvp), &ll );
if (last_coord)
g_free(last_coord);
last_coord = g_malloc(sizeof(VikCoord));
*last_coord = *coord;
if (last_successful_search_str)
g_free(last_successful_search_str);
last_successful_search_str = g_strdup(last_search_str);
done:
g_free(escaped_srch_str);
g_free(uri);
if (tmpname) {
g_remove(tmpname);
g_free(tmpname);
}
return ret;
}
Link::Link (std::string& url, std::string root) : url{url}
{
name = make_name(url, root);
download_url();
}
static HRESULT process_component(InstallEngine *This, ICifComponent *comp)
{
DWORD size_dl, size_install, phase;
char display[MAX_DISPLAYNAME_LENGTH+1];
char id[MAX_ID_LENGTH+1];
HRESULT hr;
int i;
hr = ICifComponent_GetID(comp, id, sizeof(id));
if (FAILED(hr)) return hr;
TRACE("processing component %s\n", debugstr_a(id));
hr = ICifComponent_GetDescription(comp, display, sizeof(display));
if (FAILED(hr)) return hr;
size_dl = (This->thread.operation == OP_DOWNLOAD) ? ICifComponent_GetDownloadSize(comp) : 0;
size_install = 0; /* (This->thread.operation == OP_INSTALL) ? ICifComponent_GetInstalledSize(comp) : 0; */
if (This->callback)
{
IInstallEngineCallback_OnStartComponent(This->callback, id, size_dl, size_install, display);
IInstallEngineCallback_OnComponentProgress(This->callback, id, INSTALLSTATUS_INITIALIZING, display, NULL, 0, 0);
phase = INSTALLSTATUS_INITIALIZING;
}
hr = process_component_dependencies(This, comp);
if (FAILED(hr)) return hr;
if (This->thread.operation == OP_DOWNLOAD)
{
for (i = 0;; i++)
{
DWORD flags;
char *url;
phase = INSTALLSTATUS_DOWNLOADING;
hr = get_url(comp, i, &url, &flags);
if (FAILED(hr)) goto done;
if (!url) break;
TRACE("processing url %s\n", debugstr_a(url));
hr = download_url(This, id, display, url, flags, size_dl);
heap_free(url);
if (FAILED(hr))
{
DWORD retry = 0;
if (This->callback)
IInstallEngineCallback_OnEngineProblem(This->callback, ENGINEPROBLEM_DOWNLOADFAIL, &retry);
if (!retry) goto done;
i--;
continue;
}
phase = INSTALLSTATUS_CHECKINGTRUST;
/* FIXME: check trust */
IInstallEngineCallback_OnComponentProgress(This->callback, id, INSTALLSTATUS_CHECKINGTRUST, display, NULL, 0, 0);
}
component_set_downloaded(comp, TRUE);
phase = INSTALLSTATUS_DOWNLOADFINISHED;
}
else
FIXME("Installation not yet implemented\n");
done:
IInstallEngineCallback_OnStopComponent(This->callback, id, hr, phase, display, 0);
return hr;
}
std::string file_download_cache::get_file(const std::string& url) {
// first check if the file has been downloaded.
// if it has, return the downloaded location
lock.lock();
if (url_to_file.count(url)) {
bool cache_dirty = false;
if (boost::starts_with(url, "s3://")) {
std::string last_modified = "";
try {
last_modified = webstor::get_s3_file_last_modified(url);
} catch (...) {
lock.unlock();
throw;
}
if (last_modified != url_to_file[url].last_modified) {
cache_dirty = true;
}
}
if (!cache_dirty) {
std::string ret = url_to_file[url].filename;
lock.unlock();
return ret;
}
}
lock.unlock();
// ok. we need to download the file
if (boost::starts_with(url, "s3://")) {
// if it is s3.
std::string localfile = get_temp_name();
std::string message = webstor::download_from_s3(url, localfile, "").get();
size_t i = 0;
// if message contains a permanentredirect error code, we need to try other endpoints.
while (boost::algorithm::icontains(message, "PermanentRedirect") && i < webstor::S3_END_POINTS.size()) {
message = webstor::download_from_s3(url, localfile, "", webstor::S3_END_POINTS[i]).get();
++i;
}
if (!message.empty()) {
// OK, we failed. Let's clean up anything that was downloaded
// since the real file lives in S3.
if(std::remove(localfile.c_str()) != 0) {
logstream(LOG_WARNING) << "Could not delete failed cached file: " << localfile << std::endl;
}
log_and_throw_io_failure("Fail to download from " + webstor::sanitize_s3_url(url) + ". "
+ webstor::get_s3_error_code(message));
}
lock.lock();
url_to_file[url].filename = localfile;
try {
url_to_file[url].last_modified = webstor::get_s3_file_last_modified(url);
} catch (...) {
lock.unlock();
throw;
}
lock.unlock();
return localfile;
} else {
// Ok, it is either local regular file, file:///, or remote urls http://.
// For remote urls, download_url download it into to local file.
// For local urls, download_url return as is.
std::string localfile;
int status; bool is_temp;
std::tie(status, is_temp, localfile) = download_url(url);
if (status) {
log_and_throw_io_failure("Fail to download from " + url +
". " + get_curl_error_string(status));
}
if (is_temp) {
// if it is a remote file, we check the download status code
lock.lock();
url_to_file[url].filename = localfile;
url_to_file[url].last_modified = "";
lock.unlock();
return localfile;
} else {
// purely a local file. just return it
return localfile;
}
}
}
