CL_String CL_DomElement::get_text() const
{
CL_String str;
if (has_child_nodes() == false)
return str;
CL_DomNode cur = get_first_child();
while (!cur.is_null())
{
if (cur.is_text() || cur.is_cdata_section())
str.append(cur.get_node_value());
if (cur.is_element())
str.append(cur.to_element().get_text());
cur = cur.get_next_sibling();
}
return str;
}
CL_String HTMLPage::download_url(const CL_String &page_url, const CL_String &refererer_url)
{
HTMLUrl url(page_url, refererer_url);
CL_Console::write_line("Downloading URL: %1", url.to_string());
CL_String8 request;
request = cl_format("GET %1 HTTP/1.1\r\n", url.path+url.query);
if (refererer_url.empty())
request += cl_format("Host: %1\r\nConnection: close\r\nAccept: text/plain, text/html\r\nUser-Agent: CSSTokenize/0.1\r\n\r\n", url.host);
else
request += cl_format("Host: %1\r\nConnection: close\r\nReferer: %2\r\nAccept: text/plain, text/html\r\nUser-Agent: CSSTokenize/0.1\r\n\r\n", url.host, refererer_url);
//MessageBoxW(0, CL_StringHelp::utf8_to_ucs2(cl_format("GET %1 HTTP/1.1\r\n", url.path+url.query)).c_str(), L"Download URL", MB_OK);;
CL_TCPConnection connection(CL_SocketName(url.host, url.port));
connection.set_nodelay(true);
connection.send(request.data(), request.length(), true);
CL_String response;
while (connection.get_read_event().wait(15000))
{
char buffer[16*1024];
int received = connection.read(buffer, 16*1024, false);
if (received == 0)
break;
response.append(buffer, received);
}
connection.disconnect_graceful();
CL_String response_header = response.substr(0, response.find("\r\n\r\n"));
CL_String content = response.substr(response_header.length() + 4);
if (response_header.find("Transfer-Encoding: chunked") != CL_String::npos)
{
CL_String::size_type start = 0;
while (true)
{
CL_String::size_type end = content.find("\r\n", start);
if (end == CL_String::npos)
end = content.length();
CL_String str_length = content.substr(start, end-start);
int length = CL_StringHelp::text_to_int(str_length, 16);
content = content.substr(0, start) + content.substr(end+2);
start += length;
end = content.find("\r\n", start);
if (end == CL_String::npos)
end = content.length();
content = content.substr(0, start) + content.substr(end+2);
if (length == 0)
break;
}
}
return content;
}
CL_Image HTMLPage::load_image(CL_GraphicContext &gc, const CL_String &image_url)
{
HTMLUrl url(image_url, pageurl);
CL_Console::write_line("Downloading image: %1", url.to_string());
CL_String8 request;
request = cl_format("GET %1 HTTP/1.1\r\n", url.path+url.query);
request += cl_format("Host: %1\r\nConnection: close\r\nReferer: %2\r\nAccept: image/png, image/jpeg\r\nUser-Agent: CSSTokenize/0.1\r\n\r\n", url.host, pageurl.to_string());
CL_TCPConnection connection(CL_SocketName(url.host, url.port));
connection.set_nodelay(true);
connection.send(request.data(), request.length(), true);
CL_String response;
while (connection.get_read_event().wait(15000))
{
char buffer[16*1024];
int received = connection.read(buffer, 16*1024, false);
if (received == 0)
break;
response.append(buffer, received);
}
connection.disconnect_graceful();
CL_String response_header = response.substr(0, response.find("\r\n\r\n"));
CL_String content = response.substr(response_header.length() + 4);
if (response_header.find("Transfer-Encoding: chunked") != CL_String::npos)
{
CL_String::size_type start = 0;
while (true)
{
CL_String::size_type end = content.find("\r\n", start);
if (end == CL_String::npos)
end = content.length();
CL_String str_length = content.substr(start, end-start);
int length = CL_StringHelp::text_to_int(str_length, 16);
content = content.substr(0, start) + content.substr(end+2);
start += length;
end = content.find("\r\n", start);
if (end == CL_String::npos)
end = content.length();
content = content.substr(0, start) + content.substr(end+2);
if (length == 0)
break;
}
}
CL_DataBuffer buffer(content.data(), content.length());
CL_IODevice_Memory device(buffer);
if (response_header.find("image/png") != CL_String::npos)
{
CL_PixelBuffer pb = CL_PNGProvider::load(device);
return CL_Image(gc, pb, pb.get_size());
}
else if (response_header.find("image/jpeg") != CL_String::npos)
{
CL_PixelBuffer pb = CL_JPEGProvider::load(device);
return CL_Image(gc, pb, pb.get_size());
}
else if (response_header.find("image/gif") != CL_String::npos)
{
CL_PixelBuffer pb = GIFProvider::load(device);
return CL_Image(gc, pb, pb.get_size());
}
else
{
CL_Console::write_line("Unknown image type: %1", CL_String8(buffer.get_data(), buffer.get_size()));
return CL_Image();
}
}