void HTTPPacket::setHeader(const char *name, const char *value)
{
HTTPHeader *header = getHeader(name);
if (header != NULL) {
header->setValue(value);
return;
}
addHeader(name, value);
}
// -----------------------------------------------------------------------
// AddContentFields
// -----------------------------------------------------------------------
void AddContentFields(HTTPHeader &header
, const size_t content_length
, const char *content_type
, const char *charset)
{
string tmp = content_type;
if (charset) tmp += string("; charset=") + charset;
header.AddFieldString("Content-Type", tmp.c_str());
header.AddFieldNumeric("Content-Length", content_length);
}
HTTPHeader *HTTPPacket::getHeader(const char *name)
{
int nHeaders = getNHeaders();
for (int n=0; n<nHeaders; n++) {
HTTPHeader *header = getHeader(n);
const char *headerName = header->getName();
if (StringEqualsIgnoreCase(headerName, name) == true)
return header;
}
return NULL;
}
const char *HTTPPacket::getHeaderString(string &headerStr)
{
int nHeaders = getNHeaders();
headerStr = "";
for (int n=0; n<nHeaders; n++) {
HTTPHeader *header = getHeader(n);
headerStr += header->getName();
headerStr += ": ";
headerStr += header->getValue();
headerStr += HTTP::CRLF;
}
return headerStr.c_str();
}
// proxy auth header username extraction
int proxyinstance::identify(Socket& peercon, Socket& proxycon, HTTPHeader &h, std::string &string)
{
// don't match for non-basic auth types
String t(h.getAuthType());
t.toLower();
if (t != "basic")
return DGAUTH_NOMATCH;
// extract username
string = h.getAuthData();
if (string.length() > 0) {
string.resize(string.find_first_of(':'));
return DGAUTH_OK;
}
return DGAUTH_NOMATCH;
}
// proxy auth header username extraction
int digestinstance::identify(Socket &peercon, Socket &proxycon, HTTPHeader &h, std::string &string)
{
// don't match for non-digest auth types
String t = h.getAuthType();
t.toLower();
if (t != "digest")
return DGAUTH_NOMATCH;
// extract username
string = h.getRawAuthData();
if (string.length() > 0) {
String temp(string);
temp = temp.after("username=\"");
temp = temp.before("\"");
string = temp;
return DGAUTH_OK;
}
return DGAUTH_NOMATCH;
}
// proxy auth header username extraction
int headerinstance::identify(Socket &peercon, Socket &proxycon, HTTPHeader &h, std::string &string, bool &is_real_user)
{
if (fname.length() < 0)
return DGAUTH_NOMATCH;
string = h.getAuthHeader();
if (string.length() > 0) {
return DGAUTH_OK;
}
return DGAUTH_NOMATCH;
}
void ExpPayload::onHttpResponse(const HTTPDecoder& dec, const HTTPHeader& reqHeader, const HTTPHeader& respHeader, uint32_t latency) {
(void) reqHeader;
// check whether this content-type is to be exported
if (contentTypes.find(respHeader.getContentType()) == contentTypes.end()) {return;}
// check whether we expect some payload after this response
if (reqHeader.getMethod() == HTTP_METHOD_HEAD) {return;}
// cleanup
if (openFiles.find(&dec) != openFiles.end()) {
openFiles[&dec]->close();
delete openFiles[&dec];
openFiles.erase(&dec);
}
// get details of this content-type
const ExpPayContentType& det = contentTypes[respHeader.getContentType()];
uint64_t absTS = respHeader.getTimestamp();
uint64_t relTS = getUptimeMS();
// create file-name
std::stringstream ss;
ss << absTS << "_" << rand() << "." << det.fileExt;
// get storage file-name
File file = File(det.folder, ss.str());
// add entry to stats-file
*(det.stream) << absTS << "\t" << relTS << "\t" << latency << "\t" << ss.str() << std::endl;
// create/open a new output stream
std::ofstream* stream = new std::ofstream();
stream->open(file.getAbsolutePath().c_str());
openFiles[&dec] = stream;
}
bool ClientSocket::ProcessRead()
{
std::pair<char *, size_t> net_buffer = Sinkhole::NetBuffer();
int len = recv(this->fd, net_buffer.first, net_buffer.second - 1, 0);
if (len <= 0)
return false;
net_buffer.first[len] = 0;
this->readbuffer += net_buffer.first;
size_t header_end = this->readbuffer.find("\r\n\r\n");
if (header_end != std::string::npos)
{
try
{
HTTPRequest request(this, this->readbuffer.substr(0, header_end + 2));
this->readbuffer.erase(0, header_end + 4);
request.Process();
}
catch (const HTTPException &)
{
HTTPAction *a = this->connectclass->action;
HTTPHeader reply;
reply.SetAttribute("Server", a->server_name);
reply.SetAttribute("Connection", "close");
std::string header_data = reply.ToString();
this->Write(header_data.c_str(), header_data.length() + 1);
return false;
}
}
return true;
}
void http_access_test() {
HTTPClient* httpClient = new HTTPClient();
HTTPHeader* httpHeader = new HTTPHeader();
httpHeader->insert(
std::string("Content-Type"),
std::string("application/x-www-form-urlencoded")
);
httpHeader->insert(
std::string("Connection"),
std::string("close")
);
httpClient->get(
new std::string("http://www.ex-studio.info/"),
httpHeader
);
int status_code = httpClient->status_code();
std::cout << "Status code: " << status_code << std::endl;
if(status_code != 200) {
delete httpHeader;
delete httpClient;
return;
}
HTTPHeader* recvHeader = httpClient->header();
std::string* recvBody = httpClient->body();
std::cout << "Date is " << recvHeader->get("Date") << std::endl;
std::cout << "body: " << *recvBody << std::endl;
delete httpHeader;
delete httpClient;
return;
}
// IP-based filter group determination
// never actually return NOUSER from this, because we don't actually look in the filtergroupslist.
// NOUSER stops ConnectionHandler from querying subsequent plugins.
int ipinstance::identify(Socket& peercon, Socket& proxycon, HTTPHeader &h, std::string &string)
{
// we don't get usernames out of this plugin, just a filter group
// for now, use the IP as the username
if (o.use_xforwardedfor) {
// grab the X-Forwarded-For IP if available
string = h.getXForwardedForIP();
// otherwise, grab the IP directly from the client connection
if (string.length() == 0)
string = peercon.getPeerIP();
} else {
string = peercon.getPeerIP();
}
return DGAUTH_OK;
}
void FolderSource::sendFile(File file, HTTPDecoder& dec) {
debug(DBG_FOLDER_SOURCE, DBG_LVL_INFO, "sending file: " << file.getAbsolutePath());
// get new HTTP analyzer to send the file to
HTTPAnalyzer* http = analyzer->onHttpConnection();
// send dummy request
HTTPHeader hReq;
hReq.setMethod(HTTP_METHOD_GET);
http->onHttpRequest(dec, hReq);
// content-type is the folder-name realtive to the source-folder
uint32_t rootLength = folder.getAbsolutePath().length()+1;
std::string contentType = file.getParent().getAbsolutePath();
contentType = (contentType.length() > rootLength) ? (contentType.substr(rootLength)) : ("root");
// send dummy response
HTTPHeader hResp;
hResp.setMethod(HTTP_METHOD_RESPONSE);
hResp.setCode(200);
hResp.set("content-type", contentType);
http->onHttpResponse(dec, hReq, hResp, 0);
// read the file
std::ifstream stream;
stream.open(file.getAbsolutePath().c_str(), std::ios::in);
if (stream.is_open()) {
while (true) {
// read one block;
stream.read( (char*) buffer, chunkSize );
size_t numRead = stream.gcount();
// send to analyzer
http->onHttpResponseTraffic(dec, hReq, hResp, buffer, numRead);
// EOF?
if (stream.eof()) {break;}
}
}
// finish (flush compressors)
http->onHttpDone(dec, hResp);
// close (this will delete the http analyzer)
http->onHttpClose(dec);
}
// -----------------------------------------------------------------------
// AddRequestHeaderFields
// -----------------------------------------------------------------------
void AddRequestHeaderFields(HTTPHeader &header
, const O2Profile *profile)
{
string pubkey;
profile->GetPubkeyA(pubkey);
hashT hash;
profile->GetID(hash);
string hashstr;
hash.to_string(hashstr);
string flags;
profile->GetFlags(flags);
header.AddFieldString("Connection", "close");
header.AddFieldString("Host", header.host.c_str());
header.AddFieldString("User-Agent", profile->GetUserAgentA());
header.AddFieldString(X_O2_NODE_ID, hashstr.c_str());
header.AddFieldString(X_O2_RSAPUBLICKEY, pubkey.c_str());
header.AddFieldNumeric(X_O2_PORT, profile->GetP2PPort());
header.AddFieldString(X_O2_NODENAME, profile->GetNodeNameA());
header.AddFieldString(X_O2_NODE_FLAGS, flags.c_str());
}
// -----------------------------------------------------------------------
// GetNodeInfoFromHeader
// -----------------------------------------------------------------------
bool GetNodeInfoFromHeader(const HTTPHeader &header
, const ulong ip
, const ushort port
, O2Node &node)
{
strmap::const_iterator it;
//ID
it = header.fields.find(X_O2_NODE_ID);
if (it == header.fields.end())
return false;
node.id.assign(it->second.c_str(), it->second.size());
//IP
node.ip = ip;
//Port
if (header.GetType() == HTTPHEADERTYPE_REQUEST) {
it = header.fields.find(X_O2_PORT);
if (it == header.fields.end())
return false;
node.port = (ushort)strtoul(it->second.c_str(), NULL, 10);
}
else
node.port = port;
//RSA Public Key
it = header.fields.find(X_O2_RSAPUBLICKEY);
if (it == header.fields.end())
return false;
node.pubkey.assign(it->second.c_str(), it->second.size());
//NodeName
it = header.fields.find(X_O2_NODENAME);
if (it != header.fields.end())
ToUnicode(_T(DEFAULT_XML_CHARSET), it->second, node.name);
//Flags
it = header.fields.find(X_O2_NODE_FLAGS);
if (it != header.fields.end())
ascii2unicode(it->second, node.flags);
//User-Agent
byte status;
switch (header.GetType()) {
case HTTPHEADERTYPE_REQUEST:
it = header.fields.find("User-Agent");
status = O2_NODESTATUS_LINKEDFROM;
break;
case HTTPHEADERTYPE_RESPONSE:
it = header.fields.find("Server");
status = O2_NODESTATUS_LINKEDTO;
break;
default:
return false;
}
if (it == header.fields.end())
return false;
ascii2unicode(it->second, node.ua);
// ノードのユーザーエージェントから情報を取得
// O2/0.2 (o2on/0.02.0027; Win32)
// O2/0.2 (opy2on/0.00.0001; Linux x86_64)
//など「O2/プロトコルバージョン (アプリ名/アプリバージョン; 環境等)」形式であること
size_t ProtoNamePos = node.ua.find(L"O2/", 0);
if (ProtoNamePos != 0) {
// プロトコル名が見つからない
return false;
}
size_t AppNamePeriodPos = node.ua.find(L" ", ProtoNamePos);
if ( AppNamePeriodPos == wstring::npos ) {
// プロトコル・アプリ名の区切りが見つからない
return false;
}
node.proto_ver = node.ua.substr( 3, AppNamePeriodPos - 3 );
size_t AppNameStartPos = node.ua.find(L"(", AppNamePeriodPos);
if (( AppNameStartPos == wstring::npos ) || ((AppNameStartPos + 1) == node.ua.size() )) {
// アプリ名の開始位置が見つからない
return false;
}
AppNameStartPos++;
size_t AppNameEndPos = node.ua.find(L"/", AppNameStartPos);
if ( AppNameEndPos == wstring::npos ) {
// アプリ名の終了位置が見つからない
return false;
}
node.app_name = node.ua.substr( AppNameStartPos, AppNameEndPos - AppNameStartPos );
if ((wcscmp(node.app_name.c_str(), _T(APP_NAME)) == 0) && ((AppNameEndPos + 1) < node.ua.size() )) {
// 同じアプリならバージョン番号を取得
AppNameEndPos++;
size_t VerEndPos = node.ua.find(L";", AppNameEndPos);
if ( VerEndPos == wstring::npos ) {
// バージョン番号の終了位置が見つからない
return false;
}
node.app_ver = node.ua.substr(AppNameEndPos, VerEndPos - AppNameEndPos);
}
else
{
wchar_t tmpW[128];
swprintf_s(tmpW, 128, L"異なるアプリ:%s\n", node.app_name.c_str());
TRACEW(tmpW);
}
return true;
}
void ExpPlotLatency::onHttpResponse(const HTTPDecoder& dec, const HTTPHeader& req, const HTTPHeader& resp, uint32_t latency) {
(void) dec;
(void) req;
append(resp.getContentType(), latency);
}
// ntlm auth header username extraction - also lets connection persist long enough to complete NTLM negotiation
int ntlminstance::identify(Socket& peercon, Socket& proxycon, HTTPHeader &h, std::string &string)
{
FDTunnel fdt;
Socket* upstreamcon;
Socket ntlmcon;
String url;
if (transparent) {
// we are actually sending to a second Squid, which just does NTLM
ntlmcon.connect(transparent_ip, transparent_port);
upstreamcon = &ntlmcon;
url = h.getUrl();
h.makeTransparent(false);
} else {
upstreamcon = &proxycon;
}
String at(h.getAuthType());
if (transparent && (at != "NTLM")) {
// obey forwarded-for options in what we send out
std::string clientip;
if (o.forwarded_for == 1) {
if (o.use_xforwardedfor == 1) {
// grab the X-Forwarded-For IP if available
clientip = h.getXForwardedForIP();
// otherwise, grab the IP directly from the client connection
if (clientip.length() == 0)
clientip = peercon.getPeerIP();
} else {
clientip = peercon.getPeerIP();
}
h.addXForwardedFor(clientip); // add squid-like entry
}
// in transparent mode, we need to make the initial auth required response
// appear to come from the smoothie itself as an origin server, not as a proxy.
//
// accomplish this by redirecting to a URL that results in accessing DG as if it was
// a webserver, fudging origin-server-style NTLM auth to the client whilst actually
// performing proper proxy-style auth to the parent proxy, then redirecting the client
// back to the actual URL.
if (!url.contains("sgtransntlmdest=")) {
// user has not yet been redirected
// get the browser to make a request to the proxy port on the relevant interface,
// embedding the original URL they were trying to access.
// unless they're accessing a domain for which authentication is not required,
// in which case return a no match response straight away.
if (no_auth_list >= 0)
{
#ifdef DGDEBUG
std::cout << "NTLM: Checking noauthdomains list" << std::endl;
#endif
std::string::size_type start = url.find("://");
if (start != std::string::npos)
{
start += 3;
std::string domain;
domain = url.getHostname();
#ifdef DGDEBUG
std::cout << "NTLM: URL " << url << ", domain " << domain << std::endl;
#endif
char *i;
while ((start = domain.find('.')) != std::string::npos)
{
i = o.lm.l[no_auth_list]->findInList(domain.c_str());
if (i != NULL)
{
#ifdef DGDEBUG
std::cout << "NTLM: Found domain in noauthdomains list" << std::endl;
#endif
return DGAUTH_NOMATCH;
}
domain.assign(domain.substr(start + 1));
}
if (!domain.empty())
{
domain = "." + domain;
i = o.lm.l[no_auth_list]->findInList(domain.c_str());
if (i != NULL)
{
#ifdef DGDEBUG
std::cout << "NTLM: Found domain in noauthdomains list" << std::endl;
#endif
return DGAUTH_NOMATCH;
}
}
}
}
string = "http://";
string += hostname;
string += ":";
string += String(peercon.getPort()).toCharArray();
string += "/?sgtransntlmdest=";
string += url.toCharArray();
#ifdef DGDEBUG
std::cout << "NTLM - redirecting client to " << string << std::endl;
#endif
return DGAUTH_REDIRECT;
}
#ifdef DGDEBUG
std::cout << "NTLM - forging initial auth required from origin server" << std::endl;
#endif
// obey forwarded-for options in what we send out
if (o.forwarded_for == 1) {
std::string clientip;
if (o.use_xforwardedfor == 1) {
// grab the X-Forwarded-For IP if available
clientip = h.getXForwardedForIP();
// otherwise, grab the IP directly from the client connection
if (clientip.length() == 0)
clientip = peercon.getPeerIP();
} else {
clientip = peercon.getPeerIP();
}
h.addXForwardedFor(clientip); // add squid-like entry
}
// send a variant on the original request (has to be something Squid will route to the outside
// world, and that it will require NTLM authentication for)
String domain(url.after("?sgtransntlmdest=").after("://"));
if (domain.contains("/")) domain = domain.before("/");
domain = "http://" + domain + "/";
h.setURL(domain);
h.makePersistent();
h.out(&peercon, upstreamcon, __DGHEADER_SENDALL);
// grab the auth required response and make it look like it's from the origin server
h.in(upstreamcon, true);
h.makeTransparent(true);
h.makePersistent();
// send it to the client
h.out(NULL, &peercon, __DGHEADER_SENDALL);
if (h.contentLength() != -1)
fdt.tunnel(*upstreamcon, peercon, false, h.contentLength(), true);
if (h.isPersistent()) {
// now grab the client's response to the auth request, and carry on as usual.
h.in(&peercon, true);
h.makeTransparent(false);
at = h.getAuthType();
} else
return DGAUTH_NOMATCH;
} else if (transparent && url.contains("?sgtransntlmdest=")) {
// send a variant on the original request (has to be something Squid will route to the outside
// world, and that it will require NTLM authentication for)
String domain(url.after("?sgtransntlmdest=").after("://"));
if (domain.contains("/")) domain = domain.before("/");
domain = "http://" + domain + "/";
h.setURL(domain);
}
if (at != "NTLM") {
// if no auth currently underway, then...
if (at.length() == 0) {
// allow the initial request through so the client will get the proxy's initial auth required response.
// advertise persistent connections so that parent proxy will agree to advertise NTLM support.
#ifdef DGDEBUG
std::cout << "No auth negotiation currently in progress - making initial request persistent so that proxy will advertise NTLM" << std::endl;
#endif
h.makePersistent();
}
return DGAUTH_NOMATCH;
}
#ifdef DGDEBUG
std::cout << "NTLM - sending step 1" << std::endl;
#endif
if (o.forwarded_for) {
std::string clientip;
if (o.use_xforwardedfor) {
// grab the X-Forwarded-For IP if available
clientip = h.getXForwardedForIP();
// otherwise, grab the IP directly from the client connection
if (clientip.length() == 0)
clientip = peercon.getPeerIP();
} else {
clientip = peercon.getPeerIP();
}
h.addXForwardedFor(clientip); // add squid-like entry
}
h.makePersistent();
h.out(&peercon, upstreamcon, __DGHEADER_SENDALL);
#ifdef DGDEBUG
std::cout << "NTLM - receiving step 2" << std::endl;
#endif
h.in(upstreamcon, true);
if (h.authRequired()) {
#ifdef DGDEBUG
std::cout << "NTLM - sending step 2" << std::endl;
#endif
if (transparent)
h.makeTransparent(true);
h.out(NULL, &peercon, __DGHEADER_SENDALL);
if (h.contentLength() != -1)
fdt.tunnel(*upstreamcon, peercon, false, h.contentLength(), true);
#ifdef DGDEBUG
std::cout << "NTLM - receiving step 3" << std::endl;
#endif
h.in(&peercon, true);
if (transparent) {
h.makeTransparent(false);
String domain(url.after("?sgtransntlmdest=").after("://"));
if (domain.contains("/")) domain = domain.before("/");
domain = "http://" + domain + "/";
h.setURL(domain);
}
#ifdef DGDEBUG
std::cout << "NTLM - decoding type 3 message" << std::endl;
#endif
std::string message(h.getAuthData());
ntlm_authenticate auth;
ntlm_auth *a = &(auth.a);
static char username[256]; // fixed size
static char username2[256];
char* inptr = username;
char* outptr = username2;
size_t l,o;
// copy the NTLM message into the union's buffer, simultaneously filling in the struct
if ((message.length() > sizeof(ntlm_auth)) || (message.length() < offsetof(ntlm_auth, payload))) {
syslog(LOG_ERR, "NTLM - Invalid message of length %zd, message was: %s", message.length(), message.c_str());
#ifdef DGDEBUG
std::cerr << "NTLM - Invalid message of length " << message.length() << ", message was: " << message << std::endl;
#endif
return -3;
}
memcpy((void *)auth.buf, (const void *)message.c_str(), message.length());
// verify that the message is indeed a type 3
if (strcmp("NTLMSSP",a->h.signature) == 0 && WSWAP(a->h.type) == 3) {
// grab the length & offset of the username within the message
// cope with the possibility we are a different byte order to Windows
l = SSWAP(a->user.len);
o = WSWAP(a->user.offset);
if ((l > 0) && (o >= 0) && (o + l) <= sizeof(a->payload) && (l <= 254)) {
// everything is in range
// note offsets are from start of packet - not the start of the payload area
memcpy((void *)username, (const void *)&(auth.buf[o]),l);
username[l] = '\0';
// check flags - we may need to convert from UTF-16 to something more sensible
int f = WSWAP(a->flags);
if (f & WSWAP(0x0001)) {
iconv_t ic = iconv_open("UTF-8", "UTF-16LE");
if (ic == (iconv_t)-1) {
syslog(LOG_ERR, "NTLM - Cannot initialise conversion from UTF-16LE to UTF-8: %s", strerror(errno));
#ifdef DGDEBUG
std::cerr << "NTLM - Cannot initialise conversion from UTF-16LE to UTF-8: " << strerror(errno) << std::endl;
#endif
iconv_close(ic);
return -2;
}
size_t l2 = 256;
local_iconv_adaptor(iconv, ic, &inptr, &l, &outptr, &l2);
iconv_close(ic);
username2[256 - l2] = '\0';
#ifdef DGDEBUG
std::cout << "NTLM - got username (converted from UTF-16LE) " << username2 << std::endl;
#endif
string = username2;
} else {
#ifdef DGDEBUG
std::cout << "NTLM - got username " << username << std::endl;
#endif
string = username;
}
if (!transparent)
return DGAUTH_OK;
// if in transparent mode, send a redirect to the client's original requested URL,
// having sent the final headers to the NTLM-only Squid to do with what it will
std::string tmp = peercon.getPeerIP();
h.addXForwardedFor(tmp);
h.out(&peercon, upstreamcon, __DGHEADER_SENDALL);
// also, the return code matters in ways it hasn't mattered before:
// mustn't send a redirect if it is still 407, or we get a redirection loop
h.in(upstreamcon, true);
if (h.returnCode() == 407)
{
h.makeTransparent(false);
h.out(NULL, &peercon, __DGHEADER_SENDALL);
return -10;
}
url = url.after("=");
string = url.toCharArray();
return DGAUTH_REDIRECT;
}
}
return DGAUTH_NOMATCH;
} else {
#ifdef DGDEBUG
std::cout << "NTLM - step 2 was not part of an auth handshake!" << std::endl;
for (unsigned int i = 0; i < h.header.size(); i++)
std::cout << h.header[i] << std::endl;
#endif
syslog(LOG_ERR, "NTLM - step 2 was not part of an auth handshake! (%s)", h.header[0].toCharArray());
return -1;
}
}