/* Connect to the HTTP server, specified by uuu->net_info's "port:host". * Return eIO_Success only if socket connection has succeeded and uuu->sock * is non-zero. If unsuccessful, try to adjust uuu->net_info by s_Adjust(), * and then re-try the connection attempt. */ static EIO_Status s_Connect(SHttpConnector* uuu, EReadMode read_mode) { EIO_Status status; assert(!uuu->sock); if (uuu->can_connect == eCC_None) { CORE_LOG_X(5, eLOG_Error, "[HTTP] Connector no longer usable"); return eIO_Closed; } /* the re-try loop... */ for (;;) { int/*bool*/ reset_user_header = 0; char* http_user_header = 0; char* null = 0; TSOCK_Flags flags; uuu->w_len = BUF_Size(uuu->w_buf); if (uuu->net_info->http_user_header) http_user_header = strdup(uuu->net_info->http_user_header); if (!uuu->net_info->http_user_header == !http_user_header) { ConnNetInfo_ExtendUserHeader (uuu->net_info, "User-Agent: NCBIHttpConnector" #ifdef NCBI_CXX_TOOLKIT " (C++ Toolkit)" #else " (C Toolkit)" #endif "\r\n"); reset_user_header = 1; } if (uuu->net_info->debug_printout) ConnNetInfo_Log(uuu->net_info, CORE_GetLOG()); flags = (uuu->net_info->debug_printout == eDebugPrintout_Data ? fSOCK_LogOn : fSOCK_LogDefault); if (uuu->net_info->scheme == eURL_Https) flags |= fSOCK_Secure; if (!(uuu->flags & fHCC_NoUpread)) flags |= fSOCK_ReadOnWrite; /* connect & send HTTP header */ if ((status = URL_ConnectEx (uuu->net_info->host, uuu->net_info->port, uuu->net_info->path, uuu->net_info->args, uuu->net_info->req_method, uuu->w_len, uuu->o_timeout, uuu->w_timeout, uuu->net_info->http_user_header, (int/*bool*/)(uuu->flags & fHCC_UrlEncodeArgs), flags, &uuu->sock)) != eIO_Success) { uuu->sock = 0; } if (reset_user_header) { ConnNetInfo_SetUserHeader(uuu->net_info, 0); uuu->net_info->http_user_header = http_user_header; } if (uuu->sock) break/*success*/; /* connection failed, no socket was created */ if (!s_Adjust(uuu, &null, read_mode)) break/*closed*/; } return status; }
EIO_Status CConnTest::HttpOkay(string* reason) { SConnNetInfo* net_info = ConnNetInfo_Create(0, m_DebugPrintout); if (net_info) { if (net_info->http_proxy_host[0] && net_info->http_proxy_port) m_HttpProxy = true; // Make sure there are no extras ConnNetInfo_SetUserHeader(net_info, 0); net_info->args[0] = '\0'; } PreCheck(eHttp, 0/*main*/, "Checking whether NCBI is HTTP accessible"); string host(net_info ? net_info->host : DEF_CONN_HOST); string port(net_info && net_info->port ? ':' + NStr::UIntToString(net_info->port) : kEmptyStr); SAuxData* auxdata = new SAuxData(m_Canceled, 0); CConn_HttpStream http("/Service/index.html", net_info, kEmptyStr/*user_header*/, s_GoodHeader, auxdata, s_Adjust, s_Cleanup, 0/*flags*/, m_Timeout); http.SetCanceledCallback(m_Canceled); string temp; http >> temp; EIO_Status status = ConnStatus(temp.empty(), &http); if (status == eIO_Interrupt) temp = kCanceled; else if (status == eIO_Success) temp = "OK"; else { if (status == eIO_Timeout) temp = x_TimeoutMsg(); else temp.clear(); if (NStr::CompareNocase(host, DEF_CONN_HOST) != 0 || !port.empty()) { int n = 0; temp += "Make sure that "; if (host != DEF_CONN_HOST) { n++; temp += "[CONN]HOST=\""; temp += host; temp += port.empty() ? "\"" : "\" and "; } if (!port.empty()) { n++; temp += "[CONN]PORT=\""; temp += port.c_str() + 1; temp += '"'; } _ASSERT(n); temp += n > 1 ? " are" : " is"; temp += " redefined correctly\n"; } if (m_HttpProxy) { temp += "Make sure that the HTTP proxy server \'"; temp += net_info->http_proxy_host; temp += ':'; temp += NStr::UIntToString(net_info->http_proxy_port); temp += "' specified with [CONN]HTTP_PROXY_{HOST|PORT} is correct"; } else { if (net_info->http_proxy_host[0] || net_info->http_proxy_port) { temp += "Note that your HTTP proxy seems to have been" " specified only partially, and thus cannot be used: the "; if (net_info->http_proxy_port) { temp += "host part is missing (for port :" + NStr::UIntToString(net_info->http_proxy_port); } else { temp += "port part is missing (for host \'" + string(net_info->http_proxy_host) + '\''; } temp += ")\n"; } temp += "If your network access requires the use of an HTTP proxy" " server, please contact your network administrator and set" " [CONN]HTTP_PROXY_{HOST|PORT} (both must be set) in your" " configuration accordingly"; } temp += "; and if your proxy server requires authorization, please" " check that appropriate [CONN]HTTP_PROXY_{USER|PASS} have been" " set\n"; if (net_info && (*net_info->user || *net_info->pass)) { temp += "Make sure there are no stray [CONN]{USER|PASS} that" " appear in your configuration -- NCBI services neither" " require nor use them\n"; } } PostCheck(eHttp, 0/*main*/, status, temp); ConnNetInfo_Destroy(net_info); if (reason) reason->swap(temp); return status; }
static CConn_IOStream::TConn_Pair s_SocketConnectorBuilder(const SConnNetInfo* net_info, const STimeout* timeout, const void* data, size_t size, TSOCK_Flags flags) { EIO_Status status = eIO_Success; bool proxy = false; SOCK sock = 0; _ASSERT(net_info); if ((flags & (fSOCK_LogOn | fSOCK_LogDefault)) == fSOCK_LogDefault && net_info->debug_printout == eDebugPrintout_Data) { flags &= ~fSOCK_LogDefault; flags |= fSOCK_LogOn; } if (*net_info->http_proxy_host && net_info->http_proxy_port) { status = HTTP_CreateTunnel(net_info, fHTTP_NoAutoRetry, &sock); _ASSERT(!sock ^ !(status != eIO_Success)); if (status == eIO_Success && ((flags & ~(fSOCK_LogOn | fSOCK_LogDefault)) || size)) { SOCK s; status = SOCK_CreateOnTopEx(sock, 0, &s, data, size, flags); _ASSERT(!s ^ !(status != eIO_Success)); SOCK_Destroy(sock); sock = s; } proxy = true; } if (!sock && (!proxy || net_info->http_proxy_leak)) { const char* host = (net_info->firewall && *net_info->proxy_host ? net_info->proxy_host : net_info->host); if (timeout == kDefaultTimeout) timeout = net_info->timeout; if (!proxy && net_info->debug_printout) { SConnNetInfo* x_net_info = ConnNetInfo_Clone(net_info); if (x_net_info) { x_net_info->req_method = eReqMethod_Any; x_net_info->stateless = 0; x_net_info->lb_disable = 0; x_net_info->http_proxy_host[0] = '\0'; x_net_info->http_proxy_port = 0; x_net_info->http_proxy_user[0] = '\0'; x_net_info->http_proxy_pass[0] = '\0'; x_net_info->proxy_host[0] = '\0'; ConnNetInfo_SetUserHeader(x_net_info, 0); if (x_net_info->http_referer) { free((void*) x_net_info->http_referer); x_net_info->http_referer = 0; } x_net_info->timeout = timeout; } ConnNetInfo_Log(x_net_info, eLOG_Note, CORE_GetLOG()); ConnNetInfo_Destroy(x_net_info); } status = SOCK_CreateEx(host, net_info->port, timeout, &sock, data, size, flags); _ASSERT(!sock ^ !(status != eIO_Success)); } string hostport(net_info->host); hostport += ':'; hostport += NStr::UIntToString(net_info->port); CONNECTOR c = SOCK_CreateConnectorOnTopEx(sock, 1/*own*/, hostport.c_str()); if (!c) { SOCK_Abort(sock); SOCK_Close(sock); status = eIO_Unknown; } return CConn_IOStream::TConn_Pair(c, status); }
extern NCBI_XCONNECT_EXPORT // FIXME: To remove once the API is fully official CConn_IOStream* NcbiOpenURL(const string& url, size_t buf_size) { { class CInPlaceConnIniter : protected CConnIniter { } conn_initer; } bool svc = x_IsIdentifier(url); AutoPtr<SConnNetInfo> net_info = ConnNetInfo_Create(svc ? url.c_str() : 0); if (svc) return new CConn_ServiceStream(url, fSERV_Any, net_info.get()); unsigned int host; unsigned short port; if (url.size() == CSocketAPI::StringToHostPort(url, &host, &port) && port && net_info.get()) { net_info->req_method = eReqMethod_Connect; } if (ConnNetInfo_ParseURL(net_info.get(), url.c_str())) { _ASSERT(net_info); // otherwise ConnNetInfo_ParseURL() would fail if (net_info->req_method == eReqMethod_Connect) { return new CConn_SocketStream(*net_info, 0, 0, fSOCK_LogDefault, net_info->timeout, buf_size); } switch (net_info->scheme) { case eURL_Https: case eURL_Http: return new CConn_HttpStream(net_info.get(), kEmptyStr, 0, 0, 0, 0, fHTTP_AutoReconnect, kDefaultTimeout, buf_size); case eURL_File: if (*net_info->host || net_info->port) break; /*not supported*/ _ASSERT(!*net_info->args); if (net_info->debug_printout) { net_info->req_method = eReqMethod_Any; net_info->firewall = 0; net_info->stateless = 0; net_info->lb_disable = 0; net_info->http_proxy_leak = 0; net_info->http_proxy_host[0] = '\0'; net_info->http_proxy_port = 0; net_info->http_proxy_user[0] = '\0'; net_info->http_proxy_pass[0] = '\0'; net_info->proxy_host[0] = '\0'; net_info->max_try = 0; net_info->timeout = kInfiniteTimeout; ConnNetInfo_SetUserHeader(net_info.get(), 0); if (net_info->http_referer) { free((void*) net_info->http_referer); net_info->http_referer = 0; } ConnNetInfo_Log(net_info.get(), eLOG_Note, CORE_GetLOG()); } return new CConn_FileStream(net_info->path); case eURL_Ftp: if (!net_info->user[0]) { strcpy(net_info->user, "ftp"); if (!net_info->pass[0]) strcpy(net_info->pass, "-none@"); } return new CConn_FTPDownloadStream(*net_info, 0, 0, 0, net_info->timeout, buf_size); default: break; } } return 0; }
static int run_a_test(size_t test_idx, int live, const char *svc, const char *hdr, int check_for_match, int exp_err, const char *mock_body_in, int repop, int reset) { const SSERV_Info *info = NULL; SConnNetInfo *net_info; SERV_ITER iter; const char *mock_body = NULL; char *mock_body_adj = NULL; int n_matches_perfect = 0, n_matches_near = 0; int success = 0, errors = 0; int retval = -1; s_n_hits_got = 0; /* Adjust mock data for current time, if necessary. */ adjust_mock_times(mock_body_in, &mock_body_adj); mock_body = mock_body_adj ? mock_body_adj : mock_body_in; /* Select the HTTP data source (live or mock). */ s_results[test_idx].live = live; if ( ! s_results[test_idx].live && ( ! mock_body || ! *mock_body)) { CORE_TRACE("Mock HTTP data source unavailable."); s_results[test_idx].live = 1; } if (s_results[test_idx].live) { CORE_TRACE("Using a live HTTP data source."); SERV_NAMERD_SetConnectorSource(NULL); /* use live HTTP */ } else { CORE_TRACE("Using a mock HTTP data source."); if ( ! SERV_NAMERD_SetConnectorSource(mock_body)) { CORE_LOG(eLOG_Error, "Unable to create mock HTTP data source."); retval = 1; goto out; } } /* Set up the server iterator. */ net_info = ConnNetInfo_Create(svc); if (*hdr) ConnNetInfo_SetUserHeader(net_info, hdr); iter = SERV_OpenP(svc, fSERV_All | (strpbrk(svc, "?*") ? fSERV_Promiscuous : 0), SERV_LOCALHOST, 0/*port*/, 0.0/*preference*/, net_info, 0/*skip*/, 0/*n_skip*/, 0/*external*/, 0/*arg*/, 0/*val*/); ConnNetInfo_Destroy(net_info); /* Fetch the server hits from namerd. */ if (iter) { for (; s_n_hits_got < MAX_HITS && (info = SERV_GetNextInfo(iter)); ++s_n_hits_got) { if (info->type & fSERV_Http) { CORE_LOGF(eLOG_Note, (" HTTP extra (path): %s", SERV_HTTP_PATH(&info->u.http))); } strcpy(s_hits_got[s_n_hits_got].type, SERV_TypeStr(info->type)); strcpy(s_hits_got[s_n_hits_got].xtra, (info->type & fSERV_Http) ? SERV_HTTP_PATH(&info->u.http) : ""); strcpy(s_hits_got[s_n_hits_got].loc , (info->site & fSERV_Local ) ? "yes" : "no"); strcpy(s_hits_got[s_n_hits_got].priv, (info->site & fSERV_Private ) ? "yes" : "no"); strcpy(s_hits_got[s_n_hits_got].stfl, (info->mode & fSERV_Stateful) ? "yes" : "no"); SOCK_ntoa(info->host, s_hits_got[s_n_hits_got].host, LEN_HOST); s_hits_got[s_n_hits_got].port = info->port; s_hits_got[s_n_hits_got].match = 0; char *info_str; info_str = SERV_WriteInfo(info); CORE_LOGF(eLOG_Note, (" Found server %d: %s", s_n_hits_got, info_str ? info_str : "?")); if (info_str) free(info_str); } /* Make sure endpoint data can be repopulated and reset. */ if (repop && s_n_hits_got) { /* repopulate */ CORE_LOG(eLOG_Trace, "Repopulating the service mapper."); if ( ! info && ! SERV_GetNextInfo(iter)) { CORE_LOG(eLOG_Error, "Unable to repopulate endpoint data."); errors = 1; } } if (reset && s_n_hits_got) { /* reset */ CORE_LOG(eLOG_Trace, "Resetting the service mapper."); SERV_Reset(iter); if ( ! SERV_GetNextInfo(iter)) { CORE_LOG(eLOG_Error, "No services found after reset."); errors = 1; } } SERV_Close(iter); } else { errors = 1; } /* Search for matches unless this is a standalone run. */ if (check_for_match) { /* Search for perfect matches first (order is unknown). */ int it_exp, it_got; for (it_got=0; it_got < s_n_hits_got; ++it_got) { for (it_exp=0; it_exp < s_n_hits_exp; ++it_exp) { if (s_hits_exp[it_exp].match) continue; /*if (check_match(fMatch_Default, it_exp, it_got)) {*/ if (check_match(fMatch_All, it_exp, it_got)) { CORE_LOGF(eLOG_Note, ( " Found server %d perfectly matched expected server " "%d.", it_got, it_exp)); s_hits_exp[it_exp].match = 1; s_hits_got[it_got].match = 1; ++n_matches_perfect; break; } } } /* If not all found, search again but exclude host:port from match. */ for (it_got=0; it_got < s_n_hits_got; ++it_got) { if (s_hits_got[it_got].match) continue; for (it_exp=0; it_exp < s_n_hits_exp; ++it_exp) { if (s_hits_exp[it_exp].match) continue; if (check_match(fMatch_NoHostPort, it_exp, it_got)) { CORE_LOGF(eLOG_Note, ( " Found server %d nearly matched expected server %d.", it_got, it_exp)); s_hits_exp[it_exp].match = 1; s_hits_got[it_got].match = 1; ++n_matches_near; log_match_diffs(it_exp, it_got); break; } } } /* List any non-matching servers. */ for (it_exp=0; it_exp < s_n_hits_exp; ++it_exp) { if ( ! s_hits_exp[it_exp].match) CORE_LOGF(eLOG_Note, ( " Expected server %d didn't match any found servers.", it_exp)); } for (it_got=0; it_got < s_n_hits_got; ++it_got) { if ( ! s_hits_got[it_got].match) CORE_LOGF(eLOG_Note, ( " Found server %d didn't match any expected servers.", it_got)); } CORE_LOGF(n_matches_perfect + n_matches_near == s_n_hits_got ? eLOG_Note : eLOG_Error, ("Expected %d servers; found %d (%d perfect matches, %d near " "matches, and %d non-matches).", s_n_hits_exp, s_n_hits_got, n_matches_perfect, n_matches_near, s_n_hits_got - n_matches_perfect - n_matches_near)); if (!errors && s_n_hits_got == s_n_hits_exp && s_n_hits_got == n_matches_perfect + n_matches_near) { success = 1; } retval = (success != exp_err ? 1 : 0); CORE_LOGF(eLOG_Note, ("Test result: %s.", retval ? (success ? "PASS" : "PASS (with expected error)") : (success ? "FAIL (success when error expected)" : "FAIL"))); } out: if (mock_body_adj) free(mock_body_adj); return retval == -1 ? (success != exp_err ? 1 : 0) : retval; }
static void TEST_ConnNetInfo(void) { size_t n; char* str; char buf[80]; SConnNetInfo* net_info; CORE_LOG(eLOG_Note, "ConnNetInfo test started"); net_info = ConnNetInfo_Create(0); assert(net_info); assert(ConnNetInfo_ParseURL(net_info, "ftp://*****:*****@host:8888/ro.t/p@th" "?arg/arg:arg@arg:arg/arg")); assert(net_info->scheme == eURL_Ftp); assert(strcmp(net_info->user, "user") == 0); assert(strcmp(net_info->pass, "pass") == 0); assert(strcmp(net_info->host, "host") == 0); assert( net_info->port == 8888); assert(strcmp(net_info->path, "/ro.t/p@th") == 0); assert(strcmp(net_info->args, "arg/arg:arg@arg:arg/arg") == 0); assert(ConnNetInfo_ParseURL(net_info, "https://www/path" "?arg:arg@arg#frag")); assert( net_info->scheme == eURL_Https); assert( *net_info->user == 0); assert( *net_info->pass == 0); assert(strcmp(net_info->host, "www") == 0); assert( net_info->port == 0); assert(strcmp(net_info->path, "/path") == 0); assert(strcmp(net_info->args, "arg:arg@arg#frag") == 0); assert(ConnNetInfo_ParseURL(net_info, "/path1?arg1#frag2")); assert(strcmp(net_info->args, "arg1#frag2") == 0); assert(ConnNetInfo_ParseURL(net_info, "path0/0")); assert(strcmp(net_info->path, "/path0/0") == 0); assert(strcmp(net_info->args, "#frag2") == 0); assert(ConnNetInfo_ParseURL(net_info, "#frag3")); assert(strcmp(net_info->path, "/path0/0") == 0); assert(strcmp(net_info->args, "#frag3") == 0); assert(ConnNetInfo_ParseURL(net_info, "path2")); assert(strcmp(net_info->path, "/path0/path2") == 0); assert(strcmp(net_info->args, "#frag3") == 0); assert(ConnNetInfo_ParseURL(net_info, "/path3?arg3")); assert(strcmp(net_info->path, "/path3") == 0); assert(strcmp(net_info->args, "arg3#frag3") == 0); strcpy(net_info->user, "user"); strcpy(net_info->pass, "pass"); str = ConnNetInfo_URL(net_info); assert(str); assert(strcmp(str, "https://www/path3?arg3#frag3") == 0); free(str); assert(ConnNetInfo_ParseURL(net_info, "path4/path5?arg4#")); assert(strcmp(net_info->user, "user") == 0); assert(strcmp(net_info->pass, "pass") == 0); assert(strcmp(net_info->path, "/path4/path5") == 0); assert(strcmp(net_info->args, "arg4") == 0); assert(ConnNetInfo_ParseURL(net_info, "../path6")); assert(strcmp(net_info->path, "/path4/../path6") == 0); assert(strcmp(net_info->args, "") == 0); ConnNetInfo_SetUserHeader(net_info, ""); str = UTIL_PrintableString(net_info->http_user_header, 0, buf, 0); printf("HTTP User Header after set:\n%s%s%s\n", "\"" + !str, str ? buf : "NULL", "\"" + !str); assert(!net_info->http_user_header && !str); ConnNetInfo_AppendUserHeader(net_info, "T0: V0\n" "T1:V1\r\n" "T2: V2\r\n" "T3: V3\n" "T4: V4\n" "T1: V6"); str = UTIL_PrintableString(net_info->http_user_header, 0, buf, 0); if (str) *str = '\0'; printf("HTTP User Header after append:\n%s%s%s\n", "\"" + !str, str ? buf : "NULL", "\"" + !str); assert(strcmp(net_info->http_user_header, "T0: V0\n" "T1:V1\r\n" "T2: V2\r\n" "T3: V3\n" "T4: V4\n" "T1: V6\r\n") == 0); ConnNetInfo_OverrideUserHeader(net_info, "T0\r\n" "T5: V5\n" "T1: \t \r\n" "T2:V2.1\r\n" "T3:V3\r\n" "T4: W4"); str = UTIL_PrintableString(net_info->http_user_header, 0, buf, 0); if (str) *str = '\0'; printf("HTTP User Header after override:\n%s%s%s\n", "\"" + !str, str ? buf : "NULL", "\"" + !str); assert(strcmp(net_info->http_user_header, "T0: V0\n" "T2:V2.1\r\n" "T3:V3\n" "T4: W4\r\n" "T5: V5\r\n") == 0); ConnNetInfo_ExtendUserHeader(net_info, "T0: V0\n" "T1:V1\r\n" "T2:V2\n" "T3: T3:V3\n" "T4:\n" "T5"); str = UTIL_PrintableString(net_info->http_user_header, 0, buf, 0); if (str) *str = '\0'; printf("HTTP User Header after extend:\n%s%s%s\n", "\"" + !str, str ? buf : "NULL", "\"" + !str); assert(strcmp(net_info->http_user_header, "T0: V0\n" "T2:V2.1 V2\r\n" "T3:V3 T3:V3\n" "T4: W4\r\n" "T5: V5\r\n" "T1:V1\r\n") == 0); ConnNetInfo_SetUserHeader(net_info, 0); str = UTIL_PrintableString(net_info->http_user_header, 0, buf, 0); if (str) *str = '\0'; printf("HTTP User Header after reset:\n%s%s%s\n", "\"" + !str, str ? buf : "NULL", "\"" + !str); assert(!net_info->http_user_header); for (n = 0; n < sizeof(net_info->args); n++) net_info->args[n] = "0123456789"[rand() % 10]; strncpy0(net_info->args, "a=b&b=c&c=d", sizeof(net_info->args) - 1); printf("HTTP Arg: \"%s\"\n", net_info->args); ConnNetInfo_PrependArg(net_info, "d=e", 0); ConnNetInfo_PrependArg(net_info, "e", "f"); printf("HTTP Arg after prepend: \"%s\"\n", net_info->args); ConnNetInfo_AppendArg(net_info, "f=g", 0); ConnNetInfo_AppendArg(net_info, "g", "h"); printf("HTTP Arg after append: \"%s\"\n", net_info->args); ConnNetInfo_PreOverrideArg(net_info, "a=z&b", "y"); ConnNetInfo_PreOverrideArg(net_info, "c", "x"); printf("HTTP Arg after pre-override: \"%s\"\n", net_info->args); ConnNetInfo_PostOverrideArg(net_info, "d=w&e", "v"); ConnNetInfo_PostOverrideArg(net_info, "f", "u"); printf("HTTP Arg after post-override: \"%s\"\n", net_info->args); ConnNetInfo_DeleteArg(net_info, "g"); ConnNetInfo_DeleteArg(net_info, "h=n"); printf("HTTP Arg after delete: \"%s\"\n", net_info->args); ConnNetInfo_DeleteAllArgs(net_info, "a=b&p=q&f=d"); printf("HTTP Arg after delete-all: \"%s\"\n", net_info->args); ConnNetInfo_LogEx(net_info, eLOG_Note, CORE_GetLOG()); ConnNetInfo_Destroy(net_info); CORE_LOG(eLOG_Note, "ConnNetInfo test completed"); }