void read_respond_body(HWND hWnd) { acl_assert(hdr_res_); /* 如果 HTTP 响应没有数据体则仅输出 HTTP 响应头即可 */ if (hdr_res_->hdr.content_length == 0 || (hdr_res_->hdr.content_length == -1 && hdr_res_->reply_status > 300 && hdr_res_->reply_status < 400)) { data_.i_ptr = NULL; data_.i_dlen = 0; // 如果没有数据体,也发送消息通知调用者数据结束 report(hWnd, HTTP_MSG_DAT); return; } res_ = http_res_new(hdr_res_); #define BUF_LEN 8192 char* buf; int ret; while (true) { buf = (char*) acl_mymalloc(BUF_LEN); ret = read_respond_body(buf, BUF_LEN - 1); if (ret <= 0) { acl_myfree(buf); break; } buf[ret] = 0; data_.i_ptr = buf; data_.i_dlen = (size_t) ret; report(hWnd, HTTP_MSG_DAT); if (respond_over_) break; if (nwait_ > 0) acl_doze(nwait_); } data_.i_ptr = NULL; data_.i_dlen = 0; report(hWnd, HTTP_MSG_DAT); }
int http_client::read_response_body(char* buf, size_t size) { if (hdr_res_ == NULL) { logger_error("response header not get yet"); disconnected_ = true; return -1; } if (stream_ == NULL) { logger_error("not connected yet"); disconnected_ = true; return -1; } ACL_VSTREAM* vstream = stream_->get_vstream(); if (vstream == NULL) { logger_error("connect stream null"); disconnected_ = true; return -1; } if (res_ == NULL) res_ = http_res_new(hdr_res_); // 缓冲区太大了没有任何意义 if (size >= 1024000) size = 1024000; http_off_t ret = http_res_body_get_sync(res_, vstream, buf, size); if (ret <= 0) { // 如果在读响应头时调用了 unzip_begin,则必须保证读完数据 // 后再调用 unzip_finish,否则会因为 zlib 本身的设计而导致 // 内存泄露 if (zstream_ != NULL) { string dummy(64); zstream_->unzip_finish(&dummy); } body_finish_ = true; if (ret < 0) disconnected_ = true; } return (int) ret; }
HTTP_UTIL *http_util_req_new(const char *url, const char *method) { const char *myname = "http_util_req_new"; HTTP_UTIL *http_util = (HTTP_UTIL*) acl_mycalloc(1, sizeof(HTTP_UTIL)); const char *ptr, *host; if (url == NULL || *url == 0) { acl_msg_error("%s(%d): url invalid", myname, __LINE__); return (NULL); } if (method == NULL || *method == 0) { acl_msg_error("%s(%d): method invalid", myname, __LINE__); return (NULL); } if (acl_strncasecmp(url, "http://", sizeof("http://") - 1) != 0) { acl_msg_error("%s(%d): url(%s) invalid", myname, __LINE__, url); return (NULL); } if (strcmp(method, "GET") != 0 && strcmp(method, "POST") != 0 && strcmp(method, "HEAD") != 0 && strcmp(method, "CONNECT") != 0) { acl_msg_error("%s(%d): method(%s) invalid", myname, __LINE__, method); return (NULL); } http_util->hdr_req = http_hdr_req_create(url, method, "HTTP/1.1"); http_util->hdr_res = http_hdr_res_new(); http_util->http_res = http_res_new(http_util->hdr_res); http_util->req_buf = acl_vstring_alloc(4096); host = http_hdr_req_host(http_util->hdr_req); if (host) { ptr = strchr(host, ':'); if (ptr == NULL) snprintf(http_util->server_addr, sizeof(http_util->server_addr), "%s:80", host); else ACL_SAFE_STRNCPY(http_util->server_addr, host, sizeof(http_util->server_addr)); } http_util->conn_timeout = 10; http_util->rw_timeout = 10; return (http_util); }
int http_client::read_response_body(string& out, bool clean, int* real_size) { if (real_size) *real_size = 0; if (body_finish_) return (last_ret_); if (stream_ == NULL) { logger_error("connect null"); disconnected_ = true; return -1; } ACL_VSTREAM* vstream = stream_->get_vstream(); if (vstream == NULL) { logger_error("connect stream null"); disconnected_ = true; return -1; } if (hdr_res_ == NULL) { logger_error("response header not get yet"); disconnected_ = true; return -1; } if (res_ == NULL) res_ = http_res_new(hdr_res_); if (clean) out.clear(); int saved_count = (int) out.length(); char buf[8192]; SKIP_GZIP_HEAD_AGAIN: // 对于有 GZIP 头数据,可能需要重复读 int ret = (int) http_res_body_get_sync(res_, vstream, buf, sizeof(buf)); if (zstream_ == NULL) { if (ret > 0) { out.append(buf, ret); if (real_size) *real_size = ret; } else { body_finish_ = true; // 表示数据已经读完 if (ret < 0) disconnected_ = true; last_ret_ = ret; } return ret; } if (ret <= 0) { if (zstream_->unzip_finish(&out) == false) { logger_error("unzip_finish error"); return -1; } last_ret_ = ret; // 记录返回值 body_finish_ = true; // 表示数据已经读完且解压缩完毕 if (ret < 0) disconnected_ = true; return (int) out.length() - saved_count; } if (real_size) (*real_size) += ret; // 需要先跳过 gzip 头 if (gzip_header_left_ >= ret) { gzip_header_left_ -= ret; goto SKIP_GZIP_HEAD_AGAIN; } int n; if (gzip_header_left_ > 0) { n = gzip_header_left_; gzip_header_left_ = 0; } else n = 0; if (zstream_->unzip_update(buf + n, ret - n, &out) == false) { logger_error("unzip_update error"); return -1; } return (int) out.length() - saved_count; }
void *CHttpClient::DoRequestThread(void* arg) { CHttpClient *phClient = (CHttpClient*) arg; HTTP_HDR_REQ *hdr_req = NULL; HTTP_HDR_RES *hdr_res = NULL; HTTP_RES *http_res = NULL; ACL_VSTRING *buf = acl_vstring_alloc(256); ACL_VSTREAM *server = NULL; const char *pHost = NULL; ACL_VSTREAM *fp = NULL; int ret; UINT nForward = 0; time_t begin = 0; struct timeval begin_tv, end_tv; double time_cost = 0; #undef RETURN #define RETURN(_x_) do \ { \ if (hdr_req) \ http_hdr_req_free(hdr_req); \ if (buf) \ acl_vstring_free(buf); \ if (hdr_res) \ http_hdr_res_free(hdr_res); \ if (http_res) { \ http_res->hdr_res = NULL; \ http_res_free(http_res); \ } \ if (server) \ acl_vstream_close(server); \ if (fp) \ acl_vstream_close(fp); \ gettimeofday(&end_tv, NULL); \ time_cost = stamp_sub(&end_tv, &begin_tv); \ if (time_cost >= 0) \ phClient->ReportTime(time_cost); \ phClient->ReportComplete(); \ return (_x_); \ } while(0) const char *pUrl = phClient->m_sReqUrl.GetString(); hdr_req = http_hdr_req_create(pUrl, phClient->m_bPostMethod ? "POST" : "GET", phClient->m_bHttp11 ? "HTTP/1.1" : "HTTP/1.0"); ASSERT(hdr_req); if (phClient->m_bZip) http_hdr_put_str(&hdr_req->hdr, "Accept-Encoding", "gzip, deflate"); if (phClient->m_bKeepAlive) http_hdr_entry_replace(&hdr_req->hdr, "Connection", "Keep-Alive", 1); if (phClient->m_bPostMethod) http_hdr_put_int(&hdr_req->hdr, "Content-Length", (int) phClient->m_sHttpBody.GetLength()); if (!phClient->m_sAccept.IsEmpty()) http_hdr_put_str(&hdr_req->hdr, "Accept", phClient->m_sAccept.GetString()); if (!phClient->m_sCtype.IsEmpty()) http_hdr_put_str(&hdr_req->hdr, "Content-Type", phClient->m_sCtype.GetString()); if (phClient->m_sHttpHdrAppend.GetLength() > 0) { ACL_ARGV *argv; HTTP_HDR_ENTRY *entry; int i; argv = acl_argv_split(phClient->m_sHttpHdrAppend.GetString(), "\r\n"); for (i = 0; i < argv->argc; i++) { entry = http_hdr_entry_new(argv->argv[i]); if (entry == NULL) continue; http_hdr_append_entry(&hdr_req->hdr, entry); } acl_argv_free(argv); } FORWARD: http_hdr_build_request(hdr_req, buf); pHost = http_hdr_req_host(hdr_req); ASSERT(pHost); phClient->ReportReqHdr(acl_vstring_str(buf), (int) ACL_VSTRING_LEN(buf)); CString serverAddr; if (phClient->m_bUseAddr) serverAddr.Format("%s", phClient->m_sServerAddr); if (serverAddr.GetLength() == 0) serverAddr.Format("%s", pHost); if (strchr(serverAddr.GetString(), ':') == NULL) serverAddr.AppendFormat(":80"); time(&begin); gettimeofday(&begin_tv, NULL); server = acl_vstream_connect(serverAddr.GetString(), ACL_BLOCKING, 10, 10, 4096); if (server == NULL) { CString msg; msg.Format("Connect server(%s) error", serverAddr.GetString()); phClient->ReportErrConnect(msg.GetString()); RETURN (NULL); } if (phClient->m_bLocalSave && fp == NULL) { fp = acl_vstream_fopen(phClient->m_sLocalFile.GetString(), O_WRONLY | O_CREAT | O_TRUNC, 0600, 4096); if (fp == NULL) { acl_msg_error("%s(%d): can't create file(%s)", __FILE__, __LINE__, phClient->m_sLocalFile.GetString()); RETURN (NULL); } } ret = acl_vstream_writen(server, acl_vstring_str(buf), ACL_VSTRING_LEN(buf)); if (ret == ACL_VSTREAM_EOF) { acl_msg_error("%s(%d): write error", __FILE__, __LINE__); RETURN (NULL); } if (phClient->m_bPostMethod && !phClient->m_sHttpBody.IsEmpty()) { if (acl_vstream_writen(server, phClient->m_sHttpBody.GetString(), phClient->m_sHttpBody.GetLength()) == ACL_VSTREAM_EOF) { acl_msg_error("%s(%d): write body error", __FILE__, __LINE__); RETURN (NULL); } } hdr_res = http_hdr_res_new(); if (http_hdr_res_get_sync(hdr_res, server, 10) < 0) { acl_msg_error("%s(%d): get res hdr error", __FILE__, __FILE__, __LINE__); RETURN (NULL); } if (http_hdr_res_parse(hdr_res) < 0) { acl_msg_error("%s(%d): parse hdr_res error", __FILE__, __LINE__); RETURN (NULL); } http_hdr_build(&hdr_res->hdr, buf); phClient->ReportResHdr(acl_vstring_str(buf), (int) ACL_VSTRING_LEN(buf)); phClient->ReportResContentLength((int) hdr_res->hdr.content_length); if (hdr_res->reply_status > 300 && hdr_res->reply_status < 400) { const char* pLocation; HTTP_HDR_REQ *hdr_req_tmp; if (!phClient->m_bForwardAuto) RETURN (NULL); if (nForward++ >= phClient->m_nMaxTry) { acl_msg_error("%s(%d): too many redirect, nForward(%d)", __FILE__, __LINE__, nForward); RETURN (NULL); } pLocation = http_hdr_entry_value(&hdr_res->hdr, "Location"); if (pLocation == NULL || *pLocation == 0) { acl_msg_error("%s(%d): 302 reply with no Location", __FILE__, __LINE__); RETURN (NULL); } hdr_req_tmp = http_hdr_req_rewrite(hdr_req, pLocation); if (hdr_req_tmp == NULL) RETURN (NULL); http_hdr_req_free(hdr_req); http_hdr_res_free(hdr_res); hdr_req = hdr_req_tmp; goto FORWARD; } http_res = http_res_new(hdr_res); while (1) { char tmp_buf[4096]; ret = (int) http_res_body_get_sync2(http_res, server, tmp_buf, sizeof(tmp_buf) - 1); if (ret <= 0) break; phClient->ReportResBody(tmp_buf, ret); phClient->ReportResDownLength((int)(time(NULL) - begin), ret); if (fp != NULL && acl_vstream_writen(fp, tmp_buf, ret) == ACL_VSTREAM_EOF) { acl_msg_error("%s(%d): write to file error", __FILE__, __LINE__); break; } } RETURN (NULL); }
static void get_url(const char *method, const char *url, const char *proxy, const char *dump) { /* 创建 HTTP 请求头 */ HTTP_HDR_REQ *hdr_req = http_hdr_req_create(url, method, "HTTP/1.1"); ACL_VSTREAM *stream; ACL_VSTRING *buf = acl_vstring_alloc(256); HTTP_HDR_RES *hdr_res; HTTP_RES *res; ACL_FILE *fp = NULL; const char *ptr; int ret; /* 输出 HTTP 请求头内容 */ http_hdr_print(&hdr_req->hdr, "---request hdr---"); /* 如果设定代理服务器,则连接代理服务器地址, * 否则使用 HTTP 请求头里指定的地址 */ if (*proxy) acl_vstring_strcpy(buf, proxy); else acl_vstring_strcpy(buf, http_hdr_req_host(hdr_req)); /* 获得远程 HTTP 服务器的连接地址 */ ptr = acl_vstring_memchr(buf, ':'); if (ptr == NULL) acl_vstring_strcat(buf, ":80"); else { int port; ptr++; port = atoi(ptr); if (port <= 0 || port >= 65535) { printf("http server's addr(%s) invalid\n", acl_vstring_str(buf)); acl_vstring_free(buf); http_hdr_req_free(hdr_req); return; } } /* 连接远程 http 服务器 */ stream = acl_vstream_connect(acl_vstring_str(buf) /* 服务器地址 */, ACL_BLOCKING /* 采用阻塞方式 */, 10 /* 连接超时时间为 10 秒 */, 10 /* 网络 IO 操作超时时间为 10 秒 */, 4096 /* stream 流缓冲区大小为 4096 字节 */); if (stream == NULL) { printf("connect addr(%s) error(%s)\n", acl_vstring_str(buf), acl_last_serror()); acl_vstring_free(buf); http_hdr_req_free(hdr_req); return; } /* 构建 HTTP 请求头数据 */ http_hdr_build_request(hdr_req, buf); /* 向 HTTP 服务器发送请求 */ ret = acl_vstream_writen(stream, acl_vstring_str(buf), ACL_VSTRING_LEN(buf)); if (ret == ACL_VSTREAM_EOF) { printf("write to server error(%s)\n", acl_last_serror()); acl_vstream_close(stream); acl_vstring_free(buf); http_hdr_req_free(hdr_req); return; } /* 创建一个 HTTP 响应头对象 */ hdr_res = http_hdr_res_new(); /* 读取 HTTP 服务器响应头*/ ret = http_hdr_res_get_sync(hdr_res, stream, 10 /* IO 超时时间为 10 秒 */); if (ret < 0) { printf("get http reply header error(%s)\n", acl_last_serror()); http_hdr_res_free(hdr_res); acl_vstream_close(stream); acl_vstring_free(buf); http_hdr_req_free(hdr_req); return; } if (http_hdr_res_parse(hdr_res) < 0) { printf("parse http reply header error\n"); http_hdr_print(&hdr_res->hdr, "--- reply http header ---"); http_hdr_res_free(hdr_res); acl_vstream_close(stream); acl_vstring_free(buf); http_hdr_req_free(hdr_req); return; } /* 如果需要转储至磁盘则需要先打开文件 */ if (dump != NULL) { fp = acl_fopen(dump, "w+"); if (fp == NULL) printf("open file(%s) error(%s)\n", dump, acl_last_serror()); } /* 如果 HTTP 响应没有数据体则仅输出 HTTP 响应头即可 */ if (hdr_res->hdr.content_length == 0 || (hdr_res->hdr.content_length == -1 && !hdr_res->hdr.chunked && hdr_res->reply_status > 300 && hdr_res->reply_status < 400)) { if (fp) http_hdr_fprint(ACL_FSTREAM(fp), &hdr_res->hdr, "--- reply http header ---"); else http_hdr_fprint(ACL_VSTREAM_OUT, &hdr_res->hdr, "--- reply http header ---"); http_hdr_res_free(hdr_res); acl_vstream_close(stream); acl_vstring_free(buf); http_hdr_req_free(hdr_req); return; } /* 输出 HTTP 响应头 */ http_hdr_print(&hdr_res->hdr, "--- reply http header ---"); /* 创建 HTTP 响应体对象 */ res = http_res_new(hdr_res); /* 如果有数据体则开始读取 HTTP 响应数据体部分 */ while (1) { http_off_t n; char buf2[4096]; n = http_res_body_get_sync(res, stream, buf2, sizeof(buf2) - 1); if (n <= 0) break; if (fp) { if (acl_fwrite(buf2, (size_t) n, 1, fp) == (size_t) EOF) { printf("write to dump file(%s) error(%s)\n", dump, acl_last_serror()); break; } } else { buf2[n] = 0; printf("%s", buf2); } } if (fp) acl_fclose(fp); http_res_free(res); /* 释放 HTTP 响应对象, hdr_res 会在此函数内部自动被释放 */ acl_vstream_close(stream); /* 关闭网络流 */ acl_vstring_free(buf); /* 释放内存区 */ http_hdr_req_free(hdr_req); /* 释放 HTTP 请求头对象 */ }