void print_response(HttpResponse& response, TcpClientSocket& socket) { std::cout << narrow(widen(response.getStatusLine(), CP_UTF8), CP_OEMCP) << std::endl; auto headers = response.getHeaders(); for (auto it = std::begin(headers), eit = std::end(headers); it != eit; ++it) { for (auto vit = std::begin(it->second), veit = std::end(it->second); vit != veit; ++vit) { std::cout << it->first << ": " << *vit << std::endl; } } std::cout << "======END OF HEADERS======" << std::endl; auto& buffer = response.getBuffer(); if (response.getIsChunked()) { std::cout << "[[CHUNKED ENCODING]]" << std::endl; for (auto chunk = buffer.getchunk(); !chunk.empty(); chunk = buffer.getchunk()) { std::cout << chunk; } std::cout << "[[TRAILING HEADERS]]" << std::endl; for (auto th = buffer.getline(); !th.empty(); th = buffer.getline()) { std::cout << th << std::endl; } } else if (HttpResponse::nlen != response.getContentLength()) { std::cout << "[[EXACTLY " << response.getContentLength() << " BYTES]]" << std::endl; std::cout << buffer.getcount(response.getContentLength()) << std::endl; } else { std::vector<BYTE> buff(8 * 1024, 0); for (auto last = socket.recv_upto(std::begin(buff), std::end(buff)); last != std::begin(buff); last = socket.recv_upto(std::begin(buff), std::end(buff))) { std::cout << std::string(std::begin(buff), last); } } std::cout << std::endl << "======END OF RESPONSE======" << std::endl; }
void WebClientApp::onReadComplete() { mText.push_back( "Read complete" ); console() << "HTTP version: "; switch ( mHttpResponse.getHttpVersion() ) { case HttpVersion::HTTP_0_9: console() << "0.9"; break; case HttpVersion::HTTP_1_0: console() << "1.0"; break; case HttpVersion::HTTP_1_1: console() << "1.1"; break; case HttpVersion::HTTP_2_0: console() << "2.0"; break; } console() << endl; console() << "Status code: " << mHttpResponse.getStatusCode() << endl; console() << "Reason: " << mHttpResponse.getReason() << endl; console() << "Header fields: " << endl; for ( const KeyValuePair& kvp : mHttpResponse.getHeaders() ) { console() << ">> " << kvp.first << ": " << kvp.second << endl; } console() << endl; console() << "Body:" << endl; console() << HttpResponse::bufferToString( mHttpResponse.getBody() ) << endl; mSession->close(); }
void spider::PageDownloader::handleRedirect(HttpResponse & response) const { using std::string; using std::vector; using boost::optional; HeaderCollection const& headers = response.getHeaders(); if (!headers.hasHeader("Location")) { return; } string urlString = headers.getHeader("Location").getValue(0); optional<Url> referrer = getReferrer(); vector<Url> redirectUrls { Url::parse(urlString) }; m_manager.download( referrer, redirectUrls.begin(), redirectUrls.end()); }
static ngx_int_t ngx_http_ffeadcpp_module_handler_post_read(ngx_http_request_t *r) { string cntpath = ""; cntpath.append(ffeadcpp_path.data, ffeadcpp_path.len); cntpath += "/web/"; HttpRequest* req = new HttpRequest(cntpath); //cerr << "FFEAD in ngx_http_ffeadcpp_module_handler " << cntpath << r->uri.data << endl; ngx_int_t rc; ngx_buf_t *b; ngx_chain_t out; ngx_list_part_t *part; ngx_table_elt_t *h; ngx_uint_t i; /* Get the first part of the list. There is usual only one part. */ part = &r->headers_in.headers.part; h = part->elts; /* Headers list array may consist of more than one part, so loop through all of it */ for (i = 0; /* void */ ; i++) { if (i >= part->nelts) { if (part->next == NULL) { /* The last part, search is done. */ break; } part = part->next; h = part->elts; i = 0; } // cerr << "header -> " << string(h[i].key.data, h[i].key.len) << " = " << string(h[i].value.data, h[i].value.len) << endl; req->buildRequest(string(h[i].key.data, h[i].key.len), string(h[i].value.data, h[i].value.len)); } string content; ngx_http_read_input_data(r, content); //cerr << "Input Request Data\n " << content << "\n======================\n" << endl; //cerr << "URL -> " << string(r->uri.data,r->uri.len) << endl; //cerr << "Method -> " << string(r->main->method_name.data, r->main->method_name.len) << endl; if(content!="") { req->buildRequest("Content", content.c_str()); } req->buildRequest("URL", string(r->uri.data,r->uri.len)); req->buildRequest("Method", string(r->main->method_name.data, r->main->method_name.len)); if(r->args.len > 0) { req->buildRequest("GetArguments", string(r->args.data,r->args.len)); } req->buildRequest("HttpVersion", CastUtil::lexical_cast<string>(r->http_version)); HttpResponse* respo = new HttpResponse; ServiceTask* task = new ServiceTask; task->handle(req, respo); delete task; cerr << req->toString() << endl; if(respo->isDone()) { for (int var = 0; var < (int)respo->getCookies().size(); var++) { set_custom_header_in_headers_out(r, "Set-Cookie", respo->getCookies().at(var)); } /* allocate a buffer for your response body */ b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t)); if (b == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } /* attach this buffer to the buffer chain */ out.buf = b; out.next = NULL; string data = respo->generateResponse(false); //cerr << "OUT -> " << data << endl; map<string,string>::const_iterator it; for(it=respo->getHeaders().begin();it!=respo->getHeaders().end();it++) { if(StringUtil::toLowerCopy(it->first)==StringUtil::toLowerCopy(HttpResponse::ContentLength)) { r->headers_out.content_length_n = CastUtil::lexical_cast<int>(it->second); } else { set_custom_header_in_headers_out(r, it->first, it->second); } } //cerr << "done writing headers" << endl; /* adjust the pointers of the buffer */ b->pos = data.c_str(); b->last = data.length(); b->memory = 1; /* this buffer is in memory */ b->last_buf = 1; /* this is the last buffer in the buffer chain */ /* set the status line */ r->headers_out.status = CastUtil::lexical_cast<int>(respo->getStatusCode()); delete respo; delete req; /* send the headers of your response */ rc = ngx_http_send_header(r); //cerr << "done sending headers " << rc << " " << NGX_OK <<" "<< r->header_only <<" " << NGX_ERROR << endl; if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { return rc; } /* send the buffer chain of your response */ return ngx_http_output_filter(r, &out); } else { u_char *last, *location; size_t len; ngx_str_t path; ngx_uint_t level; ngx_log_t *log; ngx_open_file_info_t of; ngx_http_core_loc_conf_t *clcf; clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); log = r->connection->log; ngx_memzero(&of, sizeof(ngx_open_file_info_t)); of.read_ahead = clcf->read_ahead; of.directio = clcf->directio; of.valid = clcf->open_file_cache_valid; of.min_uses = clcf->open_file_cache_min_uses; of.errors = clcf->open_file_cache_errors; of.events = clcf->open_file_cache_events; path.data = req->getUrl().c_str(); path.len = ngx_strlen(path.data); delete respo; delete req; rc = NGX_OK; if (ngx_http_set_disable_symlinks(r, clcf, &path, &of) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } cerr << string(path.data,path.len) << endl; if (ngx_open_cached_file(clcf->open_file_cache, &path, &of, r->pool) != NGX_OK) { switch (of.err) { case 0: rc = NGX_HTTP_INTERNAL_SERVER_ERROR; break; case NGX_ENOENT: case NGX_ENOTDIR: case NGX_ENAMETOOLONG: level = NGX_LOG_ERR; rc = NGX_HTTP_NOT_FOUND; break; case NGX_EACCES: #if (NGX_HAVE_OPENAT) case NGX_EMLINK: case NGX_ELOOP: #endif level = NGX_LOG_ERR; rc = NGX_HTTP_FORBIDDEN; break; default: level = NGX_LOG_CRIT; rc = NGX_HTTP_INTERNAL_SERVER_ERROR; break; } if (rc != NGX_HTTP_NOT_FOUND || clcf->log_not_found) { ngx_log_error(level, log, of.err, "%s \"%s\" faileddddddddddddddd", of.failed, path.data); } //return rc; } r->root_tested = !r->error_page; cerr << "found file" << endl; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, "http static fd: %d", of.fd); /*if (of.is_dir) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, log, 0, "http dir"); ngx_http_clear_location(r); r->headers_out.location = ngx_palloc(r->pool, sizeof(ngx_table_elt_t)); if (r->headers_out.location == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } len = r->uri.len + 1; if (!clcf->alias && clcf->root_lengths == NULL && r->args.len == 0) { location = path.data + clcf->root.len; *last = '/'; } else { if (r->args.len) { len += r->args.len + 1; } location = ngx_pnalloc(r->pool, len); if (location == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } last = ngx_copy(location, r->uri.data, r->uri.len); *last = '/'; if (r->args.len) { *++last = '?'; ngx_memcpy(++last, r->args.data, r->args.len); } } */ /* * we do not need to set the r->headers_out.location->hash and * r->headers_out.location->key fields */ /*r->headers_out.location->value.len = len; r->headers_out.location->value.data = location; return NGX_HTTP_MOVED_PERMANENTLY; }*/ #if !(NGX_WIN32) /* the not regular files are probably Unix specific */ if (!of.is_file) { ngx_log_error(NGX_LOG_CRIT, log, 0, "\"%s\" is not a regular file", path.data); rc = NGX_HTTP_NOT_FOUND; } #endif /* if (r->method == NGX_HTTP_POST) { return NGX_HTTP_NOT_ALLOWED; }*/ if (rc != NGX_OK) { r->headers_out.status = rc; ngx_http_send_header(r); return ngx_http_send_special (r, NGX_HTTP_LAST); } /*rc = ngx_http_discard_request_body(r); if (rc != NGX_OK) { return rc; }*/ log->action = "sending response to client"; r->headers_out.status = NGX_HTTP_OK; r->headers_out.content_length_n = of.size; r->headers_out.last_modified_time = of.mtime; /*if (ngx_http_set_etag(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; }*/ if (ngx_http_set_content_type(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } if (r != r->main && of.size == 0) { return ngx_http_send_header(r); } r->allow_ranges = 1; /* we need to allocate all before the header would be sent */ b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t)); if (b == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } b->file = ngx_pcalloc(r->pool, sizeof(ngx_file_t)); if (b->file == NULL) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } rc = ngx_http_send_header(r); if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { return rc; } b->file_pos = 0; b->file_last = of.size; b->in_file = b->file_last ? 1: 0; b->last_buf = (r == r->main) ? 1: 0; b->last_in_chain = 1; b->file->fd = of.fd; b->file->name = path; b->file->log = log; b->file->directio = of.is_directio; out.buf = b; out.next = NULL; //return ngx_http_output_filter(r, &out); ngx_http_finalize_request(r, ngx_http_output_filter(r, &out));return NGX_DONE; /*cerr << req->getUrl() << " " << cntpath << endl; cntpath = cntpath.substr(0, cntpath.length()-1); string rurl = req->getUrl(); StringUtil::replaceAll(cntpath, "//", "/"); StringUtil::replaceFirst(rurl, cntpath, ""); cerr << req->getUrl() << " " << cntpath << " " << rurl << endl; ngx_str_t nuri; nuri.data = rurl.c_str(); nuri.len = ngx_strlen(nuri.data); return ngx_http_internal_redirect(r, &nuri, NULL);*/ } }
void HttpClientApp::onRead( ci::BufferRef buffer ) { size_t sz = buffer->getSize(); mBytesRead += sz; mText.push_back( toString( sz ) + " bytes read" ); if ( !mHttpResponse.hasHeader() ) { // Parse header mHttpResponse.parseHeader( HttpResponse::bufferToString( buffer ) ); buffer = HttpResponse::removeHeader( buffer ); // Get content-length for ( const KeyValuePair& kvp : mHttpResponse.getHeaders() ) { if ( kvp.first == "Content-Length" ) { mContentLength = fromString<size_t>( kvp.second ); break; } } } // Append buffer to body mHttpResponse.append( buffer ); if ( mBytesRead < mContentLength ) { // Keep reading until we hit the content length mSession->read(); } else { mText.push_back( "Read complete" ); mText.push_back( toString( mHttpResponse.getStatusCode() ) + " " + mHttpResponse.getReason() ); if ( mHttpResponse.getStatusCode() == 200 ) { for ( const KeyValuePair& kvp : mHttpResponse.getHeaders() ) { // Choose file extension based on MIME type if ( kvp.first == "Content-Type" ) { string mime = kvp.second; if ( mime == "audio/mp3" ) { mFilename += ".mp3"; } else if ( mime == "image/jpeg" ) { mFilename += ".jpg"; } else if ( mime == "image/png" ) { mFilename += ".png"; } } else if ( kvp.first == "Connection" ) { // Close connection if requested by server if ( kvp.second == "close" ) { mSession->close(); } } } // Save the file fs::path path = getAppPath(); #if !defined ( CINDER_MSW ) path = path.parent_path(); #endif path = path / mFilename; OStreamFileRef file = writeFileStream( path ); file->write( mHttpResponse.getBody() ); mText.push_back( mFilename + " downloaded" ); } else { // Write error mText.push_back( "Response: " + HttpResponse::bufferToString( mHttpResponse.getBody() ) ); mSession->close(); } } }