static void couchLoadData(PrintTextA &print) { CouchDB db(getTestCouch()); db.use(DATABASENAME); AutoArray<Document, SmallAlloc<50> > savedDocs; Changeset chset(db.createChangeset()); natural id=10000; JSON::Value data = db.json.factory->fromString(strdata); for (JSON::Iterator iter = data->getFwIter(); iter.hasItems();) { const JSON::KeyValue &kv= iter.getNext(); Document doc; doc.edit(db.json) ("name",kv[0]) ("age",kv[1]) ("height",kv[2]) ("_id",ToString<natural>(id,16)); id+=14823; savedDocs.add(doc); chset.update(doc); } chset.commit(false); Set<StringA> uuidmap; for (natural i = 0; i < savedDocs.length(); i++) { StringA uuid = savedDocs[i]["_id"]->getStringUtf8(); // print("%1\n") << uuid; uuidmap.insert(uuid); } print("%1") << uuidmap.size(); }
int LightSpeed::AppBase::main_entry( int argc, wchar_t *argv[], ConstStrW pathname) { AutoArray<ConstStrW> params; params.reserve(argc); for (int i = 0; i < argc; i++) params.add(ConstStrW(argv[i])); appPathname = pathname; return startApp(Args(params)); }
void LinuxNetworkEventListener::workerProc() { while (!Thread::canFinish()) { try { PollBase::Result res; PollBase::WaitStatus wt = fdSelect.wait(nil, res); if (wt == PollBase::waitWakeUp) { while (pumpMessage()); } else { AutoArray<std::pair<ISleepingObject *, natural>,SmallAlloc<32> > tocall; SysTime tm = SysTime::now(); FdData *listeners = reinterpret_cast<FdData *>(res.userData); for (ListenerMap::Iterator iter = listeners->listeners.getFwIter(); iter.hasItems();) { const FdListener &l = iter.peek(); if (res.flags & l.waitMask) { tocall.add(std::make_pair(l.notify,res.flags & l.waitMask)); listeners->listeners.erase(iter); // l.notify->wakeUp(con.waitMask & l.waitMask); } else if (l.waitTimeout.expired(tm)) { tocall.add(std::make_pair(l.notify,0)); listeners->listeners.erase(iter); // l.notify->wakeUp(0); } else { iter.skip(); } } updateFdData(listeners,res.fd); for (natural i = 0; i < tocall.length(); i++) { tocall[i].first->wakeUp(tocall[i].second); } } } catch (const Exception &e) { AppBase::current().onThreadException(e); } catch (const std::exception &e) { AppBase::current().onThreadException(StdException(THISLOCATION,e)); } catch (...) { AppBase::current().onThreadException(UnknownException(THISLOCATION)); } } fdSelect.cancelAll(CleanUpProc(this)); /* fdSelect.dropAll(); while (fdSelect.hasItems()) { const LinuxFdSelect::FdInfo &con = fdSelect.getNext(); FdData *listeners = reinterpret_cast<FdData *>(con.data); if (listeners) delete listeners; fdSelect.unset(con.fd); } */ }
int LightSpeed::AppBase::main_entry( int argc, char *argv[], ConstStrW pathname) { AutoArray<String> params; AutoArray<ConstStrW> wparams; params.reserve(argc); for (int i = 0; i < argc; i++) params.add(String(argv[i])); wparams.append(params); appPathname = pathname; return startApp(Args(wparams)); }
LightSpeed::StringA flagsToDB(const LightSpeed::JSON::INode &nd) { using namespace LightSpeed; if (nd.getType() == JSON::ndString) return nd.getStringUtf8(); AutoArray<char, SmallAlloc<256> > buff; ConstStrA comma(","); ConstStrA div = comma.head(0); for(natural i = 0; i < nd.getEntryCount(); i++) { buff.append(div); div = comma; buff.append(nd[i].getString()); } return buff; }
void get(ReadIterator & source, PlanetNodeStatusMessageData &d) { get(source,d.m_x); get(source,d.m_z); get(source,d.m_loaded); AutoArray<uint32> temp; get(source,temp); d.m_servers=temp.get(); AutoArray<int> temp2; get(source,temp2); d.m_subscriptionCounts=temp2.get(); }
void put(ByteStream & target, const PlanetNodeStatusMessageData &d) { put(target,d.m_x); put(target,d.m_z); put(target,d.m_loaded); AutoArray<uint32> temp; temp.set(d.m_servers); put(target,temp); AutoArray<int> temp2; temp2.set(d.m_subscriptionCounts); put(target,temp2); }
void write(iterator rsa, iterator rse) { lock.lock(); expandBuf(rse-rsa); std::copy ( rsa, rse, buf.get() ); u_int8_t const * sa = buf.get(); u_int8_t const * se = sa + (rse-rsa); while ( sa != se ) { ssize_t const wr = ::write(fd,sa,se-sa); if ( wr <= 0 ) { switch ( errno ) { case EINTR: break; default: std::cerr << "write() failed with error code " << errno << ": " << strerror(errno) << " wr " << wr << std::endl; throw std::runtime_error("write() failed."); break; } } else { sa += wr; } } lock.unlock(); }
bool WinHttpStream::enumHeaders( IEnumHeaders &enumHdr ) { setState(stReadResponse); AutoArray<char> buffer; buffer.resize(1000); DWORD size = (DWORD)buffer.length(); while (!HttpQueryInfoA(hHTTPConn,HTTP_QUERY_RAW_HEADERS, buffer.data(),&size,0)) { DWORD res = GetLastError(); if (res == ERROR_INSUFFICIENT_BUFFER) { buffer.resize(size); } else { throw ErrNoWithDescException(THISLOCATION,GetLastError(), "Cannot retrieve headers from the request"); } } buffer.resize(size); AutoArray<char>::SplitIterator iter=buffer.split((char)0); TextParser<char> parser; while (iter.hasItems()) { ConstStrA line = iter.getNext(); if (parser("%1 : %2",line)) { ConstStrA field = parser[1].str(); ConstStrA value = parser[2].str(); if(enumHdr(field, value)) return true; } } return false; }
LightSpeed::StringA dbOrderFromJSON(const LightSpeed::JSON::INode &nd) { using namespace LightSpeed; AutoArray<char, SmallAlloc<256> > buff; for (natural i = 0; i < nd.getEntryCount();i++) { ConstStrA fld = nd[i].getStringUtf8(); if (fld.empty() || fld == ConstStrA('^') || fld.find('`') != naturalNull) throw ErrorMessageException(THISLOCATION, "Unacceptable field name"); if (i) buff.append(ConstStrA(", ")); if (fld[0] == '^') { buff.add('`');buff.append(fld.offset(1));buff.add('`'); buff.append(ConstStrA(" DESC")); } else { buff.add('`');buff.append(fld);buff.add('`'); buff.append(ConstStrA(" ASC")); } } return ConstStrA(buff); }
RangeVector(std::vector < std::pair < std::string, u_int64_t > > const & rranges) : ranges(rranges), AB ( fillRange(ranges) ), B ( AB.get() ), R(B, ((ranges.at(ranges.size()-1).second+1+63)/64)*64 ) { #if 0 for ( u_int64_t i = 0; i <= ranges.back().second; ++i ) std::cerr << Rank::getBit(B,i) << "(" << R.rank1(i)-1 << ")"; std::cerr << std::endl; #endif }
RefPtr<SourceFile> SourceManager::open(ReportingContext &cc, const char *path) { Atom *atom = strings_.add(path); AtomMap<RefPtr<SourceFile>>::Insert p = file_cache_.findForAdd(atom); if (p.found()) return p->value; FileReader reader(cc, path); if (!reader.isValid()) return nullptr; uint32_t length; AutoArray<char> chars; if (!reader.read(chars.address(), &length)) return nullptr; RefPtr<SourceFile> file = new SourceFile(chars.forget(), length, path); file_cache_.add(p, atom, file); return file; }
integer ServiceApp::postMessage(ConstStrA command, const Args & args, SeqFileOutput output) { if (instance->imOwner()) return onMessage(command,args,output); AutoArrayStream<char, StaticAlloc<65536> > buffer; if (args.length() > 127) return onCommandLineError(args); buffer.write((char)(args.length())); buffer.copy(command.getFwIter()); buffer.write(0); for(natural i = 0, cnt = args.length();i < cnt;i++){ buffer.copy(WideToUtf8Reader<ConstStrW::Iterator>(args[i].getFwIter())); buffer.write(0); } AutoArray<byte,SmallAlloc<4096> > replybuff; replybuff.resize(instance->getReplyMaxSize()); natural sz = instance->request(buffer.data(), buffer.length(), replybuff.data(), replybuff.length(), timeout); if (sz < sizeof(integer)) throw ErrorMessageException(THISLOCATION,String("Invalid reply")); else { integer res = *reinterpret_cast<integer *>(replybuff.data()); ConstStrA msg(reinterpret_cast<char *>(replybuff.data()+sizeof(integer)),sz - sizeof(integer)); output.copy(msg.getFwIter()); return res; } }
void setup(unsigned int npatl) { if ( npatl > maxpatl ) { ABstraight = AutoArray<u_int64_t>(RestMatch<sse4>::getRestWords(npatl,seedl)); Bstraight = ABstraight.get(); ABreverse = AutoArray<u_int64_t>(RestMatch<sse4>::getRestWords(npatl,seedl)); Breverse = ABreverse.get(); maxpatl = npatl; } if ( npatl != patl ) { patl = npatl; numrestwords = RestMatch<sse4>::getRestWords(patl,seedl); fullrestwords = RestMatch<sse4>::getFullRestWords(patl, seedl); fracrestsyms = RestMatch<sse4>::getFracRestSyms(patl, seedl, fullrestwords); straighttextrestoffset = RestMatch<sse4>::getTextRestOffset(patl, seedl, false); reversetextrestoffset = RestMatch<sse4>::getTextRestOffset(patl, seedl, true); straightmatchoffset = RestMatch<sse4>::getMatchOffset(patl, seedl, false); reversematchoffset = RestMatch<sse4>::getMatchOffset(patl, seedl, true); } }
bool ServiceApp::processRequest(const void *request) { const char *p = reinterpret_cast<const char*>(request); natural count = *p++; ConstStrA command(p); p += command.length() + 1; AutoArray<String,StaticAlloc<256> > params; AutoArray<ConstStrW,StaticAlloc<256> > wparams; for(natural i = 0;i < count;i++){ ConstStrA param(p); p += param.length() + 1; params.add(String(param)); wparams.add(ConstStrW(params[i])); } AutoArray<byte> output; output.resize(instance->getReplyMaxSize()); SeqOutputBuffer fakeout(output.data() + sizeof (integer), output.length() - sizeof (integer)); fakeout.setStaticObj(); integer res = -1; bool stop = false; SeqFileOutput out(&fakeout); try { res = onMessage(command, wparams, out); stop = command == ConstStrA(stopCmd); } catch(std::exception & e){ TextOut<SeqFileOutput> msg(out); msg("%1") << e.what(); res = -1; } try { *reinterpret_cast<integer*>(output.data()) = res; instance->sendReply(output.data(), fakeout.length() + sizeof (integer)); } catch(...){ } return !stop; }
static AutoArray<u_int64_t> fillRange( std::vector < std::pair < std::string, u_int64_t > > const & ranges ) { AutoArray<u_int64_t> AB ( (ranges.at(ranges.size()-1).second+1+63)/64 ); for ( u_int64_t i = 0; i < (ranges.at(ranges.size()-1).second+1+63)/64; ++i ) AB[i] = 0; Rank::BitWriter8 W(AB.get()); for ( unsigned int j = 0; j+1 < ranges.size(); ++j ) { u_int64_t len = ranges[j+1].second-ranges[j].second; W.writeBit(true); for ( u_int64_t i = 1; i < len; ++i ) W.writeBit(false); } W.writeBit(true); W.flush(); return AB; }
ConstValue LocalView::runReduce(const ConstValue &rows) const { AutoArray<KeyAndDocId, SmallAlloc<256> > keylist; AutoArray<ConstValue, SmallAlloc<256> > values; keylist.reserve(rows.length()); values.reserve(rows.length()); for (JSON::ConstIterator iter = rows->getFwConstIter(); iter.hasItems();) { const JSON::ConstValue &v = iter.getNext(); keylist.add(KeyAndDocId(v["key"],v["id"].getStringA())); values.add(v["value"]); } return reduce(keylist,values,false); }
StringA WinHttpStream::getReplyHeaders() { setState(stReadResponse); AutoArray<char> buffer; buffer.resize(1000); DWORD size = (DWORD)buffer.length(); while (!HttpQueryInfoA(hHTTPConn,HTTP_QUERY_RAW_HEADERS_CRLF, buffer.data(),&size,0)) { DWORD res = GetLastError(); if (res == ERROR_INSUFFICIENT_BUFFER) { buffer.resize(size); } else { throw ErrNoWithDescException(THISLOCATION,GetLastError(), "Cannot retrieve headers from the request"); } } buffer.resize(size); return buffer; }
static void index_VBO( const vector<vec4> & in_vertices, const vector<vec2> & in_uvs, const vector<vec3> & in_normals, AutoArray<unsigned short> & out_indices, AutoArray<vec4> & out_vertices, AutoArray<vec2> & out_uvs, AutoArray<vec3> & out_normals ) { map<PackedVertex,unsigned short> VertexToOutIndex; // For each input vertex unsigned int numVertices = in_vertices.size(); for (unsigned int i = 0; i < numVertices; i++) { PackedVertex packed; packed.position = in_vertices[i]; packed.uv = in_uvs[i]; packed.normal = in_normals[i]; // Try to find a similar vertex in out_XXXX unsigned short index; bool found = get_similar_vertex_index(packed, VertexToOutIndex, index); if(found) { // A similar vertex is already in the VBO, use it instead ! out_indices.Push(index); } else { out_vertices.Push(in_vertices[i]); out_uvs.Push(in_uvs[i]); out_normals.Push(in_normals[i]); unsigned short newindex = (unsigned short) out_vertices.Count() - 1; out_indices.Push(newindex); VertexToOutIndex[packed] = newindex; } } }
ITCPServerConnHandler::Command HttpReqImpl::readHeader() { try { AutoArray<char, SmallAlloc<8192> > linebuff; NStream::Buffer &buffer = inout->getBuffer(); natural fetched = buffer.fetch(); if (fetched == 0) { errorPage(413,ConstStrA(),"Header is too large"); return ITCPServerConnHandler::cmdRemove; } natural pos = buffer.lookup(ConstBin("\r\n"),fetched); while (pos != naturalNull) { linebuff.resize(pos+2); buffer.read(linebuff.data(),pos+2); ConstStrA line = linebuff.head(pos); if (method.empty()) { if (pos == 0) return ITCPServerConnHandler::cmdWaitRead; reqBeginTime = TimeStamp::now(); reportDuration = true; cropWhite(line); ConstStrA::SplitIterator splt = line.split(' '); method = hdrPool.add(ConstStrA(splt.getNext())); path = hdrPool.add(ConstStrA(splt.getNext())); while (path.empty()) path = hdrPool.add(ConstStrA(splt.getNext())); protocol = hdrPool.add(ConstStrA(splt.getNext())); while (protocol.empty()) protocol = hdrPool.add(ConstStrA(splt.getNext())); //parse some fields TextParser<char,StaticAlloc<256> > parser; //check http version if (parser("HTTP/%u1.%u2",protocol)) { httpMajVer = (unsigned short)((natural)parser[1]); httpMinVer = (unsigned short)((natural)parser[2]); if (httpMajVer != 1 || (httpMinVer != 0 && httpMinVer != 1)) return errorPageKA(505); useHTTP11(httpMinVer == 1); } else { return errorPageKA(400,StringA(ConstStrA("Unknown protocol: ")+ protocol)); } } else if (line.empty()) { return finishReadHeader(); } else { natural dblcolon = line.find(':'); if (dblcolon == naturalNull) { errorPage(400,ConstStrA(),"line"); return ITCPServerConnHandler::cmdRemove; } ConstStrA field = line.head(dblcolon); ConstStrA value = line.offset(dblcolon+1); cropWhite(field); cropWhite(value); requestHdrs.insert(hdrPool.add(field),hdrPool.add(value)); } pos = buffer.lookup(ConstBin("\r\n")); } return ITCPServerConnHandler::cmdWaitRead; } catch (AllocatorLimitException &) { errorPage(413,ConstStrA(),"Header is too large (4096 bytes)"); return ITCPServerConnHandler::cmdRemove; } }
HttpResponse& HttpClient::sendRequestInternal(ConstStrA url, Method method, SeqFileInput data, const HdrItem* headers, bool wo100) { //for HTTP/1.0 we need to buffer request (sad) if (useHTTP10) { //create buffer AutoArray<byte> buffer; if (data.hasItems()) { do { natural pos = buffer.length(); //reserve buffer buffer.resize(pos+4096); //read buffer natural sz = data.blockRead(buffer.data()+pos,4096,true); //adjust buffer size buffer.resize(pos+sz); //continue until end } while (data.hasItems()); } //send request return sendRequest(url,method,buffer,headers); } else { //while reading from the stream, we need to ensure, that server is ready to accept our data //so in this case, we will use Expect: 100-continue precondition try { //create request for url HttpRequest &rq = createRequest(url,method); //we don't need to use 100-continue if connection was recently opened if (!connectionReused) wo100 = true; //set content length to infinity - this should switch to chunked rq.setContentLength(naturalNull); //load headers feedHeaders(rq, headers); //set Expect: 100-continue if (!wo100) rq.setHeader(fldExpect,"100-continue"); //close headers and start body, but we must wait for response now rq.beginBody(); //so create response (do not read headers) HttpResponse& resp = createResponse(false); //now wait some time to server's response. //because it could ignore our header, we must not wait for infinity time if (wo100 || nstream->wait(INetworkResource::waitForInput,10000) != INetworkResource::waitTimeout) { //data arrived in time - read headers resp.readHeaders(); //there can other response, so if we cannot continue, return response to the caller if (!resp.canContinue()) { //check status if (resp.getStatus() == 417) //we got precondition failed - no worries, we can repeat the request //at least we know, that connection is alive return sendRequestInternal(url,method,data,headers,true); else { //other status is send to the caller return resp; } } } //until now, request could be repeated anytime for network errors //- because keep-alive can be closed by the server anytime during request //now we should prevent repeating request when network error happened //because we starting reading the stream connectionReused = false; //create a buffer for data CArray<byte, 1024> buffer; //open request as stream SeqFileOutput dout(&rq); //use blockcopy to copy data to the request dout.blockCopy(data,buffer,naturalNull); //close the output rq.closeOutput(); //now wait for response. resp.waitAfterContinue(HttpResponse::readHeadersNow); //because we could skip waiting on 100-continue above, we need to receive it now //and any other such responses while (resp.canContinue()) { resp.waitAfterContinue(HttpResponse::readHeadersNow); } //return other response to the caller return resp; } catch (const NetworkException &e) { if (connectionReused) { closeConnection(); return sendRequest(url, method, data, headers); } else { throw; } } } }
void WinHttpStream::initRequest() { TextParser<wchar_t> parser; if (!parser(L"%1://%[*-_.a-zA-Z0-9:@]2%%[/](*)*3",url)) throw FileOpenError(THISLOCATION,ERROR_FILE_NOT_FOUND,url); String protocol = parser[1].str(); String hostident = parser[2].str(); String::SplitIterator hostidentsplit = hostident.split('@'); String ident; String domain; domain = hostidentsplit.getNext(); while (hostidentsplit.hasItems()) { ident = ident + domain + ConstStrW('@'); domain = hostidentsplit.getNext(); } String path = parser[3].str(); natural port; String username; String password; bool secure; if (parser( L"%1:%u2",domain)) { domain = parser[1].str(); port = parser[2]; secure = false; } else if (protocol == ConstStrW(L"http")) { port = INTERNET_DEFAULT_HTTP_PORT; secure = false; } else if (protocol == ConstStrW(L"https")) { port = INTERNET_DEFAULT_HTTPS_PORT; secure = true; } else throw FileOpenError(THISLOCATION,ERROR_NOT_FOUND,url); if (!ident.empty()) { if (parser(L"%1:%2@",ident)) { username = parser[1].str(); password = parser[2].str(); } else { throw FileMsgException(THISLOCATION,0,url,"Invalid identification field in the url"); } } DWORD accessType; switch (settings.proxyMode) { case pmManual: accessType = INTERNET_OPEN_TYPE_PROXY; case pmDirect: accessType = INTERNET_OPEN_TYPE_DIRECT; case pmAuto: accessType = INTERNET_OPEN_TYPE_PRECONFIG; } TextFormatBuff<wchar_t> fmt; String proxyName; if (accessType == INTERNET_OPEN_TYPE_PROXY) { fmt(L"%1:%2") << settings.proxyAddr << settings.proxyPort; proxyName = fmt.write(); } if (hInternet) InternetCloseHandle(hInternet); hInternet = InternetOpenW(settings.userAgent.cStr(),accessType,proxyName.cStr(),0,0); if (hInternet == 0) throw FileMsgException(THISLOCATION,GetLastError(),url,"Cannot initialize WinInet"); if (hConnect) InternetCloseHandle(hConnect); hConnect = InternetConnectW(hInternet,domain.cStr(),(INTERNET_PORT)port, username.empty()?0:username.cStr(), password.empty()?0:password.cStr(), INTERNET_SERVICE_HTTP ,0,0); if (hConnect == 0) throw FileMsgException(THISLOCATION,GetLastError(),url,"Cannot connect remote site"); DWORD reqFlags = INTERNET_FLAG_NO_UI |INTERNET_FLAG_HYPERLINK ; if (redirDisabled) reqFlags|=INTERNET_FLAG_NO_AUTO_REDIRECT ; if (!settings.cookiesEnabled) reqFlags|=INTERNET_FLAG_NO_COOKIES; if (secure) reqFlags|=INTERNET_FLAG_SECURE; hHTTPConn = HttpOpenRequestW(hConnect,String(method).cStr(),path.cStr(), 0,0,0,reqFlags,0); if (hHTTPConn == 0) throw FileMsgException(THISLOCATION,GetLastError(),url,"Cannot connect remote site"); AutoArray<wchar_t> hdrall; for (HeaderMap::Iterator iter = hdrmap.getFwIter(); iter.hasItems();) { const HeaderMap::Entity &e = iter.getNext(); fmt(L"%1: %2\n") << e.key << e.value; hdrall.append(fmt.write()); } if (!hdrall.empty() && !HttpAddRequestHeadersW(hHTTPConn,hdrall.data(),(DWORD)hdrall.length(),HTTP_ADDREQ_FLAG_REPLACE|HTTP_ADDREQ_FLAG_ADD)) throw FileMsgException(THISLOCATION,GetLastError(),url,"AddRequest failed"); if (!HttpSendRequestW(hHTTPConn,0,0,postBuffer.data(),(DWORD)postBuffer.length())) { bool stillError = true; DWORD dwError = GetLastError(); if (dwError == ERROR_INTERNET_INVALID_CA && settings.allowUntrustedCert) { DWORD dwFlags; DWORD dwBuffLen = sizeof(dwFlags); InternetQueryOption (hHTTPConn, INTERNET_OPTION_SECURITY_FLAGS, (LPVOID)&dwFlags, &dwBuffLen); dwFlags |= SECURITY_FLAG_IGNORE_UNKNOWN_CA; InternetSetOption (hHTTPConn, INTERNET_OPTION_SECURITY_FLAGS, &dwFlags, sizeof (dwFlags) ); if (HttpSendRequestW(hHTTPConn,0,0,postBuffer.data(),(DWORD)postBuffer.length())) stillError = false; } if (stillError) throw FileMsgException(THISLOCATION,GetLastError(),url,"Failed to SendRequest"); } postBuffer.clear(); }
u_int64_t size() const { return AB.size() + sizeof(u_int64_t *) + R.size(); }
void expandBuf(u_int64_t size) { if ( buf.getN() < size ) buf = AutoArray<u_int8_t>(size,false); }