static void check_multi_info(socketmanager *smp) { int msgs_left; while (CURLMsg *msg = curl_multi_info_read(smp->curlmulti, &msgs_left)) { if (msg->msg == CURLMSG_DONE) { CURL *easy = msg->easy_handle; CURLcode res = msg->data.result; long httpcode; curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &httpcode); // This gets the mcurlconn associated with the easy handle, and removes it from the socketmanager // We then hold onto it here, it will be destructed when the scope ends unless NotifyDone claims it std::unique_ptr<mcurlconn> cs = sm.RemoveConn(easy); if (cs) { LogMsgFormat(LOGT::SOCKTRACE, "Socket Done, conn ID: %d, res: %d, http: %d", cs->id, res, httpcode); cs->NotifyDone(easy, httpcode, res, std::move(cs)); } else { // Getting here is a severe bug mcurlconn *conn; curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); LogMsgFormat(LOGT::SOCKERR, "Socket Done, yet could not be found in connection list, conn ID: %d, res: %d, http: %d", conn->id, res, httpcode); } } } if (sm.curnumsocks == 0) { LogMsgFormat(LOGT::SOCKTRACE, "No Sockets Left, Stopping Timer"); smp->st->Stop(); } }
void dump_pending_active_conn(LOGT logflags, const std::string &indent, const std::string &indentstep) { LogMsgFormat(logflags, "%sActive connections: %d", cstr(indent), std::distance(sm.connlist.begin(), sm.connlist.end())); for (auto &it : sm.connlist) { if (!it.cs) continue; LogMsgFormat(logflags, "%s%sSocket: %s, ID: %d, Error Count: %d, mcflags: 0x%X", cstr(indent), cstr(indentstep), cstr(it.cs->GetConnTypeName()), it.cs->id, it.cs->errorcount, it.cs->mcflags); } }
void dump_pending_acc_failed_conns(LOGT logflags, const std::string &indent, const std::string &indentstep, taccount *acc) { dump_acc_socket_flags(logflags, indent, acc); LogMsgFormat(logflags, "%sRestartable Failed Connections: %d", cstr(indent), acc->failed_pending_conns.size()); for (auto &it : acc->failed_pending_conns) { LogMsgFormat(logflags, "%s%sSocket: %s, ID: %d, Error Count: %d, mcflags: 0x%X", cstr(indent), cstr(indentstep), cstr(it->GetConnTypeName()), it->id, it->errorcount, it->mcflags); } }
void dump_cids_stats(const cached_id_sets &cids, LOGT logflags, const std::string &indent, const std::string &indentstep) { size_t total = 0; cids.IterateLists([&](const char *name, tweetidset cached_id_sets::* mptr, unsigned long long flag) { const tweetidset &set = cids.*mptr; LogMsgFormat(logflags, "%s%s (%s): %zu", cstr(indent), cstr(name), cstr(tweet_flags::GetValueString(flag)), set.size()); total += set.size(); }); LogMsgFormat(logflags, "%sTotal: %zu", cstr(indent), total); }
// This determines the scrollbar size and position such that, where possible // the item at the top of the visible screen is unmoved void tpanelscrollbar::RepositionItems() { tpanelscrollpane *tsp = get_paired_ptr(); if (!tsp) return; #if TPANEL_SCROLLING_COPIOUS_LOGGING LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::RepositionItems %s, START %d %d", cstr(GetThisName()), GetThumbPosition(), parent->GetCurrentDisp().size()); #endif scroll_virtual_size = 0; int cumul_size = 0; bool have_scroll_offset = false; int scroll_offset = 0; for (auto &disp : parent->GetCurrentDisp()) { wxPoint p = disp.item->GetPosition(); wxSize s = disp.item->GetSize(); if (!disp.item->IsShown()) { s.y = 0; } scroll_virtual_size += s.y; if (p.x == 0 && p.y + s.y > 0 && p.y <= 0) { // This is an item which is visible at the top of the list // We should use the *last* matching item, this is as earlier items may grow in size to overlap the top of the screen // p.y is non-positive // The scroll offset should be increased as p.y increases in magnitude below 0 scroll_offset = cumul_size - p.y; have_scroll_offset = true; } cumul_size += s.y; } if (parent->pimpl()->displayoffset == 0 && GetThumbPosition() == 0 && have_scroll_offset && !scroll_always_freeze) { // We were at the very top, we would normally be scrolling down as something has been inserted above // Scroll back to the top instead // Don't do this if scroll_always_freeze is true scroll_offset = 0; } scroll_always_freeze = false; if (!have_scroll_offset) { scroll_offset = GetThumbPosition(); } scroll_client_size = tsp->GetClientSize().y; SetScrollbar(scroll_offset, scroll_client_size, scroll_virtual_size, 1); ScrollItems(); #if TPANEL_SCROLLING_COPIOUS_LOGGING LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::RepositionItems %s, END %d %d %d %d %d", cstr(GetThisName()), GetThumbPosition(), scroll_offset, have_scroll_offset, scroll_always_freeze, cumul_size); #endif }
void dump_pending_retry_conn(LOGT logflags, const std::string &indent, const std::string &indentstep) { size_t count = 0; for (auto &it : sm.retry_conns) { if (it) count++; } LogMsgFormat(logflags, "%sConnections pending retry attempts: %d", cstr(indent), count); for (auto &it : sm.retry_conns) { if (!it) continue; LogMsgFormat(logflags, "%s%sSocket: %s, ID: %d, Error Count: %d, mcflags: 0x%X", cstr(indent), cstr(indentstep), cstr(it->GetConnTypeName()), it->id, it->errorcount, it->mcflags); } }
//return true if has been handled asynchronously bool adns::CheckAsync(CURL *ch, std::unique_ptr<mcurlconn> &&cs) { long timeout = -1; curl_easy_setopt(ch, CURLOPT_DNS_CACHE_TIMEOUT, timeout); curl_easy_setopt(ch, CURLOPT_SHARE, GetHndl()); const std::string &url = cs->url; if (url.empty()) { return false; } std::string::size_type proto_start = url.find("://"); std::string::size_type name_start; if (proto_start == std::string::npos) { name_start = 0; } else { name_start = proto_start + 3; } std::string::size_type name_end = url.find('/', name_start); std::string name = url.substr(name_start, (name_end == std::string::npos) ? std::string::npos : name_end - name_start); if (cached_names.count(name)) { return false; // name already in cache, can go now } dns_pending_conns.push_front({ name, ch, std::move(cs) }); // cs is now null, don't use again for (auto &it : dns_threads) { if (it.first == name) { LogMsgFormat(LOGT::SOCKTRACE, "DNS lookup thread already exists: %s, %s", cstr(url), cstr(name)); return true; } } LogMsgFormat(LOGT::SOCKTRACE, "Creating DNS lookup thread: %s, %s", cstr(url), cstr(name)); dns_threads.emplace_front(std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(url, name, sm, GetHndl())); adns_thread &ad = dns_threads.front().second; ad.Create(); #if defined(__GLIBC__) #if __GLIBC_PREREQ(2, 12) pthread_setname_np(ad.GetId(), "retcon-adns"); #endif #endif ad.Run(); return true; }
curl_socket_t pre_connect_func(void *clientp, curl_socket_t curlfd, curlsocktype purpose) { mcurlconn *cs = static_cast<mcurlconn *>(clientp); double lookuptime; curl_easy_getinfo(cs->GenGetCurlHandle(), CURLINFO_NAMELOOKUP_TIME, &lookuptime); LogMsgFormat(LOGT::SOCKTRACE, "DNS lookup took: %fs. Request: type: %s, conn ID: %d, url: %s", lookuptime, cstr(cs->GetConnTypeName()), cs->id, cstr(cs->url)); return 0; }
static void dump_pending_user_line(LOGT logflags, const std::string &indent, userdatacontainer *u) { time_t now = time(nullptr); LogMsgFormat(logflags, "%sUser: %" llFmtSpec "d (%s) udc_flags: 0x%X, last update: %" llFmtSpec "ds ago" ", last DB update: %" llFmtSpec "ds ago, image ready: %d, ready (nx): %d, ready (x): %d", cstr(indent), u->id, cstr(u->GetUser().screen_name), u->udc_flags, (uint64_t) (now-u->lastupdate), (uint64_t) (now-u->lastupdate_wrotetodb), u->ImgIsReady(PENDING_REQ::PROFIMG_NEED), u->IsReady(0), u->IsReady(PENDING_REQ::USEREXPIRE)); }
int retcon::OnExit() { LogMsg(LOGT::OTHERTRACE, "retcon::OnExit"); sm.DeInitMultiIOHandler(); pool.reset(); DBC_DeInit(); for (const observer_ptr<temp_file_holder> &it : temp_file_set) { LogMsgFormat(LOGT::FILEIOTRACE, "retcon::OnExit, resetting: %s", cstr(it->GetFilename())); it->Reset(); } if (!::wxRmdir(wxstrstd(tmpdir))) { LogMsgFormat(LOGT::FILEIOERR, "retcon::OnExit, could not remove temp dir: %s", cstr(tmpdir)); } DebugFinalChecks(); DeInitWxLogger(); std::exit(0); }
static void LogSocketErrorMessage(mcurlconn *mc, CURL *easy, long httpcode, CURLcode res, MCC_HTTPERRTYPE err) { if (!(currentlogflags & LOGT::SOCKERR)) { return; } const char *action = nullptr; switch (err) { case MCC_RETRY: action = " (retrying)"; break; case MCC_FAILED: action = ""; break; } char *req_url = nullptr; curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &req_url); std::string fail_type; if (res == CURLE_OK) { fail_type = string_format("Request failed: HTTP code: %ld", httpcode); } else { fail_type = string_format("Socket error: %s", cstr(curl_easy_strerror(res))); } LogMsgFormat(LOGT::SOCKERR, "%s%s%s, type: %s, url: %s, conn ID: %u", cstr(fail_type), cstr(action), cstr(mc->GetFailureLogInfo()), cstr(mc->GetConnTypeName()), cstr(req_url), mc->id); }
// This checks both the retry list and the active list for removal std::unique_ptr<mcurlconn> mcurlconn::RemoveConnCommon(const char *logprefix) { std::unique_ptr<mcurlconn> conn = sm.UnregisterRetryConn(*this); if (conn) { LogMsgFormat(LOGT::SOCKTRACE, "%s (retry list): conn ID: %d", logprefix, id); return std::move(conn); } conn = sm.RemoveConn(GenGetCurlHandle()); if (conn) { LogMsgFormat(LOGT::SOCKTRACE, "%s: conn ID: %d", logprefix, id); return std::move(conn); } LogMsgFormat(LOGT::SOCKERR, "%s: failed, conn ID: %d", logprefix, id); return nullptr; }
void log_window::OnDumpConnInfo(wxCommandEvent &event) { dump_pending_active_conn(LOGT::USERREQ, "", "\t"); dump_pending_retry_conn(LOGT::USERREQ, "", "\t"); for (auto &it : alist) { LogMsgFormat(LOGT::USERREQ, "Account: %s (%s)", cstr(it->name), cstr(it->dispname)); dump_pending_acc_failed_conns(LOGT::USERREQ, "\t", "\t", it.get()); } }
void dump_acc_id_stats(const taccount &acc, LOGT logflags, const std::string &indent, const std::string &indentstep) { auto line = [&](const char *name, size_t value) { LogMsgFormat(logflags, "%s%s: %zu", cstr(indent), cstr(name), value); }; line("Tweets IDs", acc.tweet_ids.size()); line("DM IDs", acc.dm_ids.size()); line("Pending users", acc.pendingusers.size()); line("User relations", acc.user_relations.size()); }
void tpanelscrollpane::resizehandler(wxSizeEvent &event) { #if TPANEL_SCROLLING_COPIOUS_LOGGING LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollpane::resizehandler: %s, %d, %d", cstr(GetThisName()), event.GetSize().GetWidth(), event.GetSize().GetHeight()); #endif for (auto &disp : parent->GetCurrentDisp()) { disp.item->NotifySizeChange(); } }
static void dump_pending_tweet(LOGT logflags, const std::string &indent, const std::string &indentstep, tweet *t, userdatacontainer *exclude_user) { LogMsgFormat(logflags, "%sTweet: %s", cstr(indent), cstr(tweet_log_line(t))); if (t->user && t->user.get() != exclude_user) { dump_pending_user_line(logflags, indent+indentstep, t->user.get()); } if (t->user_recipient && t->user_recipient.get() != exclude_user) { dump_pending_user_line(logflags, indent+indentstep, t->user_recipient.get()); } }
void tpanelscrollbar::OnScrollTrack(wxScrollEvent &event) { #if TPANEL_SCROLLING_COPIOUS_LOGGING LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::OnScrollTrack %s, %d", cstr(GetThisName()), event.GetPosition()); #endif int y = event.GetPosition(); ScrollItemsForPosition(y); SetThumbPosition(y); event.Skip(); }
void SetCacerts(CURL *handle) { static std::string cacertpath; if (cacertpath.empty()) { cacertpath = "./cacert.pem"; if (wxFile::Exists(wxstrstd(cacertpath))) { LogMsgFormat(LOGT::SOCKTRACE, "SetCacerts: found existing ./cacert.pem"); // There is an existing cacert.pem we can use, no need to create one } else { cacertpath = wxGetApp().datadir + "/cacert.pem"; LogMsgFormat(LOGT::SOCKTRACE, "SetCacerts: Writing out cacert.pem to: %s", cstr(cacertpath)); wxFFileOutputStream output(wxstrstd(cacertpath)); wxMemoryInputStream input(cacert_start, cacert_end - cacert_start); wxZlibInputStream zliber(input, wxZLIB_ZLIB); output.Write(zliber); LogMsgFormat(LOGT::SOCKTRACE, "SetCacerts: Writing out cacert.pem complete"); } } curl_easy_setopt(handle, CURLOPT_CAINFO, cacertpath.c_str()); }
static int sock_cb(CURL *e, curl_socket_t s, int what, socketmanager *smp, mcurlconn *cs) { LogMsgFormat(LOGT::SOCKTRACE, "Socket Interest Change Callback: %d, %d, conn ID: %d", s, what, cs ? cs->id : -1); if (what != CURL_POLL_REMOVE) { if (!cs) { curl_easy_getinfo(e, CURLINFO_PRIVATE, &cs); curl_multi_assign(smp->curlmulti, s, cs); } } smp->RegisterSockInterest(e, s, what); return 0; }
void socketmanager::RetryConn(std::unique_ptr<mcurlconn> cs) { if (cs->mcflags & mcurlconn::MCF::IN_RETRY_QUEUE) { LogMsgFormat(LOGT::SOCKERR, "socketmanager::RetryConn: Attempt to add mcurlconn to retry queue which is marked as already in queue, this is a bug: type: %s, conn ID: %d, url: %s", cstr(cs->GetConnTypeName()), cs->id, cstr(cs->url)); return; } cs->mcflags |= mcurlconn::MCF::IN_RETRY_QUEUE; cs->AddToRetryQueueNotify(); retry_conns.push_back(std::move(cs)); RetryConnLater(); }
void tpanelscrollpane::resizemsghandler(wxCommandEvent &event) { #if TPANEL_SCROLLING_COPIOUS_LOGGING LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollpane::resizemsghandler %s", cstr(GetThisName())); #endif if (tpanelscrollbar *tsb = get_paired_ptr()) { tsb->RepositionItems(); } resize_update_pending = false; Thaw(); Update(); }
adns::~adns() { size_t n = std::distance(dns_threads.begin(), dns_threads.end()); if (n) { size_t i = 0; for (auto &it : dns_threads) { LogMsgFormat(LOGT::SOCKTRACE, "Waiting for all DNS threads to terminate: %d of %d", i, n); it.second.Wait(); i++; } LogMsg(LOGT::SOCKTRACE, "All DNS threads terminated"); } RemoveShareHndl(); }
static void dump_non_acc_user_pendings(LOGT logflags, const std::string &indent, const std::string &indentstep) { std::set<uint64_t> acc_pending_users; for (auto &it : alist) { for (auto &jt : it->pendingusers) { acc_pending_users.insert(jt.second->id); } } LogMsgFormat(logflags, "%sOther pending users:", cstr(indent)); for (auto &it : ad.userconts) { if (!(it.second->pendingtweets.empty())) { if (acc_pending_users.find(it.first) == acc_pending_users.end()) { dump_pending_user(logflags, indent + indentstep, indentstep, it.second.get()); } } } if (!ad.noacc_pending_userconts.empty()) { LogMsgFormat(logflags, "%sUsers pending usable account:", cstr(indent)); for (auto &it : ad.noacc_pending_userconts) { dump_pending_user(logflags, indent + indentstep, indentstep, it.second.get()); } } }
void dump_tweet_pendings(LOGT logflags, const std::string &indent, const std::string &indentstep) { bool done_header = false; for (auto &it : ad.tweetobjs) { const tweet *t = it.second.get(); if (t && !(t->pending_ops.empty())) { if (!done_header) { done_header = true; LogMsgFormat(logflags, "%sTweets with operations pending ready state:", cstr(indent)); } dump_tweet_line(logflags, indent + indentstep, indentstep, t); } } if (!ad.noacc_pending_tweetobjs.empty()) { LogMsgFormat(logflags, "%sTweets pending usable account:", cstr(indent)); for (auto &it : ad.noacc_pending_tweetobjs) { dump_tweet_line(logflags, indent + indentstep, indentstep, it.second.get()); } } if (!ad.handlenew_pending_ops.empty()) { LogMsgFormat(logflags, "%sTotal handlenew_pending_op pending count: %zu", cstr(indent), ad.handlenew_pending_ops.size()); } }
void adns::DNSResolutionEvent(wxCommandEvent &event) { adns_thread *at = static_cast<adns_thread*>(event.GetClientData()); if (!at) return; at->Wait(); if (at->success) { LogMsgFormat(LOGT::SOCKTRACE, "Asynchronous DNS lookup succeeded: %s, %s, time: %fs", cstr(at->hostname), cstr(at->url), at->lookuptime); cached_names.insert(at->hostname); } else { LogMsgFormat(LOGT::SOCKERR, "Asynchronous DNS lookup failed: %s, (%s), error: %s (%d), time: %fs", cstr(at->hostname), cstr(at->url), cstr(curl_easy_strerror(at->result)), at->result, at->lookuptime); } std::vector<dns_pending_conn> current_dns_pending_conns; dns_pending_conns.remove_if ([&](dns_pending_conn &a) -> bool { if (a.hostname == at->hostname) { current_dns_pending_conns.emplace_back(std::move(a)); return true; } else { return false; } }); dns_threads.remove_if ([&](const std::pair<std::string, adns_thread> &a) { return at == &(a.second); }); for (auto &it : current_dns_pending_conns) { CURL *ch = it.ch; std::unique_ptr<mcurlconn> mc = std::move(it.cs); if (at->success) { LogMsgFormat(LOGT::SOCKTRACE, "Launching request as DNS lookup succeeded: type: %s, conn ID: %d, url: %s", cstr(mc->GetConnTypeName()), mc->id, cstr(mc->url)); sm->AddConn(ch, std::move(mc)); } else { LogMsgFormat(LOGT::SOCKERR, "Request failed due to asynchronous DNS lookup failure: type: %s, conn ID: %d, url: %s", cstr(mc->GetConnTypeName()), mc->id, cstr(mc->url)); mc->HandleError(ch, 0, CURLE_COULDNT_RESOLVE_HOST, std::move(mc)); } } }
static int multi_timer_cb(CURLM *multi, long timeout_ms, socketmanager *smp) { LogMsgFormat(LOGT::SOCKTRACE, "Socket Timer Callback: %d ms", timeout_ms); if (timeout_ms > 0) { smp->st->Start(timeout_ms,wxTIMER_ONE_SHOT); } else { smp->st->Stop(); } if (!timeout_ms) { smp->st->Notify(); } return 0; }
void socketmanager::RetryConnNow() { std::unique_ptr<mcurlconn> cs; while (true) { if (retry_conns.empty()) return; cs = std::move(retry_conns.front()); retry_conns.pop_front(); if (cs) break; } cs->mcflags &= ~mcurlconn::MCF::IN_RETRY_QUEUE; cs->mcflags |= mcurlconn::MCF::RETRY_NOW_ON_SUCCESS; LogMsgFormat(LOGT::SOCKTRACE, "Dequeueing request from retry queue: type: %s, conn ID: %d, url: %s", cstr(cs->GetConnTypeName()), cs->id, cstr(cs->url)); cs->RemoveFromRetryQueueNotify(); cs->DoRetry(std::move(cs)); }
void socketmanager::InitMultiIOHandler() { if (MultiIOHandlerInited) return; InitMultiIOHandlerCommon(); int pipefd[2]; pipe(pipefd); socketpollthread *th = new socketpollthread(); th->pipefd = pipefd[0]; this->pipefd = pipefd[1]; th->Create(); th->Run(); LogMsgFormat(LOGT::SOCKTRACE, "socketmanager::InitMultiIOHandler(): Created socket poll() thread: %d", th->GetId()); MultiIOHandlerInited = true; }
void tpanelscrollbar::mousewheelhandler(wxMouseEvent &event) { int pxdelta = -event.GetWheelRotation() * gc.mousewheelscrollspeed / event.GetWheelDelta(); #if TPANEL_SCROLLING_COPIOUS_LOGGING LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::mousewheelhandler %s, %d %d %d", cstr(GetThisName()), GetScrollPos(wxVERTICAL), event.GetWheelRotation(), pxdelta); #endif int y = GetThumbPosition(); SetThumbPosition(std::max(0, y + pxdelta)); ScrollItems(); // Use gc.mousewheelscrollspeed as the threshold, this is to try and make mousewheel scrolling smooth across pages OnScrollHandlerCommon(pxdelta < 0, pxdelta > 0, std::abs(gc.mousewheelscrollspeed), GetScrollPos(wxVERTICAL)); }
std::unique_ptr<mcurlconn> socketmanager::UnregisterRetryConn(mcurlconn &cs) { if (!(cs.mcflags & mcurlconn::MCF::IN_RETRY_QUEUE)) return nullptr; std::unique_ptr<mcurlconn> csptr; container_unordered_remove_if (retry_conns, [&](std::unique_ptr<mcurlconn> &it) { if (it.get() == &cs) { LogMsgFormat(LOGT::SOCKTRACE, "socketmanager::UnregisterRetryConn: Unregistered from retry queue: type: %s, conn ID: %d, url: %s", cstr(cs.GetConnTypeName()), cs.id, cstr(cs.url)); csptr = std::move(it); csptr->mcflags &= ~mcurlconn::MCF::IN_RETRY_QUEUE; csptr->RemoveFromRetryQueueNotify(); return true; } else { return false; } }); return std::move(csptr); }