static void check_multi_info(socketmanager *smp) {
int msgs_left;
while (CURLMsg *msg = curl_multi_info_read(smp->curlmulti, &msgs_left)) {
if (msg->msg == CURLMSG_DONE) {
CURL *easy = msg->easy_handle;
CURLcode res = msg->data.result;
long httpcode;
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &httpcode);
// This gets the mcurlconn associated with the easy handle, and removes it from the socketmanager
// We then hold onto it here, it will be destructed when the scope ends unless NotifyDone claims it
std::unique_ptr<mcurlconn> cs = sm.RemoveConn(easy);
if (cs) {
LogMsgFormat(LOGT::SOCKTRACE, "Socket Done, conn ID: %d, res: %d, http: %d", cs->id, res, httpcode);
cs->NotifyDone(easy, httpcode, res, std::move(cs));
} else {
// Getting here is a severe bug
mcurlconn *conn;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
LogMsgFormat(LOGT::SOCKERR, "Socket Done, yet could not be found in connection list, conn ID: %d, res: %d, http: %d", conn->id, res, httpcode);
}
}
}
if (sm.curnumsocks == 0) {
LogMsgFormat(LOGT::SOCKTRACE, "No Sockets Left, Stopping Timer");
smp->st->Stop();
}
}
void dump_pending_active_conn(LOGT logflags, const std::string &indent, const std::string &indentstep) {
LogMsgFormat(logflags, "%sActive connections: %d", cstr(indent), std::distance(sm.connlist.begin(), sm.connlist.end()));
for (auto &it : sm.connlist) {
if (!it.cs) continue;
LogMsgFormat(logflags, "%s%sSocket: %s, ID: %d, Error Count: %d, mcflags: 0x%X",
cstr(indent), cstr(indentstep), cstr(it.cs->GetConnTypeName()), it.cs->id, it.cs->errorcount, it.cs->mcflags);
}
}
void dump_pending_acc_failed_conns(LOGT logflags, const std::string &indent, const std::string &indentstep, taccount *acc) {
dump_acc_socket_flags(logflags, indent, acc);
LogMsgFormat(logflags, "%sRestartable Failed Connections: %d", cstr(indent), acc->failed_pending_conns.size());
for (auto &it : acc->failed_pending_conns) {
LogMsgFormat(logflags, "%s%sSocket: %s, ID: %d, Error Count: %d, mcflags: 0x%X",
cstr(indent), cstr(indentstep), cstr(it->GetConnTypeName()), it->id, it->errorcount, it->mcflags);
}
}
void dump_cids_stats(const cached_id_sets &cids, LOGT logflags, const std::string &indent, const std::string &indentstep) {
size_t total = 0;
cids.IterateLists([&](const char *name, tweetidset cached_id_sets::* mptr, unsigned long long flag) {
const tweetidset &set = cids.*mptr;
LogMsgFormat(logflags, "%s%s (%s): %zu", cstr(indent), cstr(name), cstr(tweet_flags::GetValueString(flag)), set.size());
total += set.size();
});
LogMsgFormat(logflags, "%sTotal: %zu", cstr(indent), total);
}
// This determines the scrollbar size and position such that, where possible
// the item at the top of the visible screen is unmoved
void tpanelscrollbar::RepositionItems() {
tpanelscrollpane *tsp = get_paired_ptr();
if (!tsp) return;
#if TPANEL_SCROLLING_COPIOUS_LOGGING
LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::RepositionItems %s, START %d %d", cstr(GetThisName()),
GetThumbPosition(), parent->GetCurrentDisp().size());
#endif
scroll_virtual_size = 0;
int cumul_size = 0;
bool have_scroll_offset = false;
int scroll_offset = 0;
for (auto &disp : parent->GetCurrentDisp()) {
wxPoint p = disp.item->GetPosition();
wxSize s = disp.item->GetSize();
if (!disp.item->IsShown()) {
s.y = 0;
}
scroll_virtual_size += s.y;
if (p.x == 0 && p.y + s.y > 0 && p.y <= 0) {
// This is an item which is visible at the top of the list
// We should use the *last* matching item, this is as earlier items may grow in size to overlap the top of the screen
// p.y is non-positive
// The scroll offset should be increased as p.y increases in magnitude below 0
scroll_offset = cumul_size - p.y;
have_scroll_offset = true;
}
cumul_size += s.y;
}
if (parent->pimpl()->displayoffset == 0 && GetThumbPosition() == 0 &&
have_scroll_offset && !scroll_always_freeze) {
// We were at the very top, we would normally be scrolling down as something has been inserted above
// Scroll back to the top instead
// Don't do this if scroll_always_freeze is true
scroll_offset = 0;
}
scroll_always_freeze = false;
if (!have_scroll_offset) {
scroll_offset = GetThumbPosition();
}
scroll_client_size = tsp->GetClientSize().y;
SetScrollbar(scroll_offset, scroll_client_size, scroll_virtual_size, 1);
ScrollItems();
#if TPANEL_SCROLLING_COPIOUS_LOGGING
LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::RepositionItems %s, END %d %d %d %d %d", cstr(GetThisName()),
GetThumbPosition(), scroll_offset, have_scroll_offset, scroll_always_freeze, cumul_size);
#endif
}
void dump_pending_retry_conn(LOGT logflags, const std::string &indent, const std::string &indentstep) {
size_t count = 0;
for (auto &it : sm.retry_conns) {
if (it) count++;
}
LogMsgFormat(logflags, "%sConnections pending retry attempts: %d", cstr(indent), count);
for (auto &it : sm.retry_conns) {
if (!it) continue;
LogMsgFormat(logflags, "%s%sSocket: %s, ID: %d, Error Count: %d, mcflags: 0x%X",
cstr(indent), cstr(indentstep), cstr(it->GetConnTypeName()), it->id, it->errorcount, it->mcflags);
}
}
//return true if has been handled asynchronously
bool adns::CheckAsync(CURL *ch, std::unique_ptr<mcurlconn> &&cs) {
long timeout = -1;
curl_easy_setopt(ch, CURLOPT_DNS_CACHE_TIMEOUT, timeout);
curl_easy_setopt(ch, CURLOPT_SHARE, GetHndl());
const std::string &url = cs->url;
if (url.empty()) {
return false;
}
std::string::size_type proto_start = url.find("://");
std::string::size_type name_start;
if (proto_start == std::string::npos) {
name_start = 0;
} else {
name_start = proto_start + 3;
}
std::string::size_type name_end = url.find('/', name_start);
std::string name = url.substr(name_start, (name_end == std::string::npos) ? std::string::npos : name_end - name_start);
if (cached_names.count(name)) {
return false; // name already in cache, can go now
}
dns_pending_conns.push_front({ name, ch, std::move(cs) });
// cs is now null, don't use again
for (auto &it : dns_threads) {
if (it.first == name) {
LogMsgFormat(LOGT::SOCKTRACE, "DNS lookup thread already exists: %s, %s", cstr(url), cstr(name));
return true;
}
}
LogMsgFormat(LOGT::SOCKTRACE, "Creating DNS lookup thread: %s, %s", cstr(url), cstr(name));
dns_threads.emplace_front(std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(url, name, sm, GetHndl()));
adns_thread &ad = dns_threads.front().second;
ad.Create();
#if defined(__GLIBC__)
#if __GLIBC_PREREQ(2, 12)
pthread_setname_np(ad.GetId(), "retcon-adns");
#endif
#endif
ad.Run();
return true;
}
curl_socket_t pre_connect_func(void *clientp, curl_socket_t curlfd, curlsocktype purpose) {
mcurlconn *cs = static_cast<mcurlconn *>(clientp);
double lookuptime;
curl_easy_getinfo(cs->GenGetCurlHandle(), CURLINFO_NAMELOOKUP_TIME, &lookuptime);
LogMsgFormat(LOGT::SOCKTRACE, "DNS lookup took: %fs. Request: type: %s, conn ID: %d, url: %s", lookuptime, cstr(cs->GetConnTypeName()), cs->id, cstr(cs->url));
return 0;
}
static void dump_pending_user_line(LOGT logflags, const std::string &indent, userdatacontainer *u) {
time_t now = time(nullptr);
LogMsgFormat(logflags, "%sUser: %" llFmtSpec "d (%s) udc_flags: 0x%X, last update: %" llFmtSpec "ds ago"
", last DB update: %" llFmtSpec "ds ago, image ready: %d, ready (nx): %d, ready (x): %d",
cstr(indent), u->id, cstr(u->GetUser().screen_name), u->udc_flags, (uint64_t) (now-u->lastupdate),
(uint64_t) (now-u->lastupdate_wrotetodb), u->ImgIsReady(PENDING_REQ::PROFIMG_NEED), u->IsReady(0), u->IsReady(PENDING_REQ::USEREXPIRE));
}
int retcon::OnExit() {
LogMsg(LOGT::OTHERTRACE, "retcon::OnExit");
sm.DeInitMultiIOHandler();
pool.reset();
DBC_DeInit();
for (const observer_ptr<temp_file_holder> &it : temp_file_set) {
LogMsgFormat(LOGT::FILEIOTRACE, "retcon::OnExit, resetting: %s", cstr(it->GetFilename()));
it->Reset();
}
if (!::wxRmdir(wxstrstd(tmpdir))) {
LogMsgFormat(LOGT::FILEIOERR, "retcon::OnExit, could not remove temp dir: %s", cstr(tmpdir));
}
DebugFinalChecks();
DeInitWxLogger();
std::exit(0);
}
static void LogSocketErrorMessage(mcurlconn *mc, CURL *easy, long httpcode, CURLcode res, MCC_HTTPERRTYPE err) {
if (!(currentlogflags & LOGT::SOCKERR)) {
return;
}
const char *action = nullptr;
switch (err) {
case MCC_RETRY:
action = " (retrying)";
break;
case MCC_FAILED:
action = "";
break;
}
char *req_url = nullptr;
curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &req_url);
std::string fail_type;
if (res == CURLE_OK) {
fail_type = string_format("Request failed: HTTP code: %ld", httpcode);
} else {
fail_type = string_format("Socket error: %s", cstr(curl_easy_strerror(res)));
}
LogMsgFormat(LOGT::SOCKERR, "%s%s%s, type: %s, url: %s, conn ID: %u",
cstr(fail_type), cstr(action), cstr(mc->GetFailureLogInfo()), cstr(mc->GetConnTypeName()), cstr(req_url), mc->id);
}
// This checks both the retry list and the active list for removal
std::unique_ptr<mcurlconn> mcurlconn::RemoveConnCommon(const char *logprefix) {
std::unique_ptr<mcurlconn> conn = sm.UnregisterRetryConn(*this);
if (conn) {
LogMsgFormat(LOGT::SOCKTRACE, "%s (retry list): conn ID: %d", logprefix, id);
return std::move(conn);
}
conn = sm.RemoveConn(GenGetCurlHandle());
if (conn) {
LogMsgFormat(LOGT::SOCKTRACE, "%s: conn ID: %d", logprefix, id);
return std::move(conn);
}
LogMsgFormat(LOGT::SOCKERR, "%s: failed, conn ID: %d", logprefix, id);
return nullptr;
}
void log_window::OnDumpConnInfo(wxCommandEvent &event) {
dump_pending_active_conn(LOGT::USERREQ, "", "\t");
dump_pending_retry_conn(LOGT::USERREQ, "", "\t");
for (auto &it : alist) {
LogMsgFormat(LOGT::USERREQ, "Account: %s (%s)", cstr(it->name), cstr(it->dispname));
dump_pending_acc_failed_conns(LOGT::USERREQ, "\t", "\t", it.get());
}
}
void dump_acc_id_stats(const taccount &acc, LOGT logflags, const std::string &indent, const std::string &indentstep) {
auto line = [&](const char *name, size_t value) {
LogMsgFormat(logflags, "%s%s: %zu", cstr(indent), cstr(name), value);
};
line("Tweets IDs", acc.tweet_ids.size());
line("DM IDs", acc.dm_ids.size());
line("Pending users", acc.pendingusers.size());
line("User relations", acc.user_relations.size());
}
void tpanelscrollpane::resizehandler(wxSizeEvent &event) {
#if TPANEL_SCROLLING_COPIOUS_LOGGING
LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollpane::resizehandler: %s, %d, %d", cstr(GetThisName()), event.GetSize().GetWidth(), event.GetSize().GetHeight());
#endif
for (auto &disp : parent->GetCurrentDisp()) {
disp.item->NotifySizeChange();
}
}
static void dump_pending_tweet(LOGT logflags, const std::string &indent, const std::string &indentstep, tweet *t, userdatacontainer *exclude_user) {
LogMsgFormat(logflags, "%sTweet: %s", cstr(indent), cstr(tweet_log_line(t)));
if (t->user && t->user.get() != exclude_user) {
dump_pending_user_line(logflags, indent+indentstep, t->user.get());
}
if (t->user_recipient && t->user_recipient.get() != exclude_user) {
dump_pending_user_line(logflags, indent+indentstep, t->user_recipient.get());
}
}
void tpanelscrollbar::OnScrollTrack(wxScrollEvent &event) {
#if TPANEL_SCROLLING_COPIOUS_LOGGING
LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::OnScrollTrack %s, %d", cstr(GetThisName()), event.GetPosition());
#endif
int y = event.GetPosition();
ScrollItemsForPosition(y);
SetThumbPosition(y);
event.Skip();
}
void SetCacerts(CURL *handle) {
static std::string cacertpath;
if (cacertpath.empty()) {
cacertpath = "./cacert.pem";
if (wxFile::Exists(wxstrstd(cacertpath))) {
LogMsgFormat(LOGT::SOCKTRACE, "SetCacerts: found existing ./cacert.pem");
// There is an existing cacert.pem we can use, no need to create one
} else {
cacertpath = wxGetApp().datadir + "/cacert.pem";
LogMsgFormat(LOGT::SOCKTRACE, "SetCacerts: Writing out cacert.pem to: %s", cstr(cacertpath));
wxFFileOutputStream output(wxstrstd(cacertpath));
wxMemoryInputStream input(cacert_start, cacert_end - cacert_start);
wxZlibInputStream zliber(input, wxZLIB_ZLIB);
output.Write(zliber);
LogMsgFormat(LOGT::SOCKTRACE, "SetCacerts: Writing out cacert.pem complete");
}
}
curl_easy_setopt(handle, CURLOPT_CAINFO, cacertpath.c_str());
}
static int sock_cb(CURL *e, curl_socket_t s, int what, socketmanager *smp, mcurlconn *cs) {
LogMsgFormat(LOGT::SOCKTRACE, "Socket Interest Change Callback: %d, %d, conn ID: %d", s, what, cs ? cs->id : -1);
if (what != CURL_POLL_REMOVE) {
if (!cs) {
curl_easy_getinfo(e, CURLINFO_PRIVATE, &cs);
curl_multi_assign(smp->curlmulti, s, cs);
}
}
smp->RegisterSockInterest(e, s, what);
return 0;
}
void socketmanager::RetryConn(std::unique_ptr<mcurlconn> cs) {
if (cs->mcflags & mcurlconn::MCF::IN_RETRY_QUEUE) {
LogMsgFormat(LOGT::SOCKERR, "socketmanager::RetryConn: Attempt to add mcurlconn to retry queue which is marked as already in queue, this is a bug: type: %s, conn ID: %d, url: %s",
cstr(cs->GetConnTypeName()), cs->id, cstr(cs->url));
return;
}
cs->mcflags |= mcurlconn::MCF::IN_RETRY_QUEUE;
cs->AddToRetryQueueNotify();
retry_conns.push_back(std::move(cs));
RetryConnLater();
}
void tpanelscrollpane::resizemsghandler(wxCommandEvent &event) {
#if TPANEL_SCROLLING_COPIOUS_LOGGING
LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollpane::resizemsghandler %s", cstr(GetThisName()));
#endif
if (tpanelscrollbar *tsb = get_paired_ptr()) {
tsb->RepositionItems();
}
resize_update_pending = false;
Thaw();
Update();
}
adns::~adns() {
size_t n = std::distance(dns_threads.begin(), dns_threads.end());
if (n) {
size_t i = 0;
for (auto &it : dns_threads) {
LogMsgFormat(LOGT::SOCKTRACE, "Waiting for all DNS threads to terminate: %d of %d", i, n);
it.second.Wait();
i++;
}
LogMsg(LOGT::SOCKTRACE, "All DNS threads terminated");
}
RemoveShareHndl();
}
static void dump_non_acc_user_pendings(LOGT logflags, const std::string &indent, const std::string &indentstep) {
std::set<uint64_t> acc_pending_users;
for (auto &it : alist) {
for (auto &jt : it->pendingusers) {
acc_pending_users.insert(jt.second->id);
}
}
LogMsgFormat(logflags, "%sOther pending users:", cstr(indent));
for (auto &it : ad.userconts) {
if (!(it.second->pendingtweets.empty())) {
if (acc_pending_users.find(it.first) == acc_pending_users.end()) {
dump_pending_user(logflags, indent + indentstep, indentstep, it.second.get());
}
}
}
if (!ad.noacc_pending_userconts.empty()) {
LogMsgFormat(logflags, "%sUsers pending usable account:", cstr(indent));
for (auto &it : ad.noacc_pending_userconts) {
dump_pending_user(logflags, indent + indentstep, indentstep, it.second.get());
}
}
}
void dump_tweet_pendings(LOGT logflags, const std::string &indent, const std::string &indentstep) {
bool done_header = false;
for (auto &it : ad.tweetobjs) {
const tweet *t = it.second.get();
if (t && !(t->pending_ops.empty())) {
if (!done_header) {
done_header = true;
LogMsgFormat(logflags, "%sTweets with operations pending ready state:", cstr(indent));
}
dump_tweet_line(logflags, indent + indentstep, indentstep, t);
}
}
if (!ad.noacc_pending_tweetobjs.empty()) {
LogMsgFormat(logflags, "%sTweets pending usable account:", cstr(indent));
for (auto &it : ad.noacc_pending_tweetobjs) {
dump_tweet_line(logflags, indent + indentstep, indentstep, it.second.get());
}
}
if (!ad.handlenew_pending_ops.empty()) {
LogMsgFormat(logflags, "%sTotal handlenew_pending_op pending count: %zu", cstr(indent), ad.handlenew_pending_ops.size());
}
}
void adns::DNSResolutionEvent(wxCommandEvent &event) {
adns_thread *at = static_cast<adns_thread*>(event.GetClientData());
if (!at) return;
at->Wait();
if (at->success) {
LogMsgFormat(LOGT::SOCKTRACE, "Asynchronous DNS lookup succeeded: %s, %s, time: %fs", cstr(at->hostname), cstr(at->url), at->lookuptime);
cached_names.insert(at->hostname);
} else {
LogMsgFormat(LOGT::SOCKERR, "Asynchronous DNS lookup failed: %s, (%s), error: %s (%d), time: %fs", cstr(at->hostname), cstr(at->url), cstr(curl_easy_strerror(at->result)), at->result, at->lookuptime);
}
std::vector<dns_pending_conn> current_dns_pending_conns;
dns_pending_conns.remove_if ([&](dns_pending_conn &a) -> bool {
if (a.hostname == at->hostname) {
current_dns_pending_conns.emplace_back(std::move(a));
return true;
} else {
return false;
}
});
dns_threads.remove_if ([&](const std::pair<std::string, adns_thread> &a) {
return at == &(a.second);
});
for (auto &it : current_dns_pending_conns) {
CURL *ch = it.ch;
std::unique_ptr<mcurlconn> mc = std::move(it.cs);
if (at->success) {
LogMsgFormat(LOGT::SOCKTRACE, "Launching request as DNS lookup succeeded: type: %s, conn ID: %d, url: %s", cstr(mc->GetConnTypeName()), mc->id, cstr(mc->url));
sm->AddConn(ch, std::move(mc));
} else {
LogMsgFormat(LOGT::SOCKERR, "Request failed due to asynchronous DNS lookup failure: type: %s, conn ID: %d, url: %s", cstr(mc->GetConnTypeName()), mc->id, cstr(mc->url));
mc->HandleError(ch, 0, CURLE_COULDNT_RESOLVE_HOST, std::move(mc));
}
}
}
static int multi_timer_cb(CURLM *multi, long timeout_ms, socketmanager *smp) {
LogMsgFormat(LOGT::SOCKTRACE, "Socket Timer Callback: %d ms", timeout_ms);
if (timeout_ms > 0) {
smp->st->Start(timeout_ms,wxTIMER_ONE_SHOT);
} else {
smp->st->Stop();
}
if (!timeout_ms) {
smp->st->Notify();
}
return 0;
}
void socketmanager::RetryConnNow() {
std::unique_ptr<mcurlconn> cs;
while (true) {
if (retry_conns.empty()) return;
cs = std::move(retry_conns.front());
retry_conns.pop_front();
if (cs) break;
}
cs->mcflags &= ~mcurlconn::MCF::IN_RETRY_QUEUE;
cs->mcflags |= mcurlconn::MCF::RETRY_NOW_ON_SUCCESS;
LogMsgFormat(LOGT::SOCKTRACE, "Dequeueing request from retry queue: type: %s, conn ID: %d, url: %s", cstr(cs->GetConnTypeName()), cs->id, cstr(cs->url));
cs->RemoveFromRetryQueueNotify();
cs->DoRetry(std::move(cs));
}
void socketmanager::InitMultiIOHandler() {
if (MultiIOHandlerInited) return;
InitMultiIOHandlerCommon();
int pipefd[2];
pipe(pipefd);
socketpollthread *th = new socketpollthread();
th->pipefd = pipefd[0];
this->pipefd = pipefd[1];
th->Create();
th->Run();
LogMsgFormat(LOGT::SOCKTRACE, "socketmanager::InitMultiIOHandler(): Created socket poll() thread: %d", th->GetId());
MultiIOHandlerInited = true;
}
void tpanelscrollbar::mousewheelhandler(wxMouseEvent &event) {
int pxdelta = -event.GetWheelRotation() * gc.mousewheelscrollspeed / event.GetWheelDelta();
#if TPANEL_SCROLLING_COPIOUS_LOGGING
LogMsgFormat(LOGT::TPANELTRACE, "TSCL: tpanelscrollbar::mousewheelhandler %s, %d %d %d", cstr(GetThisName()),
GetScrollPos(wxVERTICAL), event.GetWheelRotation(), pxdelta);
#endif
int y = GetThumbPosition();
SetThumbPosition(std::max(0, y + pxdelta));
ScrollItems();
// Use gc.mousewheelscrollspeed as the threshold, this is to try and make mousewheel scrolling smooth across pages
OnScrollHandlerCommon(pxdelta < 0, pxdelta > 0, std::abs(gc.mousewheelscrollspeed), GetScrollPos(wxVERTICAL));
}
std::unique_ptr<mcurlconn> socketmanager::UnregisterRetryConn(mcurlconn &cs) {
if (!(cs.mcflags & mcurlconn::MCF::IN_RETRY_QUEUE)) return nullptr;
std::unique_ptr<mcurlconn> csptr;
container_unordered_remove_if (retry_conns, [&](std::unique_ptr<mcurlconn> &it) {
if (it.get() == &cs) {
LogMsgFormat(LOGT::SOCKTRACE, "socketmanager::UnregisterRetryConn: Unregistered from retry queue: type: %s, conn ID: %d, url: %s", cstr(cs.GetConnTypeName()), cs.id, cstr(cs.url));
csptr = std::move(it);
csptr->mcflags &= ~mcurlconn::MCF::IN_RETRY_QUEUE;
csptr->RemoveFromRetryQueueNotify();
return true;
} else {
return false;
}
});
return std::move(csptr);
}