void ptrace_process_resume_user_thread(int is_ckpt, int is_restart)
{
if (dmtcp::PtraceInfo::instance().isPtracing() && (is_ckpt || is_restart)) {
ptrace_attach_threads(is_restart);
}
JTRACE("Waiting for Sup Attach") (GETTID());
dmtcp::PtraceInfo::instance().waitForSuperiorAttach();
JTRACE("Done Waiting for Sup Attach") (GETTID());
}
// Clean out thread data that we know to be obsolete (leftover from a task
// that UAE'd or didn't cleanup after itself correctly).
//
void TlsCleanupDeadTasks()
{
PTHREAD pthread;
TID tid;
PID pid;
tid = GETTID(&tid); // tid = caller's SS
GETPID(pid); // pid = caller's process id
DebAssert(tid != TID_EMPTY, "");
// search for thread's tid field
for (pthread = g_rgthread; pthread < g_pthreadEnd; pthread++) {
// if we find an entry with our own SS, but a different process id,
// then it must be a process that has terminated without cleaning up
// after itself. So we wipe out it's entry now.
if (pthread->tid == tid && !ISEQUALPID(pthread->pid, pid)) {
// This thread's entry contains obsolete data, since the thread has
// terminated, and the memory pointed to by our structures is either
// gone, or in use by somebody else.
// Pretend this thread did TlsSetValue(itls, NULL) on all it's itls's,
// to put us back into a stable state.
memset (pthread->rgtls, 0, ITLS_COMPONENTS * sizeof(LPVOID));
pthread->tid = TID_EMPTY;
g_tidThreadCache = TID_EMPTY;
break;
}
}
}
TraceLog::LogBuffer &TraceLog::get_logbuffer()
{
static __thread LogBuffer log_buffer;
static __thread bool inited = false;
if (!inited)
{
TBSYS_LOG(INFO, "init trace log buffer tid=%ld buffer=%p", GETTID(), &log_buffer);
memset(&log_buffer, 0, sizeof(log_buffer));
inited = true;
}
return log_buffer;
}
void ObMergeServer::on_ioth_start()
{
int64_t affinity_start_cpu = ms_config_.io_thread_start_cpu;
int64_t affinity_end_cpu = ms_config_.io_thread_end_cpu;
if (0 <= affinity_start_cpu
&& affinity_start_cpu <= affinity_end_cpu)
{
static volatile int64_t cpu = 0;
int64_t local_cpu = __sync_fetch_and_add(&cpu, 1) % (affinity_end_cpu - affinity_start_cpu + 1) + affinity_start_cpu;
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(local_cpu, &cpuset);
int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
TBSYS_LOG(INFO, "io thread setaffinity tid=%ld ret=%d cpu=%ld start=%ld end=%ld",
GETTID(), ret, local_cpu, affinity_start_cpu, affinity_end_cpu);
}
}
LPVOID PASCAL EXPORT TlsGetValue(ITLS itls)
{
PTHREAD pthread;
TID tid;
#if ID_DEBUG
PID pidTmp;
#endif //ID_DEBUG
tid = GETTID(&tid); // tid = caller's SS
DebAssert(tid != TID_EMPTY, "");
DebAssert(itls < ITLS_COMPONENTS, "TlsGetValue err1");
DebAssert(g_rgbSlot[itls] != 0, "TlsGetValue err2");
if (g_tidThreadCache == tid) {
#if ID_DEBUG
GETPID(pidTmp);
DebAssert(ISEQUALPID(pidTmp, g_pthreadCache->pid), "bad pid");
#endif //ID_DEBUG
return g_pthreadCache->rgtls[itls];
}
// cache doesn't contain desired value
// search for thread's tid field
for (pthread = g_rgthread; pthread < g_pthreadEnd; pthread++) {
if (pthread->tid == tid) {
#if ID_DEBUG
GETPID(pidTmp);
DebAssert(ISEQUALPID(pidTmp, pthread->pid), "bad pid");
#endif //ID_DEBUG
g_tidThreadCache = tid;
g_pthreadCache = pthread;
return pthread->rgtls[itls];
}
}
// thread not found.
// value must be NULL since TlsSetValue hasn't been called.
//
return NULL;
}
BOOL PASCAL EXPORT TlsSetValue(ITLS itls, LPVOID pvParam)
{
TID tid;
PTHREAD pthread, pthreadFree;
LPVOID * ppv;
#if ID_DEBUG
PID pidTmp;
#endif //ID_DEBUG
tid = GETTID(&tid); // tid = caller's SS
DebAssert(tid != TID_EMPTY, "");
DebAssert(itls < ITLS_COMPONENTS, "TlsSetValue err1");
DebAssert(g_rgbSlot[itls] != 0, "TlsSetValue err2");
// search for thread's tid field
pthreadFree = NULL;
for (pthread = g_rgthread; pthread < g_pthreadEnd; pthread++) {
if (pthread->tid == tid) {
#if ID_DEBUG
GETPID(pidTmp);
DebAssert(ISEQUALPID(pidTmp, pthread->pid), "bad pid");
#endif //ID_DEBUG
goto SetValue; // found it -- set the value
}
if ((pthreadFree == NULL) &&
(pthread->tid == TID_EMPTY)) {
// remember 1st free thread
pthreadFree = pthread;
}
}
if (pthreadFree == NULL)
return FALSE; // no free slots available for this thread
// allocate a new THREAD instance
pthread = pthreadFree;
GETPID(pthread->pid);
SetValue:
pthread->rgtls[itls] = pvParam;
// WARNING: setting of tid must be AFTER the data value is set (directly
// above). This is required for the interrupt-driven break check,
// which assumes the data is valid if the tid is non-empty.
//
pthread->tid = tid;
#if ID_DEBUG
pthread->rgfSlotUsed[ itls ] = -1;
#endif
if (pvParam != NULL)
return TRUE;
// See if all of this thread's slots are NULL.
// If so, release resources occupied by the thread.
for (ppv = &pthread->rgtls[0];
ppv < &pthread->rgtls[ITLS_COMPONENTS];
ppv++) {
if (*ppv != NULL)
return TRUE; // at least one slot is still in use by this thread
}
// This thread's last field is NULL - release the thread, flush cache
pthread->tid = TID_EMPTY;
g_tidThreadCache = TID_EMPTY;
return TRUE;
}
static void ptrace_single_step_thread(dmtcp::Inferior *inferiorInfo,
int isRestart)
{
struct user_regs_struct regs;
long peekdata;
long low, upp;
int status;
unsigned long addr;
unsigned long int eflags;
pid_t inferior = inferiorInfo->tid();
pid_t superior = GETTID();
int last_command = inferiorInfo->lastCmd();
char inferior_st = inferiorInfo->state();
while(1) {
int status;
JASSERT(_real_ptrace(PTRACE_SINGLESTEP, inferior, 0, 0) != -1)
(superior) (inferior) (JASSERT_ERRNO);
if (_real_wait4(inferior, &status, 0, NULL) == -1) {
JASSERT(_real_wait4(inferior, &status, __WCLONE, NULL) != -1)
(superior) (inferior) (JASSERT_ERRNO);
}
if (WIFEXITED(status)) {
JTRACE("thread is dead") (inferior) (WEXITSTATUS(status));
} else if(WIFSIGNALED(status)) {
JTRACE("thread terminated by signal") (inferior);
}
JASSERT(_real_ptrace(PTRACE_GETREGS, inferior, 0, ®s) != -1)
(superior) (inferior) (JASSERT_ERRNO);
peekdata = _real_ptrace(PTRACE_PEEKDATA, inferior, (void*) regs.IP_REG, 0);
long inst = peekdata & 0xffff;
#ifdef __x86_64__
/* For 64 bit architectures. */
if (inst == SIGRETURN_INST_16 && regs.AX_REG == 0xf) {
#else /* For 32 bit architectures.*/
if (inst == SIGRETURN_INST_16 && (regs.AX_REG == DMTCP_SYS_sigreturn ||
regs.AX_REG == DMTCP_SYS_rt_sigreturn)) {
#endif
if (isRestart) { /* Restart time. */
// FIXME: TODO:
if (last_command == PTRACE_SINGLESTEP) {
if (regs.AX_REG != DMTCP_SYS_rt_sigreturn) {
addr = regs.SP_REG;
} else {
addr = regs.SP_REG + 8;
addr = _real_ptrace(PTRACE_PEEKDATA, inferior, (void*) addr, 0);
addr += 20;
}
addr += EFLAGS_OFFSET;
errno = 0;
JASSERT ((eflags = _real_ptrace(PTRACE_PEEKDATA, inferior,
(void *)addr, 0)) != -1)
(superior) (inferior) (JASSERT_ERRNO);
eflags |= 0x0100;
JASSERT(_real_ptrace(PTRACE_POKEDATA, inferior, (void *)addr,
(void*) eflags) != -1)
(superior) (inferior) (JASSERT_ERRNO);
} else if (inferior_st != PTRACE_PROC_TRACING_STOP) {
/* TODO: remove in future as GROUP restore becames stable
* - Artem */
JASSERT(_real_ptrace(PTRACE_CONT, inferior, 0, 0) != -1)
(superior) (inferior) (JASSERT_ERRNO);
}
} else { /* Resume time. */
if (inferior_st != PTRACE_PROC_TRACING_STOP) {
JASSERT(_real_ptrace(PTRACE_CONT, inferior, 0, 0) != -1)
(superior) (inferior) (JASSERT_ERRNO);
}
}
/* In case we have checkpointed at a breakpoint, we don't want to
* hit the same breakpoint twice. Thus this code. */
// TODO: FIXME: Replace this code with a raise(SIGTRAP) and see what happens
if (inferior_st == PTRACE_PROC_TRACING_STOP) {
JASSERT(_real_ptrace(PTRACE_SINGLESTEP, inferior, 0, 0) != -1)
(superior) (inferior) (JASSERT_ERRNO);
if (_real_wait4(inferior, &status, 0, NULL) == -1) {
JASSERT(_real_wait4(inferior, &status, __WCLONE, NULL) != -1)
(superior) (inferior) (JASSERT_ERRNO);
}
}
break;
}
} //while(1)
}
/* This function detaches the user threads. */
static void ptrace_detach_user_threads ()
{
PtraceProcState pstate;
int status;
struct rusage rusage;
dmtcp::vector<dmtcp::Inferior*> inferiors;
inferiors = dmtcp::PtraceInfo::instance().getInferiors(GETTID());
for (size_t i = 0; i < inferiors.size(); i++) {
pid_t inferior = inferiors[i]->tid();
void *data = (void*) (unsigned long) dmtcp_get_ckpt_signal();
pstate = procfs_state(inferiors[i]->tid());
if (pstate == PTRACE_PROC_INVALID) {
JTRACE("Inferior does not exist.") (inferior);
dmtcp::PtraceInfo::instance().eraseInferior(inferior);
continue;
}
inferiors[i]->setState(pstate);
inferiors[i]->semInit();
if (inferiors[i]->isCkptThread()) {
data = NULL;
}
int ret = _real_wait4(inferior, &status, __WALL | WNOHANG, &rusage);
if (ret > 0) {
if (!WIFSTOPPED(status) || WSTOPSIG(status) != dmtcp_get_ckpt_signal()) {
inferiors[i]->setWait4Status(&status, &rusage);
}
}
pstate = procfs_state(inferiors[i]->tid());
if (pstate == PTRACE_PROC_RUNNING || pstate == PTRACE_PROC_SLEEPING) {
syscall(SYS_tkill, inferior, SIGSTOP);
_real_wait4(inferior, &status, __WALL, NULL);
JASSERT(_real_wait4(inferior, &status, __WALL | WNOHANG, NULL) == 0)
(inferior) (JASSERT_ERRNO);
}
if (_real_ptrace(PTRACE_DETACH, inferior, 0, data) == -1) {
JASSERT(errno == ESRCH)
(GETTID()) (inferior) (JASSERT_ERRNO);
dmtcp::PtraceInfo::instance().eraseInferior(inferior);
continue;
}
pstate = procfs_state(inferiors[i]->tid());
if (pstate == PTRACE_PROC_STOPPED) {
kill(inferior, SIGCONT);
}
JTRACE("Detached thread") (inferior);
}
}
static void ptrace_attach_threads(int isRestart)
{
pid_t inferior;
int status;
dmtcp::vector<dmtcp::Inferior*> inferiors;
inferiors = dmtcp::PtraceInfo::instance().getInferiors(GETTID());
if (inferiors.size() == 0) {
return;
}
JTRACE("Attaching to inferior threads") (GETTID());
// Attach to all inferior user threads.
for (size_t i = 0; i < inferiors.size(); i++) {
inferior = inferiors[i]->tid();
JASSERT(inferiors[i]->state() != PTRACE_PROC_INVALID) (GETTID()) (inferior);
if (!inferiors[i]->isCkptThread()) {
JASSERT(_real_ptrace(PTRACE_ATTACH, inferior, 0, 0) != -1)
(GETTID()) (inferior) (JASSERT_ERRNO);
JASSERT(_real_wait4(inferior, &status, __WALL, NULL) != -1)
(inferior) (JASSERT_ERRNO);
JASSERT(_real_ptrace(PTRACE_SETOPTIONS, inferior, 0,
inferiors[i]->getPtraceOptions()) != -1)
(GETTID()) (inferior) (inferiors[i]->getPtraceOptions()) (JASSERT_ERRNO);
// Run all user threads until the end of syscall(DMTCP_FAKE_SYSCALL)
dmtcp::PtraceInfo::instance().processPreResumeAttach(inferior);
ptrace_wait_for_inferior_to_reach_syscall(inferior, DMTCP_FAKE_SYSCALL);
}
}
// Attach to and run all user ckpthreads until the end of syscall(DMTCP_FAKE_SYSCALL)
for (size_t i = 0; i < inferiors.size(); i++) {
inferior = inferiors[i]->tid();
if (inferiors[i]->isCkptThread()) {
JASSERT(_real_ptrace(PTRACE_ATTACH, inferior, 0, 0) != -1)
(GETTID()) (inferior) (JASSERT_ERRNO);
JASSERT(_real_wait4(inferior, &status, __WALL, NULL) != -1)
(inferior) (JASSERT_ERRNO);
JASSERT(_real_ptrace(PTRACE_SETOPTIONS, inferior, 0,
inferiors[i]->getPtraceOptions()) != -1)
(GETTID()) (inferior) (inferiors[i]->getPtraceOptions()) (JASSERT_ERRNO);
// Wait for all inferiors to execute dummy syscall 'DMTCP_FAKE_SYSCALL'.
dmtcp::PtraceInfo::instance().processPreResumeAttach(inferior);
ptrace_wait_for_inferior_to_reach_syscall(inferior, DMTCP_FAKE_SYSCALL);
}
}
// Singlestep all user threads out of the signal handler
for (size_t i = 0; i < inferiors.size(); i++) {
int lastCmd = inferiors[i]->lastCmd();
inferior = inferiors[i]->tid();
if (!inferiors[i]->isCkptThread()) {
/* After attach, the superior needs to singlestep the inferior out of
* stopthisthread, aka the signal handler. */
ptrace_single_step_thread(inferiors[i], isRestart);
if (inferiors[i]->isStopped() && (lastCmd == PTRACE_CONT ||
lastCmd == PTRACE_SYSCALL)) {
JASSERT(_real_ptrace(lastCmd, inferior, 0, 0) != -1)
(GETTID()) (inferior) (JASSERT_ERRNO);
}
}
}
// Move ckpthreads to next step (depending on state)
for (size_t i = 0; i < inferiors.size(); i++) {
int lastCmd = inferiors[i]->lastCmd();
inferior = inferiors[i]->tid();
if (inferiors[i]->isCkptThread() && !inferiors[i]->isStopped() &&
(lastCmd == PTRACE_CONT || lastCmd == PTRACE_SYSCALL)) {
JASSERT(_real_ptrace(lastCmd, inferior, 0, 0) != -1)
(GETTID()) (inferior) (JASSERT_ERRNO);
}
}
JTRACE("thread done") (GETTID());
}
static void*
sens_dbgcli_thr(void *arg)
{
/* IPv4 domain, reliable full-duplex stream type, TCP protocol */
int sfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (-1 == sfd) {
perror("socket(...) error");
exit(1);
}
int ret;
int optval = 1;
/* socket level option to reuse port/address */
if ( setsockopt(sfd,
SOL_SOCKET,
/*
* SO_REUSEPORT doesn't seem to be portable, disable
* for now
*/
//SO_REUSEPORT | SO_REUSEADDR,
SO_REUSEADDR,
&optval,
sizeof(optval)) ) {
perror("setsockopt( SO_REUSEADDR ) error");
exit(1);
}
/* 2 seconds linger to allow data transfer completion upon close */
struct linger ling;
memset(&ling, 0, sizeof(ling));
ling.l_onoff = 1;
ling.l_linger = 2;
if ( setsockopt(sfd,
SOL_SOCKET,
SO_LINGER,
(const char *)&ling,
sizeof(ling)) ) {
perror("setsockopt( SO_LINGER ) error");
exit(1);
}
struct sockaddr_in srvaddr;
memset(&srvaddr, 0, sizeof(srvaddr));
srvaddr.sin_family = AF_INET;
/* accept connection on any IP */
srvaddr.sin_addr.s_addr = INADDR_ANY;
srvaddr.sin_port = htons(DBGCLIPORT);
if ( bind(sfd, (struct sockaddr *)&srvaddr, sizeof(srvaddr)) ) {
perror("bind(...) error ");
exit(1);
}
char prompt[16];
snprintf(prompt, sizeof(prompt), "%s", "DBGCLI > ");
printf("CLI thread, TID: %d, port: %d\n", GETTID(), DBGCLIPORT);
while (1) {
if ( listen(sfd, SIMULMAXCONN) ) {
perror("listen(...) error");
exit(1);
}
struct sockaddr_in cliaddr;
socklen_t clilen = sizeof(cliaddr);
int clih;
clih = accept(sfd, (struct sockaddr*)&cliaddr, &clilen);
if (-1 == clih) {
switch (errno) {
/* for these errs, just continue and retry */
case EINTR:
case EAGAIN:
continue;
break;
default:
perror("accept(...) error");
exit(1);
break;
}
}
printf( "Connecting with %s:%u\n",
inet_ntoa(cliaddr.sin_addr),
(unsigned)ntohs(cliaddr.sin_port) );
(void)fflush(stdout);
FILE *clicon = fdopen(clih, "w+");
if (!clicon) {
perror("fdopen(clih) error");
exit(1);
}
while (1) {
int nrcv;
char *rcvbuf = NULL;
size_t len = 0;
fprintf(clicon, "%s", prompt);
(void)fflush(clicon);
nrcv = getline(&rcvbuf, &len, clicon);
ret = sens_dbgcli_handler(clicon, rcvbuf, nrcv);
free(rcvbuf);
if (0 > ret) {
fprintf(clicon, "Exiting CLI...\n");
printf( "Disconnecting %s:%u\n",
inet_ntoa(cliaddr.sin_addr),
(unsigned)ntohs(cliaddr.sin_port) );
(void)fflush(stdout);
break;
}
}
fflush(clicon);
fclose(clicon);
close(clih);
}
pthread_exit(0);
}
