int
dmtcp::findLib_byname(string pattern, string &libpath)
{
// /proc/self/maps looks like: "<start addr>-<end addr> <mode> <offset>
// <device> <inode> <libpath>
// we need to extract libpath
ProcMapsArea area;
int ret = -1;
ProcSelfMaps procSelfMaps;
while (procSelfMaps.getNextArea(&area)) {
libpath = area.name;
// JTRACE("Inspect new /proc/seft/maps line")(libpath);
if (libpath.size() == 0) {
// JTRACE("anonymous region, skip");
continue;
}
if (libpath.find(pattern) != string::npos) {
// This is the library path that contains libtorque. This is what we
// need.
// JTRACE("Found libpath")(pattern)(libpath);
ret = 0;
break;
} else {
// JTRACE("Libpath not found")(pattern)(libpath);
}
}
return ret;
}
int
dmtcp::findLib_byfunc(string fname, string &libpath)
{
// /proc/self/maps looks like: "<start addr>-<end addr> <mode> <offset>
// <device> <inode> <libpath>
// We need to extract libpath.
ProcMapsArea area;
int ret = -1;
ProcSelfMaps procSelfMaps;
while (procSelfMaps.getNextArea(&area)) {
libpath = area.name;
// JTRACE("Inspect new /proc/seft/maps line")(libpath);
if (libpath.size() == 0) {
// JTRACE("anonymous region, skip");
continue;
}
if (libpath.find("libdmtcp") != string::npos) {
// JTRACE("dmtcp plugin, skip")(libpath);
continue;
}
void *handle = dlopen(libpath.c_str(), RTLD_LAZY);
if (handle == NULL) {
// JTRACE("Cannot open libpath, skip")(libpath);
continue;
}
void *fptr = dlsym(handle, fname.c_str());
if (fptr != NULL) {
// Able to find the requested symbol.
// JTRACE("Found libpath by content:")(fname)(libpath);
dlclose(handle);
ret = 0;
break;
}
dlclose(handle);
// JTRACE("Function not found in Libpath")(fname)(libpath);
}
return ret;
}
void mtcp_writememoryareas(int fd)
{
Area area;
//DeviceInfo dev_info;
int stack_was_seen = 0;
JTRACE("Performing checkpoint.");
// Here we want to sync the shared memory pages with the backup files
// FIXME: Why do we need this?
//JTRACE("syncing shared memory with backup files");
//sync_shared_mem();
/**************************************************************************/
/* We can't do any more mallocing at this point because malloc stuff is */
/* outside the limits of the libmtcp.so image, so it won't get */
/* checkpointed, and it's possible that we would checkpoint an */
/* inconsistent state. See note in restoreverything routine. */
/**************************************************************************/
{
if (nscdAreas == NULL) {
nscdAreas = new vector<ProcMapsArea>();
}
nscdAreas->clear();
// This block is to ensure that the object is deleted as soon as we leave
// this block.
ProcSelfMaps procSelfMaps;
// Preprocess memory regions as needed.
while (procSelfMaps.getNextArea(&area)) {
if (Util::isNscdArea(area)) {
/* Special Case Handling: nscd is enabled*/
JNOTE("NSCD daemon shared memory area present.\n"
" MTCP will now try to remap this area in read/write mode as\n"
" private (zero pages), so that glibc will automatically\n"
" stop using NSCD or ask NSCD daemon for new shared area\n")
(area.name);
nscdAreas->push_back(area);
}
}
}
if (procSelfMaps != NULL) {
// We need to explicitly delete this object here because on restart, we
// never get back to this function and the object is never released.
delete procSelfMaps;
};
/* Finally comes the memory contents */
procSelfMaps = new ProcSelfMaps();
while (procSelfMaps->getNextArea(&area)) {
// TODO(kapil): Verify that we are not doing any operation that might
// result in a change of memory layout. For example, a call to JALLOC_NEW
// will invoke mmap if the JAlloc arena is full. Similarly, for STL objects
// such as vector and string.
if ((uint64_t)area.addr == ProcessInfo::instance().restoreBufAddr()) {
JASSERT(area.size == ProcessInfo::instance().restoreBufLen())
((void*) area.addr) (area.size) (ProcessInfo::instance().restoreBufLen());
continue;
}
/* Original comment: Skip anything in kernel address space ---
* beats me what's at FFFFE000..FFFFFFFF - we can't even read it;
* Added: That's the vdso section for earlier Linux 2.6 kernels. For later
* 2.6 kernels, vdso occurs at an earlier address. If it's unreadable,
* then we simply won't copy it. But let's try to read all areas, anyway.
* **COMMENTED OUT:** if (area.addr >= HIGHEST_VA) continue;
*/
/* If it's readable, but it's VDSO, it will be dangerous to restore it.
* In 32-bit mode later Red Hat RHEL Linux 2.6.9 releases use 0xffffe000,
* the last page of virtual memory. Note 0xffffe000 >= HIGHEST_VA
* implies we're in 32-bit mode.
*/
if (area.addr >= HIGHEST_VA && area.addr == (VA)0xffffe000)
continue;
#ifdef __x86_64__
/* And in 64-bit mode later Red Hat RHEL Linux 2.6.9 releases
* use 0xffffffffff600000 for VDSO.
*/
if (area.addr >= HIGHEST_VA && area.addr == (VA)0xffffffffff600000)
continue;
#endif
/* Skip anything that has no read or execute permission. This occurs
* on one page in a Linux 2.6.9 installation. No idea why. This code
* would also take care of kernel sections since we don't have read/execute
* permission there.
*
* EDIT: We should only skip the "---p" section for the shared libraries.
* Anonymous memory areas with no rwx permission should be saved regardless
* as the process might have removed the permissions temporarily and might
* want to use it later.
*
* This happens, for example, with libpthread where the pthread library
* tries to recycle thread stacks. When a thread exits, libpthread will
* remove the access permissions from the thread stack and later, when a
* new thread is created, it will provide the proper permission to this
* area and use it as the thread stack.
*
* If we do not restore this area on restart, the area might be returned by
* some mmap() call. Later on, when pthread wants to use this area, it will
* just try to use this area which now belongs to some other object. Even
* worse, the other object can then call munmap() on that area after
* libpthread started using it as thread stack causing the parts of thread
* stack getting munmap()'d from the memory resulting in a SIGSEGV.
*
* We suspect that libpthread is using mmap() instead of mprotect to change
* the permission from "---p" to "rw-p".
*/
if (!((area.prot & PROT_READ) || (area.prot & PROT_WRITE)) &&
area.name[0] != '\0') {
continue;
}
if (Util::strStartsWith(area.name, DEV_ZERO_DELETED_STR) ||
Util::strStartsWith(area.name, DEV_NULL_DELETED_STR)) {
/* If the process has an area labelled as "/dev/zero (deleted)", we mark
* the area as Anonymous and save the contents to the ckpt image file.
* If this area has a MAP_SHARED attribute, it should be replaced with
* MAP_PRIVATE and we won't do any harm because, the /dev/zero file is
* an absolute source and sink. Anything written to it will be
* discarded and anything read from it will be all zeros.
* The following call to mmap will create "/dev/zero (deleted)" area
* mmap(addr, size, protection, MAP_SHARED | MAP_ANONYMOUS, 0, 0)
*
* The above explanation also applies to "/dev/null (deleted)"
*/
JTRACE("saving area as Anonymous") (area.name);
area.flags = MAP_PRIVATE | MAP_ANONYMOUS;
area.name[0] = '\0';
} else if (Util::isSysVShmArea(area)) {
JTRACE("saving area as Anonymous") (area.name);
area.flags = MAP_PRIVATE | MAP_ANONYMOUS;
area.name[0] = '\0';
} else if (Util::isNscdArea(area)) {
/* Special Case Handling: nscd is enabled*/
area.prot = PROT_READ | PROT_WRITE | MTCP_PROT_ZERO_PAGE;
area.flags = MAP_PRIVATE | MAP_ANONYMOUS;
Util::writeAll(fd, &area, sizeof(area));
continue;
} else if (Util::isIBShmArea(area)) {
// TODO: Don't checkpoint infiniband shared area for now.
continue;
} else if (Util::strEndsWith(area.name, DELETED_FILE_SUFFIX)) {
/* Deleted File */
} else if (area.name[0] == '/' && strstr(&area.name[1], "/") != NULL) {
/* If an absolute pathname
* Posix and SysV shared memory segments can be mapped as /XYZ
*/
}
/* Force the anonymous flag if it's a private writeable section, as the
* data has probably changed from the contents of the original images.
*/
/* We also do this for read-only private sections as it's possible
* to modify a page there, too (via mprotect).
*/
if ((area.flags & MAP_PRIVATE) /*&& (area.prot & PROT_WRITE)*/) {
area.flags |= MAP_ANONYMOUS;
}
/* Only write this image if it is not CS_RESTOREIMAGE.
* Skip any mapping for this image - it got saved as CS_RESTOREIMAGE
* at the beginning.
*/
if (strstr (area.name, "[stack]"))
stack_was_seen = 1;
// the whole thing comes after the restore image
writememoryarea(fd, &area, stack_was_seen);
}
// Release the memory.
delete procSelfMaps;
procSelfMaps = NULL;
/* It's now safe to do this, since we're done using mtcp_readmapsline() */
remap_nscd_areas(*nscdAreas);
area.addr = NULL; // End of data
area.size = -1; // End of data
Util::writeAll(fd, &area, sizeof(area));
/* That's all folks */
JASSERT(_real_close (fd) == 0);
}
void FileConnList::prepareShmList()
{
ProcSelfMaps procSelfMaps;
ProcMapsArea area;
shmAreas.clear();
unlinkedShmAreas.clear();
missingUnlinkedShmFiles.clear();
shmAreaConn.clear();
while (procSelfMaps.getNextArea(&area)) {
if ((area.flags & MAP_SHARED) && area.prot != 0) {
if (strstr(area.name, "ptraceSharedInfo") != NULL ||
strstr(area.name, "dmtcpPidMap") != NULL ||
strstr(area.name, "dmtcpSharedArea") != NULL ||
strstr(area.name, "dmtcpSharedArea") != NULL ||
strstr(area.name, "synchronization-log") != NULL ||
strstr(area.name, "infiniband") != NULL ||
strstr(area.name, "synchronization-read-log") != NULL) {
continue;
}
if (Util::isNscdArea(area) ||
Util::isIBShmArea(area) ||
Util::isSysVShmArea(area)) {
continue;
}
/* Invalidate shared memory pages so that the next read to it (when we are
* writing them to ckpt file) will cause them to be reloaded from the
* disk.
*/
JWARNING(msync(area.addr, area.size, MS_INVALIDATE) == 0)
(area.addr) (area.size) (area.name) (area.offset) (JASSERT_ERRNO);
if (jalib::Filesystem::FileExists(area.name)) {
if (_real_access(area.name, W_OK) == 0) {
JTRACE("Will checkpoint shared memory area") (area.name);
int flags = Util::memProtToOpenFlags(area.prot);
int fd = _real_open(area.name, flags, 0);
JASSERT(fd != -1) (JASSERT_ERRNO) (area.name);
FileConnection *fileConn =
new FileConnection(area.name, flags, 0, FileConnection::FILE_SHM);
add(fd, fileConn);
shmAreas.push_back(area);
shmAreaConn.push_back(fileConn);
/* Instead of unmapping the shared memory area, we make it
* non-readable. This way mtcp will skip the region while at the same
* time, we prevent JALLOC arena to grow over it.
*
* By munmapping the area, a bug was observed on CCIS linux with
* 'make check-java'. Once the region was unmapped, the JALLOC arena
* grew over it. During restart, the JALLOC'd area was reclaimed for
* remapping the shm file without informing JALLOC. Finally, during
* the second checkpoint cycle, the area was again unmapped and later
* JALLOC tried to access it, causing a SIGSEGV.
*/
JASSERT(_real_mmap(area.addr, area.size, PROT_NONE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
-1, 0) != MAP_FAILED) (JASSERT_ERRNO);
} else {
JTRACE("Will not checkpoint shared memory area") (area.name);
}
} else {
// TODO: Shared memory areas with unlinked backing files.
JASSERT(Util::strEndsWith(area.name, DELETED_FILE_SUFFIX)) (area.name);
if (Util::strStartsWith(area.name, DEV_ZERO_DELETED_STR) ||
Util::strStartsWith(area.name, DEV_NULL_DELETED_STR)) {
JWARNING(false) (area.name)
.Text("Ckpt/Restart of anonymous shared memory not supported.");
} else {
JTRACE("Will recreate shm file on restart.") (area.name);
// Remove the DELETED suffix.
area.name[strlen(area.name) - strlen(DELETED_FILE_SUFFIX)] = '\0';
unlinkedShmAreas.push_back(area);
}
}
}
}
}
