/*
* Issue fsync recursively on PGDATA and all its contents.
*
* We fsync regular files and directories wherever they are, but we follow
* symlinks only for pg_wal (or pg_xlog) and immediately under pg_tblspc.
* Other symlinks are presumed to point at files we're not responsible for
* fsyncing, and might not have privileges to write at all.
*
* serverVersion indicates the version of the server to be fsync'd.
*
* Errors are reported but not considered fatal.
*/
void
fsync_pgdata(const char *pg_data,
const char *progname,
int serverVersion)
{
bool xlog_is_symlink;
char pg_wal[MAXPGPATH];
char pg_tblspc[MAXPGPATH];
/* handle renaming of pg_xlog to pg_wal in post-10 clusters */
snprintf(pg_wal, MAXPGPATH, "%s/%s", pg_data,
serverVersion < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal");
snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data);
/*
* If pg_wal is a symlink, we'll need to recurse into it separately,
* because the first walkdir below will ignore it.
*/
xlog_is_symlink = false;
#ifndef WIN32
{
struct stat st;
if (lstat(pg_wal, &st) < 0)
fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"),
progname, pg_wal, strerror(errno));
else if (S_ISLNK(st.st_mode))
xlog_is_symlink = true;
}
#else
if (pgwin32_is_junction(pg_wal))
xlog_is_symlink = true;
#endif
/*
* If possible, hint to the kernel that we're soon going to fsync the data
* directory and its contents.
*/
#ifdef PG_FLUSH_DATA_WORKS
walkdir(pg_data, pre_sync_fname, false, progname);
if (xlog_is_symlink)
walkdir(pg_wal, pre_sync_fname, false, progname);
walkdir(pg_tblspc, pre_sync_fname, true, progname);
#endif
/*
* Now we do the fsync()s in the same order.
*
* The main call ignores symlinks, so in addition to specially processing
* pg_wal if it's a symlink, pg_tblspc has to be visited separately with
* process_symlinks = true. Note that if there are any plain directories
* in pg_tblspc, they'll get fsync'd twice. That's not an expected case
* so we don't worry about optimizing it.
*/
walkdir(pg_data, fsync_fname, false, progname);
if (xlog_is_symlink)
walkdir(pg_wal, fsync_fname, false, progname);
walkdir(pg_tblspc, fsync_fname, true, progname);
}
/*
* Callback for processing source file list.
*
* This is called once for every file in the source server. We decide what
* action needs to be taken for the file, depending on whether the file
* exists in the target and whether the size matches.
*/
void
process_source_file(const char *path, file_type_t type, size_t newsize,
const char *link_target)
{
bool exists;
char localpath[MAXPGPATH];
struct stat statbuf;
filemap_t *map = filemap;
file_action_t action = FILE_ACTION_NONE;
size_t oldsize = 0;
file_entry_t *entry;
Assert(map->array == NULL);
/*
* Completely ignore some special files in source and destination.
*/
if (strcmp(path, "postmaster.pid") == 0 ||
strcmp(path, "postmaster.opts") == 0)
return;
/*
* Skip temporary files, .../pgsql_tmp/... and .../pgsql_tmp.* in source.
* This has the effect that all temporary files in the destination will be
* removed.
*/
if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL)
return;
if (strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL)
return;
/*
* sanity check: a filename that looks like a data file better be a
* regular file
*/
if (type != FILE_TYPE_REGULAR && isRelDataFile(path))
pg_fatal("data file in source \"%s\" is not a regular file\n", path);
snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path);
/* Does the corresponding file exist in the target data dir? */
if (lstat(localpath, &statbuf) < 0)
{
if (errno != ENOENT)
pg_fatal("could not stat file \"%s\": %s\n",
localpath, strerror(errno));
exists = false;
}
else
exists = true;
switch (type)
{
case FILE_TYPE_DIRECTORY:
if (exists && !S_ISDIR(statbuf.st_mode))
{
/* it's a directory in source, but not in target. Strange.. */
pg_fatal("\"%s\" is not a directory\n", localpath);
}
if (!exists)
action = FILE_ACTION_CREATE;
else
action = FILE_ACTION_NONE;
oldsize = 0;
break;
case FILE_TYPE_SYMLINK:
if (exists &&
#ifndef WIN32
!S_ISLNK(statbuf.st_mode)
#else
!pgwin32_is_junction(localpath)
#endif
)
{
/*
* It's a symbolic link in source, but not in target.
* Strange..
*/
pg_fatal("\"%s\" is not a symbolic link\n", localpath);
}
if (!exists)
action = FILE_ACTION_CREATE;
else
action = FILE_ACTION_NONE;
oldsize = 0;
break;
case FILE_TYPE_REGULAR:
if (exists && !S_ISREG(statbuf.st_mode))
pg_fatal("\"%s\" is not a regular file\n", localpath);
if (!exists || !isRelDataFile(path))
{
/*
* File exists in source, but not in target. Or it's a
* non-data file that we have no special processing for. Copy
* it in toto.
*
* An exception: PG_VERSIONs should be identical, but avoid
* overwriting it for paranoia.
*/
if (pg_str_endswith(path, "PG_VERSION"))
{
action = FILE_ACTION_NONE;
oldsize = statbuf.st_size;
}
else
{
action = FILE_ACTION_COPY;
oldsize = 0;
}
}
else
{
/*
* It's a data file that exists in both.
*
* If it's larger in target, we can truncate it. There will
* also be a WAL record of the truncation in the source
* system, so WAL replay would eventually truncate the target
* too, but we might as well do it now.
*
* If it's smaller in the target, it means that it has been
* truncated in the target, or enlarged in the source, or
* both. If it was truncated in the target, we need to copy
* the missing tail from the source system. If it was enlarged
* in the source system, there will be WAL records in the
* source system for the new blocks, so we wouldn't need to
* copy them here. But we don't know which scenario we're
* dealing with, and there's no harm in copying the missing
* blocks now, so do it now.
*
* If it's the same size, do nothing here. Any blocks modified
* in the target will be copied based on parsing the target
* system's WAL, and any blocks modified in the source will be
* updated after rewinding, when the source system's WAL is
* replayed.
*/
oldsize = statbuf.st_size;
if (oldsize < newsize)
action = FILE_ACTION_COPY_TAIL;
else if (oldsize > newsize)
action = FILE_ACTION_TRUNCATE;
else
action = FILE_ACTION_NONE;
}
break;
}
/* Create a new entry for this file */
entry = pg_malloc(sizeof(file_entry_t));
entry->path = pg_strdup(path);
entry->type = type;
entry->action = action;
entry->oldsize = oldsize;
entry->newsize = newsize;
entry->link_target = link_target ? pg_strdup(link_target) : NULL;
entry->next = NULL;
entry->pagemap.bitmap = NULL;
entry->pagemap.bitmapsize = 0;
entry->isrelfile = isRelDataFile(path);
if (map->last)
{
map->last->next = entry;
map->last = entry;
}
else
map->first = map->last = entry;
map->nlist++;
}
static void
scan_directory(const char *basedir, const char *subdir)
{
char path[MAXPGPATH];
DIR *dir;
struct dirent *de;
snprintf(path, sizeof(path), "%s/%s", basedir, subdir);
dir = opendir(path);
if (!dir)
{
fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
while ((de = readdir(dir)) != NULL)
{
char fn[MAXPGPATH];
struct stat st;
if (strcmp(de->d_name, ".") == 0 ||
strcmp(de->d_name, "..") == 0)
continue;
/* Skip temporary files */
if (strncmp(de->d_name,
PG_TEMP_FILE_PREFIX,
strlen(PG_TEMP_FILE_PREFIX)) == 0)
continue;
/* Skip temporary folders */
if (strncmp(de->d_name,
PG_TEMP_FILES_DIR,
strlen(PG_TEMP_FILES_DIR)) == 0)
return;
snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name);
if (lstat(fn, &st) < 0)
{
fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"),
progname, fn, strerror(errno));
exit(1);
}
if (S_ISREG(st.st_mode))
{
char fnonly[MAXPGPATH];
char *forkpath,
*segmentpath;
BlockNumber segmentno = 0;
if (skipfile(de->d_name))
continue;
/*
* Cut off at the segment boundary (".") to get the segment number
* in order to mix it into the checksum. Then also cut off at the
* fork boundary, to get the relfilenode the file belongs to for
* filtering.
*/
strlcpy(fnonly, de->d_name, sizeof(fnonly));
segmentpath = strchr(fnonly, '.');
if (segmentpath != NULL)
{
*segmentpath++ = '\0';
segmentno = atoi(segmentpath);
if (segmentno == 0)
{
fprintf(stderr, _("%s: invalid segment number %d in file name \"%s\"\n"),
progname, segmentno, fn);
exit(1);
}
}
forkpath = strchr(fnonly, '_');
if (forkpath != NULL)
*forkpath++ = '\0';
if (only_relfilenode && strcmp(only_relfilenode, fnonly) != 0)
/* Relfilenode not to be included */
continue;
scan_file(fn, segmentno);
}
#ifndef WIN32
else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
#else
else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
#endif
scan_directory(path, de->d_name);
}
closedir(dir);
}
/*
* recursive part of traverse_datadir
*
* parentpath is the current subdirectory's path relative to datadir,
* or NULL at the top level.
*/
static void
recurse_dir(const char *datadir, const char *parentpath,
process_file_callback_t callback)
{
DIR *xldir;
struct dirent *xlde;
char fullparentpath[MAXPGPATH];
if (parentpath)
snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
else
snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
xldir = opendir(fullparentpath);
if (xldir == NULL)
pg_fatal("could not open directory \"%s\": %s\n",
fullparentpath, strerror(errno));
while (errno = 0, (xlde = readdir(xldir)) != NULL)
{
struct stat fst;
char fullpath[MAXPGPATH * 2];
char path[MAXPGPATH * 2];
if (strcmp(xlde->d_name, ".") == 0 ||
strcmp(xlde->d_name, "..") == 0)
continue;
snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
if (lstat(fullpath, &fst) < 0)
{
if (errno == ENOENT)
{
/*
* File doesn't exist anymore. This is ok, if the new master
* is running and the file was just removed. If it was a data
* file, there should be a WAL record of the removal. If it
* was something else, it couldn't have been anyway.
*
* TODO: But complain if we're processing the target dir!
*/
}
else
pg_fatal("could not stat file \"%s\": %s\n",
fullpath, strerror(errno));
}
if (parentpath)
snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
else
snprintf(path, sizeof(path), "%s", xlde->d_name);
if (S_ISREG(fst.st_mode))
callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
else if (S_ISDIR(fst.st_mode))
{
callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
/* recurse to handle subdirectories */
recurse_dir(datadir, path, callback);
}
#ifndef WIN32
else if (S_ISLNK(fst.st_mode))
#else
else if (pgwin32_is_junction(fullpath))
#endif
{
#if defined(HAVE_READLINK) || defined(WIN32)
char link_target[MAXPGPATH];
int len;
len = readlink(fullpath, link_target, sizeof(link_target));
if (len < 0)
pg_fatal("could not read symbolic link \"%s\": %s\n",
fullpath, strerror(errno));
if (len >= sizeof(link_target))
pg_fatal("symbolic link \"%s\" target is too long\n",
fullpath);
link_target[len] = '\0';
callback(path, FILE_TYPE_SYMLINK, 0, link_target);
/*
* If it's a symlink within pg_tblspc, we need to recurse into it,
* to process all the tablespaces. We also follow a symlink if
* it's for pg_wal. Symlinks elsewhere are ignored.
*/
if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
strcmp(path, "pg_wal") == 0)
recurse_dir(datadir, path, callback);
#else
pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
fullpath);
#endif /* HAVE_READLINK */
}
}
if (errno)
pg_fatal("could not read directory \"%s\": %s\n",
fullparentpath, strerror(errno));
if (closedir(xldir))
pg_fatal("could not close directory \"%s\": %s\n",
fullparentpath, strerror(errno));
}
