/*
* CRC-check an XLOG record. We do not believe the contents of an XLOG
* record (other than to the minimal extent of computing the amount of
* data to read in) until we've checked the CRCs.
*
* We assume all of the record has been read into memory at *record.
*/
bool
recordIsValid(XLogRecord *record, XLogRecPtr *recptr)
{
pg_crc32 crc;
int i;
uint32 len = record->xl_len;
BkpBlock bkpb;
char *blk;
/* First the rmgr data */
INIT_CRC32C(crc);
COMP_CRC32C(crc, XLogRecGetData(record), len);
/* Add in the backup blocks, if any */
blk = (char *) XLogRecGetData(record) + len;
for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
{
uint32 blen;
if (!(record->xl_info & XLR_SET_BKP_BLOCK(i)))
continue;
memcpy(&bkpb, blk, sizeof(BkpBlock));
if (bkpb.hole_offset + bkpb.hole_length > BLCKSZ)
{
ereport(WARNING,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("incorrect hole size in record at %X/%X",
recptr->xlogid, recptr->xrecoff)));
return false;
}
blen = sizeof(BkpBlock) + BLCKSZ - bkpb.hole_length;
COMP_CRC32C(crc, blk, blen);
blk += blen;
}
/* Check that xl_tot_len agrees with our calculation */
if (blk != (char *) record + record->xl_tot_len)
{
ereport(WARNING,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("incorrect total length in record at %X/%X",
recptr->xlogid, recptr->xrecoff)));
return false;
}
/* Finally include the record header */
COMP_CRC32C(crc, (char *) record + sizeof(pg_crc32), SizeOfXLogRecord - sizeof(pg_crc32));
FIN_CRC32C(crc);
if (!EQ_CRC32C(record->xl_crc, crc))
{
ereport(WARNING,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("incorrect resource manager data checksum in record at %X/%X",
recptr->xlogid, recptr->xrecoff)));
return false;
}
return true;
}
/*
* Write into the error log file. This opens the file every time,
* so that we can keep it simple to deal with concurrent write.
*/
static void
ErrorLogWrite(CdbSreh *cdbsreh)
{
HeapTuple tuple;
char filename[MAXPGPATH];
FILE *fp;
pg_crc32 crc;
Assert(OidIsValid(cdbsreh->relid));
ErrorLogFileName(filename, MyDatabaseId, cdbsreh->relid);
tuple = FormErrorTuple(cdbsreh);
INIT_CRC32C(crc);
COMP_CRC32C(crc, tuple->t_data, tuple->t_len);
FIN_CRC32C(crc);
LWLockAcquire(ErrorLogLock, LW_EXCLUSIVE);
fp = AllocateFile(filename, "a");
if (!fp)
{
mkdir(ErrorLogDir, S_IRWXU);
fp = AllocateFile(filename, "a");
}
if (!fp)
ereport(ERROR,
(errmsg("could not open \"%s\": %m", filename)));
/*
* format:
* 0-4: length
* 5-8: crc
* 9-n: tuple data
*/
if (fwrite(&tuple->t_len, 1, sizeof(tuple->t_len), fp) != sizeof(tuple->t_len))
elog(ERROR, "could not write tuple length: %m");
if (fwrite(&crc, 1, sizeof(pg_crc32), fp) != sizeof(pg_crc32))
elog(ERROR, "could not write checksum: %m");
if (fwrite(tuple->t_data, 1, tuple->t_len, fp) != tuple->t_len)
elog(ERROR, "could not write tuple data: %m");
FreeFile(fp);
LWLockRelease(ErrorLogLock);
heap_freetuple(tuple);
}
static pg_crc32
AppendOnlyStorageFormat_ComputeHeaderChecksum(
uint8 *headerPtr,
int32 headerLen)
{
pg_crc32 crc;
Assert(headerPtr != NULL);
/*
* Compute CRC of the header. The header length does not include the
* header checksum.
*/
INIT_CRC32C(crc);
COMP_CRC32C(crc, headerPtr, headerLen);
FIN_CRC32C(crc);
return crc;
}
static pg_crc32
AppendOnlyStorageFormat_ComputeBlockChecksum(
uint8 *headerPtr,
int32 headerLen,
int32 overallBlockLen)
{
int32 dataOffset;
pg_crc32 crc;
Assert(headerPtr != NULL);
/*
* The block checksum covers right after the header checksum through
* the end of the whole block (including the optional firstRowNum).
*/
dataOffset = headerLen + sizeof(pg_crc32);
/* Compute CRC of the header. */
INIT_CRC32C(crc);
COMP_CRC32C(crc, headerPtr + dataOffset, overallBlockLen - dataOffset);
FIN_CRC32C(crc);
return crc;
}
/*
* Write an empty XLOG file, containing only the checkpoint record
* already set up in ControlFile.
*/
static void
WriteEmptyXLOG(void)
{
char *buffer;
XLogPageHeader page;
XLogLongPageHeader longpage;
XLogRecord *record;
pg_crc32 crc;
char path[MAXPGPATH];
int fd;
int nbytes;
FILE *fp = NULL;
char *pch = NULL;
char buf[BUFFER_LEN];
/* Use malloc() to ensure buffer is MAXALIGNED */
buffer = (char *) malloc(XLOG_BLCKSZ);
page = (XLogPageHeader) buffer;
memset(buffer, 0, XLOG_BLCKSZ);
/* Set up the XLOG page header */
page->xlp_magic = XLOG_PAGE_MAGIC;
page->xlp_info = XLP_LONG_HEADER;
page->xlp_tli = ControlFile.checkPointCopy.ThisTimeLineID;
page->xlp_pageaddr.xlogid =
ControlFile.checkPointCopy.redo.xlogid;
page->xlp_pageaddr.xrecoff =
ControlFile.checkPointCopy.redo.xrecoff - SizeOfXLogLongPHD;
longpage = (XLogLongPageHeader) page;
longpage->xlp_sysid = ControlFile.system_identifier;
longpage->xlp_seg_size = XLogSegSize;
longpage->xlp_xlog_blcksz = XLOG_BLCKSZ;
/* Insert the initial checkpoint record */
record = (XLogRecord *) ((char *) page + SizeOfXLogLongPHD);
record->xl_prev.xlogid = 0;
record->xl_prev.xrecoff = 0;
record->xl_xid = InvalidTransactionId;
record->xl_tot_len = SizeOfXLogRecord + sizeof(CheckPoint);
record->xl_len = sizeof(CheckPoint);
record->xl_info = XLOG_CHECKPOINT_SHUTDOWN;
record->xl_rmid = RM_XLOG_ID;
memcpy(XLogRecGetData(record), &ControlFile.checkPointCopy,
sizeof(CheckPoint));
INIT_CRC32C(crc);
COMP_CRC32C(crc, &ControlFile.checkPointCopy, sizeof(CheckPoint));
COMP_CRC32C(crc, (char *) record + sizeof(pg_crc32), SizeOfXLogRecord - sizeof(pg_crc32));
FIN_CRC32C(crc);
/*
INIT_LEGACY_CRC32(crc);
COMP_LEGACY_CRC32(crc, &ControlFile.checkPointCopy, sizeof(CheckPoint));
COMP_LEGACY_CRC32(crc, (char *) record + sizeof(pg_crc32),
SizeOfXLogRecord - sizeof(pg_crc32));
FIN_LEGACY_CRC32(crc);
*/
record->xl_crc = crc;
/*
* If we make the filespace for transaction files configurable, then
* pg_resetxlog should pick up the XLOG files from the right location.
* Check the flat file for determining the right location of XLOG files
*/
fp = fopen(TXN_FILESPACE_FLATFILE, "r");
if (fp)
{
MemSet(buf, 0, BUFFER_LEN);
if (fgets(buf, BUFFER_LEN, fp))
; /* First line is Filespace OID, skip it */
MemSet(buf, 0, BUFFER_LEN);
if (fgets(buf, BUFFER_LEN, fp))
{
buf[strlen(buf)-1]='\0';
pch = strtok(buf, " "); /* The first part is DBID. Skip it */
pch = strtok(NULL, " ");
sprintf(path,"%s/%s", pch, XLOGDIR);
}
}
else
{
/* No flat file. Use the default pg_system filespace */
sprintf(path, "%s", XLOGDIR);
}
/* Write the first page */
XLogFilePath2(path, ControlFile.checkPointCopy.ThisTimeLineID,
newXlogId, newXlogSeg);
unlink(path);
fd = open(path, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
S_IRUSR | S_IWUSR);
if (fd < 0)
{
fprintf(stderr, _("%s: could not open file \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
errno = 0;
if (write(fd, buffer, XLOG_BLCKSZ) != XLOG_BLCKSZ)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
fprintf(stderr, _("%s: could not write file \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
/* Fill the rest of the file with zeroes */
memset(buffer, 0, XLOG_BLCKSZ);
for (nbytes = XLOG_BLCKSZ; nbytes < XLogSegSize; nbytes += XLOG_BLCKSZ)
{
errno = 0;
if (write(fd, buffer, XLOG_BLCKSZ) != XLOG_BLCKSZ)
{
if (errno == 0)
errno = ENOSPC;
fprintf(stderr, _("%s: could not write file \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
}
if (fsync(fd) != 0)
{
fprintf(stderr, _("%s: fsync error: %s\n"), progname, strerror(errno));
exit(1);
}
close(fd);
}
/*
* Write out the new pg_control file.
*/
static void
RewriteControlFile(void)
{
int fd;
char buffer[PG_CONTROL_SIZE]; /* need not be aligned */
/*
* Adjust fields as needed to force an empty XLOG starting at the next
* available segment.
*/
newXlogId = ControlFile.logId;
newXlogSeg = ControlFile.logSeg;
/* adjust in case we are changing segment size */
newXlogSeg *= ControlFile.xlog_seg_size;
newXlogSeg = (newXlogSeg + XLogSegSize - 1) / XLogSegSize;
/* be sure we wrap around correctly at end of a logfile */
NextLogSeg(newXlogId, newXlogSeg);
/* Now we can force the recorded xlog seg size to the right thing. */
ControlFile.xlog_seg_size = XLogSegSize;
ControlFile.checkPointCopy.redo.xlogid = newXlogId;
ControlFile.checkPointCopy.redo.xrecoff =
newXlogSeg * XLogSegSize + SizeOfXLogLongPHD;
ControlFile.checkPointCopy.time = (pg_time_t) time(NULL);
ControlFile.state = DB_SHUTDOWNED;
ControlFile.time = (pg_time_t) time(NULL);
ControlFile.checkPoint = ControlFile.checkPointCopy.redo;
ControlFile.prevCheckPoint.xlogid = 0;
ControlFile.prevCheckPoint.xrecoff = 0;
ControlFile.minRecoveryPoint.xlogid = 0;
ControlFile.minRecoveryPoint.xrecoff = 0;
/* Contents are protected with a CRC */
INIT_CRC32C(ControlFile.crc);
COMP_CRC32C(ControlFile.crc, &ControlFile, offsetof(ControlFileData, crc));
FIN_CRC32C(ControlFile.crc);
/*
INIT_LEGACY_CRC32(ControlFile.crc);
COMP_LEGACY_CRC32(ControlFile.crc,
(char *) &ControlFile,
offsetof(ControlFileData, crc));
FIN_LEGACY_CRC32(ControlFile.crc);
*/
/*
* We write out PG_CONTROL_SIZE bytes into pg_control, zero-padding the
* excess over sizeof(ControlFileData). This reduces the odds of
* premature-EOF errors when reading pg_control. We'll still fail when we
* check the contents of the file, but hopefully with a more specific
* error than "couldn't read pg_control".
*/
if (sizeof(ControlFileData) > PG_CONTROL_SIZE)
{
fprintf(stderr,
_("%s: internal error -- sizeof(ControlFileData) is too large ... fix PG_CONTROL_SIZE\n"),
progname);
exit(1);
}
memset(buffer, 0, PG_CONTROL_SIZE);
memcpy(buffer, &ControlFile, sizeof(ControlFileData));
unlink(XLOG_CONTROL_FILE);
fd = open(XLOG_CONTROL_FILE,
O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
S_IRUSR | S_IWUSR);
if (fd < 0)
{
fprintf(stderr, _("%s: could not create pg_control file: %s\n"),
progname, strerror(errno));
exit(1);
}
errno = 0;
if (write(fd, buffer, PG_CONTROL_SIZE) != PG_CONTROL_SIZE)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
fprintf(stderr, _("%s: could not write pg_control file: %s\n"),
progname, strerror(errno));
exit(1);
}
if (fsync(fd) != 0)
{
fprintf(stderr, _("%s: fsync error: %s\n"), progname, strerror(errno));
exit(1);
}
close(fd);
}
/*
* Try to read the existing pg_control file.
*
* This routine is also responsible for updating old pg_control versions
* to the current format. (Currently we don't do anything of the sort.)
*/
static bool
ReadControlFile(void)
{
int fd;
int len;
char *buffer;
pg_crc32 crc;
if ((fd = open(XLOG_CONTROL_FILE, O_RDONLY | PG_BINARY, 0)) < 0)
{
/*
* If pg_control is not there at all, or we can't read it, the odds
* are we've been handed a bad DataDir path, so give up. User can do
* "touch pg_control" to force us to proceed.
*/
fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"),
progname, XLOG_CONTROL_FILE, strerror(errno));
if (errno == ENOENT)
fprintf(stderr, _("If you are sure the data directory path is correct, execute\n"
" touch %s\n"
"and try again.\n"),
XLOG_CONTROL_FILE);
exit(1);
}
/* Use malloc to ensure we have a maxaligned buffer */
buffer = (char *) malloc(PG_CONTROL_SIZE);
len = read(fd, buffer, PG_CONTROL_SIZE);
if (len < 0)
{
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"),
progname, XLOG_CONTROL_FILE, strerror(errno));
exit(1);
}
close(fd);
if (len >= sizeof(ControlFileData) &&
((ControlFileData *) buffer)->pg_control_version == PG_CONTROL_VERSION)
{
/* Check the CRC. */
INIT_CRC32C(crc);
COMP_CRC32C(crc, buffer, offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
if (EQ_LEGACY_CRC32(crc, ((ControlFileData *) buffer)->crc))
{
/* Valid data... */
memcpy(&ControlFile, buffer, sizeof(ControlFile));
return true;
}
/* Check the CRC using old algorithm. */
INIT_LEGACY_CRC32(crc);
COMP_LEGACY_CRC32(crc,
buffer,
offsetof(ControlFileData, crc));
FIN_LEGACY_CRC32(crc);
if (EQ_LEGACY_CRC32(crc, ((ControlFileData *) buffer)->crc))
{
/* Valid data... */
memcpy(&ControlFile, buffer, sizeof(ControlFile));
return true;
}
fprintf(stderr, _("%s: pg_control exists but has invalid CRC; proceed with caution\n"),
progname);
/* We will use the data anyway, but treat it as guessed. */
memcpy(&ControlFile, buffer, sizeof(ControlFile));
guessed = true;
return true;
}
/* Looks like it's a mess. */
fprintf(stderr, _("%s: pg_control exists but is broken or unknown version; ignoring it\n"),
progname);
return false;
}
/*
* gp_read_error_log
*
* Returns set of error log tuples.
*/
Datum
gp_read_error_log(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
ReadErrorLogContext *context;
HeapTuple tuple;
Datum result;
/*
* First call setup
*/
if (SRF_IS_FIRSTCALL())
{
MemoryContext oldcontext;
FILE *fp;
text *relname;
funcctx = SRF_FIRSTCALL_INIT();
relname = PG_GETARG_TEXT_P(0);
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
context = palloc0(sizeof(ReadErrorLogContext));
funcctx->user_fctx = (void *) context;
funcctx->tuple_desc = BlessTupleDesc(GetErrorTupleDesc());
/*
* Though this function is usually executed on segment, we dispatch
* the execution if it happens to be on QD, and combine the results
* into one set.
*/
if (Gp_role == GP_ROLE_DISPATCH)
{
struct CdbPgResults cdb_pgresults = {NULL, 0};
StringInfoData sql;
int i;
initStringInfo(&sql);
/*
* construct SQL
*/
appendStringInfo(&sql,
"SELECT * FROM pg_catalog.gp_read_error_log(%s) ",
quote_literal_internal(text_to_cstring(relname)));
CdbDispatchCommand(sql.data, DF_WITH_SNAPSHOT, &cdb_pgresults);
for (i = 0; i < cdb_pgresults.numResults; i++)
{
if (PQresultStatus(cdb_pgresults.pg_results[i]) != PGRES_TUPLES_OK)
{
cdbdisp_clearCdbPgResults(&cdb_pgresults);
elog(ERROR, "unexpected result from segment: %d",
PQresultStatus(cdb_pgresults.pg_results[i]));
}
context->numTuples += PQntuples(cdb_pgresults.pg_results[i]);
}
pfree(sql.data);
context->segResults = cdb_pgresults.pg_results;
context->numSegResults = cdb_pgresults.numResults;
}
else
{
/*
* In QE, read the error log.
*/
RangeVar *relrv;
Oid relid;
relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname));
relid = RangeVarGetRelid(relrv, true);
/*
* If the relation has gone, silently return no tuples.
*/
if (OidIsValid(relid))
{
AclResult aclresult;
/*
* Requires SELECT priv to read error log.
*/
aclresult = pg_class_aclcheck(relid, GetUserId(), ACL_SELECT);
if (aclresult != ACLCHECK_OK)
aclcheck_error(aclresult, ACL_KIND_CLASS, relrv->relname);
ErrorLogFileName(context->filename, MyDatabaseId, relid);
fp = AllocateFile(context->filename, "r");
context->fp = fp;
}
}
MemoryContextSwitchTo(oldcontext);
if (Gp_role != GP_ROLE_DISPATCH && !context->fp)
{
pfree(context);
SRF_RETURN_DONE(funcctx);
}
}
funcctx = SRF_PERCALL_SETUP();
context = (ReadErrorLogContext *) funcctx->user_fctx;
/*
* Read error log, probably on segments. We don't check Gp_role, however,
* in case master also wants to read the file.
*/
if (context->fp)
{
pg_crc32 crc, written_crc;
tuple = ErrorLogRead(context->fp, &written_crc);
/*
* CRC check.
*/
if (HeapTupleIsValid(tuple))
{
INIT_CRC32C(crc);
COMP_CRC32C(crc, tuple->t_data, tuple->t_len);
FIN_CRC32C(crc);
if (!EQ_CRC32C(crc, written_crc))
{
elog(LOG, "incorrect checksum in error log %s",
context->filename);
tuple = NULL;
}
}
/*
* If we found a valid tuple, return it. Otherwise, fall through
* in the DONE routine.
*/
if (HeapTupleIsValid(tuple))
{
/*
* We need to set typmod for the executor to understand
* its type we just blessed.
*/
HeapTupleHeaderSetTypMod(tuple->t_data,
funcctx->tuple_desc->tdtypmod);
result = HeapTupleGetDatum(tuple);
SRF_RETURN_NEXT(funcctx, result);
}
}
/*
* If we got results from dispatch, return all the tuples.
*/
while (context->currentResult < context->numSegResults)
{
Datum values[NUM_ERRORTABLE_ATTR];
bool isnull[NUM_ERRORTABLE_ATTR];
PGresult *segres = context->segResults[context->currentResult];
int row = context->currentRow;
if (row >= PQntuples(segres))
{
context->currentRow = 0;
context->currentResult++;
continue;
}
context->currentRow++;
MemSet(isnull, false, sizeof(isnull));
values[0] = ResultToDatum(segres, row, 0, timestamptz_in, &isnull[0]);
values[1] = ResultToDatum(segres, row, 1, textin, &isnull[1]);
values[2] = ResultToDatum(segres, row, 2, textin, &isnull[2]);
values[3] = ResultToDatum(segres, row, 3, int4in, &isnull[3]);
values[4] = ResultToDatum(segres, row, 4, int4in, &isnull[4]);
values[5] = ResultToDatum(segres, row, 5, textin, &isnull[5]);
values[6] = ResultToDatum(segres, row, 6, textin, &isnull[6]);
values[7] = ResultToDatum(segres, row, 7, byteain, &isnull[7]);
tuple = heap_form_tuple(funcctx->tuple_desc, values, isnull);
result = HeapTupleGetDatum(tuple);
SRF_RETURN_NEXT(funcctx, result);
}
if (context->segResults != NULL)
{
int i;
for (i = 0; i < context->numSegResults; i++)
PQclear(context->segResults[i]);
/* XXX: better to copy to palloc'ed area */
free(context->segResults);
}
/*
* Close the file, if we have opened it.
*/
if (context->fp != NULL)
{
FreeFile(context->fp);
context->fp = NULL;
}
SRF_RETURN_DONE(funcctx);
}
/*
* CRC-check an XLOG record. We do not believe the contents of an XLOG
* record (other than to the minimal extent of computing the amount of
* data to read in) until we've checked the CRCs.
*
* We assume all of the record has been read into memory at *record.
*/
static bool
RecordIsValid(XLogRecord *record, XLogRecPtr recptr)
{
pg_crc32 crc;
int i;
uint32 len = record->xl_len;
BkpBlock bkpb;
char *blk;
/* First the rmgr data */
INIT_CRC32C(crc);
COMP_CRC32C(crc, XLogRecGetData(record), len);
/* Add in the backup blocks, if any */
blk = (char *) XLogRecGetData(record) + len;
for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++)
{
uint32 blen;
if (!(record->xl_info & XLR_SET_BKP_BLOCK(i)))
continue;
memcpy(&bkpb, blk, sizeof(BkpBlock));
if (bkpb.hole_offset + bkpb.hole_length > BLCKSZ)
{
printf("incorrect hole size in record at %X/%X\n",
recptr.xlogid, recptr.xrecoff);
return false;
}
blen = sizeof(BkpBlock) + BLCKSZ - bkpb.hole_length;
COMP_CRC32C(crc, blk, blen);
blk += blen;
}
/* skip total xl_tot_len check if physical log has been removed. */
#if PG_VERSION_NUM < 80300 || PG_VERSION_NUM >= 90200
if (record->xl_info & XLR_BKP_BLOCK_MASK)
#else
if (!(record->xl_info & XLR_BKP_REMOVABLE) ||
record->xl_info & XLR_BKP_BLOCK_MASK)
#endif
{
/* Check that xl_tot_len agrees with our calculation */
if (blk != (char *) record + record->xl_tot_len)
{
printf("incorrect total length in record at %X/%X\n",
recptr.xlogid, recptr.xrecoff);
return false;
}
}
/* Finally include the record header */
COMP_CRC32C(crc, (char *) record + sizeof(pg_crc32),
SizeOfXLogRecord - sizeof(pg_crc32));
FIN_CRC32C(crc);
if (!EQ_CRC32C(record->xl_crc, crc))
{
printf("incorrect resource manager data checksum in record at %X/%X\n",
recptr.xlogid, recptr.xrecoff);
return false;
}
return true;
}
bool
copy_file(const char *from_root, const char *to_root, pgFile *file)
{
char to_path[MAXPGPATH];
FILE *in;
FILE *out;
size_t read_len = 0;
int errno_tmp;
char buf[8192];
struct stat st;
pg_crc32 crc;
INIT_CRC32C(crc);
/* reset size summary */
file->read_size = 0;
file->write_size = 0;
/* open backup mode file for read */
in = fopen(file->path, "r");
if (in == NULL)
{
FIN_CRC32C(crc);
file->crc = crc;
/* maybe deleted, it's not error */
if (errno == ENOENT)
return false;
elog(ERROR, "cannot open source file \"%s\": %s", file->path,
strerror(errno));
}
/* open backup file for write */
if (check)
snprintf(to_path, lengthof(to_path), "%s/tmp", backup_path);
else
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
out = fopen(to_path, "w");
if (out == NULL)
{
int errno_tmp = errno;
fclose(in);
elog(ERROR, "cannot open destination file \"%s\": %s",
to_path, strerror(errno_tmp));
}
/* stat source file to change mode of destination file */
if (fstat(fileno(in), &st) == -1)
{
fclose(in);
fclose(out);
elog(ERROR, "cannot stat \"%s\": %s", file->path,
strerror(errno));
}
/* copy content and calc CRC */
for (;;)
{
if ((read_len = fread(buf, 1, sizeof(buf), in)) != sizeof(buf))
break;
if (fwrite(buf, 1, read_len, out) != read_len)
{
errno_tmp = errno;
/* oops */
fclose(in);
fclose(out);
elog(ERROR, "cannot write to \"%s\": %s", to_path,
strerror(errno_tmp));
}
/* update CRC */
COMP_CRC32C(crc, buf, read_len);
file->write_size += sizeof(buf);
file->read_size += sizeof(buf);
}
errno_tmp = errno;
if (!feof(in))
{
fclose(in);
fclose(out);
elog(ERROR, "cannot read backup mode file \"%s\": %s",
file->path, strerror(errno_tmp));
}
/* copy odd part. */
if (read_len > 0)
{
if (fwrite(buf, 1, read_len, out) != read_len)
{
errno_tmp = errno;
/* oops */
fclose(in);
fclose(out);
elog(ERROR, "cannot write to \"%s\": %s", to_path,
strerror(errno_tmp));
}
/* update CRC */
COMP_CRC32C(crc, buf, read_len);
file->write_size += read_len;
file->read_size += read_len;
}
/* finish CRC calculation and store into pgFile */
FIN_CRC32C(crc);
file->crc = crc;
/* update file permission */
if (chmod(to_path, st.st_mode) == -1)
{
errno_tmp = errno;
fclose(in);
fclose(out);
elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
strerror(errno_tmp));
}
fclose(in);
fclose(out);
if (check)
remove(to_path);
return true;
}
/*
* Load a single slot from disk into memory.
*/
static void
RestoreSlotFromDisk(const char *name)
{
ReplicationSlotOnDisk cp;
int i;
char slotdir[MAXPGPATH + 12];
char path[MAXPGPATH + 22];
int fd;
bool restored = false;
int readBytes;
pg_crc32 checksum;
/* no need to lock here, no concurrent access allowed yet */
/* delete temp file if it exists */
sprintf(slotdir, "pg_replslot/%s", name);
sprintf(path, "%s/state.tmp", slotdir);
if (unlink(path) < 0 && errno != ENOENT)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not remove file \"%s\": %m", path)));
sprintf(path, "%s/state", slotdir);
elog(DEBUG1, "restoring replication slot from \"%s\"", path);
fd = OpenTransientFile(path, O_RDWR | PG_BINARY, 0);
/*
* We do not need to handle this as we are rename()ing the directory into
* place only after we fsync()ed the state file.
*/
if (fd < 0)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not open file \"%s\": %m", path)));
/*
* Sync state file before we're reading from it. We might have crashed
* while it wasn't synced yet and we shouldn't continue on that basis.
*/
if (pg_fsync(fd) != 0)
{
int save_errno = errno;
CloseTransientFile(fd);
errno = save_errno;
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not fsync file \"%s\": %m",
path)));
}
/* Also sync the parent directory */
START_CRIT_SECTION();
fsync_fname(slotdir, true);
END_CRIT_SECTION();
/* read part of statefile that's guaranteed to be version independent */
readBytes = read(fd, &cp, ReplicationSlotOnDiskConstantSize);
if (readBytes != ReplicationSlotOnDiskConstantSize)
{
int saved_errno = errno;
CloseTransientFile(fd);
errno = saved_errno;
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not read file \"%s\", read %d of %u: %m",
path, readBytes,
(uint32) ReplicationSlotOnDiskConstantSize)));
}
/* verify magic */
if (cp.magic != SLOT_MAGIC)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("replication slot file \"%s\" has wrong magic %u instead of %u",
path, cp.magic, SLOT_MAGIC)));
/* verify version */
if (cp.version != SLOT_VERSION)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("replication slot file \"%s\" has unsupported version %u",
path, cp.version)));
/* boundary check on length */
if (cp.length != ReplicationSlotOnDiskV2Size)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("replication slot file \"%s\" has corrupted length %u",
path, cp.length)));
/* Now that we know the size, read the entire file */
readBytes = read(fd,
(char *) &cp + ReplicationSlotOnDiskConstantSize,
cp.length);
if (readBytes != cp.length)
{
int saved_errno = errno;
CloseTransientFile(fd);
errno = saved_errno;
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not read file \"%s\", read %d of %u: %m",
path, readBytes, cp.length)));
}
CloseTransientFile(fd);
/* now verify the CRC */
INIT_CRC32C(checksum);
COMP_CRC32C(checksum,
(char *) &cp + SnapBuildOnDiskNotChecksummedSize,
SnapBuildOnDiskChecksummedSize);
FIN_CRC32C(checksum);
if (!EQ_CRC32C(checksum, cp.checksum))
ereport(PANIC,
(errmsg("replication slot file %s: checksum mismatch, is %u, should be %u",
path, checksum, cp.checksum)));
/*
* If we crashed with an ephemeral slot active, don't restore but delete
* it.
*/
if (cp.slotdata.persistency != RS_PERSISTENT)
{
if (!rmtree(slotdir, true))
{
ereport(WARNING,
(errcode_for_file_access(),
errmsg("could not remove directory \"%s\"", slotdir)));
}
fsync_fname("pg_replslot", true);
return;
}
/*
* Verify that requirements for the specific slot type are met. That's
* important because if these aren't met we're not guaranteed to retain
* all the necessary resources for the slot.
*
* NB: We have to do so *after* the above checks for ephemeral slots,
* because otherwise a slot that shouldn't exist anymore could prevent
* restarts.
*
* NB: Changing the requirements here also requires adapting
* CheckSlotRequirements() and CheckLogicalDecodingRequirements().
*/
if (cp.slotdata.database != InvalidOid && wal_level < WAL_LEVEL_LOGICAL)
ereport(FATAL,
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("logical replication slot \"%s\" exists, but wal_level < logical",
NameStr(cp.slotdata.name)),
errhint("Change wal_level to be logical or higher.")));
else if (wal_level < WAL_LEVEL_ARCHIVE)
ereport(FATAL,
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("physical replication slot \"%s\" exists, but wal_level < archive",
NameStr(cp.slotdata.name)),
errhint("Change wal_level to be archive or higher.")));
/* nothing can be active yet, don't lock anything */
for (i = 0; i < max_replication_slots; i++)
{
ReplicationSlot *slot;
slot = &ReplicationSlotCtl->replication_slots[i];
if (slot->in_use)
continue;
/* restore the entire set of persistent data */
memcpy(&slot->data, &cp.slotdata,
sizeof(ReplicationSlotPersistentData));
/* initialize in memory state */
slot->effective_xmin = cp.slotdata.xmin;
slot->effective_catalog_xmin = cp.slotdata.catalog_xmin;
slot->candidate_catalog_xmin = InvalidTransactionId;
slot->candidate_xmin_lsn = InvalidXLogRecPtr;
slot->candidate_restart_lsn = InvalidXLogRecPtr;
slot->candidate_restart_valid = InvalidXLogRecPtr;
slot->in_use = true;
slot->active = false;
restored = true;
break;
}
if (!restored)
ereport(PANIC,
(errmsg("too many replication slots active before shutdown"),
errhint("Increase max_replication_slots and try again.")));
}
/*
* Write out the new pg_control file.
*/
static void
RewriteControlFile(void)
{
int fd;
char buffer[PG_CONTROL_SIZE]; /* need not be aligned */
/*
* Adjust fields as needed to force an empty XLOG starting at
* newXlogSegNo.
*/
XLogSegNoOffsetToRecPtr(newXlogSegNo, SizeOfXLogLongPHD,
ControlFile.checkPointCopy.redo);
ControlFile.checkPointCopy.time = (pg_time_t) time(NULL);
ControlFile.state = DB_SHUTDOWNED;
ControlFile.time = (pg_time_t) time(NULL);
ControlFile.checkPoint = ControlFile.checkPointCopy.redo;
ControlFile.prevCheckPoint = 0;
ControlFile.minRecoveryPoint = 0;
ControlFile.minRecoveryPointTLI = 0;
ControlFile.backupStartPoint = 0;
ControlFile.backupEndPoint = 0;
ControlFile.backupEndRequired = false;
/*
* Force the defaults for max_* settings. The values don't really matter
* as long as wal_level='minimal'; the postmaster will reset these fields
* anyway at startup.
*/
ControlFile.wal_level = WAL_LEVEL_MINIMAL;
ControlFile.wal_log_hints = false;
ControlFile.track_commit_timestamp = false;
ControlFile.MaxConnections = 100;
ControlFile.max_worker_processes = 8;
ControlFile.max_prepared_xacts = 0;
ControlFile.max_locks_per_xact = 64;
/* Now we can force the recorded xlog seg size to the right thing. */
ControlFile.xlog_seg_size = XLogSegSize;
/* Contents are protected with a CRC */
INIT_CRC32C(ControlFile.crc);
COMP_CRC32C(ControlFile.crc,
(char *) &ControlFile,
offsetof(ControlFileData, crc));
FIN_CRC32C(ControlFile.crc);
/*
* We write out PG_CONTROL_SIZE bytes into pg_control, zero-padding the
* excess over sizeof(ControlFileData). This reduces the odds of
* premature-EOF errors when reading pg_control. We'll still fail when we
* check the contents of the file, but hopefully with a more specific
* error than "couldn't read pg_control".
*/
if (sizeof(ControlFileData) > PG_CONTROL_SIZE)
{
fprintf(stderr,
_("%s: internal error -- sizeof(ControlFileData) is too large ... fix PG_CONTROL_SIZE\n"),
progname);
exit(1);
}
memset(buffer, 0, PG_CONTROL_SIZE);
memcpy(buffer, &ControlFile, sizeof(ControlFileData));
unlink(XLOG_CONTROL_FILE);
fd = open(XLOG_CONTROL_FILE,
O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
S_IRUSR | S_IWUSR);
if (fd < 0)
{
fprintf(stderr, _("%s: could not create pg_control file: %s\n"),
progname, strerror(errno));
exit(1);
}
errno = 0;
if (write(fd, buffer, PG_CONTROL_SIZE) != PG_CONTROL_SIZE)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
fprintf(stderr, _("%s: could not write pg_control file: %s\n"),
progname, strerror(errno));
exit(1);
}
if (fsync(fd) != 0)
{
fprintf(stderr, _("%s: fsync error: %s\n"), progname, strerror(errno));
exit(1);
}
close(fd);
}
/*
* Write an empty XLOG file, containing only the checkpoint record
* already set up in ControlFile.
*/
static void
WriteEmptyXLOG(void)
{
char *buffer;
XLogPageHeader page;
XLogLongPageHeader longpage;
XLogRecord *record;
pg_crc32c crc;
char path[MAXPGPATH];
int fd;
int nbytes;
char *recptr;
/* Use malloc() to ensure buffer is MAXALIGNED */
buffer = (char *) pg_malloc(XLOG_BLCKSZ);
page = (XLogPageHeader) buffer;
memset(buffer, 0, XLOG_BLCKSZ);
/* Set up the XLOG page header */
page->xlp_magic = XLOG_PAGE_MAGIC;
page->xlp_info = XLP_LONG_HEADER;
page->xlp_tli = ControlFile.checkPointCopy.ThisTimeLineID;
page->xlp_pageaddr = ControlFile.checkPointCopy.redo - SizeOfXLogLongPHD;
longpage = (XLogLongPageHeader) page;
longpage->xlp_sysid = ControlFile.system_identifier;
longpage->xlp_seg_size = XLogSegSize;
longpage->xlp_xlog_blcksz = XLOG_BLCKSZ;
/* Insert the initial checkpoint record */
recptr = (char *) page + SizeOfXLogLongPHD;
record = (XLogRecord *) recptr;
record->xl_prev = 0;
record->xl_xid = InvalidTransactionId;
record->xl_tot_len = SizeOfXLogRecord + SizeOfXLogRecordDataHeaderShort + sizeof(CheckPoint);
record->xl_info = XLOG_CHECKPOINT_SHUTDOWN;
record->xl_rmid = RM_XLOG_ID;
recptr += SizeOfXLogRecord;
*(recptr++) = XLR_BLOCK_ID_DATA_SHORT;
*(recptr++) = sizeof(CheckPoint);
memcpy(recptr, &ControlFile.checkPointCopy,
sizeof(CheckPoint));
INIT_CRC32C(crc);
COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord);
COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc));
FIN_CRC32C(crc);
record->xl_crc = crc;
/* Write the first page */
XLogFilePath(path, ControlFile.checkPointCopy.ThisTimeLineID, newXlogSegNo);
unlink(path);
fd = open(path, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
S_IRUSR | S_IWUSR);
if (fd < 0)
{
fprintf(stderr, _("%s: could not open file \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
errno = 0;
if (write(fd, buffer, XLOG_BLCKSZ) != XLOG_BLCKSZ)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
fprintf(stderr, _("%s: could not write file \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
/* Fill the rest of the file with zeroes */
memset(buffer, 0, XLOG_BLCKSZ);
for (nbytes = XLOG_BLCKSZ; nbytes < XLogSegSize; nbytes += XLOG_BLCKSZ)
{
errno = 0;
if (write(fd, buffer, XLOG_BLCKSZ) != XLOG_BLCKSZ)
{
if (errno == 0)
errno = ENOSPC;
fprintf(stderr, _("%s: could not write file \"%s\": %s\n"),
progname, path, strerror(errno));
exit(1);
}
}
if (fsync(fd) != 0)
{
fprintf(stderr, _("%s: fsync error: %s\n"), progname, strerror(errno));
exit(1);
}
close(fd);
}
int
main(int argc, char *argv[])
{
ControlFileData ControlFile;
int fd;
char ControlFilePath[MAXPGPATH];
char *DataDir;
pg_crc32 crc;
const char *progname;
int argno = 1;
int targetversion = -1;
uint32 tover;
set_pglocale_pgservice(argv[0], "gpmodcatversion");
progname = get_progname(argv[0]);
if (argc > 1)
{
argno = 1;
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
usage(progname);
exit(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
puts("gpmodcatversion (Greenplum Database) " PG_VERSION);
exit(0);
}
if (strcmp(argv[1], "--catversion") == 0)
{
if (argc < 2)
{
/* we didn't get the version number */
printf(_("\nA catalog version must be specified.\n\n"));
usage(progname);
exit(1);
}
else
{
char found = 0;
for (targetversion = 0;
versions[targetversion].gpversion;
targetversion++)
{
if (strcmp(argv[2], versions[targetversion].gpversion) == 0)
{
found = 1;
break;
}
}
if (!found)
{
printf(_("\n\"%s\" is not a valid version to "
"upgrade/downgrade to.\n\n"), argv[2]);
usage(progname);
exit(1);
}
argno = 3;
}
}
}
if (argc > argno)
DataDir = argv[argno];
else
DataDir = getenv("PGDATA");
if (DataDir == NULL)
{
fprintf(stderr, _("%s: no data directory specified\n"), progname);
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
usage(progname);
exit(1);
}
snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir);
#define FLAGS (O_RDWR | O_EXCL | PG_BINARY)
if ((fd = open(ControlFilePath, FLAGS, S_IRUSR | S_IWUSR)) == -1)
{
fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"),
progname, ControlFilePath, strerror(errno));
exit(2);
}
if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData))
{
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"),
progname, ControlFilePath, strerror(errno));
exit(2);
}
/* Check the CRC. */
INIT_CRC32C(crc);
COMP_CRC32C(crc, &ControlFile, offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
if (!EQ_LEGACY_CRC32(crc, ControlFile.crc))
{
/* Check the CRC using old algorithm. */
INIT_LEGACY_CRC32(crc);
COMP_LEGACY_CRC32(crc,
(char *) &ControlFile,
offsetof(ControlFileData, crc));
FIN_LEGACY_CRC32(crc);
if (!EQ_LEGACY_CRC32(crc, ControlFile.crc))
printf(_("WARNING: Calculated CRC checksum does not match value stored in file.\n"
"Either the file is corrupt, or it has a different layout than this program\n"
"is expecting. The results below are untrustworthy.\n\n"));
}
printf(_("\nCatalog version number: %u\n"),
ControlFile.catalog_version_no);
tover = versions[targetversion].vernum;
if (tover)
{
if (ControlFile.state != DB_SHUTDOWNED)
{
/* only upgrade shutdown systems */
printf(_("\n\nFATAL ERROR\n\n"
"Invalid database state for upgrade: %s\n"
"Action: shutdown the database and try again.\n"),
dbState(ControlFile.state));
close(fd);
exit(1);
}
else if (ControlFile.catalog_version_no == tover)
{
/* must be something wrong if we want to do this */
printf(_("\n\nWarning\n\n"
"Catalog version number is already set to %u\n\n"),
tover);
close(fd);
return 0;
}
printf(_("New catalog version number: %u\n"), tover);
ControlFile.catalog_version_no = tover;
/* recalcualte the CRC. */
INIT_CRC32C(crc);
COMP_CRC32C(crc, &ControlFile, offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
/*
INIT_LEGACY_CRC32(crc);
COMP_LEGACY_CRC32(crc,
(char *) &ControlFile,
offsetof(ControlFileData, crc));
FIN_LEGACY_CRC32(crc);
*/
ControlFile.crc = crc;
printf(_("Setting version number to: %u\n"), tover);
errno = 0;
if (lseek(fd, 0, SEEK_SET) < 0)
{
printf(_("\n\nFATAL ERROR\n"
"could not seek control file: %s\n"),
strerror(errno));
close(fd);
exit(1);
}
if (write(fd, &ControlFile, sizeof(ControlFile)) != sizeof(ControlFile))
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
printf(_("\n\nFATAL ERROR\n"
"could not write to control file: %s\n"),
strerror(errno));
close(fd);
exit(1);
}
if (close(fd))
{
printf(_("\n\nFATAL ERROR\n"
"could not close control file: %s\n"),
strerror(errno));
exit(1);
}
}
return 0;
}
/*
* Shared functionality between saving and creating a replication slot.
*/
static void
SaveSlotToPath(ReplicationSlot *slot, const char *dir, int elevel)
{
char tmppath[MAXPGPATH];
char path[MAXPGPATH];
int fd;
ReplicationSlotOnDisk cp;
bool was_dirty;
/* first check whether there's something to write out */
SpinLockAcquire(&slot->mutex);
was_dirty = slot->dirty;
slot->just_dirtied = false;
SpinLockRelease(&slot->mutex);
/* and don't do anything if there's nothing to write */
if (!was_dirty)
return;
LWLockAcquire(&slot->io_in_progress_lock, LW_EXCLUSIVE);
/* silence valgrind :( */
memset(&cp, 0, sizeof(ReplicationSlotOnDisk));
sprintf(tmppath, "%s/state.tmp", dir);
sprintf(path, "%s/state", dir);
fd = OpenTransientFile(tmppath, O_CREAT | O_EXCL | O_WRONLY | PG_BINARY);
if (fd < 0)
{
ereport(elevel,
(errcode_for_file_access(),
errmsg("could not create file \"%s\": %m",
tmppath)));
return;
}
cp.magic = SLOT_MAGIC;
INIT_CRC32C(cp.checksum);
cp.version = SLOT_VERSION;
cp.length = ReplicationSlotOnDiskV2Size;
SpinLockAcquire(&slot->mutex);
memcpy(&cp.slotdata, &slot->data, sizeof(ReplicationSlotPersistentData));
SpinLockRelease(&slot->mutex);
COMP_CRC32C(cp.checksum,
(char *) (&cp) + SnapBuildOnDiskNotChecksummedSize,
SnapBuildOnDiskChecksummedSize);
FIN_CRC32C(cp.checksum);
pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_WRITE);
if ((write(fd, &cp, sizeof(cp))) != sizeof(cp))
{
int save_errno = errno;
pgstat_report_wait_end();
CloseTransientFile(fd);
errno = save_errno;
ereport(elevel,
(errcode_for_file_access(),
errmsg("could not write to file \"%s\": %m",
tmppath)));
return;
}
pgstat_report_wait_end();
/* fsync the temporary file */
pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_SYNC);
if (pg_fsync(fd) != 0)
{
int save_errno = errno;
pgstat_report_wait_end();
CloseTransientFile(fd);
errno = save_errno;
ereport(elevel,
(errcode_for_file_access(),
errmsg("could not fsync file \"%s\": %m",
tmppath)));
return;
}
pgstat_report_wait_end();
CloseTransientFile(fd);
/* rename to permanent file, fsync file and directory */
if (rename(tmppath, path) != 0)
{
ereport(elevel,
(errcode_for_file_access(),
errmsg("could not rename file \"%s\" to \"%s\": %m",
tmppath, path)));
return;
}
/* Check CreateSlot() for the reasoning of using a crit. section. */
START_CRIT_SECTION();
fsync_fname(path, false);
fsync_fname(dir, true);
fsync_fname("pg_replslot", true);
END_CRIT_SECTION();
/*
* Successfully wrote, unset dirty bit, unless somebody dirtied again
* already.
*/
SpinLockAcquire(&slot->mutex);
if (!slot->just_dirtied)
slot->dirty = false;
SpinLockRelease(&slot->mutex);
LWLockRelease(&slot->io_in_progress_lock);
}
/*
* load_relmap_file -- load data from the shared or local map file
*
* Because the map file is essential for access to core system catalogs,
* failure to read it is a fatal error.
*
* Note that the local case requires DatabasePath to be set up.
*/
static void
load_relmap_file(bool shared)
{
RelMapFile *map;
char mapfilename[MAXPGPATH];
pg_crc32c crc;
int fd;
if (shared)
{
snprintf(mapfilename, sizeof(mapfilename), "global/%s",
RELMAPPER_FILENAME);
map = &shared_map;
}
else
{
snprintf(mapfilename, sizeof(mapfilename), "%s/%s",
DatabasePath, RELMAPPER_FILENAME);
map = &local_map;
}
/* Read data ... */
fd = OpenTransientFile(mapfilename,
O_RDONLY | PG_BINARY, S_IRUSR | S_IWUSR);
if (fd < 0)
ereport(FATAL,
(errcode_for_file_access(),
errmsg("could not open relation mapping file \"%s\": %m",
mapfilename)));
/*
* Note: we could take RelationMappingLock in shared mode here, but it
* seems unnecessary since our read() should be atomic against any
* concurrent updater's write(). If the file is updated shortly after we
* look, the sinval signaling mechanism will make us re-read it before we
* are able to access any relation that's affected by the change.
*/
if (read(fd, map, sizeof(RelMapFile)) != sizeof(RelMapFile))
ereport(FATAL,
(errcode_for_file_access(),
errmsg("could not read relation mapping file \"%s\": %m",
mapfilename)));
CloseTransientFile(fd);
/* check for correct magic number, etc */
if (map->magic != RELMAPPER_FILEMAGIC ||
map->num_mappings < 0 ||
map->num_mappings > MAX_MAPPINGS)
ereport(FATAL,
(errmsg("relation mapping file \"%s\" contains invalid data",
mapfilename)));
/* verify the CRC */
INIT_CRC32C(crc);
COMP_CRC32C(crc, (char *) map, offsetof(RelMapFile, crc));
FIN_CRC32C(crc);
if (!EQ_CRC32C(crc, map->crc))
ereport(FATAL,
(errmsg("relation mapping file \"%s\" contains incorrect checksum",
mapfilename)));
}
/*
* Load a single slot from disk into memory.
*/
static void
RestoreSlotFromDisk(const char *name)
{
ReplicationSlotOnDisk cp;
int i;
char path[MAXPGPATH + 22];
int fd;
bool restored = false;
int readBytes;
pg_crc32c checksum;
/* no need to lock here, no concurrent access allowed yet */
/* delete temp file if it exists */
sprintf(path, "pg_replslot/%s/state.tmp", name);
if (unlink(path) < 0 && errno != ENOENT)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not remove file \"%s\": %m", path)));
sprintf(path, "pg_replslot/%s/state", name);
elog(DEBUG1, "restoring replication slot from \"%s\"", path);
fd = OpenTransientFile(path, O_RDWR | PG_BINARY);
/*
* We do not need to handle this as we are rename()ing the directory into
* place only after we fsync()ed the state file.
*/
if (fd < 0)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not open file \"%s\": %m", path)));
/*
* Sync state file before we're reading from it. We might have crashed
* while it wasn't synced yet and we shouldn't continue on that basis.
*/
pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_RESTORE_SYNC);
if (pg_fsync(fd) != 0)
{
CloseTransientFile(fd);
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not fsync file \"%s\": %m",
path)));
}
pgstat_report_wait_end();
/* Also sync the parent directory */
START_CRIT_SECTION();
fsync_fname(path, true);
END_CRIT_SECTION();
/* read part of statefile that's guaranteed to be version independent */
pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_READ);
readBytes = read(fd, &cp, ReplicationSlotOnDiskConstantSize);
pgstat_report_wait_end();
if (readBytes != ReplicationSlotOnDiskConstantSize)
{
int saved_errno = errno;
CloseTransientFile(fd);
errno = saved_errno;
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not read file \"%s\", read %d of %u: %m",
path, readBytes,
(uint32) ReplicationSlotOnDiskConstantSize)));
}
/* verify magic */
if (cp.magic != SLOT_MAGIC)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("replication slot file \"%s\" has wrong magic number: %u instead of %u",
path, cp.magic, SLOT_MAGIC)));
/* verify version */
if (cp.version != SLOT_VERSION)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("replication slot file \"%s\" has unsupported version %u",
path, cp.version)));
/* boundary check on length */
if (cp.length != ReplicationSlotOnDiskV2Size)
ereport(PANIC,
(errcode_for_file_access(),
errmsg("replication slot file \"%s\" has corrupted length %u",
path, cp.length)));
/* Now that we know the size, read the entire file */
pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_READ);
readBytes = read(fd,
(char *) &cp + ReplicationSlotOnDiskConstantSize,
cp.length);
pgstat_report_wait_end();
if (readBytes != cp.length)
{
int saved_errno = errno;
CloseTransientFile(fd);
errno = saved_errno;
ereport(PANIC,
(errcode_for_file_access(),
errmsg("could not read file \"%s\", read %d of %u: %m",
path, readBytes, cp.length)));
}
CloseTransientFile(fd);
/* now verify the CRC */
INIT_CRC32C(checksum);
COMP_CRC32C(checksum,
(char *) &cp + SnapBuildOnDiskNotChecksummedSize,
SnapBuildOnDiskChecksummedSize);
FIN_CRC32C(checksum);
if (!EQ_CRC32C(checksum, cp.checksum))
ereport(PANIC,
(errmsg("checksum mismatch for replication slot file \"%s\": is %u, should be %u",
path, checksum, cp.checksum)));
/*
* If we crashed with an ephemeral slot active, don't restore but delete
* it.
*/
if (cp.slotdata.persistency != RS_PERSISTENT)
{
sprintf(path, "pg_replslot/%s", name);
if (!rmtree(path, true))
{
ereport(WARNING,
(errcode_for_file_access(),
errmsg("could not remove directory \"%s\"", path)));
}
fsync_fname("pg_replslot", true);
return;
}
/* nothing can be active yet, don't lock anything */
for (i = 0; i < max_replication_slots; i++)
{
ReplicationSlot *slot;
slot = &ReplicationSlotCtl->replication_slots[i];
if (slot->in_use)
continue;
/* restore the entire set of persistent data */
memcpy(&slot->data, &cp.slotdata,
sizeof(ReplicationSlotPersistentData));
/* initialize in memory state */
slot->effective_xmin = cp.slotdata.xmin;
slot->effective_catalog_xmin = cp.slotdata.catalog_xmin;
slot->candidate_catalog_xmin = InvalidTransactionId;
slot->candidate_xmin_lsn = InvalidXLogRecPtr;
slot->candidate_restart_lsn = InvalidXLogRecPtr;
slot->candidate_restart_valid = InvalidXLogRecPtr;
slot->in_use = true;
slot->active_pid = 0;
restored = true;
break;
}
if (!restored)
ereport(PANIC,
(errmsg("too many replication slots active before shutdown"),
errhint("Increase max_replication_slots and try again.")));
}
/*
* Write out a new shared or local map file with the given contents.
*
* The magic number and CRC are automatically updated in *newmap. On
* success, we copy the data to the appropriate permanent static variable.
*
* If write_wal is TRUE then an appropriate WAL message is emitted.
* (It will be false for bootstrap and WAL replay cases.)
*
* If send_sinval is TRUE then a SI invalidation message is sent.
* (This should be true except in bootstrap case.)
*
* If preserve_files is TRUE then the storage manager is warned not to
* delete the files listed in the map.
*
* Because this may be called during WAL replay when MyDatabaseId,
* DatabasePath, etc aren't valid, we require the caller to pass in suitable
* values. The caller is also responsible for being sure no concurrent
* map update could be happening.
*/
static void
write_relmap_file(bool shared, RelMapFile *newmap,
bool write_wal, bool send_sinval, bool preserve_files,
Oid dbid, Oid tsid, const char *dbpath)
{
int fd;
RelMapFile *realmap;
char mapfilename[MAXPGPATH];
/*
* Fill in the overhead fields and update CRC.
*/
newmap->magic = RELMAPPER_FILEMAGIC;
if (newmap->num_mappings < 0 || newmap->num_mappings > MAX_MAPPINGS)
elog(ERROR, "attempt to write bogus relation mapping");
INIT_CRC32C(newmap->crc);
COMP_CRC32C(newmap->crc, (char *) newmap, offsetof(RelMapFile, crc));
FIN_CRC32C(newmap->crc);
/*
* Open the target file. We prefer to do this before entering the
* critical section, so that an open() failure need not force PANIC.
*/
if (shared)
{
snprintf(mapfilename, sizeof(mapfilename), "global/%s",
RELMAPPER_FILENAME);
realmap = &shared_map;
}
else
{
snprintf(mapfilename, sizeof(mapfilename), "%s/%s",
dbpath, RELMAPPER_FILENAME);
realmap = &local_map;
}
fd = OpenTransientFile(mapfilename,
O_WRONLY | O_CREAT | PG_BINARY,
S_IRUSR | S_IWUSR);
if (fd < 0)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not open relation mapping file \"%s\": %m",
mapfilename)));
if (write_wal)
{
xl_relmap_update xlrec;
XLogRecPtr lsn;
/* now errors are fatal ... */
START_CRIT_SECTION();
xlrec.dbid = dbid;
xlrec.tsid = tsid;
xlrec.nbytes = sizeof(RelMapFile);
XLogBeginInsert();
XLogRegisterData((char *) (&xlrec), MinSizeOfRelmapUpdate);
XLogRegisterData((char *) newmap, sizeof(RelMapFile));
lsn = XLogInsert(RM_RELMAP_ID, XLOG_RELMAP_UPDATE);
/* As always, WAL must hit the disk before the data update does */
XLogFlush(lsn);
}
errno = 0;
if (write(fd, newmap, sizeof(RelMapFile)) != sizeof(RelMapFile))
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not write to relation mapping file \"%s\": %m",
mapfilename)));
}
/*
* We choose to fsync the data to disk before considering the task done.
* It would be possible to relax this if it turns out to be a performance
* issue, but it would complicate checkpointing --- see notes for
* CheckPointRelationMap.
*/
if (pg_fsync(fd) != 0)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not fsync relation mapping file \"%s\": %m",
mapfilename)));
if (CloseTransientFile(fd))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not close relation mapping file \"%s\": %m",
mapfilename)));
/*
* Now that the file is safely on disk, send sinval message to let other
* backends know to re-read it. We must do this inside the critical
* section: if for some reason we fail to send the message, we have to
* force a database-wide PANIC. Otherwise other backends might continue
* execution with stale mapping information, which would be catastrophic
* as soon as others began to use the now-committed data.
*/
if (send_sinval)
CacheInvalidateRelmap(dbid);
/*
* Make sure that the files listed in the map are not deleted if the outer
* transaction aborts. This had better be within the critical section
* too: it's not likely to fail, but if it did, we'd arrive at transaction
* abort with the files still vulnerable. PANICing will leave things in a
* good state on-disk.
*
* Note: we're cheating a little bit here by assuming that mapped files
* are either in pg_global or the database's default tablespace.
*/
if (preserve_files)
{
int32 i;
for (i = 0; i < newmap->num_mappings; i++)
{
RelFileNode rnode;
rnode.spcNode = tsid;
rnode.dbNode = dbid;
rnode.relNode = newmap->mappings[i].mapfilenode;
RelationPreserveStorage(rnode, false);
}
}
/* Success, update permanent copy */
memcpy(realmap, newmap, sizeof(RelMapFile));
/* Critical section done */
if (write_wal)
END_CRIT_SECTION();
}
/*
* Backup data file in the from_root directory to the to_root directory with
* same relative path.
* If lsn is not NULL, pages only which are modified after the lsn will be
* copied.
*/
bool
backup_data_file(const char *from_root, const char *to_root,
pgFile *file, const XLogRecPtr *lsn)
{
char to_path[MAXPGPATH];
FILE *in;
FILE *out;
BackupPageHeader header;
DataPage page; /* used as read buffer */
BlockNumber blknum = 0;
size_t read_len = 0;
pg_crc32 crc;
off_t offset;
INIT_CRC32C(crc);
/* reset size summary */
file->read_size = 0;
file->write_size = 0;
/* open backup mode file for read */
in = fopen(file->path, "r");
if (in == NULL)
{
FIN_CRC32C(crc);
file->crc = crc;
/* maybe vanished, it's not error */
if (errno == ENOENT)
return false;
elog(ERROR, "cannot open backup mode file \"%s\": %s",
file->path, strerror(errno));
}
/* open backup file for write */
if (check)
snprintf(to_path, lengthof(to_path), "%s/tmp", backup_path);
else
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
out = fopen(to_path, "w");
if (out == NULL)
{
int errno_tmp = errno;
fclose(in);
elog(ERROR, "cannot open backup file \"%s\": %s",
to_path, strerror(errno_tmp));
}
/* confirm server version */
check_server_version();
/*
* Read each page and write the page excluding hole. If it has been
* determined that the page can be copied safely, but no page map
* has been built, it means that we are in presence of a relation
* file that needs to be completely scanned. If a page map is present
* only scan the blocks needed. In each case, pages are copied without
* their hole to ensure some basic level of compression.
*/
if (file->pagemap.bitmapsize == 0)
{
for (blknum = 0;
(read_len = fread(&page, 1, sizeof(page), in)) == sizeof(page);
++blknum)
{
XLogRecPtr page_lsn;
int upper_offset;
int upper_length;
header.block = blknum;
/*
* If an invalid data page was found, fallback to simple copy to ensure
* all pages in the file don't have BackupPageHeader.
*/
if (!parse_page(&page, &page_lsn,
&header.hole_offset, &header.hole_length))
{
elog(LOG, "%s fall back to simple copy", file->path);
fclose(in);
fclose(out);
file->is_datafile = false;
return copy_file(from_root, to_root, file);
}
file->read_size += read_len;
/* if the page has not been modified since last backup, skip it */
if (lsn && !XLogRecPtrIsInvalid(page_lsn) && page_lsn < *lsn)
continue;
upper_offset = header.hole_offset + header.hole_length;
upper_length = BLCKSZ - upper_offset;
/* write data page excluding hole */
if (fwrite(&header, 1, sizeof(header), out) != sizeof(header) ||
fwrite(page.data, 1, header.hole_offset, out) != header.hole_offset ||
fwrite(page.data + upper_offset, 1, upper_length, out) != upper_length)
{
int errno_tmp = errno;
/* oops */
fclose(in);
fclose(out);
elog(ERROR, "cannot write at block %u of \"%s\": %s",
blknum, to_path, strerror(errno_tmp));
}
/* update CRC */
COMP_CRC32C(crc, &header, sizeof(header));
COMP_CRC32C(crc, page.data, header.hole_offset);
COMP_CRC32C(crc, page.data + upper_offset, upper_length);
file->write_size += sizeof(header) + read_len - header.hole_length;
}
}
else
{
datapagemap_iterator_t *iter;
iter = datapagemap_iterate(&file->pagemap);
while (datapagemap_next(iter, &blknum))
{
XLogRecPtr page_lsn;
int upper_offset;
int upper_length;
int ret;
offset = blknum * BLCKSZ;
if (offset > 0)
{
ret = fseek(in, offset, SEEK_SET);
if (ret != 0)
elog(ERROR,
"Can't seek in file offset: %llu ret:%i\n",
(long long unsigned int) offset, ret);
}
read_len = fread(&page, 1, sizeof(page), in);
header.block = blknum;
/*
* If an invalid data page was found, fallback to simple copy to ensure
* all pages in the file don't have BackupPageHeader.
*/
if (!parse_page(&page, &page_lsn,
&header.hole_offset, &header.hole_length))
{
elog(LOG, "%s fall back to simple copy", file->path);
fclose(in);
fclose(out);
file->is_datafile = false;
return copy_file(from_root, to_root, file);
}
file->read_size += read_len;
/* if the page has not been modified since last backup, skip it */
if (lsn && !XLogRecPtrIsInvalid(page_lsn) && page_lsn < *lsn)
continue;
upper_offset = header.hole_offset + header.hole_length;
upper_length = BLCKSZ - upper_offset;
/* write data page excluding hole */
if (fwrite(&header, 1, sizeof(header), out) != sizeof(header) ||
fwrite(page.data, 1, header.hole_offset, out) != header.hole_offset ||
fwrite(page.data + upper_offset, 1, upper_length, out) != upper_length)
{
int errno_tmp = errno;
/* oops */
fclose(in);
fclose(out);
elog(ERROR, "cannot write at block %u of \"%s\": %s",
blknum, to_path, strerror(errno_tmp));
}
/* update CRC */
COMP_CRC32C(crc, &header, sizeof(header));
COMP_CRC32C(crc, page.data, header.hole_offset);
COMP_CRC32C(crc, page.data + upper_offset, upper_length);
file->write_size += sizeof(header) + read_len - header.hole_length;
}
pg_free(iter);
}
/*
* update file permission
* FIXME: Should set permission on open?
*/
if (!check && chmod(to_path, FILE_PERMISSION) == -1)
{
int errno_tmp = errno;
fclose(in);
fclose(out);
elog(ERROR, "cannot change mode of \"%s\": %s", file->path,
strerror(errno_tmp));
}
fclose(in);
fclose(out);
/* finish CRC calculation and store into pgFile */
FIN_CRC32C(crc);
file->crc = crc;
/* Treat empty file as not-datafile */
if (file->read_size == 0)
file->is_datafile = false;
/* We do not backup if all pages skipped. */
if (file->write_size == 0 && file->read_size > 0)
{
if (remove(to_path) == -1)
elog(ERROR, "cannot remove file \"%s\": %s", to_path,
strerror(errno));
return false;
}
/* remove $BACKUP_PATH/tmp created during check */
if (check)
remove(to_path);
return true;
}
int
main(int argc, char *argv[])
{
ControlFileData ControlFile;
int fd;
char ControlFilePath[MAXPGPATH];
char *DataDir;
pg_crc32c crc;
char pgctime_str[128];
char ckpttime_str[128];
char sysident_str[32];
const char *strftime_fmt = "%c";
const char *progname;
set_pglocale_pgservice(argv[0], "pg_controldata");
progname = get_progname(argv[0]);
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
{
usage(progname);
exit(0);
}
if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
{
puts("pg_controldata (Greenplum Database) " PG_VERSION);
exit(0);
}
if (strcmp(argv[1], "--gp-version") == 0)
{
puts("pg_controldata (Greenplum Database) " GP_VERSION);
exit(0);
}
}
if (argc > 1)
DataDir = argv[1];
else
DataDir = getenv("PGDATA");
if (DataDir == NULL)
{
fprintf(stderr, _("%s: no data directory specified\n"), progname);
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir);
if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1)
{
fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"),
progname, ControlFilePath, strerror(errno));
exit(2);
}
if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData))
{
fprintf(stderr, _("%s: could not read file \"%s\": %s\n"),
progname, ControlFilePath, strerror(errno));
exit(2);
}
close(fd);
/* Check the CRC. */
INIT_CRC32C(crc);
COMP_CRC32C(crc, &ControlFile, offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
if (!EQ_CRC32C(crc, ControlFile.crc))
printf(_("WARNING: Calculated CRC checksum does not match value stored in file.\n"
"Either the file is corrupt, or it has a different layout than this program\n"
"is expecting. The results below are untrustworthy.\n\n"));
/*
* Use variable for format to suppress overly-anal-retentive gcc warning
* about %c
*/
strftime(pgctime_str, sizeof(pgctime_str), strftime_fmt,
localtime(&(ControlFile.time)));
strftime(ckpttime_str, sizeof(ckpttime_str), strftime_fmt,
localtime(&(ControlFile.checkPointCopy.time)));
/*
* Format system_identifier separately to keep platform-dependent format
* code out of the translatable message string.
*/
snprintf(sysident_str, sizeof(sysident_str), UINT64_FORMAT,
ControlFile.system_identifier);
printf(_("pg_control version number: %u\n"),
ControlFile.pg_control_version);
if (ControlFile.pg_control_version % 65536 == 0 && ControlFile.pg_control_version / 65536 != 0)
printf(_("WARNING: possible byte ordering mismatch\n"
"The byte ordering used to store the pg_control file might not match the one\n"
"used by this program. In that case the results below would be incorrect, and\n"
"the PostgreSQL installation would be incompatible with this data directory.\n"));
printf(_("Catalog version number: %u\n"),
ControlFile.catalog_version_no);
printf(_("Database system identifier: %s\n"),
sysident_str);
printf(_("Database cluster state: %s\n"),
dbState(ControlFile.state));
printf(_("pg_control last modified: %s\n"),
pgctime_str);
printf(_("Latest checkpoint location: %X/%X\n"),
ControlFile.checkPoint.xlogid,
ControlFile.checkPoint.xrecoff);
printf(_("Prior checkpoint location: %X/%X\n"),
ControlFile.prevCheckPoint.xlogid,
ControlFile.prevCheckPoint.xrecoff);
printf(_("Latest checkpoint's REDO location: %X/%X\n"),
ControlFile.checkPointCopy.redo.xlogid,
ControlFile.checkPointCopy.redo.xrecoff);
printf(_("Latest checkpoint's UNDO location: %X/%X\n"),
ControlFile.checkPointCopy.undo.xlogid,
ControlFile.checkPointCopy.undo.xrecoff);
printf(_("Latest checkpoint's TimeLineID: %u\n"),
ControlFile.checkPointCopy.ThisTimeLineID);
printf(_("Latest checkpoint's NextXID: %u/%u\n"),
ControlFile.checkPointCopy.nextXidEpoch,
ControlFile.checkPointCopy.nextXid);
printf(_("Latest checkpoint's NextOID: %u\n"),
ControlFile.checkPointCopy.nextOid);
printf(_("Latest checkpoint's NextMultiXactId: %u\n"),
ControlFile.checkPointCopy.nextMulti);
printf(_("Latest checkpoint's NextMultiOffset: %u\n"),
ControlFile.checkPointCopy.nextMultiOffset);
printf(_("Time of latest checkpoint: %s\n"),
ckpttime_str);
printf(_("Minimum recovery ending location: %X/%X\n"),
ControlFile.minRecoveryPoint.xlogid,
ControlFile.minRecoveryPoint.xrecoff);
printf(_("Backup start location: %X/%X\n"),
ControlFile.backupStartPoint.xlogid,
ControlFile.backupStartPoint.xrecoff);
printf(_("End-of-backup record required: %s\n"),
ControlFile.backupEndRequired ? _("yes") : _("no"));
printf(_("Maximum data alignment: %u\n"),
ControlFile.maxAlign);
/* we don't print floatFormat since can't say much useful about it */
printf(_("Database block size: %u\n"),
ControlFile.blcksz);
printf(_("Blocks per segment of large relation: %u\n"),
ControlFile.relseg_size);
printf(_("WAL block size: %u\n"),
ControlFile.xlog_blcksz);
printf(_("Bytes per WAL segment: %u\n"),
ControlFile.xlog_seg_size);
printf(_("Maximum length of identifiers: %u\n"),
ControlFile.nameDataLen);
printf(_("Maximum columns in an index: %u\n"),
ControlFile.indexMaxKeys);
printf(_("Date/time type storage: %s\n"),
(ControlFile.enableIntTimes ? _("64-bit integers") : _("floating-point numbers")));
printf(_("Maximum length of locale name: %u\n"),
ControlFile.localeBuflen);
printf(_("LC_COLLATE: %s\n"),
ControlFile.lc_collate);
printf(_("LC_CTYPE: %s\n"),
ControlFile.lc_ctype);
return 0;
}