/***********************************************************************************************************************************
Get an archive file from the repository (WAL segment, history file, etc.)
***********************************************************************************************************************************/
int
cmdArchiveGet(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
// Set the result assuming the archive file will not be found
int result = 1;
MEM_CONTEXT_TEMP_BEGIN()
{
// Check the parameters
const StringList *commandParam = cfgCommandParam();
if (strLstSize(commandParam) != 2)
{
if (strLstSize(commandParam) == 0)
THROW(ParamRequiredError, "WAL segment to get required");
if (strLstSize(commandParam) == 1)
THROW(ParamRequiredError, "path to copy WAL segment required");
THROW(ParamInvalidError, "extra parameters found");
}
// Get the segment name
String *walSegment = strBase(strLstGet(commandParam, 0));
// Destination is wherever we were told to move the WAL segment
const String *walDestination =
walPath(strLstGet(commandParam, 1), cfgOptionStr(cfgOptPgPath), STR(cfgCommandName(cfgCommand())));
// Async get can only be performed on WAL segments, history or other files must use synchronous mode
if (cfgOptionBool(cfgOptArchiveAsync) && walIsSegment(walSegment))
{
bool found = false; // Has the WAL segment been found yet?
bool queueFull = false; // Is the queue half or more full?
bool forked = false; // Has the async process been forked yet?
bool confessOnError = false; // Should we confess errors?
// Loop and wait for the WAL segment to be pushed
Wait *wait = waitNew((TimeMSec)(cfgOptionDbl(cfgOptArchiveTimeout) * MSEC_PER_SEC));
do
{
// Check for errors or missing files. For archive-get ok indicates that the process succeeded but there is no WAL
// file to download.
if (archiveAsyncStatus(archiveModeGet, walSegment, confessOnError))
{
storageRemoveP(
storageSpoolWrite(),
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s" STATUS_EXT_OK, strPtr(walSegment)), .errorOnMissing = true);
break;
}
// Check if the WAL segment is already in the queue
found = storageExistsNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment)));
// If found then move the WAL segment to the destination directory
if (found)
{
// Source is the WAL segment in the spool queue
StorageFileRead *source = storageNewReadNP(
storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment)));
// A move will be attempted but if the spool queue and the WAL path are on different file systems then a copy
// will be performed instead.
//
// It looks scary that we are disabling syncs and atomicity (in case we need to copy intead of move) but this
// is safe because if the system crashes Postgres will not try to reuse a restored WAL segment but will instead
// request it again using the restore_command. In the case of a move this hardly matters since path syncs are
// cheap but if a copy is required we could save a lot of writes.
StorageFileWrite *destination = storageNewWriteP(
storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true,
.noAtomic = true);
// Move (or copy if required) the file
storageMoveNP(storageSpoolWrite(), source, destination);
// Return success
result = 0;
// Get a list of WAL segments left in the queue
StringList *queue = storageListP(
storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .expression = WAL_SEGMENT_REGEXP_STR);
if (strLstSize(queue) > 0)
{
// Get size of the WAL segment
uint64_t walSegmentSize = storageInfoNP(storageLocal(), walDestination).size;
// Use WAL segment size to estimate queue size and determine if the async process should be launched
queueFull = strLstSize(queue) * walSegmentSize > cfgOptionUInt64(cfgOptArchiveGetQueueMax) / 2;
}
}
// If the WAL segment has not already been found then start the async process to get it. There's no point in
// forking the async process off more than once so track that as well. Use an archive lock to prevent forking if
// the async process was launched by another process.
if (!forked && (!found || !queueFull) &&
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, false))
{
// Get control info
PgControl pgControl = pgControlFromFile(cfgOptionStr(cfgOptPgPath));
// Create the queue
storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR);
// The async process should not output on the console at all
KeyValue *optionReplace = kvNew();
kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_CONSOLE_STR), VARSTRDEF("off"));
kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_STDERR_STR), VARSTRDEF("off"));
// Generate command options
StringList *commandExec = cfgExecParam(cfgCmdArchiveGetAsync, optionReplace);
strLstInsert(commandExec, 0, cfgExe());
// Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed()
// will return the list of WAL needed to fill the queue and this will be passed to the async process.
const StringList *queue = queueNeed(
walSegment, found, cfgOptionUInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize,
pgControl.version);
for (unsigned int queueIdx = 0; queueIdx < strLstSize(queue); queueIdx++)
strLstAdd(commandExec, strLstGet(queue, queueIdx));
// Release the lock so the child process can acquire it
lockRelease(true);
// Fork off the async process
if (forkSafe() == 0)
{
// Disable logging and close log file
logClose();
// Detach from parent process
forkDetach();
// Execute the binary. This statement will not return if it is successful.
THROW_ON_SYS_ERROR(
execvp(strPtr(cfgExe()), (char ** const)strLstPtr(commandExec)) == -1,
ExecuteError, "unable to execute '" CFGCMD_ARCHIVE_GET_ASYNC "'");
}
// Mark the async process as forked so it doesn't get forked again. A single run of the async process should be
// enough to do the job, running it again won't help anything.
forked = true;
}
// Exit loop if WAL was found
if (found)
break;
// Now that the async process has been launched, confess any errors that are found
confessOnError = true;
}
while (waitMore(wait));
}
// Else perform synchronous get
else
{
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
// *****************************************************************************************************************************
if (testBegin("infoPgNewLoad(), infoPgFree(), infoPgDataCurrent(), infoPgDataToLog(), infoPgAdd(), infoPgIni()"))
{
String *content = NULL;
String *fileName = strNewFmt("%s/test.ini", testPath());
String *fileName2 = strNewFmt("%s/test2.ini", testPath());
// Archive info
//--------------------------------------------------------------------------------------------------------------------------
content = strNew
(
"[db]\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file");
InfoPg *infoPg = NULL;
Ini *ini = NULL;
TEST_ASSIGN(
infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgArchive, cipherTypeNone, NULL, &ini), "load file");
TEST_RESULT_STR(strPtr(iniGet(ini, strNew("db"), strNew("db-id"))), "1", " check ini");
TEST_RESULT_INT(lstSize(infoPg->history), 1, " history record added");
InfoPgData infoPgData = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgData.id, 1, " id set");
TEST_RESULT_INT(infoPgData.version, PG_VERSION_94, " version set");
TEST_RESULT_INT(infoPgData.systemId, 6569239123849665679, " system-id set");
TEST_RESULT_INT(infoPgData.catalogVersion, 0, " catalog-version not set");
TEST_RESULT_INT(infoPgData.controlVersion, 0, " control-version not set");
TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " check pg data total");
TEST_RESULT_STR(strPtr(infoPgArchiveId(infoPg, 0)), "9.4-1", " check pg archive id");
TEST_RESULT_PTR(infoPgCipherPass(infoPg), NULL, " no cipher passphrase");
// Backup info
//--------------------------------------------------------------------------------------------------------------------------
content = strNew
(
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679,"
"\"db-version\":\"9.4\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file");
TEST_ASSIGN(
infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgBackup, cipherTypeNone, NULL, NULL), "load file");
TEST_RESULT_INT(lstSize(infoPg->history), 1, " history record added");
infoPgData = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgData.id, 1, " id set");
TEST_RESULT_INT(infoPgData.version, PG_VERSION_94, " version set");
TEST_RESULT_INT(infoPgData.systemId, 6569239123849665679, " system-id set");
TEST_RESULT_INT(infoPgData.catalogVersion, 201409291, " catalog-version set");
TEST_RESULT_INT(infoPgData.controlVersion, 942, " control-version set");
// Manifest info
//--------------------------------------------------------------------------------------------------------------------------
content = strNew
(
"[db]\n"
"db-catalog-version=201510051\n"
"db-control-version=942\n"
"db-id=2\n"
"db-system-id=6365925855999999999\n"
"db-version=\"9.5\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679,"
"\"db-version\":\"9.4\"}\n"
"2={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6365925855999999999,"
"\"db-version\":\"9.5\"}\n"
);
// Put the file and load it
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file");
TEST_ASSIGN(
infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgManifest, cipherTypeNone, NULL, NULL), "load file");
// Save the file and verify it
ini = iniNew();
TEST_RESULT_VOID(infoPgSave(infoPg, ini, storageLocalWrite(), fileName2, cipherTypeNone, NULL), "save file");
TEST_RESULT_BOOL(
bufEq(
storageGetNP(storageNewReadNP(storageLocal(), fileName)),
storageGetNP(storageNewReadNP(storageLocal(), fileName2))),
true, "files are equal");
TEST_RESULT_INT(lstSize(infoPg->history), 2, "history record added");
infoPgData = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgData.id, 2, " id set");
TEST_RESULT_INT(infoPgData.version, PG_VERSION_95, " version set");
TEST_RESULT_INT(infoPgData.systemId, 6365925855999999999, " system-id set");
TEST_RESULT_INT(infoPgData.catalogVersion, 201510051, " catalog-version set");
TEST_RESULT_INT(infoPgData.controlVersion, 942, " control-version set");
// infoPgAdd
//--------------------------------------------------------------------------------------------------------------------------
infoPgData.id = 3;
infoPgData.version = PG_VERSION_96;
infoPgData.systemId = 6399999999999999999;
infoPgData.catalogVersion = 201608131;
infoPgData.controlVersion = 960;
TEST_RESULT_VOID(infoPgAdd(infoPg, &infoPgData), "infoPgAdd");
InfoPgData infoPgDataTest = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgDataTest.id, 3, " id set");
TEST_RESULT_INT(infoPgDataTest.version, PG_VERSION_96, " version set");
TEST_RESULT_INT(infoPgDataTest.systemId, 6399999999999999999, " system-id set");
TEST_RESULT_INT(infoPgDataTest.catalogVersion, 201608131, " catalog-version set");
TEST_RESULT_INT(infoPgDataTest.controlVersion, 960, " control-version set");
// Errors
//--------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(infoPgNewLoad(storageLocal(), fileName, 10, cipherTypeNone, NULL, NULL), AssertError, "invalid InfoPg type 10");
TEST_ERROR(
infoPgNewLoad(storageLocal(), NULL, infoPgManifest, cipherTypeNone, NULL, NULL), AssertError,
"assertion 'fileName != NULL' failed");
TEST_ERROR(infoPgDataCurrent(NULL), AssertError, "assertion 'this != NULL' failed");
TEST_ERROR(infoPgAdd(NULL, &infoPgData), AssertError, "assertion 'this != NULL' failed");
TEST_ERROR(infoPgAdd(infoPg, NULL), AssertError, "assertion 'infoPgData != NULL' failed");
// infoPgFree
//--------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(infoPgFree(infoPg), "infoPgFree() - free infoPg");
// infoPgDataToLog
//--------------------------------------------------------------------------------------------------------------------------
// test max values
infoPgDataTest.id = (unsigned int)4294967295;
infoPgDataTest.version = (unsigned int)4294967295;
infoPgDataTest.systemId = 18446744073709551615U;
infoPgDataTest.catalogVersion = (uint32_t)4294967295;
infoPgDataTest.controlVersion = (uint32_t)4294967295;
TEST_RESULT_STR(
strPtr(infoPgDataToLog(&infoPgDataTest)),
"{id: 4294967295, version: 4294967295, systemId: 18446744073709551615, catalogVersion: 4294967295, controlVersion:"
" 4294967295}",
" check max format");
}
}
