/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// Create default storage object for testing
Storage *storageTest = storagePosixNew(
strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
// *****************************************************************************************************************************
if (testBegin("lockStopFileName()"))
{
// Load configuration so lock path is set
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=db");
strLstAddZ(argList, "--lock-path=/path/to/lock");
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_STR(strPtr(lockStopFileName(NULL)), "/path/to/lock/all.stop", "stop file for all stanzas");
TEST_RESULT_STR(strPtr(lockStopFileName(strNew("db"))), "/path/to/lock/db.stop", "stop file for on stanza");
}
// *****************************************************************************************************************************
if (testBegin("lockStopTest()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAdd(argList, strNewFmt("--lock-path=%s", testPath()));
strLstAddZ(argList, "start");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(lockStopTest(), "no stop files without stanza");
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=db");
strLstAdd(argList, strNewFmt("--lock-path=%s", testPath()));
strLstAddZ(argList, "start");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(lockStopTest(), "no stop files with stanza");
storagePutNP(storageNewWriteNP(storageTest, strNew("all.stop")), NULL);
TEST_ERROR(lockStopTest(), StopError, "stop file exists for all stanzas");
storagePutNP(storageNewWriteNP(storageTest, strNew("db.stop")), NULL);
TEST_ERROR(lockStopTest(), StopError, "stop file exists for stanza db");
}
FUNCTION_HARNESS_RESULT_VOID();
}
/***********************************************************************************************************************************
Determine if the WAL process list has become large enough to drop
***********************************************************************************************************************************/
static bool
archivePushDrop(const String *walPath, const StringList *const processList)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, walPath);
FUNCTION_LOG_PARAM(STRING_LIST, processList);
FUNCTION_LOG_END();
const uint64_t queueMax = cfgOptionUInt64(cfgOptArchivePushQueueMax);
uint64_t queueSize = 0;
bool result = false;
for (unsigned int processIdx = 0; processIdx < strLstSize(processList); processIdx++)
{
queueSize += storageInfoNP(
storagePg(), strNewFmt("%s/%s", strPtr(walPath), strPtr(strLstGet(processList, processIdx)))).size;
if (queueSize > queueMax)
{
result = true;
break;
}
}
FUNCTION_LOG_RETURN(BOOL, result);
}
/***********************************************************************************************************************************
Create the stop filename
***********************************************************************************************************************************/
String *
lockStopFileName(const String *stanza)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, stanza);
FUNCTION_TEST_END();
String *result = strNewFmt("%s/%s.stop", strPtr(cfgOptionStr(cfgOptLockPath)), stanza != NULL ? strPtr(stanza) : "all");
FUNCTION_TEST_RETURN(result);
}
/***********************************************************************************************************************************
Format the warning when a file is dropped
***********************************************************************************************************************************/
static String *
archivePushDropWarning(const String *walFile, uint64_t queueMax)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, walFile);
FUNCTION_TEST_PARAM(UINT64, queueMax);
FUNCTION_TEST_END();
FUNCTION_TEST_RETURN(
strNewFmt("dropped WAL file '%s' because archive queue exceeded %s", strPtr(walFile), strPtr(strSizeFormat(queueMax))));
}
/***********************************************************************************************************************************
Remote command
***********************************************************************************************************************************/
void
cmdRemote(int handleRead, int handleWrite)
{
FUNCTION_LOG_VOID(logLevelDebug);
MEM_CONTEXT_TEMP_BEGIN()
{
String *name = strNewFmt(PROTOCOL_SERVICE_REMOTE "-%u", cfgOptionUInt(cfgOptProcess));
IoRead *read = ioHandleReadNew(name, handleRead, (TimeMSec)(cfgOptionDbl(cfgOptProtocolTimeout) * 1000));
ioReadOpen(read);
IoWrite *write = ioHandleWriteNew(name, handleWrite);
ioWriteOpen(write);
ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_REMOTE_STR, read, write);
protocolServerHandlerAdd(server, storageRemoteProtocol);
protocolServerHandlerAdd(server, configProtocol);
// Acquire a lock if this command needs one. We'll use the noop that is always sent from the client right after the
// handshake to return an error.
volatile bool success = false;
TRY_BEGIN()
{
// Read the command. No need to parse it since we know this is the first noop.
ioReadLine(read);
// Only try the lock if this is process 0, i.e. the remote started from the main process
if (cfgOptionUInt(cfgOptProcess) == 0)
{
ConfigCommand commandId = cfgCommandId(strPtr(cfgOptionStr(cfgOptCommand)));
// Acquire a lock if this command requires a lock
if (cfgLockRemoteRequired(commandId))
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockRemoteType(commandId), 0, true);
}
protocolServerResponse(server, NULL);
success = true;
}
CATCH_ANY()
{
protocolServerError(server, errorCode(), STR(errorMessage()), STR(errorStackTrace()));
}
TRY_END();
// If not successful we'll just exit
if (success)
protocolServerProcess(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Free Perl objects
Don't bother freeing Perl itself since we are about to exit.
***********************************************************************************************************************************/
void
perlFree(int result)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INT, result);
FUNCTION_TEST_END();
if (my_perl != NULL)
perlEval(strNewFmt(PGBACKREST_MAIN "Cleanup(%d)", result));
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Test server with subject alternate names
***********************************************************************************************************************************/
static void
testTlsServerAltName(void)
{
if (fork() == 0)
{
harnessTlsServerInit(
TLS_TEST_PORT,
strPtr(strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-alt-name.crt", testRepoPath())),
strPtr(strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".key", testRepoPath())));
// Certificate error on invalid ca path
harnessTlsServerAccept();
harnessTlsServerClose();
// Success on valid ca file and match common name
harnessTlsServerAccept();
harnessTlsServerClose();
// Success on valid ca file and match alt name
harnessTlsServerAccept();
harnessTlsServerClose();
// Unable to find matching hostname in certificate
harnessTlsServerAccept();
harnessTlsServerClose();
// Certificate error
harnessTlsServerAccept();
harnessTlsServerClose();
// Certificate ignored
harnessTlsServerAccept();
harnessTlsServerClose();
exit(0);
}
}
/***********************************************************************************************************************************
Build list of parameters to use for perl main
***********************************************************************************************************************************/
String *
perlMain(void)
{
FUNCTION_TEST_VOID();
// Add command arguments to pass to main
String *commandParam = strNew("");
for (unsigned int paramIdx = 0; paramIdx < strLstSize(cfgCommandParam()); paramIdx++)
strCatFmt(commandParam, ",'%s'", strPtr(strLstGet(cfgCommandParam(), paramIdx)));
// Construct Perl main call
String *mainCall = strNewFmt(
"($iResult, $bErrorC, $strMessage) = " PGBACKREST_MAIN "('%s'%s)", cfgCommandName(cfgCommand()), strPtr(commandParam));
FUNCTION_TEST_RETURN(mainCall);
}
/***********************************************************************************************************************************
End the command
***********************************************************************************************************************************/
void
cmdEnd(int code, const String *errorMessage)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, code);
FUNCTION_LOG_PARAM(STRING, errorMessage);
FUNCTION_LOG_END();
ASSERT(cfgCommand() != cfgCmdNone);
// Skip this log message if it won't be output. It's not too expensive but since we skipped cmdBegin(), may as well.
if (logWill(cfgLogLevelDefault()))
{
MEM_CONTEXT_TEMP_BEGIN()
{
// Basic info on command end
String *info = strNewFmt("%s command end: ", cfgCommandName(cfgCommand()));
if (errorMessage == NULL)
{
strCat(info, "completed successfully");
if (cfgOptionValid(cfgOptLogTimestamp) && cfgOptionBool(cfgOptLogTimestamp))
strCatFmt(info, " (%" PRIu64 "ms)", timeMSec() - timeBegin);
}
else
strCat(info, strPtr(errorMessage));
LOG(cfgLogLevelDefault(), 0, strPtr(info));
}
MEM_CONTEXT_TEMP_END();
}
// Reset timeBegin in case there is another command following this one
timeBegin = timeMSec();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Initialize Perl
***********************************************************************************************************************************/
static void
perlInit(void)
{
FUNCTION_TEST_VOID();
if (!my_perl)
{
// Initialize Perl with dummy args and environment
int argc = 1;
const char *argv[1] = {strPtr(cfgExe())};
const char *env[1] = {NULL};
PERL_SYS_INIT3(&argc, (char ***)&argv, (char ***)&env);
// Create the interpreter
const char *embedding[] = {"", "-e", "0"};
my_perl = perl_alloc();
perl_construct(my_perl);
// Don't let $0 assignment update the proctitle or embedding[0]
PL_origalen = 1;
// Start the interpreter
perl_parse(my_perl, xs_init, 3, (char **)embedding, NULL);
PL_exit_flags |= PERL_EXIT_DESTRUCT_END;
perl_run(my_perl);
// Use customer loader to get all embedded modules
eval_pv("splice(@INC, 0, 0, " LOADER_SUB ");", true);
// Now that the custom loader is installed, load the main module;
eval_pv("use " PGBACKREST_MODULE ";", true);
// Set config data -- this is done separately to avoid it being included in stack traces
perlEval(strNewFmt(PGBACKREST_MAIN "ConfigSet('%s', '%s')", strPtr(cfgExe()), strPtr(perlOptionJson())));
}
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// Additional coverage not provided by testing with actual certificates
// *****************************************************************************************************************************
if (testBegin("asn1ToStr(), tlsClientHostVerify(), and tlsClientHostVerifyName()"))
{
TEST_ERROR(asn1ToStr(NULL), CryptoError, "TLS certificate name entry is missing");
TEST_ERROR(
tlsClientHostVerifyName(
strNew("host"), strNewN("ab\0cd", 5)), CryptoError, "TLS certificate name contains embedded null");
TEST_ERROR(tlsClientHostVerify(strNew("host"), NULL), CryptoError, "No certificate presented by the TLS server");
TEST_RESULT_BOOL(tlsClientHostVerifyName(strNew("host"), strNew("**")), false, "invalid pattern");
TEST_RESULT_BOOL(tlsClientHostVerifyName(strNew("host"), strNew("*.")), false, "invalid pattern");
TEST_RESULT_BOOL(tlsClientHostVerifyName(strNew("a.bogus.host.com"), strNew("*.host.com")), false, "invalid host");
}
// *****************************************************************************************************************************
if (testBegin("TlsClient verification"))
{
TlsClient *client = NULL;
// Connection errors
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(client, tlsClientNew(strNew("99.99.99.99.99"), 9443, 0, true, NULL, NULL), "new client");
TEST_ERROR(
tlsClientOpen(client), HostConnectError, "unable to get address for '99.99.99.99.99': [-2] Name or service not known");
TEST_ASSIGN(client, tlsClientNew(strNew("localhost"), 9443, 100, true, NULL, NULL), "new client");
TEST_ERROR(tlsClientOpen(client), HostConnectError, "unable to connect to 'localhost:9443': [111] Connection refused");
// Certificate location and validation errors
// -------------------------------------------------------------------------------------------------------------------------
// Add test hosts
if (system( // {uncoverable_branch}
"echo \"127.0.0.1 test.pgbackrest.org host.test2.pgbackrest.org test3.pgbackrest.org\" |"
" sudo tee -a /etc/hosts > /dev/null") != 0)
{
THROW(AssertError, "unable to add test hosts to /etc/hosts"); // {uncovered+}
}
// Start server to test various certificate errors
testTlsServerAltName();
TEST_ERROR(
tlsClientOpen(tlsClientNew(strNew("localhost"), 9443, 500, true, strNew("bogus.crt"), strNew("/bogus"))),
CryptoError, "unable to set user-defined CA certificate location: [33558530] No such file or directory");
TEST_ERROR(
tlsClientOpen(tlsClientNew(strNew("localhost"), 9443, 500, true, NULL, strNew("/bogus"))),
CryptoError, "unable to verify certificate presented by 'localhost:9443': [20] unable to get local issuer certificate");
TEST_RESULT_VOID(
tlsClientOpen(
tlsClientNew(strNew("test.pgbackrest.org"), 9443, 500, true,
strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)),
"success on valid ca file and match common name");
TEST_RESULT_VOID(
tlsClientOpen(
tlsClientNew(strNew("host.test2.pgbackrest.org"), 9443, 500, true,
strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)),
"success on valid ca file and match alt name");
TEST_ERROR(
tlsClientOpen(
tlsClientNew(strNew("test3.pgbackrest.org"), 9443, 500, true,
strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)),
CryptoError, "unable to find hostname 'test3.pgbackrest.org' in certificate common name or subject alternative names");
TEST_ERROR(
tlsClientOpen(
tlsClientNew(strNew("localhost"), 9443, 500, true, strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".crt", testRepoPath()),
NULL)),
CryptoError, "unable to verify certificate presented by 'localhost:9443': [20] unable to get local issuer certificate");
TEST_RESULT_VOID(
tlsClientOpen(tlsClientNew(strNew("localhost"), 9443, 500, false, NULL, NULL)), "success on no verify");
}
// *****************************************************************************************************************************
if (testBegin("TlsClient general usage"))
{
TlsClient *client = NULL;
// Reset statistics
tlsClientStatLocal = (TlsClientStat){0};
TEST_RESULT_STR(tlsClientStatStr(), NULL, "no stats yet");
testTlsServer();
ioBufferSizeSet(12);
TEST_ASSIGN(client, tlsClientNew(strNew(TLS_TEST_HOST), 9443, 500, true, NULL, NULL), "new client");
TEST_RESULT_VOID(tlsClientOpen(client), "open client");
const Buffer *input = BUFSTRDEF("some protocol info");
TEST_RESULT_VOID(ioWrite(tlsClientIoWrite(client), input), "write input");
ioWriteFlush(tlsClientIoWrite(client));
TEST_RESULT_STR(strPtr(ioReadLine(tlsClientIoRead(client))), "something:0", "read line");
TEST_RESULT_BOOL(ioReadEof(tlsClientIoRead(client)), false, " check eof = false");
Buffer *output = bufNew(12);
TEST_RESULT_INT(ioRead(tlsClientIoRead(client), output), 12, "read output");
TEST_RESULT_STR(strPtr(strNewBuf(output)), "some content", " check output");
TEST_RESULT_BOOL(ioReadEof(tlsClientIoRead(client)), false, " check eof = false");
output = bufNew(8);
TEST_RESULT_INT(ioRead(tlsClientIoRead(client), output), 8, "read output");
TEST_RESULT_STR(strPtr(strNewBuf(output)), "AND MORE", " check output");
TEST_RESULT_BOOL(ioReadEof(tlsClientIoRead(client)), false, " check eof = false");
output = bufNew(12);
TEST_ERROR(
ioRead(tlsClientIoRead(client), output), FileReadError,
"unable to read data from 'tls.test.pgbackrest.org:9443' after 500ms");
// -------------------------------------------------------------------------------------------------------------------------
input = BUFSTRDEF("more protocol info");
TEST_RESULT_VOID(tlsClientOpen(client), "open client again (it is already open)");
TEST_RESULT_VOID(ioWrite(tlsClientIoWrite(client), input), "write input");
ioWriteFlush(tlsClientIoWrite(client));
output = bufNew(12);
TEST_RESULT_INT(ioRead(tlsClientIoRead(client), output), 12, "read output");
TEST_RESULT_STR(strPtr(strNewBuf(output)), "0123456789AB", " check output");
TEST_RESULT_BOOL(ioReadEof(tlsClientIoRead(client)), false, " check eof = false");
output = bufNew(12);
TEST_RESULT_INT(ioRead(tlsClientIoRead(client), output), 0, "read no output after eof");
TEST_RESULT_BOOL(ioReadEof(tlsClientIoRead(client)), true, " check eof = true");
TEST_RESULT_BOOL(tlsClientStatStr() != NULL, true, "check statistics exist");
TEST_RESULT_VOID(tlsClientFree(client), "free client");
}
FUNCTION_HARNESS_RESULT_VOID();
}
/***********************************************************************************************************************************
Clean the queue and prepare a list of WAL segments that the async process should get
***********************************************************************************************************************************/
static StringList *
queueNeed(const String *walSegment, bool found, uint64_t queueSize, size_t walSegmentSize, unsigned int pgVersion)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, walSegment);
FUNCTION_LOG_PARAM(BOOL, found);
FUNCTION_LOG_PARAM(UINT64, queueSize);
FUNCTION_LOG_PARAM(SIZE, walSegmentSize);
FUNCTION_LOG_PARAM(UINT, pgVersion);
FUNCTION_LOG_END();
ASSERT(walSegment != NULL);
StringList *result = strLstNew();
MEM_CONTEXT_TEMP_BEGIN()
{
// Determine the first WAL segment for the async process to get. If the WAL segment requested by
// PostgreSQL was not found then use that. If the segment was found but the queue is not full then
// start with the next segment.
const String *walSegmentFirst =
found ? walSegmentNext(walSegment, walSegmentSize, pgVersion) : walSegment;
// Determine how many WAL segments should be in the queue. The queue total must be at least 2 or it doesn't make sense to
// have async turned on at all.
unsigned int walSegmentQueueTotal = (unsigned int)(queueSize / walSegmentSize);
if (walSegmentQueueTotal < 2)
walSegmentQueueTotal = 2;
// Build the ideal queue -- the WAL segments we want in the queue after the async process has run
StringList *idealQueue = walSegmentRange(walSegmentFirst, walSegmentSize, pgVersion, walSegmentQueueTotal);
// Get the list of files actually in the queue
StringList *actualQueue = strLstSort(
storageListP(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .errorOnMissing = true), sortOrderAsc);
// Only preserve files that match the ideal queue. error/ok files are deleted so the async process can try again.
RegExp *regExpPreserve = regExpNew(strNewFmt("^(%s)$", strPtr(strLstJoin(idealQueue, "|"))));
// Build a list of WAL segments that are being kept so we can later make a list of what is needed
StringList *keepQueue = strLstNew();
for (unsigned int actualQueueIdx = 0; actualQueueIdx < strLstSize(actualQueue); actualQueueIdx++)
{
// Get file from actual queue
const String *file = strLstGet(actualQueue, actualQueueIdx);
// Does this match a file we want to preserve?
if (regExpMatch(regExpPreserve, file))
strLstAdd(keepQueue, file);
// Else delete it
else
storageRemoveNP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(file)));
}
// Generate a list of the WAL that are needed by removing kept WAL from the ideal queue
for (unsigned int idealQueueIdx = 0; idealQueueIdx < strLstSize(idealQueue); idealQueueIdx++)
{
if (!strLstExists(keepQueue, strLstGet(idealQueue, idealQueueIdx)))
strLstAdd(result, strLstGet(idealQueue, idealQueueIdx));
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING_LIST, result);
}
/***********************************************************************************************************************************
Begin the command
***********************************************************************************************************************************/
void
cmdBegin(bool logOption)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BOOL, logOption);
FUNCTION_LOG_END();
ASSERT(cfgCommand() != cfgCmdNone);
// This is fairly expensive log message to generate so skip it if it won't be output
if (logWill(cfgLogLevelDefault()))
{
MEM_CONTEXT_TEMP_BEGIN()
{
// Basic info on command start
String *info = strNewFmt("%s command begin", cfgCommandName(cfgCommand()));
if (logOption)
{
strCatFmt(info, " %s:", PROJECT_VERSION);
// Add command parameters if they exist
const StringList *commandParamList = cfgCommandParam();
if (strLstSize(commandParamList) != 0)
{
strCatFmt(info, " [");
for (unsigned int commandParamIdx = 0; commandParamIdx < strLstSize(commandParamList); commandParamIdx++)
{
const String *commandParam = strLstGet(commandParamList, commandParamIdx);
if (commandParamIdx != 0)
strCatFmt(info, ", ");
if (strchr(strPtr(commandParam), ' ') != NULL)
commandParam = strNewFmt("\"%s\"", strPtr(commandParam));
strCat(info, strPtr(commandParam));
}
strCatFmt(info, "]");
}
// Loop though options and add the ones that are interesting
for (ConfigOption optionId = 0; optionId < CFG_OPTION_TOTAL; optionId++)
{
// Skip the option if it is not valid
if (!cfgOptionValid(optionId))
continue;
// If option was negated
if (cfgOptionNegate(optionId))
strCatFmt(info, " --no-%s", cfgOptionName(optionId));
// If option was reset
else if (cfgOptionReset(optionId))
strCatFmt(info, " --reset-%s", cfgOptionName(optionId));
// Else set and not default
else if (cfgOptionSource(optionId) != cfgSourceDefault && cfgOptionTest(optionId))
{
ConfigDefineOption optionDefId = cfgOptionDefIdFromId(optionId);
// Don't show redacted options
if (cfgDefOptionSecure(optionDefId))
strCatFmt(info, " --%s=<redacted>", cfgOptionName(optionId));
// Output boolean option
else if (cfgDefOptionType(optionDefId) == cfgDefOptTypeBoolean)
strCatFmt(info, " --%s", cfgOptionName(optionId));
// Output other options
else
{
StringList *valueList = NULL;
// Generate the values of hash options
if (cfgDefOptionType(optionDefId) == cfgDefOptTypeHash)
{
valueList = strLstNew();
const KeyValue *optionKv = cfgOptionKv(optionId);
const VariantList *keyList = kvKeyList(optionKv);
for (unsigned int keyIdx = 0; keyIdx < varLstSize(keyList); keyIdx++)
{
strLstAdd(
valueList,
strNewFmt(
"%s=%s", strPtr(varStr(varLstGet(keyList, keyIdx))),
strPtr(varStrForce(kvGet(optionKv, varLstGet(keyList, keyIdx))))));
}
}
// Generate values for list options
else if (cfgDefOptionType(optionDefId) == cfgDefOptTypeList)
{
valueList = strLstNewVarLst(cfgOptionLst(optionId));
}
// Else only one value
else
{
valueList = strLstNew();
strLstAdd(valueList, varStrForce(cfgOption(optionId)));
}
// Output options and values
for (unsigned int valueListIdx = 0; valueListIdx < strLstSize(valueList); valueListIdx++)
{
const String *value = strLstGet(valueList, valueListIdx);
strCatFmt(info, " --%s", cfgOptionName(optionId));
if (strchr(strPtr(value), ' ') != NULL)
value = strNewFmt("\"%s\"", strPtr(value));
strCatFmt(info, "=%s", strPtr(value));
}
}
}
}
}
LOG(cfgLogLevelDefault(), 0, strPtr(info));
}
MEM_CONTEXT_TEMP_END();
}
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
// Create default storage object for testing
Storage *storageTest = storagePosixNew(
strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
// *****************************************************************************************************************************
if (testBegin("infoNewLoad(), infoFileName(), infoIni()"))
{
// Initialize test variables
//--------------------------------------------------------------------------------------------------------------------------
String *content = NULL;
String *fileName = strNewFmt("%s/test.ini", testPath());
String *fileNameCopy = strNewFmt("%s/test.ini.copy", testPath());
Info *info = NULL;
content = strNew
(
"[backrest]\n"
"backrest-checksum=\"1efa53e0611604ad7d833c5547eb60ff716e758c\"\n"
"backrest-format=5\n"
"backrest-version=\"2.04\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n"
);
// Info files missing and at least one is required
//--------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT(
infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FileMissingError,
"unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING,
testPath(), testPath(), strPtr(strNewFmt("%s/test.ini", testPath())),
strPtr(strNewFmt("%s/test.ini.copy", testPath())));
// Only copy exists and one is required
//--------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put info.copy to file");
TEST_ASSIGN(info, infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), "load copy file");
TEST_RESULT_PTR(infoCipherPass(info), NULL, " cipherPass is not set");
// Remove the copy and store only the main info file and encrypt it. One is required.
//--------------------------------------------------------------------------------------------------------------------------
StorageWrite *infoWrite = storageNewWriteNP(storageLocalWrite(), fileName);
ioWriteFilterGroupSet(
storageWriteIo(infoWrite),
ioFilterGroupAdd(
ioFilterGroupNew(), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("12345678"), NULL)));
storageRemoveNP(storageLocalWrite(), fileNameCopy);
storagePutNP(
infoWrite,
BUFSTRDEF(
"[backrest]\n"
"backrest-checksum=\"9d2f6dce339751e1a056187fad67d2834b3d4ab3\"\n"
"backrest-format=5\n"
"backrest-version=\"2.04\"\n"
"\n"
"[cipher]\n"
"cipher-pass=\"ABCDEFGH\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n"));
// Only main info exists and is required
Ini *ini = NULL;
TEST_ASSIGN(info, infoNewLoad(storageLocal(), fileName, cipherTypeAes256Cbc, strNew("12345678"), &ini), "load file");
TEST_RESULT_STR(strPtr(iniGet(ini, strNew("cipher"), strNew("cipher-pass"))), "\"ABCDEFGH\"", " check ini");
TEST_RESULT_STR(strPtr(infoCipherPass(info)), "ABCDEFGH", " cipherPass is set");
// Invalid format
//--------------------------------------------------------------------------------------------------------------------------
storageRemoveNP(storageLocalWrite(), fileName);
content = strNew
(
"[backrest]\n"
"backrest-checksum=\"14617b089cb5c9b3224e739bb794e865b9bcdf4b\"\n"
"backrest-format=4\n"
"backrest-version=\"2.04\"\n"
"\n"
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679,"
"\"db-version\":\"9.4\"}\n"
);
// Only main file exists but the backrest-format is invalid
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), BUFSTR(content)), "put invalid br format to file");
TEST_ERROR_FMT(
infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FormatError,
"unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n"
"FormatError: invalid format in '%s/test.ini', expected 5 but found 4\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING,
testPath(), testPath(), testPath(), strPtr(strNewFmt("%s/test.ini.copy", testPath())));
content = strNew
(
"[backrest]\n"
"backrest-checksum=\"14617b089cb5c9b3224e739bb794e865b9bcdf4b\"\n"
"backrest-format=4\n"
"backrest-version=\"2.05\"\n"
"\n"
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679,"
"\"db-version\":\"9.4\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(
storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put invalid info to copy file");
TEST_ERROR(
infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FileOpenError,
strPtr(
strNewFmt(
"unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n"
"FormatError: invalid format in '%s/test.ini', expected 5 but found 4\n"
"ChecksumError: invalid checksum in '%s/test.ini.copy', expected 'af92308095d6141bcda6b2df6d574f98d1115163'"
" but found '14617b089cb5c9b3224e739bb794e865b9bcdf4b'",
testPath(), testPath(), testPath(), testPath())));
// Invalid checksum
//--------------------------------------------------------------------------------------------------------------------------
storageRemoveNP(storageLocalWrite(), fileName);
storageRemoveNP(storageLocalWrite(), fileNameCopy);
// change the checksum
content = strNew
(
"[backrest]\n"
"backrest-checksum=\"4306ec205f71417c301e403c4714090e61c8a999\"\n"
"backrest-format=5\n"
"backrest-version=\"1.23\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"db-system-id=6455618988686438683\n"
"db-version=\"9.6\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6455618988686438683,\"db-version\":\"9.6\"}\n"
"2={\"db-id\":6457457208275135411,\"db-version\":\"9.6\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put invalid checksum to copy");
// Empty checksum for main file
content = strNew
(
"[backrest]\n"
"backrest-checksum=\n"
"backrest-format=5\n"
"backrest-version=\"1.23\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"db-system-id=6455618988686438683\n"
"db-version=\"9.6\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6455618988686438683,\"db-version\":\"9.6\"}\n"
"2={\"db-id\":6457457208275135411,\"db-version\":\"9.6\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), BUFSTR(content)), "put empty checksum to file");
// Copy file error
TEST_ERROR(
infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), ChecksumError,
strPtr(
strNewFmt(
"unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n"
"ChecksumError: invalid checksum in '%s/test.ini', expected '4306ec205f71417c301e403c4714090e61c8a736' but"
" no checksum found\n"
"ChecksumError: invalid checksum in '%s/test.ini.copy', expected '4306ec205f71417c301e403c4714090e61c8a736'"
" but found '4306ec205f71417c301e403c4714090e61c8a999'",
testPath(), testPath(), testPath(), testPath())));
// Encryption error
//--------------------------------------------------------------------------------------------------------------------------
storageRemoveNP(storageLocalWrite(), fileName);
TEST_ERROR_FMT(
infoNewLoad(storageLocal(), fileName, cipherTypeAes256Cbc, strNew("12345678"), NULL), CryptoError,
"unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"CryptoError: '%s/test.ini.copy' cipher header invalid\n"
"HINT: Is or was the repo encrypted?",
testPath(), testPath(), strPtr(strNewFmt("%s/test.ini", testPath())), testPath());
storageRemoveNP(storageLocalWrite(), fileNameCopy);
// infoFree()
//--------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(infoFree(info), "infoFree() - free info memory context");
}
// *****************************************************************************************************************************
if (testBegin("infoSave()"))
{
const String *fileName = strNew("test.info");
const String *cipherPass = strNew("12345");
Ini *ini = iniNew();
iniSet(ini, strNew("section1"), strNew("key1"), strNew("value1"));
TEST_RESULT_VOID(infoSave(infoNew(), ini, storageTest, fileName, cipherTypeNone, NULL), "save info");
ini = NULL;
TEST_RESULT_VOID(infoNewLoad(storageTest, fileName, cipherTypeNone, NULL, &ini), " reload info");
TEST_RESULT_STR(strPtr(iniGet(ini, strNew("section1"), strNew("key1"))), "value1", " check ini");
TEST_RESULT_BOOL(storageExistsNP(storageTest, fileName), true, "check main exists");
TEST_RESULT_BOOL(storageExistsNP(storageTest, strNewFmt("%s" INFO_COPY_EXT, strPtr(fileName))), true, "check main exists");
// Add encryption
// -------------------------------------------------------------------------------------------------------------------------
ini = iniNew();
iniSet(ini, strNew("section1"), strNew("key1"), strNew("value4"));
Info *info = infoNew();
info->cipherPass = strNew("/badpass");
TEST_RESULT_VOID(infoSave(info, ini, storageTest, fileName, cipherTypeAes256Cbc, cipherPass), "save encrypted info");
ini = NULL;
TEST_RESULT_VOID(infoNewLoad(storageTest, fileName, cipherTypeAes256Cbc, cipherPass, &ini), " reload info");
TEST_RESULT_STR(strPtr(iniGet(ini, strNew("section1"), strNew("key1"))), "value4", " check ini");
TEST_RESULT_STR(strPtr(iniGet(ini, strNew("cipher"), strNew("cipher-pass"))), "\"/badpass\"", " check cipher-pass");
}
}
/***********************************************************************************************************************************
Get an archive file from the repository (WAL segment, history file, etc.)
***********************************************************************************************************************************/
int
cmdArchiveGet(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
// Set the result assuming the archive file will not be found
int result = 1;
MEM_CONTEXT_TEMP_BEGIN()
{
// Check the parameters
const StringList *commandParam = cfgCommandParam();
if (strLstSize(commandParam) != 2)
{
if (strLstSize(commandParam) == 0)
THROW(ParamRequiredError, "WAL segment to get required");
if (strLstSize(commandParam) == 1)
THROW(ParamRequiredError, "path to copy WAL segment required");
THROW(ParamInvalidError, "extra parameters found");
}
// Get the segment name
String *walSegment = strBase(strLstGet(commandParam, 0));
// Destination is wherever we were told to move the WAL segment
const String *walDestination =
walPath(strLstGet(commandParam, 1), cfgOptionStr(cfgOptPgPath), STR(cfgCommandName(cfgCommand())));
// Async get can only be performed on WAL segments, history or other files must use synchronous mode
if (cfgOptionBool(cfgOptArchiveAsync) && walIsSegment(walSegment))
{
bool found = false; // Has the WAL segment been found yet?
bool queueFull = false; // Is the queue half or more full?
bool forked = false; // Has the async process been forked yet?
bool confessOnError = false; // Should we confess errors?
// Loop and wait for the WAL segment to be pushed
Wait *wait = waitNew((TimeMSec)(cfgOptionDbl(cfgOptArchiveTimeout) * MSEC_PER_SEC));
do
{
// Check for errors or missing files. For archive-get ok indicates that the process succeeded but there is no WAL
// file to download.
if (archiveAsyncStatus(archiveModeGet, walSegment, confessOnError))
{
storageRemoveP(
storageSpoolWrite(),
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s" STATUS_EXT_OK, strPtr(walSegment)), .errorOnMissing = true);
break;
}
// Check if the WAL segment is already in the queue
found = storageExistsNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment)));
// If found then move the WAL segment to the destination directory
if (found)
{
// Source is the WAL segment in the spool queue
StorageFileRead *source = storageNewReadNP(
storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment)));
// A move will be attempted but if the spool queue and the WAL path are on different file systems then a copy
// will be performed instead.
//
// It looks scary that we are disabling syncs and atomicity (in case we need to copy intead of move) but this
// is safe because if the system crashes Postgres will not try to reuse a restored WAL segment but will instead
// request it again using the restore_command. In the case of a move this hardly matters since path syncs are
// cheap but if a copy is required we could save a lot of writes.
StorageFileWrite *destination = storageNewWriteP(
storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true,
.noAtomic = true);
// Move (or copy if required) the file
storageMoveNP(storageSpoolWrite(), source, destination);
// Return success
result = 0;
// Get a list of WAL segments left in the queue
StringList *queue = storageListP(
storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .expression = WAL_SEGMENT_REGEXP_STR);
if (strLstSize(queue) > 0)
{
// Get size of the WAL segment
uint64_t walSegmentSize = storageInfoNP(storageLocal(), walDestination).size;
// Use WAL segment size to estimate queue size and determine if the async process should be launched
queueFull = strLstSize(queue) * walSegmentSize > cfgOptionUInt64(cfgOptArchiveGetQueueMax) / 2;
}
}
// If the WAL segment has not already been found then start the async process to get it. There's no point in
// forking the async process off more than once so track that as well. Use an archive lock to prevent forking if
// the async process was launched by another process.
if (!forked && (!found || !queueFull) &&
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, false))
{
// Get control info
PgControl pgControl = pgControlFromFile(cfgOptionStr(cfgOptPgPath));
// Create the queue
storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR);
// The async process should not output on the console at all
KeyValue *optionReplace = kvNew();
kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_CONSOLE_STR), VARSTRDEF("off"));
kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_STDERR_STR), VARSTRDEF("off"));
// Generate command options
StringList *commandExec = cfgExecParam(cfgCmdArchiveGetAsync, optionReplace);
strLstInsert(commandExec, 0, cfgExe());
// Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed()
// will return the list of WAL needed to fill the queue and this will be passed to the async process.
const StringList *queue = queueNeed(
walSegment, found, cfgOptionUInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize,
pgControl.version);
for (unsigned int queueIdx = 0; queueIdx < strLstSize(queue); queueIdx++)
strLstAdd(commandExec, strLstGet(queue, queueIdx));
// Release the lock so the child process can acquire it
lockRelease(true);
// Fork off the async process
if (forkSafe() == 0)
{
// Disable logging and close log file
logClose();
// Detach from parent process
forkDetach();
// Execute the binary. This statement will not return if it is successful.
THROW_ON_SYS_ERROR(
execvp(strPtr(cfgExe()), (char ** const)strLstPtr(commandExec)) == -1,
ExecuteError, "unable to execute '" CFGCMD_ARCHIVE_GET_ASYNC "'");
}
// Mark the async process as forked so it doesn't get forked again. A single run of the async process should be
// enough to do the job, running it again won't help anything.
forked = true;
}
// Exit loop if WAL was found
if (found)
break;
// Now that the async process has been launched, confess any errors that are found
confessOnError = true;
}
while (waitMore(wait));
}
// Else perform synchronous get
else
{
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
// *****************************************************************************************************************************
if (testBegin("infoPgNewLoad(), infoPgFree(), infoPgDataCurrent(), infoPgDataToLog(), infoPgAdd(), infoPgIni()"))
{
String *content = NULL;
String *fileName = strNewFmt("%s/test.ini", testPath());
String *fileName2 = strNewFmt("%s/test2.ini", testPath());
// Archive info
//--------------------------------------------------------------------------------------------------------------------------
content = strNew
(
"[db]\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file");
InfoPg *infoPg = NULL;
Ini *ini = NULL;
TEST_ASSIGN(
infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgArchive, cipherTypeNone, NULL, &ini), "load file");
TEST_RESULT_STR(strPtr(iniGet(ini, strNew("db"), strNew("db-id"))), "1", " check ini");
TEST_RESULT_INT(lstSize(infoPg->history), 1, " history record added");
InfoPgData infoPgData = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgData.id, 1, " id set");
TEST_RESULT_INT(infoPgData.version, PG_VERSION_94, " version set");
TEST_RESULT_INT(infoPgData.systemId, 6569239123849665679, " system-id set");
TEST_RESULT_INT(infoPgData.catalogVersion, 0, " catalog-version not set");
TEST_RESULT_INT(infoPgData.controlVersion, 0, " control-version not set");
TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " check pg data total");
TEST_RESULT_STR(strPtr(infoPgArchiveId(infoPg, 0)), "9.4-1", " check pg archive id");
TEST_RESULT_PTR(infoPgCipherPass(infoPg), NULL, " no cipher passphrase");
// Backup info
//--------------------------------------------------------------------------------------------------------------------------
content = strNew
(
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679,"
"\"db-version\":\"9.4\"}\n"
);
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file");
TEST_ASSIGN(
infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgBackup, cipherTypeNone, NULL, NULL), "load file");
TEST_RESULT_INT(lstSize(infoPg->history), 1, " history record added");
infoPgData = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgData.id, 1, " id set");
TEST_RESULT_INT(infoPgData.version, PG_VERSION_94, " version set");
TEST_RESULT_INT(infoPgData.systemId, 6569239123849665679, " system-id set");
TEST_RESULT_INT(infoPgData.catalogVersion, 201409291, " catalog-version set");
TEST_RESULT_INT(infoPgData.controlVersion, 942, " control-version set");
// Manifest info
//--------------------------------------------------------------------------------------------------------------------------
content = strNew
(
"[db]\n"
"db-catalog-version=201510051\n"
"db-control-version=942\n"
"db-id=2\n"
"db-system-id=6365925855999999999\n"
"db-version=\"9.5\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679,"
"\"db-version\":\"9.4\"}\n"
"2={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6365925855999999999,"
"\"db-version\":\"9.5\"}\n"
);
// Put the file and load it
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file");
TEST_ASSIGN(
infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgManifest, cipherTypeNone, NULL, NULL), "load file");
// Save the file and verify it
ini = iniNew();
TEST_RESULT_VOID(infoPgSave(infoPg, ini, storageLocalWrite(), fileName2, cipherTypeNone, NULL), "save file");
TEST_RESULT_BOOL(
bufEq(
storageGetNP(storageNewReadNP(storageLocal(), fileName)),
storageGetNP(storageNewReadNP(storageLocal(), fileName2))),
true, "files are equal");
TEST_RESULT_INT(lstSize(infoPg->history), 2, "history record added");
infoPgData = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgData.id, 2, " id set");
TEST_RESULT_INT(infoPgData.version, PG_VERSION_95, " version set");
TEST_RESULT_INT(infoPgData.systemId, 6365925855999999999, " system-id set");
TEST_RESULT_INT(infoPgData.catalogVersion, 201510051, " catalog-version set");
TEST_RESULT_INT(infoPgData.controlVersion, 942, " control-version set");
// infoPgAdd
//--------------------------------------------------------------------------------------------------------------------------
infoPgData.id = 3;
infoPgData.version = PG_VERSION_96;
infoPgData.systemId = 6399999999999999999;
infoPgData.catalogVersion = 201608131;
infoPgData.controlVersion = 960;
TEST_RESULT_VOID(infoPgAdd(infoPg, &infoPgData), "infoPgAdd");
InfoPgData infoPgDataTest = infoPgDataCurrent(infoPg);
TEST_RESULT_INT(infoPgDataTest.id, 3, " id set");
TEST_RESULT_INT(infoPgDataTest.version, PG_VERSION_96, " version set");
TEST_RESULT_INT(infoPgDataTest.systemId, 6399999999999999999, " system-id set");
TEST_RESULT_INT(infoPgDataTest.catalogVersion, 201608131, " catalog-version set");
TEST_RESULT_INT(infoPgDataTest.controlVersion, 960, " control-version set");
// Errors
//--------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(infoPgNewLoad(storageLocal(), fileName, 10, cipherTypeNone, NULL, NULL), AssertError, "invalid InfoPg type 10");
TEST_ERROR(
infoPgNewLoad(storageLocal(), NULL, infoPgManifest, cipherTypeNone, NULL, NULL), AssertError,
"assertion 'fileName != NULL' failed");
TEST_ERROR(infoPgDataCurrent(NULL), AssertError, "assertion 'this != NULL' failed");
TEST_ERROR(infoPgAdd(NULL, &infoPgData), AssertError, "assertion 'this != NULL' failed");
TEST_ERROR(infoPgAdd(infoPg, NULL), AssertError, "assertion 'infoPgData != NULL' failed");
// infoPgFree
//--------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(infoPgFree(infoPg), "infoPgFree() - free infoPg");
// infoPgDataToLog
//--------------------------------------------------------------------------------------------------------------------------
// test max values
infoPgDataTest.id = (unsigned int)4294967295;
infoPgDataTest.version = (unsigned int)4294967295;
infoPgDataTest.systemId = 18446744073709551615U;
infoPgDataTest.catalogVersion = (uint32_t)4294967295;
infoPgDataTest.controlVersion = (uint32_t)4294967295;
TEST_RESULT_STR(
strPtr(infoPgDataToLog(&infoPgDataTest)),
"{id: 4294967295, version: 4294967295, systemId: 18446744073709551615, catalogVersion: 4294967295, controlVersion:"
" 4294967295}",
" check max format");
}
}
/***********************************************************************************************************************************
Test run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("cfgLoadLogSetting()"))
{
cfgInit();
TEST_RESULT_VOID(cfgLoadLogSetting(), "load log settings all defaults");
TEST_RESULT_INT(logLevelStdOut, logLevelOff, "console logging is off");
TEST_RESULT_INT(logLevelStdErr, logLevelOff, "stderr logging is off");
TEST_RESULT_INT(logLevelFile, logLevelOff, "file logging is off");
TEST_RESULT_BOOL(logTimestamp, true, "timestamp logging is on");
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgCommandSet(cfgCmdLocal);
cfgOptionValidSet(cfgOptLogLevelConsole, true);
cfgOptionSet(cfgOptLogLevelConsole, cfgSourceParam, varNewStrZ("info"));
cfgOptionValidSet(cfgOptLogLevelStderr, true);
cfgOptionSet(cfgOptLogLevelStderr, cfgSourceParam, varNewStrZ("error"));
cfgOptionValidSet(cfgOptLogLevelFile, true);
cfgOptionSet(cfgOptLogLevelFile, cfgSourceParam, varNewStrZ("debug"));
cfgOptionValidSet(cfgOptLogTimestamp, true);
cfgOptionSet(cfgOptLogTimestamp, cfgSourceParam, varNewBool(false));
TEST_RESULT_VOID(cfgLoadLogSetting(), "load log settings no defaults");
TEST_RESULT_INT(logLevelStdOut, logLevelInfo, "console logging is info");
TEST_RESULT_INT(logLevelStdErr, logLevelError, "stderr logging is error");
TEST_RESULT_INT(logLevelFile, logLevelDebug, "file logging is debugging");
TEST_RESULT_BOOL(logTimestamp, false, "timestamp logging is off");
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgCommandSet(cfgCmdLocal);
cfgOptionValidSet(cfgOptLogLevelStderr, true);
cfgOptionSet(cfgOptLogLevelStderr, cfgSourceParam, varNewStrZ("info"));
TEST_RESULT_VOID(cfgLoadLogSetting(), "load log settings reset stderr");
TEST_RESULT_INT(logLevelStdErr, logLevelError, "stderr logging is error");
}
// *****************************************************************************************************************************
if (testBegin("cfgLoadUpdateOption()"))
{
String *exe = strNew("/path/to/pgbackrest");
String *exeOther = strNew("/other/path/to/pgbackrest");
cfgInit();
cfgCommandSet(cfgCmdBackup);
cfgExeSet(exe);
cfgOptionValidSet(cfgOptRepoHost, true);
cfgOptionValidSet(cfgOptPgHost, true);
TEST_RESULT_VOID(cfgLoadUpdateOption(), "hosts are not set so don't update commands");
cfgOptionSet(cfgOptRepoHost, cfgSourceParam, varNewStrZ("repo-host"));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "repo remote command is updated");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoHostCmd)), strPtr(exe), " check repo1-host-cmd");
cfgOptionSet(cfgOptRepoHostCmd, cfgSourceParam, varNewStr(exeOther));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "repo remote command was already set");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoHostCmd)), strPtr(exeOther), " check repo1-host-cmd");
cfgOptionSet(cfgOptRepoHost, cfgSourceParam, NULL);
// -------------------------------------------------------------------------------------------------------------------------
cfgOptionValidSet(cfgOptPgHostCmd, true);
cfgOptionSet(cfgOptPgHost, cfgSourceParam, varNewStrZ("pg1-host"));
cfgOptionValidSet(cfgOptPgHost + 1, true);
cfgOptionSet(cfgOptPgHost + 1, cfgSourceParam, varNewStrZ("pg2-host"));
cfgOptionValidSet(cfgOptPgHostCmd + 1, true);
cfgOptionSet(cfgOptPgHostCmd + 1, cfgSourceParam, varNewStr(exeOther));
cfgOptionValidSet(cfgOptPgHost + cfgOptionIndexTotal(cfgOptPgHost) - 1, true);
cfgOptionSet(cfgOptPgHost + cfgOptionIndexTotal(cfgOptPgHost) - 1, cfgSourceParam, varNewStrZ("pgX-host"));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "pg remote command is updated");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptPgHostCmd)), strPtr(exe), " check pg1-host-cmd");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptPgHostCmd + 1)), strPtr(exeOther), " check pg2-host-cmd is already set");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptPgHostCmd + 2)), NULL, " check pg3-host-cmd is not set");
TEST_RESULT_STR(
strPtr(cfgOptionStr(cfgOptPgHostCmd + cfgOptionIndexTotal(cfgOptPgHost) - 1)), strPtr(exe), " check pgX-host-cmd");
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgOptionValidSet(cfgOptDbTimeout, true);
cfgOptionSet(cfgOptDbTimeout, cfgSourceParam, varNewDbl(100));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "pg timeout set but not protocol timeout");
cfgOptionValidSet(cfgOptProtocolTimeout, true);
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceDefault, varNewDbl(101));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "protocol timeout > pg timeout");
cfgOptionSet(cfgOptDbTimeout, cfgSourceParam, varNewDbl(100000));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "protocol timeout set automatically");
TEST_RESULT_DOUBLE(cfgOptionDbl(cfgOptProtocolTimeout), 100030, " check protocol timeout");
cfgOptionValidSet(cfgOptProtocolTimeout, true);
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceParam, varNewDbl(50.5));
TEST_ERROR(
cfgLoadUpdateOption(), OptionInvalidValueError,
"'50.5' is not valid for 'protocol-timeout' option\n"
"HINT 'protocol-timeout' option (50.5) should be greater than 'db-timeout' option (100000).");
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceParam, varNewDbl(45));
cfgOptionSet(cfgOptDbTimeout, cfgSourceDefault, varNewDbl(3600));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "set default pg timeout to be less than protocol timeout");
TEST_RESULT_DOUBLE(cfgOptionDbl(cfgOptProtocolTimeout), 45, " check protocol timeout");
TEST_RESULT_DOUBLE(cfgOptionDbl(cfgOptDbTimeout), 15, " check db timeout");
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceParam, varNewDbl(11));
cfgOptionSet(cfgOptDbTimeout, cfgSourceDefault, varNewDbl(3600));
TEST_RESULT_VOID(cfgLoadUpdateOption(), "set default pg timeout to be less than test protocol timeout");
TEST_RESULT_DOUBLE(cfgOptionDbl(cfgOptProtocolTimeout), 11, " check protocol timeout");
TEST_RESULT_DOUBLE(cfgOptionDbl(cfgOptDbTimeout), 5.5, " check db timeout");
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgCommandSet(cfgCmdBackup);
cfgExeSet(exe);
cfgOptionValidSet(cfgOptPgHost, true);
TEST_RESULT_VOID(cfgLoadUpdateOption(), "only repo-host is valid");
cfgOptionValidSet(cfgOptRepoHost, true);
cfgOptionSet(cfgOptRepoHost, cfgSourceParam, varNewStrZ("repo-host"));
cfgOptionValidSet(cfgOptPgHost + 4, true);
cfgOptionSet(cfgOptPgHost + 4, cfgSourceParam, varNewStrZ("pg5-host"));
TEST_ERROR(cfgLoadUpdateOption(), ConfigError, "pg and repo hosts cannot both be configured as remote");
// -------------------------------------------------------------------------------------------------------------------------
StringList *argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("help"));
strLstAdd(argList, strNew("backup"));
strLstAdd(argList, strNew("process-max"));
harnessLogLevelSet(logLevelWarn);
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load help config -- no retention warning");
TEST_RESULT_BOOL(cfgCommandHelp(), true, " command is help");
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--no-log-timestamp"));
strLstAdd(argList, strNew("expire"));
harnessLogLevelSet(logLevelWarn);
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for retention warning");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), set option"
" 'repo1-retention-full' to the maximum.");
TEST_RESULT_BOOL(cfgOptionTest(cfgOptRepoRetentionArchive), false, " repo1-retention-archive not set");
strLstAdd(argList, strNew("--repo1-retention-full=1"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load config no retention warning");
TEST_RESULT_INT(cfgOptionInt(cfgOptRepoRetentionArchive), 1, " repo1-retention-archive set");
// Munge repo-type for coverage. This will go away when there are multiple repos.
cfgOptionSet(cfgOptRepoType, cfgSourceParam, NULL);
TEST_RESULT_VOID(cfgLoadUpdateOption(), "load config no repo-type");
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--no-log-timestamp"));
strLstAdd(argList, strNew("--repo1-retention-archive-type=incr"));
strLstAdd(argList, strNew("expire"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for retention warning");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full'"
" to the maximum.\n"
"P00 WARN: WAL segments will not be expired: option 'repo1-retention-archive-type=incr' but option"
" 'repo1-retention-archive' is not set");
TEST_RESULT_BOOL(cfgOptionTest(cfgOptRepoRetentionArchive), false, " repo1-retention-archive not set");
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--no-log-timestamp"));
strLstAdd(argList, strNew("--repo1-retention-archive-type=diff"));
strLstAdd(argList, strNew("expire"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for retention warning");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), set option"
" 'repo1-retention-full' to the maximum.\n"
"P00 WARN: WAL segments will not be expired: option 'repo1-retention-archive-type=diff' but neither option"
" 'repo1-retention-archive' nor option 'repo1-retention-diff' is set");
TEST_RESULT_BOOL(cfgOptionTest(cfgOptRepoRetentionArchive), false, " repo1-retention-archive not set");
strLstAdd(argList, strNew("--repo1-retention-diff=2"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for retention warning");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), set option"
" 'repo1-retention-full' to the maximum.");
TEST_RESULT_INT(cfgOptionInt(cfgOptRepoRetentionArchive), 2, " repo1-retention-archive set to retention-diff");
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--no-log-timestamp"));
strLstAdd(argList, strNew("--repo1-retention-archive-type=diff"));
strLstAdd(argList, strNew("--repo1-retention-archive=3"));
strLstAdd(argList, strNew("--repo1-retention-full=1"));
strLstAdd(argList, strNew("expire"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for retention warning");
harnessLogResult(
"P00 WARN: option 'repo1-retention-diff' is not set for 'repo1-retention-archive-type=diff'\n"
" HINT: to retain differential backups indefinitely (without warning), set option 'repo1-retention-diff'"
" to the maximum.");
// -------------------------------------------------------------------------------------------------------------------------
setenv("PGBACKREST_REPO1_S3_KEY", "mykey", true);
setenv("PGBACKREST_REPO1_S3_KEY_SECRET", "mysecretkey", true);
// Invalid bucket name with verification enabled fails
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--repo1-type=s3"));
strLstAdd(argList, strNew("--repo1-s3-bucket=bogus.bucket"));
strLstAdd(argList, strNew("--repo1-s3-region=region"));
strLstAdd(argList, strNew("--repo1-s3-endpoint=endpoint"));
strLstAdd(argList, strNew("--repo1-path=/repo"));
strLstAdd(argList, strNew("archive-get"));
TEST_ERROR(
harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), OptionInvalidValueError,
"'bogus.bucket' is not valid for option 'repo1-s3-bucket'"
"\nHINT: RFC-2818 forbids dots in wildcard matches"
"\nHINT: TLS/SSL verification cannot proceed with this bucket name"
"\nHINT: remove dots from the bucket name");
// Invalid bucket name with verification disabled succeeds
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--repo1-type=s3"));
strLstAdd(argList, strNew("--repo1-s3-bucket=bogus.bucket"));
strLstAdd(argList, strNew("--repo1-s3-region=region"));
strLstAdd(argList, strNew("--repo1-s3-endpoint=endpoint"));
strLstAdd(argList, strNew("--no-repo1-s3-verify-ssl"));
strLstAdd(argList, strNew("--repo1-path=/repo"));
strLstAdd(argList, strNew("archive-get"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "invalid bucket with no verification");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoS3Bucket)), "bogus.bucket", " check bucket value");
// Valid bucket name
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--repo1-type=s3"));
strLstAdd(argList, strNew("--repo1-s3-bucket=cool-bucket"));
strLstAdd(argList, strNew("--repo1-s3-region=region"));
strLstAdd(argList, strNew("--repo1-s3-endpoint=endpoint"));
strLstAdd(argList, strNew("--repo1-path=/repo"));
strLstAdd(argList, strNew("archive-get"));
TEST_RESULT_VOID(harnessCfgLoad(strLstSize(argList), strLstPtr(argList)), "valid bucket name");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoS3Bucket)), "cool-bucket", " check bucket value");
unsetenv("PGBACKREST_REPO1_S3_KEY");
unsetenv("PGBACKREST_REPO1_S3_KEY_SECRET");
}
// *****************************************************************************************************************************
if (testBegin("cfgLoadLogFile()"))
{
cfgInit();
cfgOptionValidSet(cfgOptLogLevelFile, true);
cfgOptionSet(cfgOptLogLevelFile, cfgSourceParam, varNewStrZ("detail"));
// On the error case is tested here, success is tested in cfgLoad()
TEST_RESULT_VOID(cfgLoadLogFile(strNew("/BOGUS")), "attempt to open bogus log file");
TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptLogLevelFile)), "off", "log-level-file should now be off");
}
// *****************************************************************************************************************************
if (testBegin("cfgLoad()"))
{
// Command does not have umask
// -------------------------------------------------------------------------------------------------------------------------
StringList *argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("info"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "load config and don't set umask");
// Set a distinct umask value and test that the umask is reset by configLoad since default for neutral-umask=y
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--log-level-console=off"));
strLstAdd(argList, strNew("--log-level-stderr=off"));
strLstAdd(argList, strNew("--log-level-file=off"));
strLstAdd(argList, strNew("archive-get"));
umask(0111);
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for neutral-umask");
TEST_RESULT_INT(umask(0111), 0000, " umask was reset");
// Set a distinct umask value and test that the umask is not reset by configLoad with option --no-neutral-umask
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--no-neutral-umask"));
strLstAdd(argList, strNew("--log-level-console=off"));
strLstAdd(argList, strNew("--log-level-stderr=off"));
strLstAdd(argList, strNew("--log-level-file=off"));
strLstAdd(argList, strNew("archive-get"));
umask(0111);
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "load config for no-neutral-umask");
TEST_RESULT_INT(umask(0), 0111, " umask was not reset");
// No command
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "no command");
// Help command only
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("help"));
ioBufferSizeSet(333);
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "help command");
TEST_RESULT_SIZE(ioBufferSize(), 333, "buffer size not updated by help command");
// Help command for backup
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("help"));
strLstAdd(argList, strNew("backup"));
strLstAdd(argList, strNew("--log-level-console=off"));
strLstAdd(argList, strNew("--log-level-stderr=off"));
strLstAdd(argList, strNew("--log-level-file=off"));
strLstAdd(argList, strNew("--repo1-retention-full=2"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "help command for backup");
TEST_RESULT_SIZE(ioBufferSize(), 4 * 1024 * 1024, "buffer size set to option default");
// Command takes lock and opens log file
// -------------------------------------------------------------------------------------------------------------------------
struct stat statLog;
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--pg1-path=/path"));
strLstAdd(argList, strNew("--repo1-retention-full=1"));
strLstAdd(argList, strNewFmt("--log-path=%s", testPath()));
strLstAdd(argList, strNew("--log-level-console=off"));
strLstAdd(argList, strNew("--log-level-stderr=off"));
strLstAdd(argList, strNew("--log-level-file=warn"));
strLstAdd(argList, strNew("backup"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "lock and open log file");
TEST_RESULT_INT(lstat(strPtr(strNewFmt("%s/db-backup.log", testPath())), &statLog), 0, " check log file exists");
// Local command opens log file with special filename
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--command=backup"));
strLstAdd(argList, strNewFmt("--log-path=%s", testPath()));
strLstAdd(argList, strNew("--process=1"));
strLstAdd(argList, strNew("--host-id=1"));
strLstAdd(argList, strNew("--type=backup"));
strLstAdd(argList, strNew("--log-level-file=warn"));
strLstAdd(argList, strNew("local"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "open log file");
TEST_RESULT_INT(
lstat(strPtr(strNewFmt("%s/db-backup-local-001.log", testPath())), &statLog), 0, " check log file exists");
// Remote command opens log file with special filename
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--command=backup"));
strLstAdd(argList, strNewFmt("--log-path=%s", testPath()));
strLstAdd(argList, strNew("--type=backup"));
strLstAdd(argList, strNew("--log-level-file=warn"));
strLstAdd(argList, strNew("--process=0"));
strLstAdd(argList, strNew("remote"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "open log file");
TEST_RESULT_INT(
lstat(strPtr(strNewFmt("%s/all-backup-remote-000.log", testPath())), &statLog), 0, " check log file exists");
}
FUNCTION_HARNESS_RESULT_VOID();
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("strNew(), strNewBuf(), strNewN(), strEmpty(), strPtr(), strSize(), and strFree()"))
{
// We don't want this struct to grow since there are generally a lot of strings, so make sure it doesn't grow without us
// knowing about it
TEST_RESULT_UINT(sizeof(StringConst), TEST_64BIT() ? 16 : 12, "check StringConst struct size");
// Test the size macro
TEST_RESULT_VOID(CHECK_SIZE(555), "valid size");
TEST_ERROR(CHECK_SIZE(STRING_SIZE_MAX + 1), AssertError, "string size must be <= 1073741824 bytes");
String *string = strNew("static string");
TEST_RESULT_STR(strPtr(string), "static string", "new with static string");
TEST_RESULT_INT(strSize(string), 13, "check size");
TEST_RESULT_BOOL(strEmpty(string), false, "is not empty");
TEST_RESULT_INT(strlen(strPtr(string)), 13, "check size with strlen()");
TEST_RESULT_CHAR(strPtr(string)[2], 'a', "check character");
TEST_RESULT_VOID(strFree(string), "free string");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR(strPtr(strNewN("testmorestring", 4)), "test", "new string with size limit");
// -------------------------------------------------------------------------------------------------------------------------
Buffer *buffer = bufNew(8);
memcpy(bufPtr(buffer), "12345678", 8);
bufUsedSet(buffer, 8);
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "12345678", "new string from buffer");
// -------------------------------------------------------------------------------------------------------------------------
string = strNewFmt("formatted %s %04d", "string", 1);
TEST_RESULT_STR(strPtr(string), "formatted string 0001", "new with formatted string");
TEST_RESULT_PTR(strPtr(NULL), NULL, "null string pointer");
TEST_RESULT_VOID(strFree(string), "free string");
TEST_RESULT_VOID(strFree(NULL), "free null string");
}
// *****************************************************************************************************************************
if (testBegin("STRING_STATIC()"))
{
TEST_RESULT_STR(strPtr(TEST_STRING), "a very interesting string!", "check static string");
TEST_RESULT_STR(strPtr(strSubN(TEST_STRING, 0, 6)), "a very", "read-only strSub() works");
}
// *****************************************************************************************************************************
if (testBegin("strBase() and strPath()"))
{
TEST_RESULT_STR(strPtr(strBase(STRDEF(""))), "", "empty string");
TEST_RESULT_STR(strPtr(strBase(STRDEF("/"))), "", "/ only");
TEST_RESULT_STR(strPtr(strBase(STRDEF("/file"))), "file", "root file");
TEST_RESULT_STR(strPtr(strBase(STRDEF("/dir1/dir2/file"))), "file", "subdirectory file");
TEST_RESULT_STR(strPtr(strPath(STRDEF(""))), "", "empty string");
TEST_RESULT_STR(strPtr(strPath(STRDEF("/"))), "/", "/ only");
TEST_RESULT_STR(strPtr(strPath(STRDEF("/file"))), "/", "root path");
TEST_RESULT_STR(strPtr(strPath(STRDEF("/dir1/dir2/file"))), "/dir1/dir2", "subdirectory file");
}
// *****************************************************************************************************************************
if (testBegin("strCat(), strCatChr(), and strCatFmt()"))
{
String *string = strNew("XXXX");
String *string2 = strNew("ZZZZ");
TEST_RESULT_STR(strPtr(strCat(string, "YYYY")), "XXXXYYYY", "cat string");
TEST_RESULT_SIZE(string->extra, 4, "check extra");
TEST_RESULT_STR(strPtr(strCatFmt(string, "%05d", 777)), "XXXXYYYY00777", "cat formatted string");
TEST_RESULT_SIZE(string->extra, 6, "check extra");
TEST_RESULT_STR(strPtr(strCatChr(string, '!')), "XXXXYYYY00777!", "cat chr");
TEST_RESULT_SIZE(string->extra, 5, "check extra");
TEST_RESULT_STR(strPtr(string2), "ZZZZ", "check unaltered string");
}
// *****************************************************************************************************************************
if (testBegin("strDup()"))
{
const String *string = STRDEF("duplicated string");
String *stringDup = strDup(string);
TEST_RESULT_STR(strPtr(stringDup), strPtr(string), "duplicated strings match");
TEST_RESULT_PTR(strDup(NULL), NULL, "duplicate null string");
}
// *****************************************************************************************************************************
if (testBegin("strBeginsWith() and strBeginsWithZ()"))
{
TEST_RESULT_BOOL(strBeginsWith(STRDEF(""), STRDEF("aaa")), false, "empty string");
TEST_RESULT_BOOL(strBeginsWith(STRDEF("astring"), STRDEF("")), true, "empty begins with");
TEST_RESULT_BOOL(strBeginsWithZ(STRDEF("astring"), "astr"), true, "partial begins with");
TEST_RESULT_BOOL(strBeginsWithZ(STRDEF("astring"), "astring"), true, "equal strings");
}
// *****************************************************************************************************************************
if (testBegin("strEndsWith() and strEndsWithZ()"))
{
TEST_RESULT_BOOL(strEndsWith(STRDEF(""), STRDEF(".doc")), false, "empty string");
TEST_RESULT_BOOL(strEndsWith(STRDEF("astring"), STRDEF("")), true, "empty ends with");
TEST_RESULT_BOOL(strEndsWithZ(STRDEF("astring"), "ing"), true, "partial ends with");
TEST_RESULT_BOOL(strEndsWithZ(STRDEF("astring"), "astring"), true, "equal strings");
}
// *****************************************************************************************************************************
if (testBegin("strEq(), strEqZ(), strCmp(), strCmpZ()"))
{
TEST_RESULT_BOOL(strEq(STRDEF("equalstring"), STRDEF("equalstring")), true, "strings equal");
TEST_RESULT_BOOL(strEq(STRDEF("astring"), STRDEF("anotherstring")), false, "strings not equal");
TEST_RESULT_BOOL(strEq(STRDEF("astring"), STRDEF("bstring")), false, "equal length strings not equal");
TEST_RESULT_INT(strCmp(STRDEF("equalstring"), STRDEF("equalstring")), 0, "strings equal");
TEST_RESULT_INT(strCmp(STRDEF("a"), STRDEF("b")), -1, "a < b");
TEST_RESULT_INT(strCmp(STRDEF("b"), STRDEF("a")), 1, "b > a");
TEST_RESULT_BOOL(strEqZ(STRDEF("equalstring"), "equalstring"), true, "strings equal");
TEST_RESULT_BOOL(strEqZ(STRDEF("astring"), "anotherstring"), false, "strings not equal");
TEST_RESULT_BOOL(strEqZ(STRDEF("astring"), "bstring"), false, "equal length strings not equal");
TEST_RESULT_INT(strCmpZ(STRDEF("equalstring"), "equalstring"), 0, "strings equal");
TEST_RESULT_INT(strCmpZ(STRDEF("a"), "b"), -1, "a < b");
TEST_RESULT_INT(strCmpZ(STRDEF("b"), "a"), 1, "b > a");
}
// *****************************************************************************************************************************
if (testBegin("strFirstUpper(), strFirstLower(), strUpper(), strLower()"))
{
TEST_RESULT_STR(strPtr(strFirstUpper(strNew(""))), "", "empty first upper");
TEST_RESULT_STR(strPtr(strFirstUpper(strNew("aaa"))), "Aaa", "first upper");
TEST_RESULT_STR(strPtr(strFirstUpper(strNew("Aaa"))), "Aaa", "first already upper");
TEST_RESULT_STR(strPtr(strFirstLower(strNew(""))), "", "empty first lower");
TEST_RESULT_STR(strPtr(strFirstLower(strNew("AAA"))), "aAA", "first lower");
TEST_RESULT_STR(strPtr(strFirstLower(strNew("aAA"))), "aAA", "first already lower");
TEST_RESULT_STR(strPtr(strLower(strNew("K123aBc"))), "k123abc", "all lower");
TEST_RESULT_STR(strPtr(strLower(strNew("k123abc"))), "k123abc", "already lower");
TEST_RESULT_STR(strPtr(strLower(strNew("C"))), "c", "char lower");
TEST_RESULT_STR(strPtr(strLower(strNew(""))), "", "empty lower");
TEST_RESULT_STR(strPtr(strUpper(strNew("K123aBc"))), "K123ABC", "all upper");
TEST_RESULT_STR(strPtr(strUpper(strNew("K123ABC"))), "K123ABC", "already upper");
TEST_RESULT_STR(strPtr(strUpper(strNew("c"))), "C", "char upper");
TEST_RESULT_STR(strPtr(strUpper(strNew(""))), "", "empty upper");
}
// *****************************************************************************************************************************
if (testBegin("strQuote()"))
{
TEST_RESULT_STR(strPtr(strQuote(STRDEF("abcd"), STRDEF("'"))), "'abcd'", "quote string");
}
// *****************************************************************************************************************************
if (testBegin("strReplaceChr()"))
{
TEST_RESULT_STR(strPtr(strReplaceChr(strNew("ABCD"), 'B', 'R')), "ARCD", "replace chr");
}
// *****************************************************************************************************************************
if (testBegin("strSub() and strSubN()"))
{
TEST_RESULT_STR(strPtr(strSub(STRDEF("ABCD"), 2)), "CD", "sub string");
TEST_RESULT_STR(strPtr(strSubN(STRDEF("ABCD"), 1, 2)), "BC", "sub string with length");
}
// *****************************************************************************************************************************
if (testBegin("strTrim()"))
{
TEST_RESULT_STR(strPtr(strTrim(strNew(""))), "", "trim empty");
TEST_RESULT_STR(strPtr(strTrim(strNew("X"))), "X", "no trim (one char)");
TEST_RESULT_STR(strPtr(strTrim(strNew("no-trim"))), "no-trim", "no trim (string)");
TEST_RESULT_STR(strPtr(strTrim(strNew(" \t\r\n"))), "", "all whitespace");
TEST_RESULT_STR(strPtr(strTrim(strNew(" \tbegin-only"))), "begin-only", "trim begin");
TEST_RESULT_STR(strPtr(strTrim(strNew("end-only\t "))), "end-only", "trim end");
TEST_RESULT_STR(strPtr(strTrim(strNew("\n\rboth\r\n"))), "both", "trim both");
TEST_RESULT_STR(strPtr(strTrim(strNew("begin \r\n\tend"))), "begin \r\n\tend", "ignore whitespace in middle");
}
// *****************************************************************************************************************************
if (testBegin("strChr() and strTrunc()"))
{
TEST_RESULT_INT(strChr(STRDEF("abcd"), 'c'), 2, "c found");
TEST_RESULT_INT(strChr(STRDEF("abcd"), 'C'), -1, "capital C not found");
TEST_RESULT_INT(strChr(STRDEF("abcd"), 'i'), -1, "i not found");
TEST_RESULT_INT(strChr(STRDEF(""), 'x'), -1, "empty string - x not found");
String *val = strNew("abcdef");
TEST_ERROR(
strTrunc(val, (int)(strSize(val) + 1)), AssertError,
"assertion 'idx >= 0 && (size_t)idx <= this->size' failed");
TEST_ERROR(strTrunc(val, -1), AssertError, "assertion 'idx >= 0 && (size_t)idx <= this->size' failed");
TEST_RESULT_STR(strPtr(strTrunc(val, strChr(val, 'd'))), "abc", "simple string truncated");
strCat(val, "\r\n to end");
TEST_RESULT_STR(strPtr(strTrunc(val, strChr(val, 'n'))), "abc\r\n to e", "complex string truncated");
TEST_RESULT_STR(strPtr(strTrunc(val, strChr(val, 'a'))), "", "complete string truncated - empty string");
TEST_RESULT_INT(strSize(val), 0, "0 size");
TEST_RESULT_STR(strPtr(strTrunc(val, 0)), "", "test coverage of empty string - no error thrown for index 0");
}
// *****************************************************************************************************************************
if (testBegin("strToLog() and strObjToLog()"))
{
TEST_RESULT_STR(strPtr(strToLog(STRDEF("test"))), "{\"test\"}", "format string");
TEST_RESULT_STR(strPtr(strToLog(NULL)), "null", "format null string");
char buffer[256];
TEST_RESULT_UINT(strObjToLog(NULL, (StrObjToLogFormat)strToLog, buffer, sizeof(buffer)), 4, "format null string");
TEST_RESULT_STR(buffer, "null", "check null string");
TEST_RESULT_UINT(strObjToLog(STRDEF("teststr"), (StrObjToLogFormat)strToLog, buffer, sizeof(buffer)), 11, "format string");
TEST_RESULT_STR(buffer, "{\"teststr\"}", "check string");
}
// *****************************************************************************************************************************
if (testBegin("strSizeFormat()"))
{
TEST_RESULT_STR(strPtr(strSizeFormat(0)), "0B", "zero bytes");
TEST_RESULT_STR(strPtr(strSizeFormat(1023)), "1023B", "1023 bytes");
TEST_RESULT_STR(strPtr(strSizeFormat(1024)), "1KB", "1 KB");
TEST_RESULT_STR(strPtr(strSizeFormat(2200)), "2.1KB", "2.1 KB");
TEST_RESULT_STR(strPtr(strSizeFormat(1048576)), "1MB", "1 MB");
TEST_RESULT_STR(strPtr(strSizeFormat(20162900)), "19.2MB", "19.2 MB");
TEST_RESULT_STR(strPtr(strSizeFormat(1073741824)), "1GB", "1 GB");
TEST_RESULT_STR(strPtr(strSizeFormat(1073741824 + 107374183)), "1.1GB", "1.1 GB");
TEST_RESULT_STR(strPtr(strSizeFormat(UINT64_MAX)), "17179869183GB", "uint64 max");
}
// *****************************************************************************************************************************
if (testBegin("strLstNew(), strLstAdd*(), strLstGet(), strLstMove(), strLstSize(), and strLstFree()"))
{
// Add strings to the list
// -------------------------------------------------------------------------------------------------------------------------
StringList *list = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
list = strLstNew();
for (int listIdx = 0; listIdx <= LIST_INITIAL_SIZE; listIdx++)
{
if (listIdx == 0)
{
TEST_RESULT_PTR(strLstAdd(list, NULL), list, "add null item");
}
else
TEST_RESULT_PTR(strLstAdd(list, strNewFmt("STR%02d", listIdx)), list, "add item %d", listIdx);
}
strLstMove(list, MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_INT(strLstSize(list), 9, "list size");
// Read them back and check values
// -------------------------------------------------------------------------------------------------------------------------
for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)
{
if (listIdx == 0)
{
TEST_RESULT_PTR(strLstGet(list, listIdx), NULL, "check null item");
}
else
TEST_RESULT_STR(strPtr(strLstGet(list, listIdx)), strPtr(strNewFmt("STR%02u", listIdx)), "check item %u", listIdx);
}
TEST_RESULT_VOID(strLstFree(list), "free string list");
TEST_RESULT_VOID(strLstFree(NULL), "free null string list");
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("cfgExecParam()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/db path", testPath()));
strLstAddZ(argList, "--log-subprocess");
strLstAddZ(argList, "--no-config");
strLstAddZ(argList, "--reset-neutral-umask");
strLstAddZ(argList, "--repo-cipher-type=aes-256-cbc");
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Set repo1-cipher-pass to make sure it is not passed on the command line
cfgOptionValidSet(cfgOptRepoCipherPass, true);
cfgOptionSet(cfgOptRepoCipherPass, cfgSourceConfig, varNewStrZ("1234"));
TEST_RESULT_STR(
strPtr(strLstJoin(cfgExecParam(cfgCmdLocal, NULL), "|")),
strPtr(
strNewFmt(
"--no-config|--log-subprocess|--pg1-path=\"%s/db path\"|--repo1-cipher-type=aes-256-cbc|--repo1-path=%s/repo"
"|--stanza=test1|local",
testPath(), testPath())),
"exec archive-get -> local");
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
strLstAddZ(argList, "--db-include=1");
strLstAddZ(argList, "--db-include=2");
strLstAddZ(argList, "--recovery-option=a=b");
strLstAddZ(argList, "--recovery-option=c=d");
strLstAddZ(argList, "restore");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
KeyValue *optionReplace = kvNew();
kvPut(optionReplace, varNewStr(strNew("repo1-path")), varNewStr(strNew("/replace/path")));
kvPut(optionReplace, varNewStr(strNew("stanza")), NULL);
TEST_RESULT_STR(
strPtr(strLstJoin(cfgExecParam(cfgCmdRestore, optionReplace), "|")),
strPtr(
strNewFmt(
"--db-include=1|--db-include=2|--pg1-path=%s/db|--recovery-option=a=b|--recovery-option=c=d"
"|--repo1-path=/replace/path|restore",
testPath())),
"exec restore -> restore");
}
FUNCTION_HARNESS_RESULT_VOID();
}
/***********************************************************************************************************************************
Do cleanup and return result code
***********************************************************************************************************************************/
int
exitSafe(int result, bool error, SignalType signalType)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INT, result);
FUNCTION_LOG_PARAM(BOOL, error);
FUNCTION_LOG_PARAM(ENUM, signalType);
FUNCTION_LOG_END();
// Report error if one was thrown
if (error)
{
// Don't log the error if it has already been logged by Perl
#ifdef HAVE_LIBPERL
if (strcmp(errorMessage(), PERL_EMBED_ERROR) != 0)
{
#endif
LogLevel logLevel = errorCode() == errorTypeCode(&AssertError) ? logLevelAssert : logLevelError;
// Assert errors always output a stack trace
if (logLevel == logLevelAssert)
LOG(logLevel, errorCode(), "%s\nSTACK TRACE:\n%s", errorMessage(), errorStackTrace());
else
{
// Log just the error to non-debug levels
LOG_INTERNAL(logLevel, LOG_LEVEL_MIN, logLevelDetail, 0, errorCode(), errorMessage());
// Log the stack trace debug levels
if (logAny(logLevelDebug))
{
LOG_INTERNAL(
logLevel, logLevelDebug, LOG_LEVEL_MAX, 0, errorCode(), "%s\nSTACK TRACE:\n%s", errorMessage(),
errorStackTrace());
}
}
#ifdef HAVE_LIBPERL
}
#endif
result = errorCode();
}
// Free protocol objects but ignore errors
TRY_BEGIN()
{
protocolFree();
}
TRY_END();
// Free Perl but ignore errors
#ifdef HAVE_LIBPERL
TRY_BEGIN()
{
perlFree(result);
}
TRY_END();
#endif
// Log command end if a command is set
if (cfgCommand() != cfgCmdNone)
{
String *errorMessage = NULL;
// On error generate an error message
if (result != 0)
{
// On process terminate
if (result == errorTypeCode(&TermError))
{
errorMessage = strNew("terminated on signal ");
// Terminate from a child
if (signalType == signalTypeNone)
strCat(errorMessage, "from child process");
// Else terminated directly
else
strCatFmt(errorMessage, "[SIG%s]", exitSignalName(signalType));
}
// Standard error exit message
else if (error)
errorMessage = strNewFmt("aborted with exception [%03d]", result);
}
cmdEnd(result, errorMessage);
}
// Release any locks but ignore errors
TRY_BEGIN()
{
lockRelease(false);
}
TRY_END();
// Return result - caller should immediate pass this result to exit()
FUNCTION_LOG_RETURN(INT, result);
}
