/*********************************************************************************************************************************** Test for the existence of a stop file ***********************************************************************************************************************************/ void lockStopTest(void) { FUNCTION_LOG_VOID(logLevelDebug); MEM_CONTEXT_TEMP_BEGIN() { // Check the current stanza (if any) if (cfgOptionTest(cfgOptStanza)) { if (storageExistsNP(storageLocal(), lockStopFileName(cfgOptionStr(cfgOptStanza)))) THROW_FMT(StopError, "stop file exists for stanza %s", strPtr(cfgOptionStr(cfgOptStanza))); } // Check all stanzas if (storageExistsNP(storageLocal(), lockStopFileName(NULL))) THROW(StopError, "stop file exists for all stanzas"); } MEM_CONTEXT_TEMP_END(); FUNCTION_LOG_RETURN_VOID(); }
/*********************************************************************************************************************************** Get an archive file from the repository (WAL segment, history file, etc.) ***********************************************************************************************************************************/ int cmdArchiveGet(void) { FUNCTION_LOG_VOID(logLevelDebug); // Set the result assuming the archive file will not be found int result = 1; MEM_CONTEXT_TEMP_BEGIN() { // Check the parameters const StringList *commandParam = cfgCommandParam(); if (strLstSize(commandParam) != 2) { if (strLstSize(commandParam) == 0) THROW(ParamRequiredError, "WAL segment to get required"); if (strLstSize(commandParam) == 1) THROW(ParamRequiredError, "path to copy WAL segment required"); THROW(ParamInvalidError, "extra parameters found"); } // Get the segment name String *walSegment = strBase(strLstGet(commandParam, 0)); // Destination is wherever we were told to move the WAL segment const String *walDestination = walPath(strLstGet(commandParam, 1), cfgOptionStr(cfgOptPgPath), STR(cfgCommandName(cfgCommand()))); // Async get can only be performed on WAL segments, history or other files must use synchronous mode if (cfgOptionBool(cfgOptArchiveAsync) && walIsSegment(walSegment)) { bool found = false; // Has the WAL segment been found yet? bool queueFull = false; // Is the queue half or more full? bool forked = false; // Has the async process been forked yet? bool confessOnError = false; // Should we confess errors? // Loop and wait for the WAL segment to be pushed Wait *wait = waitNew((TimeMSec)(cfgOptionDbl(cfgOptArchiveTimeout) * MSEC_PER_SEC)); do { // Check for errors or missing files. For archive-get ok indicates that the process succeeded but there is no WAL // file to download. if (archiveAsyncStatus(archiveModeGet, walSegment, confessOnError)) { storageRemoveP( storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s" STATUS_EXT_OK, strPtr(walSegment)), .errorOnMissing = true); break; } // Check if the WAL segment is already in the queue found = storageExistsNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment))); // If found then move the WAL segment to the destination directory if (found) { // Source is the WAL segment in the spool queue StorageFileRead *source = storageNewReadNP( storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment))); // A move will be attempted but if the spool queue and the WAL path are on different file systems then a copy // will be performed instead. // // It looks scary that we are disabling syncs and atomicity (in case we need to copy intead of move) but this // is safe because if the system crashes Postgres will not try to reuse a restored WAL segment but will instead // request it again using the restore_command. In the case of a move this hardly matters since path syncs are // cheap but if a copy is required we could save a lot of writes. StorageFileWrite *destination = storageNewWriteP( storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true, .noAtomic = true); // Move (or copy if required) the file storageMoveNP(storageSpoolWrite(), source, destination); // Return success result = 0; // Get a list of WAL segments left in the queue StringList *queue = storageListP( storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .expression = WAL_SEGMENT_REGEXP_STR); if (strLstSize(queue) > 0) { // Get size of the WAL segment uint64_t walSegmentSize = storageInfoNP(storageLocal(), walDestination).size; // Use WAL segment size to estimate queue size and determine if the async process should be launched queueFull = strLstSize(queue) * walSegmentSize > cfgOptionUInt64(cfgOptArchiveGetQueueMax) / 2; } } // If the WAL segment has not already been found then start the async process to get it. There's no point in // forking the async process off more than once so track that as well. Use an archive lock to prevent forking if // the async process was launched by another process. if (!forked && (!found || !queueFull) && lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, false)) { // Get control info PgControl pgControl = pgControlFromFile(cfgOptionStr(cfgOptPgPath)); // Create the queue storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR); // The async process should not output on the console at all KeyValue *optionReplace = kvNew(); kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_CONSOLE_STR), VARSTRDEF("off")); kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_STDERR_STR), VARSTRDEF("off")); // Generate command options StringList *commandExec = cfgExecParam(cfgCmdArchiveGetAsync, optionReplace); strLstInsert(commandExec, 0, cfgExe()); // Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed() // will return the list of WAL needed to fill the queue and this will be passed to the async process. const StringList *queue = queueNeed( walSegment, found, cfgOptionUInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize, pgControl.version); for (unsigned int queueIdx = 0; queueIdx < strLstSize(queue); queueIdx++) strLstAdd(commandExec, strLstGet(queue, queueIdx)); // Release the lock so the child process can acquire it lockRelease(true); // Fork off the async process if (forkSafe() == 0) { // Disable logging and close log file logClose(); // Detach from parent process forkDetach(); // Execute the binary. This statement will not return if it is successful. THROW_ON_SYS_ERROR( execvp(strPtr(cfgExe()), (char ** const)strLstPtr(commandExec)) == -1, ExecuteError, "unable to execute '" CFGCMD_ARCHIVE_GET_ASYNC "'"); } // Mark the async process as forked so it doesn't get forked again. A single run of the async process should be // enough to do the job, running it again won't help anything. forked = true; } // Exit loop if WAL was found if (found) break; // Now that the async process has been launched, confess any errors that are found confessOnError = true; } while (waitMore(wait)); } // Else perform synchronous get else {
/*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ void testRun(void) { // Create default storage object for testing Storage *storageTest = storagePosixNew( strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); // ***************************************************************************************************************************** if (testBegin("infoNewLoad(), infoFileName(), infoIni()")) { // Initialize test variables //-------------------------------------------------------------------------------------------------------------------------- String *content = NULL; String *fileName = strNewFmt("%s/test.ini", testPath()); String *fileNameCopy = strNewFmt("%s/test.ini.copy", testPath()); Info *info = NULL; content = strNew ( "[backrest]\n" "backrest-checksum=\"1efa53e0611604ad7d833c5547eb60ff716e758c\"\n" "backrest-format=5\n" "backrest-version=\"2.04\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n" ); // Info files missing and at least one is required //-------------------------------------------------------------------------------------------------------------------------- TEST_ERROR_FMT( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FileMissingError, "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING "\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING, testPath(), testPath(), strPtr(strNewFmt("%s/test.ini", testPath())), strPtr(strNewFmt("%s/test.ini.copy", testPath()))); // Only copy exists and one is required //-------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put info.copy to file"); TEST_ASSIGN(info, infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), "load copy file"); TEST_RESULT_PTR(infoCipherPass(info), NULL, " cipherPass is not set"); // Remove the copy and store only the main info file and encrypt it. One is required. //-------------------------------------------------------------------------------------------------------------------------- StorageWrite *infoWrite = storageNewWriteNP(storageLocalWrite(), fileName); ioWriteFilterGroupSet( storageWriteIo(infoWrite), ioFilterGroupAdd( ioFilterGroupNew(), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("12345678"), NULL))); storageRemoveNP(storageLocalWrite(), fileNameCopy); storagePutNP( infoWrite, BUFSTRDEF( "[backrest]\n" "backrest-checksum=\"9d2f6dce339751e1a056187fad67d2834b3d4ab3\"\n" "backrest-format=5\n" "backrest-version=\"2.04\"\n" "\n" "[cipher]\n" "cipher-pass=\"ABCDEFGH\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n")); // Only main info exists and is required Ini *ini = NULL; TEST_ASSIGN(info, infoNewLoad(storageLocal(), fileName, cipherTypeAes256Cbc, strNew("12345678"), &ini), "load file"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("cipher"), strNew("cipher-pass"))), "\"ABCDEFGH\"", " check ini"); TEST_RESULT_STR(strPtr(infoCipherPass(info)), "ABCDEFGH", " cipherPass is set"); // Invalid format //-------------------------------------------------------------------------------------------------------------------------- storageRemoveNP(storageLocalWrite(), fileName); content = strNew ( "[backrest]\n" "backrest-checksum=\"14617b089cb5c9b3224e739bb794e865b9bcdf4b\"\n" "backrest-format=4\n" "backrest-version=\"2.04\"\n" "\n" "[db]\n" "db-catalog-version=201409291\n" "db-control-version=942\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679," "\"db-version\":\"9.4\"}\n" ); // Only main file exists but the backrest-format is invalid TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), BUFSTR(content)), "put invalid br format to file"); TEST_ERROR_FMT( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FormatError, "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FormatError: invalid format in '%s/test.ini', expected 5 but found 4\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING, testPath(), testPath(), testPath(), strPtr(strNewFmt("%s/test.ini.copy", testPath()))); content = strNew ( "[backrest]\n" "backrest-checksum=\"14617b089cb5c9b3224e739bb794e865b9bcdf4b\"\n" "backrest-format=4\n" "backrest-version=\"2.05\"\n" "\n" "[db]\n" "db-catalog-version=201409291\n" "db-control-version=942\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679," "\"db-version\":\"9.4\"}\n" ); TEST_RESULT_VOID( storagePutNP( storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put invalid info to copy file"); TEST_ERROR( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FileOpenError, strPtr( strNewFmt( "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FormatError: invalid format in '%s/test.ini', expected 5 but found 4\n" "ChecksumError: invalid checksum in '%s/test.ini.copy', expected 'af92308095d6141bcda6b2df6d574f98d1115163'" " but found '14617b089cb5c9b3224e739bb794e865b9bcdf4b'", testPath(), testPath(), testPath(), testPath()))); // Invalid checksum //-------------------------------------------------------------------------------------------------------------------------- storageRemoveNP(storageLocalWrite(), fileName); storageRemoveNP(storageLocalWrite(), fileNameCopy); // change the checksum content = strNew ( "[backrest]\n" "backrest-checksum=\"4306ec205f71417c301e403c4714090e61c8a999\"\n" "backrest-format=5\n" "backrest-version=\"1.23\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6455618988686438683\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-id\":6455618988686438683,\"db-version\":\"9.6\"}\n" "2={\"db-id\":6457457208275135411,\"db-version\":\"9.6\"}\n" ); TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put invalid checksum to copy"); // Empty checksum for main file content = strNew ( "[backrest]\n" "backrest-checksum=\n" "backrest-format=5\n" "backrest-version=\"1.23\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6455618988686438683\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-id\":6455618988686438683,\"db-version\":\"9.6\"}\n" "2={\"db-id\":6457457208275135411,\"db-version\":\"9.6\"}\n" ); TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), BUFSTR(content)), "put empty checksum to file"); // Copy file error TEST_ERROR( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), ChecksumError, strPtr( strNewFmt( "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "ChecksumError: invalid checksum in '%s/test.ini', expected '4306ec205f71417c301e403c4714090e61c8a736' but" " no checksum found\n" "ChecksumError: invalid checksum in '%s/test.ini.copy', expected '4306ec205f71417c301e403c4714090e61c8a736'" " but found '4306ec205f71417c301e403c4714090e61c8a999'", testPath(), testPath(), testPath(), testPath()))); // Encryption error //-------------------------------------------------------------------------------------------------------------------------- storageRemoveNP(storageLocalWrite(), fileName); TEST_ERROR_FMT( infoNewLoad(storageLocal(), fileName, cipherTypeAes256Cbc, strNew("12345678"), NULL), CryptoError, "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING "\n" "CryptoError: '%s/test.ini.copy' cipher header invalid\n" "HINT: Is or was the repo encrypted?", testPath(), testPath(), strPtr(strNewFmt("%s/test.ini", testPath())), testPath()); storageRemoveNP(storageLocalWrite(), fileNameCopy); // infoFree() //-------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID(infoFree(info), "infoFree() - free info memory context"); } // ***************************************************************************************************************************** if (testBegin("infoSave()")) { const String *fileName = strNew("test.info"); const String *cipherPass = strNew("12345"); Ini *ini = iniNew(); iniSet(ini, strNew("section1"), strNew("key1"), strNew("value1")); TEST_RESULT_VOID(infoSave(infoNew(), ini, storageTest, fileName, cipherTypeNone, NULL), "save info"); ini = NULL; TEST_RESULT_VOID(infoNewLoad(storageTest, fileName, cipherTypeNone, NULL, &ini), " reload info"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("section1"), strNew("key1"))), "value1", " check ini"); TEST_RESULT_BOOL(storageExistsNP(storageTest, fileName), true, "check main exists"); TEST_RESULT_BOOL(storageExistsNP(storageTest, strNewFmt("%s" INFO_COPY_EXT, strPtr(fileName))), true, "check main exists"); // Add encryption // ------------------------------------------------------------------------------------------------------------------------- ini = iniNew(); iniSet(ini, strNew("section1"), strNew("key1"), strNew("value4")); Info *info = infoNew(); info->cipherPass = strNew("/badpass"); TEST_RESULT_VOID(infoSave(info, ini, storageTest, fileName, cipherTypeAes256Cbc, cipherPass), "save encrypted info"); ini = NULL; TEST_RESULT_VOID(infoNewLoad(storageTest, fileName, cipherTypeAes256Cbc, cipherPass, &ini), " reload info"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("section1"), strNew("key1"))), "value4", " check ini"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("cipher"), strNew("cipher-pass"))), "\"/badpass\"", " check cipher-pass"); } }