/*********************************************************************************************************************************** Clean the queue and prepare a list of WAL segments that the async process should get ***********************************************************************************************************************************/ static StringList * queueNeed(const String *walSegment, bool found, uint64_t queueSize, size_t walSegmentSize, unsigned int pgVersion) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, walSegment); FUNCTION_LOG_PARAM(BOOL, found); FUNCTION_LOG_PARAM(UINT64, queueSize); FUNCTION_LOG_PARAM(SIZE, walSegmentSize); FUNCTION_LOG_PARAM(UINT, pgVersion); FUNCTION_LOG_END(); ASSERT(walSegment != NULL); StringList *result = strLstNew(); MEM_CONTEXT_TEMP_BEGIN() { // Determine the first WAL segment for the async process to get. If the WAL segment requested by // PostgreSQL was not found then use that. If the segment was found but the queue is not full then // start with the next segment. const String *walSegmentFirst = found ? walSegmentNext(walSegment, walSegmentSize, pgVersion) : walSegment; // Determine how many WAL segments should be in the queue. The queue total must be at least 2 or it doesn't make sense to // have async turned on at all. unsigned int walSegmentQueueTotal = (unsigned int)(queueSize / walSegmentSize); if (walSegmentQueueTotal < 2) walSegmentQueueTotal = 2; // Build the ideal queue -- the WAL segments we want in the queue after the async process has run StringList *idealQueue = walSegmentRange(walSegmentFirst, walSegmentSize, pgVersion, walSegmentQueueTotal); // Get the list of files actually in the queue StringList *actualQueue = strLstSort( storageListP(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .errorOnMissing = true), sortOrderAsc); // Only preserve files that match the ideal queue. error/ok files are deleted so the async process can try again. RegExp *regExpPreserve = regExpNew(strNewFmt("^(%s)$", strPtr(strLstJoin(idealQueue, "|")))); // Build a list of WAL segments that are being kept so we can later make a list of what is needed StringList *keepQueue = strLstNew(); for (unsigned int actualQueueIdx = 0; actualQueueIdx < strLstSize(actualQueue); actualQueueIdx++) { // Get file from actual queue const String *file = strLstGet(actualQueue, actualQueueIdx); // Does this match a file we want to preserve? if (regExpMatch(regExpPreserve, file)) strLstAdd(keepQueue, file); // Else delete it else storageRemoveNP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(file))); } // Generate a list of the WAL that are needed by removing kept WAL from the ideal queue for (unsigned int idealQueueIdx = 0; idealQueueIdx < strLstSize(idealQueue); idealQueueIdx++) { if (!strLstExists(keepQueue, strLstGet(idealQueue, idealQueueIdx))) strLstAdd(result, strLstGet(idealQueue, idealQueueIdx)); } } MEM_CONTEXT_TEMP_END(); FUNCTION_LOG_RETURN(STRING_LIST, result); }
/*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ void testRun(void) { // Create default storage object for testing Storage *storageTest = storagePosixNew( strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); // ***************************************************************************************************************************** if (testBegin("infoNewLoad(), infoFileName(), infoIni()")) { // Initialize test variables //-------------------------------------------------------------------------------------------------------------------------- String *content = NULL; String *fileName = strNewFmt("%s/test.ini", testPath()); String *fileNameCopy = strNewFmt("%s/test.ini.copy", testPath()); Info *info = NULL; content = strNew ( "[backrest]\n" "backrest-checksum=\"1efa53e0611604ad7d833c5547eb60ff716e758c\"\n" "backrest-format=5\n" "backrest-version=\"2.04\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n" ); // Info files missing and at least one is required //-------------------------------------------------------------------------------------------------------------------------- TEST_ERROR_FMT( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FileMissingError, "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING "\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING, testPath(), testPath(), strPtr(strNewFmt("%s/test.ini", testPath())), strPtr(strNewFmt("%s/test.ini.copy", testPath()))); // Only copy exists and one is required //-------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put info.copy to file"); TEST_ASSIGN(info, infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), "load copy file"); TEST_RESULT_PTR(infoCipherPass(info), NULL, " cipherPass is not set"); // Remove the copy and store only the main info file and encrypt it. One is required. //-------------------------------------------------------------------------------------------------------------------------- StorageWrite *infoWrite = storageNewWriteNP(storageLocalWrite(), fileName); ioWriteFilterGroupSet( storageWriteIo(infoWrite), ioFilterGroupAdd( ioFilterGroupNew(), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("12345678"), NULL))); storageRemoveNP(storageLocalWrite(), fileNameCopy); storagePutNP( infoWrite, BUFSTRDEF( "[backrest]\n" "backrest-checksum=\"9d2f6dce339751e1a056187fad67d2834b3d4ab3\"\n" "backrest-format=5\n" "backrest-version=\"2.04\"\n" "\n" "[cipher]\n" "cipher-pass=\"ABCDEFGH\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n")); // Only main info exists and is required Ini *ini = NULL; TEST_ASSIGN(info, infoNewLoad(storageLocal(), fileName, cipherTypeAes256Cbc, strNew("12345678"), &ini), "load file"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("cipher"), strNew("cipher-pass"))), "\"ABCDEFGH\"", " check ini"); TEST_RESULT_STR(strPtr(infoCipherPass(info)), "ABCDEFGH", " cipherPass is set"); // Invalid format //-------------------------------------------------------------------------------------------------------------------------- storageRemoveNP(storageLocalWrite(), fileName); content = strNew ( "[backrest]\n" "backrest-checksum=\"14617b089cb5c9b3224e739bb794e865b9bcdf4b\"\n" "backrest-format=4\n" "backrest-version=\"2.04\"\n" "\n" "[db]\n" "db-catalog-version=201409291\n" "db-control-version=942\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679," "\"db-version\":\"9.4\"}\n" ); // Only main file exists but the backrest-format is invalid TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), BUFSTR(content)), "put invalid br format to file"); TEST_ERROR_FMT( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FormatError, "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FormatError: invalid format in '%s/test.ini', expected 5 but found 4\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING, testPath(), testPath(), testPath(), strPtr(strNewFmt("%s/test.ini.copy", testPath()))); content = strNew ( "[backrest]\n" "backrest-checksum=\"14617b089cb5c9b3224e739bb794e865b9bcdf4b\"\n" "backrest-format=4\n" "backrest-version=\"2.05\"\n" "\n" "[db]\n" "db-catalog-version=201409291\n" "db-control-version=942\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679," "\"db-version\":\"9.4\"}\n" ); TEST_RESULT_VOID( storagePutNP( storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put invalid info to copy file"); TEST_ERROR( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), FileOpenError, strPtr( strNewFmt( "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FormatError: invalid format in '%s/test.ini', expected 5 but found 4\n" "ChecksumError: invalid checksum in '%s/test.ini.copy', expected 'af92308095d6141bcda6b2df6d574f98d1115163'" " but found '14617b089cb5c9b3224e739bb794e865b9bcdf4b'", testPath(), testPath(), testPath(), testPath()))); // Invalid checksum //-------------------------------------------------------------------------------------------------------------------------- storageRemoveNP(storageLocalWrite(), fileName); storageRemoveNP(storageLocalWrite(), fileNameCopy); // change the checksum content = strNew ( "[backrest]\n" "backrest-checksum=\"4306ec205f71417c301e403c4714090e61c8a999\"\n" "backrest-format=5\n" "backrest-version=\"1.23\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6455618988686438683\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-id\":6455618988686438683,\"db-version\":\"9.6\"}\n" "2={\"db-id\":6457457208275135411,\"db-version\":\"9.6\"}\n" ); TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileNameCopy), BUFSTR(content)), "put invalid checksum to copy"); // Empty checksum for main file content = strNew ( "[backrest]\n" "backrest-checksum=\n" "backrest-format=5\n" "backrest-version=\"1.23\"\n" "\n" "[db]\n" "db-id=1\n" "db-system-id=6455618988686438683\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-id\":6455618988686438683,\"db-version\":\"9.6\"}\n" "2={\"db-id\":6457457208275135411,\"db-version\":\"9.6\"}\n" ); TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), BUFSTR(content)), "put empty checksum to file"); // Copy file error TEST_ERROR( infoNewLoad(storageLocal(), fileName, cipherTypeNone, NULL, NULL), ChecksumError, strPtr( strNewFmt( "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "ChecksumError: invalid checksum in '%s/test.ini', expected '4306ec205f71417c301e403c4714090e61c8a736' but" " no checksum found\n" "ChecksumError: invalid checksum in '%s/test.ini.copy', expected '4306ec205f71417c301e403c4714090e61c8a736'" " but found '4306ec205f71417c301e403c4714090e61c8a999'", testPath(), testPath(), testPath(), testPath()))); // Encryption error //-------------------------------------------------------------------------------------------------------------------------- storageRemoveNP(storageLocalWrite(), fileName); TEST_ERROR_FMT( infoNewLoad(storageLocal(), fileName, cipherTypeAes256Cbc, strNew("12345678"), NULL), CryptoError, "unable to load info file '%s/test.ini' or '%s/test.ini.copy':\n" "FileMissingError: " STORAGE_ERROR_READ_MISSING "\n" "CryptoError: '%s/test.ini.copy' cipher header invalid\n" "HINT: Is or was the repo encrypted?", testPath(), testPath(), strPtr(strNewFmt("%s/test.ini", testPath())), testPath()); storageRemoveNP(storageLocalWrite(), fileNameCopy); // infoFree() //-------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID(infoFree(info), "infoFree() - free info memory context"); } // ***************************************************************************************************************************** if (testBegin("infoSave()")) { const String *fileName = strNew("test.info"); const String *cipherPass = strNew("12345"); Ini *ini = iniNew(); iniSet(ini, strNew("section1"), strNew("key1"), strNew("value1")); TEST_RESULT_VOID(infoSave(infoNew(), ini, storageTest, fileName, cipherTypeNone, NULL), "save info"); ini = NULL; TEST_RESULT_VOID(infoNewLoad(storageTest, fileName, cipherTypeNone, NULL, &ini), " reload info"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("section1"), strNew("key1"))), "value1", " check ini"); TEST_RESULT_BOOL(storageExistsNP(storageTest, fileName), true, "check main exists"); TEST_RESULT_BOOL(storageExistsNP(storageTest, strNewFmt("%s" INFO_COPY_EXT, strPtr(fileName))), true, "check main exists"); // Add encryption // ------------------------------------------------------------------------------------------------------------------------- ini = iniNew(); iniSet(ini, strNew("section1"), strNew("key1"), strNew("value4")); Info *info = infoNew(); info->cipherPass = strNew("/badpass"); TEST_RESULT_VOID(infoSave(info, ini, storageTest, fileName, cipherTypeAes256Cbc, cipherPass), "save encrypted info"); ini = NULL; TEST_RESULT_VOID(infoNewLoad(storageTest, fileName, cipherTypeAes256Cbc, cipherPass, &ini), " reload info"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("section1"), strNew("key1"))), "value4", " check ini"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("cipher"), strNew("cipher-pass"))), "\"/badpass\"", " check cipher-pass"); } }