/*********************************************************************************************************************************** Get an archive file from the repository (WAL segment, history file, etc.) ***********************************************************************************************************************************/ int cmdArchiveGet(void) { FUNCTION_LOG_VOID(logLevelDebug); // Set the result assuming the archive file will not be found int result = 1; MEM_CONTEXT_TEMP_BEGIN() { // Check the parameters const StringList *commandParam = cfgCommandParam(); if (strLstSize(commandParam) != 2) { if (strLstSize(commandParam) == 0) THROW(ParamRequiredError, "WAL segment to get required"); if (strLstSize(commandParam) == 1) THROW(ParamRequiredError, "path to copy WAL segment required"); THROW(ParamInvalidError, "extra parameters found"); } // Get the segment name String *walSegment = strBase(strLstGet(commandParam, 0)); // Destination is wherever we were told to move the WAL segment const String *walDestination = walPath(strLstGet(commandParam, 1), cfgOptionStr(cfgOptPgPath), STR(cfgCommandName(cfgCommand()))); // Async get can only be performed on WAL segments, history or other files must use synchronous mode if (cfgOptionBool(cfgOptArchiveAsync) && walIsSegment(walSegment)) { bool found = false; // Has the WAL segment been found yet? bool queueFull = false; // Is the queue half or more full? bool forked = false; // Has the async process been forked yet? bool confessOnError = false; // Should we confess errors? // Loop and wait for the WAL segment to be pushed Wait *wait = waitNew((TimeMSec)(cfgOptionDbl(cfgOptArchiveTimeout) * MSEC_PER_SEC)); do { // Check for errors or missing files. For archive-get ok indicates that the process succeeded but there is no WAL // file to download. if (archiveAsyncStatus(archiveModeGet, walSegment, confessOnError)) { storageRemoveP( storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s" STATUS_EXT_OK, strPtr(walSegment)), .errorOnMissing = true); break; } // Check if the WAL segment is already in the queue found = storageExistsNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment))); // If found then move the WAL segment to the destination directory if (found) { // Source is the WAL segment in the spool queue StorageFileRead *source = storageNewReadNP( storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strPtr(walSegment))); // A move will be attempted but if the spool queue and the WAL path are on different file systems then a copy // will be performed instead. // // It looks scary that we are disabling syncs and atomicity (in case we need to copy intead of move) but this // is safe because if the system crashes Postgres will not try to reuse a restored WAL segment but will instead // request it again using the restore_command. In the case of a move this hardly matters since path syncs are // cheap but if a copy is required we could save a lot of writes. StorageFileWrite *destination = storageNewWriteP( storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true, .noAtomic = true); // Move (or copy if required) the file storageMoveNP(storageSpoolWrite(), source, destination); // Return success result = 0; // Get a list of WAL segments left in the queue StringList *queue = storageListP( storageSpool(), STORAGE_SPOOL_ARCHIVE_IN_STR, .expression = WAL_SEGMENT_REGEXP_STR); if (strLstSize(queue) > 0) { // Get size of the WAL segment uint64_t walSegmentSize = storageInfoNP(storageLocal(), walDestination).size; // Use WAL segment size to estimate queue size and determine if the async process should be launched queueFull = strLstSize(queue) * walSegmentSize > cfgOptionUInt64(cfgOptArchiveGetQueueMax) / 2; } } // If the WAL segment has not already been found then start the async process to get it. There's no point in // forking the async process off more than once so track that as well. Use an archive lock to prevent forking if // the async process was launched by another process. if (!forked && (!found || !queueFull) && lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, false)) { // Get control info PgControl pgControl = pgControlFromFile(cfgOptionStr(cfgOptPgPath)); // Create the queue storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR); // The async process should not output on the console at all KeyValue *optionReplace = kvNew(); kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_CONSOLE_STR), VARSTRDEF("off")); kvPut(optionReplace, VARSTR(CFGOPT_LOG_LEVEL_STDERR_STR), VARSTRDEF("off")); // Generate command options StringList *commandExec = cfgExecParam(cfgCmdArchiveGetAsync, optionReplace); strLstInsert(commandExec, 0, cfgExe()); // Clean the current queue using the list of WAL that we ideally want in the queue. queueNeed() // will return the list of WAL needed to fill the queue and this will be passed to the async process. const StringList *queue = queueNeed( walSegment, found, cfgOptionUInt64(cfgOptArchiveGetQueueMax), pgControl.walSegmentSize, pgControl.version); for (unsigned int queueIdx = 0; queueIdx < strLstSize(queue); queueIdx++) strLstAdd(commandExec, strLstGet(queue, queueIdx)); // Release the lock so the child process can acquire it lockRelease(true); // Fork off the async process if (forkSafe() == 0) { // Disable logging and close log file logClose(); // Detach from parent process forkDetach(); // Execute the binary. This statement will not return if it is successful. THROW_ON_SYS_ERROR( execvp(strPtr(cfgExe()), (char ** const)strLstPtr(commandExec)) == -1, ExecuteError, "unable to execute '" CFGCMD_ARCHIVE_GET_ASYNC "'"); } // Mark the async process as forked so it doesn't get forked again. A single run of the async process should be // enough to do the job, running it again won't help anything. forked = true; } // Exit loop if WAL was found if (found) break; // Now that the async process has been launched, confess any errors that are found confessOnError = true; } while (waitMore(wait)); } // Else perform synchronous get else {
/*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ void testRun(void) { // ***************************************************************************************************************************** if (testBegin("infoPgNewLoad(), infoPgFree(), infoPgDataCurrent(), infoPgDataToLog(), infoPgAdd(), infoPgIni()")) { String *content = NULL; String *fileName = strNewFmt("%s/test.ini", testPath()); String *fileName2 = strNewFmt("%s/test2.ini", testPath()); // Archive info //-------------------------------------------------------------------------------------------------------------------------- content = strNew ( "[db]\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-id\":6569239123849665679,\"db-version\":\"9.4\"}\n" ); TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file"); InfoPg *infoPg = NULL; Ini *ini = NULL; TEST_ASSIGN( infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgArchive, cipherTypeNone, NULL, &ini), "load file"); TEST_RESULT_STR(strPtr(iniGet(ini, strNew("db"), strNew("db-id"))), "1", " check ini"); TEST_RESULT_INT(lstSize(infoPg->history), 1, " history record added"); InfoPgData infoPgData = infoPgDataCurrent(infoPg); TEST_RESULT_INT(infoPgData.id, 1, " id set"); TEST_RESULT_INT(infoPgData.version, PG_VERSION_94, " version set"); TEST_RESULT_INT(infoPgData.systemId, 6569239123849665679, " system-id set"); TEST_RESULT_INT(infoPgData.catalogVersion, 0, " catalog-version not set"); TEST_RESULT_INT(infoPgData.controlVersion, 0, " control-version not set"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " check pg data total"); TEST_RESULT_STR(strPtr(infoPgArchiveId(infoPg, 0)), "9.4-1", " check pg archive id"); TEST_RESULT_PTR(infoPgCipherPass(infoPg), NULL, " no cipher passphrase"); // Backup info //-------------------------------------------------------------------------------------------------------------------------- content = strNew ( "[db]\n" "db-catalog-version=201409291\n" "db-control-version=942\n" "db-id=1\n" "db-system-id=6569239123849665679\n" "db-version=\"9.4\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679," "\"db-version\":\"9.4\"}\n" ); TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file"); TEST_ASSIGN( infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgBackup, cipherTypeNone, NULL, NULL), "load file"); TEST_RESULT_INT(lstSize(infoPg->history), 1, " history record added"); infoPgData = infoPgDataCurrent(infoPg); TEST_RESULT_INT(infoPgData.id, 1, " id set"); TEST_RESULT_INT(infoPgData.version, PG_VERSION_94, " version set"); TEST_RESULT_INT(infoPgData.systemId, 6569239123849665679, " system-id set"); TEST_RESULT_INT(infoPgData.catalogVersion, 201409291, " catalog-version set"); TEST_RESULT_INT(infoPgData.controlVersion, 942, " control-version set"); // Manifest info //-------------------------------------------------------------------------------------------------------------------------- content = strNew ( "[db]\n" "db-catalog-version=201510051\n" "db-control-version=942\n" "db-id=2\n" "db-system-id=6365925855999999999\n" "db-version=\"9.5\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6569239123849665679," "\"db-version\":\"9.4\"}\n" "2={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6365925855999999999," "\"db-version\":\"9.5\"}\n" ); // Put the file and load it TEST_RESULT_VOID( storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), harnessInfoChecksum(content)), "put info to file"); TEST_ASSIGN( infoPg, infoPgNewLoad(storageLocal(), fileName, infoPgManifest, cipherTypeNone, NULL, NULL), "load file"); // Save the file and verify it ini = iniNew(); TEST_RESULT_VOID(infoPgSave(infoPg, ini, storageLocalWrite(), fileName2, cipherTypeNone, NULL), "save file"); TEST_RESULT_BOOL( bufEq( storageGetNP(storageNewReadNP(storageLocal(), fileName)), storageGetNP(storageNewReadNP(storageLocal(), fileName2))), true, "files are equal"); TEST_RESULT_INT(lstSize(infoPg->history), 2, "history record added"); infoPgData = infoPgDataCurrent(infoPg); TEST_RESULT_INT(infoPgData.id, 2, " id set"); TEST_RESULT_INT(infoPgData.version, PG_VERSION_95, " version set"); TEST_RESULT_INT(infoPgData.systemId, 6365925855999999999, " system-id set"); TEST_RESULT_INT(infoPgData.catalogVersion, 201510051, " catalog-version set"); TEST_RESULT_INT(infoPgData.controlVersion, 942, " control-version set"); // infoPgAdd //-------------------------------------------------------------------------------------------------------------------------- infoPgData.id = 3; infoPgData.version = PG_VERSION_96; infoPgData.systemId = 6399999999999999999; infoPgData.catalogVersion = 201608131; infoPgData.controlVersion = 960; TEST_RESULT_VOID(infoPgAdd(infoPg, &infoPgData), "infoPgAdd"); InfoPgData infoPgDataTest = infoPgDataCurrent(infoPg); TEST_RESULT_INT(infoPgDataTest.id, 3, " id set"); TEST_RESULT_INT(infoPgDataTest.version, PG_VERSION_96, " version set"); TEST_RESULT_INT(infoPgDataTest.systemId, 6399999999999999999, " system-id set"); TEST_RESULT_INT(infoPgDataTest.catalogVersion, 201608131, " catalog-version set"); TEST_RESULT_INT(infoPgDataTest.controlVersion, 960, " control-version set"); // Errors //-------------------------------------------------------------------------------------------------------------------------- TEST_ERROR(infoPgNewLoad(storageLocal(), fileName, 10, cipherTypeNone, NULL, NULL), AssertError, "invalid InfoPg type 10"); TEST_ERROR( infoPgNewLoad(storageLocal(), NULL, infoPgManifest, cipherTypeNone, NULL, NULL), AssertError, "assertion 'fileName != NULL' failed"); TEST_ERROR(infoPgDataCurrent(NULL), AssertError, "assertion 'this != NULL' failed"); TEST_ERROR(infoPgAdd(NULL, &infoPgData), AssertError, "assertion 'this != NULL' failed"); TEST_ERROR(infoPgAdd(infoPg, NULL), AssertError, "assertion 'infoPgData != NULL' failed"); // infoPgFree //-------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID(infoPgFree(infoPg), "infoPgFree() - free infoPg"); // infoPgDataToLog //-------------------------------------------------------------------------------------------------------------------------- // test max values infoPgDataTest.id = (unsigned int)4294967295; infoPgDataTest.version = (unsigned int)4294967295; infoPgDataTest.systemId = 18446744073709551615U; infoPgDataTest.catalogVersion = (uint32_t)4294967295; infoPgDataTest.controlVersion = (uint32_t)4294967295; TEST_RESULT_STR( strPtr(infoPgDataToLog(&infoPgDataTest)), "{id: 4294967295, version: 4294967295, systemId: 18446744073709551615, catalogVersion: 4294967295, controlVersion:" " 4294967295}", " check max format"); } }