int ModeListMirrorBuddyGroups::execute()
{
const int mgmtTimeoutMS = 2500;
int retVal = APPCODE_RUNTIME_ERROR;
App* app = Program::getApp();
DatagramListener* dgramLis = app->getDatagramListener();
NodeStoreServers* mgmtNodes = app->getMgmtNodes();
std::string mgmtHost = app->getConfig()->getSysMgmtdHost();
unsigned short mgmtPortUDP = app->getConfig()->getConnMgmtdPortUDP();
StringMap* cfg = app->getConfig()->getUnknownConfigArgs();
UInt16List buddyGroupIDs;
UInt16List primaryTargetIDs;
UInt16List secondaryTargetIDs;
if(ModeHelper::checkInvalidArgs(cfg) )
return APPCODE_INVALID_CONFIG;
// check mgmt node
if(!NodesTk::waitForMgmtHeartbeat(
NULL, dgramLis, mgmtNodes, mgmtHost, mgmtPortUDP, mgmtTimeoutMS) )
{
std::cerr << "Management node communication failed: " << mgmtHost << std::endl;
return APPCODE_RUNTIME_ERROR;
}
// download buddy groups
Node* mgmtNode = mgmtNodes->referenceFirstNode();
if(!NodesTk::downloadMirrorBuddyGroups(mgmtNode, NODETYPE_Storage, &buddyGroupIDs,
&primaryTargetIDs, &secondaryTargetIDs, false) )
{
std::cerr << "Download of mirror buddy groups failed." << std::endl;
retVal = APPCODE_RUNTIME_ERROR;
goto cleanup_mgmt;
}
// print results
printGroups(buddyGroupIDs, primaryTargetIDs, secondaryTargetIDs);
retVal = APPCODE_NO_ERROR;
cleanup_mgmt:
mgmtNodes->releaseNode(&mgmtNode);
return retVal;
}
bool GetNodeInfoMsgEx::processIncoming(struct sockaddr_in* fromAddr, Socket* sock, char* respBuf,
size_t bufLen, HighResolutionStats* stats)
{
LogContext log("GetNodeInfoMsg incoming");
std::string peer = fromAddr ? Socket::ipaddrToStr(&fromAddr->sin_addr) : sock->getPeername();
LOG_DEBUG_CONTEXT(log, 4, std::string("Received a GetNodeInfoMsg") + peer);
GeneralNodeInfo info;
App *app = Program::getApp();
getCPUInfo(&(info.cpuName), &(info.cpuSpeed), &(info.cpuCount));
getMemInfo(&(info.memTotal), &(info.memFree));
info.logFilePath = app->getConfig()->getLogStdFile();
Node *localNode = app->getLocalNode();
std::string nodeID = localNode->getID();
uint16_t nodeNumID = localNode->getNumID();
GetNodeInfoRespMsg getNodeInfoRespMsg(nodeID.c_str(), nodeNumID, info);
getNodeInfoRespMsg.serialize(respBuf, bufLen);
sock->sendto(respBuf, getNodeInfoRespMsg.getMsgLength(), 0, (struct sockaddr*) fromAddr,
sizeof(struct sockaddr_in));
LOG_DEBUG_CONTEXT(log, 5, std::string("Sent a message with type: " ) + StringTk::uintToStr(
getNodeInfoRespMsg.getMsgType() ) + std::string(" to admon"));
app->getClientOpStats()->updateNodeOp(sock->getPeerIP(), MetaOpCounter_GETNODEINFO);
return true;
}
bool RequestExceededQuotaMsgEx::processIncoming(struct sockaddr_in* fromAddr, Socket* sock,
char* respBuf, size_t bufLen, HighResolutionStats* stats)
{
LogContext log("RequestExceededQuotaMsgEx incoming");
std::string peer = fromAddr ? Socket::ipaddrToStr(&fromAddr->sin_addr) : sock->getPeername();
LOG_DEBUG_CONTEXT(log, Log_DEBUG, std::string("Received a RequestExceededQuotaMsgEx from: ") +
peer);
App* app = Program::getApp();
Config* cfg = app->getConfig();
// get exceeded quota ID list
RequestExceededQuotaRespMsg respMsg(getQuotaDataType(), getExceededType() );
if(cfg->getQuotaEnableEnforcment() )
app->getQuotaManager()->getExceededQuotaStore()->getExceededQuota(
respMsg.getExceededQuotaIDs(), getQuotaDataType(), getExceededType() );
else
log.log(Log_ERR, "Unable to provide exceeded quota IDs. "
"The storage daemon on " + sock->getPeername() + " has quota enforcement enabled, "
"but not this management daemon. Fix this configuration problem or quota enforcement will "
"not work.");
respMsg.serialize(respBuf, bufLen);
sock->sendto(respBuf, respMsg.getMsgLength(), 0,
(struct sockaddr*)fromAddr, sizeof(struct sockaddr_in) );
return true;
}
/**
* checks if all required functions of the installed libzfs are available
*
* @param blockDevice a QuotaBlockDevice for a test the get quota workflow
* @param targetNumID the targetNumID of the storage target
* @return true if all function pointers are available, false if a error occurred
*/
bool QuotaTk::checkRequiredLibZfsFunctions(QuotaBlockDevice* blockDevice, uint16_t targetNumID)
{
App* app = Program::getApp();
ZfsSession session;
// check if all function names in the lib available
if(!session.initZfsSession(app->getDlOpenHandleLibZfs() ) )
{
if(!app->getConfig()->getQuotaDisableZfsSupport() )
app->getLogger()->logErr("InitLibZFS", "Cannot initialize libzfs. "
"The quota system for all zfs blockdevices/pools on this machine won't work.");
return false;
}
// check if the signature of all function pointers are compatible
std::string propertyString;
QuotaData quotaData(0, QuotaDataType_USER); // test request for user root
if(!requestQuotaFromZFS(blockDevice, targetNumID, &propertyString, "aData, &session) )
return false;
// check function pointers which are required in case of errors happens
std::string errorDec( (*session.libzfs_error_description)(session.getlibZfsHandle() ) );
std::string errorAct( (*session.libzfs_error_action)(session.getlibZfsHandle() ) );
return true;
}
bool CreateDefDirInodesMsgEx::processIncoming(struct sockaddr_in* fromAddr, Socket* sock,
char* respBuf, size_t bufLen, HighResolutionStats* stats)
{
std::string peer = fromAddr ? Socket::ipaddrToStr(&fromAddr->sin_addr) : sock->getPeername();
LOG_DEBUG("CreateDefDirInodesMsg incoming", 4,
std::string("Received a CreateDefDirInodesMsg from: ") + peer);
LogContext log("CreateDefDirInodesMsg incoming");
App* app = Program::getApp();
Config* cfg = app->getConfig();
uint16_t localNodeNumID = app->getLocalNode()->getNumID();
StringList inodeIDs;
StringList failedInodeIDs;
FsckDirInodeList createdInodes;
this->parseInodeIDs(&inodeIDs);
for ( StringListIter iter = inodeIDs.begin(); iter != inodeIDs.end(); iter++ )
{
std::string inodeID = *iter;
int mode = S_IFDIR | S_IRWXU;
unsigned userID = 0; // root
unsigned groupID = 0; // root
uint16_t ownerNodeID = localNodeNumID;
UInt16Vector stripeTargets;
unsigned defaultChunkSize = cfg->getTuneDefaultChunkSize();
unsigned defaultNumStripeTargets = cfg->getTuneDefaultNumStripeTargets();
Raid0Pattern stripePattern(defaultChunkSize, stripeTargets, defaultNumStripeTargets);
// we try to create a new directory inode, with default values
DirInode dirInode(inodeID, mode, userID, groupID, ownerNodeID, stripePattern);
if ( dirInode.storeAsReplacementFile(inodeID) == FhgfsOpsErr_SUCCESS )
{
// try to refresh the metainfo (maybe a .cont directory was already present)
dirInode.refreshMetaInfo();
StatData statData;
dirInode.getStatData(statData);
FsckDirInode fsckDirInode(inodeID, "", 0, localNodeNumID, statData.getFileSize(),
statData.getNumHardlinks(), stripeTargets, FsckStripePatternType_RAID0, localNodeNumID);
createdInodes.push_back(fsckDirInode);
}
else
failedInodeIDs.push_back(inodeID);
}
CreateDefDirInodesRespMsg respMsg(&failedInodeIDs, &createdInodes);
respMsg.serialize(respBuf, bufLen);
sock->sendto(respBuf, respMsg.getMsgLength(), 0, (struct sockaddr*) fromAddr,
sizeof(struct sockaddr_in));
return true;
}
int Upgrade::runUpgrade()
{
App* app = Program::getApp();
Config* config = app->getConfig();
std::string storageDir = config->getStoreMetaDirectory();
this->metaIdMap = Program::getApp()->getMetaStringIdMap();
LogContext log("");
log.logErr(""); // just an EOL
log.logErr("* Writing log file to: " + config->getLogStdFile() );
log.logErr(" Progress may be monitored by watching this file.");
// loads target string to numeric ID map
std::string targetIDMapFileName = Program::getApp()->getConfig()->getTargetIdMapFile();
if (!loadIDMap(targetIDMapFileName, &this->targetIdMap) )
return 1;
bool deleteOldFiles = config->getDeleteOldFiles();
if (!deleteOldFiles)
{
bool initRes = initInodesBackupDir(storageDir);
if (!initRes)
return 1;
}
if (!setLocalNodeNumID(this->metaIdMap) )
return 1;
log.logErr(""); // just an EOL
log.logErr("* Starting the upgrade process: ...");
log.logErr(""); // just an EOL
bool retVal = migrateDentries(storageDir); // walks and calls migrateStructureSubdir()
if(retVal == false || this->allOK == false)
{
log.logErr("");
log.logErr("Not all dentries could be migrated. Remaining inodes will be ignored!");
retVal = false;
goto out;
}
if (!migrateInodes(storageDir) ) // migrate all (remaining) inodes
retVal = false;
out:
// storage format file
if(!StorageTkEx::createStorageFormatFile(storageDir) )
throw InvalidConfigException("Unable to create storage format file in: " + storageDir);
printFinalInfo();
return retVal;
}
/**
* Note: Creates the file only if it does not exist yet.
*
* @return true if format file was created or existed already
*/
bool StorageTkEx::createStorageFormatFile(const std::string pathStr)
{
App* app = Program::getApp();
Config* cfg = app->getConfig();
StringMap formatProperties;
formatProperties[STORAGETK_FORMAT_XATTR] = cfg->getStoreUseExtendedAttribs() ? "true" : "false";
return StorageTk::createStorageFormatFile(pathStr, STORAGETK_FORMAT_CURRENT_VERSION,
&formatProperties);
}
int ModeHelp::execute()
{
App* app = Program::getApp();
Config* cfg = app->getConfig();
RunMode runMode = cfg->determineRunMode();
if(runMode == RunMode_INVALID)
printGeneralHelp();
else
printSpecificHelp(runMode);
std::cout << std::endl;
return APPCODE_INVALID_CONFIG;
}
/**
* Delete the inode on disk or move it into the backup dir.
*
*/
bool Upgrade::backupOrDeleteDirInode(DirInode* inode)
{
App* app = Program::getApp();
bool deleteOldEntries = app->getConfig()->getDeleteOldFiles();
bool retVal = true;
if (deleteOldEntries)
{
const char* logContext = "Delete DirInode";
int unlinkRes = unlink(inode->getPathToDirInode() );
if (unlinkRes)
{
retVal = false;
this->allOK = false;
LogContext(logContext).logErr("");
LogContext(logContext).logErr(std::string("Failed to delete old DirInode: ") +
inode->getPathToDirInode() + std::string(" :") + System::getErrString() );
}
}
else
{ // move to backup dir
const char* logContext = "Backup DirInode";
std::string inodeBackupDir = this->inodesBackupPath + inode->getOldHashDirPath();
int renameRes = rename(inode->getPathToDirInode(), inodeBackupDir.c_str() );
if (renameRes && errno != EEXIST)
{
retVal = false;
this->allOK = false;
LogContext(logContext).logErr("");
LogContext(logContext).logErr(std::string("Failed to rename DirInode to backup dir: ") +
inode->getPathToDirInode() + std::string(": ") + System::getErrString() );
}
}
return retVal;
}
bool HeartbeatRequestMsgEx::processIncoming(struct sockaddr_in* fromAddr, Socket* sock,
char* respBuf, size_t bufLen, HighResolutionStats* stats)
{
//const char* logContext = "HeartbeatRequest incoming";
//std::string peer = fromAddr ? Socket::ipaddrToStr(&fromAddr->sin_addr) : sock->getPeername();
//LOG_DEBUG_CONTEXT(log, 5, std::string("Received a HeartbeatRequestMsg from: ") + peer);
//IGNORE_UNUSED_VARIABLE(logContext);
App* app = Program::getApp();
Config* cfg = app->getConfig();
Node* localNode = app->getLocalNode();
std::string localNodeID = localNode->getID();
uint16_t localNodeNumID = localNode->getNumID();
uint16_t rootNodeID = app->getMetaNodes()->getRootNodeNumID();
NicAddressList nicList(localNode->getNicList() );
const BitStore* nodeFeatureFlags = localNode->getNodeFeatures();
HeartbeatMsg hbMsg(localNodeID.c_str(), localNodeNumID, NODETYPE_Meta, &nicList,
nodeFeatureFlags);
hbMsg.setRootNumID(rootNodeID);
hbMsg.setPorts(cfg->getConnMetaPortUDP(), cfg->getConnMetaPortTCP() );
hbMsg.setFhgfsVersion(BEEGFS_VERSION_CODE);
hbMsg.serialize(respBuf, bufLen);
if(fromAddr)
{ // datagram => reply via dgramLis send method
app->getDatagramListener()->sendto(respBuf, hbMsg.getMsgLength(), 0,
(struct sockaddr*)fromAddr, sizeof(*fromAddr) );
}
else
sock->sendto(respBuf, hbMsg.getMsgLength(), 0, NULL, 0);
return true;
}
/**
* Compatibility and validity check of storage format file contents.
*
* @param pathStr path to the main storage working directory (not including a filename)
* @throws exception if format file was not valid (eg didn't exist or contained wrong version).
*/
void StorageTkEx::checkStorageFormatFile(const std::string pathStr) throw(InvalidConfigException)
{
App* app = Program::getApp();
Config* cfg = app->getConfig();
StringMap formatProperties;
StringMapIter formatIter;
StorageTk::checkAndUpdateStorageFormatFile(pathStr, STORAGETK_FORMAT_MIN_VERSION,
STORAGETK_FORMAT_CURRENT_VERSION, &formatProperties);
formatIter = formatProperties.find(STORAGETK_FORMAT_XATTR);
if(formatIter == formatProperties.end() )
{
throw InvalidConfigException(std::string("Property missing from storage format file: ") +
STORAGETK_FORMAT_XATTR " (dir: " + pathStr + ")");
}
if(cfg->getStoreUseExtendedAttribs() != StringTk::strToBool(formatIter->second) )
{
throw InvalidConfigException("Mismatch of extended attributes settings in storage format file"
" and daemon config file.");
}
}
/**
* Note: Remember to call freeDirSettings() when you're done.
*/
bool ModeCreateDir::initDirSettings(DirSettings* settings)
{
App* app = Program::getApp();
StringMap* cfg = app->getConfig()->getUnknownConfigArgs();
settings->preferredNodes = NULL;
settings->path = NULL;
StringMapIter iter;
// parse and validate command line args
settings->mode = 0755;
iter = cfg->find(MODECREATEDIR_ARG_PERMISSIONS);
if(iter != cfg->end() )
{
settings->mode = StringTk::strOctalToUInt(iter->second);
settings->mode = settings->mode & 0777; // trim invalid flags from mode
cfg->erase(iter);
}
settings->mode |= S_IFDIR; // make sure mode contains the "dir" flag
settings->userID = 0;
iter = cfg->find(MODECREATEDIR_ARG_USERID);
if(iter != cfg->end() )
{
settings->userID = StringTk::strToUInt(iter->second);
cfg->erase(iter);
}
settings->groupID = 0;
iter = cfg->find(MODECREATEDIR_ARG_GROUPID);
if(iter != cfg->end() )
{
settings->groupID = StringTk::strToUInt(iter->second);
cfg->erase(iter);
}
settings->noMirroring = false;
iter = cfg->find(MODECREATEDIR_ARG_NOMIRROR);
if(iter != cfg->end() )
{
settings->noMirroring = true;
cfg->erase(iter);
}
// parse preferred nodes
StringList preferredNodesUntrimmed;
settings->preferredNodes = new UInt16List();
iter = cfg->find(MODECREATEDIR_ARG_NODES);
if(iter != cfg->end() )
{
StringTk::explode(iter->second, ',', &preferredNodesUntrimmed);
cfg->erase(iter);
// trim nodes (copy to other list)
for(StringListIter nodesIter = preferredNodesUntrimmed.begin();
nodesIter != preferredNodesUntrimmed.end();
nodesIter++)
{
std::string nodeTrimmedStr = StringTk::trim(*nodesIter);
uint16_t nodeTrimmed = StringTk::strToUInt(nodeTrimmedStr);
settings->preferredNodes->push_back(nodeTrimmed);
}
}
bool useMountedPath = true;
iter = cfg->find(MODECREATEDIR_ARG_UNMOUNTEDPATH);
if(iter != cfg->end() )
{
useMountedPath = false;
cfg->erase(iter);
}
iter = cfg->begin();
if(iter == cfg->end() )
{
std::cerr << "No path specified." << std::endl;
return false;
}
else
{
std::string pathStr = iter->first;
if(useMountedPath)
{ // make path relative to mount root
std::string mountRoot;
std::string relativePath;
bool pathRelativeRes = ModeHelper::makePathRelativeToMount(
pathStr, true, false, &mountRoot, &relativePath);
if(!pathRelativeRes)
return false;
pathStr = relativePath;
std::cout << "Mount: '" << mountRoot << "'; Path: '" << relativePath << "'" << std::endl;
}
settings->path = new Path(pathStr);
settings->path->setAbsolute(true);
if(settings->path->getIsEmpty() )
{
std::cerr << "Invalid path specified." << std::endl;
return false;
}
}
return true;
}
int ModeCreateDir::execute()
{
const int mgmtTimeoutMS = 2500;
int retVal = APPCODE_RUNTIME_ERROR;
App* app = Program::getApp();
AbstractDatagramListener* dgramLis = app->getDatagramListener();
NodeStoreServers* mgmtNodes = app->getMgmtNodes();
NodeStoreServers* metaNodes = app->getMetaNodes();
NodeStoreServers* storageNodes = Program::getApp()->getStorageNodes();
std::string mgmtHost = app->getConfig()->getSysMgmtdHost();
unsigned short mgmtPortUDP = app->getConfig()->getConnMgmtdPortUDP();
NodeList metaNodesList;
uint16_t rootNodeID;
NodeList storageNodesList;
DirSettings settings;
// check privileges
if(!ModeHelper::checkRootPrivileges() )
return APPCODE_RUNTIME_ERROR;
// check mgmt node
if(!NodesTk::waitForMgmtHeartbeat(
NULL, dgramLis, mgmtNodes, mgmtHost, mgmtPortUDP, mgmtTimeoutMS) )
{
std::cerr << "Management node communication failed: " << mgmtHost << std::endl;
return APPCODE_RUNTIME_ERROR;
}
// download nodes
Node* mgmtNode = mgmtNodes->referenceFirstNode();
if(!NodesTk::downloadNodes(mgmtNode, NODETYPE_Meta, &metaNodesList, false, &rootNodeID) )
{
std::cerr << "Node download failed." << std::endl;
mgmtNodes->releaseNode(&mgmtNode);
return APPCODE_RUNTIME_ERROR;
}
NodesTk::applyLocalNicCapsToList(app->getLocalNode(), &metaNodesList);
NodesTk::moveNodesFromListToStore(&metaNodesList, metaNodes);
metaNodes->setRootNodeNumID(rootNodeID, false);
if(!NodesTk::downloadNodes(mgmtNode, NODETYPE_Storage, &storageNodesList, false, NULL) )
{
std::cerr << "Node download failed." << std::endl;
mgmtNodes->releaseNode(&mgmtNode);
return APPCODE_RUNTIME_ERROR;
}
NodesTk::applyLocalNicCapsToList(app->getLocalNode(), &storageNodesList);
NodesTk::moveNodesFromListToStore(&storageNodesList, storageNodes);
// check arguments
if(!initDirSettings(&settings) )
{
mgmtNodes->releaseNode(&mgmtNode);
return APPCODE_RUNTIME_ERROR;
}
// find owner node
Node* ownerNode = NULL;
EntryInfo entryInfo;
FhgfsOpsErr findRes = MetadataTk::referenceOwner(settings.path, true, metaNodes, &ownerNode,
&entryInfo);
if(findRes != FhgfsOpsErr_SUCCESS)
{
std::cerr << "Unable to find metadata node for path: " << settings.path->getPathAsStr() <<
std::endl;
std::cerr << "Error: " << FhgfsOpsErrTk::toErrString(findRes) << std::endl;
retVal = APPCODE_RUNTIME_ERROR;
goto cleanup_settings;
}
// create the dir
if(communicate(ownerNode, &entryInfo, &settings) )
{
std::cout << "Operation succeeded." << std::endl;
retVal = APPCODE_NO_ERROR;
}
// cleanup
metaNodes->releaseNode(&ownerNode);
cleanup_settings:
freeDirSettings(&settings);
mgmtNodes->releaseNode(&mgmtNode);
return retVal;
}
int ModeGetNodes::execute()
{
const int mgmtTimeoutMS = 2500;
int retVal = APPCODE_RUNTIME_ERROR;
App* app = Program::getApp();
DatagramListener* dgramLis = app->getDatagramListener();
NodeStoreServers* mgmtNodes = app->getMgmtNodes();
std::string mgmtHost = app->getConfig()->getSysMgmtdHost();
unsigned short mgmtPortUDP = app->getConfig()->getConnMgmtdPortUDP();
StringMap* cfg = app->getConfig()->getUnknownConfigArgs();
NodeList nodes;
StringSet unreachableNodes; // keys are nodeIDs, values unused
uint16_t rootNodeID;
// check arguments
StringMapIter iter = cfg->find(MODEGETNODES_ARG_PRINTDETAILS);
if(iter != cfg->end() )
{
cfgPrintDetails = true;
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_PRINTNICDETAILS);
if(iter != cfg->end() )
{
cfgPrintNicDetails = true;
cfgPrintDetails = true; // implied in this case
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_PRINTFHGFSVERSION);
if(iter != cfg->end() )
{
cfgPrintFhgfsVersion = true;
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_CHECKREACHABILITY);
if(iter != cfg->end() )
{
cfgCheckReachability = true;
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_REACHABILITYRETRIES);
if(iter != cfg->end() )
{
cfgReachabilityNumRetries = StringTk::strToUInt(iter->second);
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_REACHABILITYTIMEOUT_MS);
if(iter != cfg->end() )
{
cfgReachabilityRetryTimeoutMS = StringTk::strToUInt(iter->second);
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_PING);
if(iter != cfg->end() )
{
cfgPing = true;
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_PINGRETRIES);
if(iter != cfg->end() )
{
cfgPingRetries = StringTk::strToUInt(iter->second);
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_CONNTESTNUM);
if(iter != cfg->end() )
{
cfgConnTestNum = StringTk::strToUInt(iter->second);
cfg->erase(iter);
}
iter = cfg->find(MODEGETNODES_ARG_CONNROUTE);
if(iter != cfg->end() )
{
cfgPrintConnRoute = true;
cfg->erase(iter);
}
NodeType nodeType = ModeHelper::nodeTypeFromCfg(cfg);
if(nodeType == NODETYPE_Invalid)
{
std::cerr << "Invalid or missing node type." << std::endl;
return APPCODE_INVALID_CONFIG;
}
if(ModeHelper::checkInvalidArgs(cfg) )
return APPCODE_INVALID_CONFIG;
// check mgmt node
if(!NodesTk::waitForMgmtHeartbeat(
NULL, dgramLis, mgmtNodes, mgmtHost, mgmtPortUDP, mgmtTimeoutMS) )
{
std::cerr << "Management node communication failed: " << mgmtHost << std::endl;
return APPCODE_RUNTIME_ERROR;
}
Node* mgmtNode = mgmtNodes->referenceFirstNode();
if(!NodesTk::downloadNodes(mgmtNode, nodeType, &nodes, false, &rootNodeID) )
{
std::cerr << "Node download failed." << std::endl;
retVal = APPCODE_RUNTIME_ERROR;
goto cleanup_mgmt;
}
NodesTk::applyLocalNicCapsToList(app->getLocalNode(), &nodes); /* (downloaded node objects
don't know the local nic caps initially) */
// check reachability
if(cfgCheckReachability)
ModeHelperGetNodes::checkReachability(nodeType, &nodes, &unreachableNodes,
cfgReachabilityNumRetries, cfgReachabilityRetryTimeoutMS);
// ping
if(cfgPing)
ModeHelperGetNodes::pingNodes(nodeType, &nodes, cfgPingRetries);
// conn test
if(cfgConnTestNum)
ModeHelperGetNodes::connTest(nodeType, &nodes, cfgConnTestNum);
// print nodes
printNodes(nodeType, &nodes, &unreachableNodes, rootNodeID);
retVal = APPCODE_NO_ERROR;
// clean up
NodesTk::deleteListNodes(&nodes);
cleanup_mgmt:
mgmtNodes->releaseNode(&mgmtNode);
return retVal;
}
/**
* Establish the given number of connections simultaneously to each of the nodes.
* This is intended to test general limit and performance impacts with a high number of established
* connections.
*
* @param numPingRetries 0-based number of ping repetitions per node
*/
void ModeHelperGetNodes::connTest(NodeType nodeType, NodeList* nodeList, unsigned numConns)
{
// note: this works by sending heartbeat requests to all nodes via stream connection and
// measuring response time.
if(nodeList->empty() )
{
std::cerr << "Skipping conn test (no nodes available)." << std::endl;
return;
}
App* app = Program::getApp();
Node* localNode = app->getLocalNode();
Node* node = *nodeList->begin();
NodeConnPool* connPool = node->getConnPool();
NicAddressList origNicList = connPool->getNicList();
NicListCapabilities origNicCaps;
unsigned i = 0;
Socket** socks = (Socket**)calloc(1, numConns * sizeof(Socket*) );
NetworkInterfaceCard::supportedCapabilities(&origNicList, &origNicCaps);
connPool->setMaxConns(numConns+1); // temporarily inc conn limit of node
NicAddressList localNicList(localNode->getNicList() );
NicListCapabilities localNicCaps;
NetworkInterfaceCard::supportedCapabilities(&localNicList, &localNicCaps);
if(localNicCaps.supportsRDMA)
{
std::cout << "Beginning RDMA conn test. Node: " << node->getID() << std::endl;
NicListCapabilities tmpRdmaNicCaps = localNicCaps;
tmpRdmaNicCaps.supportsSDP = false;
connPool->setLocalNicCaps(&tmpRdmaNicCaps);
for(i=0; i < numConns; i++)
{
// log status message for each 500 established conns
if( (i % 500) == 0)
std::cout << "Established connections: " << i << std::endl;
try
{
// establish new connection
socks[i] = connPool->acquireStreamSocket();
}
catch(SocketException& e)
{
std::cerr << "Connect error: " << e.what() << std::endl;
}
if(!socks[i])
{
std::cerr << "RDMA Connect #" << (i+1) << " failed." << std::endl;
break;
}
if(socks[i]->getSockType() != NICADDRTYPE_RDMA)
{
std::cerr << "Connection #" << (i+1) << " is not of type RDMA." << std::endl;
break;
}
}
std::cout << "Hit ENTER to drop " << i << " RDMA connections... "; // (no endl here)
getchar();
for(i=0; i < numConns; i++)
{
if(socks[i])
connPool->invalidateStreamSocket(socks[i]);
}
} // end of RDMA
// TCP test...
std::cout << "Beginning TCP conn test. Node: " << node->getID() << std::endl;
NicListCapabilities tmpTcpNicCaps = localNicCaps;
tmpTcpNicCaps.supportsRDMA = false;
tmpTcpNicCaps.supportsSDP = false;
connPool->setLocalNicCaps(&tmpTcpNicCaps);
memset(socks, 0, numConns * sizeof(Socket*) );
for(i=0; i < numConns; i++)
{
// log status message for each 500 established conns
if( (i % 500) == 0)
std::cout << "Established connections: " << i << std::endl;
try
{
// establish new connection
socks[i] = connPool->acquireStreamSocket();
}
catch(SocketException& e)
{
std::cerr << "Connect error: " << e.what() << std::endl;
}
if(!socks[i])
{
std::cerr << "TCP Connect #" << (i+1) << " failed." << std::endl;
break;
}
if(socks[i]->getSockType() != NICADDRTYPE_STANDARD)
{
std::cerr << "Connection #" << (i+1) << " is not of type STANDARD." << std::endl;
break;
}
}
std::cout << "Hit ENTER to drop " << i << " TCP connections... "; // (no std::endl here)
getchar();
for(i=0; i < numConns; i++)
{
if(socks[i])
connPool->invalidateStreamSocket(socks[i]);
}
// cleanup
free(socks);
connPool->setMaxConns(app->getConfig()->getConnMaxInternodeNum() );
connPool->setLocalNicCaps(&origNicCaps);
std::cout << std::endl;
}
/**
* Migrate a single dirEntry
*/
bool Upgrade::migrateDirEntry(std::string storageDir, std::string parentEntryID,
DirInode* parentDirInode, std::string relDentriesPath, std::string entryName)
{
const char* logContext = "migrate DirEntry";
App* app = Program::getApp();
Config* cfg = app->getConfig();
bool deleteOldFiles = cfg->getDeleteOldFiles();
MetaStore* metaStore = app->getMetaStore();
bool retVal;
FileInode* fileInode = NULL;
EntryInfo entryInfo;
std::string absDentriesPath = storageDir + "/" META_DENTRIES_SUBDIR_NAME_OLD "/" +
relDentriesPath;
std::string entryPathOld = absDentriesPath + "/" + entryName;
std::string inodePath;
std::string relFileInodePath;
DirEntry* fhgfs_dirEntry = DirEntry::createFromFile(absDentriesPath, entryName);
if (!fhgfs_dirEntry)
{
LogContext(logContext).logErr("Failed to create DirEntry from file: " + entryPathOld);
return false;
}
bool dentryMapRes = fhgfs_dirEntry->mapOwnerNodeStrID(this->metaIdMap);
if (dentryMapRes == false)
{
LogContext(logContext).logErr("OwnerNodeID mapping failed for: " + entryPathOld);
retVal = false;
goto out;
}
fhgfs_dirEntry->getEntryInfo(parentEntryID, 0, &entryInfo);
if (DirEntryType_ISFILE(entryInfo.getEntryType() ) )
{
FhgfsOpsErr mkDentryRes;
fileInode = FileInode::createFromEntryInfo(&entryInfo);
if (!fileInode)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to deserialize inode for: " + entryPathOld);
retVal = false;
goto out;
}
bool mapRes = fileInode->mapStripePatternStringIDs(&this->targetIdMap);
if (mapRes == false)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("targetID mapping failed for: " + entryPathOld);
retVal = false;
goto out;
}
inodePath = fileInode->getPathToInodeFile();
relFileInodePath = fileInode->getReInodeFilePathPath();
// deletes fileInode!
mkDentryRes = metaStore->mkMetaFileUnlocked(parentDirInode, entryName,
entryInfo.getEntryType(), fileInode);
if (mkDentryRes != FhgfsOpsErr_SUCCESS && mkDentryRes != FhgfsOpsErr_EXISTS)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to create updated dentry for: " + entryPathOld);
retVal = false;
goto out;
}
retVal = true;
}
else
{
FhgfsOpsErr mkDirRes = parentDirInode->makeDirEntry(fhgfs_dirEntry);
fhgfs_dirEntry = NULL;
if(mkDirRes != FhgfsOpsErr_SUCCESS && mkDirRes != FhgfsOpsErr_EXISTS)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to create updated dir-dentry for: " + entryPathOld);
retVal = false;
goto out;
}
retVal = true;
}
// delete or backup migrated dentries and their inodes
if (retVal == true)
{
/* Always first the dentry, then the inode. If we would do it the other way around and would
* need to re-run the upgrade tool, it will complain loudly if it cannot find an inode for
* a dentry. */
if (deleteOldFiles)
{
// first the dentry
int unlinkDentryRes = unlink(entryPathOld.c_str() );
if (unlinkDentryRes)
{
this->allOK = false;
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to unlink old dentry: " + entryPathOld);
}
else
if (DirEntryType_ISFILE(entryInfo.getEntryType() ) )
{
// now the inode (for non-directories).
int unlinkRes = unlink(inodePath.c_str() );
if (unlinkRes)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to unlink old fileInode: " + inodePath + " :" +
System::getErrString() );
}
}
}
else
{
// first the dentry
std::string backupPath = storageDir +
"/" META_DENTRIES_SUBDIR_NAME_OLD META_UPGRADE_BACKUP_EXT "/" +
relDentriesPath + "/" + entryName;
int renameRes = rename(entryPathOld.c_str(), backupPath.c_str() );
if (renameRes)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to rename: " + entryPathOld + " to: " +
backupPath + " SysErr: " + System::getErrString() );
this->allOK = false;
}
else
if (DirEntryType_ISFILE(entryInfo.getEntryType() ) )
{
// now the inode
std::string inodeBackupFilePath = this->inodesBackupPath + "/" + relFileInodePath;
int renameRes = rename(inodePath.c_str(), inodeBackupFilePath.c_str() );
if (renameRes)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to rename FileInode to backup dir: " +
inodePath + " :" + System::getErrString() );
}
}
}
/* No else here, as there is no reason to move dentries around, the 'structure' dir is the
* backup.
* Note: The only reason to move inodes is the migrateInodes step - it only shall migrate
* inodes not already handled by the dentry-migration */
}
out:
SAFE_DELETE(fhgfs_dirEntry);
return retVal;
}
/**
* Converts the directory entry (dentry) files in the dentries-subdir to the new format and stores
* into the correct dir. Also inlines inodes.
*/
bool Upgrade::migrateDentries(std::string storageDir)
{
const char* logContext = "migrate dentries";
App* app = Program::getApp();
bool deleteOldFiles = app->getConfig()->getDeleteOldFiles();
uint64_t numUserDirs = 0;
struct dirent* dirEntry = NULL;
std::string oldDentriesPath = storageDir + "/" META_DENTRIES_SUBDIR_NAME_OLD;
if (!deleteOldFiles)
{
std::string backupDir = oldDentriesPath + META_UPGRADE_BACKUP_EXT;
if (!doMkDir(backupDir) )
return false;
}
LogContext(logContext).logErr("* Migrating dentries... (Processing " +
StringTk::uint64ToStr(HASHDIRS_NUM_OLD) + " hash directories...)" );
LogContext(logContext).log(Log_NOTICE, "Progress: ");
bool retVal = true;
// walk over all hash subdirs ('dentries', formerly called 'structure')
for(unsigned i=0; i < HASHDIRS_NUM_OLD; i++)
{
std::string subDirHash = StringTk::uintToHexStr(i);
std::string subdirPath = oldDentriesPath + "/" + subDirHash;
LogContext(logContext).log(Log_NOTICE, subDirHash + " ", true);
if (!deleteOldFiles)
{
std::string backupPath = storageDir +
"/" META_DENTRIES_SUBDIR_NAME_OLD META_UPGRADE_BACKUP_EXT "/" + subDirHash;
if (!doMkDir(backupPath) )
continue;
}
DIR* dirHandle = opendir(subdirPath.c_str() );
if(!dirHandle && errno == ENOENT)
continue;
else
if(!dirHandle)
{
LogContext(logContext).logErr("Unable to open hash directory: " + subdirPath + ". " +
"SysErr: " + System::getErrString() );
goto err_cleanup;
}
errno = 0;
// walk over all parentEntryIDs
while( (dirEntry = StorageTk::readdirFiltered(dirHandle) ) )
{
std::string parentEntryID = dirEntry->d_name; // has the .cont suffix
std::string relDentriesPath = subDirHash + "/" + parentEntryID;
if (dirEntry->d_type != DT_DIR && dirEntry->d_type != DT_UNKNOWN)
this->hasInvalidDirectoryType = true;
bool migrateRes = migrateDentryDir(storageDir, parentEntryID, relDentriesPath);
if(!migrateRes)
{
std::string absPath = storageDir + "/" META_DENTRIES_SUBDIR_NAME_OLD "/" +
relDentriesPath;
LogContext(logContext).logErr("Unable to convert directory: " + absPath + ".");
retVal = false;
this->allOK = false;
continue;
}
numUserDirs++;
}
if(!dirEntry && errno)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Unable to fetch directory entry from: " + oldDentriesPath +
". " + "SysErr: " + System::getErrString() );
goto err_closedir;
}
closedir(dirHandle);
// everything was ok, delete the hash dir
{
int rmDirRes = rmdir(subdirPath.c_str() );
if (rmDirRes)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to rmdir the dentries hash dir: " + subdirPath +
": " + System::getErrString() );
}
}
continue;
err_closedir:
closedir(dirHandle);
retVal = false;
this->allOK = false;
continue;
}
LogContext(logContext).logErr(""); // just an EOL
LogContext(logContext).logErr("* Converted user directories: " +
StringTk::uint64ToStr(numUserDirs) );
if (retVal == true && this->allOK == true)
{
int rmDirRes = rmdir(oldDentriesPath.c_str() );
if (rmDirRes)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to rmdir the old structure dir: " + oldDentriesPath +
" :" + System::getErrString() );
}
}
return retVal;
err_cleanup:
return false;
}
/**
* Migrate a DirInode
*
* Note: We create this inode in the disposal dir, as there is no corresponding DirEntry for this
* inode.
*/
bool Upgrade::migrateFileInode(std::string inodeID)
{
const char* logContext = "migrage FileInode";
App* app = Program::getApp();
bool deleteOldFiles = app->getConfig()->getDeleteOldFiles();
MetaStore* metaStore = app->getMetaStore();
bool retVal;
EntryInfo entryInfo(0, "", inodeID, "", DirEntryType_REGULARFILE, 0); // just basic values
FileInode* fileInode = FileInode::createFromEntryInfo(&entryInfo);
if (!fileInode)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to deserialize FileInode: " + inodeID);
return false;
}
bool mapRes = fileInode->mapStripePatternStringIDs(&this->targetIdMap);
if (mapRes == false)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("StripePattern mapping failed for: " + inodeID);
SAFE_DELETE(fileInode);
return false;
}
std::string inodePath = fileInode->getPathToInodeFile();
std::string hashDirInodePath = fileInode->getReInodeFilePathPath();
FhgfsOpsErr mkDentryRes = metaStore->makeFileInode(fileInode, false);
if (mkDentryRes != FhgfsOpsErr_SUCCESS && mkDentryRes != FhgfsOpsErr_EXISTS)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to create updated inode for fileInode ID: " + inodeID);
retVal = false;
goto out;
}
retVal = true;
if (deleteOldFiles)
{
int unlinkRes = unlink(inodePath.c_str() );
if (unlinkRes)
{
LogContext(logContext).logErr("Failed to unlink old fileInode: " + inodePath + " :"
+ System::getErrString() );
}
}
else
{ // move to backup dir
std::string inodeBackupDir = this->inodesBackupPath + "/" + hashDirInodePath;
int renameRes = rename(inodePath.c_str(), inodeBackupDir.c_str() );
if (renameRes)
LogContext(logContext).logErr("Failed to rename FileInode to backup dir: " +
inodePath + " :" + System::getErrString() );
}
out:
return retVal;
}
/**
* Migrate dentries of the given dir
*
* @path path to dentries subdir (where we will read the dentry files).
* @newTmpPath path to new temporary structure subdir (where we will put the new link files);
* will be created.
*/
bool Upgrade::migrateDentryDir(std::string storageDir, std::string parentEntryID,
std::string relDentriesPath)
{
const char* logContext = "Migrate DentriesDir (structure)";
App* app = Program::getApp();
bool deleteOldFiles = app->getConfig()->getDeleteOldFiles();
if (!deleteOldFiles)
{
std::string backupPath = storageDir +
"/" META_DENTRIES_SUBDIR_NAME_OLD META_UPGRADE_BACKUP_EXT "/" + relDentriesPath;
if (!doMkDir(backupPath) )
return false;
}
// walk the old .cont dir and migrate each contained file
unsigned migrateErrors = 0;
int parentLen = parentEntryID.length();
if (parentLen < DENTRYDIR_CONT_LEN + 1)
{
LogContext(logContext).logErr("Invalid parentID (too small: " + parentEntryID + ")" );
this->allOK = false;
return false;
}
std::string contSubStr = parentEntryID.substr(parentLen - DENTRYDIR_CONT_LEN, DENTRYDIR_CONT_LEN);
if (contSubStr != DENTRYDIR_CONT)
{
LogContext(logContext).logErr(std::string("Invalid parentEntryID (.cont missing): ") +
contSubStr);
this->allOK = false;
return false;
}
// remove .cont from parentEntryID
std::string parentIDWithoutCont(parentEntryID);
parentIDWithoutCont.resize(parentLen - DENTRYDIR_CONT_LEN);
DirInode* parentDirInode;
bool dirInodeRes = migrateDirInode(parentIDWithoutCont, &parentDirInode);
if (!dirInodeRes)
{
this->allOK = false;
return false;
}
std::string absDentriesPath = storageDir + "/" META_DENTRIES_SUBDIR_NAME_OLD "/" +
relDentriesPath;
DIR* dirHandle = opendir(absDentriesPath.c_str() );
if(!dirHandle)
{
LogContext(logContext).logErr("Unable to open structure subdir: " + absDentriesPath + ". " +
"SysErr: " + System::getErrString() );
this->allOK = false;
return false;
}
// uint64_t numEntries = 0;
struct dirent* dirEntry = NULL;
errno = 0; // recommended by posix (readdir(3p) )
// read all entries and move them to new tmp dir
do {
dirEntry = StorageTk::readdirFiltered(dirHandle);
if (!dirEntry)
break;
std::string entryName = dirEntry->d_name;
bool migrateRes = migrateDirEntry(storageDir, parentIDWithoutCont, parentDirInode,
relDentriesPath, entryName);
if (migrateRes == false)
migrateErrors++;
} while(dirEntry);
bool retVal;
if (!migrateErrors)
{
retVal = true;
backupOrDeleteDirInode(parentDirInode);
}
else
{
retVal = false;
this->allOK = false;
}
SAFE_DELETE(parentDirInode);
closedir(dirHandle);
if (retVal == true)
{
int rmDirRes = rmdir(absDentriesPath.c_str() );
if (rmDirRes)
{
LogContext(logContext).logErr("");
LogContext(logContext).logErr("Failed to rmdir the contents dir: " + absDentriesPath +
": " + System::getErrString() );
this->allOK = false;
retVal = false;
}
}
return retVal;
}
void InternodeSyncer::syncLoop()
{
App* app = Program::getApp();
Config* cfg = app->getConfig();
MgmtdTargetStateStore* mgmtdTargetStateStore = app->getTargetStateStore();
MgmtdTargetStateStore* mgmtdMetaStateStore = app->getMetaStateStore();
const int sleepIntervalMS = 5*1000; // 5sec
const unsigned updateStatesMS = cfg->getSysUpdateTargetStatesSecs() * 1000;
const unsigned updateCapacityPoolsMS = 3 * updateStatesMS;
const unsigned saveTargetMappingsMS = 15*60*1000; // 15min
const unsigned idleDisconnectIntervalMS = 70*60*1000; /* 70 minutes (must be less than half the
streamlis idle disconnect interval to avoid cases where streamlis disconnects first) */
Time lastStatesUpdateT;
Time lastCapacityPoolsUpdateT;
Time lastTargetMappingsSaveT;
Time lastIdleDisconnectT;
while(!waitForSelfTerminateOrder(sleepIntervalMS) )
{
bool poolsUpdateForced = getAndResetForcePoolsUpdate();
if( poolsUpdateForced ||
(lastStatesUpdateT.elapsedMS() > updateStatesMS) )
{
bool statesModified = false;
const unsigned targetPOfflineTimeoutMS = (3 * sleepIntervalMS) + (2 * updateStatesMS);
statesModified |= mgmtdTargetStateStore->autoOfflineTargets(
targetPOfflineTimeoutMS, targetOfflineTimeoutMS, app->getMirrorBuddyGroupMapper() );
statesModified |= mgmtdMetaStateStore->autoOfflineTargets(
targetPOfflineTimeoutMS, targetOfflineTimeoutMS, NULL); /* NULL here needs to be set
to meta mirror buddy group mapper when it's available */
statesModified |= mgmtdTargetStateStore->resolveDoubleResync();
// if target states have changed, inform other nodes about it
if (statesModified)
app->getHeartbeatMgr()->notifyAsyncRefreshTargetStates();
lastStatesUpdateT.setToNow();
}
if( poolsUpdateForced ||
(lastCapacityPoolsUpdateT.elapsedMS() > updateCapacityPoolsMS) )
{
updateMetaCapacityPools();
updateStorageCapacityPools(poolsUpdateForced);
lastCapacityPoolsUpdateT.setToNow();
}
if(lastTargetMappingsSaveT.elapsedMS() > saveTargetMappingsMS)
{
saveTargetMappings();
lastTargetMappingsSaveT.setToNow();
}
if(lastIdleDisconnectT.elapsedMS() > idleDisconnectIntervalMS)
{
dropIdleConns();
lastIdleDisconnectT.setToNow();
}
}
}