int ProcessFdSet(nfhs_t pv_Nfhs, nfhs_t *pp_Nfds, fh_set *pp_Fhs, fd_set *pp_Fds) { nfhs_t lv_Fd = 0; if(pp_Fhs != NULL) { FD_ZERO(pp_Fds); for (int i = 0; i < pv_Nfhs; i++) { if(i < (int)pp_Fhs->FhArray.size()) { STFS_ExternalFileHandle *lp_Efh = STFS_ExternalFileHandle::GetExternalFileHandle(pp_Fhs->FhArray[i]); if (! lp_Efh) { TRACE_PRINTF2(1, "Error in obtaining External File Handle for %ld\n", pp_Fhs->FhArray[i]); return -1; } //Check for local file. //Check for Nonblocking or not if so ignore. if(((lp_Efh->OpenFlagsGet())&O_NONBLOCK) == O_NONBLOCK) { lv_Fd = lp_Efh->GetFD(); if(lv_Fd >= 0) { FD_SET(lv_Fd, pp_Fds); if(lv_Fd > *pp_Nfds) { *pp_Nfds = lv_Fd; } } } } } } return 0; }
int ProcessFhSet(nfhs_t pv_Nfhs, fh_set *pp_Fhs, fd_set *pp_Fds) { int lv_Fd = 0; struct fh_set lv_Fhs; if(pp_Fhs != NULL) { lv_Fhs.FhArray = pp_Fhs->FhArray; STFS_FH_ZERO(pp_Fhs); for (int i = 0; i < pv_Nfhs; i++) { if(i < (int)lv_Fhs.FhArray.size()) { STFS_ExternalFileHandle *lp_Efh = STFS_ExternalFileHandle::GetExternalFileHandle(lv_Fhs.FhArray[i]); if (! lp_Efh) { TRACE_PRINTF2(1, "Error in obtaining External File Handle for %ld\n", lv_Fhs.FhArray[i]); return -1; } if(((lp_Efh->OpenFlagsGet())&O_NONBLOCK) == O_NONBLOCK) { lv_Fd = lp_Efh->GetFD(); if(lv_Fd >= 0) { if(FD_ISSET(lv_Fd, pp_Fds)) { //Insert into fh_set pp_Fhs->FhArray.push_back(pp_Fhs->FhArray[i]); } } } } } } return 0; }
/////////////////////////////////////////////////////////////////////////////// /// // STFSd_read /// /// \brief Reads from an STFS file /// /////////////////////////////////////////////////////////////////////////////// ssize_t STFSd_read(STFS_OpenIdentifier *pp_OpenId, void *pp_Buf, size_t pv_Count) { const char *WHERE = "STFSd_read"; STFS_ScopeTrace lv_st(WHERE); STFS_Session *lp_Session = STFS_Session::GetSession(); if(!lp_Session) { return -1; } if(!pp_OpenId || !pp_Buf) { //Invalid argument errno return -1; } if (pv_Count > (size_t)STFS_SSIZE_MAX) { return -1; } //If count is > buffer size then what? Handle this at the send.cpp level //Lookup Open file, Need to have added check to see if node matches STFS_ExternalFileOpenerContainer *lp_EfoContainer = STFS_ExternalFileOpenerContainer::GetInstance(); if (!lp_EfoContainer) { TRACE_PRINTF2(1,"%s\n", "Null EFO Container"); return -1; } STFS_ExternalFileOpenerInfo *lp_Efoi = lp_EfoContainer->Get(pp_OpenId); if (!lp_Efoi) { TRACE_PRINTF3(1, "Open Id: %d,%ld not found in the EFO Container\n", pp_OpenId->sqOwningDaemonNodeId, pp_OpenId->openIdentifier ); return -1; } STFS_ExternalFileMetadata *lp_Efm = lp_Efoi->efm_; if (!lp_Efm) { TRACE_PRINTF1(1,"Null EFM Found in the lp_Efoi Entry\n"); return -1; } TraceOpeners(lp_EfoContainer, lp_Efm); //Check to see if the file is local //If not local then Don't do anything. Return 0. //Do Read return 0; }
/////////////////////////////////////////////////////////////////////////////// /// // STFSd_close /// /// \brief Processes a file close request given the open identifier /// /// \param STFS_OpenIdentifier *pp_OpenId /// /////////////////////////////////////////////////////////////////////////////// int STFSd_close(STFS_OpenIdentifier *pp_OpenId) { const char *WHERE = "STFSd_close"; STFS_ScopeTrace lv_st(WHERE); if (!pp_OpenId) { return -1; } STFS_ExternalFileOpenerContainer *lp_EfoContainer = STFS_ExternalFileOpenerContainer::GetInstance(); if (!lp_EfoContainer) { TRACE_PRINTF2(1,"%s\n", "Null EFO Container"); return -1; } STFS_ExternalFileOpenerInfo *lp_Efoi = lp_EfoContainer->Get(pp_OpenId); if (!lp_Efoi) { TRACE_PRINTF3(1, "Open Id: %d,%ld not found in the EFO Container\n", pp_OpenId->sqOwningDaemonNodeId, pp_OpenId->openIdentifier ); return -1; } STFS_ExternalFileMetadata *lp_Efm = lp_Efoi->efm_; if (!lp_Efm) { TRACE_PRINTF1(1,"Null EFM Found in the Efoi Entry\n"); return -1; } TraceOpeners(lp_EfoContainer, lp_Efm); int lv_Ret = lp_Efm->Close(true); if (lv_Ret < 0) { return lv_Ret; } lv_Ret = lp_EfoContainer->Delete(pp_OpenId); if (lv_Ret < 0) { return lv_Ret; } TraceOpeners(lp_EfoContainer, lp_Efm); lv_Ret = STFS_ExternalFileMetadata::DeleteFromContainer(lp_Efm); if (lv_Ret < 0) { return lv_Ret; } return lv_Ret; }
/////////////////////////////////////////////////////////////////////////////// /// // STFSd_stat /// /// \brief Processes a stat request /// /// \param stfs_nodeid_t pv_Nid /// \param char *pp_Path /// \param Stfs_StatSet *pp_StatSet /// \param stfs_statmask_t pv_Mask /// /////////////////////////////////////////////////////////////////////////////// int STFSd_stat (stfs_nodeid_t pv_Nid, char *pp_Path, struct STFS_StatSet *pp_StatSet, stfs_statmask_t pv_Mask) { const char *WHERE = "STFSd_stat"; STFS_ScopeTrace lv_st(WHERE,2); int lv_Count = 0; int lv_OpenRet = 0; long *lp_PrevIndex = new long(0); while((lv_Count < STFS_MAX_OPENERS) && (lv_OpenRet == 0)) { strcpy(pp_StatSet->Path[*lp_PrevIndex], pp_Path); lv_OpenRet = STFS_util::STFS_stat(pv_Nid, pp_StatSet->Path[*lp_PrevIndex], lp_PrevIndex, pv_Mask, &pp_StatSet->Stat[*lp_PrevIndex]); if(lv_OpenRet < 0) { //TODO: Error Checking return -1; } if(lv_OpenRet != 1) { lv_Count++; } } TRACE_PRINTF2(4,"Number of files found by STFS_stat: %d\n", lv_Count); pp_StatSet->Count = lv_Count; return 0; }
ssize_t STFSLIB_lseek( stfs_fhndl_t pv_Fhandle, off_t pv_Offset, int pv_Whence) { const char *WHERE = "STFSLIB_lseek"; STFS_ScopeTrace lv_st(WHERE); STFS_ExternalFileHandle *lp_Efh = STFS_ExternalFileHandle::GetExternalFileHandle(pv_Fhandle); if (! lp_Efh) { TRACE_PRINTF2(1, "Error in obtaining External File Handle for %ld\n", pv_Fhandle); return -1; } lp_Efh->ResetErrors(); ssize_t lv_Stat = lp_Efh->Lseek(pv_Offset, pv_Whence); if (lv_Stat < 0) { return -1; } return lv_Stat; }
/////////////////////////////////////////////////////////////////////////////// /// // STSFSLIB_write() /// /// \brief The actual implementation of an STFS_write /// /// This method is the driver for an STFS_write() function call. /// it also implements the protocol for determining whether or not /// to create a new fragment. Only STFSLIB can make this call, not /// STFS-d, so the logic is put here. We could put a lot of "if /// !ExecutingInDaemon" escape clauses, but there's no need to put /// those in the mainline remote write path. /// /// /// \param pv_Offset byte offset count based on whence /// \param pv_Whence values are as follow: \n /// SEEK_SET - The offset is set to offset bytes. \n /// SEEK_CUR - The offset is set to its current /// location plus offset bytes. \n /// SEEK_END - The offset is set to the size of the /// file plus offset bytes. /// /// \retval off_t SUCCESS: The resulting offset location from beginning /// of current fragment file.\n /// FAILURE: -1 /// /////////////////////////////////////////////////////////////////////////////// ssize_t STFSLIB_write( stfs_fhndl_t pv_Fhandle, const void *pp_Buf, size_t pv_Count) { const char *WHERE = "STFSLIB_write"; STFS_ScopeTrace lv_st(WHERE); if (!pp_Buf) { errno = EINVAL; return -1; } STFS_ExternalFileHandle *lp_Efh = STFS_ExternalFileHandle::GetExternalFileHandle(pv_Fhandle); if (! lp_Efh) { TRACE_PRINTF2(1, "Error in obtaining External File Handle for %ld\n", pv_Fhandle); return -1; } lp_Efh->ResetErrors(); bool lv_WriteCompleted = false; ssize_t lv_totalCountWritten = 0; void *lp_currBuffPtr = (void *) pp_Buf; ssize_t lv_writeSize = pv_Count; while (!lv_WriteCompleted) { lp_Efh->SetCurrentFragmentBasedOnCurrOffset(); ssize_t lv_Stat = lp_Efh->Write( lp_currBuffPtr, lv_writeSize); if (lv_Stat == lv_writeSize) { // The usual case: we wrote everything as expected. Prepare // to exit the loop and go home for the day! lv_totalCountWritten += lv_writeSize; lv_WriteCompleted = true; } else if (lv_Stat < 0) { // Hmm, do we need to create a new fragment? if (errno == ENOSPC) { //gotcha! Create a new fragment and continue writing the //buffer STFS_ExternalFileMetadata *lp_EFM = lp_Efh->EfmGet(); // make sure that the lib has the same opinion as the // daemon on the new fragment's offset size_t lv_newFragExpectedOffset = lp_Efh->CurrentOffsetGet(); #ifdef SQ_PACK STFS_OpenIdentifier lp_openIdentifier = lp_Efh->OpenIdentifierGet(); lv_Stat = SendToSTFSd_createFrag(lp_EFM, STFS_util::GetMyNodeId(), STFS_util::GetMyPID(), &lp_openIdentifier); #else lv_Stat = STFSd_createFragment(lp_EFM, STFS_util::GetMyNodeId(), STFS_util::GetMyPID(), lp_Efh->CurrentOffsetGet()); #endif if (lv_Stat < 0) { // Fragment not created. Treat it like a normal ENOSPC // error after all. If the error on the createFrag was // something else, then we'll hope that it was put in // the open error structure... errno = ENOSPC; } else { // open the new fragment lv_Stat = lp_Efh->OpenNewFragments(); if (lv_Stat < 0) { errno=ENOSPC; } // Now check to see if the new fragment has the correct // starting offset. The new fragment isn't yet the // current one, so we have to extract the current number // of fragments, use it as the index to get the last // fragment's FFM, and call OffsetGet on that fragment. size_t lv_newFragRealStartOffset = lp_Efh->GetLastFFMStartOffset(); if (lv_newFragRealStartOffset != lv_newFragExpectedOffset) { // Something's wrong because the daemon has a // different opinion on the previous fragment's // size or our calculations are out of sync. This // is a data integrity problem // This code should really be fixed to change the // if condition to be a Permassert or the // following Assert (false) to kill the library // process. STFS_util::SoftwareFailureHandler (WHERE); } } } if (lv_Stat < 0) { // we've got a real, live error here! // before we just return this error, check to see if we've // written anything! If so, we want to return that // value. It's up to the library caller to decide whether // to retry the I/O in that case. if (lv_totalCountWritten > 0) { return lv_totalCountWritten; } else { // a real error! Make sure errno wasn't overwritten by // this point! Any additional information or subsequent // errors should be stuffed in the error structure by // lower calls. return -1; } } // Not ENOSPC check } // lv_Stat < 0 check else { //Rev. Following line is comparing signed vs unsigned. ASSERT ((size_t)(lv_totalCountWritten+lv_Stat) < pv_Count); // partial write! Hmmmmm, might mean we need a new fragment // (or it might mean there was partial write). We need to // save the number of bytes we wrote and then try again to // write the data. We also have to save our current // location in the buffer and calculate the new number of // bytes to write. // Careful here: it's possible that a single large write // might create more than one fragment! Make sure that // whatever code gets added uses lv_CountWritten and lv_Stat // with the proper context. lv_totalCountWritten += lv_Stat; lp_currBuffPtr = (char *)lp_currBuffPtr + lv_Stat; lv_writeSize -= lv_Stat; } } // While !lv_WriteCompleted return lv_totalCountWritten; }
/////////////////////////////////////////////////////////////////////////////// /// // STFSd_open /// /// \brief Opens an existing STFS file /// /// \param const char *pp_Path, /// \param int pv_OFlag, /// \param pv_OpenerNodeId, /// \param pv_OpenerPID, /// \param STFS_ExternalFileMetadata *&pp_Efm, /// \param STFS_FragmentFileMetadata *&pp_Ffm, /// \param STFS_OpenIdentifier *&pp_OpenId) /// /////////////////////////////////////////////////////////////////////////////// int STFSd_open(const char *pp_Path, int pv_OFlag, int pv_OpenerNodeId, int pv_OpenerPID, STFS_ExternalFileMetadata *&pp_Efm, STFS_FragmentFileMetadata *&pp_Ffm, STFS_OpenIdentifier *&pp_OpenId) { const char *WHERE = "STFSd_open"; STFS_ScopeTrace lv_st(WHERE); STFS_Session *lp_Session = STFS_Session::GetSession(); int lv_Ret = 0; if (!pp_Path) { errno = EINVAL; char lp_ErrMess[] = "Error Path is NULL"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -1; } char lp_Path[STFS_PATH_MAX]; strcpy(lp_Path, pp_Path); STFS_ExternalFileMetadata *lp_Efm = STFS_ExternalFileMetadata::GetFromContainer(lp_Path); if(!lp_Efm) { TRACE_PRINTF2(1,"%s\n", "Error getting lp_Efm"); errno = EINVAL; char lp_ErrMess[] = "Error getting lp_Efm"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -2; } TRACE_PRINTF4(2, "FileName: %s, Opener Node Id: %d, Opener SQ PID: %d\n", lp_Efm->ExternalFileNameGet(), pv_OpenerNodeId, pv_OpenerPID); STFS_util::ValidateEFM(lp_Efm); if(!lp_Efm->FileAvailableGet()) { TRACE_PRINTF2(1, "File: %s is no longer available\n", pp_Path); errno = ENOENT; char lp_ErrMess[] = "Error file is no longer available"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -1; } int lv_FragmentOffset = 0; //If O_APPEND, last fragment will be set open, if((pv_OFlag & O_APPEND) == O_APPEND) { lv_FragmentOffset = lp_Efm->GetNumFragments(); if(lv_FragmentOffset == 0) { TRACE_PRINTF2(1,"%s\n", "Error Number of Fragments returned is 0"); errno = ENOENT; char lp_ErrMess[] = "Error number of fragments returned is 0"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -3; } lv_FragmentOffset -= 1; } STFS_FragmentFileMetadata *lp_Ffm = lp_Efm->GetFragment(lv_FragmentOffset); if(!lp_Ffm) { errno = ENOENT; char lp_ErrMess[] = "Error obtaining fragment"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -4; } STFS_ExternalFileHandle *lp_Efh = lp_Efm->Open(true); if (!lp_Efh) { // TBD cleanup errno = ENOENT; char lp_ErrMess[] = "Error allocating EFH"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -5; } lv_Ret = StoreOpenerInfo(lp_Efm, pv_OpenerNodeId, pv_OpenerPID, pp_OpenId); if (lv_Ret < 0) { //TBD cleanup errno = ENOENT; char lp_ErrMess[] = "Error storing opener information"; lp_Session->SetError(true, errno, 0, lp_ErrMess, strlen(lp_ErrMess)); return -6; } pp_Efm = lp_Efm; pp_Ffm = lp_Ffm; //Build Success Reply return 0; }
int STFSLIB_close(stfs_fhndl_t pv_Fhandle) { const char *WHERE = "STFSLIB_close"; STFS_ScopeTrace lv_st(WHERE); STFS_ExternalFileHandle *lp_Efh = STFS_ExternalFileHandle::GetExternalFileHandle(pv_Fhandle); if (! lp_Efh) { errno = EBADF; TRACE_PRINTF2(1, "Error in obtaining External File Handle for %ld\n", pv_Fhandle); return -1; } lp_Efh->ResetErrors(); /// Rev: Do we close . Set the state. Continue on errors. // Closes the fragments int lv_Ret = lp_Efh->Close(); if (lv_Ret < 0) { TRACE_PRINTF3(1, "Error %d while closing handle: %ld\n", lv_Ret, pv_Fhandle); // continue on error when closing } STFS_ExternalFileMetadata *lp_Efm = lp_Efh->EfmGet(); if (!lp_Efm) { // corruption in some STFS maintained data structures. STFS_util::SoftwareFailureHandler(WHERE); return -1; } lv_Ret = lp_Efm->Close(false); if (lv_Ret < 0) { TRACE_PRINTF3(1,"Error %d while closing handle: %ld\n", lv_Ret, pv_Fhandle); return -1; } lv_Ret = STFS_ExternalFileMetadata::DeleteFromContainer(lp_Efm); if (lv_Ret < 0) { STFS_util::SoftwareFailureHandler(WHERE); return lv_Ret; } // send message to the STFS daemon #ifdef SQ_PACK lv_Ret = SendToSTFSd_close(&(lp_Efh->OpenIdentifierGet()), STFS_util::GetMyNodeId(), STFS_util::GetMyPID()); if (lv_Ret < 0) { // 'errno' should be set here return lv_Ret; } #else lv_Ret = STFSd_close(&(lp_Efh->OpenIdentifierGet())); if (lv_Ret < 0) { return lv_Ret; } #endif // remove Efh from the Container lv_Ret = STFS_ExternalFileHandle::DeleteFromContainer(lp_Efh); if (lv_Ret < 0) { STFS_util::SoftwareFailureHandler(WHERE); return lv_Ret; } return 0; }
////////////////////////////////////////////////////////////////////////////// /// // STFSLIB_read /// /// \brief Driver code for an STFS_read() library call /// ////////////////////////////////////////////////////////////////////////////// ssize_t STFSLIB_read( stfs_fhndl_t pv_Fhandle, void *pp_Buf, size_t pv_Count) { const char *WHERE = "STFSLIB_read"; STFS_ScopeTrace lv_st(WHERE); if (!pp_Buf) { errno = EFAULT; return -1; } STFS_ExternalFileHandle *lp_Efh = STFS_ExternalFileHandle::GetExternalFileHandle(pv_Fhandle); if (! lp_Efh) { TRACE_PRINTF2(1, "Error in obtaining External File Handle for %ld\n", pv_Fhandle); return -1; } lp_Efh->ResetErrors(); size_t lv_totalBytesRead = 0; bool lv_readCompleted = false; char * lp_currBufPtr = (char *) pp_Buf; size_t lv_readSize = pv_Count; while (!lv_readCompleted) { ssize_t lv_Stat = 0; int lv_Ret = 0; //Check if the current fragment is local. If not, then send req to daemon lv_Ret = lp_Efh->IsCurrFragmentLocal(); if(lv_Ret < 0) { //error evaluating fragment return -1; } else if(lv_Ret == 1) { lv_Stat = lp_Efh->Read((char *)lp_currBufPtr, lv_readSize); } else { #ifdef SQ_PACK lv_Stat= SendToSTFSd_read(); #else lv_Stat = STFSd_read(&(lp_Efh->OpenIdentifierGet()), (char *)lp_currBufPtr, lv_readSize); #endif //The currentFFH_ is not local so we will need to read // through remote daemon } if (( (size_t)lv_Stat + lv_totalBytesRead) == pv_Count) { // the usual case; we read everything as expected. Prepare to exit our // read-loop lv_totalBytesRead += lv_Stat; lv_readCompleted = true; } else if (lv_Stat == 0) { // Didn't read anything else in this fragment. Can we roll to another? // Gotcha! There might be a new fragment here! Check to see if there // are any new fragments then loop around and read again. // If we're moving to a new FFH that's already in our list, the // reposition at the top of the loop should move to it automatically, // so we just skip ahead. // // But if this is the last FFH, we need to see if either this process // or the owning STFSd knows about any more fragments. We take a lazy // approach to doing this, hoping that somewhere along the way, // someone has already retrieved this information for us: // // 1) See if the EFM already knows about other fragments. Occurs // this process owns read access to the file in another open, or // another opener in this process already got an updated list of // FFMs. // // 2) If the number of FFHs matches the number of FFMs and we're in // the last FFM, check with the owning daemon to update the // metadata for this file. if (lp_Efh->CurrentFFHIsLast() == true) { if (lp_Efh->NewFFHsNeeded() == true) { // Someone else found fragments for us. Logically open them for // this efh and try the read again. lv_Stat = lp_Efh ->OpenNewFragments(); if (lv_Stat < 0) { return (lv_totalBytesRead>0 ? lv_totalBytesRead : lv_Stat); } } else { // We have all the FFHs that this lib process knows about, // check to see if new fragments available bool lv_newFragsDetected = LibRead_GetMoreFragments (lp_Efh); if (lv_newFragsDetected == false) { // a real EOF! Clean up errno after all the calls we've made errno = 0; return lv_totalBytesRead; } // We found new fragments. Open them and try to read the rest of // the data. lv_Stat = lp_Efh ->OpenNewFragments(); if (lv_Stat < 0) { // hope that any error was set below return (lv_totalBytesRead>0 ? lv_totalBytesRead : -1); } } // else } } // 0 count error handling else if (lv_Stat<0) { // Error! We might be done here. We can't just return this error // though. We have to check to see if we read anything. If so, return // that value and when the caller comes back, the error might (or might // not) happen again. return (lv_totalBytesRead>0 ? lv_totalBytesRead : -1); } else { // partial read! Adjust our counters and try again // find out... ASSERT ( (size_t) lv_Stat < pv_Count); lv_totalBytesRead += lv_Stat; lp_currBufPtr = (char *)lp_currBufPtr + lv_Stat; lv_readSize -= lv_Stat; } if (lp_Efh->SetCurrentFragmentBasedOnCurrOffset() == true) { // switched frags! We're reading sequentially, so we need to do an // explicit lseek in case this fragment's FD was previously used for // some other I/O operation. lv_Stat = lp_Efh->SeekToCurrFragmentStart(); if (lv_Stat < 0) { // physical position failed! return (lv_totalBytesRead>0 ? lv_totalBytesRead : lv_Stat); } } } //while !readcompleted return lv_totalBytesRead; }