size_t CEnvCanGribForecast::GetLatestHH(CHttpConnectionPtr& pConnection)const { ERMsg msg; size_t HH = NOT_INIT; vector<pair<CTRef, size_t>> latest; for (size_t h = 0; h < 24; h += 6) { string remotePath = GetRemoteFilePath(h, 0, (m_type == GT_HRDPS) ? "*P000-00.grib2":"*P000.grib2"); CFileInfoVector fileListTmp; if (FindFiles(pConnection, remotePath, fileListTmp) && !fileListTmp.empty()) { string fileTitle = GetFileName(fileListTmp.front().m_filePath); latest.push_back(make_pair(GetTRef(fileTitle), h)); } } sort(latest.begin(), latest.end()); if (!latest.empty()) HH = latest.back().second; return HH; }
// Generate //------------------------------------------------------------------------------ bool ToolManifest::Generate( const Node * mainExecutable, const Dependencies & dependencies ) { m_Files.Clear(); m_TimeStamp = 0; m_Files.SetCapacity( 1 + dependencies.GetSize() ); // unify "main executable" and "extra files" // (loads contents of file into memory, and creates hashes) if ( !AddFile( mainExecutable ) ) { return false; // AddFile will have emitted error } for ( size_t i=0; i<dependencies.GetSize(); ++i ) { const FileNode & n = *( dependencies[ i ].GetNode()->CastTo< FileNode >() ); if ( !AddFile( &n ) ) { return false; // AddFile will have emitted error } } // create a hash for the whole tool chain const size_t numFiles( m_Files.GetSize() ); const size_t memSize( numFiles * sizeof( uint32_t ) * 2 ); uint32_t * mem = (uint32_t *)ALLOC( memSize ); uint32_t * pos = mem; for ( size_t i=0; i<numFiles; ++i ) { const File & f = m_Files[ i ]; // file contents *pos = f.m_Hash; ++pos; // file name & sub-path (relative to remote folder) AStackString<> relativePath; GetRemoteFilePath( (uint32_t)i, relativePath, false ); // false = don't use full path *pos = xxHash::Calc32( relativePath ); ++pos; } m_ToolId = xxHash::Calc64( mem, memSize ); FREE( mem ); // update time stamp (most recent file in manifest) for ( size_t i=0; i<numFiles; ++i ) { const File & f = m_Files[ i ]; ASSERT( f.m_TimeStamp ); // should have had an error before if the file was missing m_TimeStamp = Math::Max( m_TimeStamp, f.m_TimeStamp ); } return true; }
// ReceiveFileData //------------------------------------------------------------------------------ bool ToolManifest::ReceiveFileData( uint32_t fileId, const void * data, size_t & dataSize ) { MutexHolder mh( m_Mutex ); File & f = m_Files[ fileId ]; // gracefully handle multiple receipts of the same data if ( f.m_Content ) { ASSERT( f.m_SyncState == File::SYNCHRONIZED ); return true; } ASSERT( f.m_SyncState == File::SYNCHRONIZING ); // prepare name for this file AStackString<> fileName; GetRemoteFilePath( fileId, fileName ); // prepare destination AStackString<> pathOnly( fileName.Get(), fileName.FindLast( NATIVE_SLASH ) ); if ( !FileIO::EnsurePathExists( pathOnly ) ) { return false; // FAILED } // write to disk FileStream fs; if ( !fs.Open( fileName.Get(), FileStream::WRITE_ONLY ) ) { return false; // FAILED } if ( fs.Write( data, dataSize ) != dataSize ) { return false; // FAILED } fs.Close(); // open read-only AutoPtr< FileStream > fileStream( FNEW( FileStream ) ); if ( fileStream.Get()->Open( fileName.Get(), FileStream::READ_ONLY ) == false ) { return false; // FAILED } // This file is now synchronized f.m_FileLock = fileStream.Release(); // NOTE: Keep file open to prevent deletion f.m_SyncState = File::SYNCHRONIZED; // is completely synchronized? const File * const end = m_Files.End(); for ( const File * it = m_Files.Begin(); it != end; ++it ) { if ( it->m_SyncState != File::SYNCHRONIZED ) { // still some files to be received return true; // file stored ok } } // all files received m_Synchronized = true; return true; // file stored ok }
// Deserialize //------------------------------------------------------------------------------ void ToolManifest::Deserialize( IOStream & ms ) { ms.Read( m_ToolId ); ASSERT( m_Files.IsEmpty() ); uint32_t numFiles( 0 ); ms.Read( numFiles ); m_Files.SetCapacity( numFiles ); for ( size_t i=0; i<(size_t)numFiles; ++i ) { AStackString<> name; uint64_t timeStamp( 0 ); uint32_t hash( 0 ); uint32_t contentSize( 0 ); ms.Read( name ); ms.Read( timeStamp ); ms.Read( hash ); ms.Read( contentSize ); m_Files.Append( File( name, timeStamp, hash, nullptr, contentSize ) ); } // determine if any files are remaining from a previous run size_t numFilesAlreadySynchronized = 0; for ( size_t i=0; i<(size_t)numFiles; ++i ) { AStackString<> localFile; GetRemoteFilePath( (uint32_t)i, localFile ); // is this file already present? AutoPtr< FileStream > fileStream( FNEW( FileStream ) ); FileStream & f = *( fileStream.Get() ); if ( f.Open( localFile.Get() ) == false ) { continue; // file not found } if ( f.GetFileSize() != m_Files[ i ].m_ContentSize ) { continue; // file is not complete } AutoPtr< char > mem( (char *)ALLOC( (size_t)f.GetFileSize() ) ); if ( f.Read( mem.Get(), (size_t)f.GetFileSize() ) != f.GetFileSize() ) { continue; // problem reading file } if( Murmur3::Calc32( mem.Get(), (size_t)f.GetFileSize() ) != m_Files[ i ].m_Hash ) { continue; // file contents unexpected } // file present and ok m_Files[ i ].m_FileLock = fileStream.Release(); // NOTE: keep file open to prevent deletions m_Files[ i ].m_SyncState = File::SYNCHRONIZED; numFilesAlreadySynchronized++; } // Generate Environment ASSERT( m_RemoteEnvironmentString == nullptr ); // PATH= AStackString<> basePath; GetRemotePath( basePath ); AStackString<> paths; paths.Format( "PATH=%s", basePath.Get() ); // TMP= AStackString<> normalTmp; Env::GetEnvVariable( "TMP", normalTmp ); AStackString<> tmp; tmp.Format( "TMP=%s", normalTmp.Get() ); // SystemRoot= AStackString<> sysRoot( "SystemRoot=C:\\Windows" ); char * mem = (char *)ALLOC( paths.GetLength() + 1 + tmp.GetLength() + 1 + sysRoot.GetLength() + 1 + 1 ); m_RemoteEnvironmentString = mem; AString::Copy( paths.Get(), mem, paths.GetLength() + 1 ); // including null mem += ( paths.GetLength() + 1 ); // including null AString::Copy( tmp.Get(), mem, tmp.GetLength() + 1 ); // including null mem += ( tmp.GetLength() + 1 ); // including null AString::Copy( sysRoot.Get(), mem, sysRoot.GetLength() + 1 ); // including null mem += ( sysRoot.GetLength() + 1 ); // including null *mem = 0; ++mem; // double null // are all files already present? if ( numFilesAlreadySynchronized == m_Files.GetSize() ) { m_Synchronized = true; } }