size_t CEnvCanGribForecast::GetLatestHH(CHttpConnectionPtr& pConnection)const
{
ERMsg msg;
size_t HH = NOT_INIT;
vector<pair<CTRef, size_t>> latest;
for (size_t h = 0; h < 24; h += 6)
{
string remotePath = GetRemoteFilePath(h, 0, (m_type == GT_HRDPS) ? "*P000-00.grib2":"*P000.grib2");
CFileInfoVector fileListTmp;
if (FindFiles(pConnection, remotePath, fileListTmp) && !fileListTmp.empty())
{
string fileTitle = GetFileName(fileListTmp.front().m_filePath);
latest.push_back(make_pair(GetTRef(fileTitle), h));
}
}
sort(latest.begin(), latest.end());
if (!latest.empty())
HH = latest.back().second;
return HH;
}
// Generate
//------------------------------------------------------------------------------
bool ToolManifest::Generate( const Node * mainExecutable, const Dependencies & dependencies )
{
m_Files.Clear();
m_TimeStamp = 0;
m_Files.SetCapacity( 1 + dependencies.GetSize() );
// unify "main executable" and "extra files"
// (loads contents of file into memory, and creates hashes)
if ( !AddFile( mainExecutable ) )
{
return false; // AddFile will have emitted error
}
for ( size_t i=0; i<dependencies.GetSize(); ++i )
{
const FileNode & n = *( dependencies[ i ].GetNode()->CastTo< FileNode >() );
if ( !AddFile( &n ) )
{
return false; // AddFile will have emitted error
}
}
// create a hash for the whole tool chain
const size_t numFiles( m_Files.GetSize() );
const size_t memSize( numFiles * sizeof( uint32_t ) * 2 );
uint32_t * mem = (uint32_t *)ALLOC( memSize );
uint32_t * pos = mem;
for ( size_t i=0; i<numFiles; ++i )
{
const File & f = m_Files[ i ];
// file contents
*pos = f.m_Hash;
++pos;
// file name & sub-path (relative to remote folder)
AStackString<> relativePath;
GetRemoteFilePath( (uint32_t)i, relativePath, false ); // false = don't use full path
*pos = xxHash::Calc32( relativePath );
++pos;
}
m_ToolId = xxHash::Calc64( mem, memSize );
FREE( mem );
// update time stamp (most recent file in manifest)
for ( size_t i=0; i<numFiles; ++i )
{
const File & f = m_Files[ i ];
ASSERT( f.m_TimeStamp ); // should have had an error before if the file was missing
m_TimeStamp = Math::Max( m_TimeStamp, f.m_TimeStamp );
}
return true;
}
// ReceiveFileData
//------------------------------------------------------------------------------
bool ToolManifest::ReceiveFileData( uint32_t fileId, const void * data, size_t & dataSize )
{
MutexHolder mh( m_Mutex );
File & f = m_Files[ fileId ];
// gracefully handle multiple receipts of the same data
if ( f.m_Content )
{
ASSERT( f.m_SyncState == File::SYNCHRONIZED );
return true;
}
ASSERT( f.m_SyncState == File::SYNCHRONIZING );
// prepare name for this file
AStackString<> fileName;
GetRemoteFilePath( fileId, fileName );
// prepare destination
AStackString<> pathOnly( fileName.Get(), fileName.FindLast( NATIVE_SLASH ) );
if ( !FileIO::EnsurePathExists( pathOnly ) )
{
return false; // FAILED
}
// write to disk
FileStream fs;
if ( !fs.Open( fileName.Get(), FileStream::WRITE_ONLY ) )
{
return false; // FAILED
}
if ( fs.Write( data, dataSize ) != dataSize )
{
return false; // FAILED
}
fs.Close();
// open read-only
AutoPtr< FileStream > fileStream( FNEW( FileStream ) );
if ( fileStream.Get()->Open( fileName.Get(), FileStream::READ_ONLY ) == false )
{
return false; // FAILED
}
// This file is now synchronized
f.m_FileLock = fileStream.Release(); // NOTE: Keep file open to prevent deletion
f.m_SyncState = File::SYNCHRONIZED;
// is completely synchronized?
const File * const end = m_Files.End();
for ( const File * it = m_Files.Begin(); it != end; ++it )
{
if ( it->m_SyncState != File::SYNCHRONIZED )
{
// still some files to be received
return true; // file stored ok
}
}
// all files received
m_Synchronized = true;
return true; // file stored ok
}
// Deserialize
//------------------------------------------------------------------------------
void ToolManifest::Deserialize( IOStream & ms )
{
ms.Read( m_ToolId );
ASSERT( m_Files.IsEmpty() );
uint32_t numFiles( 0 );
ms.Read( numFiles );
m_Files.SetCapacity( numFiles );
for ( size_t i=0; i<(size_t)numFiles; ++i )
{
AStackString<> name;
uint64_t timeStamp( 0 );
uint32_t hash( 0 );
uint32_t contentSize( 0 );
ms.Read( name );
ms.Read( timeStamp );
ms.Read( hash );
ms.Read( contentSize );
m_Files.Append( File( name, timeStamp, hash, nullptr, contentSize ) );
}
// determine if any files are remaining from a previous run
size_t numFilesAlreadySynchronized = 0;
for ( size_t i=0; i<(size_t)numFiles; ++i )
{
AStackString<> localFile;
GetRemoteFilePath( (uint32_t)i, localFile );
// is this file already present?
AutoPtr< FileStream > fileStream( FNEW( FileStream ) );
FileStream & f = *( fileStream.Get() );
if ( f.Open( localFile.Get() ) == false )
{
continue; // file not found
}
if ( f.GetFileSize() != m_Files[ i ].m_ContentSize )
{
continue; // file is not complete
}
AutoPtr< char > mem( (char *)ALLOC( (size_t)f.GetFileSize() ) );
if ( f.Read( mem.Get(), (size_t)f.GetFileSize() ) != f.GetFileSize() )
{
continue; // problem reading file
}
if( Murmur3::Calc32( mem.Get(), (size_t)f.GetFileSize() ) != m_Files[ i ].m_Hash )
{
continue; // file contents unexpected
}
// file present and ok
m_Files[ i ].m_FileLock = fileStream.Release(); // NOTE: keep file open to prevent deletions
m_Files[ i ].m_SyncState = File::SYNCHRONIZED;
numFilesAlreadySynchronized++;
}
// Generate Environment
ASSERT( m_RemoteEnvironmentString == nullptr );
// PATH=
AStackString<> basePath;
GetRemotePath( basePath );
AStackString<> paths;
paths.Format( "PATH=%s", basePath.Get() );
// TMP=
AStackString<> normalTmp;
Env::GetEnvVariable( "TMP", normalTmp );
AStackString<> tmp;
tmp.Format( "TMP=%s", normalTmp.Get() );
// SystemRoot=
AStackString<> sysRoot( "SystemRoot=C:\\Windows" );
char * mem = (char *)ALLOC( paths.GetLength() + 1 +
tmp.GetLength() + 1 +
sysRoot.GetLength() + 1 +
1 );
m_RemoteEnvironmentString = mem;
AString::Copy( paths.Get(), mem, paths.GetLength() + 1 ); // including null
mem += ( paths.GetLength() + 1 ); // including null
AString::Copy( tmp.Get(), mem, tmp.GetLength() + 1 ); // including null
mem += ( tmp.GetLength() + 1 ); // including null
AString::Copy( sysRoot.Get(), mem, sysRoot.GetLength() + 1 ); // including null
mem += ( sysRoot.GetLength() + 1 ); // including null
*mem = 0; ++mem; // double null
// are all files already present?
if ( numFilesAlreadySynchronized == m_Files.GetSize() )
{
m_Synchronized = true;
}
}
