bool CClientThread::ProcessPacket(CRData *data)
{
uchar id;
if( data->getUChar(&id)==true )
{
switch(id)
{
case ID_GET_GAMELIST:
{
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident))
{
Log("Identity check failed -1", LL_DEBUG);
return false;
}
#endif
hFile=0;
std::vector<std::wstring> games=get_maps();
Log("Sending game list", LL_DEBUG);
EnableNagle();
CWData data;
data.addUChar( ID_GAMELIST );
data.addUInt( (unsigned int)games.size() );
stack.Send(clientpipe, data);
for(size_t i=0;i<games.size();++i)
{
std::string version;
std::wstring udir;
version=getFile(wnarrow(map_file(games[i]+L"\\version.uri",true,&udir)));
if( udir!=L"" )
games[i]+=L"|"+udir;
std::string game=Server->ConvertToUTF8(games[i]);
stack.Send(clientpipe, (char*)game.c_str(), game.size() );
stack.Send(clientpipe, (char*)version.c_str(), version.size() );
}
Log("done.", LL_DEBUG);
DisableNagle();
}break;
case ID_GET_FILE_RESUME:
case ID_GET_FILE:
case ID_GET_FILE_RESUME_HASH:
{
std::string s_filename;
if(data->getStr(&s_filename)==false)
break;
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident))
{
Log("Identity check failed -2", LL_DEBUG);
return false;
}
#endif
std::wstring o_filename=Server->ConvertToUnicode(s_filename);
_i64 start_offset=0;
bool offset_set=data->getInt64(&start_offset);
Log("Sending file "+Server->ConvertToUTF8(o_filename), LL_DEBUG);
std::wstring filename=map_file(o_filename);
Log("Mapped name: "+Server->ConvertToUTF8(filename), LL_DEBUG);
if(filename.empty())
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -1", LL_DEBUG);
return false;
}
Log("Info: Base dir lost -1", LL_DEBUG);
break;
}
cmd_id=id;
if( id==ID_GET_FILE_RESUME_HASH )
{
hash_func.init();
}
#ifdef _WIN32
if(filename.size()<2 || (filename[0]!='\\' && filename[1]!='\\' ) )
{
filename=L"\\\\?\\"+filename;
}
if(bufmgr==NULL)
{
bufmgr=new CBufMgr(NBUFFERS,READSIZE);
}
#endif
#ifndef LINUX
#ifndef BACKUP_SEM
hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED|FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#else
hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED|FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#endif
if(hFile == INVALID_HANDLE_VALUE)
{
hFile=NULL;
#ifdef CHECK_BASE_PATH
std::wstring basePath=map_file(getuntil(L"/",o_filename)+L"/");
if(!isDirectory(basePath))
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG);
return false;
}
Log("Info: Base dir lost", LL_DEBUG);
break;
}
#endif
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Info: Couldn't open file", LL_DEBUG);
break;
}
currfilepart=0;
sendfilepart=0;
sent_bytes=start_offset;
LARGE_INTEGER filesize;
GetFileSizeEx(hFile, &filesize);
curr_filesize=filesize.QuadPart;
next_checkpoint=start_offset+c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
if( offset_set==false || id==ID_GET_FILE_RESUME || id==ID_GET_FILE_RESUME_HASH )
{
CWData data;
data.addUChar(ID_FILESIZE);
data.addUInt64(filesize.QuadPart);
int rc=SendInt(data.getDataPtr(), data.getDataSize());
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: SendSize", LL_DEBUG);
CloseHandle(hFile);
hFile=NULL;
return false;
}
}
if(filesize.QuadPart==0)
{
CloseHandle(hFile);
hFile=NULL;
break;
}
for(_i64 i=start_offset;i<filesize.QuadPart && stopped==false;i+=READSIZE)
{
bool last;
if(i+READSIZE<filesize.QuadPart)
last=false;
else
{
last=true;
Log("Reading last file part", LL_DEBUG);
}
while(bufmgr->nfreeBufffer()==0 && stopped==false)
{
int rc;
SleepEx(0,true);
rc=SendData();
if(rc==-1)
{
Log("Error: Send failed in file loop -1", LL_DEBUG);
CloseThread(hFile);
}
else if(rc==0)
SleepEx(1,true);
}
if( stopped==false )
ReadFilePart(hFile, i, last);
if(FileServ::isPause() )
{
DWORD starttime=GetTickCount();
while(GetTickCount()-starttime<5000)
{
SleepEx(500,true);
int rc=SendData();
if(rc==-1)
{
Log("Error: Send failed in file pause loop -2", LL_DEBUG);
CloseThread(hFile);
}
}
}
}
while(bufmgr->nfreeBufffer()!=NBUFFERS && stopped==false)
{
SleepEx(0,true);
int rc;
rc=SendData();
if( rc==2 && bufmgr->nfreeBufffer()!=NBUFFERS )
{
Log("Error: File end and not all Buffers are free!-1", LL_WARNING);
}
if(rc==-1)
{
Log("Error: Send failed in off file loop -3", LL_DEBUG);
CloseHandle(hFile);
hFile=NULL;
break;
}
else if(rc==0)
SleepEx(1,true);
}
if( stopped==false )
{
Log("Closed file.", LL_DEBUG);
CloseHandle(hFile);
hFile=NULL;
}
#else //LINUX
hFile=open64(Server->ConvertToUTF8(filename).c_str(), O_RDONLY|O_LARGEFILE);
if(hFile == INVALID_HANDLE_VALUE)
{
#ifdef CHECK_BASE_PATH
std::wstring basePath=map_file(getuntil(L"/",o_filename)+L"/");
if(!isDirectory(basePath))
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG);
return false;
}
Log("Info: Base dir lost", LL_DEBUG);
break;
}
#endif
hFile=0;
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Info: Couldn't open file", LL_DEBUG);
break;
}
currfilepart=0;
sendfilepart=0;
struct stat64 stat_buf;
fstat64(hFile, &stat_buf);
off64_t filesize=stat_buf.st_size;
curr_filesize=filesize;
if( offset_set==false || id==ID_GET_FILE_RESUME || id==ID_GET_FILE_RESUME_HASH )
{
CWData data;
data.addUChar(ID_FILESIZE);
data.addUInt64(filesize);
int rc=SendInt(data.getDataPtr(), data.getDataSize() );
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: SendSize", LL_DEBUG);
CloseHandle(hFile);
hFile=0;
return false;
}
}
if(filesize==0)
{
CloseHandle(hFile);
hFile=0;
break;
}
off64_t foffset=start_offset;
unsigned int s_bsize=8192;
if(id==ID_GET_FILE || id==ID_GET_FILE_RESUME )
{
s_bsize=32768;
next_checkpoint=curr_filesize;
}
else
{
next_checkpoint=start_offset+c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
}
char *buf=new char[s_bsize];
bool has_error=false;
while( foffset < filesize )
{
size_t count=(std::min)((size_t)s_bsize, (size_t)(next_checkpoint-foffset));
if( clientpipe==NULL && ( id==ID_GET_FILE || id==ID_GET_FILE_RESUME ) )
{
ssize_t rc=sendfile64(int_socket, hFile, &foffset, count);
if(rc>0)
{
foffset+=rc;
}
else
{
Log("Error: Reading and sending from file failed", LL_DEBUG);
CloseHandle(hFile);
delete []buf;
return false;
}
}
else
{
ssize_t rc=read(hFile, buf, count);
if(rc>0)
{
rc=SendInt(buf, rc);
if(rc==SOCKET_ERROR)
{
Log("Error: Sending data failed");
CloseHandle(hFile);
delete []buf;
return false;
}
else if(id==ID_GET_FILE_RESUME_HASH)
{
hash_func.update((unsigned char*)buf, rc);
}
foffset+=rc;
}
else
{
Log("Error: Reading from file failed", LL_DEBUG);
CloseHandle(hFile);
delete []buf;
return false;
}
if(id==ID_GET_FILE_RESUME_HASH && foffset==next_checkpoint)
{
hash_func.finalize();
SendInt((char*)hash_func.raw_digest_int(), 16);
next_checkpoint+=c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
hash_func.init();
}
}
if(FileServ::isPause() )
{
Sleep(500);
}
}
CloseHandle(hFile);
delete []buf;
hFile=0;
#endif
}break;
case ID_GET_FILE_BLOCKDIFF:
{
bool b=GetFileBlockdiff(data);
if(!b)
return false;
}break;
case ID_BLOCK_REQUEST:
{
if(state==CS_BLOCKHASH)
{
Handle_ID_BLOCK_REQUEST(data);
}
}break;
}
}
if( stopped==true )
return false;
else
return true;
}
void BackupServerPrepareHash::operator()(void)
{
while(true)
{
working=false;
std::string data;
size_t rc=pipe->Read(&data);
if(data=="exitnow")
{
output->Write("exitnow");
std::string t;
exitpipe_hash->Read(&t);
Server->destroy(exitpipe_hash);
exitpipe->Write("ok");
Server->Log("server_prepare_hash Thread finished");
delete this;
return;
}
else if(data=="exit")
{
output->Write("exit");
Server->destroy(exitpipe);
Server->Log("server_prepare_hash Thread finished");
delete this;
return;
}
else if(data=="flush")
{
continue;
}
if(rc>0)
{
working=true;
CRData rd(&data);
std::string temp_fn;
rd.getStr(&temp_fn);
int backupid;
rd.getInt(&backupid);
char incremental;
rd.getChar(&incremental);
std::string tfn;
rd.getStr(&tfn);
std::string hashpath;
rd.getStr(&hashpath);
std::string hashoutput_fn;
rd.getStr(&hashoutput_fn);
bool diff_file=!hashoutput_fn.empty();
std::string old_file_fn;
rd.getStr(&old_file_fn);
IFile *tf=Server->openFile(os_file_prefix(Server->ConvertToUnicode(temp_fn)), MODE_READ);
IFile *old_file=NULL;
if(diff_file)
{
old_file=Server->openFile(os_file_prefix(Server->ConvertToUnicode(old_file_fn)), MODE_READ);
if(old_file==NULL)
{
ServerLogger::Log(clientid, "Error opening file \""+old_file_fn+"\" from pipe for reading. File: old_file", LL_ERROR);
if(tf!=NULL) Server->destroy(tf);
continue;
}
}
if(tf==NULL)
{
ServerLogger::Log(clientid, "Error opening file \""+temp_fn+"\" from pipe for reading file. File: temp_fn", LL_ERROR);
has_error=true;
if(old_file!=NULL)
{
Server->destroy(old_file);
}
}
else
{
ServerLogger::Log(clientid, "PT: Hashing file \""+ExtractFileName(tfn)+"\"", LL_DEBUG);
std::string h;
if(!diff_file)
{
h=hash(tf);
}
else
{
h=hash_with_patch(old_file, tf);
}
Server->destroy(tf);
if(old_file!=NULL)
{
Server->destroy(old_file);
}
CWData data;
data.addString(temp_fn);
data.addInt(backupid);
data.addChar(incremental);
data.addString(tfn);
data.addString(hashpath);
data.addString(h);
data.addString(hashoutput_fn);
data.addString(old_file_fn);
output->Write(data.getDataPtr(), data.getDataSize() );
}
}
}
}
bool CClientThread::GetFileBlockdiff(CRData *data)
{
std::string s_filename;
if(data->getStr(&s_filename)==false)
return false;
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident))
{
Log("Identity check failed -2", LL_DEBUG);
return false;
}
#endif
std::wstring o_filename=Server->ConvertToUnicode(s_filename);
_i64 start_offset=0;
data->getInt64(&start_offset);
curr_hash_size=0;
data->getInt64(&curr_hash_size);
Log("Sending file "+Server->ConvertToUTF8(o_filename), LL_DEBUG);
std::wstring filename=map_file(o_filename);
Log("Mapped name: "+Server->ConvertToUTF8(filename), LL_DEBUG);
if(filename.empty())
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -1", LL_DEBUG);
return false;
}
Log("Info: Base dir lost -1", LL_DEBUG);
return true;
}
hash_func.init();
#ifdef _WIN32
if(filename.size()<2 || (filename[0]!='\\' && filename[1]!='\\' ) )
filename=L"\\\\?\\"+filename;
#endif
#ifdef _WIN32
#ifndef BACKUP_SEM
hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#else
hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#endif
#else //_WIN32
hFile=open64(Server->ConvertToUTF8(filename).c_str(), O_RDONLY|O_LARGEFILE);
#endif //_WIN32
if(hFile == INVALID_HANDLE_VALUE)
{
hFile=(HANDLE)NULL;
#ifdef CHECK_BASE_PATH
std::wstring basePath=map_file(getuntil(L"/",o_filename)+L"/");
if(!isDirectory(basePath))
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG);
return false;
}
Log("Info: Base dir lost", LL_DEBUG);
return true;
}
#endif
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Info: Couldn't open file", LL_DEBUG);
return true;
}
currfilepart=0;
sendfilepart=0;
sent_bytes=0;
#ifdef _WIN32
LARGE_INTEGER filesize;
GetFileSizeEx(hFile, &filesize);
curr_filesize=filesize.QuadPart;
#else
struct stat64 stat_buf;
fstat64(hFile, &stat_buf);
curr_filesize=stat_buf.st_size;
#endif
next_checkpoint=start_offset+c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
CWData sdata;
sdata.addUChar(ID_FILESIZE);
sdata.addUInt64(curr_filesize);
SendInt(sdata.getDataPtr(), sdata.getDataSize());
if(mutex==NULL)
{
mutex=Server->createMutex();
cond=Server->createCondition();
}
state=CS_BLOCKHASH;
if(chunk_send_thread_ticket==ILLEGAL_THREADPOOL_TICKET)
{
IFile * tf=Server->openFileFromHandle((void*)hFile);
if(tf==NULL)
{
Log("Could not open file from handle", LL_ERROR);
return false;
}
chunk_send_thread_ticket=Server->getThreadPool()->execute(new ChunkSendThread(this, tf, curr_hash_size) );
}
else
{
IScopedLock lock(mutex);
update_file=Server->openFileFromHandle((void*)hFile);
if(update_file==NULL)
{
Log("Could not open update file from handle", LL_ERROR);
}
cond->notify_all();
}
hFile=(HANDLE)NULL;
return true;
}
void LMDBFileIndex::put_internal(const SIndexKey& key, int64 value, int flags, bool log, bool handle_enosp)
{
CWData vdata;
vdata.addVarInt(value);
MDB_val mdb_tkey;
mdb_tkey.mv_data=const_cast<void*>(static_cast<const void*>(&key));
mdb_tkey.mv_size=sizeof(SIndexKey);
MDB_val mdb_tvalue;
mdb_tvalue.mv_data=vdata.getDataPtr();
mdb_tvalue.mv_size=vdata.getDataSize();
int rc = mdb_put(txn, dbi, &mdb_tkey, &mdb_tvalue, flags);
if(rc==MDB_MAP_FULL && handle_enosp)
{
mdb_txn_abort(txn);
if(_has_error)
{
Server->Log("LMDB had error during increase (on put). Aborting...", LL_ERROR);
start_transaction();
return;
}
{
read_transaction_lock.reset();
IScopedWriteLock lock(mutex);
destroy_env();
map_size*=2;
Server->Log("Increased LMDB database size to "+PrettyPrintBytes(map_size)+" (on put)", LL_DEBUG);
if(!create_env())
{
Server->Log("Error creating env after database file size increase", LL_ERROR);
_has_error=true;
start_transaction();
return;
}
}
start_transaction();
replay_transaction_log();
put_internal(key, value, flags, false, true);
}
else if(rc==MDB_BAD_TXN && handle_enosp)
{
mdb_txn_abort(txn);
if(_has_error)
{
Server->Log("LMDB had error on BAD_TXN (on put). Aborting...", LL_ERROR);
start_transaction();
return;
}
start_transaction();
replay_transaction_log();
put_internal(key, value, flags, false, false);
}
else if(rc)
{
Server->Log("LMDB: Failed to put data ("+(std::string)mdb_strerror(rc)+")", LL_ERROR);
_has_error=true;
}
if(!_has_error && log)
{
STransactionLogItem item = { key, value, flags};
transaction_log.push_back(item);
}
}
