bool CClientThread::ProcessPacket(CRData *data) { uchar id; if( data->getUChar(&id)==true ) { switch(id) { case ID_GET_GAMELIST: { #ifdef CHECK_IDENT std::string ident; data->getStr(&ident); if(!FileServ::checkIdentity(ident)) { Log("Identity check failed -1", LL_DEBUG); return false; } #endif hFile=0; std::vector<std::wstring> games=get_maps(); Log("Sending game list", LL_DEBUG); EnableNagle(); CWData data; data.addUChar( ID_GAMELIST ); data.addUInt( (unsigned int)games.size() ); stack.Send(clientpipe, data); for(size_t i=0;i<games.size();++i) { std::string version; std::wstring udir; version=getFile(wnarrow(map_file(games[i]+L"\\version.uri",true,&udir))); if( udir!=L"" ) games[i]+=L"|"+udir; std::string game=Server->ConvertToUTF8(games[i]); stack.Send(clientpipe, (char*)game.c_str(), game.size() ); stack.Send(clientpipe, (char*)version.c_str(), version.size() ); } Log("done.", LL_DEBUG); DisableNagle(); }break; case ID_GET_FILE_RESUME: case ID_GET_FILE: case ID_GET_FILE_RESUME_HASH: { std::string s_filename; if(data->getStr(&s_filename)==false) break; #ifdef CHECK_IDENT std::string ident; data->getStr(&ident); if(!FileServ::checkIdentity(ident)) { Log("Identity check failed -2", LL_DEBUG); return false; } #endif std::wstring o_filename=Server->ConvertToUnicode(s_filename); _i64 start_offset=0; bool offset_set=data->getInt64(&start_offset); Log("Sending file "+Server->ConvertToUTF8(o_filename), LL_DEBUG); std::wstring filename=map_file(o_filename); Log("Mapped name: "+Server->ConvertToUTF8(filename), LL_DEBUG); if(filename.empty()) { char ch=ID_BASE_DIR_LOST; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -1", LL_DEBUG); return false; } Log("Info: Base dir lost -1", LL_DEBUG); break; } cmd_id=id; if( id==ID_GET_FILE_RESUME_HASH ) { hash_func.init(); } #ifdef _WIN32 if(filename.size()<2 || (filename[0]!='\\' && filename[1]!='\\' ) ) { filename=L"\\\\?\\"+filename; } if(bufmgr==NULL) { bufmgr=new CBufMgr(NBUFFERS,READSIZE); } #endif #ifndef LINUX #ifndef BACKUP_SEM hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED|FILE_FLAG_SEQUENTIAL_SCAN, NULL); #else hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED|FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_SEQUENTIAL_SCAN, NULL); #endif if(hFile == INVALID_HANDLE_VALUE) { hFile=NULL; #ifdef CHECK_BASE_PATH std::wstring basePath=map_file(getuntil(L"/",o_filename)+L"/"); if(!isDirectory(basePath)) { char ch=ID_BASE_DIR_LOST; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG); return false; } Log("Info: Base dir lost", LL_DEBUG); break; } #endif char ch=ID_COULDNT_OPEN; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG); return false; } Log("Info: Couldn't open file", LL_DEBUG); break; } currfilepart=0; sendfilepart=0; sent_bytes=start_offset; LARGE_INTEGER filesize; GetFileSizeEx(hFile, &filesize); curr_filesize=filesize.QuadPart; next_checkpoint=start_offset+c_checkpoint_dist; if(next_checkpoint>curr_filesize) next_checkpoint=curr_filesize; if( offset_set==false || id==ID_GET_FILE_RESUME || id==ID_GET_FILE_RESUME_HASH ) { CWData data; data.addUChar(ID_FILESIZE); data.addUInt64(filesize.QuadPart); int rc=SendInt(data.getDataPtr(), data.getDataSize()); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: SendSize", LL_DEBUG); CloseHandle(hFile); hFile=NULL; return false; } } if(filesize.QuadPart==0) { CloseHandle(hFile); hFile=NULL; break; } for(_i64 i=start_offset;i<filesize.QuadPart && stopped==false;i+=READSIZE) { bool last; if(i+READSIZE<filesize.QuadPart) last=false; else { last=true; Log("Reading last file part", LL_DEBUG); } while(bufmgr->nfreeBufffer()==0 && stopped==false) { int rc; SleepEx(0,true); rc=SendData(); if(rc==-1) { Log("Error: Send failed in file loop -1", LL_DEBUG); CloseThread(hFile); } else if(rc==0) SleepEx(1,true); } if( stopped==false ) ReadFilePart(hFile, i, last); if(FileServ::isPause() ) { DWORD starttime=GetTickCount(); while(GetTickCount()-starttime<5000) { SleepEx(500,true); int rc=SendData(); if(rc==-1) { Log("Error: Send failed in file pause loop -2", LL_DEBUG); CloseThread(hFile); } } } } while(bufmgr->nfreeBufffer()!=NBUFFERS && stopped==false) { SleepEx(0,true); int rc; rc=SendData(); if( rc==2 && bufmgr->nfreeBufffer()!=NBUFFERS ) { Log("Error: File end and not all Buffers are free!-1", LL_WARNING); } if(rc==-1) { Log("Error: Send failed in off file loop -3", LL_DEBUG); CloseHandle(hFile); hFile=NULL; break; } else if(rc==0) SleepEx(1,true); } if( stopped==false ) { Log("Closed file.", LL_DEBUG); CloseHandle(hFile); hFile=NULL; } #else //LINUX hFile=open64(Server->ConvertToUTF8(filename).c_str(), O_RDONLY|O_LARGEFILE); if(hFile == INVALID_HANDLE_VALUE) { #ifdef CHECK_BASE_PATH std::wstring basePath=map_file(getuntil(L"/",o_filename)+L"/"); if(!isDirectory(basePath)) { char ch=ID_BASE_DIR_LOST; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG); return false; } Log("Info: Base dir lost", LL_DEBUG); break; } #endif hFile=0; char ch=ID_COULDNT_OPEN; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG); return false; } Log("Info: Couldn't open file", LL_DEBUG); break; } currfilepart=0; sendfilepart=0; struct stat64 stat_buf; fstat64(hFile, &stat_buf); off64_t filesize=stat_buf.st_size; curr_filesize=filesize; if( offset_set==false || id==ID_GET_FILE_RESUME || id==ID_GET_FILE_RESUME_HASH ) { CWData data; data.addUChar(ID_FILESIZE); data.addUInt64(filesize); int rc=SendInt(data.getDataPtr(), data.getDataSize() ); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: SendSize", LL_DEBUG); CloseHandle(hFile); hFile=0; return false; } } if(filesize==0) { CloseHandle(hFile); hFile=0; break; } off64_t foffset=start_offset; unsigned int s_bsize=8192; if(id==ID_GET_FILE || id==ID_GET_FILE_RESUME ) { s_bsize=32768; next_checkpoint=curr_filesize; } else { next_checkpoint=start_offset+c_checkpoint_dist; if(next_checkpoint>curr_filesize) next_checkpoint=curr_filesize; } char *buf=new char[s_bsize]; bool has_error=false; while( foffset < filesize ) { size_t count=(std::min)((size_t)s_bsize, (size_t)(next_checkpoint-foffset)); if( clientpipe==NULL && ( id==ID_GET_FILE || id==ID_GET_FILE_RESUME ) ) { ssize_t rc=sendfile64(int_socket, hFile, &foffset, count); if(rc>0) { foffset+=rc; } else { Log("Error: Reading and sending from file failed", LL_DEBUG); CloseHandle(hFile); delete []buf; return false; } } else { ssize_t rc=read(hFile, buf, count); if(rc>0) { rc=SendInt(buf, rc); if(rc==SOCKET_ERROR) { Log("Error: Sending data failed"); CloseHandle(hFile); delete []buf; return false; } else if(id==ID_GET_FILE_RESUME_HASH) { hash_func.update((unsigned char*)buf, rc); } foffset+=rc; } else { Log("Error: Reading from file failed", LL_DEBUG); CloseHandle(hFile); delete []buf; return false; } if(id==ID_GET_FILE_RESUME_HASH && foffset==next_checkpoint) { hash_func.finalize(); SendInt((char*)hash_func.raw_digest_int(), 16); next_checkpoint+=c_checkpoint_dist; if(next_checkpoint>curr_filesize) next_checkpoint=curr_filesize; hash_func.init(); } } if(FileServ::isPause() ) { Sleep(500); } } CloseHandle(hFile); delete []buf; hFile=0; #endif }break; case ID_GET_FILE_BLOCKDIFF: { bool b=GetFileBlockdiff(data); if(!b) return false; }break; case ID_BLOCK_REQUEST: { if(state==CS_BLOCKHASH) { Handle_ID_BLOCK_REQUEST(data); } }break; } } if( stopped==true ) return false; else return true; }
void BackupServerPrepareHash::operator()(void) { while(true) { working=false; std::string data; size_t rc=pipe->Read(&data); if(data=="exitnow") { output->Write("exitnow"); std::string t; exitpipe_hash->Read(&t); Server->destroy(exitpipe_hash); exitpipe->Write("ok"); Server->Log("server_prepare_hash Thread finished"); delete this; return; } else if(data=="exit") { output->Write("exit"); Server->destroy(exitpipe); Server->Log("server_prepare_hash Thread finished"); delete this; return; } else if(data=="flush") { continue; } if(rc>0) { working=true; CRData rd(&data); std::string temp_fn; rd.getStr(&temp_fn); int backupid; rd.getInt(&backupid); char incremental; rd.getChar(&incremental); std::string tfn; rd.getStr(&tfn); std::string hashpath; rd.getStr(&hashpath); std::string hashoutput_fn; rd.getStr(&hashoutput_fn); bool diff_file=!hashoutput_fn.empty(); std::string old_file_fn; rd.getStr(&old_file_fn); IFile *tf=Server->openFile(os_file_prefix(Server->ConvertToUnicode(temp_fn)), MODE_READ); IFile *old_file=NULL; if(diff_file) { old_file=Server->openFile(os_file_prefix(Server->ConvertToUnicode(old_file_fn)), MODE_READ); if(old_file==NULL) { ServerLogger::Log(clientid, "Error opening file \""+old_file_fn+"\" from pipe for reading. File: old_file", LL_ERROR); if(tf!=NULL) Server->destroy(tf); continue; } } if(tf==NULL) { ServerLogger::Log(clientid, "Error opening file \""+temp_fn+"\" from pipe for reading file. File: temp_fn", LL_ERROR); has_error=true; if(old_file!=NULL) { Server->destroy(old_file); } } else { ServerLogger::Log(clientid, "PT: Hashing file \""+ExtractFileName(tfn)+"\"", LL_DEBUG); std::string h; if(!diff_file) { h=hash(tf); } else { h=hash_with_patch(old_file, tf); } Server->destroy(tf); if(old_file!=NULL) { Server->destroy(old_file); } CWData data; data.addString(temp_fn); data.addInt(backupid); data.addChar(incremental); data.addString(tfn); data.addString(hashpath); data.addString(h); data.addString(hashoutput_fn); data.addString(old_file_fn); output->Write(data.getDataPtr(), data.getDataSize() ); } } } }
bool CClientThread::GetFileBlockdiff(CRData *data) { std::string s_filename; if(data->getStr(&s_filename)==false) return false; #ifdef CHECK_IDENT std::string ident; data->getStr(&ident); if(!FileServ::checkIdentity(ident)) { Log("Identity check failed -2", LL_DEBUG); return false; } #endif std::wstring o_filename=Server->ConvertToUnicode(s_filename); _i64 start_offset=0; data->getInt64(&start_offset); curr_hash_size=0; data->getInt64(&curr_hash_size); Log("Sending file "+Server->ConvertToUTF8(o_filename), LL_DEBUG); std::wstring filename=map_file(o_filename); Log("Mapped name: "+Server->ConvertToUTF8(filename), LL_DEBUG); if(filename.empty()) { char ch=ID_BASE_DIR_LOST; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -1", LL_DEBUG); return false; } Log("Info: Base dir lost -1", LL_DEBUG); return true; } hash_func.init(); #ifdef _WIN32 if(filename.size()<2 || (filename[0]!='\\' && filename[1]!='\\' ) ) filename=L"\\\\?\\"+filename; #endif #ifdef _WIN32 #ifndef BACKUP_SEM hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); #else hFile=CreateFileW(filename.c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_SEQUENTIAL_SCAN, NULL); #endif #else //_WIN32 hFile=open64(Server->ConvertToUTF8(filename).c_str(), O_RDONLY|O_LARGEFILE); #endif //_WIN32 if(hFile == INVALID_HANDLE_VALUE) { hFile=(HANDLE)NULL; #ifdef CHECK_BASE_PATH std::wstring basePath=map_file(getuntil(L"/",o_filename)+L"/"); if(!isDirectory(basePath)) { char ch=ID_BASE_DIR_LOST; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG); return false; } Log("Info: Base dir lost", LL_DEBUG); return true; } #endif char ch=ID_COULDNT_OPEN; int rc=SendInt(&ch, 1); if(rc==SOCKET_ERROR) { Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG); return false; } Log("Info: Couldn't open file", LL_DEBUG); return true; } currfilepart=0; sendfilepart=0; sent_bytes=0; #ifdef _WIN32 LARGE_INTEGER filesize; GetFileSizeEx(hFile, &filesize); curr_filesize=filesize.QuadPart; #else struct stat64 stat_buf; fstat64(hFile, &stat_buf); curr_filesize=stat_buf.st_size; #endif next_checkpoint=start_offset+c_checkpoint_dist; if(next_checkpoint>curr_filesize) next_checkpoint=curr_filesize; CWData sdata; sdata.addUChar(ID_FILESIZE); sdata.addUInt64(curr_filesize); SendInt(sdata.getDataPtr(), sdata.getDataSize()); if(mutex==NULL) { mutex=Server->createMutex(); cond=Server->createCondition(); } state=CS_BLOCKHASH; if(chunk_send_thread_ticket==ILLEGAL_THREADPOOL_TICKET) { IFile * tf=Server->openFileFromHandle((void*)hFile); if(tf==NULL) { Log("Could not open file from handle", LL_ERROR); return false; } chunk_send_thread_ticket=Server->getThreadPool()->execute(new ChunkSendThread(this, tf, curr_hash_size) ); } else { IScopedLock lock(mutex); update_file=Server->openFileFromHandle((void*)hFile); if(update_file==NULL) { Log("Could not open update file from handle", LL_ERROR); } cond->notify_all(); } hFile=(HANDLE)NULL; return true; }
void LMDBFileIndex::put_internal(const SIndexKey& key, int64 value, int flags, bool log, bool handle_enosp) { CWData vdata; vdata.addVarInt(value); MDB_val mdb_tkey; mdb_tkey.mv_data=const_cast<void*>(static_cast<const void*>(&key)); mdb_tkey.mv_size=sizeof(SIndexKey); MDB_val mdb_tvalue; mdb_tvalue.mv_data=vdata.getDataPtr(); mdb_tvalue.mv_size=vdata.getDataSize(); int rc = mdb_put(txn, dbi, &mdb_tkey, &mdb_tvalue, flags); if(rc==MDB_MAP_FULL && handle_enosp) { mdb_txn_abort(txn); if(_has_error) { Server->Log("LMDB had error during increase (on put). Aborting...", LL_ERROR); start_transaction(); return; } { read_transaction_lock.reset(); IScopedWriteLock lock(mutex); destroy_env(); map_size*=2; Server->Log("Increased LMDB database size to "+PrettyPrintBytes(map_size)+" (on put)", LL_DEBUG); if(!create_env()) { Server->Log("Error creating env after database file size increase", LL_ERROR); _has_error=true; start_transaction(); return; } } start_transaction(); replay_transaction_log(); put_internal(key, value, flags, false, true); } else if(rc==MDB_BAD_TXN && handle_enosp) { mdb_txn_abort(txn); if(_has_error) { Server->Log("LMDB had error on BAD_TXN (on put). Aborting...", LL_ERROR); start_transaction(); return; } start_transaction(); replay_transaction_log(); put_internal(key, value, flags, false, false); } else if(rc) { Server->Log("LMDB: Failed to put data ("+(std::string)mdb_strerror(rc)+")", LL_ERROR); _has_error=true; } if(!_has_error && log) { STransactionLogItem item = { key, value, flags}; transaction_log.push_back(item); } }