void SockSpray::Send(DataBlock *blocks[], LNE_UINT count)
{
LNE_ASSERT_RETURN_VOID(blocks != NULL && count > 0);
bool to_handle = true;
// ignore when shutdown
shutdown_lock_.Lock();
if(shutdown_state_.already || shutdown_state_.query)
to_handle = false;
shutdown_lock_.Unlock();
if(!to_handle)
return;
// append to queue
DataBlock *block;
send_lock_.Lock();
for(LNE_UINT i = 0; to_handle && i < count; ++i) {
block = blocks[i];
block->AddRef();
if(send_blocks_.Append(block) != LNERR_OK) {
block->Release();
to_handle = false;
} else if(send_blocks_.count() > limit_write_cache_)
to_handle = false;
}
if(!to_handle) {
shutdown_lock_.Lock();
shutdown_state_.query = true;
shutdown_lock_.Unlock();
}
send_lock_.Unlock();
if(to_handle)
__HandleWrite();
}
void *DataChunker::alloc(S32 size)
{
if (size > mChunkSize)
{
DataBlock * temp = (DataBlock*)dMalloc(DataChunker::PaddDBSize + size);
AssertFatal(temp, "Malloc failed");
constructInPlace(temp);
if (mCurBlock)
{
temp->next = mCurBlock->next;
mCurBlock->next = temp;
}
else
{
mCurBlock = temp;
temp->curIndex = mChunkSize;
}
return temp->getData();
}
if(!mCurBlock || size + mCurBlock->curIndex > mChunkSize)
{
const U32 paddDBSize = (sizeof(DataBlock) + 3) & ~3;
DataBlock *temp = (DataBlock*)dMalloc(paddDBSize+ mChunkSize);
AssertFatal(temp, "Malloc failed");
constructInPlace(temp);
temp->next = mCurBlock;
mCurBlock = temp;
}
void *ret = mCurBlock->getData() + mCurBlock->curIndex;
mCurBlock->curIndex += (size + 3) & ~3; // dword align
return ret;
}
// Convert endiannes
bool RadioDataFile::fixEndian(DataBlock& buf, unsigned int bytes)
{
if (!bytes)
return false;
unsigned int n = buf.length() / bytes;
if (bytes == 2) {
for (uint16_t* p = (uint16_t*)buf.data(); n; n--, p++)
#ifdef LITTLE_ENDIAN
*p = be16toh(*p);
#else
*p = le16toh(*p);
#endif
return true;
}
if (bytes == 4) {
for (uint32_t* p = (uint32_t*)buf.data(); n; n--, p++)
#ifdef LITTLE_ENDIAN
*p = be32toh(*p);
#else
*p = le32toh(*p);
#endif
return true;
}
if (bytes == 8) {
for (uint64_t* p = (uint64_t*)buf.data(); n; n--, p++)
#ifdef LITTLE_ENDIAN
*p = be64toh(*p);
#else
*p = le64toh(*p);
#endif
return true;
}
return false;
}
TEST(TestFileReader,readSmallFile){
string path = "/tmp/small.txt";
ofstream outfile (path.c_str(),std::ofstream::binary);
FileReader * fr;
DataBlock * block;
char data[]="0123456789";
// Creo un archivo de pruebas: 10 bytes * 10k = 100k
for(int i=0; i<10000;i++)
outfile.write((char*)&data,10);
outfile.close();
fr = new FileReader(path);
ASSERT_TRUE(fr->hasBlocksLeft());
block = fr->getBlock();
//blockSize = 16k (7 bloques)
ASSERT_EQ(block->getSizeInBytes(),16384);
vector<unsigned char>::iterator it = block->getIterator();
unsigned char c = '0';
for(int i=0; i<10;i++){
ASSERT_EQ(c,*it);
c+=1;
it++;
}
for(int i=0;i<6;i++){
ASSERT_TRUE(fr->hasBlocksLeft());
delete block;
block = fr->getBlock();
}
ASSERT_FALSE(fr->hasBlocksLeft());
ASSERT_EQ(block->getSizeInBytes(),1696);
delete block;
delete fr;
}
fulliautomatix_data* AsterixData::getData()
{
fulliautomatix_data *firstData=NULL,*lastData=NULL;
int byteoffset = 0;
std::list<DataBlock*>::iterator it;
for ( it=m_lDataBlocks.begin() ; it != m_lDataBlocks.end(); it++ )
{
DataBlock* db = (DataBlock*)(*it);
if (db != NULL)
{
if (!lastData)
{
firstData = lastData = db->getData(byteoffset);
}
else
{
lastData->next = db->getData(byteoffset);
}
while(lastData->next)
{
lastData = lastData->next;
}
byteoffset = lastData->bytenr + lastData->length;
}
}
return firstData;
}
/*
* zmienianie zawartosci pliku
*/
bool Inode::SetData(std::string text)
{
for(unsigned int i = 0; i < this->i_blocks; i++)
{
DataBlock *blockData = GetBlock(i);
if(!blockData->setData(text.substr(i * Synchro::blockSize).c_str()) == true)
return false;
}
// gdy zakonczy sie powodzeniem
return true;
}
// Break received audio data into manageable chunks, forward them to decoder
void FaxTerminal::rxData(const DataBlock& data, unsigned long tStamp)
{
unsigned int pos = 0;
while (pos < data.length())
{
// feed the decoder with small chunks of data (16 bytes/ms)
int len = data.length() - pos;
if (len > FAX_DATA_CHUNK)
len = FAX_DATA_CHUNK;
rxBlock(((char *)data.data())+pos, len);
pos += len;
}
}
// CIFfile::AddDataBlock()
int CIFfile::AddDataBlock( DataBlock const& block ) {
if (block.Header().empty()) {
mprinterr("Internal Error: Attempting to add empty CIF data block.\n");
return 1;
}
CIF_DataType::const_iterator it = cifdata_.find( block.Header() );
if (it != cifdata_.end()) {
mprinterr("Error: Duplicate CIF block found: '%s'\n", block.Header().c_str());
return 1;
}
cifdata_.insert( std::pair<std::string, DataBlock>(block.Header(), block) );
return 0;
}
void Column::append(int rows)
{
if (rows == 0)
return;
BlockMap::reverse_iterator itr = _blocks.rbegin();
if (itr == _blocks.rend()) // no blocks
{
ull firstId = DataBlock::capacity();
DataBlock *firstBlock = _mem->construct<DataBlock>(anonymous_instance)();
_blocks.insert(BlockEntry(firstId, firstBlock));
itr = _blocks.rbegin();
}
BlockEntry entry = *itr;
DataBlock *block = entry.second.get();
ull id = entry.first;
int room = DataBlock::capacity() - block->rowCount();
int rowsLeft = rows - room;
if (rowsLeft <= 0)
{
block->insert(block->rowCount(), rows);
_rowCount += rows;
return;
}
block->insert(block->rowCount(), room);
_rowCount += room;
int newBlocksRequired = rowsLeft / DataBlock::capacity();
if (rowsLeft % DataBlock::capacity())
newBlocksRequired++;
for (int i = 0; i < newBlocksRequired; i++)
{
try {
DataBlock *newBlock = _mem->construct<DataBlock>(anonymous_instance)();
int toInsert = std::min(rowsLeft, DataBlock::capacity());
newBlock->insert(0, toInsert);
rowsLeft -= toInsert;
id += DataBlock::capacity();
_blocks.insert(BlockEntry(id, newBlock));
_rowCount += toInsert;
}
catch (boost::interprocess::bad_alloc &e)
{
cout << e.what();
cout << "setRowCount" << _rowCount << "\n";
throw;
}
}
}
void DNN::define(const char* filename)
{
DataBlock data;
data.load(filename);
strvec layers = split(data("layers"),',');
std::cout << "We have " << layers.size() << " layers\n";
activations.resize(layers.size());
weights.resize(layers.size()-1);
bias.resize(layers.size()-1);
unsigned int i;
for(i=0;i<layers.size();++i)
{
unsigned int num_neurons = atof(layers[i].c_str());
activations[i] = Matrix(num_neurons,1,0.0f);
if(i > 0)
{
weights[i-1] = Matrix(activations[i].rows(),
activations[i-1].rows(),
1.0f);
bias[i-1] = Matrix(activations[i].rows(),1,1.0f);
}
}
for(i=0;i<layers.size();++i)
{
std::cout << activations[i].rows() << " inputs in layer " << i << "\n";
}
for(i=0;i<layers.size()-1;++i)
{
std::cout << "Weight matrix dimensions: " << weights[i].rows() << " " << weights[i].cols();
std::cout << " in layer " << i << "\n";
}
learningRate = atof(data("learning_rate").c_str());
}
LNE_NAMESPACE_USING
void TestClient()
{
TimeValue tv(3);
SockAddr addr("www.google.com:80");
SockConnector *connector = SockConnector::NewInstance(addr, &tv);
if(connector == NULL) {
printf("connector cannot create\n");
return;
}
SockPad skpad;
if(connector->Connect(skpad) != LNERR_OK) {
printf("connector cannot connect\n");
connector->Release();
return;
}
SockWaves *stream = SockWaves::NewInstance(skpad);
const char *query = "GET / HTTP/1.1\r\n\r\n";
DataBlock *block = DataBlock::NewInstance(1024 * 1024);
strcpy(block->buffer(), query);
block->set_size(static_cast<LNE_UINT>(strlen(block->buffer())));
stream->Send(block);
while(stream->Recv(block, tv) == LNERR_OK) {
block->buffer()[block->size()] = '\0';
puts(block->buffer());
}
block->Release();
SockAddr addr_sock, addr_peer;
stream->GetSockAddr(addr_sock);
stream->GetPeerAddr(addr_peer);
printf("connect %s => %s\n", addr_sock.addr_text(), addr_peer.addr_text());
stream->Release();
connector->Release();
}
void ParameterLoader<T>::ParseAndRequest(multiverso::DataBlockBase *data_block)
{
if (m_parse_and_request_count == 0)
{
m_start_time = clock();
}
fprintf(m_log_file, "%lf\n", (clock() - m_start_time) / (double)CLOCKS_PER_SEC);
multiverso::Log::Info("Rank %d ParameterLoader begin %d\n", multiverso::Multiverso::ProcessRank(), m_parse_and_request_count);
DataBlock *data = reinterpret_cast<DataBlock*>(data_block);
SkipGramMixtureNeuralNetwork<T>* sg_mixture_neural_network = reinterpret_cast<SkipGramMixtureNeuralNetwork<T>*>(m_sgmixture_neural_networks[m_parse_and_request_count % 2]);
++m_parse_and_request_count;
data->UpdateNextRandom();
sg_mixture_neural_network->PrepareParmeter(data);
std::vector<int>& input_layer_nodes = sg_mixture_neural_network->GetInputLayerNodes();
std::vector<int>& output_layer_nodes = sg_mixture_neural_network->GetOutputLayerNodes();
assert(sg_mixture_neural_network->status == 0);
sg_mixture_neural_network->status = 1;
for (int i = 0; i < input_layer_nodes.size(); ++i)
{
int word_id = input_layer_nodes[i];
for (int j = 0; j < m_words_sense_info->word_sense_cnts_info[word_id]; ++j)
RequestRow(kInputEmbeddingTableId, m_words_sense_info->p_input_embedding[word_id] + j);
}
for (int i = 0; i < output_layer_nodes.size(); ++i)
RequestRow(kEmbeddingOutputTableId, output_layer_nodes[i]);
RequestRow(kWordCountActualTableId, 0);
for (int i = 0; i < input_layer_nodes.size(); ++i)
{
int word_id = input_layer_nodes[i];
if (m_words_sense_info->word_sense_cnts_info[word_id] > 1)
RequestRow(kWordSensePriorTableId, m_words_sense_info->p_wordidx2sense_idx[word_id]);
}
std::vector<int> & tables = data->GetTables();
for (int i = 0; i < tables.size(); ++i)
RequestTable(tables[i]);
multiverso::Log::Info("Rank %d ParameterLoader finish %d\n", multiverso::Multiverso::ProcessRank(), m_parse_and_request_count - 1);
fprintf(m_log_file, "%lf\n", (clock() - m_start_time) / (double)CLOCKS_PER_SEC);
assert(sg_mixture_neural_network->status == 1);
sg_mixture_neural_network->status = 2;
}
PyObject* AsterixData::getData()
{
PyObject* hp = PyList_New(0);
std::list<DataBlock*>::iterator it;
for ( it=m_lDataBlocks.begin() ; it != m_lDataBlocks.end(); it++ )
{
DataBlock* db = (DataBlock*)(*it);
if (db != NULL)
{
db->getData(hp);
}
}
return hp;
}
unsigned long FaxConsumer::Consume(const DataBlock& data, unsigned long tStamp, unsigned long flags)
{
if (data.null() || !m_wrap)
return 0;
m_wrap->rxData(data,tStamp);
return invalidStamp();
}
// Read a record from file
bool RadioDataFile::read(uint64_t& ts, DataBlock& buffer, DebugEnabler* dbg, int* error)
{
int e = 0;
if (error)
*error = 0;
else
error = &e;
uint8_t hdr[12];
int rd = m_file.readData(hdr,sizeof(hdr));
// EOF ?
if (rd == 0) {
buffer.resize(0);
return true;
}
if (rd != sizeof(hdr))
return ioError(false,dbg,error,rd > 0 ? "Incomplete read (invalid size?)" : 0);
uint32_t len = 0;
uint32_t* u = (uint32_t*)hdr;
if (m_littleEndian == m_header.m_littleEndian)
len = *u;
else if (m_littleEndian)
len = be32toh(*u);
else
len = le32toh(*u);
uint64_t* p = (uint64_t*)&hdr[4];
if (m_littleEndian == m_header.m_littleEndian)
ts = *p;
else if (m_littleEndian)
len = be64toh(*p);
else
len = le64toh(*p);
buffer.resize(len);
if (!len)
return ioError(false,dbg,0,"Empty record");
rd = m_file.readData((void*)buffer.data(),len);
if (rd != (int)len)
return ioError(false,dbg,error,rd > 0 ? "Incomplete read (invalid size?)" : 0);
#ifdef XDEBUG
String sHdr, s;
sHdr.hexify(hdr,sizeof(hdr),' ');
s.hexify((void*)buffer.data(),rd,' ');
Debug(dbg,DebugAll,"RadioDataFile[%s] read %d hdr=%s data=%s [%p]",
c_str(),rd + (int)sizeof(hdr),sHdr.c_str(),s.c_str(),this);
#endif
return true;
}
/*
* pobieranie calej zawartosci pliku w postaci stringa
*/
std::string Inode::GetData()
{
std::string temp;
for(unsigned int i = 0; i < this->i_blocks; i++)
{
DataBlock *tempBlock = GetBlock(i);
char* blockData = tempBlock->getData();
for(int j = 0; j < Synchro::blockSize; j++)
{
if(blockData[j] != '\0')
temp += (blockData[j]);
}
temp += ' ';
}
return temp;
}
void YStunAttributeError::toBuffer(DataBlock& buffer)
{
u_int8_t header[8] = {0,0,0,0,
0,0,(u_int8_t)(m_code / 100),(u_int8_t)(m_code % 100)};
setHeader(header,type(),4 + m_text.length());
DataBlock tmp(header,sizeof(header));
buffer += tmp;
buffer.append(m_text);
}
bool DatabaseAdapter_DataBlock::Reader::load( const Index& elemno, DataBlock& blk)
{
m_dbkey.resize( m_domainKeySize);
m_dbkey.addElem( elemno);
std::string blkstr;
if (!m_database->readValue( m_dbkey.ptr(), m_dbkey.size(), blkstr, DatabaseOptions())) return false;
blk.init( elemno, blkstr.c_str(), blkstr.size(), blkstr.size());
return true;
}
bool DatabaseAdapter_DataBlock::Cursor::getBlock( const DatabaseCursorInterface::Slice& key, DataBlock& blk)
{
if (!key.defined()) return false;
char const* ki = key.ptr()+m_domainKeySize;
char const* ke = ki + key.size()-m_domainKeySize;
Index elemno = unpackIndex( ki, ke);
DatabaseCursorInterface::Slice blkslice = m_cursor->value();
blk.init( elemno, blkslice.ptr(), blkslice.size());
return true;
}
template<class T> uint64_t test_const(T val) {
using testfunc = double (*)();
static const Abi test_abi = {
.gpUnreserved = RegSet{},
.gpReserved = x64::abi().gp(),
.simdUnreserved = RegSet{xmm0},
.simdReserved = x64::abi().simd() - RegSet{xmm0},
.calleeSaved = x64::abi().calleeSaved,
.sf = x64::abi().sf
};
static uint8_t code[1000];
// None of these tests should use any data.
static uint8_t data_buffer[0];
CodeBlock main;
main.init(code, sizeof(code), "test");
DataBlock data;
data.init(data_buffer, sizeof(data), "data");
Vunit unit;
Vasm vasm{unit};
Vtext text { main, data };
auto& v = vasm.main();
unit.entry = v;
v << copy{v.cns(val), Vreg{xmm0}};
v << ret{RegSet{xmm0}};
optimizeX64(vasm.unit(), test_abi, true /* regalloc */);
CGMeta fixups;
LeaseHolder writer{Translator::WriteLease()};
EXPECT_TRUE(writer.canWrite());
emitX64(unit, text, fixups, nullptr);
// The above code might use fixups.literals but shouldn't use anything else.
fixups.literals.clear();
EXPECT_TRUE(fixups.empty());
union { double d; uint64_t c; } u;
u.d = ((testfunc)code)();
return u.c;
}
int MessageBlock::resize(size_t nsize)
{
try {
DataBlock* ndb = new DataBlock(nsize);
memcpy(ndb->addr(), rptr(), std::min(nsize, size()));
wptr_ = size();
rptr_ = 0;
dblock_->dec_ref();
dblock_ = ndb;
}
catch (std::exception& ex)
{
SYSLOG_ERROR("MessageBlock resize from %ld to %ld failed: %s", (long)size(), (long)nsize, ex.what());
return -1;
}
return 0;
}
/*
* appends Asterix data description to strResult
*/
bool AsterixData::getText(std::string& strResult, const unsigned int formatType)
{
static int i = 1;
std::list<DataBlock*>::iterator it;
for ( it=m_lDataBlocks.begin() ; it != m_lDataBlocks.end(); it++ )
{
DataBlock* db = (DataBlock*)(*it);
if (db != NULL)
{
switch(formatType)
{
case CAsterixFormat::ETxt:
strResult += format("\n\n-------------------------\nData Block %d", i++);
break;
}
db->getText(strResult, formatType);
}
}
return true;
}
inline void SerialData::Append(SerialData * pIns)
{
DataNode * pNewNode = new DataNode(pIns->m_pPrev,m_pData,m_pPrev);
m_pData->Release();
m_pData = pIns->m_pData;
pIns->m_pData->AddRef();
if (m_pPrev)
m_pPrev->Release();
m_pPrev = pNewNode;
}
IterationReport
BlackoilMultiSegmentModel<Grid>::solveWellEq(const std::vector<ADB>& mob_perfcells,
const std::vector<ADB>& b_perfcells,
SolutionState& state,
WellState& well_state)
{
IterationReport iter_report = Base::solveWellEq(mob_perfcells, b_perfcells, state, well_state);
if (iter_report.converged) {
// We must now update the state.segp and state.segqs members,
// that the base version does not know about.
const int np = numPhases();
const int nseg_total =well_state.numSegments();
{
// We will set the segp primary variable to the new ones,
// but we do not change the derivatives here.
ADB::V new_segp = Eigen::Map<ADB::V>(well_state.segPress().data(), nseg_total);
// Avoiding the copy below would require a value setter method
// in AutoDiffBlock.
std::vector<ADB::M> old_segp_derivs = state.segp.derivative();
state.segp = ADB::function(std::move(new_segp), std::move(old_segp_derivs));
}
{
// Need to reshuffle well rates, from phase running fastest
// to wells running fastest.
// The transpose() below switches the ordering.
const DataBlock segrates = Eigen::Map<const DataBlock>(well_state.segPhaseRates().data(), nseg_total, np).transpose();
ADB::V new_segqs = Eigen::Map<const V>(segrates.data(), nseg_total * np);
std::vector<ADB::M> old_segqs_derivs = state.segqs.derivative();
state.segqs = ADB::function(std::move(new_segqs), std::move(old_segqs_derivs));
}
// This is also called by the base version, but since we have updated
// state.segp we must call it again.
asImpl().computeWellConnectionPressures(state, well_state);
}
return iter_report;
}
// CIFfile::Read()
int CIFfile::Read(FileName const& fnameIn, int debugIn) {
if (file_.OpenFileRead( fnameIn )) return 1;
const char* ptr = file_.Line();
mode currentMode = UNKNOWN;
while (ptr != 0) {
/// There are 3 places we can be; a data block, a looped data block,
/// or unknown.
if ( currentMode == UNKNOWN ) {
// Are we at a data block yet?
if (ptr[0] == '_')
currentMode = SERIAL;
else if ( strncmp(ptr, "loop_", 5) == 0 )
currentMode = LOOP;
else
ptr = file_.Line();
} else if ( currentMode == SERIAL ) {
// SERIAL data block
DataBlock serial;
while ( ptr != 0 && ptr[0] == '_' ) {
serial.AddSerialDataRecord(ptr, file_);
ptr = file_.Line();
}
if (debugIn > 1) serial.ListData();
currentMode = UNKNOWN;
if (AddDataBlock( serial )) return 1;
} else if ( currentMode == LOOP ) {
DataBlock loop;
ptr = file_.Line();
if (ptr == 0 || ptr[0] != '_')
return LineError("In CIF file, malformed loop.", file_.LineNumber(), ptr);
while (ptr != 0 && ptr[0] == '_') {
loop.AddLoopColumn(ptr, file_);
ptr = file_.Line();
}
// Should now be positioned at loop data
if (ptr == 0)
return LineError("In CIF file, no loop data.", file_.LineNumber(), ptr);
while (ptr != 0 && ptr[0] != '_' && ptr[0] != '#') {
loop.AddLoopData(ptr, file_);
ptr = file_.Line();
}
if (debugIn > 1) loop.ListData();
currentMode = UNKNOWN;
if (AddDataBlock( loop )) return 1;
}
}
if (debugIn > 0)
mprintf("\tCIF file '%s', %i lines.\n", file_.Filename().full(), file_.LineNumber());
return 0;
}
LNE_NAMESPACE_USING
void TestObject()
{
LNE_UINT i, v;
ObjectList<LNE_UINT> list;
for(int i = 0; i < 3; i++)
list.PushFront(i + 1);
while(list.PopFront(v) == LNERR_OK) {
printf("%u\n", v);
}
DataBlock *block;
ObjectQueue<DataBlock *> queue;
DataBlockPool *pool = DataBlockPool::NewInstance();
for(i = 0; i < 10000; ++i)
queue.Append(pool->Alloc());
i = 0;
while(queue.Extract(block) == LNERR_OK) {
++i;
block->Release();
}
printf("data count: %u\n", i);
pool->Release();
}
virtual unsigned long Consume(const DataBlock& data, unsigned long tStamp, unsigned long flags)
{
if (!ref())
return 0;
unsigned long len = 0;
if (m_valid && getTransSource() && m_buffer.convert(data,m_sFmt,m_dFmt)) {
if (tStamp == invalidStamp()) {
unsigned int delta = data.length();
if (delta > m_buffer.length())
delta = m_buffer.length();
tStamp = m_timestamp + delta;
}
m_timestamp = tStamp;
len = getTransSource()->Forward(m_buffer,tStamp,flags);
}
deref();
return len;
}
void BlockRenderDataTexturedVertexBuffer::PrepareVertexArray(DataBlock& dataBlock)
{
ATLASSERT(OpenGL::BufferObjectBase::IsAvailable() == true);
Model::ElevationData& elevationData = dataBlock.GetElevationData();
std::vector<VertexBufferEntry>& vecVertices = m_vertData.Vertices();
vecVertices.resize(elevationData.RawData().size());
unsigned int uiBlockSize = elevationData.Size()+1;
for (size_t i=0, iMax=vecVertices.size(); i<iMax; i++)
{
VertexBufferEntry& entry = vecVertices[i];
unsigned int x, y;
elevationData.ArrayMapper().IndexToCoord(i, x, y);
entry.vertex[0] = static_cast<GLfloat>(x);
entry.vertex[1] = static_cast<GLfloat>(elevationData.Height(x, y));
entry.vertex[2] = static_cast<GLfloat>(y);
// TODO
Vector3d vNormal(0.0, 1.0, 0.0);
entry.normal[0] = static_cast<GLfloat>(vNormal.X());
entry.normal[1] = static_cast<GLfloat>(vNormal.Y());
entry.normal[2] = static_cast<GLfloat>(vNormal.Z());
entry.texcoords[0] = float(x) / uiBlockSize;
entry.texcoords[1] = float(y) / uiBlockSize;
entry.color[0] = entry.color[1] = entry.color[2] = 255;
entry.color[3] = 0;
}
// upload with next frame
m_taskManager.UploadTaskGroup().Add(
std::bind(&BlockRenderDataTexturedVertexBuffer::UploadVertexArray, this));
}
Packet::Packet(DataBlock &data, size_t size) : _rindex(0), _windex(0), _size(size)
{
_data = &data;
data.seize();
}
void Trainer::TrainIteration(multiverso::DataBlockBase *data_block)
{
if (process_id_ == -1)
process_id_ = multiverso::Multiverso::ProcessRank();
if (trainer_id_ == 0)
//Record the starting time of the Trainiteration
{
fprintf(log_file_, "%lf\n", (clock()) / (double)CLOCKS_PER_SEC);
fflush(log_file_);
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
++train_count_;
//Compute the total number of processes
if (process_count_ == -1)
process_count_ = multiverso::Multiverso::TotalProcessCount();
DataBlock *data = reinterpret_cast<DataBlock*>(data_block);
std::vector<int> input_nodes(data->input_nodes.begin(), data->input_nodes.end());
std::vector<int> output_nodes(data->output_nodes.begin(), data->output_nodes.end());
//A trainer only copy or add apart of parameters
//This trainer should copy or add the parameters according to
//local_input_nodes and local_output_nodes
std::vector<int> local_input_nodes;
std::vector<int> local_output_nodes;
for (int i = trainer_id_; i < input_nodes.size(); i += option_->thread_cnt)
local_input_nodes.push_back(input_nodes[i]);
for (int i = trainer_id_; i < output_nodes.size(); i += option_->thread_cnt)
local_output_nodes.push_back(output_nodes[i]);
if (trainer_id_ == 0)
{
multiverso::Log::Info("Rank %d [Trainer]------ input_size=%d, output_size=%d, negativesample_size=%d \n",
process_id_, input_nodes.size(), output_nodes.size(), data->negativesample_pools.size());
}
//Step 1, Copy the parameter from multiverso to WordEmbedding_
//One trainer only copy a part of parameters
multiverso::Log::Info("Rank %d [Trainer]------Train %d Copyparameter Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
CopyParameter(local_input_nodes, local_output_nodes);
if (trainer_id_ == 0)
{
multiverso::Row<int64> ©_row = GetRow<int64>(kWordCountActualTableId, 0);
WordEmbedding_->word_count_actual = copy_row.At(0);
WordEmbedding_->UpdateLearningRate();
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d Copyparameter end TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//Wait for all the trainers to finish copying parameter
barrier_->Wait();
//Step 2, After finishing copying parameter,
//Use WordEmbedding_ to train a part of data_block
int64 last_word_count = word_count;
clock_t start = clock();
multiverso::Log::Info("Rank %d [Trainer]------Train %d TrainNN Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
if (trainer_id_ == 0)
multiverso::Log::Info("Rank %d [Trainer]------Train %d Datablock's sentence number:%d ...\n",
process_id_, trainer_id_, data->Size());
WordEmbedding_->Train(data, trainer_id_, option_->thread_cnt,
word_count, hidden_act_, hidden_err_);
if (word_count > last_word_count)
{
multiverso::Log::Info("[Trainer]------TrainNNSpeed: Words/thread/second %lfk\n",
((double)word_count - last_word_count) /
(clock() - start) * (double)CLOCKS_PER_SEC / 1000);
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d TrainNN end TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//Wait for all the trainers to finish training
barrier_->Wait();
multiverso::Log::Info("Rank %d [Trainer]------Train %d AddDeltaParameter Begin TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//Step 3, After finishing training, add the delta of parameters to multiverso
AddDeltaParameter(local_input_nodes, local_output_nodes);
if (trainer_id_ == 0)
{
multiverso::Row<int64> ©_row = GetRow<int64>(kWordCountActualTableId, 0);
Add<int64>(kWordCountActualTableId, 0, 0, WordEmbedding_->word_count_actual - copy_row.At(0));
}
multiverso::Log::Info("Rank %d [Trainer]------Train %d AddDeltaParameter end TrainIteration%d ...\n",
process_id_, trainer_id_, train_count_);
//If the data_block is the last one,Dump the input-embedding weights
if (data->Type() == DataBlockType::Test && trainer_id_ == 0)
{
SaveEmbedding(option_->output_file, option_->output_binary);
}
if (data->Type() == DataBlockType::Test && process_id_ == 0 && trainer_id_ == 0)
{
SaveEmbedding("tmp.bin", 1);
char s[128] = { 0 };
sprintf_s(s, "check.py tmp.bin %d >> records.txt", clock() / CLOCKS_PER_SEC);
system(s);
}
if (trainer_id_ == 0)
{
fprintf(log_file_, "%lf\n",
(clock()) / (double)CLOCKS_PER_SEC);
fflush(log_file_);
}
}
