static void AddGames(DBProvider *pdb, int session_id, int player_id0, int player_id1)
{
int gamenum = 0;
listOLD *plGame, *pl = lMatch.plNext;
while ((plGame = pl->p) != NULL)
{
int game_id = GetNextId(pdb, "game");
moverecord *pmr = plGame->plNext->p;
xmovegameinfo *pmgi = &pmr->g;
char *buf = g_strdup_printf("INSERT INTO game(game_id, session_id, player_id0, player_id1, "
"score_0, score_1, result, added, game_number, crawford) "
"VALUES (%d, %d, %d, %d, %d, %d, %d, CURRENT_TIME, %d, %d )",
game_id, session_id, player_id0, player_id1,
pmgi->anScore[0], pmgi->anScore[1], pmgi->nPoints, ++gamenum, pmr->g.fCrawfordGame);
if (pdb->UpdateCommand(buf))
{
AddStats(pdb, game_id, player_id0, 0, "gamestat", ms.nMatchTo, &(pmgi->sc));
AddStats(pdb, game_id, player_id1, 1, "gamestat", ms.nMatchTo, &(pmgi->sc));
}
g_free(buf);
pl = pl->plNext;
}
}
extern void CommandRelationalAddMatch(char *sz)
{
DBProvider *pdb;
char *buf, *buf2, *date;
char warnings[1024] = "";
int session_id, existing_id, player_id0, player_id1;
char *arg = NULL;
gboolean quiet = FALSE;
arg = NextToken(&sz);
if (arg)
quiet = !strcmp(arg, "quiet");
if (ListEmpty(&lMatch))
{
outputl( _("No match is being played.") );
return;
}
/* Warn if match is not finished or fully analyzed */
if (!quiet && !GameOver())
strcat(warnings, _("The match is not finished\n"));
if (!quiet && !MatchAnalysed())
strcat(warnings, _("All of the match is not analyzed\n"));
if (*warnings)
{
strcat(warnings, _("\nAdd match anyway?"));
if (!GetInputYN(warnings))
return;
}
if ((pdb = ConnectToDB(dbProviderType)) == NULL)
return;
existing_id = RelationalMatchExists(pdb);
if (existing_id != -1)
{
if (!quiet && !GetInputYN(_("Match exists, overwrite?")))
return;
/* Remove any game stats and games */
buf2 = g_strdup_printf("FROM game WHERE session_id = %d", existing_id);
buf = g_strdup_printf("DELETE FROM gamestat WHERE game_id in (SELECT game_id %s)", buf2);
pdb->UpdateCommand(buf);
g_free(buf);
buf = g_strdup_printf("DELETE %s", buf2);
pdb->UpdateCommand(buf);
g_free(buf);
g_free(buf2);
/* Remove any match stats and session */
buf = g_strdup_printf("DELETE FROM matchstat WHERE session_id = %d", existing_id);
pdb->UpdateCommand(buf);
g_free(buf);
buf = g_strdup_printf("DELETE FROM session WHERE session_id = %d", existing_id);
pdb->UpdateCommand(buf);
g_free(buf);
}
session_id = GetNextId(pdb, "session");
player_id0 = AddPlayer(pdb, ap[0].szName);
player_id1 = AddPlayer(pdb, ap[1].szName);
if (session_id == -1 || player_id0 == -1 || player_id1 == -1)
{
outputl( _("Error adding match.") );
return;
}
if( mi.nYear )
date = g_strdup_printf("%04d-%02d-%02d", mi.nYear, mi.nMonth, mi.nDay);
else
date = NULL;
buf = g_strdup_printf("INSERT INTO session(session_id, checksum, player_id0, player_id1, "
"result, length, added, rating0, rating1, event, round, place, annotator, comment, date) "
"VALUES (%d, '%s', %d, %d, %d, %d, CURRENT_TIMESTAMP, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')",
session_id, GetMatchCheckSum(), player_id0, player_id1,
MatchResult(ms.nMatchTo), ms.nMatchTo, NS(mi.pchRating[0]), NS(mi.pchRating[1]),
NS(mi.pchEvent), NS(mi.pchRound), NS(mi.pchPlace), NS(mi.pchAnnotator), NS(mi.pchComment), NS(date));
updateStatisticsMatch ( &lMatch );
if (pdb->UpdateCommand(buf))
{
if (AddStats(pdb, session_id, player_id0, 0, "matchstat", ms.nMatchTo, &scMatch) &&
AddStats(pdb, session_id, player_id1, 1, "matchstat", ms.nMatchTo, &scMatch))
{
if (storeGameStats)
AddGames(pdb, session_id, player_id0, player_id1);
pdb->Commit();
}
}
g_free(buf);
g_free(date);
pdb->Disconnect();
}
LOCAL_C TInt testAsyncAccess(TChar dc1, TChar dc2)
//
// Test one drive against the other.
//
{
TFileOps f1;
TFileOps f2;
f1.Open(dc1, 1);
if (dc1 != dc2)
f2.Open(dc2, 2);
TInt op1 = 0;
TInt op2 = 0;
RTimer timer;
TRequestStatus tstat;
TTime startTime;
TTime endTime;
TTimeIntervalMicroSeconds timeTaken;
timer.CreateLocal();
timer.After(tstat, KTimeBM * KSecond);
startTime.HomeTime();
while (tstat == KRequestPending)
{
TInt num = f1.Write();
num += f2.Write();
if (num == 0)
User::WaitForAnyRequest();
}
op1 = f1.End();
op2 = f2.End();
endTime.HomeTime();
timeTaken=endTime.MicroSecondsFrom(startTime);
TInt64 dtime = timeTaken.Int64();
TTest::Printf(_L("%c: %8d writes in %6d mS = %8d bytes per second\n"),
(TUint)dc1, op1, I64LOW(dtime)/1000, GetSpeed(op1, dtime));
if (dc1 != dc2)
TTest::Printf(_L("%c: %8d writes in %6d mS = %8d bytes per second\n"),
(TUint)dc2, op2, I64LOW(dtime)/1000, GetSpeed(op2, dtime));
AddStats(gWrStats, MAKE_TINT64(0, op1 + op2) * MAKE_TINT64(0, KBufLen) * MAKE_TINT64(0, KSecond), dtime);
// now the reads!
f1.Reset();
f2.Reset();
timer.After(tstat, KTimeBM * KSecond);
startTime.HomeTime();
while (tstat == KRequestPending)
{
f1.Read();
f2.Read();
User::WaitForAnyRequest();
}
op1 = f1.End();
op2 = f2.End();
endTime.HomeTime();
timeTaken=endTime.MicroSecondsFrom(startTime);
dtime = timeTaken.Int64();
TTest::Printf(_L("%c: %8d reads in %6d mS = %8d bytes per second\n"),
(TUint)dc1, op1, I64LOW(dtime)/1000, GetSpeed(op1, dtime));
if (dc1 != dc2)
TTest::Printf(_L("%c: %8d reads in %6d mS = %8d bytes per second\n"),
(TUint)dc2, op2, I64LOW(dtime)/1000, GetSpeed(op2, dtime));
AddStats(gRdStats, MAKE_TINT64(0, op1 + op2) * MAKE_TINT64(0, KBufLen) * MAKE_TINT64(0, KSecond), dtime);
test.Printf(_L("\n"));
test.Printf(_L("average write throughput = %d bytes/sec\n"), GetSpeed(gWrStats));
test.Printf(_L("average read throughput = %d bytes/sec\n"), GetSpeed(gRdStats));
test.Printf(_L("\n"));
gWrStats.Init();
gRdStats.Init();
timer.Cancel();
timer.Close();
f1.Close();
f2.Close();
// delay for a second to allow the close to complete before dismounting.
User::After(1000000);
return KErrNone;
}
LOCAL_C TInt testAsyncAccess(TAny* aData)
//
/// Test read file handling.
///
/// @param aData pointer to the thread data area
{
TThreadData& data = *(TThreadData*)aData;
TFileName fileName = data.iFile;
TBool dowrite = (data.iData != NULL);
TBuf8<KBufLen>* buffer = gBufferArr[data.iNum];
TRequestStatus* status = gStatusArr[data.iNum];
RFs myFs;
TInt r = myFs.Connect();
TEST(r==KErrNone);
r = myFs.SetSessionPath(gSessionPath);
if (r != KErrNone)
TTest::Fail(HERE, _L("SetSessionPath returned %d"), r);
TVolumeInfo vol;
TInt drv;
r = myFs.CharToDrive(fileName[0], drv);
if (r != KErrNone)
TTest::Fail(HERE, _L("CharToDrive(%c) returned %d"), fileName[0], r);
r = myFs.Volume(vol, drv);
if (r != KErrNone)
TTest::Fail(HERE, _L("Volume() returned %d"), r);
TInt64 maxwrite = vol.iFree / 2 - KBufLen;
if (maxwrite < KBufLen*2)
TTest::Fail(HERE, _L("Not enough space to do test, only %d KB available"),
TInt(vol.iFree/1024));
RFile f;
RTimer timer;
TRequestStatus tstat;
TTime startTime;
TTime endTime;
TTimeIntervalMicroSeconds timeTaken;
TInt wrnum = 0;
TInt rdnum = 0;
TInt opnum = 0;
TInt opfin = 0;
TInt i;
timer.CreateLocal();
if (dowrite)
{
r = f.Replace(myFs, fileName, EFileStreamText | EFileWrite);
TEST(r==KErrNone);
// wait for both tasks to have a chance to complete opening the files
User::After(1000);
for (i = 0; i < KNumBuf; i++)
buffer[i].Fill('_', KBufLen);
timer.After(tstat, KTimeBM * KSecond);
startTime.HomeTime();
while (tstat == KRequestPending)
{
TInt pos = TInt((wrnum * KBufLen) % maxwrite);
TInt bnum = opnum++ % KNumBuf;
f.Write(pos, buffer[bnum], status[bnum]);
if (opnum - opfin > KMaxLag)
{
while (status[opfin % KNumBuf] == KRequestPending)
User::WaitForRequest(status[opfin % KNumBuf]);
opfin++;
}
++wrnum;
}
while (opfin < opnum)
{
while (status[opfin % KNumBuf] == KRequestPending)
User::WaitForRequest(status[opfin % KNumBuf]);
opfin++;
}
endTime.HomeTime();
TTimeIntervalMicroSeconds timeTaken=endTime.MicroSecondsFrom(startTime);
TInt64 dtime = timeTaken.Int64();
TInt64 dsize = wrnum * KBufLen * TInt64(KSecond);
TInt32 speed = TInt32((dsize + dtime/2) / dtime);
AddStats(gWrStats, dsize, dtime);
TTest::Printf(_L("%8d writes in %6d mS = %8d bytes per second\n"),
wrnum, TInt32(dtime)/1000, speed);
}
else
{
r = f.Open(myFs, fileName, EFileStreamText);
TEST(r==KErrNone);
timer.After(tstat, KTimeBM * KSecond);
startTime.HomeTime();
while (tstat == KRequestPending)
{
TInt pos = TInt((rdnum * KBufLen) % maxwrite);
TInt bnum = opnum++ % KNumBuf;
f.Read(pos, buffer[bnum], status[bnum]);
if (opnum - opfin > KMaxLag)
{
User::WaitForRequest(status[opfin++ % KNumBuf]);
}
++rdnum;
}
while (opfin < opnum)
{
if (status[opfin % KNumBuf] == KRequestPending)
User::WaitForRequest(status[opfin % KNumBuf]);
opfin++;
}
endTime.HomeTime();
timeTaken=endTime.MicroSecondsFrom(startTime);
TInt64 dtime = timeTaken.Int64();
TInt64 dsize = rdnum * KBufLen * TInt64(KSecond);
TInt32 speed = TInt32((dsize + dtime/2) / dtime);
AddStats(gRdStats, dsize, dtime);
// wait to allow the dust to settle
User::After(KSecond);
TTest::Printf(_L("%8d reads in %6d mS = %8d bytes per second\n"),
rdnum, TInt32(dtime)/1000, speed);
myFs.Delete(fileName);
}
timer.Cancel();
timer.Close();
f.Close();
myFs.Close();
return r;
}
/* PPlexStream: compute perplexity and related statistics */
static void ProcessTextStream(char *fn, int nSize)
{
int i;
FILE *f;
LabId lab=0;
double ppl;
int numPLabs;
Boolean isPipe;
char word[256];
if (fn!=NULL) {
if ((f=FOpen(fn, LMTextFilter, &isPipe))==NULL)
HError(16610,"ProcessTextStream: unable to open file %s", fn);
} else {
f = stdin;
}
if (trace>0) {
printf("Processing text stream: %s\n", (fn==NULL)?"<stdin>":fn);
fflush(stdout);
}
numPLabs = 0;
ZeroStats(&sent);
sent.nUtt = 1; sent.nTok = 0;
while ((fscanf(f, "%200s", word))==1) {
if (strlen(word)>=200)
HError(-16640, "ProcessTextStream: word too long, will be split: %s\n", word);
lab = GetEQLab(GetLabId(word, TRUE));
if (IS_SST(lab)) {
numPLabs = 0;
for (i=0; i<(nSize-1); i++) pLab[numPLabs++] = sstId;
ZeroStats(&sent);
sent.nUtt = 1; sent.nTok = 1;
continue;
}
if (IS_UNK(lab)) {
if (trace&T_OOV)
printf("mapping OOV: %s\n", lab->name);
StoreOOV(&sent,lab,1); lab = unkId;
}
pLab[numPLabs++] = lab; sent.nTok++;
if (numPLabs>=LBUF_SIZE) {
HError(16645,"ProcessTextStream: word buffer size exceeded - too many words without a sentence end (%d)",LBUF_SIZE);
CalcPerplexity(&sent,pLab,numPLabs,nSize);
numPLabs = 0;
}
if (IS_SEN(lab)) {
CalcPerplexity(&sent,pLab,numPLabs,nSize);
AddStats(&sent, &totl);
if (trace&T_SEL) { /* compact info for sentence selection */
ppl = exp(-(sent.logpp)/(double) (sent.nWrd));
printf("#! %.4f", ppl);
for (i=nSize-1; i<numPLabs; i++)
printf(" %s", pLab[i]->name);
printf("\n"); fflush(stdout);
}
ZeroStats(&sent);
}
}
AddStats(&sent,&totl);
if (fn!=NULL)
FClose(f,isPipe);
}
/* ProcessLabelFile: compute perplexity and related statistics from labels */
static void ProcessLabelFile(char *fn, int nSize)
{
LLink ll;
double ppl;
LabList *ref;
LabId lab;
Transcription *tr;
int i,numPLabs,nLabel;
tr = LOpen(&tempHeap, fn, lff);
if (tr->numLists < 1) {
HError(-16635,"ProcessLabelFile: transcription file %s is Empty",fn);
return;
}
ref = GetLabelList(tr, 1);
if (ref->head->succ == ref->tail) {
HError(-16635,"ProcessLabelFile: transcription file %s is Empty",fn);
return;
}
if (trace>0) {
printf("Processing label file: %s\n", fn);
fflush(stdout);
}
nLabel = CountLabs(ref);
ZeroStats(&sent);
sent.nTok = nLabel + 2; sent.nUtt = 1;
/* copy labels into pLab, mapping OOVs */
numPLabs = 0;
if (sstId!=NULL) /* add sentence start marker(s) */
for (i=0; i<(nSize-1); i++) pLab[numPLabs++] = sstId;
for (i=0,ll=ref->head->succ; i<nLabel; i++,ll=ll->succ) {
lab = GetEQLab(ll->labid);
if ((i==0) && IS_SST(lab)) {
sent.nTok--; continue;
}
if ((i==(nLabel-1)) && IS_SEN(lab)) {
sent.nTok--; continue;
}
if (IS_UNK(lab)) {
if (trace&T_OOV)
printf("mapping OOV: %s\n", lab->name);
StoreOOV(&sent,lab,1); lab = unkId;
}
pLab[numPLabs++] = lab;
if (numPLabs>=LBUF_SIZE) {
HError(16650, "Maximum utterance length in a label file exceeded (limit is compiled to be %d tokens)",
LBUF_SIZE);
}
}
if (senId!=NULL) /* add sentence end marker */
pLab[numPLabs++] = senId;
CalcPerplexity(&sent, pLab, numPLabs, nSize);
AddStats(&sent, &totl);
if (trace&T_SEL) { /* compact info for sentence selection */
ppl = exp(-(sent.logpp)/(double) (sent.nWrd));
printf("#! %.4f", ppl);
for (i=0, ll=ref->head->succ; i<nLabel; i++, ll=ll->succ)
printf(" %s", ll->labid->name);
printf("\n"); fflush(stdout);
}
}
// update the tree, do pruning
virtual void Update(const std::vector<bst_gpair> &gpair,
IFMatrix *p_fmat,
const BoosterInfo &info,
const std::vector<RegTree*> &trees) {
if (trees.size() == 0) return;
// number of threads
// thread temporal space
std::vector< std::vector<TStats> > stemp;
std::vector<RegTree::FVec> fvec_temp;
// setup temp space for each thread
int nthread;
#pragma omp parallel
{
nthread = omp_get_num_threads();
}
fvec_temp.resize(nthread, RegTree::FVec());
stemp.resize(nthread, std::vector<TStats>());
#pragma omp parallel
{
int tid = omp_get_thread_num();
int num_nodes = 0;
for (size_t i = 0; i < trees.size(); ++i) {
num_nodes += trees[i]->param.num_nodes;
}
stemp[tid].resize(num_nodes, TStats(param));
std::fill(stemp[tid].begin(), stemp[tid].end(), TStats(param));
fvec_temp[tid].Init(trees[0]->param.num_feature);
}
// if it is C++11, use lazy evaluation for Allreduce,
// to gain speedup in recovery
#if __cplusplus >= 201103L
auto lazy_get_stats = [&]()
#endif
{
// start accumulating statistics
utils::IIterator<RowBatch> *iter = p_fmat->RowIterator();
iter->BeforeFirst();
while (iter->Next()) {
const RowBatch &batch = iter->Value();
utils::Check(batch.size < std::numeric_limits<unsigned>::max(),
"too large batch size ");
const bst_omp_uint nbatch = static_cast<bst_omp_uint>(batch.size);
#pragma omp parallel for schedule(static)
for (bst_omp_uint i = 0; i < nbatch; ++i) {
RowBatch::Inst inst = batch[i];
const int tid = omp_get_thread_num();
const bst_uint ridx = static_cast<bst_uint>(batch.base_rowid + i);
RegTree::FVec &feats = fvec_temp[tid];
feats.Fill(inst);
int offset = 0;
for (size_t j = 0; j < trees.size(); ++j) {
AddStats(*trees[j], feats, gpair, info, ridx,
BeginPtr(stemp[tid]) + offset);
offset += trees[j]->param.num_nodes;
}
feats.Drop(inst);
}
}
// aggregate the statistics
int num_nodes = static_cast<int>(stemp[0].size());
#pragma omp parallel for schedule(static)
for (int nid = 0; nid < num_nodes; ++nid) {
for (int tid = 1; tid < nthread; ++tid) {
stemp[0][nid].Add(stemp[tid][nid]);
}
}
};
#if __cplusplus >= 201103L
reducer.Allreduce(BeginPtr(stemp[0]), stemp[0].size(), lazy_get_stats);
#else
reducer.Allreduce(BeginPtr(stemp[0]), stemp[0].size());
#endif
// rescale learning rate according to size of trees
float lr = param.learning_rate;
param.learning_rate = lr / trees.size();
int offset = 0;
for (size_t i = 0; i < trees.size(); ++i) {
for (int rid = 0; rid < trees[i]->param.num_roots; ++rid) {
this->Refresh(BeginPtr(stemp[0]) + offset, rid, trees[i]);
}
offset += trees[i]->param.num_nodes;
}
// set learning rate back
param.learning_rate = lr;
}
