int main(){
bool go_on = true;
int c;
char s[1000];
int width = 100;
int height = 20;
int block;
int next_block;
int socket_num = -1;
bool bool_change = true;
bool bool_win = false;
int mysock;
int inputsock;
WINDOW *win;
Act act = single;
/* WINDOW *my_menu_win;
MENU *my_menu;
ITEM **my_items;
printf("Choose a mode to play\n");
printf("1. Create an online game\n");
printf("2. Join an online game\n");
printf( "3. Play sigle mode\n");
cin >> c;
*/
initscr();
start_color();
cbreak(); // disable key buffering
keypad(stdscr, TRUE);
noecho(); // disable echoing keypad(stdscr, TRUE);
mvaddstr(1,4,"Choose a mode to play");
mvaddstr(3,4,"1.Create an online game");
mvaddstr(4,4,"2.Join an online game");
mvaddstr(5,4,"3.Play in sinlge mode");
while(go_on){
c = getch();
switch(c){
case '1':
act = server;
socket_set(mysock, inputsock);
srand( time(NULL));
go_on = false;
break;
case '2':
act = client;
socket_set(mysock);
srand( time(NULL) + 5);
go_on = false;
break;
case '3':
srand( time(NULL));
go_on = false;
break;
default:
break;
}
}
clear();
go_on = true;
nodelay(stdscr, TRUE);
AbstractBlock *ptrBlock;
Container container(14, 20);
Poisition last_poi;
Timer time;
block = rand() % RAND_NUM;
while( go_on ){
while( (c=getch()) == ERR && go_on){
if(bool_change){
next_block = rand() % RAND_NUM;
switch(block){
case 0:
ptrBlock = new TBlock(container, act);
ptrBlock->paint();
break;
case 1:
ptrBlock = new LBlock(container, act);
ptrBlock->paint();
break;
case 2:
ptrBlock = new ZBlock(container, act);
ptrBlock->paint();
break;
case 3:
ptrBlock = new IBlock(container, act);
ptrBlock->paint();
break;
case 4:
ptrBlock = new RZBlock(container, act);
ptrBlock->paint();
break;
case 5:
ptrBlock = new OBlock(container, act);
ptrBlock->paint();
break;
case 6:
ptrBlock = new RLBlock(container, act);
ptrBlock->paint();
break;
}
block = next_block;
bool_change = false;
paint_next(next_block);
if(container.getLose()){
go_on = false;
time.stopTime();
break;
}
}
if(down(*ptrBlock, last_poi, container, act)){
bool_change = true;
delete ptrBlock;
}
if(act == server){
if(send_receive(inputsock, container) == 1){
go_on = false;
bool_win = true;
}
}
else if(act == client){
if(send_receive(mysock, container) == 1){
go_on = false;
bool_win = true;
}
}
time.setTime();
}
switch(c){
case KEY_UP:
break;
case KEY_DOWN:
if(accDown(*ptrBlock, last_poi, container, act)){
bool_change = true;
delete ptrBlock;
}
break;
case KEY_LEFT:
left(*ptrBlock, last_poi, container);
break;
case KEY_RIGHT:
right(*ptrBlock, last_poi, container);
break;
case ' ':
rotate(*ptrBlock, last_poi, container);
break;
case 'q':
go_on = false;
break;
default:
break;
}
}
if(act == client){
send_Lose(mysock, container);
}
else if(act == server){
send_Lose(inputsock, container);
}
time.displayTime();
end_game(bool_win);
endwin();
close(mysock);
close(inputsock);
return 0;
}
bool dbEval(const string& dbName, BSONObj& cmd, BSONObjBuilder& result, string& errmsg) {
BSONElement e = cmd.firstElement();
uassert( 10046 , "eval needs Code" , e.type() == Code || e.type() == CodeWScope || e.type() == String );
const char *code = 0;
switch ( e.type() ) {
case String:
case Code:
code = e.valuestr();
break;
case CodeWScope:
code = e.codeWScopeCode();
break;
default:
verify(0);
}
verify( code );
if ( ! globalScriptEngine ) {
errmsg = "db side execution is disabled";
return false;
}
auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName );
ScriptingFunction f = s->createFunction(code);
if ( f == 0 ) {
errmsg = (string)"compile failed: " + s->getError();
return false;
}
if ( e.type() == CodeWScope )
s->init( e.codeWScopeScopeData() );
s->localConnect( dbName.c_str() );
BSONObj args;
{
BSONElement argsElement = cmd.getField("args");
if ( argsElement.type() == Array ) {
args = argsElement.embeddedObject();
if ( edebug ) {
out() << "args:" << args.toString() << endl;
out() << "code:\n" << code << endl;
}
}
}
int res;
{
Timer t;
res = s->invoke(f, &args, 0, cmdLine.quota ? 10 * 60 * 1000 : 0 );
int m = t.millis();
if ( m > cmdLine.slowMS ) {
out() << "dbeval slow, time: " << dec << m << "ms " << dbName << endl;
if ( m >= 1000 ) log() << code << endl;
else OCCASIONALLY log() << code << endl;
}
}
if ( res ) {
result.append("errno", (double) res);
errmsg = "invoke failed: ";
errmsg += s->getError();
return false;
}
s->append( result , "retval" , "return" );
return true;
}
bool
TIFFOutput::open (const std::string &name, const ImageSpec &userspec,
OpenMode mode)
{
if (mode == AppendMIPLevel) {
error ("%s does not support MIP levels", format_name());
return false;
}
close (); // Close any already-opened file
m_spec = userspec; // Stash the spec
// Check for things this format doesn't support
if (m_spec.width < 1 || m_spec.height < 1) {
error ("Image resolution must be at least 1x1, you asked for %d x %d",
m_spec.width, m_spec.height);
return false;
}
if (m_spec.tile_width) {
if (m_spec.tile_width % 16 != 0 ||
m_spec.tile_height % 16 != 0 ||
m_spec.tile_height == 0) {
error("Tile size must be a multiple of 16, you asked for %d x %d", m_spec.tile_width, m_spec.tile_height);
return false;
}
}
if (m_spec.depth < 1)
m_spec.depth = 1;
// Open the file
#ifdef _WIN32
std::wstring wname = Strutil::utf8_to_utf16 (name);
m_tif = TIFFOpenW (wname.c_str(), mode == AppendSubimage ? "a" : "w");
#else
m_tif = TIFFOpen (name.c_str(), mode == AppendSubimage ? "a" : "w");
#endif
if (! m_tif) {
error ("Can't open \"%s\" for output.", name.c_str());
return false;
}
// N.B. Clamp position at 0... TIFF is internally incapable of having
// negative origin.
TIFFSetField (m_tif, TIFFTAG_XPOSITION, (float)std::max (0, m_spec.x));
TIFFSetField (m_tif, TIFFTAG_YPOSITION, (float)std::max (0, m_spec.y));
TIFFSetField (m_tif, TIFFTAG_IMAGEWIDTH, m_spec.width);
TIFFSetField (m_tif, TIFFTAG_IMAGELENGTH, m_spec.height);
if ((m_spec.full_width != 0 || m_spec.full_height != 0) &&
(m_spec.full_width != m_spec.width || m_spec.full_height != m_spec.height)) {
TIFFSetField (m_tif, TIFFTAG_PIXAR_IMAGEFULLWIDTH, m_spec.full_width);
TIFFSetField (m_tif, TIFFTAG_PIXAR_IMAGEFULLLENGTH, m_spec.full_height);
}
if (m_spec.tile_width) {
TIFFSetField (m_tif, TIFFTAG_TILEWIDTH, m_spec.tile_width);
TIFFSetField (m_tif, TIFFTAG_TILELENGTH, m_spec.tile_height);
} else {
// Scanline images must set rowsperstrip
TIFFSetField (m_tif, TIFFTAG_ROWSPERSTRIP, 32);
}
TIFFSetField (m_tif, TIFFTAG_SAMPLESPERPIXEL, m_spec.nchannels);
int orientation = m_spec.get_int_attribute("Orientation", 1);
TIFFSetField (m_tif, TIFFTAG_ORIENTATION, orientation);
int bps, sampformat;
switch (m_spec.format.basetype) {
case TypeDesc::INT8:
bps = 8;
sampformat = SAMPLEFORMAT_INT;
break;
case TypeDesc::UINT8:
bps = 8;
sampformat = SAMPLEFORMAT_UINT;
break;
case TypeDesc::INT16:
bps = 16;
sampformat = SAMPLEFORMAT_INT;
break;
case TypeDesc::UINT16:
bps = 16;
sampformat = SAMPLEFORMAT_UINT;
break;
case TypeDesc::HALF:
// Silently change requests for unsupported 'half' to 'float'
m_spec.set_format (TypeDesc::FLOAT);
case TypeDesc::FLOAT:
bps = 32;
sampformat = SAMPLEFORMAT_IEEEFP;
break;
case TypeDesc::DOUBLE:
bps = 64;
sampformat = SAMPLEFORMAT_IEEEFP;
break;
default:
// Everything else, including UNKNOWN -- default to 8 bit
bps = 8;
sampformat = SAMPLEFORMAT_UINT;
m_spec.set_format (TypeDesc::UINT8);
break;
}
TIFFSetField (m_tif, TIFFTAG_BITSPERSAMPLE, bps);
TIFFSetField (m_tif, TIFFTAG_SAMPLEFORMAT, sampformat);
int photo = (m_spec.nchannels > 1 ? PHOTOMETRIC_RGB : PHOTOMETRIC_MINISBLACK);
TIFFSetField (m_tif, TIFFTAG_PHOTOMETRIC, photo);
// ExtraSamples tag
if (m_spec.nchannels > 3) {
bool unass = m_spec.get_int_attribute("oiio:UnassociatedAlpha", 0);
short e = m_spec.nchannels-3;
std::vector<unsigned short> extra (e);
for (int c = 0; c < e; ++c) {
if (m_spec.alpha_channel == (c+3))
extra[c] = unass ? EXTRASAMPLE_UNASSALPHA : EXTRASAMPLE_ASSOCALPHA;
else
extra[c] = EXTRASAMPLE_UNSPECIFIED;
}
TIFFSetField (m_tif, TIFFTAG_EXTRASAMPLES, e, &extra[0]);
}
// Default to LZW compression if no request came with the user spec
if (! m_spec.find_attribute("compression"))
m_spec.attribute ("compression", "lzw");
ImageIOParameter *param;
const char *str = NULL;
// Did the user request separate planar configuration?
m_planarconfig = PLANARCONFIG_CONTIG;
if ((param = m_spec.find_attribute("planarconfig", TypeDesc::STRING)) ||
(param = m_spec.find_attribute("tiff:planarconfig", TypeDesc::STRING))) {
str = *(char **)param->data();
if (str && Strutil::iequals (str, "separate")) {
m_planarconfig = PLANARCONFIG_SEPARATE;
if (! m_spec.tile_width) {
// I can only seem to make separate planarconfig work when
// rowsperstrip is 1.
TIFFSetField (m_tif, TIFFTAG_ROWSPERSTRIP, 1);
}
}
}
TIFFSetField (m_tif, TIFFTAG_PLANARCONFIG, m_planarconfig);
// Automatically set date field if the client didn't supply it.
if (! m_spec.find_attribute("DateTime")) {
time_t now;
time (&now);
struct tm mytm;
Sysutil::get_local_time (&now, &mytm);
std::string date = Strutil::format ("%4d:%02d:%02d %2d:%02d:%02d",
mytm.tm_year+1900, mytm.tm_mon+1, mytm.tm_mday,
mytm.tm_hour, mytm.tm_min, mytm.tm_sec);
m_spec.attribute ("DateTime", date);
}
// Write ICC profile, if we have anything
const ImageIOParameter* icc_profile_parameter = m_spec.find_attribute(ICC_PROFILE_ATTR);
if (icc_profile_parameter != NULL) {
unsigned char *icc_profile = (unsigned char*)icc_profile_parameter->data();
uint32 length = icc_profile_parameter->type().size();
if (icc_profile && length)
TIFFSetField (m_tif, TIFFTAG_ICCPROFILE, length, icc_profile);
}
if (Strutil::iequals (m_spec.get_string_attribute ("oiio:ColorSpace"), "sRGB"))
m_spec.attribute ("Exif:ColorSpace", 1);
// Deal with all other params
for (size_t p = 0; p < m_spec.extra_attribs.size(); ++p)
put_parameter (m_spec.extra_attribs[p].name().string(),
m_spec.extra_attribs[p].type(),
m_spec.extra_attribs[p].data());
std::vector<char> iptc;
encode_iptc_iim (m_spec, iptc);
if (iptc.size()) {
iptc.resize ((iptc.size()+3) & (0xffff-3)); // round up
TIFFSetField (m_tif, TIFFTAG_RICHTIFFIPTC, iptc.size()/4, &iptc[0]);
}
std::string xmp = encode_xmp (m_spec, true);
if (! xmp.empty())
TIFFSetField (m_tif, TIFFTAG_XMLPACKET, xmp.size(), xmp.c_str());
TIFFCheckpointDirectory (m_tif); // Ensure the header is written early
m_checkpointTimer.start(); // Initialize the to the fileopen time
m_checkpointItems = 0; // Number of tiles or scanlines we've written
m_dither = (m_spec.format == TypeDesc::UINT8) ?
m_spec.get_int_attribute ("oiio:dither", 0) : 0;
return true;
}
/*
* TestIndexPerformance() - Test driver for indices of a given type
*
* This function tests Insert and Delete performance together with
* key scan
*/
static void TestIndexPerformance(const IndexType &index_type) {
// This is where we read all values in and verify them
std::vector<ItemPointer *> location_ptrs;
// INDEX
std::unique_ptr<index::Index> index(BuildIndex(false, index_type));
// Parallel Test by default 1 Million key
// Number of threads doing insert or delete
size_t num_thread = 4;
// Number of keys inserted by each thread
size_t num_key = 1024 * 256;
Timer<> timer;
///////////////////////////////////////////////////////////////////
// Start InsertTest1
///////////////////////////////////////////////////////////////////
timer.Start();
// First two arguments are used for launching tasks
// All remaining arguments are passed to the thread body
LaunchParallelTest(num_thread, InsertTest1, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(num_thread * num_key, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("InsertTest1 :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// Start DeleteTest1
///////////////////////////////////////////////////////////////////
timer.Start();
LaunchParallelTest(num_thread, DeleteTest1, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(0, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("DeleteTest1 :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// Start InsertTest2
///////////////////////////////////////////////////////////////////
timer.Start();
LaunchParallelTest(num_thread, InsertTest2, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(num_thread * num_key, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("InsertTest2 :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// Start DeleteTest2
///////////////////////////////////////////////////////////////////
timer.Start();
LaunchParallelTest(num_thread, DeleteTest2, index.get(), num_thread, num_key);
// Perform garbage collection
if (index->NeedGC() == true) {
index->PerformGC();
}
index->ScanAllKeys(location_ptrs);
EXPECT_EQ(0, location_ptrs.size());
location_ptrs.clear();
timer.Stop();
LOG_INFO("DeleteTest :: Type=%s; Duration=%.2lf",
IndexTypeToString(index_type).c_str(), timer.GetDuration());
///////////////////////////////////////////////////////////////////
// End of all tests
///////////////////////////////////////////////////////////////////
delete tuple_schema;
return;
}
void commit( set<DiskLoc>* dupsToDrop,
CurOp* op,
bool mayInterrupt ) {
Timer timer;
IndexCatalogEntry* entry = _real->_btreeState;
bool dupsAllowed = !entry->descriptor()->unique() ||
ignoreUniqueIndex(entry->descriptor());
bool dropDups = entry->descriptor()->dropDups() || inDBRepair;
BtreeBuilder<V> btBuilder(dupsAllowed, entry);
BSONObj keyLast;
scoped_ptr<BSONObjExternalSorter::Iterator> i( _phase1.sorter->iterator() );
// verifies that pm and op refer to the same ProgressMeter
ProgressMeter& pm = op->setMessage("Index Bulk Build: (2/3) btree bottom up",
"Index: (2/3) BTree Bottom Up Progress",
_phase1.nkeys,
10);
while( i->more() ) {
RARELY killCurrentOp.checkForInterrupt( !mayInterrupt );
ExternalSortDatum d = i->next();
try {
if ( !dupsAllowed && dropDups ) {
LastError::Disabled led( lastError.get() );
btBuilder.addKey(d.first, d.second);
}
else {
btBuilder.addKey(d.first, d.second);
}
}
catch( AssertionException& e ) {
if ( dupsAllowed ) {
// unknown exception??
throw;
}
if( e.interrupted() ) {
killCurrentOp.checkForInterrupt();
}
if ( ! dropDups )
throw;
/* we could queue these on disk, but normally there are very few dups,
* so instead we keep in ram and have a limit.
*/
if ( dupsToDrop ) {
dupsToDrop->insert(d.second);
uassert( 10092,
"too may dups on index build with dropDups=true",
dupsToDrop->size() < 1000000 );
}
}
pm.hit();
}
pm.finished();
op->setMessage("Index Bulk Build: (3/3) btree-middle",
"Index: (3/3) BTree Middle Progress");
LOG(timer.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit";
btBuilder.commit( mayInterrupt );
if ( btBuilder.getn() != _phase1.nkeys && ! dropDups ) {
warning() << "not all entries were added to the index, probably some "
<< "keys were too large" << endl;
}
}
void dd::GibbsSampling::learn(const int & n_epoch, const int & n_sample_per_epoch,
const double & stepsize, const double & decay,
const double reg_param, const double reg1_param,
const bool is_quiet){
//const std::string meta_file, const bool is_quiet){
Timer t_total;
double current_stepsize = stepsize;
Timer t;
int nvar = this->factorgraphs[0].n_var;
int nnode = n_numa_nodes + 1;
int nweight = this->factorgraphs[0].n_weight;
// int num_sources_per_var[nvar];
// for (int i = 0; i < nvar; i++) {
// num_sources_per_var[i] = 1;
// }
// std::string full_feature_names[nvar];
// int feature_values[nvar];
// std::map<std::string, int*> feature_var_map;
// std::map<std::string,int*>::iterator it;
// int num_feature_names = 0;
//
// std::cout<<"Opening file "<<meta_file<<std::endl;
// std::ifstream myfile;
// std::string data;
// myfile.open (meta_file);
// if (myfile.good()) {
// while ( std::getline(myfile,data)) {
// istringstream iss(data);
// std::string temp;
// std::getline(iss, temp, '\t');
// int feature_id = std::stoi(temp, nullptr);
// std::string table_name;
// std::getline(iss, table_name, '\t');
// std::string feature_data;
// std::getline(iss, feature_data);
// feature_data = feature_data.substr(1, feature_data.length() - 2);
// temp = feature_data.substr(14 + feature_data.find("source_count\":"), feature_data.find(",feature_name") - feature_data.find("source_count\":") - 14 );
// num_sources_per_var[feature_id] = std::stoi(temp, nullptr);
// temp = feature_data.substr(13 + feature_data.find("feature_name:"), feature_data.find(",feature_value") - feature_data.find("feature_name:") - 13 );
// full_feature_names[feature_id] = table_name + "." + temp;
// temp = feature_data.substr(14 + feature_data.find("feature_value:"), feature_data.find("}") - feature_data.find("feature_value:") - 14 );
// feature_values[feature_id] = (int)std::stod(temp, nullptr);
// it = feature_var_map.find(full_feature_names[feature_id]);
// if (it != feature_var_map.end()) {
// int* arr = it;
// arr[feature_values[feature_id]] = feature_id;
// } else {
// feature_var_map[full_feature_names[feature_id]] = new int[10]; // NOTE: Hardcoded, change later.
// }
// feature_var_map[full_feature_names[feature_id]] = ;
// std::cout << feature_id << " -- " << table_name << " - " << full_feature_names[feature_id] << " ;; " << feature_values[feature_id] << " ,, " << num_sources_per_var[feature_id] << std::endl;
// }
// }
// myfile.close();
// single node samplers
std::vector<SingleNodeSampler> single_node_samplers;
for(int i=0;i<=n_numa_nodes;i++){
single_node_samplers.push_back(SingleNodeSampler(&this->factorgraphs[i],
n_thread_per_numa, i, false, 0, learn_non_evidence));
}
std::unique_ptr<double[]> ori_weights(new double[nweight]);
memcpy(ori_weights.get(), this->factorgraphs[0].infrs->weight_values, sizeof(double)*nweight);
// learning epochs
for(int i_epoch=0;i_epoch<n_epoch;i_epoch++){
if (!is_quiet) {
std::cout << std::setprecision(2) << "LEARNING EPOCH " << i_epoch * nnode << "~"
<< ((i_epoch+1) * nnode) << "...." << std::flush;
}
t.restart();
// set stepsize
for(int i=0;i<nnode;i++){
single_node_samplers[i].p_fg->stepsize = current_stepsize;
}
// performs stochastic gradient descent with sampling
for(int i=0;i<nnode;i++){
single_node_samplers[i].sample_sgd();
}
// wait the samplers to finish
for(int i=0;i<nnode;i++){
single_node_samplers[i].wait_sgd();
}
FactorGraph & cfg = this->factorgraphs[0];
// sum the weights and store in the first factor graph
// the average weights will be calculated and assigned to all factor graphs
for(int i=1;i<=n_numa_nodes;i++){
FactorGraph & cfg_other = this->factorgraphs[i];
for(int j=0;j<nweight;j++){
cfg.infrs->weight_values[j] += cfg_other.infrs->weight_values[j];
}
}
// calculate average weights and regularize weights
for(int j=0;j<nweight;j++){
cfg.infrs->weight_values[j] /= nnode;
if(cfg.infrs->weights_isfixed[j] == false){
cfg.infrs->weight_values[j] *= (1.0/(1.0+reg_param*current_stepsize));
double l1delta = reg1_param * current_stepsize;
if (cfg.infrs->weight_values[j] > l1delta) {
cfg.infrs->weight_values[j] -= l1delta;
} else if (cfg.infrs->weight_values[j] < - l1delta) {
cfg.infrs->weight_values[j] += l1delta;
} else {
cfg.infrs->weight_values[j] = 0;
}
}
}
// set weights for other factor graph to be the same as the first factor graph
for(int i=1;i<=n_numa_nodes;i++){
FactorGraph &cfg_other = this->factorgraphs[i];
for(int j=0;j<nweight;j++){
if(cfg.infrs->weights_isfixed[j] == false){
cfg_other.infrs->weight_values[j] = cfg.infrs->weight_values[j];
}
}
}
// calculate the norms of the difference of weights from the current epoch
// and last epoch
double lmax = -1000000;
double l2=0.0;
for(int i=0;i<nweight;i++){
double diff = fabs(ori_weights[i] - cfg.infrs->weight_values[i]);
ori_weights[i] = cfg.infrs->weight_values[i];
l2 += diff*diff;
if(lmax < diff){
lmax = diff;
}
}
lmax = lmax/current_stepsize;
double elapsed = t.elapsed();
if (!is_quiet) {
std::cout << "" << elapsed << " sec.";
std::cout << "," << (nvar*nnode)/elapsed << " vars/sec." << ",stepsize=" << current_stepsize << ",lmax=" << lmax << ",l2=" << sqrt(l2)/current_stepsize << std::endl;
}
current_stepsize = current_stepsize * decay;
}
double elapsed = t_total.elapsed();
std::cout << "TOTAL LEARNING TIME: " << elapsed << " sec." << std::endl;
}
bool dbEval(const string& dbName, BSONObj& cmd, BSONObjBuilder& result, string& errmsg) {
BSONElement e = cmd.firstElement();
uassert( 10046 , "eval needs Code" , e.type() == Code || e.type() == CodeWScope || e.type() == String );
const char *code = 0;
switch ( e.type() ) {
case String:
case Code:
code = e.valuestr();
break;
case CodeWScope:
code = e.codeWScopeCode();
break;
default:
verify(0);
}
verify( code );
if ( ! globalScriptEngine ) {
errmsg = "db side execution is disabled";
return false;
}
const string userToken = ClientBasic::getCurrent()->getAuthorizationSession()
->getAuthenticatedUserNamesToken();
auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" + userToken );
ScriptingFunction f = s->createFunction(code);
if ( f == 0 ) {
errmsg = (string)"compile failed: " + s->getError();
return false;
}
if ( e.type() == CodeWScope )
s->init( e.codeWScopeScopeDataUnsafe() );
s->localConnect( dbName.c_str() );
BSONObj args;
{
BSONElement argsElement = cmd.getField("args");
if ( argsElement.type() == Array ) {
args = argsElement.embeddedObject();
if ( edebug ) {
log() << "args:" << args.toString() << endl;
log() << "code:\n" << code << endl;
}
}
}
int res;
{
Timer t;
res = s->invoke(f, &args, 0, storageGlobalParams.quota ? 10 * 60 * 1000 : 0);
int m = t.millis();
if (m > serverGlobalParams.slowMS) {
log() << "dbeval slow, time: " << dec << m << "ms " << dbName << endl;
if ( m >= 1000 ) log() << code << endl;
else OCCASIONALLY log() << code << endl;
}
}
if (res || s->isLastRetNativeCode()) {
result.append("errno", (double) res);
errmsg = "invoke failed: ";
if (s->isLastRetNativeCode())
errmsg += "cannot return native function";
else
errmsg += s->getError();
return false;
}
s->append( result , "retval" , "__returnValue" );
return true;
}
void SimpleRecordStoreV1::_compactExtent(OperationContext* txn,
const DiskLoc diskloc,
int extentNumber,
RecordStoreCompactAdaptor* adaptor,
const CompactOptions* compactOptions,
CompactStats* stats ) {
log() << "compact begin extent #" << extentNumber
<< " for namespace " << _ns << " " << diskloc;
unsigned oldObjSize = 0; // we'll report what the old padding was
unsigned oldObjSizeWithPadding = 0;
Extent *e = _extentManager->getExtent( diskloc );
e->assertOk();
fassert( 17437, e->validates(diskloc) );
{
// the next/prev pointers within the extent might not be in order so we first
// page the whole thing in sequentially
log() << "compact paging in len=" << e->length/1000000.0 << "MB" << endl;
Timer t;
size_t length = e->length;
touch_pages( reinterpret_cast<const char*>(e), length );
int ms = t.millis();
if( ms > 1000 )
log() << "compact end paging in " << ms << "ms "
<< e->length/1000000.0/t.seconds() << "MB/sec" << endl;
}
{
log() << "compact copying records" << endl;
long long datasize = 0;
long long nrecords = 0;
DiskLoc L = e->firstRecord;
if( !L.isNull() ) {
while( 1 ) {
Record *recOld = recordFor(L);
RecordData oldData = recOld->toRecordData();
L = getNextRecordInExtent(txn, L);
if ( compactOptions->validateDocuments && !adaptor->isDataValid( oldData ) ) {
// object is corrupt!
log() << "compact skipping corrupt document!";
stats->corruptDocuments++;
}
else {
unsigned dataSize = adaptor->dataSize( oldData );
unsigned docSize = dataSize;
nrecords++;
oldObjSize += docSize;
oldObjSizeWithPadding += recOld->netLength();
unsigned lenWHdr = docSize + Record::HeaderSize;
unsigned lenWPadding = lenWHdr;
switch( compactOptions->paddingMode ) {
case CompactOptions::NONE:
if ( _details->isUserFlagSet(Flag_UsePowerOf2Sizes) )
lenWPadding = quantizePowerOf2AllocationSpace(lenWPadding);
break;
case CompactOptions::PRESERVE:
// if we are preserving the padding, the record should not change size
lenWPadding = recOld->lengthWithHeaders();
break;
case CompactOptions::MANUAL:
lenWPadding = compactOptions->computeRecordSize(lenWPadding);
if (lenWPadding < lenWHdr || lenWPadding > BSONObjMaxUserSize / 2 ) {
lenWPadding = lenWHdr;
}
break;
}
CompactDocWriter writer( recOld, dataSize, lenWPadding );
StatusWith<DiskLoc> status = insertRecord( txn, &writer, false );
uassertStatusOK( status.getStatus() );
datasize += recordFor( status.getValue() )->netLength();
adaptor->inserted( dataFor( txn, status.getValue() ), status.getValue() );
}
if( L.isNull() ) {
// we just did the very last record from the old extent. it's still pointed to
// by the old extent ext, but that will be fixed below after this loop
break;
}
// remove the old records (orphan them) periodically so our commit block doesn't get too large
bool stopping = false;
RARELY stopping = !txn->checkForInterruptNoAssert().isOK();
if( stopping || txn->recoveryUnit()->isCommitNeeded() ) {
*txn->recoveryUnit()->writing(&e->firstRecord) = L;
Record *r = recordFor(L);
txn->recoveryUnit()->writingInt(r->prevOfs()) = DiskLoc::NullOfs;
txn->recoveryUnit()->commitIfNeeded();
txn->checkForInterrupt();
}
}
} // if !L.isNull()
invariant( _details->firstExtent(txn) == diskloc );
invariant( _details->lastExtent(txn) != diskloc );
DiskLoc newFirst = e->xnext;
_details->setFirstExtent( txn, newFirst );
*txn->recoveryUnit()->writing(&_extentManager->getExtent( newFirst )->xprev) = DiskLoc();
_extentManager->freeExtent( txn, diskloc );
txn->recoveryUnit()->commitIfNeeded();
{
double op = 1.0;
if( oldObjSize )
op = static_cast<double>(oldObjSizeWithPadding)/oldObjSize;
log() << "compact finished extent #" << extentNumber << " containing " << nrecords
<< " documents (" << datasize/1000000.0 << "MB)"
<< " oldPadding: " << op << ' ' << static_cast<unsigned>(op*100.0)/100;
}
}
}
int main( int argc, char* argv[] )
{
Environment env( argc, argv );
try
{
const string inputBasisFile =
Input("--inputBasisFile","input basis file",
string("../data/number_theory/SVPChallenge40.txt"));
const bool trans = Input("--transpose","transpose input?",true);
const string outputBasisFile =
Input("--outputBasisFile","output basis file",string("BKZ"));
const string shortestVecFile =
Input
("--shortestVecFile","shortest vector file",string("shortest"));
const Real delta = Input("--delta","delta for LLL",Real(0.9999));
const Real eta =
Input
("--eta","eta for LLL",
Real(1)/Real(2) + Pow(limits::Epsilon<Real>(),Real(0.9)));
const Int varInt = Input("--variant","0: weak LLL, 1: normal LLL, 2: deep insertion LLL, 3: deep reduction LLL",1);
const Int blocksize =
Input("--blocksize","BKZ blocksize",20);
const bool variableBsize =
Input("--variableBsize","variable blocksize?",false);
const bool variableEnumType =
Input("--variableEnumType","variable enum type?",false);
const Int multiEnumWindow =
Input("--multiEnumWindow","window for y-sparse enumeration",15);
const Int phaseLength =
Input("--phaseLength","YSPARSE_ENUM phase length",10);
const double enqueueProb =
Input("--enqueueProb","enqueue probability?",1.);
const Int progressLevel =
Input("--progressLevel","YSPARSE_ENUM progress level",4);
const bool presort = Input("--presort","presort columns?",false);
const bool smallestFirst =
Input("--smallestFirst","sort smallest first?",true);
const bool recursiveLLL = Input("--recursiveLLL","recursive LLL?",true);
const bool recursiveBKZ =
Input("--recursiveBKZ","recursive BKZ?",false);
const Int cutoff = Input("--cutoff","recursive cutoff",10);
const bool earlyAbort = Input("--earlyAbort","early abort BKZ?",false);
const Int numEnumsBeforeAbort =
Input("--numEnumsBeforeAbort","num enums before early aborting",1000);
const bool subBKZ =
Input("--subBKZ","use BKZ w/ lower blocksize for subproblems?",true);
const bool subEarlyAbort =
Input("--subEarlyAbort","early abort subproblem?",false);
const bool jumpstartBKZ =
Input("--jumpstartBKZ","jumpstart BKZ?",false);
const Int startColBKZ = Input("--startColBKZ","BKZ start column",0);
const bool timeLLL = Input("--timeLLL","time LLL?",false);
const bool timeBKZ = Input("--timeBKZ","time BKZ?",true);
const bool progressLLL =
Input("--progressLLL","print LLL progress?",false);
const bool progressBKZ =
Input("--progressBKZ","print BKZ progress?",true);
const bool print = Input("--print","output all matrices?",true);
const bool logFailedEnums =
Input("--logFailedEnums","log failed enumerations in BKZ?",false);
const bool logStreakSizes =
Input("--logStreakSizes","log enum streak sizes in BKZ?",false);
const bool logNontrivialCoords =
Input("--logNontrivialCoords","log nontrivial enum coords?",false);
const bool logNorms = Input("--logNorms","log norms of B?",true);
const bool logProjNorms =
Input("--logProjNorms","log proj norms of B?",true);
const bool checkpoint =
Input("--checkpoint","checkpoint each tour?",true);
const Real targetRatio =
Input("--targetRatio","targeted ratio of GH(L)",Real(1.05));
const bool timeEnum = Input("--timeEnum","time enum?",true);
const bool innerEnumProgress =
Input("--innerEnumProgress","inner enum progress?",false);
const bool probEnum =
Input("--probEnum","probabalistic enumeration *after* BKZ?",true);
const bool fullEnum = Input("--fullEnum","SVP via full enum?",false);
#ifdef EL_HAVE_MPC
const mpfr_prec_t prec =
Input("--prec","MPFR precision",mpfr_prec_t(1024));
#endif
ProcessInput();
PrintInputReport();
#ifdef EL_HAVE_MPC
mpfr::SetPrecision( prec );
#endif
Matrix<Real> B;
if( trans )
{
Matrix<Real> BTrans;
Read( BTrans, inputBasisFile );
Transpose( BTrans, B );
}
else
Read( B, inputBasisFile );
const Int m = B.Height();
const Int n = B.Width();
const Real BOrigOne = OneNorm( B );
Output("|| B_orig ||_1 = ",BOrigOne);
if( print )
Print( B, "BOrig" );
auto blocksizeLambda =
[&]( Int j )
{
// With k-sparse
if( j <= 3 )
return 146;
else if( j <= 10 )
return 62;
else if( j <= 20 )
return 60;
else if( j <= 50 )
return 55;
else
return 45;
// Full enum
/*
if( j == 0 )
return 80;
else if( j == 1 )
return 75;
else if( j == 2 )
return 70;
else if( j <= 10 )
return 62;
else if( j <= 20 )
return 60;
else if( j <= 50 )
return 55;
else
return 45;
*/
};
auto enumTypeLambda =
[&]( Int j )
{
if( j <= 3 )
return YSPARSE_ENUM;
else
return FULL_ENUM;
//return FULL_ENUM;
};
BKZCtrl<Real> ctrl;
ctrl.blocksize = blocksize;
ctrl.variableBlocksize = variableBsize;
ctrl.blocksizeFunc = function<Int(Int)>(blocksizeLambda);
ctrl.variableEnumType = variableEnumType;
ctrl.enumTypeFunc = function<EnumType(Int)>(enumTypeLambda);
ctrl.multiEnumWindow = multiEnumWindow;
ctrl.time = timeBKZ;
ctrl.progress = progressBKZ;
ctrl.recursive = recursiveBKZ;
ctrl.jumpstart = jumpstartBKZ;
ctrl.startCol = startColBKZ;
ctrl.enumCtrl.enumType = FULL_ENUM;
ctrl.enumCtrl.time = timeEnum;
ctrl.enumCtrl.innerProgress = innerEnumProgress;
ctrl.enumCtrl.phaseLength = phaseLength;
ctrl.enumCtrl.enqueueProb = enqueueProb;
ctrl.enumCtrl.progressLevel = progressLevel;
ctrl.earlyAbort = earlyAbort;
ctrl.numEnumsBeforeAbort = numEnumsBeforeAbort;
ctrl.subBKZ = subBKZ;
ctrl.subEarlyAbort = subEarlyAbort;
ctrl.logFailedEnums = logFailedEnums;
ctrl.logStreakSizes = logStreakSizes;
ctrl.logNontrivialCoords = logNontrivialCoords;
ctrl.logNorms = logNorms;
ctrl.logProjNorms = logProjNorms;
ctrl.checkpoint = checkpoint;
ctrl.lllCtrl.delta = delta;
ctrl.lllCtrl.eta = eta;
ctrl.lllCtrl.variant = static_cast<LLLVariant>(varInt);
ctrl.lllCtrl.recursive = recursiveLLL;
ctrl.lllCtrl.cutoff = cutoff;
ctrl.lllCtrl.presort = presort;
ctrl.lllCtrl.smallestFirst = smallestFirst;
ctrl.lllCtrl.progress = progressLLL;
ctrl.lllCtrl.time = timeLLL;
// TODO(poulson): Make this less fragile
/*
ctrl.enumCtrl.customMinInfNorms = true;
ctrl.enumCtrl.customMaxInfNorms = true;
ctrl.enumCtrl.customMinOneNorms = true;
ctrl.enumCtrl.customMaxOneNorms = true;
const Int startIndex = Max(n/2-1,0);
const Int numPhases = ((n-startIndex)+phaseLength-1) / phaseLength;
Output("numPhases=",numPhases);
ctrl.enumCtrl.minInfNorms.resize( numPhases, 0 );
ctrl.enumCtrl.maxInfNorms.resize( numPhases, 1 );
ctrl.enumCtrl.minOneNorms.resize( numPhases, 0 );
ctrl.enumCtrl.maxOneNorms.resize( numPhases, 1 );
// NOTE: This is tailored to SVP 146 where the ranges are
// 0: [72,82)
// 1: [82,92)
// 2: [92,102)
// 3: [102,112)
// 4: [112,122)
// 5: [122,132)
// 6: [132,142)
// 7: [142,146)
ctrl.enumCtrl.maxOneNorms[0] = 0;
ctrl.enumCtrl.maxOneNorms[1] = 1;
ctrl.enumCtrl.maxOneNorms[2] = 1;
ctrl.enumCtrl.maxOneNorms[3] = 1;
ctrl.enumCtrl.maxOneNorms[4] = 1;
ctrl.enumCtrl.maxOneNorms[5] = 2;
ctrl.enumCtrl.maxOneNorms[6] = 3;
ctrl.enumCtrl.maxOneNorms[7] = 3;
ctrl.enumCtrl.maxInfNorms[0] = 1;
ctrl.enumCtrl.maxInfNorms[1] = 1;
ctrl.enumCtrl.maxInfNorms[2] = 1;
ctrl.enumCtrl.maxInfNorms[3] = 1;
ctrl.enumCtrl.maxInfNorms[4] = 1;
ctrl.enumCtrl.maxInfNorms[5] = 1;
ctrl.enumCtrl.maxInfNorms[6] = 2;
ctrl.enumCtrl.maxInfNorms[7] = 2;
*/
const double startTime = mpi::Time();
Matrix<Real> R;
auto info = BKZ( B, R, ctrl );
const double runTime = mpi::Time() - startTime;
Output
(" BKZ(",blocksize,",",delta,",",eta,") took ",runTime," seconds");
Output(" achieved delta: ",info.delta);
Output(" achieved eta: ",info.eta);
Output(" num swaps: ",info.numSwaps);
Output(" num enums: ",info.numEnums);
Output(" num failed enums: ",info.numEnumFailures);
Output(" log(vol(L)): ",info.logVol);
const Real GH = LatticeGaussianHeuristic( info.rank, info.logVol );
const Real challenge = targetRatio*GH;
Output(" GH(L): ",GH);
Output(" targetRatio*GH(L): ",challenge);
if( print )
{
Print( B, "B" );
Print( R, "R" );
}
Write( B, outputBasisFile, ASCII, "BKZ" );
const Real BOneNorm = OneNorm( B );
Output("|| B ||_1 = ",BOneNorm);
auto b0 = B( ALL, IR(0) );
const Real b0Norm = FrobeniusNorm( b0 );
Output("|| b_0 ||_2 = ",b0Norm);
if( print )
Print( b0, "b0" );
bool succeeded = false;
if( b0Norm <= challenge )
{
Output
("SVP Challenge solved via BKZ: || b_0 ||_2=",b0Norm,
" <= targetRatio*GH(L)=",challenge);
succeeded = true;
Write( b0, shortestVecFile, ASCII, "b0" );
}
else
Output
("SVP Challenge NOT solved via BKZ: || b_0 ||_2=",b0Norm,
" > targetRatio*GH(L)=",challenge);
if( !succeeded || fullEnum )
{
const Int start = 0;
const Int numCols = n;
const Range<Int> subInd( start, start+numCols );
auto BSub = B( ALL, subInd );
auto RSub = R( subInd, subInd );
const Real target = ( start == 0 ? challenge : RSub(0,0) );
Timer timer;
Matrix<F> v;
EnumCtrl<Real> enumCtrl;
enumCtrl.enumType = ( probEnum ? GNR_ENUM : FULL_ENUM );
timer.Start();
Real result;
if( fullEnum )
{
result =
ShortestVectorEnumeration( BSub, RSub, target, v, enumCtrl );
Output("shortest vector result = ",result);
}
else
{
result =
ShortVectorEnumeration( BSub, RSub, target, v, enumCtrl );
Output("short vector result = ",result);
}
Output("Enumeration: ",timer.Stop()," seconds");
if( result < target )
{
Print( BSub, "BSub" );
Print( v, "v" );
Matrix<Real> x;
Zeros( x, m, 1 );
Gemv( NORMAL, Real(1), BSub, v, Real(0), x );
Print( x, "x" );
const Real xNorm = FrobeniusNorm( x );
Output("|| x ||_2 = ",xNorm);
Output("Claimed || x ||_2 = ",result);
Write( x, shortestVecFile, ASCII, "x" );
EnrichLattice( BSub, v );
Print( B, "BNew" );
}
else
Output("Enumeration failed after ",timer.Stop()," seconds");
}
}
catch( std::exception& e ) { ReportException(e); }
return 0;
}
void run() {
try { boost::filesystem::remove(fn); }
catch(...) { }
Lock::GlobalWrite lk;
{
MongoMMF f;
unsigned long long len = 256 * 1024 * 1024;
verify( f.create(fn, len, /*sequential*/false) );
{
char *p = (char *) f.getView();
verify(p);
// write something to the private view as a test
if( cmdLine.dur )
MemoryMappedFile::makeWritable(p, 6);
strcpy(p, "hello");
}
if( cmdLine.dur ) {
char *w = (char *) f.view_write();
strcpy(w + 6, "world");
}
MongoFileFinder ff;
ASSERT( ff.findByPath(fn) );
ASSERT( ff.findByPath("asdf") == 0 );
}
{
MongoFileFinder ff;
ASSERT( ff.findByPath(fn) == 0 );
}
int N = 10000;
#if !defined(_WIN32) && !defined(__linux__)
// seems this test is slow on OS X.
N = 100;
#endif
// we make a lot here -- if we were leaking, presumably it would fail doing this many.
Timer t;
for( int i = 0; i < N; i++ ) {
MongoMMF f;
verify( f.open(fn, i%4==1) );
{
char *p = (char *) f.getView();
verify(p);
if( cmdLine.dur )
MemoryMappedFile::makeWritable(p, 4);
strcpy(p, "zzz");
}
if( cmdLine.dur ) {
char *w = (char *) f.view_write();
if( i % 2 == 0 )
++(*w);
verify( w[6] == 'w' );
}
}
if( t.millis() > 10000 ) {
log() << "warning: MMap LeakTest is unusually slow N:" << N << ' ' << t.millis() << "ms" << endl;
}
}
int main(int argc, char* argv[])
{
if (argc == 2)
{
//initiate variables
unsigned char hash[20];
char hexstr[41];
unsigned int size = strlen(argv[1]);
//convert to hash and then to string
calc(argv[1], size, hash);
toHexString(hash, hexstr);
//print final string
std::cout << hexstr << std::endl;
}
else if (argc == 3)
{
//check if files exist and open them
std::ifstream dict(argv[1]);
std::ifstream pass(argv[2]);
if (pass.good() && dict.good())
{
//load dictionary
HashMap dictTable = HashMap(dict);
std::cout << dictTable.mNumElements << " elements added to table" << std::endl;
std::cout << "Dictionary loaded in " << dictTable.mElapsed << " seconds." << std::endl;
//open file for writing
std::string line;
std::ofstream solved;
solved.open("pass_solved.txt");
int count = 0;
if (pass.is_open() && solved.is_open())
{
while (getline(pass, line))
{
dictTable.getPass(line, count);
count++;
}
Timer timer;
timer.start();
tbb::parallel_invoke(
[&dictTable] {dictTable.force(0, 4); },
[&dictTable] {dictTable.force(4, 8); },
[&dictTable] {dictTable.force(8, 12); },
[&dictTable] {dictTable.force(12, 16); },
[&dictTable] {dictTable.force(16, 20); },
[&dictTable] {dictTable.force(20, 24); },
[&dictTable] {dictTable.force(24, 28); },
[&dictTable] {dictTable.force(28, 32); },
[&dictTable] {dictTable.force(32, 36); });
std::cout << "Passwords decrypted in " << timer.getElapsed() << " seconds" << std::endl;
for (int i = 0; i < signed int(dictTable.mSolved.size()); i++)
{
try
{
solved << dictTable.mSolved.at(i)->mHexStr << "," << dictTable.mSolved.at(i)->mSolution << std::endl;
}
catch (const std::out_of_range& oor)
{
std::cerr << "Out of Range error: " << oor.what() << '\n';
}
}
}
else
{
void run() {
string fn = "/tmp/testfile.map";
boost::filesystem::remove(fn);
MemoryMappedFile f;
char *p = (char *) f.create(fn, 1024 * 1024 * 1024, true);
verify(p);
strcpy(p, "hello");
{
void *x = f.testGetCopyOnWriteView();
Timer tt;
for( int i = 11; i < 1000000000; i++ )
p[i] = 'z';
cout << "fill 1GB time: " << tt.millis() << "ms" << endl;
f.testCloseCopyOnWriteView(x);
}
/* test a lot of view/unviews */
{
Timer t;
char *q;
for( int i = 0; i < 1000; i++ ) {
q = (char *) f.testGetCopyOnWriteView();
verify( q );
if( i == 999 ) {
strcpy(q+2, "there");
}
f.testCloseCopyOnWriteView(q);
}
cout << "view unview: " << t.millis() << "ms" << endl;
}
f.flush(true);
/* plain old mmaped writes */
{
Timer t;
for( int i = 0; i < 10; i++ ) {
memset(p+100, 'c', 200 * 1024 * 1024);
}
cout << "traditional writes: " << t.millis() << "ms" << endl;
}
f.flush(true);
/* test doing some writes */
{
Timer t;
char *q = (char *) f.testGetCopyOnWriteView();
for( int i = 0; i < 10; i++ ) {
verify( q );
memset(q+100, 'c', 200 * 1024 * 1024);
}
f.testCloseCopyOnWriteView(q);
cout << "inc style some writes: " << t.millis() << "ms" << endl;
}
/* test doing some writes */
{
Timer t;
for( int i = 0; i < 10; i++ ) {
char *q = (char *) f.testGetCopyOnWriteView();
verify( q );
memset(q+100, 'c', 200 * 1024 * 1024);
f.testCloseCopyOnWriteView(q);
}
cout << "some writes: " << t.millis() << "ms" << endl;
}
/* more granular */
{
Timer t;
for( int i = 0; i < 100; i++ ) {
char *q = (char *) f.testGetCopyOnWriteView();
verify( q );
memset(q+100, 'c', 20 * 1024 * 1024);
f.testCloseCopyOnWriteView(q);
}
cout << "more granular some writes: " << t.millis() << "ms" << endl;
}
p[10] = 0;
cout << p << endl;
}
/** We need to remap the private views periodically. otherwise they would become very large.
Call within write lock. See top of file for more commentary.
*/
void REMAPPRIVATEVIEW() {
Timer t;
_REMAPPRIVATEVIEW();
stats.curr->_remapPrivateViewMicros += t.micros();
}
static void _REMAPPRIVATEVIEW() {
// todo: Consider using ProcessInfo herein and watching for getResidentSize to drop. that could be a way
// to assure very good behavior here.
static unsigned startAt;
static unsigned long long lastRemap;
LOG(4) << "journal REMAPPRIVATEVIEW" << endl;
verify( Lock::isW() );
verify( !commitJob.hasWritten() );
// we want to remap all private views about every 2 seconds. there could be ~1000 views so
// we do a little each pass; beyond the remap time, more significantly, there will be copy on write
// faults after remapping, so doing a little bit at a time will avoid big load spikes on
// remapping.
unsigned long long now = curTimeMicros64();
double fraction = (now-lastRemap)/2000000.0;
if( cmdLine.durOptions & CmdLine::DurAlwaysRemap )
fraction = 1;
lastRemap = now;
#if defined(_WIN32)
// Note that this negatively affects performance.
// We must grab the exclusive lock here because remapThePrivateView() on Windows needs
// to grab it as well, due to the lack of a non-atomic way to remap a memory mapped file.
// See SERVER-5723 for performance improvement.
// See SERVER-5680 to see why this code is necessary.
LockMongoFilesExclusive lk;
#else
LockMongoFilesShared lk;
#endif
set<MongoFile*>& files = MongoFile::getAllFiles();
unsigned sz = files.size();
if( sz == 0 )
return;
{
// be careful not to use too much memory if the write rate is
// extremely high
double f = privateMapBytes / ((double)UncommittedBytesLimit);
if( f > fraction ) {
fraction = f;
}
privateMapBytes = 0;
}
unsigned ntodo = (unsigned) (sz * fraction);
if( ntodo < 1 ) ntodo = 1;
if( ntodo > sz ) ntodo = sz;
const set<MongoFile*>::iterator b = files.begin();
const set<MongoFile*>::iterator e = files.end();
set<MongoFile*>::iterator i = b;
// skip to our starting position
for( unsigned x = 0; x < startAt; x++ ) {
i++;
if( i == e ) i = b;
}
unsigned startedAt = startAt;
startAt = (startAt + ntodo) % sz; // mark where to start next time
Timer t;
for( unsigned x = 0; x < ntodo; x++ ) {
dassert( i != e );
if( (*i)->isMongoMMF() ) {
MongoMMF *mmf = (MongoMMF*) *i;
verify(mmf);
if( mmf->willNeedRemap() ) {
mmf->willNeedRemap() = false;
mmf->remapThePrivateView();
}
i++;
if( i == e ) i = b;
}
}
LOG(2) << "journal REMAPPRIVATEVIEW done startedAt: " << startedAt << " n:" << ntodo << ' ' << t.millis() << "ms" << endl;
}
bool KMeans::trainModel(MatrixFloat &data){
if( numClusters == 0 ){
errorLog << "trainModel(MatrixFloat &data) - Failed to train model. NumClusters is zero!" << std::endl;
return false;
}
if( clusters.getNumRows() != numClusters ){
errorLog << "trainModel(MatrixFloat &data) - Failed to train model. The number of rows in the cluster matrix does not match the number of clusters! You should need to initalize the clusters matrix first before calling this function!" << std::endl;
return false;
}
if( clusters.getNumCols() != numInputDimensions ){
errorLog << "trainModel(MatrixFloat &data) - Failed to train model. The number of columns in the cluster matrix does not match the number of input dimensions! You should need to initalize the clusters matrix first before calling this function!" << std::endl;
return false;
}
Timer timer;
UINT currentIter = 0;
UINT numChanged = 0;
bool keepTraining = true;
Float theta = 0;
Float lastTheta = 0;
Float delta = 0;
Float startTime = 0;
thetaTracker.clear();
finalTheta = 0;
numTrainingIterationsToConverge = 0;
trained = false;
converged = false;
//Scale the data if needed
ranges = data.getRanges();
if( useScaling ){
data.scale(0,1);
}
//Init the assign and count Vectors
//Assign is set to K+1 so that the nChanged values in the eStep at the first iteration will be updated correctly
for(UINT m=0; m<numTrainingSamples; m++) assign[m] = numClusters+1;
for(UINT k=0; k<numClusters; k++) count[k] = 0;
//Run the training loop
timer.start();
while( keepTraining ){
startTime = timer.getMilliSeconds();
//Compute the E step
numChanged = estep( data );
//Compute the M step
mstep( data );
//Update the iteration counter
currentIter++;
//Compute theta if needed
if( computeTheta ){
theta = calculateTheta(data);
delta = lastTheta - theta;
lastTheta = theta;
}else theta = delta = 0;
//Check convergance
if( numChanged == 0 && currentIter > minNumEpochs ){ converged = true; keepTraining = false; }
if( currentIter >= maxNumEpochs ){ keepTraining = false; }
if( fabs( delta ) < minChange && computeTheta && currentIter > minNumEpochs ){ converged = true; keepTraining = false; }
if( computeTheta ) thetaTracker.push_back( theta );
trainingLog << "Epoch: " << currentIter << "/" << maxNumEpochs;
trainingLog << " Epoch time: " << (timer.getMilliSeconds()-startTime)/1000.0 << " seconds";
trainingLog << " Theta: " << theta << " Delta: " << delta << std::endl;
}
trainingLog << "Model Trained at epoch: " << currentIter << " with a theta value of: " << theta << std::endl;
finalTheta = theta;
numTrainingIterationsToConverge = currentIter;
trained = true;
//Setup the cluster labels
clusterLabels.resize(numClusters);
for(UINT i=0; i<numClusters; i++){
clusterLabels[i] = i+1;
}
clusterLikelihoods.resize(numClusters,0);
clusterDistances.resize(numClusters,0);
return true;
}
void NonLinearDynamic :: updateComponent(TimeStep *tStep, NumericalCmpn cmpn, Domain *d)
//
// Updates some component, which is used by numerical method
// to newly reached state. used mainly by numerical method
// when new tanget stiffness is needed during finding
// of new equlibrium stage.
//
{
#ifdef TIME_REPORT
Timer timer;
#endif
switch ( cmpn ) {
case NonLinearLhs:
// Prevent assembly if already assembled ( totIterations > 0 )
// Allow if MANRMSteps != 0
if ( ( totIterations == 0 ) || MANRMSteps ) {
#ifdef VERBOSE
OOFEM_LOG_DEBUG("Updating effective stiffness matrix\n");
#endif
#ifdef TIME_REPORT
timer.startTimer();
#endif
#if 1
effectiveStiffnessMatrix->zero();
this->assemble(*effectiveStiffnessMatrix, tStep, EffectiveTangentAssembler(false, 1 + this->delta * a1, this->a0 + this->eta * this->a1),
EModelDefaultEquationNumbering(), d);
#else
this->assemble(effectiveStiffnessMatrix, tStep, TangentStiffnessMatrix,
EModelDefaultEquationNumbering(), d);
effectiveStiffnessMatrix->times(1. + this->delta * a1);
effectiveStiffnessMatrix->add(this->a0 + this->eta * this->a1, this->massMatrix);
#endif
#ifdef TIME_REPORT
timer.stopTimer();
OOFEM_LOG_DEBUG( "User time consumed by updating nonlinear LHS: %.2fs\n", timer.getUtime() );
#endif
}
break;
case InternalRhs:
#ifdef VERBOSE
OOFEM_LOG_DEBUG("Updating internal RHS\n");
#endif
{
#ifdef TIME_REPORT
timer.startTimer();
#endif
if ( ( currentIterations != 0 ) || ( totIterations == 0 ) ) {
this->giveInternalForces(internalForces, true, d->giveNumber(), tStep);
// Updating the residual vector @ NR-solver
help.beScaled(a0 + eta * a1, incrementOfDisplacement);
massMatrix->times(help, rhs2);
forcesVector = internalForces;
forcesVector.add(rhs2);
forcesVector.subtract(previousInternalForces);
if ( delta != 0 ) {
help.beScaled(delta * a1, incrementOfDisplacement);
this->timesMtrx(help, rhs2, TangentStiffnessMatrix, this->giveDomain(1), tStep);
//this->assembleVector(rhs2, tStep, MatrixProductAssembler(TangentAssembler(), help), VM_Total,
// EModelDefaultEquationNumbering(), this->giveDomain(1));
forcesVector.add(rhs2);
}
}
#ifdef TIME_REPORT
timer.stopTimer();
OOFEM_LOG_DEBUG( "User time consumed by updating internal RHS: %.2fs\n", timer.getUtime() );
#endif
}
break;
default:
OOFEM_ERROR("Unknown Type of component.");
}
}