// AMD reports maxCpuidIdFunction > 4 but consider functions 2..4 to be
// "reserved". cache characteristics are returned via ext. functions.
static void DetectCacheAndTLB()
{
x86_x64::CpuidRegs regs = { 0 };
regs.eax = 0x80000005;
if(x86_x64::cpuid(®s))
{
AddCache(L1Cache(regs.ecx, x86_x64::Cache::kData));
AddCache(L1Cache(regs.edx, x86_x64::Cache::kInstruction));
AddTLB(TLB1(regs.eax, 0, 2*MiB, x86_x64::Cache::kInstruction));
AddTLB(TLB1(regs.eax, 16, 2*MiB, x86_x64::Cache::kData));
AddTLB(TLB1(regs.ebx, 0, 4*KiB, x86_x64::Cache::kInstruction));
AddTLB(TLB1(regs.ebx, 16, 4*KiB, x86_x64::Cache::kData));
}
regs.eax = 0x80000006;
if(x86_x64::cpuid(®s))
{
AddCache(L2Cache(regs.ecx, x86_x64::Cache::kUnified));
AddCache(L3Cache(regs.edx, x86_x64::Cache::kUnified));
AddTLB2Pair(regs.eax, 2*MiB);
AddTLB2Pair(regs.ebx, 4*KiB);
}
}
static void DetectCacheAndTLB(size_t& descriptorFlags)
{
const Descriptors descriptors = GetDescriptors();
for(Descriptors::const_iterator it = descriptors.begin(); it != descriptors.end(); ++it)
{
const Descriptor descriptor = *it;
if(HandleSpecialDescriptor(descriptor, descriptorFlags))
continue;
const Characteristics* characteristics = CharacteristicsFromDescriptor(*it);
if(!characteristics)
continue;
if((descriptorFlags & SKIP_CACHE_DESCRIPTORS) && !characteristics->IsTLB())
continue;
x86_x64::Cache cache;
cache.Initialize(characteristics->Level(), characteristics->Type());
cache.numEntries = characteristics->NumEntries();
cache.entrySize = characteristics->EntrySize();
cache.associativity = characteristics->associativity;
cache.sharedBy = 1; // (safe default)
if(characteristics->IsTLB())
AddTLB(cache);
else
AddCache(cache);
}
}
static bool DetectCache()
{
// note: level order is unspecified (see Intel AP-485)
for(u32 count = 0; ; count++)
{
x86_x64::CpuidRegs regs = { 0 };
regs.eax = 4;
regs.ecx = count;
if(!x86_x64::cpuid(®s))
return false;
const x86_x64::Cache::Type type = (x86_x64::Cache::Type)bits(regs.eax, 0, 4);
if(type == x86_x64::Cache::kNull) // no more remaining
break;
const size_t level = (size_t)bits(regs.eax, 5, 7);
const size_t partitions = (size_t)bits(regs.ebx, 12, 21)+1;
const size_t sets = (size_t)bits(regs.ecx, 0, 31)+1;
x86_x64::Cache cache;
cache.Initialize(level, type);
cache.entrySize = (size_t)bits(regs.ebx, 0, 11)+1; // (yes, this also uses +1 encoding)
cache.associativity = (size_t)bits(regs.ebx, 22, 31)+1;
cache.sharedBy = (size_t)bits(regs.eax, 14, 25)+1;
cache.numEntries = cache.associativity * partitions * sets;
AddCache(cache);
}
return true;
}
//------------------------------------------------------------------------------
IContainer* TSynchroAbonent::GetEvent(int& id_sender)
{
// найти среди кеша
IContainer* pC = mSynchroPoint->GetEvent(mSelfID, id_sender);
if(pC)
AddCache(pC, id_sender);
return pC;
}
apr_status_t kahanaIOLoadFile( apr_pool_t *p, const char *path, size_t *size, bool *is_cache, void **data )
{
apr_status_t rc = APR_SUCCESS;
struct stat finfo;
char *d = NULL;
*size = 0;
if( stat( path, &finfo ) == -1 ){
rc = errno;
kahanaLogPut( NULL, NULL, "failed to stat(): %s", strerror( rc ) );
}
else
{
apr_file_t *file = NULL;
if( ( d = CacheFile( p, path, finfo.st_mtime, size ) ) ){
*is_cache = TRUE;
*size = finfo.st_size;
*data = d;
}
else if( ( rc = apr_file_open( &file, path, APR_READ, APR_UREAD|APR_OS_DEFAULT, p ) ) ){
kahanaLogPut( NULL, NULL, "failed to apr_file_open(): %s", STRERROR_APR( rc ) );
}
else
{
if( ( rc = kahanaIOReadFile( p, file, &d ) ) == APR_SUCCESS )
{
*is_cache = FALSE;
*size = finfo.st_size;
*data = d;
rc = AddCache( path, data, finfo.st_size, finfo.st_mtime );
}
apr_file_close( file );
}
}
return rc;
}
IPath::SearchResult IPathFinder::GetPath(
const MoveDef& moveDef,
const CPathFinderDef& pfDef,
const CSolidObject* owner,
float3 startPos,
IPath::Path& path,
const unsigned int maxNodes
) {
startPos.ClampInBounds();
// Clear the path
path.path.clear();
path.squares.clear();
path.pathCost = PATHCOST_INFINITY;
// initial calculations
if (isEstimator) {
maxBlocksToBeSearched = std::min(MAX_SEARCHED_NODES_PE - 8U, maxNodes);
} else {
maxBlocksToBeSearched = std::min(MAX_SEARCHED_NODES_PF - 8U, maxNodes);
}
mStartBlock.x = startPos.x / BLOCK_PIXEL_SIZE;
mStartBlock.y = startPos.z / BLOCK_PIXEL_SIZE;
mStartBlockIdx = BlockPosToIdx(mStartBlock);
assert((unsigned)mStartBlock.x < nbrOfBlocks.x && (unsigned)mStartBlock.y < nbrOfBlocks.y);
// Check cache (when there is one)
int2 goalBlock;
goalBlock.x = pfDef.goalSquareX / BLOCK_SIZE;
goalBlock.y = pfDef.goalSquareZ / BLOCK_SIZE;
const CPathCache::CacheItem* ci = GetCache(mStartBlock, goalBlock, pfDef.sqGoalRadius, moveDef.pathType, pfDef.synced);
if (ci != nullptr) {
path = ci->path;
return ci->result;
}
// Start up a new search
IPath::SearchResult result = InitSearch(moveDef, pfDef, owner);
// If search was successful, generate new path
if (result == IPath::Ok || result == IPath::GoalOutOfRange) {
FinishSearch(moveDef, pfDef, path);
// Save to cache
AddCache(&path, result, mStartBlock, goalBlock, pfDef.sqGoalRadius, moveDef.pathType, pfDef.synced);
if (LOG_IS_ENABLED(L_DEBUG)) {
LOG_L(L_DEBUG, "==== %s: Search completed ====", (isEstimator) ? "PE" : "PF");
LOG_L(L_DEBUG, "Tested blocks: %u", testedBlocks);
LOG_L(L_DEBUG, "Open blocks: %u", openBlockBuffer.GetSize());
LOG_L(L_DEBUG, "Path length: " _STPF_, path.path.size());
LOG_L(L_DEBUG, "Path cost: %f", path.pathCost);
LOG_L(L_DEBUG, "==============================");
}
} else {
if (LOG_IS_ENABLED(L_DEBUG)) {
LOG_L(L_DEBUG, "==== %s: Search failed! ====", (isEstimator) ? "PE" : "PF");
LOG_L(L_DEBUG, "Tested blocks: %u", testedBlocks);
LOG_L(L_DEBUG, "Open blocks: %u", openBlockBuffer.GetSize());
LOG_L(L_DEBUG, "============================");
}
}
return result;
}
ground_truth CbirGVT::GetGvtImagesManyNew(const vector<size_t>& idxv,
const vector<size_t>& nnv,
size_t nobj,
const ground_truth& allowed) {
string msg = "CbirGVT::GetGvtImagesMany() : ";
if (!idxv.size())
return ground_truth();
if (idxv.size()!=nnv.size()) {
ShowError(msg+"vector dimensions differ");
return ground_truth();
}
GetDataBase()->WriteLog(msg+"starting with "+ToStr(idxv.size())+" samples");
vector<size_t> idxvres;
vector<int> nnvres;
vector<string> image_url_vec;
map<size_t,size_t> idx2nn;
multimap<size_t,size_t> retmap;
size_t nallow = 0;
for (size_t a=0; a<idxv.size(); a++) {
bool hit = false;
vector<size_t> cres = FindCache(idxv[a], nnv[a], hit);
if (hit) {
cout << msg << "a=" << a << " idx=" << idxv[a] << " nn=" << nnv[a]
<< " cache hit, returning " << ToStr(cres.size()) << " images"
<< endl;
for (size_t i=0; i<cres.size(); i++) {
size_t idx = cres[i];
bool allow = allowed[idx];
nallow += allow;
if (debug>2)
cout << " i=" << i << " index=" << idx << " allow=" << allow << endl;
if (allow)
retmap.insert(make_pair(i, idx));
}
continue;
}
idxvres.push_back(idxv[a]);
nnvres.push_back(nnv[a]);
idx2nn[idxv[a]] = nnv[a];
map<string,string> oi = GetDataBase()->ReadOriginsInfo(idxv[a],
false, false);
string image_url = oi["url"];
if (image_url=="")
ShowError(msg+"image URL not resolved for image #"+ToStr(idxv[a]));
image_url_vec.push_back(image_url);
}
GetDataBase()->WriteLog(msg+"found "+ToStr(nallow)+" images in the cache,"
" continuing with "+ToStr(nnvres.size())+" samples");
if (nnvres.size()) {
string access_token;
ImageCollectionServiceSoapBindingProxy *imageCollection =
(ImageCollectionServiceSoapBindingProxy*)GetGvtImagesCommon(access_token);
if (!imageCollection)
return ground_truth();
string collection_name = "b12n";
gvt__imageSimilarityUrls imageSimilarity_payloads;
imageSimilarity_payloads.CollectionId = &collection_name;
imageSimilarity_payloads.maxResults = nnvres;
imageSimilarity_payloads.imageUrl = image_url_vec;
gvt__imageSimilarityUrlsResponse imageSimilarity_responses;
// imageSimilarityUrls(collectionId, string[] imageUrls, int[] numResults);
if (imageCollection->imageSimilarityUrls(&imageSimilarity_payloads,
&imageSimilarity_responses)
== SOAP_OK) {
GetDataBase()->WriteLog(msg+"ending");
gvt__imageSimilarityResult *imageSimilarity_result
= imageSimilarity_responses.return_;
if (debug>1)
cout << *(imageSimilarity_result->message) << endl;
vector<gvt__imageSimilarityId*> &similar_images
= imageSimilarity_result->results;
vector<gvt__imageSimilarityId*>::iterator it = similar_images.begin();
float prevsim = 0.0;
size_t idxx = 0, nnx = 0;
size_t j = 0;
for (;it < similar_images.end(); it++) {
gvt__imageSimilarityId *imageSimilarityId = *it;
bool set_idxx = false;
if (imageSimilarityId->similarity>prevsim)
set_idxx = true;
prevsim = imageSimilarityId->similarity;
if (debug>2)
cout << *(imageSimilarityId->imageId) << "\t"
<< imageSimilarityId->similarity;
bool ok = false;
string uin = *imageSimilarityId->imageId, u = uin;
size_t p = u.find("static.flickr.com");
if (p!=string::npos) {
p = u.find('/', p);
if (p!=string::npos) {
p = u.find('/', p+1);
u.erase(0, p+1);
p = u.find('_');
if (p!=string::npos) {
u.erase(p);
u = string(10-u.size(), '0')+u;
int idxr = GetDataBase()->ToIndex(u);
if (idxr>=0) {
if (set_idxx) {
idxx = idxr;
nnx = idx2nn[idxx];
j = 0;
}
bool allow = allowed[idxr];
nallow += allow;
if (debug>2)
cout << " j=" << j << " index=" << idxr
<< " allow=" << allow << endl;
if (allow)
retmap.insert(make_pair(j, idxr));
ok = true;
AddCache(idxx, nnx, idxr);
j++;
}
}
}
}
if (debug>2)
cout << endl;
if (!ok && debug) {
if (false)
ShowError(msg+"failed with <"+uin+"> -> <"+u+">");
else
cerr << msg+"failed with <"+uin+"> -> <"+u+">" << endl;
}
}
}
else {
soap_print_fault(imageCollection, stderr);
}
delete imageCollection;
}
ground_truth ret(GetDataBase()->Size());
size_t nretpos = 0;
for (multimap<size_t,size_t>::const_iterator mi=retmap.begin();
mi!=retmap.end() && nretpos<nobj; mi++) {
if (debug>2)
cout << " checking " << mi->first << " " << mi->second << " "
<< (int)ret[mi->second] << " " << nretpos << endl;
nretpos += ret[mi->second]!=1;
ret[mi->second] = 1;
}
if (debug)
cout << msg << "returning " << nretpos << " images from total of "
<< nallow << " found" << endl;
return ret;
}
ground_truth CbirGVT::GetGvtImagesOneByOne(size_t idx, size_t nn) {
string msg = "CbirGVT::GetGvtImagesOneByOne("+ToStr(idx)+","+ToStr(nn)+") : ";
ground_truth ret(GetDataBase()->Size());
bool hit = false;
vector<size_t> cres = FindCache(idx, nn, hit);
if (hit) {
cout << msg << "cache hit, returning " << ToStr(cres.size()) << " images"
<< endl;
for (size_t i=0; i<cres.size(); i++)
ret[i] = 1;
return ret;
}
string access_token;
ImageCollectionServiceSoapBindingProxy *imageCollection =
(ImageCollectionServiceSoapBindingProxy*)GetGvtImagesCommon(access_token);
if (!imageCollection)
return ground_truth();
string collection_name = "b12n";
GetDataBase()->WriteLog(msg+"starting");
size_t nretpos = 0;
map<string,string> oi = GetDataBase()->ReadOriginsInfo(idx, false, false);
string image_url = oi["url"];
gvt__imageSimilarityUrl imageSimilarity_payload;
imageSimilarity_payload.CollectionId = &collection_name;
imageSimilarity_payload.maxResults = nn;
imageSimilarity_payload.imageUrl = &image_url;
gvt__imageSimilarityUrlResponse imageSimilarity_response;
if (imageCollection->imageSimilarityUrl(&imageSimilarity_payload,
&imageSimilarity_response)
== SOAP_OK) {
GetDataBase()->WriteLog(msg+"ending");
gvt__imageSimilarityResult *imageSimilarity_result
= imageSimilarity_response.return_;
if (debug>1)
cout << *(imageSimilarity_result->message) << endl;
vector<gvt__imageSimilarityId*> &similar_images
= imageSimilarity_result->results;
vector<gvt__imageSimilarityId*>::iterator it = similar_images.begin();
for (;it < similar_images.end(); it++) {
gvt__imageSimilarityId *imageSimilarityId = *it;
if (debug>2)
cout << *(imageSimilarityId->imageId) << "\t"
<< imageSimilarityId->similarity << endl;
bool ok = false;
string uin = *imageSimilarityId->imageId, u = uin;
size_t p = u.find("static.flickr.com");
if (p!=string::npos) {
p = u.find('/', p);
if (p!=string::npos) {
p = u.find('/', p+1);
u.erase(0, p+1);
p = u.find('_');
if (p!=string::npos) {
u.erase(p);
u = string(10-u.size(), '0')+u;
int idxr = GetDataBase()->ToIndex(u);
if (idxr>=0) {
nretpos++;
ret[idxr] = 1;
ok = true;
AddCache(idx, nn, idxr);
}
}
}
}
if (!ok && debug) {
if (false)
ShowError(msg+"failed with <"+uin+"> -> <"+u+">");
else
cerr << msg+"failed with <"+uin+"> -> <"+u+">" << endl;
}
}
}
else {
soap_print_fault(imageCollection, stderr);
}
delete imageCollection;
if (debug)
cout << msg << "returning " << nretpos << " images" << endl;
return ret;
}
void ThreadFunc(void* args){
//checking if we have right args
if(!args)
return;
//getting thread info
ThreadInfo* t = (ThreadInfo*)args;
uint CurImage = 0;
t->CurImage = 0;
//main loop
do{
//checking if thread is paused
WaitForSingleObject(t->Running, INFINITE);
//checking if thread is not terminated
if(!t->Terminate){
//checking if we need list and downloading it
if(!t->ResultList){
t->ResultList = flickcurl_photos_search_params(t->Flickcurl,
&t->SearchParams, &t->ListParams);
if(t->ResultList)
t->TotalImages = t->ResultList->total_count;
CurImage = 0;
t->ListParams.page++; //updating page
}
//checking if anything were found
if(t->ResultList){
t->CurImage++;
//creating new image
Image* img = new(Image);
InitImage(img);
//downloading photo meta if needed
if(t->WantMeta)
img->Meta = flickcurl_photos_getInfo(t->Flickcurl, t->ResultList->photos[CurImage]->id);
//downloading photo exif if needed
if(t->WantEXIF)
img->EXIF = flickcurl_photos_getExif(t->Flickcurl, t->ResultList->photos[CurImage]->id,
t->ResultList->photos[CurImage]->fields[PHOTO_FIELD_secret].string);
//downloading photo itself if needed
if(t->WantImage)
img->Img = DownloadImage(t, t->ResultList->photos[CurImage]);
//checking if we've got any data
if(img->EXIF || img->Img != "" || img->Meta)
AddCache(t, img); //adding to cache
else
FreeImage(img); //or killing intance
//updating image number
if(CurImage < t->ResultList->photos_count - 1)
CurImage++;
else{
CurImage = 0;
flickcurl_free_photos_list(t->ResultList);
t->ResultList = 0;
}
}else
if(!t->Terminate)
ResetEvent(t->Running); //pause thread if nothing were found
}
}while(!t->Terminate);
};