bool isDiscontinuous(const BoolArray &image, const ICoord pt) { //This function checks to see if a point is discontinuous. It does //so by checking the pixels surrounding the current pixel. if(image[pt]==0 || pt(0)==0 || pt(0)==image.width()-1 || pt(1)==0 || pt(1)==image.height()-1) return false; int p1=image[ICoord(pt(0)-1,pt(1)+1)]; int p2=image[ICoord(pt(0),pt(1)+1)]; int p3=image[ICoord(pt(0)+1,pt(1)+1)]; int p4=image[ICoord(pt(0)-1,pt(1))]; int p6=image[ICoord(pt(0)+1,pt(1))]; int p7=image[ICoord(pt(0)-1,pt(1)-1)]; int p8=image[ICoord(pt(0),pt(1)-1)]; int p9=image[ICoord(pt(0)+1,pt(1)-1)]; int numNonZero=0; for(int a=-1; a<=1; a++) for(int b=-1; b<=1; b++) numNonZero+=image[pt+ICoord(a,b)]; if(numNonZero==1 || numNonZero==2) return true; int compVal=p1 + 2*p2 + 4*p3 + 8*p4 + 16*p6 + 32*p7 + 64*p8 + 128*p9; if(compVal==3 || compVal==6 || compVal==96 || compVal==192 || compVal==20 || compVal==9 || compVal==40 || compVal==144) return true; return false; // //3 pixel cases *** CHECK may not be necessary //top left /* if ((p1==1) && (p3==0) && (p4==0) && (p6==0) && (p7==0) && (p8==0) && (p9==0) && (p2==1)) return true; //top right if ((p1==0) && (p3==1) && (p4==0) && (p6==0) && (p7==0) && (p8==0) && (p9==0) && (p2==1)) return true; //bottom left if ((p1==0) && (p3==0) && (p4==0) && (p6==0) && (p7==1) && (p8==1) && (p9==0) && (p2==0)) return true; //bottom right if ((p1==0) && (p3==0) && (p4==0) && (p6==0) && (p7==0) && (p8==1) && (p9==1) && (p2==0)) return true; //right top if ((p1==0) && (p3==1) && (p4==0) && (p6==1) && (p7==0) && (p8==0) && (p9==0) && (p2==0)) return true; //left top if ((p1==1) && (p3==0) && (p4==1) && (p6==0) && (p7==0) && (p8==0) && (p9==0) && (p2==0)) return true; //left bottom if ((p1==0) && (p3==0) && (p4==1) && (p6==0) && (p7==1) && (p8==0) && (p9==0) && (p2==0)) return true; //left bottom if ((p1==0) && (p3==0) && (p4==0) && (p6==1) && (p7==0) && (p8==0) && (p9==1) && (p2==0)) return true;*/ }
inline void MarkMultiples(BoolArray<>& sieve, const size_t n, const size_t i) { size_t cur = i + ((i + 1) * n); // Start at the square of the number. while (cur < sieve.Size()) { // Since we're multiplying two odd numbers, the result is always odd, and thus in our sieve. sieve.ClearBit(cur); cur += n; } }
BoolArray closeLines(const BoolArray &image) { // Fill in single pixels gaps in lines. BoolArray temp=image; for (BoolArray::const_iterator i=image.begin(); i!=image.end(); ++i) { if (isBroken(image,i.coord())) { temp[i.coord()]=1; } } return temp; }
void MappingLevel::calculateMappings(IDefRecordElement const * diskRecord, unsigned numKeyedDisk, IDefRecordElement const * activityRecord, unsigned numKeyedActivity) { if(diskRecord->getKind() != DEKrecord) throw makeFailure(IRecordLayoutTranslator::Failure::BadStructure)->append("Disk record metadata had unexpected structure (expected record)")->appendScopeDesc(topLevel ? NULL : scope.str()); if(activityRecord->getKind() != DEKrecord) throw makeFailure(IRecordLayoutTranslator::Failure::BadStructure)->append("Activity record metadata had unexpected structure (expected record)")->appendScopeDesc(topLevel ? NULL : scope.str()); unsigned numActivityChildren = activityRecord->numChildren(); unsigned numDiskChildren = diskRecord->numChildren(); bool activityHasInternalFpos = false; if(topLevel && (numActivityChildren > numKeyedActivity)) { IDefRecordElement const * lastChild = activityRecord->queryChild(numActivityChildren-1); if((lastChild->queryName() == internalFposAtom) && (lastChild->queryType()->isInteger())) activityHasInternalFpos = true; } if((numActivityChildren - (activityHasInternalFpos ? 1 : 0)) > numDiskChildren) //if last activity field might be unmatched __internal_fpos__, should be more lenient by 1 as would fill that in (see below) throw makeFailure(IRecordLayoutTranslator::Failure::MissingDiskField)->append("Activity record requires more fields than index provides")->appendScopeDesc(topLevel ? NULL : scope.str()); if(numKeyedActivity > numKeyedDisk) throw makeFailure(IRecordLayoutTranslator::Failure::UnkeyedDiskField)->append("Activity record requires more keyed fields than index provides")->appendScopeDesc(topLevel ? NULL : scope.str()); BoolArray activityFieldMapped; activityFieldMapped.ensure(numActivityChildren); for(unsigned i=0; i<numActivityChildren; ++i) activityFieldMapped.append(false); FieldSearcher searcher(activityRecord); for(unsigned diskFieldNum = 0; diskFieldNum < numDiskChildren; ++diskFieldNum) { checkField(diskRecord, diskFieldNum, "Disk"); bool diskFieldKeyed = (diskFieldNum < numKeyedDisk); unsigned activityFieldNum; if(searcher.search(diskRecord->queryChild(diskFieldNum)->queryName(), activityFieldNum)) { bool activityFieldKeyed = (activityFieldNum < numKeyedActivity); if(activityFieldKeyed && !diskFieldKeyed) throw makeFailure(IRecordLayoutTranslator::Failure::UnkeyedDiskField)->append("Field ")->appendFieldName(topLevel ? NULL : scope.str(), activityRecord->queryChild(activityFieldNum))->append(" is keyed in activity but not on disk"); checkField(activityRecord, activityFieldNum, "Activity"); attemptMapping(diskRecord, diskFieldNum, diskFieldKeyed, activityRecord, activityFieldNum, activityFieldKeyed); activityFieldMapped.replace(true, activityFieldNum); } else { mappings.append(*new FieldMapping(FieldMapping::None, diskRecord, diskFieldNum, diskFieldKeyed, NULL, 0, false)); } } for(unsigned activityFieldNum=0; activityFieldNum<numActivityChildren; ++activityFieldNum) if(!activityFieldMapped.item(activityFieldNum)) { checkField(activityRecord, activityFieldNum, "Activity"); if((activityFieldNum != numActivityChildren-1) || !activityHasInternalFpos) //if last activity field is unmatched __internal_fpos__, this is not an error, we need do nothing and it will get correctly set to zero throw makeFailure(IRecordLayoutTranslator::Failure::MissingDiskField)->append("Field ")->appendFieldName(topLevel ? NULL : scope.str(), activityRecord->queryChild(activityFieldNum))->append(" is required by activity but not present on disk index"); } }
bool testAndNotBoolArray() { cout << "[testing AndNotBoolArray] sizeof(uword)=" << sizeof(uword) << endl; BoolArray<uword> b1 = BoolArray<uword>::bitmapOf(1,1); BoolArray<uword> b = BoolArray<uword>::bitmapOf(2,1,100); BoolArray<uword> bout; b.logicalandnot(b1,bout); if (bout.numberOfOnes() != 1) { return false; } return true; }
void removeNoisePoints(double tol, BoolArray & image, DoubleArray minMaxRatio) { int w=image.width(); int h=image.height(); for (int a=0; a<w; a++) { for (int b=0; b<h; b++) { if ( (image[ICoord(a,b)]==1) && (minMaxRatio[ICoord(a,b)]>tol)) { // std::cout<<"Noise removed at "<<a<<" "<<h-1-b<<std::endl; image[ICoord(a,b)]=0; } } } }
std::vector<size_t> PrimeFinder::FindPrimes(const size_t until) { // Find prime number using the sieve of Eratosthenes BoolArray<> sieve = GetSieve(until); size_t expected = GetExpectedPrimeCount(until); // Check the rest of the range (from sqrt(until) to until) for primes: std::vector<size_t> primes; primes.reserve(expected); primes.push_back(2); // Add 2, the only even prime number, and therefore the only one not in our sieve. for (size_t i = 0; i < sieve.Size(); i++) if (sieve[i]) primes.push_back(GetNumberAtIndex(i)); return primes; }
BoolArray removeDeadLines(const BoolArray & connected) { BoolArray temp=connected.clone(); int numRemoved=-1; while (numRemoved!=0) { numRemoved=0; for (int a = 1; a < connected.width()-1; a++) { for (int b = 1; b < connected.height()-1; b++) { if ( (temp[ICoord(a,b)]==1) && isDiscontinuous(temp,ICoord(a,b))) { temp[ICoord(a,b)]=0; numRemoved++; } } } } return temp; }
bool testCardinalityBoolArray() { cout << "[testing CardinalityBoolArray] sizeof(uword)=" << sizeof(uword) << endl; BoolArray<uword> b1 = BoolArray<uword>::bitmapOf(1,1); if (b1.numberOfOnes() != 1) { return false; } b1.inplace_logicalnot(); if (b1.numberOfOnes() != 1) { return false; } BoolArray<uword> b = BoolArray<uword>::bitmapOf(2,1,100); if(b.numberOfOnes() != 2) { return false; } BoolArray<uword> bout; b.logicalnot(bout); if(bout.numberOfOnes() != 99) { return false; } b.inplace_logicalnot(); if(b.numberOfOnes() != 99) { return false; } return true; }
bool isBroken(const BoolArray &image,ICoord pt) { // Is there a single pixel gap in a line at the given point? if (image[pt] || pt(0)==0 || pt(0)==image.width()-1 || pt(1)==0 || pt(1)==image.height()-1) return false; int p1=image[ICoord(pt(0)-1,pt(1)+1)]; int p2=image[ICoord(pt(0),pt(1)+1)]; int p3=image[ICoord(pt(0)+1,pt(1)+1)]; int p4=image[ICoord(pt(0)-1,pt(1))]; int p6=image[ICoord(pt(0)+1,pt(1))]; int p7=image[ICoord(pt(0)-1,pt(1)-1)]; int p8=image[ICoord(pt(0),pt(1)-1)]; int p9=image[ICoord(pt(0)+1,pt(1)-1)]; int compNum=p1 + 2*p2 + 4*p3 + 8*p4 + 16*p6 + 32*p7 + 64*p8 + 128*p9; /* if ( ((p1==0) && (p2==0) && (p3==1) && (p4==0) && (p6==0) && (p7==1) && (p8==0) && (p9==0)) || ((p1==1) && (p2==0) && (p3==0) && (p4==0) && (p6==0) && (p7==0) && (p8==0) && (p9==1)) || ((p1==0) && (p2==0) && (p3==0) && (p4==1) && (p6==1) && (p7==0) && (p8==0) && (p9==0)) || ((p1==0) && (p2==1) && (p3==0) && (p4==0) && (p6==0) && (p7==0) && (p8==1) && (p9==0)) || ((p1==0) && (p2==0) && (p3==1) && (p4==0) && (p6==0) && (p7==0) && (p8==1) && (p9==0)) || ((p1==1) && (p2==0) && (p3==0) && (p4==0) && (p6==0) && (p7==0) && (p8==1) && (p9==0)) || ((p1==0) && (p2==0) && (p3==0) && (p4==1) && (p6==0) && (p7==0) && (p8==0) && (p9==1)) || ((p1==0) && (p2==0) && (p3==1) && (p4==1) && (p6==0) && (p7==0) && (p8==0) && (p9==0)) || ((p1==1) && (p2==0) && (p3==0) && (p4==0) && (p6==1) && (p7==0) && (p8==0) && (p9==0)) || ((p1==0) && (p2==1) && (p3==0) && (p4==0) && (p6==0) && (p7==1) && (p8==0) && (p9==0)) || ((p1==0) && (p2==1) && (p3==0) && (p4==0) && (p6==0) && (p7==0) && (p8==0) && (p9==1)) || ((p1==0) && (p2==0) && (p3==0) && (p4==0) && (p6==1) && (p7==1) && (p8==0) && (p9==0)) )*/ if(compNum==36 || compNum==129 || compNum==24 || compNum==66 || compNum==68 || compNum==65 || compNum==136 || compNum==12 || compNum==17 || compNum==34 || compNum==130 || compNum==48) return true; return false; }
BoolArray getAndReduceDiscontinuities (const BoolArray& connected1, const BoolArray newThresh, int d, int halfsize) { BoolArray connected=connected1.clone(); int w = connected.width(); int h = connected.height(); int size=halfsize*2+1; //make size odd BoolArray box(ICoord(size,size),false);// make small box CloserClass cl = CloserClass(); // int halfsize = size/2; Skeletonizing skel = Skeletonizing(); for(BoolArray::iterator pt=connected.begin(); pt!=connected.end(); ++pt) { int a=pt.coord()(0), b=pt.coord()(1); if (connected[pt]==1 && isDiscontinuous(connected,pt.coord())) { for (int c=-halfsize; c<=halfsize; c++) { for (int d=-halfsize; d<=halfsize; d++) { // TODO: Use if.. else to avoid extra computation if(a+c>=0 && a+c<w && b+d>=0 && b+d<h) { if ( (abs(c)<size/4) && (abs(d)<size/4) && box[ICoord(c+halfsize,d+halfsize)]==0) box[ICoord(c+halfsize,d+halfsize)]=newThresh[ICoord(a+c,b+d)]; else box[ICoord(c+halfsize,d+halfsize)]=connected[ICoord(a+c,b+d)]; } } } box=skel.skeletonize(cl.close(box,size/4)); for (int g=-halfsize; g<=halfsize; g++) { for (int h2=-halfsize; h2<=halfsize; h2++) { if(a+g>=0 && a+g<w && b+h2>=0 && b+h2<h) { if (connected[ICoord(g+a,h2+b)]==0) { connected[ICoord(g+a,h2+b)]=box[ICoord(g+halfsize,h2+halfsize)]; } } } } } } return skel.skeletonize(cl.close(connected,d)); }
void prepareKey(IDistributedFile *index) { IDistributedFile *f = index; IDistributedSuperFile *super = f->querySuperFile(); unsigned nparts = f->numParts(); // includes tlks if any, but unused in array performPartLookup.ensure(nparts); bool checkTLKConsistency = (nullptr != super) && !localKey && (0 != (TIRsorted & indexBaseHelper->getFlags())); if (nofilter) { while (nparts--) performPartLookup.append(true); if (!checkTLKConsistency) return; } else { while (nparts--) performPartLookup.append(false); // parts to perform lookup set later } Owned<IDistributedFileIterator> iter; if (super) { iter.setown(super->getSubFileIterator(true)); verifyex(iter->first()); f = &iter->query(); } unsigned width = f->numParts(); if (!localKey) --width; assertex(width); unsigned tlkCrc = 0; bool first = true; unsigned superSubIndex=0; bool fileCrc = false, rowCrc = false; for (;;) { Owned<IDistributedFilePart> part = f->getPart(width); if (checkTLKConsistency) { unsigned _tlkCrc; if (part->getCrc(_tlkCrc)) fileCrc = true; else if (part->queryAttributes().hasProp("@crc")) // NB: key "@crc" is not a crc on the file, but data within. { _tlkCrc = part->queryAttributes().getPropInt("@crc"); rowCrc = true; } else if (part->queryAttributes().hasProp("@tlkCrc")) // backward compat. { _tlkCrc = part->queryAttributes().getPropInt("@tlkCrc"); rowCrc = true; } else { if (rowCrc || fileCrc) { checkTLKConsistency = false; Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, some crc attributes are missing", super->queryLogicalName()); queryJobChannel().fireException(e); } } if (rowCrc && fileCrc) { checkTLKConsistency = false; Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, due to mixed crc types.", super->queryLogicalName()); queryJobChannel().fireException(e); } if (checkTLKConsistency) { if (first) { tlkCrc = _tlkCrc; first = false; } else if (tlkCrc != _tlkCrc) throw MakeActivityException(this, 0, "Sorted output on super files comprising of non coparitioned sub keys is not supported (TLK's do not match)"); } } if (!nofilter) { Owned<IKeyIndex> keyIndex; unsigned copy; for (copy=0; copy<part->numCopies(); copy++) { RemoteFilename rfn; OwnedIFile ifile = createIFile(part->getFilename(rfn,copy)); if (ifile->exists()) { StringBuffer remotePath; rfn.getRemotePath(remotePath); unsigned crc = 0; part->getCrc(crc); keyIndex.setown(createKeyIndex(remotePath.str(), crc, false, false)); break; } } if (!keyIndex) throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", index->queryLogicalName()); unsigned fixedSize = indexBaseHelper->queryDiskRecordSize()->querySerializedDiskMeta()->getFixedSize(); // used only if fixed Owned <IKeyManager> tlk = createLocalKeyManager(keyIndex, fixedSize, NULL); indexBaseHelper->createSegmentMonitors(tlk); tlk->finishSegmentMonitors(); tlk->reset(); while (tlk->lookup(false)) { if (tlk->queryFpos()) performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1)); } } if (!super||!iter->next()) break; superSubIndex++; f = &iter->query(); if (width != f->numParts()-1) throw MakeActivityException(this, 0, "Super key %s, with mixture of sub key width are not supported.", f->queryLogicalName()); } }