void TsEmInfo::ExecuteEM(int grayScale,int clusterNum)
{
float loglik = 0.0;
float beforeLogLik = 0.0;
int nStep = 0;
while (1)
{
// E Step
CalculateDistribution(grayScale, clusterNum, prb_em, mu_em, v_em, p_em, x_em);
CalculateScal(grayScale, scal_em, clusterNum, prb_em);
loglik = GetLogLikehood(grayScale, histogram_em, scal_em);
// Exit
if ((nStep> 0 && fabs(beforeLogLik - loglik) < 0.0001) || nStep > 40)
{
break;
}
// M Step
ExecuteMaxiumStep(grayScale,clusterNum);
beforeLogLik = loglik;
++nStep;
}
}
int main(void)
{
InitialFromBinary();
//LocationMatchTest2();
//CreateProteinInteractionMatrix();
//CreatePhenotypeMatchProteinRelationFile();
Begin();
CalculateDistribution();
CalculatePerformance();
CalculatePrecision();
return 0;
}
MgReader* MgFeatureNumericFunctions::Execute()
{
CHECKNULL((MgReader*)m_reader, L"MgFeatureNumericFunctions.Execute");
CHECKNULL(m_customFunction, L"MgFeatureNumericFunctions.Execute");
Ptr<MgReader> reader;
MG_LOG_TRACE_ENTRY(L"MgFeatureNumericFunctions::Execute");
// TODO: Can this be optimized to process them as they are read?
// TODO: Should we put a limit on double buffer
INT32 funcCode = -1;
bool supported = MgServerFeatureUtil::FindCustomFunction(m_customFunction, funcCode);
if (supported)
{
// In case we have int64 but is a custom function use double to evaluate it.
// Since we don't have a type which can make operations with big numbers we will fix only Unique/Min/Max functions
// Even if we treat int64 different from double we don't solve the issue with truncation error.
// If we emulate SUM adding two big numbers we will get overflow even we use int64, e.g.:
// Int64 val1 = 9223372036854775806;
// Int64 val2 = 9223372036854775807;
// Int64 sum = val1 + val2; // the sum value will be -3 so is overflow error
// In the future we will need to implement a type which can handle int64 numbers without getting overflow a sum/avg is calculated.
// Since all other functions calculate a sum we will use double, at least to be able to get the right result for small numbers.
if (!(m_type == MgPropertyType::Int64 && (funcCode == MINIMUM || funcCode == MAXIMUM || funcCode == UNIQUE)))
{
VECTOR values, distValues;
while(m_reader->ReadNext())
{
// TODO: Add support for Geometry extents
double val = GetValue();
values.push_back(val);
}
// Calulate the distribution on the collected values
CalculateDistribution(values, distValues);
// Create FeatureReader from distribution values
reader = GetReader(distValues);
}
else
{
VECTOR_INT64 values, distValues;
while(m_reader->ReadNext())
{
INT64 int64Val = 0;
if (!m_reader->IsNull(m_propertyName))
int64Val = m_reader->GetInt64(m_propertyName);
values.push_back(int64Val);
}
// Calulate the distribution on the collected values
CalculateDistribution(values, distValues);
// Create FeatureReader from distribution values
Ptr<MgInt64DataReaderCreator> drCreator = new MgInt64DataReaderCreator(m_propertyAlias);
reader = drCreator->Execute(distValues);
}
}
else
{
// just return an emty reader
VECTOR distValues;
reader = GetReader(distValues);
}
return reader.Detach();
}
bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileScanner wayScanner;
FileWriter writer;
std::vector<TypeData> wayTypeData;
size_t maxLevel;
progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag()));
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
if (!CalculateDistribution(typeConfig,
parameter,
progress,
wayTypeData,
maxLevel)) {
return false;
}
// Calculate number of types which have data
uint32_t indexEntries=0;
for (const auto& type : typeConfig->GetWayTypes())
{
if (wayTypeData[type->GetIndex()].HasEntries()) {
indexEntries++;
}
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
writer.Write(indexEntries);
for (const auto &type : typeConfig->GetWayTypes()) {
size_t i=type->GetIndex();
if (wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteTypeId(type->GetWayId(),
typeConfig->GetWayTypeIdBytes());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
TypeInfoSet indexTypes(*typeConfig);
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
wayScanner.GotoBegin();
for (const auto &type : typeConfig->GetWayTypes()) {
if (wayTypeData[type->GetIndex()].HasEntries() &&
wayTypeData[type->GetIndex()].indexLevel==l) {
indexTypes.Set(type);
}
}
if (indexTypes.Empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount());
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (!indexTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset);
}
}
}
for (const auto &type : indexTypes) {
size_t index=type->GetIndex();
if (!WriteBitmap(progress,
writer,
*typeConfig->GetTypeInfo(index),
wayTypeData[index],
typeCellOffsets[index])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
