const RsArrayType* TypeSystem::getArrayType(const std::string& name, size_t size)
{
const RsType* bt = getTypeInfo(name);
assert( bt );
return getArrayType(bt, size);
}
bool FillExtrusionLayer::Impl::hasLayoutDifference(const Layer::Impl& other) const {
assert(other.getTypeInfo() == getTypeInfo());
const auto& impl = static_cast<const style::FillExtrusionLayer::Impl&>(other);
return filter != impl.filter ||
visibility != impl.visibility ||
paint.hasDataDrivenPropertyDifference(impl.paint);
}
bool JsObjectWrapper::getInfo(NPIdentifier name, TypeMemberInfo& info) {
if(!getTypeInfo())
return false;
if(NPN_IdentifierIsString(name)) {
NPUTF8* id = NPN_UTF8FromIdentifier(name);
MembersByName_t::const_iterator it = m_byName.find(id);
NPN_MemFree(id);
if(it == m_byName.end())
return false;
info = it->second;
return true;
}
else if(m_indexerLength >= 0) {
uint32_t index = NPN_IntFromIdentifier(name);
if(index < m_indexerLength) {
info.dispatchType = DT_PropertyGet | DT_PropertySet;
//info.memberId = index;
return true;
}
}
return false;
}
float Task::getExpectedDuration()
{
if (expectedDuration == -1.0f) {
expectedDuration = 0.0f;
const type_info *id = getTypeInfo();
pthread_mutex_lock((pthread_mutex_t*) mutex);
map<type_info const*, TaskStatistics*, TypeInfoSort>::iterator i = statistics.find(id);
if (i != statistics.end()) {
TaskStatistics *stats = i->second;
// to get "valid" statistics, we wait until we have enough samples,
// and we ignore the min and max values
const int MIN_SAMPLES = 64;
if (stats->n >= MIN_SAMPLES) {
if (!stats->corrected) {
float sum = stats->durationSum - stats->maxDuration - stats->minDuration;
float squareSum = stats->durationSum - stats->maxDuration * stats->maxDuration - stats->minDuration * stats->minDuration;
stats->durationSum = (sum * stats->n) / (stats->n - 2) ;
stats->durationSquareSum = (squareSum * stats->n) / (stats->n - 2);
stats->corrected = true;
}
float mean = stats->durationSum / stats->n;
float squareMean = stats->durationSquareSum / stats->n;
float variance = squareMean - mean * mean;
float standardDeviation = sqrt(variance);
expectedDuration = (mean + 2.0f * standardDeviation) * getComplexity();
}
}
pthread_mutex_unlock((pthread_mutex_t*) mutex);
}
return expectedDuration;
}
char *VmhConfig::getGuestPass(char *type)
{
GuestTypeInfo *typeInfo = getTypeInfo(type);
if ((typeInfo != NULL) && (strlen(typeInfo->guestPass) > 0))
return typeInfo->guestPass;
else
return guestPass;
}
char *VmhConfig::getHostTag(char *type)
{
GuestTypeInfo *typeInfo = getTypeInfo(type);
if (typeInfo != NULL)
return typeInfo->hostTag;
else
return NULL;
}
Node<MemberNode>::Link TypeParser::member(Visibility vis, bool isStatic) {
Trace mbTrace = current().trace;
auto parsedAsDecl = declaration();
auto mbNode = Node<MemberNode>::make(parsedAsDecl->getIdentifier(), parsedAsDecl->getTypeInfo().getEvalTypeList(), isStatic, vis == INVALID ? PRIVATE : vis);
mbNode->setTrace(mbTrace);
if (parsedAsDecl->getChildren().size() > 0) mbNode->setInit(Node<ExpressionNode>::staticPtrCast(parsedAsDecl->removeChild(0)));
return mbNode;
}
char *VmhConfig::getInstaller(char *type)
{
GuestTypeInfo *typeInfo = getTypeInfo(type);
if (typeInfo != NULL)
return typeInfo->installPackage;
else
return NULL;
}
char *VmhConfig::getUninstallArgs(char *type)
{
GuestTypeInfo *typeInfo = getTypeInfo(type);
if (typeInfo != NULL)
return typeInfo->uninstallArgs;
else
return NULL;
}
Component* ComponentFactory::createComponent(const std::string& typeName) {
auto typeInfo = getTypeInfo(typeName);
if (typeInfo->isAbstract()) {
throw std::runtime_error("Cannot create instance of abstract component type '" + typeName + "'");
}
return typeInfo->createComponent();
}
SimulatorResourceModel::TypeInfo SimulatorResourceModel::getType(const std::string &name) const
{
if (m_attributes.end() != m_attributes.find(name))
{
return getTypeInfo(m_attributes.find(name)->second);
}
return SimulatorResourceModel::TypeInfo();
}
bool SimulatorResourceModel::updateValue(const std::string &name,
const AttributeValueVariant &value)
{
if (name.empty())
{
return false;
}
if (m_attributes.end() == m_attributes.find(name))
{
return false;
}
if (!(getTypeInfo(m_attributes[name]) == getTypeInfo(value)))
{
return false;
}
m_attributes[name] = value;
return true;
}
void Spider::doFirstConnection()
{
_socket->async_read(_buffer.data(), 4, [this]() {
uint16_t sizeName;
_proto = _buffer.getValue<uint16_t>();
if (_proto != PROTOCOL_VERSION) {
std::cerr << "Wrong protocole" << std::endl;
dieInDignity();
return ;
}
if ((sizeName = _buffer.getValue<uint16_t>()) >= SIZE_STRING) {
std::cerr << "Wrong string size allowed" << std::endl;
dieInDignity();
return ;
}
_buffer.reset();
_socket->async_read(_buffer.data(), sizeName, [this, sizeName]() {
std::fill(_str, _str + SIZE_STRING, 0);
_buffer.getValue<char>(_str, sizeName);
setName(sizeName);
_buffer.reset();
_socket->async_read(_buffer.data(), 4, [this]() {
try {
if (_dumpFile)
_dumpFile->createFile(_name);
}
catch (const std::exception &e) {
std::cerr << e.what() << std::endl;
}
uint16_t *reponse = new uint16_t;
*reponse = 2 << 8 | 2;
_socket->async_write(reponse, 2, [this, reponse]() mutable {
delete reponse;
boost::asio::spawn(_web.get_ioservice(), [this](boost::asio::yield_context yield) {
getTypeInfo(yield);
});
});
});
});
});
}
bool JsObjectWrapper::enumeration(NPIdentifier** values, uint32_t* count) {
Debug::println("JsObjectWrapper::enumeration");
if(!getTypeInfo())
return false;
NPIdentifier* names = (NPIdentifier*)NPN_MemAlloc(m_byName.size() * sizeof(NPIdentifier));
MembersByName_t::const_iterator it = m_byName.begin();
MembersByName_t::const_iterator itEnd = m_byName.end();
size_t i = 0;
for(; it != itEnd; ++it) {
const TypeMemberInfo& info = it->second;
names[i++] = NPN_GetStringIdentifier(info.name.c_str());
}
*values = names;
*count = m_byName.size();
return true;
}
void Task::setActualDuration(float duration)
{
TaskStatistics *stats;
const type_info *id = getTypeInfo();
pthread_mutex_lock((pthread_mutex_t*) mutex);
map<type_info const*, TaskStatistics*, TypeInfoSort>::iterator i = statistics.find(id);
if (i == statistics.end()) {
stats = new TaskStatistics();
statistics.insert(make_pair(id, stats));
} else {
stats = i->second;
}
duration = duration / getComplexity();
stats->durationSum += duration;
stats->durationSquareSum += duration * duration;
stats->minDuration = min(duration, stats->minDuration);
stats->maxDuration = max(duration, stats->maxDuration);
stats->n += 1;
pthread_mutex_unlock((pthread_mutex_t*) mutex);
}
std::unique_ptr<RenderLayer> FillExtrusionLayerFactory::createRenderLayer(Immutable<style::Layer::Impl> impl) noexcept {
assert(impl->getTypeInfo() == getTypeInfo());
return std::make_unique<RenderFillExtrusionLayer>(staticImmutableCast<style::FillExtrusionLayer::Impl>(std::move(impl)));
}
const RsPointerType* TypeSystem::getPointerType(const std::string & name)
{
return getPointerType(getTypeInfo(name));
}
/*******************************************************************
* Function: ArrayIndexAnalysis::flowFunction
* Purpose : compute out set from in set based on the type of statement
* Initial : Nurudeen A. Lameed on July 21, 2009
********************************************************************
Revisions and bug fixes:
*/
FlowSet ArrayIndexAnalysis::flowFunction(const Statement* stmt, const FlowSet& in)
{
// declare out flow set
FlowSet out;
// copy in flow set into out flow set
out.insert(in.begin(), in.end());
switch(stmt->getStmtType())
{
case Statement::ASSIGN:
{
const AssignStmt* aStmt = dynamic_cast<const AssignStmt*>(stmt);
// get the left handsides
Expression::ExprVector lhs = aStmt->getLeftExprs();
SymbolExpr* symbol = 0;
if (lhs.size() == 1 && lhs[0]->getExprType() == Expression::SYMBOL)
{
symbol = dynamic_cast<SymbolExpr*>(lhs[0]);
}
else
{
// get all left symbols/expressions
for (size_t i=0, size = lhs.size(); i < size; ++i)
{
// update out set, if it is a parameterized expression
computeOutSet(lhs[i], out);
}
}
// check the right hand side
Expression* rhs = aStmt->getRightExpr();
// is symbol an array/matrix with known size?
if ( allMatrixSymbols.find(symbol) != allMatrixSymbols.end() ) // symbol is used as an array ...
{
// get the type information
TypeInfo typ = getTypeInfo(aStmt, symbol);
if ( typ.getSizeKnown())
{
const TypeInfo::DimVector& dimV = typ.getMatSize();
TypeInfo::DimVector bounds;
// remove dimensions with value 1; (e.g. row or column vectors)
std::remove_copy_if(dimV.begin(), dimV.end(), std::back_inserter(bounds), equalOne);
// update the table
arrayBoundsMap[symbol] = bounds;
// update the flow set using the bounds
updateFlowSet(symbol, bounds, out);
}
}
// update outset (for parameterized expression)
computeOutSet(rhs, out);
// update outset (for index update
if (isIndex(symbol))
{
// compute flow set for index update
computeOutSet(symbol, rhs, out);
}
}
break;
case Statement::EXPR:
{
// convert the statement to an expression
const ExprStmt* aStmt = dynamic_cast<const ExprStmt*>(stmt);
// compute out set for the statement
computeOutSet(aStmt->getExpression(), out);
}
break;
default: // do nothing;
{
}
}
return out;
}
SPtr<ManagedSerializableTypeInfo> ScriptAssemblyManager::getTypeInfo(MonoClass* monoClass)
{
if(!mBaseTypesInitialized)
BS_EXCEPT(InvalidStateException, "Calling getTypeInfo without previously initializing base types.");
MonoPrimitiveType monoPrimitiveType = MonoUtil::getPrimitiveType(monoClass->_getInternalClass());
// If enum get the enum base data type
bool isEnum = MonoUtil::isEnum(monoClass->_getInternalClass());
if (isEnum)
monoPrimitiveType = MonoUtil::getEnumPrimitiveType(monoClass->_getInternalClass());
// Determine field type
switch(monoPrimitiveType)
{
case MonoPrimitiveType::Boolean:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::Bool;
return typeInfo;
}
case MonoPrimitiveType::Char:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::Char;
return typeInfo;
}
case MonoPrimitiveType::I8:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::I8;
return typeInfo;
}
case MonoPrimitiveType::U8:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::U8;
return typeInfo;
}
case MonoPrimitiveType::I16:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::I16;
return typeInfo;
}
case MonoPrimitiveType::U16:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::U16;
return typeInfo;
}
case MonoPrimitiveType::I32:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::I32;
return typeInfo;
}
case MonoPrimitiveType::U32:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::U32;
return typeInfo;
}
case MonoPrimitiveType::I64:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::I64;
return typeInfo;
}
case MonoPrimitiveType::U64:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::U64;
return typeInfo;
}
case MonoPrimitiveType::String:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::String;
return typeInfo;
}
case MonoPrimitiveType::R32:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::Float;
return typeInfo;
}
case MonoPrimitiveType::R64:
{
SPtr<ManagedSerializableTypeInfoPrimitive> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoPrimitive>();
typeInfo->mType = ScriptPrimitiveType::Double;
return typeInfo;
}
case MonoPrimitiveType::Class:
if(monoClass->isSubClassOf(ScriptResource::getMetaData()->scriptClass)) // Resource
{
SPtr<ManagedSerializableTypeInfoRef> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoRef>();
typeInfo->mTypeNamespace = monoClass->getNamespace();
typeInfo->mTypeName = monoClass->getTypeName();
typeInfo->mRTIITypeId = 0;
if(monoClass == ScriptResource::getMetaData()->scriptClass)
typeInfo->mType = ScriptReferenceType::BuiltinResourceBase;
else if (monoClass == ScriptManagedResource::getMetaData()->scriptClass)
typeInfo->mType = ScriptReferenceType::ManagedResourceBase;
else if (monoClass->isSubClassOf(ScriptManagedResource::getMetaData()->scriptClass))
typeInfo->mType = ScriptReferenceType::ManagedResource;
else if (monoClass->isSubClassOf(ScriptResource::getMetaData()->scriptClass))
{
typeInfo->mType = ScriptReferenceType::BuiltinResource;
::MonoReflectionType* type = MonoUtil::getType(monoClass->_getInternalClass());
BuiltinResourceInfo* builtinInfo = getBuiltinResourceInfo(type);
if (builtinInfo == nullptr)
{
assert(false && "Unable to find information about a built-in resource. Did you update BuiltinResourceTypes list?");
return nullptr;
}
typeInfo->mRTIITypeId = builtinInfo->typeId;
}
return typeInfo;
}
else if (monoClass->isSubClassOf(mSceneObjectClass) || monoClass->isSubClassOf(mComponentClass)) // Game object
{
SPtr<ManagedSerializableTypeInfoRef> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoRef>();
typeInfo->mTypeNamespace = monoClass->getNamespace();
typeInfo->mTypeName = monoClass->getTypeName();
typeInfo->mRTIITypeId = 0;
if (monoClass == mComponentClass)
typeInfo->mType = ScriptReferenceType::BuiltinComponentBase;
else if (monoClass == mManagedComponentClass)
typeInfo->mType = ScriptReferenceType::ManagedComponentBase;
else if (monoClass->isSubClassOf(mSceneObjectClass))
typeInfo->mType = ScriptReferenceType::SceneObject;
else if (monoClass->isSubClassOf(mManagedComponentClass))
typeInfo->mType = ScriptReferenceType::ManagedComponent;
else if (monoClass->isSubClassOf(mComponentClass))
{
typeInfo->mType = ScriptReferenceType::BuiltinComponent;
::MonoReflectionType* type = MonoUtil::getType(monoClass->_getInternalClass());
BuiltinComponentInfo* builtinInfo = getBuiltinComponentInfo(type);
if(builtinInfo == nullptr)
{
assert(false && "Unable to find information about a built-in component. Did you update BuiltinComponents list?");
return nullptr;
}
typeInfo->mRTIITypeId = builtinInfo->typeId;
}
return typeInfo;
}
else
{
SPtr<ManagedSerializableObjectInfo> objInfo;
if (getSerializableObjectInfo(monoClass->getNamespace(), monoClass->getTypeName(), objInfo))
return objInfo->mTypeInfo;
}
break;
case MonoPrimitiveType::ValueType:
{
SPtr<ManagedSerializableObjectInfo> objInfo;
if (getSerializableObjectInfo(monoClass->getNamespace(), monoClass->getTypeName(), objInfo))
return objInfo->mTypeInfo;
}
break;
case MonoPrimitiveType::Generic:
if(monoClass->getFullName() == mSystemGenericListClass->getFullName()) // Full name is part of CIL spec, so it is just fine to compare like this
{
SPtr<ManagedSerializableTypeInfoList> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoList>();
MonoProperty* itemProperty = monoClass->getProperty("Item");
MonoClass* itemClass = itemProperty->getReturnType();
if (itemClass != nullptr)
typeInfo->mElementType = getTypeInfo(itemClass);
if (typeInfo->mElementType == nullptr)
return nullptr;
return typeInfo;
}
else if(monoClass->getFullName() == mSystemGenericDictionaryClass->getFullName())
{
SPtr<ManagedSerializableTypeInfoDictionary> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoDictionary>();
MonoMethod* getEnumerator = monoClass->getMethod("GetEnumerator");
MonoClass* enumClass = getEnumerator->getReturnType();
MonoProperty* currentProp = enumClass->getProperty("Current");
MonoClass* keyValuePair = currentProp->getReturnType();
MonoProperty* keyProperty = keyValuePair->getProperty("Key");
MonoProperty* valueProperty = keyValuePair->getProperty("Value");
MonoClass* keyClass = keyProperty->getReturnType();
if(keyClass != nullptr)
typeInfo->mKeyType = getTypeInfo(keyClass);
MonoClass* valueClass = valueProperty->getReturnType();
if(valueClass != nullptr)
typeInfo->mValueType = getTypeInfo(valueClass);
if (typeInfo->mKeyType == nullptr || typeInfo->mValueType == nullptr)
return nullptr;
return typeInfo;
}
break;
case MonoPrimitiveType::Array:
{
SPtr<ManagedSerializableTypeInfoArray> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoArray>();
::MonoClass* elementClass = ScriptArray::getElementClass(monoClass->_getInternalClass());
if(elementClass != nullptr)
{
MonoClass* monoElementClass = MonoManager::instance().findClass(elementClass);
if(monoElementClass != nullptr)
typeInfo->mElementType = getTypeInfo(monoElementClass);
}
if (typeInfo->mElementType == nullptr)
return nullptr;
typeInfo->mRank = ScriptArray::getRank(monoClass->_getInternalClass());
return typeInfo;
}
default:
break;
}
return nullptr;
}
void ScriptAssemblyManager::loadAssemblyInfo(const String& assemblyName)
{
if(!mBaseTypesInitialized)
initializeBaseTypes();
initializeBuiltinComponentInfos();
initializeBuiltinResourceInfos();
// Process all classes and fields
UINT32 mUniqueTypeId = 1;
MonoAssembly* curAssembly = MonoManager::instance().getAssembly(assemblyName);
if(curAssembly == nullptr)
return;
SPtr<ManagedSerializableAssemblyInfo> assemblyInfo = bs_shared_ptr_new<ManagedSerializableAssemblyInfo>();
assemblyInfo->mName = assemblyName;
mAssemblyInfos[assemblyName] = assemblyInfo;
MonoClass* resourceClass = ScriptResource::getMetaData()->scriptClass;
MonoClass* managedResourceClass = ScriptManagedResource::getMetaData()->scriptClass;
// Populate class data
const Vector<MonoClass*>& allClasses = curAssembly->getAllClasses();
for(auto& curClass : allClasses)
{
if ((curClass->isSubClassOf(mComponentClass) || curClass->isSubClassOf(resourceClass) ||
curClass->hasAttribute(mSerializeObjectAttribute)) &&
curClass != mComponentClass && curClass != resourceClass &&
curClass != mManagedComponentClass && curClass != managedResourceClass)
{
SPtr<ManagedSerializableTypeInfoObject> typeInfo = bs_shared_ptr_new<ManagedSerializableTypeInfoObject>();
typeInfo->mTypeNamespace = curClass->getNamespace();
typeInfo->mTypeName = curClass->getTypeName();
typeInfo->mTypeId = mUniqueTypeId++;
MonoPrimitiveType monoPrimitiveType = MonoUtil::getPrimitiveType(curClass->_getInternalClass());
if(monoPrimitiveType == MonoPrimitiveType::ValueType)
typeInfo->mValueType = true;
else
typeInfo->mValueType = false;
SPtr<ManagedSerializableObjectInfo> objInfo = bs_shared_ptr_new<ManagedSerializableObjectInfo>();
objInfo->mTypeInfo = typeInfo;
objInfo->mMonoClass = curClass;
assemblyInfo->mTypeNameToId[objInfo->getFullTypeName()] = typeInfo->mTypeId;
assemblyInfo->mObjectInfos[typeInfo->mTypeId] = objInfo;
}
}
// Populate field & property data
for(auto& curClassInfo : assemblyInfo->mObjectInfos)
{
SPtr<ManagedSerializableObjectInfo> objInfo = curClassInfo.second;
UINT32 mUniqueFieldId = 1;
const Vector<MonoField*>& fields = objInfo->mMonoClass->getAllFields();
for(auto& field : fields)
{
if(field->isStatic())
continue;
SPtr<ManagedSerializableTypeInfo> typeInfo = getTypeInfo(field->getType());
if (typeInfo == nullptr)
continue;
SPtr<ManagedSerializableFieldInfo> fieldInfo = bs_shared_ptr_new<ManagedSerializableFieldInfo>();
fieldInfo->mFieldId = mUniqueFieldId++;
fieldInfo->mName = field->getName();
fieldInfo->mMonoField = field;
fieldInfo->mTypeInfo = typeInfo;
fieldInfo->mParentTypeId = objInfo->mTypeInfo->mTypeId;
MonoMemberVisibility visibility = field->getVisibility();
if (visibility == MonoMemberVisibility::Public)
{
if (!field->hasAttribute(mDontSerializeFieldAttribute))
fieldInfo->mFlags |= ScriptFieldFlag::Serializable;
if (!field->hasAttribute(mHideInInspectorAttribute))
fieldInfo->mFlags |= ScriptFieldFlag::Inspectable;
fieldInfo->mFlags |= ScriptFieldFlag::Animable;
}
else
{
if (field->hasAttribute(mSerializeFieldAttribute))
fieldInfo->mFlags |= ScriptFieldFlag::Serializable;
if (field->hasAttribute(mShowInInspectorAttribute))
fieldInfo->mFlags |= ScriptFieldFlag::Inspectable;
}
if (field->hasAttribute(mRangeAttribute))
fieldInfo->mFlags |= ScriptFieldFlag::Range;
if (field->hasAttribute(mStepAttribute))
fieldInfo->mFlags |= ScriptFieldFlag::Step;
objInfo->mFieldNameToId[fieldInfo->mName] = fieldInfo->mFieldId;
objInfo->mFields[fieldInfo->mFieldId] = fieldInfo;
}
const Vector<MonoProperty*>& properties = objInfo->mMonoClass->getAllProperties();
for (auto& property : properties)
{
SPtr<ManagedSerializableTypeInfo> typeInfo = getTypeInfo(property->getReturnType());
if (typeInfo == nullptr)
continue;
SPtr<ManagedSerializablePropertyInfo> propertyInfo = bs_shared_ptr_new<ManagedSerializablePropertyInfo>();
propertyInfo->mFieldId = mUniqueFieldId++;
propertyInfo->mName = property->getName();
propertyInfo->mMonoProperty = property;
propertyInfo->mTypeInfo = typeInfo;
propertyInfo->mParentTypeId = objInfo->mTypeInfo->mTypeId;
if (!property->isIndexed())
{
MonoMemberVisibility visibility = property->getVisibility();
if (visibility == MonoMemberVisibility::Public)
propertyInfo->mFlags |= ScriptFieldFlag::Animable;
if (property->hasAttribute(mSerializeFieldAttribute))
propertyInfo->mFlags |= ScriptFieldFlag::Serializable;
if (property->hasAttribute(mShowInInspectorAttribute))
propertyInfo->mFlags |= ScriptFieldFlag::Inspectable;
}
if (property->hasAttribute(mRangeAttribute))
propertyInfo->mFlags |= ScriptFieldFlag::Range;
if (property->hasAttribute(mStepAttribute))
propertyInfo->mFlags |= ScriptFieldFlag::Step;
objInfo->mFieldNameToId[propertyInfo->mName] = propertyInfo->mFieldId;
objInfo->mFields[propertyInfo->mFieldId] = propertyInfo;
}
}
// Form parent/child connections
for(auto& curClass : assemblyInfo->mObjectInfos)
{
MonoClass* base = curClass.second->mMonoClass->getBaseClass();
while(base != nullptr)
{
SPtr<ManagedSerializableObjectInfo> baseObjInfo;
if(getSerializableObjectInfo(base->getNamespace(), base->getTypeName(), baseObjInfo))
{
curClass.second->mBaseClass = baseObjInfo;
baseObjInfo->mDerivedClasses.push_back(curClass.second);
break;
}
base = base->getBaseClass();
}
}
}
int main (int argc, char ** argv) {
//For varriable definitions:
//gbounds = global bounds string, lbounds = local bounds string, offs = offset string, tstring = temp string to hold temperary stuff
char gbounds[1007], lbounds[1007], offs[1007],tstring[100];
//size = number of cores, gidx = adios group index
int rank, size, gidx, i, j, k, ii;
//data = pointer to read-in data
void * data = NULL;
uint64_t s[] = {0,0,0,0,0,0,0,0,0,0}; //starting offset
uint64_t c[] = {1,1,1,1,1,1,1,1,1,1}; //chunk block array
uint64_t bytes_read = 0;
int element_size;
int64_t new_adios_group, m_adios_file;
uint64_t var_size; //portion_bound,
uint64_t adios_groupsize, adios_totalsize;
int read_buffer; //possible maximum size you the user would like for each chunk in MB
int write_buffer = 1536; //actual buffer size you use in MB
int itime;
int WRITEME=1;
uint64_t chunk_size; //chunk size in # of elements
char *var_path, *var_name; // full path cut into dir path and name
MPI_Init(&argc,&argv);
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&size);
// timing numbers
// we will time:
// 0: adios_open, adios_group_size
// 1: the total time to read in the data
// 2: times around each write (will only work if we do NOT buffer....
// 3: the time in the close
// 4: fopen, fclose
// 5: total time
// timers: the total I/O time
int timers = 6;
double start_time[timers], end_time[timers], total_time[timers];
if (TIMING==100) {
for (itime=0;itime<timers;itime++) {
start_time[itime] = 0;
end_time[itime] = 0;
total_time[itime]=0;
}
//MPI_Barrier(MPI_COMM_WORLD);
start_time[5] = MPI_Wtime();
}
if(rank==0)
printf("converting...\n");
if (argc < 5) {
if (rank==0) printf("Usage: %s <BP-file> <ADIOS-file> read_buffer(MB) write_buffer(MB) METHOD (LUSTRE_strip_count) (LUSTRE_strip_size) (LUSTRE_block_size)\n", argv[0]);
return 1;
}
if(TIMING==100)
start_time[4] = MPI_Wtime();
ADIOS_FILE * f = adios_fopen (argv[1], MPI_COMM_SELF);
if(TIMING==100){
end_time[4] = MPI_Wtime();
total_time[4] = end_time[4]-start_time[4];
}
adios_init_noxml(comm); // no xml will be used to write the new adios file
read_buffer = atoi(argv[3]);
write_buffer = atoi(argv[4]);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, write_buffer); // allocate MB buffer
if (f == NULL) {
printf("rank=%d, file cant be opened\n", rank);
if (DEBUG) printf ("%s\n", adios_errmsg());
return -1;
}
for (gidx = 0; gidx < f->groups_count; gidx++) { //group part
adios_groupsize = 0;
ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]);
if (g == NULL) {
if (DEBUG) printf ("%s\n", adios_errmsg());
printf("rank %d: group cannot be opened.\n", rank);
return -1;
}
/* First create all of the groups */
// now I need to create this group in the file that will be written
adios_declare_group(&new_adios_group,f->group_namelist[gidx],"",adios_flag_yes);
if(strcmp(argv[5],"MPI_LUSTRE")!=0) //see whether or not the user uses MPI_LUSTRE method
adios_select_method (new_adios_group, argv[5], "", ""); //non-MPI_LUSTRE methods... like MPI, POSIX....
else{
char lustre_pars[1000];
strcpy(lustre_pars, "");
strcat(lustre_pars, "stripe_count=");
sprintf(tstring, "%d", atoi(argv[6]));
strcat(lustre_pars, tstring);
strcat(lustre_pars, ",stripe_size=");
sprintf(tstring, "%d", atoi(argv[7]));
strcat(lustre_pars, tstring);
strcat(lustre_pars, ",block_size=");
sprintf(tstring, "%d", atoi(argv[8]));
strcat(lustre_pars, tstring);
if(rank==0)
printf("lustre_pars=%s\n", lustre_pars);
adios_select_method (new_adios_group, argv[5], lustre_pars, ""); //Use MPI Lustre method
}
// variable definition part
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
getbasename (g->var_namelist[i], &var_path, &var_name);
if (v->ndim == 0)
{
// scalars: every process does them the same.
adios_define_var(new_adios_group,var_name,var_path,v->type,0,0,0);
getTypeInfo( v->type, &element_size); //element_size is size per element based on its type
if (v->type == adios_string) { //special case when the scalar is string.
adios_groupsize += strlen(v->value);
} else {
adios_groupsize += element_size;
}
}
else
{
// vector variables
getTypeInfo( v->type, &element_size);
var_size=1;
for (ii=0;ii<v->ndim;ii++) {
var_size*=v->dims[ii];
}
uint64_t total_size = var_size; //total_size tells you the total number of elements in the current vector variable
var_size*=element_size; //var_size tells you the size of the current vector variable in bytess
//re-initialize the s and c variables
for(j=0; j<v->ndim; j++){
s[j] = 0;
c[j] = 1;
}
//find the approximate chunk_size you would like to use.
chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
//set the chunk block array with the total size as close to chunk_size as possible
calcC(chunk_size, v, c);
strcpy(lbounds,"");
for(j=0; j<v->ndim; j++){
sprintf(tstring, "%" PRId64 ",", c[j]);
strcat(lbounds, tstring);
}
printf("rank=%d, name=%s, chunk_size1=%" PRId64 " c[]=%s\n",rank,g->var_namelist[i],chunk_size,lbounds);
chunk_size = 1;
for(ii=0; ii<v->ndim; ii++) //reset chunk_size based on the created c. Now the chunk_size is exact.
chunk_size *= c[ii];
//current step points to where the process is in processing the vector. First sets with respect to rank.
uint64_t current_step = rank*chunk_size;
//First advance the starting point s by current_step. Of course, you don't do it if the current_step exceeds total_size.
if(current_step<total_size)
rS(v, s, current_step, rank);
uint64_t elements_defined = 0; //First, the number of elements you have defined is 0.
//You (the process) process your part of the vector when your current_step is smaller than the total_size
while(current_step < total_size)
{
//ts, temporary s, is introduced for the sake of the inner do while loop below. Copy s to ts.
uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
arrCopy(s, ts);
//for every outer while iteration, you always have the whole chunk_size remained to process.
uint64_t remain_chunk = chunk_size;
if(current_step+chunk_size>total_size) //except when you are nearing the end of the vector....
remain_chunk = total_size-current_step;
//tc, temporary c, is introduced for the sake of the inner do while loop below. Copy s to tc.
uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
arrCopy(c, tc);
do{
//how much of the remain chunk you wanna process? initially you think you can do all of it....
uint64_t used_chunk = remain_chunk;
//you feel like you should process the vector with tc block size, but given ts, you might go over bound.
uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
//so you verify it by setting a new legit chunck block uc, and getting a new remain_chunk.
remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);
//you check whether or not ts+uc goes over the bound. This is just checking to make sure there's no error.
//Thereotically, there should be no problem at all.
checkOverflow(0, v, ts, uc);
//the below code fragment simply calculates gbounds, and sets place holders for lbounds and offs.
strcpy(gbounds,"");
strcpy(lbounds,"");
strcpy(offs,"");
for(j=0; j<v->ndim-1; j++){
sprintf(tstring, "%d,", (int)v->dims[j]);
strcat(gbounds, tstring);
//sprintf(tstring, "ldim%d_%s,", j, var_name);
sprintf(tstring, "ldim%d,", j);
strcat(lbounds, tstring);
//sprintf(tstring, "offs%d_%s,", j, var_name);
sprintf(tstring, "offs%d,", j);
strcat(offs, tstring);
}
sprintf(tstring, "%d", (int)v->dims[v->ndim-1]);
strcat(gbounds, tstring);
//sprintf(tstring, "ldim%d_%s", v->ndim-1, var_name);
sprintf(tstring, "ldim%d", v->ndim-1);
strcat(lbounds, tstring);
//sprintf(tstring, "offs%d_%s", v->ndim-1, var_name);
sprintf(tstring, "offs%d", v->ndim-1);
strcat(offs, tstring);
//sprintf(tstring, "%d", v->ndim);
for(j=0; j<v->ndim; j++){
//sprintf(tstring, "ldim%d_%s", j, var_name);
sprintf(tstring, "ldim%d", j);
adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
//sprintf(tstring, "offs%d_%s", j, var_name);
sprintf(tstring, "offs%d", j);
adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
}
adios_define_var(new_adios_group,var_name,var_path,v->type,lbounds,gbounds,offs);
if (DEBUG){
strcpy(lbounds,"");
strcpy(offs,"");
for(j=0; j<v->ndim; j++){
sprintf(tstring, "%" PRId64 ",", ts[j]);
strcat(offs, tstring);
sprintf(tstring, "%" PRId64 ",", uc[j]);
strcat(lbounds, tstring);
}
printf("rank=%d, name=%s, gbounds=%s: lbounds=%s: offs=%s \n",rank,g->var_namelist[i],gbounds, lbounds, offs);
}
used_chunk -= remain_chunk; //you get the actual used_chunk here.
elements_defined += used_chunk;
if(remain_chunk!=0){
rS(v, ts, used_chunk, rank); //advance ts by used_chunk.
for(k=0; k<10; k++)
tc[k] = 1;
calcC(remain_chunk, v, tc); //based on the remain_chunk, calculate new tc chunk block remained to process.
}
adios_groupsize+= used_chunk*element_size+2*v->ndim*8;
}while(remain_chunk!=0);
current_step += size*chunk_size; //once a whole chunk_size is processed, advance the current_step in roll-robin manner.
if(current_step<total_size){ //advance s in the same way.
rS(v, s, size*chunk_size, rank);
}
}
//beside checkOverflow above, here you check whether or not the total number of elements processed across processes matches
//the total number of elements in the original vector.
if(DEBUG){
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = elements_defined;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0 && rb[0]!=total_size)
printf("some array define mismatch. please use debug mode\n");
free(sb); free(rb);
}
}
free (var_name);
free (var_path);
} // finished declaring all of the variables
// Now we can define the attributes....
for (i = 0; i < g->attrs_count; i++) {
enum ADIOS_DATATYPES atype;
int asize;
void *adata;
adios_get_attr_byid (g, i, &atype, &asize, &adata);
// if (DEBUG) printf("attribute name=%s\n",g->attr_namelist[i]);
adios_define_attribute(new_adios_group,g->attr_namelist[i],"",atype,adata,0);
}
/*------------------------------ NOW WE WRITE -------------------------------------------- */
// Now we have everything declared... now we need to write them out!!!!!!
if (WRITEME==1) {
// open up the file for writing....
if (DEBUG) printf("rank=%d, opening file = %s, with group %s, size=%" PRId64 "\n",rank,argv[2],f->group_namelist[gidx],adios_groupsize);
if(TIMING==100)
start_time[0] = MPI_Wtime();
adios_open(&m_adios_file, f->group_namelist[gidx],argv[2],"w",comm);
adios_group_size( m_adios_file, adios_groupsize, &adios_totalsize);
//get both the total adios_totalsize and total adios_groupsize summed across processes.
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = adios_groupsize;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
uint64_t* sb2 = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb2 = (uint64_t *) malloc(sizeof(uint64_t));
sb2[0] = adios_totalsize;
MPI_Reduce(sb2,rb2,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0){
printf("total adios_totalsize = %" PRId64 "\n", *rb2);
printf("total adios_groupsize = %" PRId64 "\n", *rb);
}
free(sb); free(rb); free(sb2); free(rb2);
if (TIMING==100) {
end_time[0] = MPI_Wtime();
total_time[0]+=end_time[0] - start_time[0]; //variable definition time taken
}
// now we have to write out the variables.... since they are all declared now
// This will be the place we actually write out the data!!!!!!!!
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
getbasename (g->var_namelist[i], &var_path, &var_name);
if (v->ndim == 0)
{
if (DEBUG) {
printf ("ADIOS WRITE SCALAR: rank=%d, name=%s value=",
rank,g->var_namelist[i]);
print_data (v->value, 0, v->type);
printf ("\n");
}
if (TIMING==100) {
start_time[2] = MPI_Wtime();
}
adios_write(m_adios_file,g->var_namelist[i],v->value);
if (TIMING==100) {
end_time[2] = MPI_Wtime();
total_time[2]+=end_time[2] - start_time[2]; //IO write time...
}
}
else
{
for(j=0; j<v->ndim; j++){
s[j] = 0;
c[j] = 1;
}
getTypeInfo( v->type, &element_size);
uint64_t total_size = 1;
for (ii=0;ii<v->ndim;ii++)
total_size*=v->dims[ii];
chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
calcC(chunk_size, v, c);
chunk_size = 1;
for(ii=0; ii<v->ndim; ii++)
chunk_size *= c[ii];
uint64_t current_step = rank*chunk_size;
if(current_step<total_size)
rS(v, s, current_step, rank);
uint64_t elements_written = 0;
while(current_step < total_size)
{
uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
arrCopy(s, ts);
uint64_t remain_chunk = chunk_size;
if(current_step+chunk_size>total_size)
remain_chunk = total_size-current_step;
uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
arrCopy(c, tc);
do{
uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
uint64_t used_chunk = remain_chunk;
remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);
checkOverflow(1, v, ts, uc);
used_chunk -= remain_chunk;
elements_written += used_chunk;
//allocated space for data read-in
data = (void *) malloc(used_chunk*element_size);
if (TIMING==100) {
start_time[1] = MPI_Wtime();
}
if(PERFORMANCE_CHECK) printf("rank=%d, read start\n",rank);
bytes_read = adios_read_var_byid(g,v->varid,ts,uc,data);
if(PERFORMANCE_CHECK) printf("rank=%d, read end\n",rank);
if (TIMING==100) {
end_time[1] = MPI_Wtime();
total_time[1]+=end_time[1] -start_time[1]; //IO read time
}
if (DEBUG)
printf ("ADIOS WRITE: rank=%d, name=%s datasize=%" PRId64 "\n",rank,g->var_namelist[i],bytes_read);
if (TIMING==100) {
start_time[2] = MPI_Wtime();
}
if (DEBUG){
printf("rank=%d, write ts=",rank);
int k;
for(k=0; k<v->ndim; k++)
printf("%" PRId64 ",", ts[k]);
printf(" uc=");
for(k=0; k<v->ndim; k++)
printf("%" PRId64 ",", uc[k]);
printf("\n");
}
//local bounds and offets placeholders are not written out with actual values.
if(PERFORMANCE_CHECK) printf("rank=%d, adios write start\n", rank);
for(k=0; k<v->ndim; k++){
//sprintf(tstring, "ldim%d_%s", k, var_name);
sprintf(tstring, "ldim%d", k);
if (DEBUG) {
printf ("ADIOS WRITE DIMENSION: rank=%d, name=%s value=",
rank,tstring);
print_data (&uc[k], 0, adios_unsigned_long);
printf ("\n");
}
adios_write(m_adios_file, tstring, &uc[k]);
//sprintf(tstring, "offs%d_%s", k, var_name);
sprintf(tstring, "offs%d", k);
if (DEBUG) {
printf ("ADIOS WRITE OFFSET: rank=%d, name=%s value=",
rank,tstring);
print_data (&ts[k], 0, adios_unsigned_long);
printf ("\n");
}
adios_write(m_adios_file, tstring, &ts[k]);
}
adios_write(m_adios_file,g->var_namelist[i],data);
if(PERFORMANCE_CHECK) printf("rank=%d, adios write end\n", rank);
if (TIMING==100) {
end_time[2] = MPI_Wtime();
total_time[2]+=end_time[2] - start_time[2]; //IO write time
}
free(data);
if(remain_chunk!=0){
rS(v, ts, used_chunk, rank);
for(k=0; k<10; k++)
tc[k] = 1;
calcC(remain_chunk, v, tc);
}
}while(remain_chunk!=0);
current_step += size*chunk_size;
if(current_step<total_size)
rS(v, s, size*chunk_size,rank);
}
if(DEBUG){
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = elements_written;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0 && rb[0]!=total_size)
printf("some array read mismatch. please use debug mode\n");
free(sb); free(rb);
}
}
free (var_name);
free (var_path);
}// end of the writing of the variable..
if (TIMING==100) {
start_time[3] = MPI_Wtime();
}
if(PERFORMANCE_CHECK) printf("rank=%d, adios_close start\n", rank);
adios_close(m_adios_file);
if(PERFORMANCE_CHECK) printf("rank=%d, adios_close end\n", rank);
if (TIMING==100) {
end_time[3] = MPI_Wtime();
total_time[3]+=end_time[3] - start_time[3];
}
adios_gclose(g);
} //end of WRITEME
} // end of all of the groups
if(rank==0)
printf("conversion done!\n");
if(TIMING==100)
start_time[4] = MPI_Wtime();
adios_fclose(f);
if(TIMING==100){
end_time[4] = MPI_Wtime();
total_time[4] = total_time[4]+end_time[4]-start_time[4];
}
adios_finalize(rank);
// now, we write out the timing data, for each category, we give max, min, avg, std, all in seconds, across all processes.
if(TIMING==100){
// 0: adios_open, adios_group_size
// 1: the total time to read in the data
// 2: times around each write (will only work if we do NOT buffer....
// 3: the time in the close
// 4: fopen, fclose
// 5: total time
end_time[5] = MPI_Wtime();
total_time[5] = end_time[5] - start_time[5];
double sb[7];
sb[0] = total_time[1]; sb[1] = total_time[4]; //read_var, fopen+fclose
sb[2] = sb[0]+sb[1];
sb[3] = total_time[0]; sb[4] = total_time[2]+total_time[3]; //adios_open+adios_group_size, write+close
sb[5] = sb[3]+sb[4];
sb[6] = total_time[5]; //total
double * rb = NULL;
if(rank==0)
rb = (double *)malloc(size*7*sizeof(double));
//MPI_Barrier(comm);
MPI_Gather(sb, 7, MPI_DOUBLE, rb, 7, MPI_DOUBLE, 0, comm);
if(rank==0){
double read_avg1 = 0;
double read_avg2 = 0;
double tread_avg = 0;
double write_avg1 = 0;
double write_avg2 = 0;
double twrite_avg = 0;
double total_avg = 0;
for(j=0; j<size; j++){
read_avg1 += rb[7*j];
read_avg2 += rb[7*j+1];
tread_avg += rb[7*j+2];
write_avg1 += rb[7*j+3];
write_avg2 += rb[7*j+4];
twrite_avg += rb[7*j+5];
total_avg += rb[7*j+6];
}
read_avg1 /= size;
read_avg2 /= size;
tread_avg /= size;
write_avg1 /= size;
write_avg2 /= size;
twrite_avg /= size;
total_avg /= size;
double read1_max = rb[0];
double read1_min = rb[0];
double read1_std = rb[0]-read_avg1; read1_std *= read1_std;
double read2_max = rb[1];
double read2_min = rb[1];
double read2_std = rb[1]-read_avg2; read2_std *= read2_std;
double tread_max = rb[2];
double tread_min = rb[2];
double tread_std = rb[2]-tread_avg; tread_std *= tread_std;
double write1_max = rb[3];
double write1_min = rb[3];
double write1_std = rb[3]-write_avg1; write1_std *= write1_std;
double write2_max = rb[4];
double write2_min = rb[4];
double write2_std = rb[4]-write_avg2; write2_std *= write2_std;
double twrite_max = rb[5];
double twrite_min = rb[5];
double twrite_std = rb[5]-twrite_avg; twrite_std *= twrite_std;
double total_max = rb[6];
double total_min = rb[6];
double total_std = rb[6]-total_avg; total_std *= total_std;
for(j=1; j<size; j++){
if(rb[7*j]>read1_max)
read1_max = rb[7*j];
else if(rb[7*j]<read1_min)
read1_min = rb[7*j];
double std = rb[7*j]-read_avg1; std *= std;
read1_std += std;
if(rb[7*j+1]>read2_max)
read2_max = rb[7*j+1];
else if(rb[7*j+1]<read2_min)
read2_min = rb[7*j+1];
std = rb[7*j+1]-read_avg2; std *= std;
read2_std += std;
if(rb[7*j+2]>tread_max)
tread_max = rb[7*j+2];
else if(rb[7*j+2]<tread_min)
tread_min = rb[7*j+2];
std = rb[7*j+2]-tread_avg; std *= std;
tread_std += std;
if(rb[7*j+3]>write1_max)
write1_max = rb[7*j+3];
else if(rb[7*j+3]<write1_min)
write1_min = rb[7*j+3];
std = rb[7*j+3]-write_avg1; std *= std;
write1_std += std;
if(rb[7*j+4]>write2_max)
write2_max = rb[7*j+4];
else if(rb[7*j+4]<write2_min)
write2_min = rb[7*j+4];
std = rb[7*j+4]-write_avg2; std *= std;
write2_std += std;
if(rb[7*j+5]>twrite_max)
twrite_max = rb[7*j+5];
else if(rb[7*j+5]<twrite_min)
twrite_min = rb[7*j+5];
std = rb[7*j+5]-twrite_avg; std *= std;
twrite_std += std;
if(rb[7*j+6]>total_max)
total_max = rb[7*j+6];
else if(rb[7*j+6]<total_min)
total_min = rb[7*j+6];
std = rb[7*j+6]-total_avg; std *= std;
total_std += std;
}
read1_std /= size; read1_std = sqrt(read1_std);
read2_std /= size; read2_std = sqrt(read2_std);
tread_std /= size; tread_std = sqrt(tread_std);
write1_std /= size; write1_std = sqrt(write1_std);
write2_std /= size; write2_std = sqrt(write2_std);
twrite_std /= size; twrite_std = sqrt(twrite_std);
total_std /= size; total_std = sqrt(total_std);
printf("---type--- max\tmin\tavg\tstd\n");
printf("---read_var--- %lf\t%lf\t%lf\t%lf\n", read1_max, read1_min, read_avg1, read1_std);
printf("---fopen+fclose--- %lf\t%lf\t%lf\t%lf\n", read2_max, read2_min, read_avg2, read2_std);
printf("---total_read--- %lf\t%lf\t%lf\t%lf\n", tread_max, tread_min, tread_avg, tread_std);
printf("---adios_open+adios_groupsize--- %lf\t%lf\t%lf\t%lf\n", write1_max, write1_min, write_avg1, write1_std);
printf("---write+close--- %lf\t%lf\t%lf\t%lf\n", write2_max, write2_min, write_avg2, write2_std);
printf("---total_write--- %lf\t%lf\t%lf\t%lf\n", twrite_max, twrite_min, twrite_avg, twrite_std);
printf("---total--- %lf\t%lf\t%lf\t%lf\n", total_max, total_min, total_avg, total_std);
free(rb);
}
}
// if (TIMING==100 && rank==0) {
// printf("------------------------------------------------------------------\n");
// printf("Define variables = %lf\n",total_time[0]);
// printf("Read variables = %lf\n",total_time[1]);
// printf("Write variables = %lf\n",total_time[2]);
// printf("Close File for write = %lf\n",total_time[3]);
// printf("Total write time = %lf\n",total_time[2] + total_time[3]);
// for (itime=0;itime<timers-1;itime++)
// total_time[timers-1]+=total_time[itime];
// printf("Total I/O time = %lf\n",total_time[timers-1]);
// }
MPI_Finalize();
return(0);
}
int readVar(ADIOS_GROUP *gp, ADIOS_VARINFO *vi, const char * name)
{
int i,j;
uint64_t start_t[MAX_DIMS], count_t[MAX_DIMS]; // processed <0 values in start/count
uint64_t s[MAX_DIMS], c[MAX_DIMS]; // for block reading of smaller chunks
hsize_t h5_start[MAX_DIMS], h5_count[MAX_DIMS], h5_stride[MAX_DIMS];
hsize_t h5_dims[MAX_DIMS];
uint64_t nelems; // number of elements to read
int elemsize; // size in bytes of one element
uint64_t st, ct;
void *data;
uint64_t sum; // working var to sum up things
int maxreadn; // max number of elements to read once up to a limit (10MB of data)
int actualreadn; // our decision how much to read at once
int readn[MAX_DIMS]; // how big chunk to read in in each dimension?
int64_t bytes_read; // retval from adios_get_var()
int incdim; // used in incremental reading in
hid_t grp_id, space_id, dataset, global_memspace, dataspace;
hid_t local_memspace, h5_ndim ;
hid_t h5_err;
hid_t h5_type_id;
if (getTypeInfo(vi->type, &elemsize)) {
fprintf(stderr, "Adios type %d (%s) not supported in bpls. var=%s\n",
vi->type, adios_type_to_string(vi->type), name);
return 10;
}
h5_err = bp_getH5TypeId (vi->type, &h5_type_id);
h5_ndim = (hsize_t) vi->ndim;
for (j=0;j<h5_ndim;j++)
h5_dims[j] = vi->dims[j];
// create the hdf5 dataspace.
// create the counter arrays with the appropriate lengths
// transfer start and count arrays to format dependent arrays
for (j=0; j<vi->ndim; j++) {
icount[j]=-1;
h5_stride[j]= (hsize_t) 1;
}
nelems = 1;
for (j=0; j<vi->ndim; j++) {
if (istart[j] < 0) // negative index means last-|index|
st = vi->dims[j]+istart[j];
else
st = istart[j];
if (icount[j] < 0) // negative index means last-|index|+1-start
ct = vi->dims[j]+icount[j]+1-st;
else
ct = icount[j];
if (verbose>2)
printf(" j=%d, st=%llu ct=%llu\n", j, st, ct);
start_t[j] = st;
count_t[j] = ct;
nelems *= ct;
if (verbose>1)
printf(" s[%d]=%llu, c[%d]=%llu, n=%llu\n", j, start_t[j], j, count_t[j], nelems);
}
if (verbose>1) {
printf(" total size of data to read = %llu\n", nelems*elemsize);
}
maxreadn = MAX_BUFFERSIZE/elemsize;
if (nelems < maxreadn)
maxreadn = nelems;
// special case: string. Need to use different elemsize
if (vi->type == adios_string) {
if (vi->value)
elemsize = strlen(vi->value)+1;
maxreadn = elemsize;
}
// allocate data array
data = (void *) malloc (maxreadn*elemsize); // SAK: don't want the +8....
//+8 for just to be sure
// determine strategy how to read in:
// - at once
// - loop over 1st dimension
// - loop over 1st & 2nd dimension
// - etc
if (verbose>1) printf("Read size strategy:\n");
sum = (uint64_t) 1;
actualreadn = (uint64_t) 1;
for (i=vi->ndim-1; i>=0; i--) {
if (sum >= (uint64_t) maxreadn) {
readn[i] = 1;
} else {
readn[i] = maxreadn / (int)sum; // sum is small for 4 bytes here
// this may be over the max count for this dimension
if (readn[i] > count_t[i])
readn[i] = count_t[i];
}
if (verbose>1) printf(" dim %d: read %d elements\n", i, readn[i]);
sum = sum * (uint64_t) count_t[i];
actualreadn = actualreadn * readn[i];
}
if (verbose>1) printf(" read %d elements at once, %lld in total (nelems=%lld)\n", actualreadn, sum, nelems);
// init s and c
for (j=0; j<vi->ndim; j++) {
s[j]=start_t[j];
c[j]=readn[j];
h5_count[j] = (hsize_t) c[j];
h5_start[j] = (hsize_t) s[j];
}
// read until read all 'nelems' elements
sum = 0;
while (sum < nelems) {
// how many elements do we read in next?
actualreadn = 1;
for (j=0; j<vi->ndim; j++)
actualreadn *= c[j];
if (verbose>2) {
printf("adios_read_var name=%s ", name);
//PRINT_DIMS(" start", s, vi->ndim, j);
//PRINT_DIMS(" count", c, vi->ndim, j);
printf(" read %d elems\n", actualreadn);
}
// read a slice finally
bytes_read = adios_read_var_byid (gp, vi->varid, s, c, data);
if (bytes_read < 0) {
fprintf(stderr, "Error when reading variable %s. errno=%d : %s \n", name, adios_errno, adios_errmsg());
free(data);
return 11;
}
// now we must place this inside the hdf5 file... we know the offset (s)
// we know the count, c
// we know the global rank v->ndim
// we know the global dimensions v->dims
// get the hdf5 calls for writing the hyperslab.....
dataset = H5Dopen(HDF5_FILE,name);
if (dataset> 0) {
global_memspace = H5Dget_space(dataset);
hid_t rank_old = H5Sget_simple_extent_ndims(global_memspace);
hsize_t *maxdims = (hsize_t *) malloc (h5_ndim * sizeof (hsize_t));
h5_err = H5Sget_simple_extent_dims(global_memspace,maxdims,NULL);
free(maxdims);
} else {
global_memspace = H5Screate_simple (h5_ndim, h5_dims, NULL);
hid_t cparms = H5Pcreate(H5P_DATASET_CREATE);
h5_err = H5Pset_chunk(cparms,h5_ndim,h5_count);
dataset = H5Dcreate(HDF5_FILE,name,h5_type_id, global_memspace,cparms);
H5Pclose(cparms);
}
local_memspace = H5Screate_simple (h5_ndim, h5_count, NULL);
for (j=0;j<vi->ndim;j++)
h5_err = H5Sselect_hyperslab (global_memspace, H5S_SELECT_SET
,h5_start, h5_stride, h5_count, NULL);
h5_err = H5Dwrite(dataset,h5_type_id,local_memspace,global_memspace,H5P_DEFAULT, data);
H5Sclose(local_memspace);
H5Dclose(dataset);
//H5Tclose(h5_type_id);
if (verbose>2) printf(" read %lld bytes\n", bytes_read);
// print slice
// prepare for next read
sum += actualreadn;
incdim=1; // largest dim should be increased
for (j=vi->ndim-1; j>=0; j--) {
if (incdim==1) {
if (s[j]+c[j] == start_t[j]+count_t[j]) {
// reached the end of this dimension
s[j] = start_t[j];
c[j] = readn[j];
h5_count[j] = (hsize_t) c[j];
h5_start[j] = (hsize_t) s[j];
incdim = 1; // next smaller dim can increase too
} else {
// move up in this dimension up to total count
s[j] += readn[j];
h5_start[j] = (hsize_t) s[j];
if (s[j]+c[j] > start_t[j]+count_t[j]) {
// do not reach over the limit
c[j] = start_t[j]+count_t[j]-s[j];
h5_count[j] = (hsize_t) c[j];
}
incdim = 0;
}
}
}
} // end while sum < nelems
H5Sclose(global_memspace);
free(data);
return 0;
}
const std::string& DataSourceTypeInfo<UnknownType>::getType() { return getTypeInfo()->getTypeName(); }
const RsPointerType* TypeSystem::getPointerType(const std::string& name, AddressDesc desc)
{
const RsType* bt = getTypeInfo(name);
return getPointerType(bt, desc);
}
