DummyDataSource::DummyDataSource()
{
int nTochek = 200000;
QVector<DataType> buffer;
QVector<double> xData;
QVector<double> yData;
for (int i = 0; i < nTochek; ++i) {
xData.push_back(1+(i*1+0.5)/(double)nTochek);
yData.push_back(sin(18*(i*1+1.)/nTochek));
}
mDataNameVector.push_back("someThing");
buffer.clear();
buffer.push_back(DataType(xData, yData));
mDataVector.push_back(buffer);
QVector<double> xData2;
QVector<double> yData2;
for (int i = 0; i < nTochek; ++i) {
xData2.push_back(-1+(i*1-1.5)/(double)nTochek);
yData2.push_back(cos(9*(i*2-1.5)/nTochek/4.));
}
mDataNameVector.push_back("someThing2");
buffer.clear();
buffer.push_back(DataType(xData2, yData2));
mDataVector.push_back(buffer);
QVector<double> xData3;
QVector<double> yData3;
for (int i = 0; i < nTochek; ++i) {
xData3.push_back(1.5+i/double(nTochek));
yData3.push_back(cos(6*(i*1+0.1)/(double)nTochek));
}
mDataNameVector.push_back("someThing3");
buffer.clear();
buffer.push_back(DataType(xData3, yData3));
mDataVector.push_back(buffer);
QVector<double> xData4;
QVector<double> yData4;
for (int i = 0; i < nTochek; ++i) {
xData4.push_back(-0.75+i/(double)nTochek);
yData4.push_back(1);
}
mDataNameVector.push_back("someThing4");
buffer.clear();
buffer.push_back(DataType(xData4, yData4));
mDataVector.push_back(buffer);
QVector<double> xData5;
QVector<double> yData5;
for (int i = 0; i < nTochek; ++i) {
xData5.push_back(i/(double)nTochek);
yData5.push_back(0.5+sin(30*(9-i)/nTochek));
}
mDataNameVector.push_back("someThing5");
buffer.clear();
buffer.push_back(DataType(xData5, yData5));
mDataVector.push_back(buffer);
}
DataType TypeParser::parseClosureType() {
assert(_helper->nextIf(Token::Type::PunctuationOpenBrace));
DataType type(DataType::Kind::Closure);
// make sure to define the environment pointer
type.addParameter(DataType::wrapInPointer(DataType::Kind::Void));
this->parseTypeList(Token::Type::PunctuationCloseBrace, true, [&] (const DataType& subtype) {
type.addParameter(subtype);
});
if (!_helper->nextIf(Token::Type::PunctuationCloseBrace)) {
assert(0 && "Message: failed to parse a closure type, missing close brace");
return DataType();
}
if (!_helper->nextIf(Token::Type::OperatorArrow)) {
type.setReturnType(DataType(DataType::Kind::Void));
return type;
}
type.setReturnType(this->parseType());
return type;
}
DataType TypeParser::parseArrayType(bool genericParam) {
assert(_helper->next().type() == Token::Type::PunctuationOpenBracket);
DataType type = DataType(genericParam ? DataType::Kind::GenericArray : DataType::Kind::Array);
if (_helper->peek().type() != Token::Type::PunctuationCloseBracket) {
uint32_t value = 0;
if (!this->parseIntegerSpecifierValue(&value)) {
assert(0 && "Message: unable to parse array size");
return DataType();
}
type.setArrayCount(value);
}
if (!_helper->nextIf(Token::Type::PunctuationCloseBracket)) {
assert(0 && "Message: array type element count should be followed by a close bracket");
return DataType();
}
type.addSubtype(this->parseType(genericParam));
return this->applyTypePostfixes(type, genericParam);
}
void variance_scaling_initializer::fill(AbsDistMat& matrix) {
// Check if fan-in and fan-out parameters are valid
if (m_fan_in <= 0 || m_fan_out <= 0) {
std::stringstream err;
err << "attempted variance scaling initialization "
<< "with invalid parameters "
<< "(fan_in=" << m_fan_in << ",fan_out=" << m_fan_out << ")";
LBANN_ERROR(err.str());
}
// Get variance
const auto& variance = get_variance(m_fan_in, m_fan_out);
// Fill matrix with values drawn from probability distribution
switch (m_prob_dist) {
case probability_distribution::gaussian:
gaussian_fill(matrix, matrix.Height(), matrix.Width(),
DataType(0), std::sqrt(variance));
break;
case probability_distribution::uniform:
uniform_fill(matrix, matrix.Height(), matrix.Width(),
DataType(0), std::sqrt(3*variance));
break;
default:
std::stringstream err;
err << "variance scaling initializer is only supported with "
<< "Gaussian and uniform probability distributions "
<< "(dist=" << El::Int(m_prob_dist) << ")";
LBANN_ERROR(err.str());
}
}
// SET DATA TYPE FOR THIS BUTTON GIVEN A "STRING"
// Returns 0 if OK, -1 if unknown data type.
//
int MyButton::SetDataTypeStr(const char *s) {
dtype = DataType(0);
if ( strstr(s, "int") ) dtype = DataType(((int)dtype) | (int)INT);
if ( strstr(s, "float") ) dtype = DataType(((int)dtype) | (int)FLOAT);
if ( strstr(s, "string") ) dtype = DataType(((int)dtype) | (int)STRING);
return ( dtype == 0 ) ? -1 : 0;
}
void RysIntegral<DataType>::perform_contraction_new_inner(const int nsize, const int ac, const DataType* prim, const int pdim0, const int pdim1, DataType* cont,
const std::vector<std::vector<double>>& coeff0, const std::vector<int>& upper0, const std::vector<int>& lower0, const int cdim0,
const std::vector<std::vector<double>>& coeff1, const std::vector<int>& upper1, const std::vector<int>& lower1, const int cdim1) {
const int worksize = pdim1 * ac;
DataType* const work = stack_->get<DataType>(worksize);
DataType* current_cont = cont;
for (int n = 0; n != nsize; ++n) { // loop of cdim * cdim
const DataType* current_prim = &prim[ac * pdim1 * pdim0 * n];
for (int i = 0; i != cdim0; ++i) {
const int begin0 = lower0[i];
const int end0 = upper0[i];
std::fill_n(work, worksize, DataType(0.0));
for (int j = begin0; j != end0; ++j)
blas::ax_plus_y_n(coeff0[i][j], current_prim+j*worksize, worksize, work);
for (int k = 0; k != cdim1; ++k, current_cont += ac) {
const int begin1 = lower1[k];
const int end1 = upper1[k];
std::fill_n(current_cont, ac, DataType(0.0));
for (int j = begin1; j != end1; ++j)
blas::ax_plus_y_n(coeff1[k][j], work+j*ac, ac, current_cont);
}
}
}
stack_->release(worksize, work);
}
static void unpack(uint64_t msg, const std::vector<BitMapType>& bitmap,
std::vector<DataType>& data)
{
data.clear();
data.resize(bitmap.size());
for (int i=bitmap.size()-1; i>=0; --i)
{
DataType bitmask = DataType(DataType(1) << bitmap[i]) -1;
data[i]=(msg & bitmask);
//std::cout<<"Unpacking message:"<<std::bitset<48>(msg)<<" with bitmap: "<<std::bitset<sizeof(DataType)*8>(bitmask)<<std::endl;
msg >>= bitmap[i];
}
};
DataType DataType::create(const QString &dataTypeCd, const QString &dataTypeNameZh, const QString &dataTypeNameEn, const QString &engUnitCd, const QString &createdBy, const QString &updatedBy)
{
DataTypeObject obj;
obj.data_type_cd = dataTypeCd;
obj.data_type_name_zh = dataTypeNameZh;
obj.data_type_name_en = dataTypeNameEn;
obj.eng_unit_cd = engUnitCd;
obj.created_by = createdBy;
obj.updated_by = updatedBy;
if (!obj.create()) {
return DataType();
}
return DataType(obj);
}
void hypergradient_adam::step_compute(AbsDistMat& values,
const AbsDistMat& gradient) {
// Precompute the bias correction.
m_current_beta1 *= m_beta1;
m_current_beta2 *= m_beta2;
const DataType correction = std::sqrt(DataType(1) - m_current_beta2) /
(DataType(1) - m_current_beta1);
// Get local matrix data
const int local_height = values.LocalHeight();
const int local_width = values.LocalWidth();
DataType* __restrict__ values_buffer = values.Buffer();
const int values_ldim = values.LDim();
const DataType* __restrict__ gradient_buffer = gradient.LockedBuffer();
const int gradient_ldim = gradient.LDim();
DataType* __restrict__ moment1_buffer = m_moment1->Buffer();
const int moment1_ldim = m_moment1->LDim();
DataType* __restrict__ moment2_buffer = m_moment2->Buffer();
const int moment2_ldim = m_moment2->LDim();
DataType* __restrict__ old_gradient_buffer = m_old_gradient->Buffer();
const int old_gradient_ldim = m_old_gradient->LDim();
// Compute the learning rate update.
DataType lr_update = El::Dot(gradient, *m_old_gradient);
m_learning_rate += m_hyper_learning_rate * lr_update;
// Check if matrix data is contiguous.
if (values_ldim != local_height
|| gradient_ldim != local_height
|| moment1_ldim != local_height
|| moment2_ldim != local_height
|| old_gradient_ldim != local_height) {
// Non-contiguous data.
LBANN_OMP_PARALLEL_FOR_COLLAPSE2
for (int j = 0; j < local_width; ++j) {
for (int i = 0; i < local_height; ++i) {
DataType& x = values_buffer[i+j*values_ldim];
const DataType g = gradient_buffer[i+j*gradient_ldim] + m_eps;
DataType& m1 = moment1_buffer[i+j*moment1_ldim];
DataType& m2 = moment2_buffer[i+j*moment2_ldim];
DataType& old_c = old_gradient_buffer[i+j*old_gradient_ldim];
m1 = m_beta1 * m1 + (DataType(1) - m_beta1) * g;
m2 = m_beta2 * m2 + (DataType(1) - m_beta2) * g * g;
old_c = correction * m1 / (std::sqrt(m2) + m_eps);
x -= m_learning_rate * old_c;
}
}
} else {
DataType TypeParser::parseFunctionOrTupleType() {
assert(_helper->nextIf(Token::Type::PunctuationOpenParen));
DataType type;
this->parseTypeListWithLabels(Token::Type::PunctuationCloseParen, [&] (const DataType& subtype) {
type.addSubtype(subtype);
});
if (!_helper->nextIf(Token::Type::PunctuationCloseParen)) {
assert(0 && "Message: failed to parse a tuple/function type");
return DataType();
}
if (!_helper->nextIf(Token::Type::OperatorArrow)) {
type.setKind(DataType::Kind::Tuple);
return type;
}
type.setKind(DataType::Kind::Function);
// TODO: clean this up - move the subtypes to parameters
type.parameters = type.subtypes;
type.subtypes.clear();
type.setReturnType(this->parseType());
return type;
}
void weights::reconcile_values(Al::request& req) {
auto& values = get_values();
if (values.RedundantSize() > 1) {
values *= DataType(1) / values.RedundantSize();
m_comm->nb_allreduce(values, values.RedundantComm(), req);
}
}
void PKCS12Serializable::Serialize(IStream &stream) const
{
// key
Key *keyPtr = getKey().get();
bool isAnyKeyPresent = (getKey().get() != NULL);
// logics if PKCS is correct or not is on the service side.
// sending number of keys and certificates to allow proper parsing on the service side.
// (what if no key or cert present? attempt to deserialize a not present key/cert would
// throw an error and close the connection).
Serialization::Serialize(stream, static_cast<size_t>(isAnyKeyPresent?1:0));
if (keyPtr) {
Serialization::Serialize(stream, DataType(keyPtr->getType()));
Serialization::Serialize(stream, keyPtr->getDER());
}
bool isAnyCertPresent = (getCertificate().get() != NULL);
Serialization::Serialize(stream, static_cast<size_t>(isAnyCertPresent?1:0));
if (isAnyCertPresent)
Serialization::Serialize(stream, getCertificate().get()->getDER());
// CA chain
Serialization::Serialize(stream, getCaCertificateShPtrVector().size());
for (auto it : getCaCertificateShPtrVector())
Serialization::Serialize(stream, it->getDER());
};
DataType& SparseStorage<DataType>::elem(int rr, int cc) {
int oldsize = sparsity().size();
int ind = sparsityRef().getNZ(rr, cc);
if (oldsize != sparsity().size())
data().insert(begin()+ind, DataType(0));
return at(ind);
}
RemoteInformation deserializeDataSourceData( const ::zeq::Event& event )
{
if( event.getType() != EVENT_DATASOURCE_DATA )
return RemoteInformation();
auto data = GetVolumeInformation( event.getData( ));
RemoteInformation info;
livre::VolumeInformation& vi = info.second;
info.first.low() = data->eventLow();
info.first.high() = data->eventHigh();
vi.isBigEndian = data->isBigEndian();
vi.compCount = data->compCount();
vi.dataType = DataType( data->dataType( ));
vi.overlap = _deserializeVector3< unsigned >( data->overlap( ));
vi.maximumBlockSize = _deserializeVector3< unsigned >(
data->maximumBlockSize( ));
vi.minPos = _deserializeVector3< float >( data->minPos( ));
vi.maxPos = _deserializeVector3< float >( data->maxPos( ));
vi.voxels = _deserializeVector3< unsigned >( data->voxels( ));
vi.worldSize = _deserializeVector3< float >( data->worldSize( ));
vi.boundingBox.getMin() = _deserializeVector3< float >(
data->boundingBoxMin( ));
vi.boundingBox.getMax() = _deserializeVector3< float >(
data->boundingBoxMax( ));
vi.worldSpacePerVoxel = data->worldSpacePerVoxel();
const Vector3ui& blockSize = vi.maximumBlockSize - vi.overlap * 2;
Vector3ui blocksSize = vi.voxels / blockSize;
blocksSize = blocksSize / ( 1u << data->depth( ));
vi.rootNode = RootNode( data->depth(), blocksSize );
return info;
}
Data Func::eval(VarTable *vars,
FuncTable *funcs,
vector<Data> &argv)
{
VarTable new_vars(vars);
m_args->apply(&new_vars, argv);
bool retstat = false;
Data ret = m_block->eval(&new_vars, funcs, &retstat);
DataType type = m_var_decl->get_type();
if (type != ret.get_type())
{
if (type.get_type() == "int")
{
printf("Error: function conversion from float to int\n");
exit(1);
}
int ival = ret.get_int();
ret.set_float((float)ival);
ret.set_type(DataType("float", 0));
}
return ret;
}
DataType& SparseStorage<DataType>::elem(casadi_int rr, casadi_int cc) {
casadi_int oldsize = sparsity().nnz();
casadi_int ind = sparsity_.add_nz(rr, cc);
if (oldsize != sparsity().nnz())
nonzeros().insert(nonzeros().begin()+ind, DataType(0));
return nonzeros().at(ind);
}
// [RH][TODO]
// merge this with space/metric_object.hpp -> metric_object
//
array()
{
for (size_t ci = 0; ci < N; ++ci)
{
(*this)[ci] = DataType(0);
}
}
BasicPtree() :
_key(KeyType()),
_data(DataType()),
_children(ChildrenList()),
_meta_data(NULL),
_data_cmp(PredEqualData()),
_key_cmp(PredEqualKey())
{}
NekVector<DataType>
Multiply(const NekMatrix<LhsDataType, MatrixType>& lhs,
const NekVector<DataType>& rhs)
{
NekVector<DataType> result(lhs.GetRows(), DataType(0));
Multiply(result, lhs, rhs);
return result;
}
static DataType FromProtoType(proto::NDArrayView::DataType type)
{
if (!proto::NDArrayView::DataType_IsValid(type))
{
InvalidArgument("NDArrayView::DataType is invalid.");
}
return DataType(type);
}
DataType VariableNode::dataType() const {
if (!_newVariable) {
std::cout << "variable undefined!" << std::endl;
return DataType();
}
return _newVariable->type;
}
bool FRawCurveTracks::AddCurveDataImpl(TArray<DataType> & Curves, USkeleton::AnimCurveUID Uid, int32 CurveFlags)
{
if(GetCurveDataImpl<DataType>(Curves, Uid) == NULL)
{
Curves.Add(DataType(Uid, CurveFlags));
return true;
}
return false;
}
DataType TypeParser::parseTypeSpecifiers(const DataType& type) {
if (type.kind() == DataType::Kind::Character) {
return this->parseCharacterTypeSpecifiers(type);
}
DataType newType(type);
uint32_t value = 0;
// width specifier (or, colon)
if (_helper->nextIf(Token::Type::PunctuationColon)) {
if (_helper->peek().type() != Token::Type::PunctuationColon) {
if (!this->parseIntegerSpecifierValue(&value)) {
return DataType();
}
newType.setWidthSpecifier(value);
}
}
// alignment (or, colon)
if (_helper->nextIf(Token::Type::PunctuationColon)) {
if (_helper->peek().type() != Token::Type::PunctuationColon) {
if (!this->parseIntegerSpecifierValue(&value)) {
return DataType();
}
newType.setAlignmentSpecifier(value);
}
}
// vector
if (_helper->nextIf(Token::Type::PunctuationColon)) {
if (!this->parseIntegerSpecifierValue(&value)) {
return DataType();
}
newType.setVectorSizeSpecifier(value);
}
return newType;
}
DataType TypeParser::parsePointerType(bool genericParam) {
assert(_helper->next().type() == Token::Type::OperatorStar);
DataType type = DataType(genericParam ? DataType::Kind::GenericPointer : DataType::Kind::Pointer);
type = this->parseTypePostfixes(type);
type.addSubtype(this->parseType(genericParam));
return this->applyTypePostfixes(type, genericParam);
}
void ThreadedDyscoColumn<DataType>::getValues(casacore::uInt rowNr, casacore::Array<DataType>* dataPtr)
{
if(!areOffsetsInitialized())
{
// Trying to read before first block was written -- return zero
// TODO if a few rows were written of the first block, those are
// incorrectly returned. This is a rare case but can be fixed.
for(typename casacore::Array<DataType>::contiter i=dataPtr->cbegin(); i!=dataPtr->cend(); ++i)
*i = DataType();
}
else {
size_t blockIndex = getBlockIndex(rowNr);
if(blockIndex >= nBlocksInFile())
{
// Trying to read a row that was not stored yet -- return zero
for(typename casacore::Array<DataType>::contiter i=dataPtr->cbegin(); i!=dataPtr->cend(); ++i)
*i = DataType();
}
else {
mutex::scoped_lock lock(_mutex);
// Wait until the block to be read is not in the write cache
typename cache_t::const_iterator cacheItemPtr = _cache.find(blockIndex);
while(cacheItemPtr != _cache.end())
{
_cacheChangedCondition.wait(lock);
cacheItemPtr = _cache.find(blockIndex);
}
lock.unlock();
if(_currentBlock != blockIndex)
{
if(_isCurrentBlockChanged)
storeBlock();
loadBlock(blockIndex);
}
// The time block encoder is now initialized and contains the unpacked block.
_timeBlockBuffer->GetData(getRowWithinBlock(rowNr), dataPtr->data());
}
}
}
DataType FunctionMap::getFunctionsNamed(String name) {
/* create a dataType to return */
DataType retType = DataType(TYPE_OVERLOAD);
/* loop through all elements in the map */
for (std::map<String, Node*>::iterator it = lookup.begin(); it != lookup.end(); it++) {
/* check for a name match */
if (name == it->first.substring(0, (it->first).indexOf("("))) {
/* if one was found, add it to the subtypes */
retType.subtypes->push_back(DataType(*(it->second->type())));
}
}
/* If there are no subtypes, return null */
if (retType.subtypes->size() == 0) {
retType = DataType(TYPE_NONE);
/* If there is one, just return that one */
} else if (retType.subtypes->size() == 1) {
retType = (*(retType.subtypes))[0];
}
return retType;
}
Variant HHVM_METHOD(XMLReader, __get,
const Variant& name) {
auto* data = Native::data<XMLReader>(this_);
const xmlChar *retchar = nullptr;
int retint = 0;
XMLPropertyAccessor *propertyMap = xmlreader_properties_map.get(name);
if (!propertyMap) {
this_->raiseUndefProp(name.getStringData());
return init_null();
}
if (data->m_ptr) {
if (propertyMap->getter_char) {
retchar = propertyMap->getter_char(data->m_ptr);
} else if (propertyMap->getter_int) {
retint = propertyMap->getter_int(data->m_ptr);
}
}
switch (DataType(propertyMap->return_type)) {
case KindOfBoolean:
return retint ? true : false;
case KindOfInt64:
return retint;
case KindOfString:
if (retchar) {
return String((char*)retchar, CopyString);
} else {
return empty_string_variant();
}
case KindOfUninit:
case KindOfNull:
case KindOfDouble:
case KindOfPersistentString:
case KindOfArray:
case KindOfPersistentArray:
case KindOfVec:
case KindOfPersistentVec:
case KindOfDict:
case KindOfPersistentDict:
case KindOfKeyset:
case KindOfPersistentKeyset:
case KindOfObject:
case KindOfResource:
case KindOfRef:
return init_null();
case KindOfClass:
break;
}
not_reached();
}
bool FRawCurveTracks::DuplicateCurveDataImpl(TArray<DataType> & Curves, USkeleton::AnimCurveUID ToCopyUid, USkeleton::AnimCurveUID NewUid)
{
DataType* ExistingCurve = GetCurveDataImpl<DataType>(Curves, ToCopyUid);
if(ExistingCurve && GetCurveDataImpl<DataType>(Curves, NewUid) == NULL)
{
// Add the curve to the track and set its data to the existing curve
Curves.Add(DataType(NewUid, ExistingCurve->GetCurveTypeFlags()));
Curves.Last().CopyCurve(*ExistingCurve);
return true;
}
return false;
}
DataType Buffer< DataType >::pullData(int pos)
{
if (pos != -1){
if (pos < 0 || pos >= m_cloudBuffer.size()){
printMessage("Pulling at index out of range");
return DataType(nullptr);
}
m_pullDataPosition = pos;
}
std::unique_lock<std::mutex> cloudBufferLock(*m_cloudBufferMutex);
if (!m_bufferingActive || m_bufferFillLevel == 0){
//we do not have any data ready?
return DataType(nullptr);
}
//wait until buffer is filled
while (m_bufferFillLevel != m_cloudBuffer.size()){
if (m_producerFinished){
//producer have finished => so we can just
// take the left data out
printMessage("break!");
break;
}
m_cloudBufferUpdated->wait(cloudBufferLock);
}
//retrieve data
auto result = m_cloudBuffer[m_pullDataPosition];
//do we want to clear the data retrieved from the buffer?
if (m_releaseDataAfterPull){
m_cloudBuffer[m_pullDataPosition].reset();
m_bufferFillLevel--;
}
//calc next pull index
m_pullDataPosition = (m_pullDataPosition + 1) % m_cloudBuffer.size();
m_cloudBufferFree->notify_all();
return result;
}
void NekMultiplyUnspecializedMatrixType(DataType* result,
const NekMatrix<LhsDataType, MatrixType>& lhs,
const DataType* rhs)
{
for(unsigned int i = 0; i < lhs.GetRows(); ++i)
{
DataType accum = DataType(0);
for(unsigned int j = 0; j < lhs.GetColumns(); ++j)
{
accum += lhs(i,j)*rhs[j];
}
result[i] = accum;
}
}
