void draw(RealVector& input,unsigned int& label) const{
input.resize(5);
label = Rng::coinToss(0.5)*2+1;
for(std::size_t i = 0; i != 5; ++i){
input(i) = Rng::uni(-1,1);
}
}
///////////////////////////////////////////////////////////////////////////////
//
/// Finds the zeroes of P(x) in the domain \f$a < x <= b\f$, putting the
/// results into Z
//
///////////////////////////////////////////////////////////////////////////////
void AntisymmetricExpFit::find_zeroes(RealVector &P, const Real &a, const Real &b, RealVector &Z) {
const int N = P.size();
RealMatrix cm(N-1,N-1);
int i,j;
// --- Form the companion matrix
// -----------------------------
cm.fill(0.0);
for(j=0; j<N-1; ++j) {
cm(0,j) = -P(N-2-j)/P(N-1);
}
for(i=1; i<N-1; ++i) {
cm(i,i-1) = 1.0;
}
// --- find eigenvalues of
// --- companion matrix and
// --- extract roots in [0:1]
// --------------------------
EigenSolver<RealMatrix> roots(cm,false);
Z.resize(N);
i = 0;
for(j = 0; j<roots.eigenvalues().size(); ++j) {
if(fabs(roots.eigenvalues()(j).imag()) < 1e-15 &&
roots.eigenvalues()(j).real() > a &&
roots.eigenvalues()(j).real() <= b) {
Z(i) = roots.eigenvalues()(j).real();
++i;
}
}
Z.conservativeResize(i);
}
void draw(RealVector& input, unsigned int& label)const
{
label = Rng::discrete(0, 4);
input.resize(2);
input(0) = m_noise * Rng::gauss() + 3.0 * std::cos((double)label);
input(1) = m_noise * Rng::gauss() + 3.0 * std::sin((double)label);
}
void draw(RealVector& point) const
{
point.resize(2);
size_t cluster = random::discrete(random::globalRng,0, 4);
double alpha = 0.4 * M_PI * cluster;
point(0) = 3.0 * cos(alpha) + 0.75 * random::gauss(random::globalRng);
point(1) = 3.0 * sin(alpha) + 0.75 * random::gauss(random::globalRng);
}
void OnlineRNNet::weightedParameterDerivative(RealMatrix const& pattern, const RealMatrix& coefficients, RealVector& gradient) {
SIZE_CHECK(pattern.size1()==1);//we can only process a single input at a time.
SIZE_CHECK(coefficients.size1()==1);
SIZE_CHECK(pattern.size2() == inputSize());
SIZE_CHECK(pattern.size2() == coefficients.size2());
gradient.resize(mpe_structure->parameters());
std::size_t numNeurons = mpe_structure->numberOfNeurons();
std::size_t numUnits = mpe_structure->numberOfUnits();
//first check wether this is the first call of the derivative after a change of internal structure. in this case we have to allocate A LOT
//of memory for the derivative and set it to zero.
if(m_unitGradient.size1() != mpe_structure->parameters() || m_unitGradient.size2() != numNeurons) {
m_unitGradient.resize(mpe_structure->parameters(),numNeurons);
m_unitGradient.clear();
}
//for the next steps see Kenji Doya, "Recurrent Networks: Learning Algorithms"
//calculate the derivative for all neurons f'
RealVector neuronDerivatives(numNeurons);
for(std::size_t i=0; i!=numNeurons; ++i) {
neuronDerivatives(i)=mpe_structure->neuronDerivative(m_activation(i+inputSize()+1));
}
//calculate the derivative for every weight using the derivative of the last time step
auto hiddenWeights = columns(
mpe_structure->weights(),
inputSize()+1,numUnits
);
//update the new gradient with the effect of last timestep
noalias(m_unitGradient) = prod(m_unitGradient,trans(hiddenWeights));
//add the effect of the current time step
std::size_t param = 0;
for(std::size_t i = 0; i != numNeurons; ++i) {
for(std::size_t j = 0; j != numUnits; ++j) {
if(mpe_structure->connection(i,j)) {
m_unitGradient(param,i) += m_lastActivation(j);
++param;
}
}
}
//multiply with outer derivative of the neurons
for(std::size_t i = 0; i != m_unitGradient.size1(); ++i) {
noalias(row(m_unitGradient,i)) = element_prod(row(m_unitGradient,i),neuronDerivatives);
}
//and formula 4 (the gradient itself)
noalias(gradient) = prod(
columns(m_unitGradient,numNeurons-outputSize(),numNeurons),
row(coefficients,0)
);
//sanity check
SIZE_CHECK(param == mpe_structure->parameters());
}
virtual void weightedParameterDerivative( RealMatrix const& input, RealMatrix const& coefficients, State const& state, RealVector& derivative)const
{
derivative.resize(1);
derivative(0)=0;
for (size_t p = 0; p < coefficients.size1(); p++)
{
derivative(0) +=sum(row(coefficients,p));
}
}
void Softmax::weightedParameterDerivative(
BatchInputType const& patterns, BatchOutputType const& coefficients, State const& state, RealVector& gradient
)const{
SIZE_CHECK(patterns.size2() == inputSize());
SIZE_CHECK(coefficients.size2()==outputSize());
SIZE_CHECK(coefficients.size1()==patterns.size1());
gradient.resize(0);
}
void RNNet::weightedParameterDerivative(
BatchInputType const& patterns, BatchInputType const& coefficients,
State const& state, RealVector& gradient
)const{
//SIZE_CHECK(pattern.size() == coefficients.size());
InternalState const& s = state.toState<InternalState>();
gradient.resize(numberOfParameters());
gradient.clear();
std::size_t numUnits = mpe_structure->numberOfUnits();
std::size_t numNeurons = mpe_structure->numberOfNeurons();
std::size_t warmUpLength=m_warmUpSequence.size();
for(std::size_t b = 0; b != patterns.size(); ++b){
Sequence const& sequence = s.timeActivation[b];
std::size_t sequenceLength = s.timeActivation[b].size();
RealMatrix errorDerivative(sequenceLength,numNeurons);
errorDerivative.clear();
//copy errors
for (std::size_t t = warmUpLength+1; t != sequenceLength; ++t)
for(std::size_t i = 0; i != outputSize(); ++i)
errorDerivative(t,i+numNeurons-outputSize())=coefficients[b][t-warmUpLength-1](i);
//backprop through time
for (std::size_t t = (int)sequence.size()-1; t > 0; t--){
for (std::size_t j = 0; j != numNeurons; ++j){
double derivative = mpe_structure->neuronDerivative(sequence[t](j+mpe_structure->inputs()+1));
errorDerivative(t,j)*=derivative;
}
noalias(row(errorDerivative,t-1)) += prod(
trans(columns(mpe_structure->weights(), inputSize()+1,numUnits)),
row(errorDerivative,t)
);
}
//update gradient for batch element i
std::size_t param = 0;
for (std::size_t i = 0; i != numNeurons; ++i){
for (std::size_t j = 0; j != numUnits; ++j){
if(!mpe_structure->connection(i,j))continue;
for(std::size_t t=1;t != sequence.size(); ++t)
gradient(param)+=errorDerivative(t,i) * sequence[t-1](j);
++param;
}
}
//sanity check
SIZE_CHECK(param == mpe_structure->parameters());
}
}
///////////////////////////////////////////////////////////////////////////////
//
/// Changes the variable of a polynomial represented in P and stores the
/// result in Q so that \f$P(x) = Q(x')\f$ where
///
/// \f$x = \alpha x' + (1 - \alpha)\f$
//
///////////////////////////////////////////////////////////////////////////////
void AntisymmetricExpFit::change_variable(RealVector &P, const Real &alpha, RealVector &Q) {
const int M = P.size(); // degree of P + 1
const int SMAX = 96; // maximum degree of Q
int m; // index into P
int r; // index into Q
Real beta = 1.0-alpha; // offset
Real lgb; // log of alpha^r beta^(m-r) (m choose r)
Q.resize(SMAX);
Q.fill(0.0);
for(r = 0; r<SMAX; ++r) {
lgb = r*log(alpha);
for(m = r; m<M; ++m) {
Q(r) += exp(lgb)*P(m);
lgb += log(beta) + log((m+1.0)/(m+1.0-r));
}
}
}
void CompressedDataColumn::fill(RealVector& values, int nRows) {
values.resize(nRows);
if (formatType == DENSE) {
values.assign(data->begin(), data->end());
} else {
bool isSparse = formatType == SPARSE;
values.assign(nRows, 0.0);
int* indicators = getColumns();
size_t n = getNumberOfEntries();
for (size_t i = 0; i < n; ++i) {
const int k = indicators[i];
if (isSparse) {
values[k] = data->at(i);
} else {
values[k] = 1.0;
}
}
}
}
void CMACMap::weightedParameterDerivative(
RealMatrix const& patterns,
RealMatrix const& coefficients,
State const&,//not needed
RealVector &gradient
) const {
SIZE_CHECK(patterns.size2() == m_inputSize);
SIZE_CHECK(coefficients.size2() == m_outputSize);
SIZE_CHECK(coefficients.size1() == patterns.size1());
std::size_t numPatterns = patterns.size1();
gradient.resize(m_parameters.size());
gradient.clear();
for(std::size_t i = 0; i != numPatterns; ++i) {
std::vector<std::size_t> indizes = getIndizes(row(patterns,i));
for (std::size_t o=0; o!=m_outputSize; ++o) {
for (std::size_t j=0; j != m_tilings; ++j) {
gradient(indizes[j] + o*m_parametersPerTiling) += coefficients(i,o);
}
}
}
}
void SigmoidModel::weightedParameterDerivative(
BatchInputType const& patterns, BatchOutputType const& coefficients, State const& state, RealVector& gradient
)const{
SIZE_CHECK( patterns.size2() == 1 );
SIZE_CHECK( coefficients.size2() == 1 );
SIZE_CHECK( coefficients.size1() == patterns.size1() );
InternalState const& s = state.toState<InternalState>();
gradient.resize(2);
gradient(0)=0;
gradient(1)=0;
//calculate derivative
for(std::size_t i = 0; i != patterns.size1(); ++i){
double derivative = sigmoidDerivative( s.result(i) );
double slope= coefficients(i,0)*derivative*patterns(i,0); //w.r.t. slope
if ( m_transformForUnconstrained )
slope *= m_parameters(0);
gradient(0)+=slope;
if ( m_useOffset ) {
gradient(1) -= coefficients(i,0)*derivative; //w.r.t. bias parameter
}
}
}
void DAESolver::calculate_rhs(Real a_step_interval)
{
const Time a_current_time(the_current_time_);
const VariableArray::size_type a_size(the_system_size_);
const Real alphah(alpha_ / a_step_interval);
const Real betah(beta_ / a_step_interval);
const Real gammah(gamma_ / a_step_interval);
gsl_complex comp;
RealVector aTIF;
aTIF.resize(the_system_size_ * 3);
for (VariableArray::size_type c(0); c < the_system_size_; ++c)
{
const Real z(the_w_[c] * 0.091232394870892942792
- the_w_[c + a_size] * 0.14125529502095420843
- the_w_[c + 2 * a_size] * 0.030029194105147424492);
if (c < the_function_differential_size_)
the_value_differential_[c] = the_value_differential_buffer_[c] + z;
else
{
const VariableArray::size_type an_index(
c - the_function_differential_size_);
the_value_algebraic_[an_index]
= the_value_algebraic_buffer_[an_index] + z;
}
}
// ========= 1 ===========
the_current_time_
= a_current_time + a_step_interval * (4.0 - SQRT6) / 10.0;
set_variable_velocity(the_taylor_series_[4]);
for (FunctionArray::size_type c(
the_function_differential_size_); c < the_system_size_; c++)
{
Real tmp_value = (*the_function_[c])(the_value_differential_,
the_value_algebraic_, the_current_time_);
aTIF[c] = tmp_value * 4.3255798900631553510;
aTIF[c + a_size] = tmp_value * -4.1787185915519047273;
aTIF[c + a_size * 2] = tmp_value * -0.50287263494578687595;
}
for (VariableArray::size_type c(0);
c < the_function_differential_size_; c++)
{
aTIF[c] = the_taylor_series_[4][c] * 4.3255798900631553510;
aTIF[c + a_size] = the_taylor_series_[4][c] * -4.1787185915519047273;
aTIF[c + a_size * 2]
= the_taylor_series_[4][c] * -0.50287263494578687595;
}
for (VariableArray::size_type c(0); c < a_size; ++c)
{
const Real z(the_w_[c] * 0.24171793270710701896
+ the_w_[c + a_size] * 0.20412935229379993199
+ the_w_[c + 2 * a_size] * 0.38294211275726193779);
if (c < the_function_differential_size_)
{
the_value_differential_[c]
= the_value_differential_buffer_[c] + z;
}
else
{
const VariableArray::size_type an_index(
c - the_function_differential_size_);
the_value_algebraic_[an_index]
= the_value_algebraic_buffer_[an_index] + z;
}
}
// ========= 2 ===========
the_current_time_
= a_current_time + a_step_interval * (4.0 + SQRT6) / 10.0;
set_variable_velocity(the_taylor_series_[4]);
for (FunctionArray::size_type c(the_function_differential_size_);
c < the_system_size_; c++)
{
Real tmp_value = (*the_function_[c])(
the_value_differential_, the_value_algebraic_, the_current_time_);
aTIF[c] += tmp_value * 0.33919925181580986954;
aTIF[c + a_size] -= tmp_value * 0.32768282076106238708;
aTIF[c + a_size * 2] += tmp_value * 2.5719269498556054292;
}
for (VariableArray::size_type c(0);
c < the_function_differential_size_; c++)
{
aTIF[c] += the_taylor_series_[4][c] * 0.33919925181580986954;
aTIF[c + a_size] -= the_taylor_series_[4][c] * 0.32768282076106238708;
aTIF[c + a_size * 2]
+= the_taylor_series_[4][c] * 2.5719269498556054292;
}
for (VariableArray::size_type c(0); c < a_size; ++c)
{
const Real z(the_w_[c] * 0.96604818261509293619 + the_w_[c + a_size]);
if (c < the_function_differential_size_)
{
the_value_differential_[c]
= the_value_differential_buffer_[c] + z;
}
else
{
const VariableArray::size_type an_index(
c - the_function_differential_size_);
the_value_algebraic_[an_index]
= the_value_algebraic_buffer_[an_index] + z;
}
}
// ========= 3 ===========
the_current_time_ = a_current_time + a_step_interval;
set_variable_velocity(the_taylor_series_[4]);
for (FunctionArray::size_type c(the_function_differential_size_);
c < the_system_size_; c++)
{
Real tmp_value = (*the_function_[c])(
the_value_differential_, the_value_algebraic_, the_current_time_);
aTIF[c] += tmp_value * 0.54177053993587487119;
aTIF[c + a_size] += tmp_value * 0.47662355450055045196;
aTIF[c + a_size * 2] -= tmp_value * 0.59603920482822492497;
gsl_vector_set(the_velocity_vector1_, c, aTIF[c]);
GSL_SET_COMPLEX(&comp, aTIF[c + a_size], aTIF[c + a_size * 2]);
gsl_vector_complex_set(the_velocity_vector2_, c, comp);
}
for (VariableArray::size_type c(0);
c < the_function_differential_size_; c++)
{
aTIF[c] += the_taylor_series_[4][c] * 0.54177053993587487119;
aTIF[c + a_size] += the_taylor_series_[4][c] * 0.47662355450055045196;
aTIF[c + a_size * 2]
-= the_taylor_series_[4][c] * 0.59603920482822492497;
gsl_vector_set(the_velocity_vector1_,
c, aTIF[c] - the_w_[c] * gammah);
GSL_SET_COMPLEX(&comp,
aTIF[c + a_size]
- the_w_[c + a_size] * alphah
+ the_w_[c + a_size * 2] * betah,
aTIF[c + a_size * 2]
- the_w_[c + a_size] * betah
- the_w_[c + a_size * 2] * alphah);
gsl_vector_complex_set(the_velocity_vector2_, c, comp);
}
the_current_time_ = a_current_time;
}