inline tensor5s gru_impl(const tensor5& input, const std::size_t n_units, const bool use_bias, const bool reset_after, const bool return_sequences, const float_vec& weights, const float_vec& recurrent_weights, const float_vec& bias, const std::string& activation, const std::string& recurrent_activation) { const std::size_t n_timesteps = input.shape().width_; const std::size_t n_features = input.shape().depth_; // weight matrices const EigenIndex n = EigenIndex(n_units); const RowMajorMatrix<Dynamic, Dynamic> W = eigen_row_major_mat_from_values(n_features, n_units * 3, weights); const RowMajorMatrix<Dynamic, Dynamic> U = eigen_row_major_mat_from_values(n_units, n_units * 3, recurrent_weights); // kernel bias RowVector<Dynamic> b_x(n_units * 3); if (use_bias && bias.size() >= 1 * n_units * 3) std::copy_n(bias.cbegin(), n_units * 3, b_x.data()); else b_x.setZero(); // recurrent kernel bias RowVector<Dynamic> b_h(n_units * 3); if (use_bias && bias.size() >= 2 * n_units * 3) std::copy_n(bias.cbegin() + static_cast<float_vec::const_iterator::difference_type>(n_units * 3), n_units * 3, b_h.data()); else b_h.setZero(); // initialize cell output states h RowVector<Dynamic> h(1, n_units); h.setZero(); // write input to eigen matrix of shape (timesteps, n_features) RowMajorMatrix<Dynamic, Dynamic> x(n_timesteps, n_features); for (std::size_t a_t = 0; a_t < n_timesteps; ++a_t) for (std::size_t a_f = 0; a_f < n_features; ++a_f) x(EigenIndex(a_t), EigenIndex(a_f)) = input.get(0, 0, 0, a_t, a_f); // kernel applied to inputs (with bias), produces shape (timesteps, n_units * 3) RowMajorMatrix<Dynamic, Dynamic> Wx = x * W; Wx.rowwise() += b_x; // get activation functions auto act_func = get_activation_func(activation); auto act_func_recurrent = get_activation_func(recurrent_activation); // computing GRU output tensor5s gru_result; if (return_sequences) gru_result = { tensor5(shape5(1, 1, 1, n_timesteps, n_units), float_type(0)) }; else gru_result = { tensor5(shape5(1, 1, 1, 1, n_units), float_type(0)) }; for (EigenIndex k = 0; k < EigenIndex(n_timesteps); ++k) { RowVector<Dynamic> r; RowVector<Dynamic> z; RowVector<Dynamic> m; // in the formulae below, the following notations are used: // A b matrix product // a o b Hadamard (element-wise) product // x input vector // h state vector // W_{x,a} block of the kernel weight matrix corresponding to "a" // W_{h,a} block of the recurrent kernel weight matrix corresponding to "a" // b_{x,a} part of the kernel bias vector corresponding to "a" // b_{h,a} part of the recurrent kernel bias corresponding to "a" // z update gate vector // r reset gate vector if (reset_after) { // recurrent kernel applied to timestep (with bias), produces shape (1, n_units * 3) RowMajorMatrix<1, Dynamic> Uh = h * U; Uh += b_h; // z = sigmoid(W_{x,z} x + b_{i,z} + W_{h,z} h + b_{h,z}) z = (Wx.block(k, 0 * n, 1, n) + Uh.block(0, 0 * n, 1, n)).unaryExpr(act_func_recurrent); // r = sigmoid(W_{x,r} x + b_{i,r} + W_{h,r} h + b_{h,r}) r = (Wx.block(k, 1 * n, 1, n) + Uh.block(0, 1 * n, 1, n)).unaryExpr(act_func_recurrent); // m = tanh(W_{x,m} x + b_{i,m} + r * (W_{h,m} h + b_{h,m})) m = (Wx.block(k, 2 * n, 1, n) + (r.array() * Uh.block(0, 2 * n, 1, n).array()).matrix()).unaryExpr(act_func); } else { // z = sigmoid(W_{x,z} x + b_{x,z} + W_{h,z} h + b_{h,z}) z = (Wx.block(k, 0 * n, 1, n) + h * U.block(0, 0 * n, n, n) + b_h.block(0, 0 * n, 1, n)).unaryExpr(act_func_recurrent); // r = sigmoid(W_{x,r} x + b_{x,r} + W_{h,r} h + b_{h,r}) r = (Wx.block(k, 1 * n, 1, n) + h * U.block(0, 1 * n, n, n) + b_h.block(0, 1 * n, 1, n)).unaryExpr(act_func_recurrent); // m = tanh(W_{x,m} x + b_{x,m} + W_{h,m} (r o h) + b_{h,m})) m = (Wx.block(k, 2 * n, 1, n) + (r.array() * h.array()).matrix() * U.block(0, 2 * n, n, n) + b_h.block(0, 2 * n, 1, n)).unaryExpr(act_func); } // output vector: h' = (1 - z) o m + z o h h = ((1 - z.array()) * m.array() + z.array() * h.array()).matrix(); if (return_sequences) for (EigenIndex idx = 0; idx < n; ++idx) gru_result.front().set(0, 0, 0, std::size_t(k), std::size_t(idx), h(idx)); else if (k == EigenIndex(n_timesteps) - 1) for (EigenIndex idx = 0; idx < n; ++idx) gru_result.front().set(0, 0, 0, 0, std::size_t(idx), h(idx)); } return gru_result; }