void forward_rnn_layer_gpu(layer l, network_state state)
{
network_state s = {0};
s.train = state.train;
int i;
layer input_layer = *(l.input_layer);
layer self_layer = *(l.self_layer);
layer output_layer = *(l.output_layer);
fill_ongpu(l.outputs * l.batch * l.steps, 0, output_layer.delta_gpu, 1);
fill_ongpu(l.hidden * l.batch * l.steps, 0, self_layer.delta_gpu, 1);
fill_ongpu(l.hidden * l.batch * l.steps, 0, input_layer.delta_gpu, 1);
if(state.train) fill_ongpu(l.hidden * l.batch, 0, l.state_gpu, 1);
for (i = 0; i < l.steps; ++i) {
s.input = state.input;
forward_connected_layer_gpu(input_layer, s);
s.input = l.state_gpu;
forward_connected_layer_gpu(self_layer, s);
float *old_state = l.state_gpu;
if(state.train) l.state_gpu += l.hidden*l.batch;
if(l.shortcut){
copy_ongpu(l.hidden * l.batch, old_state, 1, l.state_gpu, 1);
}else{
fill_ongpu(l.hidden * l.batch, 0, l.state_gpu, 1);
}
axpy_ongpu(l.hidden * l.batch, 1, input_layer.output_gpu, 1, l.state_gpu, 1);
axpy_ongpu(l.hidden * l.batch, 1, self_layer.output_gpu, 1, l.state_gpu, 1);
s.input = l.state_gpu;
forward_connected_layer_gpu(output_layer, s);
state.input += l.inputs*l.batch;
increment_layer(&input_layer, 1);
increment_layer(&self_layer, 1);
increment_layer(&output_layer, 1);
}
}
void forward_lstm_layer_gpu(layer l, network state) {
network s = { 0 };
s.train = state.train;
int i;
layer wf = *(l.wf);
layer wi = *(l.wi);
layer wg = *(l.wg);
layer wo = *(l.wo);
layer uf = *(l.uf);
layer ui = *(l.ui);
layer ug = *(l.ug);
layer uo = *(l.uo);
fill_gpu(l.outputs * l.batch * l.steps, 0, wf.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, wi.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, wg.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, wo.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, uf.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, ui.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, ug.delta_gpu, 1, state.st);
fill_gpu(l.outputs * l.batch * l.steps, 0, uo.delta_gpu, 1, state.st);
if (state.train) {
fill_gpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1, state.st);
}
for (i = 0; i < l.steps; ++i) {
s.input_gpu = l.h_gpu;
forward_connected_layer_gpu(wf, s);
forward_connected_layer_gpu(wi, s);
forward_connected_layer_gpu(wg, s);
forward_connected_layer_gpu(wo, s);
s.input_gpu = state.input_gpu;
forward_connected_layer_gpu(uf, s);
forward_connected_layer_gpu(ui, s);
forward_connected_layer_gpu(ug, s);
forward_connected_layer_gpu(uo, s);
copy_gpu(l.outputs * l.batch, wf.output_gpu, 1, l.f_gpu, 1, state.st);
axpy_gpu(l.outputs * l.batch, 1, uf.output_gpu, 1, l.f_gpu, 1, state.st);
copy_gpu(l.outputs * l.batch, wi.output_gpu, 1, l.i_gpu, 1, state.st);
axpy_gpu(l.outputs * l.batch, 1, ui.output_gpu, 1, l.i_gpu, 1, state.st);
copy_gpu(l.outputs * l.batch, wg.output_gpu, 1, l.g_gpu, 1, state.st);
axpy_gpu(l.outputs * l.batch, 1, ug.output_gpu, 1, l.g_gpu, 1, state.st);
copy_gpu(l.outputs * l.batch, wo.output_gpu, 1, l.o_gpu, 1, state.st);
axpy_gpu(l.outputs * l.batch, 1, uo.output_gpu, 1, l.o_gpu, 1, state.st);
activate_array_gpu(l.f_gpu, l.outputs * l.batch, LOGISTIC, state.st);
activate_array_gpu(l.i_gpu, l.outputs * l.batch, LOGISTIC, state.st);
activate_array_gpu(l.g_gpu, l.outputs * l.batch, TANH, state.st);
activate_array_gpu(l.o_gpu, l.outputs * l.batch, LOGISTIC, state.st);
copy_gpu(l.outputs * l.batch, l.i_gpu, 1, l.temp_gpu, 1, state.st);
mul_gpu(l.outputs * l.batch, l.g_gpu, 1, l.temp_gpu, 1, state.st);
mul_gpu(l.outputs * l.batch, l.f_gpu, 1, l.c_gpu, 1, state.st);
axpy_gpu(l.outputs * l.batch, 1, l.temp_gpu, 1, l.c_gpu, 1, state.st);
copy_gpu(l.outputs * l.batch, l.c_gpu, 1, l.h_gpu, 1, state.st);
activate_array_gpu(l.h_gpu, l.outputs * l.batch, TANH, state.st);
mul_gpu(l.outputs * l.batch, l.o_gpu, 1, l.h_gpu, 1, state.st);
copy_gpu(l.outputs * l.batch, l.c_gpu, 1, l.cell_gpu, 1, state.st);
copy_gpu(l.outputs * l.batch, l.h_gpu, 1, l.output_gpu, 1, state.st);
state.input_gpu += l.inputs * l.batch;
l.output_gpu += l.outputs * l.batch;
l.cell_gpu += l.outputs * l.batch;
increment_layer(&wf, 1);
increment_layer(&wi, 1);
increment_layer(&wg, 1);
increment_layer(&wo, 1);
increment_layer(&uf, 1);
increment_layer(&ui, 1);
increment_layer(&ug, 1);
increment_layer(&uo, 1);
}
}
