Verilog-NN : Long Short Term Memory : LSTM

 

module LSTMCell(input clk,
                input [3:0] input_data,
                input [3:0] prev_hidden_state,
                input [3:0] prev_cell_state,
                output reg [3:0] next_hidden_state,
                output reg [3:0] next_cell_state);
  
  reg [3:0] forget_gate, input_gate, output_gate, cell_input;
  
  // weights for inputs
  parameter w0_f = 0.3;
  parameter w1_f = 0.4;
  parameter w2_f = 0.2;
  parameter w0_i = 0.5;
  parameter w1_i = 0.7;
  parameter w2_i = 0.9;
  parameter w0_o = 0.1;
  parameter w1_o = 0.3;
  parameter w2_o = 0.5;
  parameter w0_c = 0.2;
  parameter w1_c = 0.4;
  parameter w2_c = 0.6;
  
  // biases
  parameter b_f = 0.1;
  parameter b_i = 0.2;
  parameter b_o = 0.3;
  parameter b_c = 0.4;
  
  always @(posedge clk) begin
    forget_gate <= w0_f * input_data + w1_f * prev_hidden_state + w2_f * prev_cell_state + b_f;
    input_gate <= w0_i * input_data + w1_i * prev_hidden_state + w2_i * prev_cell_state + b_i;
    output_gate <= w0_o * input_data + w1_o * prev_hidden_state + w2_o * prev_cell_state + b_o;
    cell_input <= w0_c * input_data + w1_c * prev_hidden_state + w2_c * prev_cell_state + b_c;
    
    // apply activation functions (sigmoid)
    forget_gate <= 1 / (1 + exp(-forget_gate));
    input_gate <= 1 / (1 + exp(-input_gate));
    output_gate <= 1 / (1 + exp(-output_gate));
    cell_input <= tanh(cell_input);
    
    next_cell_state <= forget_gate * prev_cell_state + input_gate * cell_input;
    next_hidden_state <= output_gate * tanh(next_cell_state);
  end
endmodule

 

https://towardsdatascience.com/the-mostly-complete-chart-of-neural-networks-explained-3fb6f2367464

The mostly complete chart of Neural Networks, explained

 

 

module lstm_cell (
  input clk,
  input rst,
  input [15:0] x,
  input [15:0] h_prev,
  input [15:0] c_prev,
  output [15:0] h,
  output [15:0] c
);
  reg [15:0] i_t, f_t, o_t, g_t, s_t;
  reg [15:0] i_t_temp, f_t_temp, o_t_temp, g_t_temp, s_t_temp;
  reg [15:0] c_temp;
  reg [15:0] h_temp;

  // Gates
  assign i_t_temp = x + h_prev;
  assign f_t_temp = x + h_prev;
  assign o_t_temp = x + h_prev;
  assign g_t_temp = x + h_prev;

  // Sigmoid Non-Linearity
  always @(posedge clk or posedge rst) begin
    if (rst) begin
      i_t <= 16'h0000;
      f_t <= 16'h0000;
      o_t <= 16'h0000;
      g_t <= 16'h0000;
    end else begin
      i_t <= i_t_temp;
      f_t <= f_t_temp;
      o_t <= o_t_temp;
      g_t <= g_t_temp;
    end
  end

  // LSTM Cell
  always @(posedge clk or posedge rst) begin
    if (rst) begin
      s_t <= 16'h0000;
      c <= 16'h0000;
      h <= 16'h0000;
    end else begin
      s_t_temp <= c_prev * f_t + g_t * i_t;
      c_temp <= s_t_temp;
      h_temp <= o_t * c_temp;
      s_t <= s_t_temp;
      c <= c_temp;
      h <= h_temp;
    end
  end
endmodule