brain.js/src/recurrent/gru.js at develop · Makho96/brain.js · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import Matrix from './matrix';
import RandomMatrix from './matrix/random-matrix';
import RNN from './rnn';

export default class GRU extends RNN {
  getModel(hiddenSize, prevSize) {
    return {
      // update Gate
      //wzxh
      updateGateInputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08),
      //wzhh
      updateGateHiddenMatrix: new RandomMatrix(hiddenSize, hiddenSize, 0.08),
      //bz
      updateGateBias: new Matrix(hiddenSize, 1),

      // reset Gate
      //wrxh
      resetGateInputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08),
      //wrhh
      resetGateHiddenMatrix: new RandomMatrix(hiddenSize, hiddenSize, 0.08),
      //br
      resetGateBias: new Matrix(hiddenSize, 1),

      // cell write parameters
      //wcxh
      cellWriteInputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08),
      //wchh
      cellWriteHiddenMatrix: new RandomMatrix(hiddenSize, hiddenSize, 0.08),
      //bc
      cellWriteBias: new Matrix(hiddenSize, 1)
    };
  }

  /**
   *
   * @param {Equation} equation
   * @param {Matrix} inputMatrix
   * @param {Matrix} previousResult
   * @param {Object} hiddenLayer
   * @returns {Matrix}
   */
  getEquation(equation, inputMatrix, previousResult, hiddenLayer) {
    let sigmoid = equation.sigmoid.bind(equation);
    let add = equation.add.bind(equation);
    let multiply = equation.multiply.bind(equation);
    let multiplyElement = equation.multiplyElement.bind(equation);
    let tanh = equation.tanh.bind(equation);
    let allOnes = equation.allOnes.bind(equation);
    let cloneNegative = equation.cloneNegative.bind(equation);

    // update gate
    let updateGate = sigmoid(
      add(
        add(
          multiply(
            hiddenLayer.updateGateInputMatrix,
            inputMatrix
          ),
          multiply(
            hiddenLayer.updateGateHiddenMatrix,
            previousResult
          )
        ),
        hiddenLayer.updateGateBias
      )
    );

    // reset gate
    let resetGate = sigmoid(
        add(
          add(
            multiply(
              hiddenLayer.resetGateInputMatrix,
              inputMatrix
            ),
            multiply(
              hiddenLayer.resetGateHiddenMatrix,
              previousResult
            )
          ),
          hiddenLayer.resetGateBias
        )
    );

    // cell
    let cell = tanh(
      add(
        add(
          multiply(
            hiddenLayer.cellWriteInputMatrix,
            inputMatrix
          ),
          multiply(
            hiddenLayer.cellWriteHiddenMatrix,
            multiplyElement(
              resetGate,
              previousResult
            )
          )
        ),
        hiddenLayer.cellWriteBias
      )
    );

    // compute hidden state as gated, saturated cell activations
    // negate updateGate
    return add(
      multiplyElement(
        add(
          allOnes(updateGate.rows, updateGate.columns),
          cloneNegative(updateGate)
        ),
        cell
      ),
      multiplyElement(
        previousResult,
        updateGate
      )
    );
  }
}