303 lines
7.1 KiB
Go
303 lines
7.1 KiB
Go
package nn
|
|
|
|
import (
|
|
"gonum.org/v1/gonum/mat"
|
|
"math/rand"
|
|
"time"
|
|
)
|
|
|
|
type Network struct {
|
|
layers []uint // number of neurons of each layer
|
|
|
|
fed bool // whether the network has the state fed by a forward pass
|
|
|
|
neurons []*mat.Dense // neuron value vector for each layer (size=L)
|
|
biases []*mat.Dense // neuron bias vector for each layer (size=L-1)
|
|
weights []*mat.Dense // weights between each 2-layers (size=L-1)
|
|
}
|
|
|
|
const MaxLayerCount = 255
|
|
const LearningRate = 1
|
|
|
|
// Empty creates a new network where each argument is the number of neurons of a layer
|
|
// each param represents a layer, including input/output
|
|
func Empty(_layers ...uint) (*Network, error) {
|
|
|
|
var L = len(_layers)
|
|
// check size
|
|
if L < 2 {
|
|
return nil, ErrMissingLayers
|
|
} else if L > MaxLayerCount {
|
|
return nil, ErrTooMuchLayers
|
|
}
|
|
|
|
net := &Network{
|
|
layers: _layers,
|
|
fed: false,
|
|
neurons: make([]*mat.Dense, 0),
|
|
biases: make([]*mat.Dense, 0),
|
|
weights: make([]*mat.Dense, 0),
|
|
}
|
|
|
|
for i, layer := range _layers {
|
|
|
|
// check neuron count
|
|
if layer < 1 {
|
|
return nil, ErrEmptyLayer
|
|
}
|
|
|
|
// create neurons
|
|
net.neurons = append(net.neurons, mat.NewDense(int(layer), 1, nil))
|
|
|
|
// do not create weights nor biases for first layer
|
|
// (no previous layer to bound to)
|
|
if i == 0 {
|
|
continue
|
|
}
|
|
|
|
// create biases
|
|
biases := make([]float64, 0, layer+1)
|
|
for b := uint(0); b < layer; b++ {
|
|
rand.Seed(time.Now().UnixNano())
|
|
biases = append(biases, rand.Float64())
|
|
}
|
|
biasesVec := mat.NewDense(int(layer), 1, biases)
|
|
net.biases = append(net.biases, biasesVec)
|
|
|
|
rows, cols := int(layer), int(_layers[i-1])
|
|
weights := make([]float64, 0, rows*cols+1)
|
|
for v := 0; v < rows*cols; v++ {
|
|
rand.Seed(time.Now().UnixNano())
|
|
weights = append(weights, rand.Float64())
|
|
}
|
|
weightsMat := mat.NewDense(rows, cols, weights)
|
|
net.weights = append(net.weights, weightsMat)
|
|
|
|
}
|
|
|
|
return net, nil
|
|
|
|
}
|
|
|
|
// reset neuron values
|
|
func (net *Network) reset() {
|
|
net.fed = false
|
|
|
|
for i, _ := range net.neurons {
|
|
net.neurons[i] = mat.NewDense(int(net.layers[i]), 1, nil)
|
|
}
|
|
}
|
|
|
|
// forward processes a forward propagation from an input vector
|
|
// and lets the network in the final processing state
|
|
func (net *Network) forward(_input ...float64) error {
|
|
|
|
// check input size
|
|
if len(_input) < net.neurons[0].ColView(0).Len() {
|
|
return ErrMissingInput
|
|
}
|
|
// reset neuron values
|
|
net.reset()
|
|
|
|
// forward input to first layer
|
|
for n, l := 0, net.neurons[0].ColView(0).Len(); n < l; n++ {
|
|
net.neurons[0].Set(n, 0, _input[n])
|
|
}
|
|
|
|
// process each layer from the previous one
|
|
for l, ll := 1, len(net.layers); l < ll; l++ {
|
|
|
|
// Z = w^l . a^(l-1) + b^l
|
|
z := net.neurons[l]
|
|
|
|
a := net.neurons[l-1] // neurons of previous layer
|
|
w := net.weights[l-1] // shifted by 1 because no weights between layers -1 and 0
|
|
b := net.biases[l-1] // same shift as weights
|
|
|
|
z.Mul(w, a)
|
|
z.Add(z, b)
|
|
z.Apply(sigmoid, z)
|
|
}
|
|
|
|
net.fed = true
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Cost returns the cost from the given output
|
|
func (net *Network) Cost(_expect ...float64) (float64, error) {
|
|
|
|
costVec, err := net.costVec(_expect...)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return mat.Sum(costVec), nil
|
|
}
|
|
|
|
// costVec returns the cost derivative for each output (as a vector)
|
|
// from the given _expect data
|
|
func (net *Network) costVec(_expect ...float64) (*mat.Dense, error) {
|
|
|
|
if !net.fed {
|
|
return nil, ErrNoState
|
|
}
|
|
|
|
out := net.neurons[len(net.neurons)-1]
|
|
|
|
// check output size
|
|
if len(_expect) < out.ColView(0).Len() {
|
|
return nil, ErrMissingOutput
|
|
}
|
|
|
|
// build expect vector
|
|
expect := mat.NewDense(len(_expect), 1, _expect)
|
|
|
|
// process cost = 1/2 * learningRate * (out - expect)^2
|
|
cost := new(mat.Dense)
|
|
cost.Sub(out, expect) // out - expect
|
|
cost.MulElem(cost, cost) // (out - expect)^2
|
|
cost.Mul(cost, mat.NewDense(1, 1, []float64{0.5 * LearningRate})) // 1/2 *learningRate * (out - expect)^2
|
|
|
|
return cost, nil
|
|
}
|
|
|
|
// errorVec returns the cost derivative (also called ERROR) for each
|
|
// output (as a vector) from the given _expect data
|
|
func (net *Network) errorVec(_expect ...float64) (*mat.Dense, error) {
|
|
|
|
if !net.fed {
|
|
return nil, ErrNoState
|
|
}
|
|
|
|
outLayer := net.neurons[len(net.neurons)-1]
|
|
|
|
// check output size
|
|
if len(_expect) < outLayer.ColView(0).Len() {
|
|
return nil, ErrMissingOutput
|
|
}
|
|
|
|
// build expect vector
|
|
expect := mat.NewDense(len(_expect), 1, _expect)
|
|
|
|
// calc cost derivative = 2 * learningRate * (expect - out)
|
|
cost := new(mat.Dense)
|
|
cost.Sub(expect, outLayer)
|
|
cost.Mul(cost, mat.NewDense(1, 1, []float64{2 * LearningRate}))
|
|
|
|
// return diff (derivative of cost)
|
|
return cost, nil
|
|
}
|
|
|
|
// backward processes the backpropagation from the current network state
|
|
// and the expected data : _expect
|
|
func (net *Network) backward(_expect ...float64) error {
|
|
|
|
out := net.neurons[len(net.neurons)-1]
|
|
|
|
// 1. fail on no state (no forward pass applied first)
|
|
if !net.fed {
|
|
return ErrNoState
|
|
}
|
|
|
|
// fail on invalid _expect size
|
|
if len(_expect) != out.ColView(0).Len() {
|
|
return ErrMissingOutput
|
|
}
|
|
|
|
// calc ERROR = 0.5 * learningRate * (expect - out)
|
|
// *it is in fact the cost derivative
|
|
errors, err := net.errorVec(_expect...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// FOR EACH LAYER (from last to 1)
|
|
for l := len(net.layers) - 1; l > 0; l-- {
|
|
|
|
neurons := net.neurons[l]
|
|
previous := net.neurons[l-1]
|
|
weights := net.weights[l-1] // from l-1 to l
|
|
biases := net.biases[l-1] // at l
|
|
|
|
// calc GRADIENTS = sigmoid'( neuron[l-1] )
|
|
gradients := new(mat.Dense)
|
|
gradients.Apply(derivateSigmoid, neurons)
|
|
gradients.MulElem(gradients, errors)
|
|
gradients.Mul(gradients, mat.NewDense(1, 1, []float64{LearningRate}))
|
|
|
|
// calc WEIGHTS DELTAS = gradients . previous^T
|
|
wdeltas := new(mat.Dense)
|
|
wdeltas.Mul(gradients, previous.T())
|
|
|
|
// update weights
|
|
weights.Add(weights, wdeltas)
|
|
|
|
// adjust biases
|
|
biases.Add(biases, gradients)
|
|
|
|
// update ERRORS
|
|
previousErrors := new(mat.Dense)
|
|
previousErrors.Clone(errors)
|
|
errors.Reset()
|
|
errors.Mul(weights.T(), previousErrors)
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Guess uses the trained network to guess output from an input
|
|
func (net *Network) Guess(_input ...float64) ([]float64, error) {
|
|
|
|
// process feed forward
|
|
err := net.forward(_input...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// extract output
|
|
return net.Output()
|
|
|
|
}
|
|
|
|
// Train uses the trained network to train with the _input and tries to learn
|
|
// to guess the _expect instead
|
|
func (net *Network) Train(_input []float64, _expect []float64) error {
|
|
|
|
out := net.neurons[len(net.neurons)-1]
|
|
|
|
// check output size
|
|
if len(_expect) != out.ColView(0).Len() {
|
|
return ErrMissingOutput
|
|
}
|
|
|
|
// process guess subroutine
|
|
_, err := net.Guess(_input...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// process backward propagation
|
|
return net.backward(_expect...)
|
|
|
|
}
|
|
|
|
// Output returns the output data (only if the network has been fed)
|
|
func (net Network) Output() ([]float64, error) {
|
|
|
|
if !net.fed {
|
|
return nil, ErrNoState
|
|
}
|
|
|
|
out := net.neurons[len(net.neurons)-1]
|
|
output := make([]float64, 0, net.layers[len(net.layers)-1])
|
|
for n, l := 0, out.ColView(0).Len(); n < l; n++ {
|
|
output = append(output, out.At(n, 0))
|
|
}
|
|
|
|
return output, nil
|
|
|
|
}
|