go
/
neuralnet
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

network structure (t) | feed-forward (t) | cost calculation (?) | backprop (!)

This commit is contained in:
Adrien Marquès 2018-10-10 19:11:20 +02:00
commit e31b864c32
7 changed files with 487 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*.test

8
LICENSE Normal file
View File

@ -0,0 +1,8 @@
MIT License
Copyright (c) 2018 Adrien Marquès (aka xdrm-brackets)
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

8
README.md Normal file
View File

@ -0,0 +1,8 @@
# Go N N
[![Go version](https://img.shields.io/badge/go_version-1.10.3-blue.svg)](https://golang.org/doc/go1.10)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
[![Go Report Card](https://goreportcard.com/badge/git.xdrm.io/go/nn)](https://goreportcard.com/report/git.xdrm.io/go/nn)
[![Go doc](https://godoc.org/git.xdrm.io/go/nn?status.svg)](https://godoc.org/git.xdrm.io/go/nn)
Another neural-network implementation featuring back-propagation.

24
errors.go Normal file
View File

@ -0,0 +1,24 @@
package nn
import (
"errors"
)
// ErrMissingLayers is raised when there is less than 3 layers
var ErrMissingLayers = errors.New("a network needs at least 2 layers")
// ErrMissingInput is raised when there is missing inputs for a network
var ErrMissingInput = errors.New("missing input data")
// ErrMissingOutput is raised when there is missing outputs for a network
var ErrMissingOutput = errors.New("missing output data")
// ErrTooMuchLayers is raised when the max number of layers is reached
var ErrTooMuchLayers = errors.New("too much layers")
// ErrEmptyLayer is raised when there is an empty layer
var ErrEmptyLayer = errors.New("cannot create an empty layer")
// ErrNoState is raised when trying to apply a stateful process
// on an empty network (no forward pass applied)
var ErrNoState = errors.New("you must process a forward pass beforehand")

9
funcs.go Normal file
View File

@ -0,0 +1,9 @@
package nn
import (
"math"
)
var sigmoid = func(i, j int, x float64) float64 { return 1 / (1 + math.Exp(-x)) }
var sigmoidToDerivative = func(y float64) float64 { return y * (1 - y) }
var derivateSigmoid = func(i, j int, y float64) float64 { return y * (1 - y) }

264
network.go Normal file
View File

@ -0,0 +1,264 @@
package nn
import (
"fmt"
"gonum.org/v1/gonum/mat"
"math"
"math/rand"
"time"
)
type Network struct {
layers []uint // number of neurons of each layer
fed bool // whether the network has the state fed by a forward pass
Neurons []*mat.VecDense // neuron value vector for each layer (size=L)
Biases []*mat.VecDense // neuron bias vector for each layer (size=L-1)
Weights []*mat.Dense // weights between each 2-layers (size=L-1)
}
const MaxLayerCount = 255
const LearningRate = 1
// Empty creates a new network where each argument is the number of neurons of a layer
// each param represents a layer, including input/output
func Empty(_layers ...uint) (*Network, error) {
var L = len(_layers)
// check size
if L < 2 {
return nil, ErrMissingLayers
} else if L > MaxLayerCount {
return nil, ErrTooMuchLayers
}
net := &Network{
layers: _layers,
fed: false,
Neurons: make([]*mat.VecDense, 0),
Biases: make([]*mat.VecDense, 0),
Weights: make([]*mat.Dense, 0),
}
for i, layer := range _layers {
// check neuron count
if layer < 1 {
return nil, ErrEmptyLayer
}
// create neurons
net.Neurons = append(net.Neurons, mat.NewVecDense(int(layer), nil))
// do not create weights nor biases for first layer
// (no previous layer to bound to)
if i == 0 {
continue
}
// create biases
biases := make([]float64, 0, layer+1)
for b := uint(0); b < layer; b++ {
rand.Seed(time.Now().UnixNano())
biases = append(biases, rand.Float64())
}
biasesVec := mat.NewVecDense(int(layer), biases)
net.Biases = append(net.Biases, biasesVec)
rows, cols := int(layer), int(_layers[i-1])
weights := make([]float64, 0, rows*cols+1)
for v := 0; v < rows*cols; v++ {
rand.Seed(time.Now().UnixNano())
weights = append(weights, rand.Float64())
}
weightsMat := mat.NewDense(rows, cols, weights)
net.Weights = append(net.Weights, weightsMat)
}
return net, nil
}
// reset neuron values
func (net *Network) reset() {
net.fed = false
for i, _ := range net.Neurons {
net.Neurons[i] = mat.NewVecDense(int(net.layers[i]), nil)
}
}
// Forward processes a forward propagation from an input vector
// and lets the network in the final processing state
func (net *Network) Forward(_input ...float64) ([]float64, error) {
// check input size
if len(_input) < net.Neurons[0].Len() {
return nil, ErrMissingInput
}
// reset neuron values
net.reset()
// forward input to first layer
for n, l := 0, net.Neurons[0].Len(); n < l; n++ {
net.Neurons[0].SetVec(n, _input[n])
}
// process each layer from the previous one
for l, ll := 1, len(net.layers); l < ll; l++ {
// Z = w^l . a^(l-1) + b^l
z := new(mat.Dense)
a := net.Neurons[l-1] // neurons of previous layer
w := net.Weights[l-1] // shifted by 1 because no weights between layers -1 and 0
b := net.Biases[l-1] // same shift as weights
z.Mul(w, a)
z.Add(z, b)
z.Apply(sigmoid, z)
// copy values (first line = vector)
net.Neurons[l].CloneVec(z.ColView(0))
}
net.fed = true
// format output
outputLayer := net.Neurons[len(net.Neurons)-1]
output := make([]float64, 0, net.layers[len(net.layers)-1])
for n, l := 0, outputLayer.Len(); n < l; n++ {
output = append(output, outputLayer.AtVec(n))
}
return output, nil
}
// Cost returns the cost from the given output
func (net *Network) Cost(_expect ...float64) (float64, error) {
outputLayer := net.Neurons[len(net.Neurons)-1]
// check output size
if len(_expect) < outputLayer.Len() {
return 0, ErrMissingOutput
}
var Cost float64
// process cost
for n, l := 0, outputLayer.Len(); n < l; n++ {
Cost += math.Pow(outputLayer.AtVec(n)-_expect[n], 2) * LearningRate
}
return Cost, nil
}
// CostDerVec returns the cost derivative for each output (as a vector)
// from the given _expect data
func (net *Network) CostDerVec(_expect ...float64) (*mat.VecDense, error) {
outLayer := net.Neurons[len(net.Neurons)-1]
// check output size
if len(_expect) < outLayer.Len() {
return nil, ErrMissingOutput
}
Cost := mat.NewVecDense(outLayer.Len(), nil)
// process cost
for n, expect := range _expect {
Cost.SetVec(n, LearningRate*2*(outLayer.AtVec(n)-expect))
}
return Cost, nil
}
// Backward processes the backpropagation from the current network state
// and the expected data : _expect
func (net *Network) Backward(_expect ...float64) error {
// 0. fail on no state (no forward pass applied first)
if !net.fed {
return ErrNoState
}
// 1. Prepare receiver network
delta, err := Empty(net.layers...)
if err != nil {
return err
}
// 2. Get cost
cost, err := net.CostDerVec(_expect...)
if err != nil {
return err
}
// replace delta neuron values with the cost derivative
deltaOutLayer := delta.Neurons[len(delta.Neurons)-1]
for n, nl := 0, deltaOutLayer.Len(); n < nl; n++ {
deltaOutLayer.SetVec(n, cost.AtVec(n))
}
// 3. for each layer (except last)
for l := len(net.layers) - 1; l > 0; l-- {
// process weights/biases between l and (l-1)
for prev := 0; prev < int(net.layers[l-1]); prev++ {
// init sum to get the previous layers' neuron cost derivative
prevCostDer := float64(0)
for cur := 0; cur < int(net.layers[l]); cur++ {
sigmoidDer := sigmoidToDerivative(net.Neurons[l].AtVec(cur))
curCostDer := delta.Neurons[l].AtVec(cur)
// bias = sigmoid' . (cost derivative of current neuron)
if prev == 0 {
bias := sigmoidDer
bias *= curCostDer
delta.Biases[l-1].SetVec(cur, bias)
}
// weight = a^prev . sigmoid' . (cost derivative of current neuron)
weight := net.Neurons[l-1].AtVec(prev)
weight *= sigmoidDer
weight *= curCostDer
delta.Weights[l-1].Set(cur, prev, weight)
// add each weight to derivative of the previous neuron : weight * sigmoid' * (cost derivative of current neuron)
prevCostDer += weight * sigmoidDer * curCostDer
}
// update previous layer neuron cost derivative
delta.Neurons[l-1].SetVec(prev, prevCostDer)
}
}
// 4. Apply backpropagation
// each bias
for b, bias := range net.Biases {
bias.SubVec(bias, delta.Biases[b])
}
// each weight
for w, weight := range net.Weights {
weight.Sub(weight, delta.Weights[w])
}
outLayer := net.Neurons[len(net.Neurons)-1]
for i, l := 0, deltaOutLayer.Len(); i < l; i++ {
fmt.Printf("[out.%d.deriv] = %f - %f = %f\n", i, outLayer.AtVec(i), delta.Neurons[len(delta.Neurons)-1].AtVec(i), _expect[i])
}
return nil
}

173
network_test.go Normal file
View File

@ -0,0 +1,173 @@
package nn
import (
"gonum.org/v1/gonum/floats"
"testing"
)
func TestEmptyNetworkErrors(t *testing.T) {
F255 := make([]uint, 255)
for i, _ := range F255 {
F255[i] = 1
}
F256 := make([]uint, 256)
for i, _ := range F256 {
F256[i] = 1
}
tests := []struct {
Args []uint
Err error
}{
{[]uint{}, ErrMissingLayers},
{[]uint{1}, ErrMissingLayers},
{[]uint{1, 2}, nil},
{F255, nil},
{F256, ErrTooMuchLayers},
{[]uint{1, 1, 0}, ErrEmptyLayer},
}
for _, test := range tests {
_, err := Empty(test.Args...)
// unexpected error
if test.Err == nil && err != nil {
t.Errorf("Unexpected error <%s>", err)
continue
}
// expected error
if test.Err != nil && err != test.Err {
t.Errorf("Expected error <%s> but got <%s>", test.Err, err)
continue
}
}
}
func TestEmptyNetworkSizes(t *testing.T) {
tests := [][]uint{
{1, 2, 3},
{4, 6, 9, 10},
{1, 1, 1},
}
for _, test := range tests {
net, err := Empty(test...)
// unexpected error
if err != nil {
t.Errorf("Unexpected error <%s>", err)
continue
}
// 1. Check neuron layer count
if len(net.Neurons) != len(test) {
t.Errorf("Expected %d layers of neurons, got %d", len(test), len(net.Neurons))
continue
}
// 2. Check bias layer count (layers-1)
if len(net.Biases) != len(test)-1 {
t.Errorf("Expected %d layers of biases, got %d", len(test)-1, len(net.Biases))
continue
}
// 3. Check weights layer count (layers-1)
if len(net.Weights) != len(test)-1 {
t.Errorf("Expected %d layers of weights, got %d", len(test)-1, len(net.Weights))
continue
}
// 4. Check each neuron layer count
for n, neuron := range net.Neurons {
if uint(neuron.Len()) != test[n] {
t.Errorf("Expected %d neurons on layer %d, got %d", test[n], n, neuron.Len())
}
}
// 5. Check each bias layer count
for b, bias := range net.Biases {
if uint(bias.Len()) != test[b+1] {
t.Errorf("Expected %d biases on layer %d, got %d", test[b+1], b, bias.Len())
}
}
// 6. Check each weight layer count
for w, weight := range net.Weights {
rows, cols := weight.Dims()
if uint(rows) != test[w+1] { // invalid rows
t.Errorf("Expected %d weights' rows from layer %d to %d, got %d", test[w+1], w, w+1, rows)
}
if uint(cols) != test[w] { // invalid cols
t.Errorf("Expected %d weights' cols from layer %d to %d, got %d", test[w], w, w+1, cols)
}
}
}
}
func TestForwardPass(t *testing.T) {
tests := []struct {
Layers []uint
X []float64
}{
{
[]uint{2, 2, 1},
[]float64{0.5, 0.2},
}, {
[]uint{4, 5, 10},
[]float64{0.5, 0.2, 0.8, 0.4},
},
}
for _, test := range tests {
net, err := Empty(test.Layers...)
if err != nil {
t.Errorf("Unexpected error <%s>", err)
break
}
// apply forward pass
_, err = net.Forward(test.X...)
if err != nil {
t.Errorf("Unexpected error <%s>", err)
break
}
// check layer by layer (begin at 1)
for l, ll := 1, len(net.layers); l < ll; l++ {
// each neuron = ( each previous neuron times its weight ) + neuron bias
for n, nl := 0, net.Neurons[l].Len(); n < nl; n++ {
sum := net.Biases[l-1].AtVec(n)
// sum each previous neuron*its weight
for i, il := 0, net.Neurons[l-1].Len(); i < il; i++ {
sum += net.Neurons[l-1].AtVec(i) * net.Weights[l-1].At(n, i)
}
sum = sigmoid(0, 0, sum)
// check sum
if !floats.EqualWithinAbs(net.Neurons[l].AtVec(n), sum, 1e9) {
t.Fatalf("Expected neuron %d.%d to be %f, got %f", l, n, sum, net.Neurons[l].AtVec(n))
}
}
}
}
}