commit e31b864c32f8190a5ca8e8a1791070f9d82cb454
Author: xdrm-brackets <xdrm.brackets.dev@gmail.com>
Date:   Wed Oct 10 19:11:20 2018 +0200

    network structure (t) | feed-forward (t) | cost calculation (?) | backprop (!)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..c56069f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.test
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..baf57ff
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,8 @@
+MIT License
+Copyright (c) 2018 Adrien Marquès (aka xdrm-brackets)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0110755
--- /dev/null
+++ b/README.md
@@ -0,0 +1,8 @@
+# Go N N
+
+[![Go version](https://img.shields.io/badge/go_version-1.10.3-blue.svg)](https://golang.org/doc/go1.10)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Go Report Card](https://goreportcard.com/badge/git.xdrm.io/go/nn)](https://goreportcard.com/report/git.xdrm.io/go/nn)
+[![Go doc](https://godoc.org/git.xdrm.io/go/nn?status.svg)](https://godoc.org/git.xdrm.io/go/nn)
+
+Another neural-network implementation featuring back-propagation.
\ No newline at end of file
diff --git a/errors.go b/errors.go
new file mode 100644
index 0000000..bb05017
--- /dev/null
+++ b/errors.go
@@ -0,0 +1,24 @@
+package nn
+
+import (
+	"errors"
+)
+
+// ErrMissingLayers is raised when there is less than 3 layers
+var ErrMissingLayers = errors.New("a network needs at least 2 layers")
+
+// ErrMissingInput is raised when there is missing inputs for a network
+var ErrMissingInput = errors.New("missing input data")
+
+// ErrMissingOutput is raised when there is missing outputs for a network
+var ErrMissingOutput = errors.New("missing output data")
+
+// ErrTooMuchLayers is raised when the max number of layers is reached
+var ErrTooMuchLayers = errors.New("too much layers")
+
+// ErrEmptyLayer is raised when there is an empty layer
+var ErrEmptyLayer = errors.New("cannot create an empty layer")
+
+// ErrNoState is raised when trying to apply a stateful process
+// on an empty network (no forward pass applied)
+var ErrNoState = errors.New("you must process a forward pass beforehand")
diff --git a/funcs.go b/funcs.go
new file mode 100644
index 0000000..fd40be8
--- /dev/null
+++ b/funcs.go
@@ -0,0 +1,9 @@
+package nn
+
+import (
+	"math"
+)
+
+var sigmoid = func(i, j int, x float64) float64 { return 1 / (1 + math.Exp(-x)) }
+var sigmoidToDerivative = func(y float64) float64 { return y * (1 - y) }
+var derivateSigmoid = func(i, j int, y float64) float64 { return y * (1 - y) }
diff --git a/network.go b/network.go
new file mode 100644
index 0000000..a98f6c2
--- /dev/null
+++ b/network.go
@@ -0,0 +1,264 @@
+package nn
+
+import (
+	"fmt"
+	"gonum.org/v1/gonum/mat"
+	"math"
+	"math/rand"
+	"time"
+)
+
+type Network struct {
+	layers []uint // number of neurons of each layer
+
+	fed bool // whether the network has the state fed by a forward pass
+
+	Neurons []*mat.VecDense // neuron value vector for each layer (size=L)
+	Biases  []*mat.VecDense // neuron bias vector for each layer (size=L-1)
+	Weights []*mat.Dense    // weights between each 2-layers (size=L-1)
+}
+
+const MaxLayerCount = 255
+const LearningRate = 1
+
+// Empty creates a new network where each argument is the number of neurons of a layer
+// each param represents a layer, including input/output
+func Empty(_layers ...uint) (*Network, error) {
+
+	var L = len(_layers)
+	// check size
+	if L < 2 {
+		return nil, ErrMissingLayers
+	} else if L > MaxLayerCount {
+		return nil, ErrTooMuchLayers
+	}
+
+	net := &Network{
+		layers:  _layers,
+		fed:     false,
+		Neurons: make([]*mat.VecDense, 0),
+		Biases:  make([]*mat.VecDense, 0),
+		Weights: make([]*mat.Dense, 0),
+	}
+
+	for i, layer := range _layers {
+
+		// check neuron count
+		if layer < 1 {
+			return nil, ErrEmptyLayer
+		}
+
+		// create neurons
+		net.Neurons = append(net.Neurons, mat.NewVecDense(int(layer), nil))
+
+		// do not create weights nor biases for first layer
+		// (no previous layer to bound to)
+		if i == 0 {
+			continue
+		}
+
+		// create biases
+		biases := make([]float64, 0, layer+1)
+		for b := uint(0); b < layer; b++ {
+			rand.Seed(time.Now().UnixNano())
+			biases = append(biases, rand.Float64())
+		}
+		biasesVec := mat.NewVecDense(int(layer), biases)
+		net.Biases = append(net.Biases, biasesVec)
+
+		rows, cols := int(layer), int(_layers[i-1])
+		weights := make([]float64, 0, rows*cols+1)
+		for v := 0; v < rows*cols; v++ {
+			rand.Seed(time.Now().UnixNano())
+			weights = append(weights, rand.Float64())
+		}
+		weightsMat := mat.NewDense(rows, cols, weights)
+		net.Weights = append(net.Weights, weightsMat)
+
+	}
+
+	return net, nil
+
+}
+
+// reset neuron values
+func (net *Network) reset() {
+	net.fed = false
+
+	for i, _ := range net.Neurons {
+		net.Neurons[i] = mat.NewVecDense(int(net.layers[i]), nil)
+	}
+}
+
+// Forward processes a forward propagation from an input vector
+// and lets the network in the final processing state
+func (net *Network) Forward(_input ...float64) ([]float64, error) {
+
+	// check input size
+	if len(_input) < net.Neurons[0].Len() {
+		return nil, ErrMissingInput
+	}
+	// reset neuron values
+	net.reset()
+
+	// forward input to first layer
+	for n, l := 0, net.Neurons[0].Len(); n < l; n++ {
+		net.Neurons[0].SetVec(n, _input[n])
+	}
+
+	// process each layer from the previous one
+	for l, ll := 1, len(net.layers); l < ll; l++ {
+
+		// Z = w^l . a^(l-1) + b^l
+		z := new(mat.Dense)
+
+		a := net.Neurons[l-1] // neurons of previous layer
+		w := net.Weights[l-1] // shifted by 1 because no weights between layers -1 and 0
+		b := net.Biases[l-1]  // same shift as weights
+
+		z.Mul(w, a)
+		z.Add(z, b)
+		z.Apply(sigmoid, z)
+
+		// copy values (first line = vector)
+		net.Neurons[l].CloneVec(z.ColView(0))
+	}
+
+	net.fed = true
+
+	// format output
+	outputLayer := net.Neurons[len(net.Neurons)-1]
+	output := make([]float64, 0, net.layers[len(net.layers)-1])
+	for n, l := 0, outputLayer.Len(); n < l; n++ {
+		output = append(output, outputLayer.AtVec(n))
+	}
+
+	return output, nil
+
+}
+
+// Cost returns the cost from the given output
+func (net *Network) Cost(_expect ...float64) (float64, error) {
+
+	outputLayer := net.Neurons[len(net.Neurons)-1]
+
+	// check output size
+	if len(_expect) < outputLayer.Len() {
+		return 0, ErrMissingOutput
+	}
+
+	var Cost float64
+
+	// process cost
+	for n, l := 0, outputLayer.Len(); n < l; n++ {
+		Cost += math.Pow(outputLayer.AtVec(n)-_expect[n], 2) * LearningRate
+	}
+
+	return Cost, nil
+}
+
+// CostDerVec returns the cost derivative for each output (as a vector)
+// from the given _expect data
+func (net *Network) CostDerVec(_expect ...float64) (*mat.VecDense, error) {
+
+	outLayer := net.Neurons[len(net.Neurons)-1]
+
+	// check output size
+	if len(_expect) < outLayer.Len() {
+		return nil, ErrMissingOutput
+	}
+
+	Cost := mat.NewVecDense(outLayer.Len(), nil)
+
+	// process cost
+	for n, expect := range _expect {
+		Cost.SetVec(n, LearningRate*2*(outLayer.AtVec(n)-expect))
+	}
+
+	return Cost, nil
+}
+
+// Backward processes the backpropagation from the current network state
+// and the expected data : _expect
+func (net *Network) Backward(_expect ...float64) error {
+
+	// 0. fail on no state (no forward pass applied first)
+	if !net.fed {
+		return ErrNoState
+	}
+
+	// 1. Prepare receiver network
+	delta, err := Empty(net.layers...)
+	if err != nil {
+		return err
+	}
+
+	// 2. Get cost
+	cost, err := net.CostDerVec(_expect...)
+	if err != nil {
+		return err
+	}
+	// replace delta neuron values with the cost derivative
+	deltaOutLayer := delta.Neurons[len(delta.Neurons)-1]
+	for n, nl := 0, deltaOutLayer.Len(); n < nl; n++ {
+		deltaOutLayer.SetVec(n, cost.AtVec(n))
+	}
+
+	// 3. for each layer (except last)
+	for l := len(net.layers) - 1; l > 0; l-- {
+
+		// process weights/biases between l and (l-1)
+		for prev := 0; prev < int(net.layers[l-1]); prev++ {
+
+			// init sum to get the previous layers' neuron cost derivative
+			prevCostDer := float64(0)
+
+			for cur := 0; cur < int(net.layers[l]); cur++ {
+
+				sigmoidDer := sigmoidToDerivative(net.Neurons[l].AtVec(cur))
+				curCostDer := delta.Neurons[l].AtVec(cur)
+
+				// bias = sigmoid' . (cost derivative of current neuron)
+				if prev == 0 {
+					bias := sigmoidDer
+					bias *= curCostDer
+					delta.Biases[l-1].SetVec(cur, bias)
+				}
+
+				// weight = a^prev . sigmoid' . (cost derivative of current neuron)
+				weight := net.Neurons[l-1].AtVec(prev)
+				weight *= sigmoidDer
+				weight *= curCostDer
+				delta.Weights[l-1].Set(cur, prev, weight)
+
+				// add each weight to derivative of the previous neuron : weight * sigmoid' * (cost derivative of current neuron)
+				prevCostDer += weight * sigmoidDer * curCostDer
+
+			}
+
+			// update previous layer neuron cost derivative
+			delta.Neurons[l-1].SetVec(prev, prevCostDer)
+
+		}
+
+	}
+
+	// 4. Apply backpropagation
+
+	// each bias
+	for b, bias := range net.Biases {
+		bias.SubVec(bias, delta.Biases[b])
+	}
+	// each weight
+	for w, weight := range net.Weights {
+		weight.Sub(weight, delta.Weights[w])
+	}
+
+	outLayer := net.Neurons[len(net.Neurons)-1]
+	for i, l := 0, deltaOutLayer.Len(); i < l; i++ {
+		fmt.Printf("[out.%d.deriv] = %f - %f = %f\n", i, outLayer.AtVec(i), delta.Neurons[len(delta.Neurons)-1].AtVec(i), _expect[i])
+	}
+
+	return nil
+
+}
diff --git a/network_test.go b/network_test.go
new file mode 100644
index 0000000..1af82ff
--- /dev/null
+++ b/network_test.go
@@ -0,0 +1,173 @@
+package nn
+
+import (
+	"gonum.org/v1/gonum/floats"
+	"testing"
+)
+
+func TestEmptyNetworkErrors(t *testing.T) {
+
+	F255 := make([]uint, 255)
+	for i, _ := range F255 {
+		F255[i] = 1
+	}
+	F256 := make([]uint, 256)
+	for i, _ := range F256 {
+		F256[i] = 1
+	}
+
+	tests := []struct {
+		Args []uint
+		Err  error
+	}{
+		{[]uint{}, ErrMissingLayers},
+		{[]uint{1}, ErrMissingLayers},
+		{[]uint{1, 2}, nil},
+		{F255, nil},
+		{F256, ErrTooMuchLayers},
+		{[]uint{1, 1, 0}, ErrEmptyLayer},
+	}
+
+	for _, test := range tests {
+
+		_, err := Empty(test.Args...)
+
+		// unexpected error
+		if test.Err == nil && err != nil {
+			t.Errorf("Unexpected error <%s>", err)
+			continue
+		}
+
+		// expected error
+		if test.Err != nil && err != test.Err {
+			t.Errorf("Expected error <%s> but got <%s>", test.Err, err)
+			continue
+		}
+
+	}
+
+}
+
+func TestEmptyNetworkSizes(t *testing.T) {
+
+	tests := [][]uint{
+		{1, 2, 3},
+		{4, 6, 9, 10},
+		{1, 1, 1},
+	}
+
+	for _, test := range tests {
+
+		net, err := Empty(test...)
+
+		// unexpected error
+		if err != nil {
+			t.Errorf("Unexpected error <%s>", err)
+			continue
+		}
+
+		// 1. Check neuron layer count
+		if len(net.Neurons) != len(test) {
+			t.Errorf("Expected %d layers of neurons, got %d", len(test), len(net.Neurons))
+			continue
+		}
+
+		// 2. Check bias layer count (layers-1)
+		if len(net.Biases) != len(test)-1 {
+			t.Errorf("Expected %d layers of biases, got %d", len(test)-1, len(net.Biases))
+			continue
+		}
+
+		// 3. Check weights layer count (layers-1)
+		if len(net.Weights) != len(test)-1 {
+			t.Errorf("Expected %d layers of weights, got %d", len(test)-1, len(net.Weights))
+			continue
+		}
+
+		// 4. Check each neuron layer count
+		for n, neuron := range net.Neurons {
+			if uint(neuron.Len()) != test[n] {
+				t.Errorf("Expected %d neurons on layer %d, got %d", test[n], n, neuron.Len())
+			}
+		}
+		// 5. Check each bias layer count
+		for b, bias := range net.Biases {
+
+			if uint(bias.Len()) != test[b+1] {
+				t.Errorf("Expected %d biases on layer %d, got %d", test[b+1], b, bias.Len())
+			}
+
+		}
+
+		// 6. Check each weight layer count
+		for w, weight := range net.Weights {
+
+			rows, cols := weight.Dims()
+
+			if uint(rows) != test[w+1] { // invalid rows
+				t.Errorf("Expected %d weights' rows from layer %d to %d, got %d", test[w+1], w, w+1, rows)
+			}
+			if uint(cols) != test[w] { // invalid cols
+				t.Errorf("Expected %d weights' cols from layer %d to %d, got %d", test[w], w, w+1, cols)
+			}
+
+		}
+
+	}
+
+}
+
+func TestForwardPass(t *testing.T) {
+
+	tests := []struct {
+		Layers []uint
+		X      []float64
+	}{
+		{
+			[]uint{2, 2, 1},
+			[]float64{0.5, 0.2},
+		}, {
+			[]uint{4, 5, 10},
+			[]float64{0.5, 0.2, 0.8, 0.4},
+		},
+	}
+
+	for _, test := range tests {
+
+		net, err := Empty(test.Layers...)
+		if err != nil {
+			t.Errorf("Unexpected error <%s>", err)
+			break
+		}
+
+		// apply forward pass
+		_, err = net.Forward(test.X...)
+		if err != nil {
+			t.Errorf("Unexpected error <%s>", err)
+			break
+		}
+
+		// check layer by layer (begin at 1)
+		for l, ll := 1, len(net.layers); l < ll; l++ {
+
+			// each neuron = ( each previous neuron times its weight ) + neuron bias
+			for n, nl := 0, net.Neurons[l].Len(); n < nl; n++ {
+				sum := net.Biases[l-1].AtVec(n)
+
+				// sum each previous neuron*its weight
+				for i, il := 0, net.Neurons[l-1].Len(); i < il; i++ {
+					sum += net.Neurons[l-1].AtVec(i) * net.Weights[l-1].At(n, i)
+				}
+
+				sum = sigmoid(0, 0, sum)
+
+				// check sum
+				if !floats.EqualWithinAbs(net.Neurons[l].AtVec(n), sum, 1e9) {
+					t.Fatalf("Expected neuron %d.%d to be %f, got %f", l, n, sum, net.Neurons[l].AtVec(n))
+				}
+			}
+
+		}
+
+	}
+}