Updated gitignore

.gitignore

@@ -9,3 +9,8 @@
 
 layer.exe
 main.exe
+network.exe
+build/Debug/main.o
+build/Debug/outDebug.exe
+matrices.exe
+tempCodeRunnerFile.cpp

layer.h

@@ -1,3 +1,6 @@
+#ifndef LAYER_H_
+#define LAYER_H_
+
 #include "matrices.h"
 #include <cassert>
 #include <math.h>
@@ -5,25 +8,83 @@
 #define assertm(exp, msg) assert((void(msg), exp))
 
 class Layer {
-    private:
+    public:
     Matrix input;
     Matrix weights;
     Matrix raw_output;
     Matrix activated_output;
     Matrix biases;
 
-    float learning_rate = 0.1;
+    // Planning for back propagation
+    // Each layer needs the derivative of Z with respect to W, derivative of A with respect to Z and derivative of loss with respect to A
+    // Let's call them dzw, daz and dca
+    Matrix daz;
 
     static inline float Sigmoid(float);
     static inline float SigmoidPrime(float);
 
-    public:
-    inline Layer(int); // Number of neurons
-
     inline void Forward(); // Forward Pass with sigmoid
     inline void Forward(float (*activation)(float)); // Forward Pass with custom activation function
+
+    inline void BackPropagate(Matrix);
+    inline void BackPropagate(Matrix, Matrix, float (*activation)(float)); // To backpropagate, we need the derivative of loss with respect to A and the derivative of used activation function
+
+    inline void Feed(Matrix);
+
+    // Constructors
+    // Input size, Size
+    Layer(int, int);
+    Layer();
 };
 
+void Layer::BackPropagate(Matrix dzw, Matrix dca, float (*derivative)(float)){
+    // Calculate daz ; derivative of activation function
+    this->daz = this->activated_output.Function(derivative);
+    // this->daz.Print("daz");
+
+    // We need to transpose dzw and extend down
+    // dzw.Print("dzw");
+    dzw = dzw.Transpose().ExtendDown(dca.values.size());
+    // dzw.Print("dzw extended transposed");
+
+    Matrix dcw = this->daz.Hadamard(&dca).ExtendRight(this->input.values.size());
+    // dcw.Print("daz . dca");
+    dcw = dcw.Hadamard(&dzw);
+    // dcw.Print("daz . dca . dzw : DCW");
+
+    // this->weights.Print("weights");
+
+    // Apply dcw to weights
+    float learning_rate = 0.1F;
+    Matrix reduced_dcw = dcw.Multiply(learning_rate);
+    // We SUBSTRACT the derivative of loss with respect to the weights.
+    this->weights = this->weights.Substract(&reduced_dcw);
+    // this->weights.Print("New weights");
+}
+
+Layer::Layer(){
+
+}
+
+Layer::Layer(int input_size, int size){
+    this->input = Matrix(input_size, 1);
+
+    // Every neuron has a weight for every input
+    this->weights = Matrix(size, input_size);
+    this->weights.Randomize(-1.0F, 1.0F);
+
+    this->raw_output = Matrix(size, 1);
+    this->activated_output = this->raw_output;
+
+    // One bias per neuron
+    this->biases = Matrix(size, 1);
+    this->biases.Randomize(-1.0F, 1.0F);
+}
+
+void Layer::Feed(Matrix a){
+    this->input = a;
+}
+
 float Layer::Sigmoid(float x){
     return 1 / (1 + exp(-x));
 }
@@ -34,9 +95,9 @@ float Layer::SigmoidPrime(float x){
 }
 
 void Layer::Forward(float (*activation)(float)){
-    // Multiply inputs by weights
+    // Multiply weight matrix by input matrix
     // W x I + B = Z
-    this->raw_output = this->input.Multiply(&this->weights).Add(&this->biases);
+    this->raw_output = this->weights.Multiply(&this->input).Add(&this->biases);
 
     // Now through activation function
     // A = F(Z)
@@ -45,4 +106,6 @@ void Layer::Forward(float (*activation)(float)){
 
 void Layer::Forward(){
     this->Forward(&Layer::Sigmoid);
-}
+}
+
+#endif

matrices.h

@@ -1,3 +1,6 @@
+#ifndef MATRICES_H_
+#define MATRICES_H_
+
 #include <string>
 #include <vector>
 #include <cassert>
@@ -6,9 +9,8 @@
 #define assertm(exp, msg) assert((void(msg), exp))
 
 class Matrix{
-    private:
-        std::vector<std::vector<float>> values;
     public:
+        std::vector<std::vector<float>> values;
         inline void Randomize();
         inline void Randomize(float, float);
 
@@ -19,7 +21,7 @@ class Matrix{
         inline Matrix Multiply(float);
         inline Matrix Multiply(const Matrix*);
 
-        inline void Hadamard(const Matrix*);
+        inline Matrix Hadamard(const Matrix*);
 
         inline Matrix Add(float);
         inline Matrix Add(const Matrix*);
@@ -29,11 +31,15 @@ class Matrix{
 
         inline Matrix Function(float (*f)(float));
 
+        inline Matrix ExtendRight(int);
+        inline Matrix ExtendDown(int);
+
         inline void Print(std::string_view);
 
         inline Matrix Transpose();
 
         // Operators
+        inline Matrix();
         inline Matrix operator=(const Matrix*);
         inline Matrix operator+(const Matrix*);
         inline Matrix operator-(const Matrix*);
@@ -48,6 +54,32 @@ class Matrix{
         inline Matrix(const Matrix*);
 };
 
+Matrix::Matrix(){
+    
+}
+
+Matrix Matrix::ExtendRight(int new_size){
+    // Extend the matrix to the right
+    Matrix result(this->values.size(), new_size);
+    for(int n = 0; n < result.values.size(); n++){
+        for(int m = 0; m < result.values[n].size(); m++){
+            result.values[n][m] = this->values[n][0];
+        }
+    }
+    return result;
+}
+
+Matrix Matrix::ExtendDown(int new_size){
+    // Extend the matrix down
+    Matrix result(new_size, this->values[0].size());
+    for(int n = 0; n < result.values.size(); n++){
+        for(int m = 0; m < result.values[n].size(); m++){
+            result.values[n][m] = this->values[0][m];
+        }
+    }
+    return result;
+}
+
 Matrix Matrix::operator=(const Matrix* other){
     return this->Swap(other);
 }
@@ -107,16 +139,18 @@ Matrix Matrix::Swap(const Matrix* other){
     return *this;
 }
 
-void Matrix::Hadamard(const Matrix* other){
+Matrix Matrix::Hadamard(const Matrix* other){
     // Matrices need to be the same size
     assertm(this->values.size() == other->values.size() && 
         this->values[0].size() == other->values[0].size(),
         "Matrices need to be the same size");
-    for(int m = 0; m < this->values.size(); m++){
+    Matrix result = this;
-        for(int n = 0; n < this->values[m].size(); n++){
+    for(int m = 0; m < result.values.size(); m++){
-            this->values[m][n] *= other->values[m][n];
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] = this->values[m][n] * other->values[m][n];
         }
     }
+    return result;
 }
 
 // Multiply 2 matrices (AxB = this x other)
@@ -253,4 +287,6 @@ void Matrix::Randomize(float min, float max){
             this->values[m][n] = min + ((float)rand()/(float)(RAND_MAX)) * (max - min);
         }
     }
-}
+}
+
+#endif

network.h

@@ -0,0 +1,69 @@
+#include "layer.h"
+#include "matrices.h"
+#include <vector>
+
+class Network {
+    public:
+    Matrix input;
+    float (*activation)(float) = Layer::Sigmoid; // Activation function is sigmoid by default
+
+    std::vector<Layer> hidden_layers;
+    Layer output_layer;
+
+    inline void Feed(Matrix);
+    inline Matrix GetOutput();
+
+    inline void Forward();
+
+    inline void BackPropagate(Matrix);
+
+    // Constructors
+    // Input size, Array of hidden sizes, Output size
+    Network(int, std::vector<int>, int);
+};
+
+void Network::BackPropagate(Matrix target){
+    // Calculate derivative of loss in respect to A (dca) for output layer
+    // loss = (A - Y)^2
+    // derivative = 2(A - Y)
+    Matrix loss = this->output_layer.activated_output.Substract(&target);
+    loss = loss.Hadamard(&loss);
+    // loss.Print("Loss");
+    Matrix dca = this->output_layer.activated_output.Substract(&target);
+    dca = dca.Multiply(2.0F);
+    // dca.Print("DCA");
+
+    this->output_layer.BackPropagate(this->hidden_layers[this->hidden_layers.size() - 1].activated_output, dca, &Layer::SigmoidPrime);
+}
+
+Network::Network(int input_size, std::vector<int> hidden_sizes, int output_size){
+    this->input = Matrix(input_size, 1);
+
+    this->hidden_layers.push_back(Layer(input_size, hidden_sizes[0]));
+    for(int i = 1; i < hidden_sizes.size(); i++){
+        // For every hidden layer, create a layer of specified size
+        this->hidden_layers.push_back(Layer(hidden_sizes[i-1], hidden_sizes[i]));
+    }
+    this->output_layer = Layer(hidden_sizes[hidden_sizes.size() - 1], output_size);
+}
+
+Matrix Network::GetOutput(){
+    return this->output_layer.activated_output;
+}
+
+void Network::Feed(Matrix a){
+    this->input = a;
+}
+
+void Network::Forward(){
+    // Feeding first layer
+    this->hidden_layers[0].Feed(this->input);
+    this->hidden_layers[0].Forward();
+    for(int i = 1; i < this->hidden_layers.size(); i++){
+        // Feeding A(L-1) and forwarding
+        this->hidden_layers[i].Feed(this->hidden_layers[i - 1].activated_output);
+        this->hidden_layers[i].Forward();
+    }
+    this->output_layer.Feed(this->hidden_layers[this->hidden_layers.size() - 1].activated_output);
+    this->output_layer.Forward();
+}