Updated gitignore

.gitignore

@@ -1,10 +1,5 @@
 # ---> VisualStudioCode
 .vscode/*
-!.vscode/settings.json
-!.vscode/tasks.json
-!.vscode/launch.json
-!.vscode/extensions.json
-!.vscode/*.code-snippets
 
 # Local History for Visual Studio Code
 .history/
@@ -12,3 +7,10 @@
 # Built Visual Studio Code Extensions
 *.vsix
 
+layer.exe
+main.exe
+network.exe
+build/Debug/main.o
+build/Debug/outDebug.exe
+matrices.exe
+tempCodeRunnerFile.cpp

layer.h

@@ -0,0 +1,111 @@
+#ifndef LAYER_H_
+#define LAYER_H_
+
+#include "matrices.h"
+#include <cassert>
+#include <math.h>
+
+#define assertm(exp, msg) assert((void(msg), exp))
+
+class Layer {
+    public:
+    Matrix input;
+    Matrix weights;
+    Matrix raw_output;
+    Matrix activated_output;
+    Matrix biases;
+
+    // Planning for back propagation
+    // Each layer needs the derivative of Z with respect to W, derivative of A with respect to Z and derivative of loss with respect to A
+    // Let's call them dzw, daz and dca
+    Matrix daz;
+
+    static inline float Sigmoid(float);
+    static inline float SigmoidPrime(float);
+
+    inline void Forward(); // Forward Pass with sigmoid
+    inline void Forward(float (*activation)(float)); // Forward Pass with custom activation function
+
+    inline void BackPropagate(Matrix);
+    inline void BackPropagate(Matrix, Matrix, float (*activation)(float)); // To backpropagate, we need the derivative of loss with respect to A and the derivative of used activation function
+
+    inline void Feed(Matrix);
+
+    // Constructors
+    // Input size, Size
+    Layer(int, int);
+    Layer();
+};
+
+void Layer::BackPropagate(Matrix dzw, Matrix dca, float (*derivative)(float)){
+    // Calculate daz ; derivative of activation function
+    this->daz = this->activated_output.Function(derivative);
+    // this->daz.Print("daz");
+
+    // We need to transpose dzw and extend down
+    // dzw.Print("dzw");
+    dzw = dzw.Transpose().ExtendDown(dca.values.size());
+    // dzw.Print("dzw extended transposed");
+
+    Matrix dcw = this->daz.Hadamard(&dca).ExtendRight(this->input.values.size());
+    // dcw.Print("daz . dca");
+    dcw = dcw.Hadamard(&dzw);
+    // dcw.Print("daz . dca . dzw : DCW");
+
+    // this->weights.Print("weights");
+
+    // Apply dcw to weights
+    float learning_rate = 0.1F;
+    Matrix reduced_dcw = dcw.Multiply(learning_rate);
+    // We SUBSTRACT the derivative of loss with respect to the weights.
+    this->weights = this->weights.Substract(&reduced_dcw);
+    // this->weights.Print("New weights");
+}
+
+Layer::Layer(){
+
+}
+
+Layer::Layer(int input_size, int size){
+    this->input = Matrix(input_size, 1);
+
+    // Every neuron has a weight for every input
+    this->weights = Matrix(size, input_size);
+    this->weights.Randomize(-1.0F, 1.0F);
+
+    this->raw_output = Matrix(size, 1);
+    this->activated_output = this->raw_output;
+
+    // One bias per neuron
+    this->biases = Matrix(size, 1);
+    this->biases.Randomize(-1.0F, 1.0F);
+}
+
+void Layer::Feed(Matrix a){
+    this->input = a;
+}
+
+float Layer::Sigmoid(float x){
+    return 1 / (1 + exp(-x));
+}
+
+float Layer::SigmoidPrime(float x){
+    float buffer = Layer::Sigmoid(x);
+    return buffer * (1 - buffer);
+}
+
+void Layer::Forward(float (*activation)(float)){
+    // Multiply weight matrix by input matrix
+    // W x I + B = Z
+    this->raw_output = this->weights.Multiply(&this->input).Add(&this->biases);
+
+    // Now through activation function
+    // A = F(Z)
+    this->activated_output = this->raw_output.Function(activation);
+}
+
+void Layer::Forward(){
+    this->Forward(&Layer::Sigmoid);
+}
+
+#endif

main.cpp

@@ -0,0 +1,8 @@
+#include <iostream>
+#include "layer.h"
+
+int main(){
+    
+
+    return 0;
+}

matrices.h

@@ -0,0 +1,292 @@
+#ifndef MATRICES_H_
+#define MATRICES_H_
+
+#include <string>
+#include <vector>
+#include <cassert>
+#include <iostream>
+
+#define assertm(exp, msg) assert((void(msg), exp))
+
+class Matrix{
+    public:
+        std::vector<std::vector<float>> values;
+        inline void Randomize();
+        inline void Randomize(float, float);
+
+        inline void Set(float);
+
+        inline Matrix Swap(const Matrix*);
+
+        inline Matrix Multiply(float);
+        inline Matrix Multiply(const Matrix*);
+
+        inline Matrix Hadamard(const Matrix*);
+
+        inline Matrix Add(float);
+        inline Matrix Add(const Matrix*);
+
+        inline Matrix Substract(float);
+        inline Matrix Substract(const Matrix*);
+
+        inline Matrix Function(float (*f)(float));
+
+        inline Matrix ExtendRight(int);
+        inline Matrix ExtendDown(int);
+
+        inline void Print(std::string_view);
+
+        inline Matrix Transpose();
+
+        // Operators
+        inline Matrix();
+        inline Matrix operator=(const Matrix*);
+        inline Matrix operator+(const Matrix*);
+        inline Matrix operator-(const Matrix*);
+        inline Matrix operator*(const Matrix*);
+
+        inline Matrix operator+(float);
+        inline Matrix operator-(float);
+        inline Matrix operator*(float);
+
+        // Constructors
+        inline Matrix(int, int);
+        inline Matrix(const Matrix*);
+};
+
+Matrix::Matrix(){
+    
+}
+
+Matrix Matrix::ExtendRight(int new_size){
+    // Extend the matrix to the right
+    Matrix result(this->values.size(), new_size);
+    for(int n = 0; n < result.values.size(); n++){
+        for(int m = 0; m < result.values[n].size(); m++){
+            result.values[n][m] = this->values[n][0];
+        }
+    }
+    return result;
+}
+
+Matrix Matrix::ExtendDown(int new_size){
+    // Extend the matrix down
+    Matrix result(new_size, this->values[0].size());
+    for(int n = 0; n < result.values.size(); n++){
+        for(int m = 0; m < result.values[n].size(); m++){
+            result.values[n][m] = this->values[0][m];
+        }
+    }
+    return result;
+}
+
+Matrix Matrix::operator=(const Matrix* other){
+    return this->Swap(other);
+}
+
+Matrix Matrix::operator+(const Matrix* other){
+    return this->Add(other);
+}
+
+Matrix Matrix::operator-(const Matrix* other){
+    return this->Substract(other);
+}
+
+Matrix Matrix::operator*(const Matrix* other){
+    return this->Multiply(other);
+}
+
+Matrix Matrix::operator+(float value){
+    return this->Add(value);
+}
+
+Matrix Matrix::operator-(float value){
+    return this->Substract(value);
+}
+
+Matrix Matrix::operator*(float value){
+    return this->Multiply(value);
+}
+
+Matrix Matrix::Function(float (*f)(float)){
+    Matrix result = this;
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            // Execute function on every value
+            result.values[m][n] = f(result.values[m][n]);
+        }
+    }
+    return result;
+}
+
+// Constructs a zero matrix
+Matrix::Matrix(int rows, int cols){
+    for(int m = 0; m < rows; m++){
+        std::vector<float> buffer = {};
+        for(int n = 0; n < cols; n++){
+            buffer.push_back(0.0F);
+        }
+        this->values.push_back(buffer);
+    }
+}
+
+Matrix::Matrix(const Matrix* other){
+    this->values = other->values;
+}
+
+Matrix Matrix::Swap(const Matrix* other){
+    this->values = other->values;
+    return *this;
+}
+
+Matrix Matrix::Hadamard(const Matrix* other){
+    // Matrices need to be the same size
+    assertm(this->values.size() == other->values.size() && 
+        this->values[0].size() == other->values[0].size(),
+        "Matrices need to be the same size");
+    Matrix result = this;
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] = this->values[m][n] * other->values[m][n];
+        }
+    }
+    return result;
+}
+
+// Multiply 2 matrices (AxB = this x other)
+Matrix Matrix::Multiply(const Matrix* other){
+    // Matrices need to be of right size
+    assertm(this->values[0].size() == other->values.size(),"Wrong matrix size");
+
+    // Resulting size is this->M x other->N
+    Matrix result(this->values.size(), other->values[0].size());
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            // Sum multiplications
+            float buffer = 0.0F;
+            for(int i = 0; i < this->values[0].size(); i++){
+                buffer += this->values[m][i] * other->values[i][n];
+            }
+            result.values[m][n] = buffer;
+        }
+    }
+    return result;
+}
+
+// Add 2 matrices
+Matrix Matrix::Add(const Matrix* other){
+    // Matrices need to be the same size
+    assertm(this->values.size() == other->values.size() && 
+        this->values[0].size() == other->values[0].size(),
+        "Wrong matrix size");
+
+    Matrix result = this;
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] += other->values[m][n];
+        }
+    }
+    return result;
+}
+
+// Substract 2 matrices
+Matrix Matrix::Substract(const Matrix* other){
+    // Matrices need to be the same size
+    assertm(this->values.size() == other->values.size() && 
+        this->values[0].size() == other->values[0].size(),
+        "Wrong matrix size");
+
+    Matrix result = this;
+
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] -= other->values[m][n];
+        }
+    }
+
+    return result;
+}
+
+// Print a matrix in terminal, with a title
+void Matrix::Print(std::string_view title){
+    std::cout << title << std::endl;
+    for(int m = 0; m < values.size(); m++){
+        std::cout << '|';
+        for(int n = 0; n < values[m].size(); n++){
+            std::cout << values[m][n] << "|";
+        }
+        std::cout << std::endl;
+    }
+}
+
+// Add a constant value to every matrix case
+Matrix Matrix::Add(float value){
+    Matrix result = this;
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] += value;
+        }
+    }
+    return result;
+}
+
+// Substract a constant value to every matrix case
+Matrix Matrix::Substract(float value){
+    Matrix result = this;
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] -= value;
+        }
+    }
+    return result;
+}
+
+// Multiply every matrix case by a given factor
+Matrix Matrix::Multiply(float value){
+    Matrix result = this;
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] *= value;
+        }
+    }
+    return result;
+}
+
+// Set a matrix to a given value
+void Matrix::Set(float value){
+    for(int m = 0; m < this->values.size(); m++){
+        for(int n = 0; n < this->values[m].size(); n++){
+            this->values[m][n] = value;
+        }
+    }
+}
+
+// Transpose a matrix
+Matrix Matrix::Transpose(){
+    // Transposed matrix size is inverted
+    Matrix result(this->values[0].size(), this->values.size());
+
+    for(int m = 0; m < result.values.size(); m++){
+        for(int n = 0; n < result.values[m].size(); n++){
+            result.values[m][n] = this->values[n][m];
+        }
+    }
+    
+    return result;
+}
+
+// Randomize a matrix from 0.0F to 10.0F
+void Matrix::Randomize(){
+    this->Randomize(0.0F, 10.0F);
+}
+
+// Randomize a matrix from min to max
+void Matrix::Randomize(float min, float max){
+    for(int m = 0; m < this->values.size(); m++){
+        for(int n = 0; n < this->values[m].size(); n++){
+            this->values[m][n] = min + ((float)rand()/(float)(RAND_MAX)) * (max - min);
+        }
+    }
+}
+
+#endif

network.h

@@ -0,0 +1,69 @@
+#include "layer.h"
+#include "matrices.h"
+#include <vector>
+
+class Network {
+    public:
+    Matrix input;
+    float (*activation)(float) = Layer::Sigmoid; // Activation function is sigmoid by default
+
+    std::vector<Layer> hidden_layers;
+    Layer output_layer;
+
+    inline void Feed(Matrix);
+    inline Matrix GetOutput();
+
+    inline void Forward();
+
+    inline void BackPropagate(Matrix);
+
+    // Constructors
+    // Input size, Array of hidden sizes, Output size
+    Network(int, std::vector<int>, int);
+};
+
+void Network::BackPropagate(Matrix target){
+    // Calculate derivative of loss in respect to A (dca) for output layer
+    // loss = (A - Y)^2
+    // derivative = 2(A - Y)
+    Matrix loss = this->output_layer.activated_output.Substract(&target);
+    loss = loss.Hadamard(&loss);
+    // loss.Print("Loss");
+    Matrix dca = this->output_layer.activated_output.Substract(&target);
+    dca = dca.Multiply(2.0F);
+    // dca.Print("DCA");
+
+    this->output_layer.BackPropagate(this->hidden_layers[this->hidden_layers.size() - 1].activated_output, dca, &Layer::SigmoidPrime);
+}
+
+Network::Network(int input_size, std::vector<int> hidden_sizes, int output_size){
+    this->input = Matrix(input_size, 1);
+
+    this->hidden_layers.push_back(Layer(input_size, hidden_sizes[0]));
+    for(int i = 1; i < hidden_sizes.size(); i++){
+        // For every hidden layer, create a layer of specified size
+        this->hidden_layers.push_back(Layer(hidden_sizes[i-1], hidden_sizes[i]));
+    }
+    this->output_layer = Layer(hidden_sizes[hidden_sizes.size() - 1], output_size);
+}
+
+Matrix Network::GetOutput(){
+    return this->output_layer.activated_output;
+}
+
+void Network::Feed(Matrix a){
+    this->input = a;
+}
+
+void Network::Forward(){
+    // Feeding first layer
+    this->hidden_layers[0].Feed(this->input);
+    this->hidden_layers[0].Forward();
+    for(int i = 1; i < this->hidden_layers.size(); i++){
+        // Feeding A(L-1) and forwarding
+        this->hidden_layers[i].Feed(this->hidden_layers[i - 1].activated_output);
+        this->hidden_layers[i].Forward();
+    }
+    this->output_layer.Feed(this->hidden_layers[this->hidden_layers.size() - 1].activated_output);
+    this->output_layer.Forward();
+}