GaussSeidelPCLayer.h

Go to the documentation of this file.

#pragma once
 
#include <vector>
#include <stdexcept>
#include <random>
#include <memory>
#include <cstdlib>
#include <deepity/utils/Activations.h>
#include <deepity/utils/AdamOptimizer.h>
#include <deepity/layers/Layer.h>
#include <deepity/utils/MemoryArena.h>

 
namespace Deep
{
    class GaussSeidelPCNDiagnostics;

    class GaussSeidelPCLayer : public Layer
    {
    public:
        GaussSeidelPCLayer(int size, int nextSize, int batchSize = 1,
                           float learningRate = 1e-6f, float inferenceRate = 0.1f, float lmbda = 1e-2f,
                           void (*act)(float *, size_t) = relu,
                           void (*dAct)(float *, size_t, bool) = dRelu);

        GaussSeidelPCLayer(int size, int nextSize, int batchSize = 1,
                           float learningRate = 1e-6f, float inferenceRate = 0.1f, float lmbda = 1e-2f,
                           ActivationType aType = ActivationType::RELU, ActivationType dType = ActivationType::dRELU);

        float CalculateState() noexcept override
        {
            UpdateState();
            return 0.0f;
        }

        void UpdateState() noexcept override;

        void ComputePrediction() noexcept;

        float ComputeError() noexcept;

        void UpdateWeights() noexcept override;

        void Flush() noexcept override {}

        void ClampState(const std::vector<float> &inputData) noexcept;
        void UnclampState() noexcept;

        float *GetBeliefs() noexcept override { return z; }
        const float *GetMu() const noexcept { return mu; }
        const float *GetErrors() const noexcept override { return e; }
        size_t GetInputSize() const noexcept override { return size; }
        size_t GetOutputSize() const noexcept override { return nextSize; }
        size_t GetBatchSize() const noexcept override { return batchSize; }

        const float *GetWeights() const noexcept { return W; }
        float *GetWeights() noexcept { return W; }
        const float *GetBiases() const noexcept { return b; }
        float *GetBiases() noexcept { return b; }

        float GetLearningRate() const noexcept { return lr; }
        float GetInferenceRate() const noexcept { return ir; }
        float GetLambda() const noexcept { return lmbda; }

        void SetLearningRate(float lr) noexcept { this->lr = lr; }
        void SetInferenceRate(float ir) noexcept { this->ir = ir; }
        void SetLambda(float l) noexcept { this->lmbda = l; }
        void SetOptimizer(const OptimizerType o) noexcept { opt = o; }

        void SetLayerAbove(GaussSeidelPCLayer *above) noexcept { layerAbove = above; }
        void SetLayerBelow(GaussSeidelPCLayer *below) noexcept { layerBelow = below; }

        void ResetState() noexcept;

        const GaussSeidelPCLayer &GetLayerAbove() const noexcept { return *layerAbove; }
        const GaussSeidelPCLayer &GetLayerBelow() const noexcept { return *layerBelow; }

        void RandomizeWeights(std::mt19937 &twister) noexcept;

        ActivationType GetActivationType() const noexcept { return To_AType(activation); }
        ActivationType GetDerivativeType() const noexcept { return To_AType(activationDerivative); }

        size_t GetRequiredFloats() const noexcept;
        void BindMemory(MemoryArena &arena);
 
    private:
        std::unique_ptr<MemoryArena> localArena;
        float *W;
        float *b;
        float *e;
        float *z;

        int batchSize;
 
        float *mu;      // this layer's OWN outgoing prediction -- stays
                        // CLEAN/activated at all times; never mutated
                        // into a derivative in place
        float *muDeriv; // scratch buffer for mu's derivative, computed
                        // fresh whenever needed (UpdateState(),
                        // UpdateWeights()) -- copy mu here, derive in
                        // place, leaving the real mu untouched
        float *dz_dt;
        float *z_deriv;
        float *bottom_up;
        float *E; // feedback-alignment matrix -- SEPARATE from W, same
                  // shape, randomly initialized once, NEVER updated.
                  // Matches ngc-learn's REAL wiring: e2.dmu >> E2.inputs;
                  // E2.outputs >> z1.j, where E2 is an independently
                  // initialized StaticSynapse -- NOT W transposed. This
                  // is feedback alignment (Lillicrap et al.), not
                  // backprop-style transposed-weight feedback.

        float lr;
        float ir;
        float lmbda;
        bool isClamped = false;

        GaussSeidelPCLayer *layerAbove;
        GaussSeidelPCLayer *layerBelow;
        ActivationFn activation;
        DerivativeFn activationDerivative;
        ActivationType activationType;
        OptimizerType opt = OptimizerType::SGD;

        float *grad_W = nullptr;
        float *grad_b = nullptr;
        float *m_W = nullptr;
        float *v_W = nullptr;
        float *m_b = nullptr;
        float *v_b = nullptr;
        int t = 0;
 
        friend class GaussSeidelPCNDiagnostics;
    };
 
} // namespace Deep