DiscriminativePCLayer.h

Go to the documentation of this file.

#pragma once
 
#include <vector>
#include <stdexcept>
#include <random>
#include <memory>
#include <cstdlib>
#include <deepity/utils/Activations.h>
#include <deepity/layers/Layer.h>
#include <deepity/utils/MemoryArena.h>
#include <deepity/utils/AdamOptimizer.h>

 
namespace Deep
{
        class PCNDiagnostics;
        class DiscriminativePCLayer : public Layer
        {
        public:
                DiscriminativePCLayer(int size, int nextSize, int batchSize = 1,
                                      float learningRate = 1e-6, float inferenceRate = 0.1f, float precisionRate = 0.01f, float lmbda = 1e-2f,
                                      void (*act)(float *, size_t) = relu,
                                      void (*dAct)(float *, size_t, bool) = dRelu);

                DiscriminativePCLayer(int size, int nextSize, int batchSize = 1,
                                      float learningRate = 1e-6, float inferenceRate = 0.1f, float precisionRate = 0.01f, float lmbda = 1e-2f,
                                      ActivationType aType = ActivationType::RELU, ActivationType dType = ActivationType::dRELU);

                float CalculateState() noexcept override;

                void UpdateState() noexcept override;

                void UpdateWeights() noexcept override;

                void UpdatePrecision() noexcept;

                void Flush() noexcept override {} // no buffer

                void ResyncLogPrecision() noexcept;

                void ClampState(const std::vector<float> &inputData) noexcept;
                void UnclampState() noexcept;

                float *GetBeliefs() noexcept override { return z; }
                const float *GetErrors() const noexcept override { return e; }

                const float *GetMu() const noexcept { return mu; }
                size_t GetInputSize() const noexcept override { return size; }
                size_t GetOutputSize() const noexcept override { return nextSize; }
                size_t GetBatchSize() const noexcept override { return batchSize; }

                const float *GetWeights() const noexcept { return W; }
                float *GetWeights() noexcept { return W; }
                const float *GetBiases() const noexcept { return b; }
                float *GetBiases() noexcept { return b; }

                const float *GetPrecisions() const { return p; }

                float GetLearningRate() const noexcept { return lr; }
                float GetInferenceRate() const noexcept { return ir; }
                float GetPrecisionRate() const noexcept { return pr; }
                float GetLambda() const noexcept { return lmbda; }

                void SetLearningRate(float lr) noexcept { this->lr = lr; }
                void SetInferenceRate(float ir) noexcept { this->ir = ir; }
                void SetPrecisionRate(float pr) noexcept { this->pr = pr; }
                void SetLambda(float l) noexcept { this->lmbda = l; }
                void SetOptimizer(OptimizerType o) noexcept { this->opt = o; }

                void SetLayerAbove(DiscriminativePCLayer *above) noexcept { layerAbove = above; }
                void SetLayerBelow(DiscriminativePCLayer *below) noexcept { layerBelow = below; }

                void ResetState() noexcept;

                const DiscriminativePCLayer &GetLayerAbove() const noexcept { return *layerAbove; }
                const DiscriminativePCLayer &GetLayerBelow() const noexcept { return *layerBelow; }

                void RandomizeWeights(std::mt19937 &twister) noexcept;

                ActivationType GetActivationType() const noexcept { return To_AType(activation); }
                ActivationType GetDerivativeType() const noexcept { return To_AType(activationDerivative); }

                size_t GetRequiredFloats() const noexcept;

                void BindMemory(MemoryArena &arena);

                void ComputeMuOnly() noexcept;
 
        private:
                std::unique_ptr<MemoryArena> localArena;
                float *W;
                float *b;
                float *e;
                float *z;
                float *p;
                float *log_p;

                float *cachedMu = nullptr;
                bool muCacheValid = false;

                float *zF = nullptr;
                float *zFDeriv = nullptr;
                float *feedbackScratch = nullptr;

                int batchSize;
 
                // @internal --- For inference ---
                float *mu;
                float *dz_dt;
                float *bottom_up;
                // ------

                float lr;
                float ir;
                float pr;
                float lmbda;
                bool isClamped = false;

                OptimizerType opt = OptimizerType::SGD;
                int t = 0;
                float *grad_W = nullptr;
                float *m_W = nullptr;
                float *v_W = nullptr;
                float *grad_b = nullptr;
                float *m_b = nullptr;
                float *v_b = nullptr;

                DiscriminativePCLayer *layerAbove;
                DiscriminativePCLayer *layerBelow;
                ActivationFn activation;
                DerivativeFn activationDerivative;

                ActivationType activationType;
 
                friend class PCNDiagnostics;
        };
 
} // namespace Deep
 
/*
 * MHSA USING PCNs:
 * In a standard Transformer, Multi-Head Self-Attention (MHSA) computes Queries, Keys,
 * and Values in a rigid, one-shot forward pass to route token context. If implemented
 * using a PCN, the attention scores and context vectors would become dynamic latent
 * states (z). When a sequence is input, the attention routing wouldn't be calculated
 * instantly; instead, it would iteratively settle over artificial time. The network
 * would adjust its attention states to minimize the prediction error between the
 * bottom-up token data and the top-down expectations from higher layers. The Q, K,
 * and V projection matrices would serve as the local weights (W) for this mechanism,
 * and would be updated entirely in parallel via local Hebbian rules once the attention
 * routing reached its minimum-energy equilibrium.
 */