SimplePCLayer.h

Go to the documentation of this file.

#pragma once
 
#include <vector>
#include <stdexcept>
#include <random>
#include <memory>
#include <cstdlib>
#include <deepity/utils/Activations.h>
#include <deepity/utils/AdamOptimizer.h>
#include <deepity/layers/Layer.h>
#include <deepity/utils/DeviceMemoryArena.h>
#include <deepity/utils/MemoryArena.h>
#include <deepity/backend/IComputeBackend.h>

 
namespace Deep
{
    class SimplePCNDiagnostics;

    class SimplePCLayer : public Layer
    {
    public:
        SimplePCLayer(size_t size, size_t nextSize, size_t batchSize = 1,
                      float learningRate = 1e-6f, float inferenceRate = 0.1f, float lmbda = 1e-2f,
                      void (*act)(float *, size_t) = relu,
                      void (*dAct)(float *, size_t, bool) = dRelu,
                      IComputeBackend *backend = nullptr);

        SimplePCLayer(size_t size, size_t nextSize, size_t batchSize = 1,
                      float learningRate = 1e-6f, float inferenceRate = 0.1f, float lmbda = 1e-2f,
                      ActivationType aType = ActivationType::RELU, ActivationType dType = ActivationType::dRELU,
                      IComputeBackend *backend = nullptr);

        float CalculateState(bool needEnergy = true) noexcept;
 
        float CalculateState() noexcept override { return CalculateState(true); }

        void UpdateState() noexcept override;

        void UpdateWeights() noexcept override;

        void Flush() noexcept override {}

        void ClampState(const std::vector<float> &inputData) noexcept;
        void UnclampState() noexcept;

        bool IsClamped() const noexcept { return isClamped; }

        float *GetBeliefs() noexcept override { return z; }

        const float *GetMu() const noexcept { return mu; }

        const float *GetErrors() const noexcept override { return e; }
        size_t GetInputSize() const noexcept override { return size; }
        size_t GetOutputSize() const noexcept override { return nextSize; }
        size_t GetBatchSize() const noexcept override { return batchSize; }

        const float *GetWeights() const noexcept { return W; }
        float *GetWeights() noexcept { return W; }
        const float *GetBiases() const noexcept { return b; }
        float *GetBiases() noexcept { return b; }

        float GetLearningRate() const noexcept { return lr; }
        float GetInferenceRate() const noexcept { return ir; }
        float GetLambda() const noexcept { return lmbda; }

        void SetLearningRate(float lr) noexcept;
        void SetInferenceRate(float ir) noexcept { this->ir = ir; }
        void SetLambda(float l) noexcept { this->lmbda = l; }

        void SetOptimizer(const OptimizerType o) noexcept { opt = o; }

        void SetMuCacheThreshold(float threshold) noexcept { muCacheThreshold = threshold; }
        float GetMuCacheThreshold() const noexcept { return muCacheThreshold; }

        void ComputeMuOnly() noexcept;

        void SetLayerAbove(SimplePCLayer *above) noexcept { layerAbove = above; }
        void SetLayerBelow(SimplePCLayer *below) noexcept { layerBelow = below; }

        void ResetState() noexcept;

        const SimplePCLayer &GetLayerAbove() const noexcept { return *layerAbove; }
        const SimplePCLayer &GetLayerBelow() const noexcept { return *layerBelow; }

        void RandomizeWeights(std::mt19937 &twister, const char *distribution = "normal") noexcept;

        ActivationType GetActivationType() const noexcept { return To_AType(activation); }
        ActivationType GetDerivativeType() const noexcept { return To_AType(activationDerivative); }

        size_t GetRequiredFloats() const noexcept;
        template <typename ArenaT>
        void BindMemory(ArenaT &arena);
 
    private:
        std::unique_ptr<MemoryArena> localArena;

        using BackendDeleter = void (*)(IComputeBackend *);
        std::unique_ptr<IComputeBackend, BackendDeleter> backend;
 
        float *W;
        float *b;
        float *e;
        float *z;
 
        int batchSize;
 
        float *mu;
        float *cachedMu;        // separate from mu -- mu gets mutated in-place by
                                // UpdateState() every step (converted to its
                                // derivative), so caching must copy a preserved
                                // value back INTO mu each skipped step, not just
                                // skip writing to mu entirely
        float *zF;              // this layer's OWN activated belief,
                                // phi(z) -- needed both for the forward
                                // GEMM (mu=zF@W+b) AND the weight
                                // gradient (dW uses zF, not raw z),
                                // matching ngc-learn's documented
                                // convention: mu_l = W_l . phi(z_{l-1})
        float *zFDeriv;         // SEPARATE scratch for f'(z), computed
                                // from RAW z (activated=false flag) --
                                // zF itself must stay clean/activated
                                // for the weight-gradient step
        float *feedbackScratch; // own_state_size scratch for the raw
                                // feedback GEMM's output -- the f'(z)
                                // multiply applies ONLY to the feedback
                                // term, not the -e term already in
                                // dz_dt, so it can't accumulate directly
                                // into dz_dt via the GEMM itself
        float lr;
        float ir;
        float lmbda;
        bool isClamped = false;
 
        int *t_device = nullptr;
        float *lr_device = nullptr;
 
        float muCacheThreshold = -1.0f; // -1 = disabled. 0 = exact clamped-only
                                        // (today's validated behavior). >0 =
                                        // approximate, extends to unclamped
                                        // layers too.
        bool muCacheValid = false;      // true once mu computed at least once
                                        // since the most recent ClampState()
 
        SimplePCLayer *layerAbove;
        SimplePCLayer *layerBelow;
        ActivationFn activation;
        DerivativeFn activationDerivative;
        DerivativeFn2 activationDerivativeInto; // fused two-buffer derivative
                                                // (dst, src, n), resolved once
                                                // at construction alongside
                                                // activation/activationDerivative
        ActivationType activationType;
        OptimizerType opt = OptimizerType::SGD;
        // @private Optional Adam weights
        float *grad_W = nullptr;
        float *grad_b = nullptr;
        float *m_W = nullptr;
        float *v_W = nullptr;
        float *m_b = nullptr;
        float *v_b = nullptr;
        int t = 0;
 
        friend class SimplePCNDiagnostics;
    };
 
} // namespace Deep