//+------------------------------------------------------------------+
//|                                                      ProjectName |
//|                                      Copyright 2020, CompanyName |
//|                                       http://www.companyname.net |
//+------------------------------------------------------------------+
#ifndef NEURONET_DECLARATIONS_MQH
#ifndef NEURONET_DEFINITIONS_MQH
#define NEURONET_BUILDING_FACADE
#include "NeuroNet_Definitions.mqh"
#undef NEURONET_BUILDING_FACADE
#endif // NEURONET_DEFINITIONS_MQH
#define NEURONET_DECLARATIONS_MQH
#ifndef NEURONET_DEFINITIONS_MQH
#include "NeuroNet_Definitions.mqh"
#endif // NEURONET_DEFINITIONS_MQH
//+------------------------------------------------------------------+
//| NeuroNet class declarations in original dependency order
//+------------------------------------------------------------------+
class CConnection : public CObject
  {
public:
   float             weight;        ///< Current weight
   float             deltaWeight;   ///< last delta of weight used in SGD optimization
   float             mt;            ///< First moment in Adam optimization
   float             vt;            ///< Second moment in Adam optimization

   /** Constructor @param[in] w initial weight */ CConnection(float w) { weight = w; deltaWeight = 0; mt = 0; vt = 0; }
   /** Destructor */ ~CConnection() {};
   //--- methods for working with files
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual int       Type(void)   const   {  return defConnect;   }  ///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class CArrayCon
///\brief Array of connections to anothe neuron
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/7447#para51">the link.</A>
//+------------------------------------------------------------------+
class CArrayCon  :    public CArrayObj
  {
public:
   /** Constructor */
                     CArrayCon(void) {};
   /** Destructor */ ~CArrayCon(void) {};
   //---
   virtual bool      CreateElement(int const index); ///< Method for cearing new element by index @param[in] index Index of new element @return logical result of operation
   virtual void      IncreaseTotal()   {  m_data_total++; } ///< Increase number of elements in array
   virtual int       Type(void)  const { return defArrayConnects; }  ///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class CLayer
/// Class of neurons collection in one layer of Neural Net.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/7447#para53">the link.</A>
//+------------------------------------------------------------------+
class CLayer;
class CNeuronBaseOCL;
//+------------------------------------------------------------------+
///\class CNeuronBase
/// The base class of neuron.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/7447#para52">the link.</A>
//+------------------------------------------------------------------+
class CNeuronBase    :  public CObject
  {
protected:
   float             outputVal;  ///< Output value
   float             prevVal;    ///< Previous output value
   uint              m_myIndex;  ///< Index of neuron in layer
   float             gradient;   ///< Current gradient of neuron
   CArrayCon         *Connections;  ///< Array of connections with neurons in next layer
   ENUM_ACTIVATION   activation;    ///< Activation type (#ENUM_ACTIVATION)
   ENUM_OPTIMIZATION optimization;  ///< Optimization method (#ENUM_OPTIMIZATION)
   int               t;             ///< Count of iterations
   //---
   virtual bool      feedForward(CLayer *prevLayer)               {  ReturnFalse;    }   ///< Feed Forward method.@param prevLayer Pointer to previos layer.
   virtual bool      calcHiddenGradients(CLayer *&nextLayer)     {  ReturnFalse;     }   ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer.
   virtual bool      updateInputWeights(CLayer *&prevLayer)       {  ReturnFalse;    }   ///< Method for updating weights.@param prevLayer Pointer to previos layer.
   virtual float     activationFunction(float x);                                        ///< Method to calculate activation function.@param x Input data. @return Result of activation function.
   virtual float     SigmoidFunction(float x)                    {  return (float)MathPow(1 + exp(-x), -1); } ///< Calculating Sigmoid \f$\frac{1}{1+e^x}\f$.@param x Input data.@return Result of calculation
   virtual float     TanhFunction(float x)                       {  return (float)tanh(x);   }              ///< Calculating \f$tanh(x)\f$.@param x Input data.@return Result of calculation
   virtual CLayer    *getOutputLayer(void)                        {  return NULL;      }  ///< Method for getting a pointer to the resulting neural layer. Not used in fully connected neural networks.@return Pointer to layer.
public:
   /** Constructor */
                     CNeuronBase(void);
   /** Destructor */~CNeuronBase(void);
   virtual bool      Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type);  ///< Method of initialization class.@param numOutputs Number of connections to next layer.@param myIndex Index of neuron in layer.@param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) {  activation = value; }      ///< Set the type of activation function (#ENUM_ACTIVATION)
   //---
   //static float            lr;
   static float            alpha;        ///< Multiplier to momentum in SGD optimization
   //---
   virtual void      setOutputVal(float val)                     {  prevVal = outputVal;   outputVal = val;    } ///< Set the output value
   virtual float     getOutputVal()                               {  return outputVal; }                      ///< Return result of feed forward operations.@return Output value
   virtual float     getPrevVal()                                 {  return prevVal;   }                      ///< Return result of feed forward operations at previous iteration.@return Previous output value
   virtual void      setGradient(float val)                      {  gradient = val;     }                    ///< Set gradient value to neuron.
   virtual float     getGradient()                                {  return gradient;  }                      ///< Return gradient of neuron.@return Gradient
   virtual CArrayCon *getConnections()                            {  return Connections;}                     ///< Method to get access to array of connections.@return Pointer to connections array
   virtual float     activationFunctionDerivative(float x);   ///< Calculate derivative of activation function.@param[in] x Input data@return Derivative
   virtual float     SigmoidFunctionDerivative(float x)          {  return x * (1 - x);   } ///< Calculate derivative of Sigmoid function.@param x Input data@return Derivative
   virtual float     TanhFunctionDerivative(float x)             {  return (1 + x) * (1 - x);  } ///< Calculate derivative of \f$tanh(x)\f$.@param x Input data@return Derivative
   //---
   virtual bool      feedForward(CObject *&SourceObject);            ///< Dispatch method for defining the subroutine for Feed Forward process.@param SourceObject Pointer to previos layer.
   virtual bool      calcHiddenGradients(CObject *&TargetObject);    ///< Dispatch method for defining the subroutine for transfer gradient to previous layer.@param TargetObject Pointer to next layer.
   virtual bool      updateInputWeights(CObject *&SourceObject);     ///< Dispatch method for defining the subroutine for updating weights.@param SourceObject Pointer to previos layer.
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle)///< Load method @param[in] file_handle handle of file @return logical result of operation
     {
      activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
      optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
      t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
      return(Connections.Load(file_handle));
     }
   //---
   virtual int       Type(void)        const                      {  return defNeuronBase;                  }///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class CNeuron
/// Class of neuron for full connected layers.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/7447#para52">the link.</A>
//+------------------------------------------------------------------+
class CNeuron  :  public CNeuronBase
  {
private:
   virtual bool      feedForward(CLayer *prevLayer);                          ///< Feed Forward method.@param prevLayer Pointer to previos layer.
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);                 ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer.
   virtual bool      updateInputWeights(CLayer *&prevLayer);                  ///< Method for updating weights.@param prevLayer Pointer to previos layer.

public:
   /** Constructor */
                     CNeuron(void)  {};
   /** Destructor */~CNeuron(void) { Connections.Shutdown(); }
   //---
   virtual bool      calcOutputGradients(float targetVals);                  ///< Method of output gradients calculation.@param targetVals Traget value
   virtual float     sumDOW(CLayer *&nextLayer) ;                             ///< A method for collecting gradients from the next layer.@param[in] nextLayer Pointer to next layer@return Total gradient to neuron.
   virtual int       Type(void)   const   {  return defNeuron;   } ///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class COpenCLMy
/// Class for working with OpenCL
//+------------------------------------------------------------------+
class COpenCLMy   :  public COpenCL
  {
protected:
   uaLongChar              uLocalDimension;
   uaLongChar              uMaxWorkGroup;

public:
   /** Constructor */
                     COpenCLMy(void)   {};
   /** Destructor */~COpenCLMy(void)   { for(int i = 0; i < m_kernels_total; i++) KernelFree(i); }
   bool              Initialize(const string program, const bool show_log = true);
   template<typename T>
   int               AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags);
   ///< Method for creating OpenCL buffer from array.@param data[] Array of data.@param data_array_offset Offset of data in array.@param data_array_count Number of data items in array.@param flags Buffer's properties (CL_MEM_READ_WRITE, CL_MEM_WRITE_ONLY, CL_MEM_READ_ONLY, CL_MEM_ALLOC_HOST_PTR)
   int               AddBuffer(uint size_in_bytes, const uint flags);
   //---
   long              GetMaxLocalSize(const int dimension) const { return uLocalDimension.data[dimension]; }
   long              GetMaxWorkGroupSize(void) const { return uMaxWorkGroup.data[0]; }
   ulong             GetBufferSize(const int buffer_index) const;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CLayer: public CArrayObj
  {
private:
   uint              iOutputs;                                 ///< Number of output connections from 1 neuron to neurons in next layer.
   int               iFileHandle;                              ///< File handle for download result of previous study.
   COpenCLMy         *OpenCL;                                   ///< Class for working with OpenCL

public:
   /** Constructor */
                     CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL);
   ///< @param[in] outputs Number of output connections from 1 neuron to neurons in next layer @param[in] handle File handle for download result of previous study @param[in] OpenCL Pointer to class for working with OpenCL
   /** Destructor */~CLayer(void) {};
   //---
   virtual bool      CreateElement(int const index);           ///< Method for creating new element in layer
   virtual void      IncreaseTotal()   {  m_data_total++;   }  ///< Method for increase number of items in layer
   virtual int       Type(void)  const { return defLayer; }    ///< Identificator of class.@return Type of class
   virtual bool      Load(const int file_handle);              ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual uint      Outputs(void)  { return iOutputs;   }
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);
   virtual bool      WeightsUpdate(CLayer *source, float tau);
   virtual bool      ClearStates(void);
   CObject           *At(const int index) const
     {
      if(index >= m_data_total || (-index) > m_data_total)
         return NULL;
      if(index < 0)
         return(m_data[m_data_total + index]);
      return(m_data[index]);
     }
   virtual CNeuronBaseOCL* operator[](const int index) const { return (CNeuronBaseOCL*)At(index); }
   virtual CObject*        AsObject(void) { return GetPointer(this); }
  };
//+------------------------------------------------------------------+
///\class CArrayLayer
/// Class of layers collection in Neural Net.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/7447#para53">the link.</A>
//+------------------------------------------------------------------+
class CArrayLayer  :    public CArrayObj
  {
public:
   /** Constructor */
                     CArrayLayer(void) {};
   /** Destructor */~CArrayLayer(void) {};
   //---
   virtual bool      CreateElement(uint neurons,  uint outputs);  ///< Method for creating new element
   virtual int       Type(void)  const { return defArrayLayer; }  ///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class CNeuronProof
/// Class of pooling layer
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8234#para42">the link.</A>
//+------------------------------------------------------------------+
class CNeuronProof : public CNeuronBase
  {
protected:
   CLayer            *OutputLayer;                                      ///< Layer of output data. Used for connection with next layer.
   int               iWindow;                                           ///< Input window size
   int               iStep;                                             ///< Size of step

   virtual bool      feedForward(CLayer *prevLayer);                    ///< Feed Forward method.@param prevLayer Pointer to previos layer.
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);           ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer.

public:
   /** Constructor */
                     CNeuronProof(void) {};
   /** Destructor */~CNeuronProof(void);
   virtual bool      Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
   ///< Method of initialization class.@param numOutputs Number of connections to next layer.@param myIndex Index of neuron in layer.@param window Size of input window @param step Step size.@param units_countNumber of neurons.@param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual CLayer    *getOutputLayer(void)  { return OutputLayer;  }    ///< Method for getting a pointer to the resulting neural layer. Not used in fully connected neural networks.@return Pointer to layer.
   virtual bool      calcInputGradients(CLayer *prevLayer);             ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index); ///< Method to transfer gradients to neuron in previous layer @param[in] prevNeuron Pointer to neuron.@param[in] index Index of neuron in previous layer
   //--- methods for working with files
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual int       Type(void)   const   {  return defNeuronProof;   }///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class CNeuronConv
/// Class of convolution layer
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8234#para43">the link.</A>
//+------------------------------------------------------------------+
class CNeuronConv  :  public CNeuronProof
  {
protected:
   float             param;   //PReLU param
   virtual bool      feedForward(CLayer *prevLayer);                    ///< Feed Forward method.@param prevLayer Pointer to previos layer.
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);           ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer.
   virtual float     activationFunction(float x);                      ///< Method to calculate activation function.@param x Input data. @return Result of activation function.
   virtual bool      updateInputWeights(CLayer *&prevLayer);            ///< Method for updating weights.@param prevLayer Pointer to previos layer.
public:
   /** Constructor */
                     CNeuronConv() :   param((float)0.01) { };
   /** Destructor */~CNeuronConv(void)             { };
   //---
   virtual bool      calcInputGradients(CLayer *prevLayer);                  ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index);///< Method to transfer gradients to neuron in previous layer @param[in] prevNeuron Pointer to neuron.@param[in] index Index of neuron in previous layer
   virtual float     activationFunctionDerivative(float x);                 ///< Calculate derivative of activation function.@param[in] x Input data@return Derivative
   virtual int       Type(void)   const   {  return defNeuronConv;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
  };
//+------------------------------------------------------------------+
///\class CLayerDescription
/// Class of layer decription. Used to describe the structure of a neural network from the main program.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8234#para44">the link.</A>
//+------------------------------------------------------------------+
class CLayerDescription    :  public CObject
  {
public:
   /** Constructor */
                     CLayerDescription(void);
   /** Destructor */~CLayerDescription(void) {};
   //---
   uint              type;          ///< Type of neurons in layer (\ref ObjectTypes)
   uint              count;         ///< Number of neurons
   uint              window;        ///< Size of input window
   uint              window_out;    ///< Size of output window
   uint              step;          ///< Step size
   uint              layers;        ///< Layers count
   uint              batch;         ///< Batch Size
   ENUM_ACTIVATION   activation;    ///< Type of activation function (#ENUM_ACTIVATION)
   ENUM_OPTIMIZATION optimization;  ///< Type of optimization method (#ENUM_OPTIMIZATION)
   float             probability;   ///< Probability of neurons shutdown, only Dropout used
   uint              windows[];
   uint              units[];
   uint              heads[];
   vector<float>     radius;
   uint              variables;
   //---
   virtual bool      Copy(CLayerDescription *source);
   //---
   virtual bool      operator= (CLayerDescription *source)  { return Copy(source); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CBufferFloat;
class CNeuronRAGMemory;
//+------------------------------------------------------------------+
///\class CNet
/// The main class of the neural network. Contains basic methods for the functioning of a neural network.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/7447#para53">the link.</A>
//+------------------------------------------------------------------+
class CNet : public CObject
  {
protected:
   bool                    backPropOCL(CBufferFloat *targetVals, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, bool layer_by_layer = false, ENUM_ACTIVATION SecondActivation = None);       ///< Back propagation method for GPU calculation. @param[in] targetVals Target values
   int                     backPropCount;
   float                   recentAverageError;                 ///< Average error
   CArrayLayer            *layers;                             ///< Array of layers
   COpenCLMy              *opencl;                             ///< Class for working with OpenCL
   bool                    bOwnOpenCL;                         ///< This net created and must release OpenCL
   virtual bool            OpenCLInit(void);

public:
   /** Constructor */
                     CNet(void) : bOwnOpenCL(false) { Create(NULL); }
                     CNet(CArrayObj *Description) : bOwnOpenCL(false) { Create(Description); }
   bool                    Create(CArrayObj *Description);
   /** Destructor */      ~CNet(void);
   bool                    feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true, CBufferFloat *SecondInput = NULL); ///< Feed Forward method.@param[in] prevLayer Pointer to previos layer. @param[in] window Window of input data. @param[in] tem Use Time Embedding.
   bool                    feedForward(CNet *inputNet, int inputLayer = -1, CBufferFloat *SecondInput = NULL);
   bool                    feedForward(CNet *inputNet, int inputLayer = -1, CNet *secondNet = NULL, int secondLayer = -1);
   bool                    feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true, CNet *secondNet = NULL, int secondLayer = -1);
   bool                    backProp(CArrayFloat *targetVals, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL);               ///< Back propagation method. @param[in] targetVals Target values
   bool                    backProp(CArrayFloat *targetVals, CNet *secondNet = NULL, int secondLayer = -1);               ///< Back propagation method. @param[in] targetVals Target values
   bool                    backPropGradient(CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, int LastLayer = -1, bool load_weights = true);     ///< Back propagation method for GPU calculation. @param[in] targetVals Target values
   bool                    backPropGradient(CNet *secondNet, int secondLayer = -1, int LastLayer = -1, bool load_weights = true);
   void                    getResults(CBufferFloat *&resultVals);                ///< Method to get results of feed forward process.@param[out] resultVals Array of result values
   void                    getResults(vector<float> &resultVals);                ///< Method to get results of feed forward process.@param[out] resultVals Array of result values
   void                    getResults(float &resultVals[]);                      ///< Method to get results of feed forward process.@param[out] resultVals Array of result values
   float                   getRecentAverageError()  { return recentAverageError; } ///< Method to check quality of study. @return Average error
   bool                    Save(string file_name, float error, float undefine, float forecast, datetime time, bool common = true);
   ///< Save method. @param[in] file_name File name to save @param[in] error Average error @param[in] undefine Undefined percent @param[in] Foecast percent @param[in] time Last study time @param[in] common Common flag
   virtual bool            Save(const int file_handle);
   bool                    Load(string file_name, float &error, float &undefine, float &forecast, datetime &time, bool common = true);
   ///< Load method. @param[in] file_name File name to save @param[out] error Average error @param[out] undefine Undefined percent @param[out] Foecast percent @param[out] time Last study time @param[in] common Common flag
   virtual bool            Load(const int file_handle);
   //---
   static float            recentAverageSmoothingFactor;             ///< Smoothing factor of average error
   virtual int             Type(void)   const   {  return defNet;   }///< Identificator of class.@return Type of class
   virtual bool            TrainMode(bool flag);                     ///< Set Training Mode Flag
   virtual bool            GetLayerOutput(uint layer, CBufferFloat *&result); ///< Retutn Output data of layer. @param[in] layer Number of layer @param[out] return Buffer with data
   virtual bool            GetLayerOutputDevice(uint layer, CBufferFloat *&result); ///< Return a non-owning existing OpenCL output buffer; the layer/net retains its lifetime.
   virtual bool            GetLayerOutput(uint layer, vector<float> &result); ///< Retutn Output data of layer. @param[in] layer Number of layer @param[out] return Buffer with data
   bool                    BindRAGMemory(const uint layer,CNeuronRAGMemory *memory); ///< Bind a compatible library-owned RAG token snapshot to a VLADriver RAG-MPI layer.
   bool                    UnbindRAGMemory(const uint layer); ///< Detach a RAG snapshot before its COW buffers are replaced.
   //---
   virtual void            SetOpenCL(COpenCLMy *obj);
   virtual COpenCLMy*      GetOpenCL(void)   {  return opencl; }
   //---
   virtual bool            WeightsUpdate(CNet *net, float tau);
   //--- Soft Actor-Critic
   virtual bool            GetLogProbs(vector<float> &log_probs);
   virtual bool            AlphasGradient(CNet *PolicyNet);
   virtual bool            CalcLogProbs(CBufferFloat *buffer);
   virtual bool            SetResult(float &result[]);
   virtual bool            Clear(void);
  };
//+------------------------------------------------------------------+
///\class CNeuronLSTM
/// Class of recurrent LSTM unit
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8385#para5">the link.</A>
//+------------------------------------------------------------------+
class CNeuronLSTM    :  public CNeuronProof
  {
protected:
   CLayer            *ForgetGate;                                    ///< Object of forget gate
   CLayer            *InputGate;                                     ///< Object of input gate
   CLayer            *OutputGate;                                    ///< Object of output gate
   CLayer            *NewContent;                                    ///< Object of new content
   CArrayFloat       *Memory;                                        ///< Memory array
   CArrayFloat       *PrevMemory;                                    ///< Ravious iteration memory array
   CArrayFloat       *Input;                                         ///< Input data
   CArrayFloat       *InputGradient;                                 ///< Gradient on previous layer
   //---
   virtual bool      feedForward(CLayer *prevLayer);                    ///< Feed Forward method. Detailed description on <A HREF="https://www.mql5.com/ru/articles/8385#para52">the link.</A>@param prevLayer Pointer to previos layer.
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);           ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer.
   virtual bool      updateInputWeights(CLayer *&prevLayer);            ///< Method for updating weights.@param prevLayer Pointer to previos layer.
   virtual bool      updateInputWeights(CLayer *gate, CArrayFloat *input_data); ///< Method for updating gates' weights.@param gate Pointer to gate. @param input_data Pointer to tensor with input data.
   virtual bool      InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type);
   ///< Method of gate initialization @param[in] layer Pointer to gate @param[in] numUnits Number of units in gate @param[in] numOutputs Number of outputs @param[in] optimization_type Type of optimization (#ENUM_OPTIMIZATION)
   virtual CArrayFloat *CalculateGate(CLayer *gate, CArrayFloat *sequence); ///< Method of calculation gate iteration @param[in] gate Pointer to gate @param[in] sequence Input data @return Array of output data

public:
   /** Constructor */
                     CNeuronLSTM(void);
   /** Destructor */~CNeuronLSTM(void);
   virtual bool      Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
   ///< Unit initialization method. Detailed description on <A HREF="https://www.mql5.com/ru/articles/8385#para51">the link.</A>
   //---
   virtual CLayer    *getOutputLayer(void)  { return OutputLayer;  }    ///< Method for getting a pointer to the resulting neural layer. Not used in fully connected neural networks.@return Pointer to layer.
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   //--- methods for working with files
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual int       Type(void)   const   {  return defNeuronLSTM;   }///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
///\class CBufferFloat
/// Class of OpenCL buffer data. Used for transfer data from CPU to GPU and back.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8435#para44">the link.</A>
//+------------------------------------------------------------------+
class CBufferFloat     :  public CArrayFloat
  {
protected:
   COpenCLMy         *OpenCL;                ///< Object for working with OpenCL
   int               m_myIndex;              ///< Index of buffer
public:
   /** Constructor */
                     CBufferFloat(void);
   /** Destructor */~CBufferFloat(void);
   //---
   virtual bool      BufferInit(uint count, float value);  ///< Method for buffer initialization @param[in] count Number of items @param[in] value Initialization value
   virtual bool      BufferInitLike(CBufferFloat* master);  ///< Method for buffer initialization @param[in] count Number of items @param[in] value Initialization value
   virtual bool      BufferCreate(COpenCL *opencl);       ///< Method for creating new buffer @param[in] opencl Pointer to #COpenCLMy object
   virtual bool      BufferFree(void);                      ///< Method for deleting buffer from GPU
   virtual bool      BufferRead(void);                      ///< Method for reading buffer data from GPU
   virtual bool      BufferWrite(void);                     ///< Method for writing buffer data to GPU
   virtual bool      BufferSet(int index) {  if(!OpenCL.BufferFree(m_myIndex)) ReturnFalse; m_myIndex = index; return true; } ///< Change buffer index number @param index New index
   template<typename T>
   int               GetData(vector<T> &values);             ///< Read data from buffer to vector @param[out] values Vector to read data
   template<typename T>
   int               GetData(T &values[]);             ///< Read data from buffer to array @param[out] values Array to read data
   template<typename T>
   int               GetData(matrix<T> &values);             ///< Read data from buffer to vector @param[out] values Vector to read data
   virtual int       GetData(CArrayFloat *values);         ///< Read data from buffer to array @param[out] values Array to read data
   virtual int       GetIndex(void) const {  return m_myIndex; }  ///< Get buffer index @return Index
   virtual float     Maximum(void) {  return m_data[Argmax()];}
   virtual float     Minimum(void) {  return m_data[Argmin()];}
   virtual int       Argmax(void) {  return ArrayMaximum(m_data, 0, m_data_total);}
   virtual int       Argmin(void) {  return ArrayMinimum(m_data, 0, m_data_total);}
   //---
   template<typename T>
   bool              AssignArray(const T &src[]);
   virtual bool      AssignArray(const matrix<float> &src);
   virtual bool      AssignArray(const vector<float> &src);
   virtual bool      AssignArray(const CArrayFloat *obj);
   virtual bool      AssignArray(const CBufferFloat *obj)  { return AssignArray((CArrayFloat*)obj); }
   virtual bool      AddArray(const CBufferFloat *src)  { return CArrayFloat::AddArray((CArrayFloat*)src); }
   virtual bool      AddArray(const vector<float> &src);
   template<typename T>
   bool              AddArray(const T &src[]);
   //---
   virtual bool      Fill(const float value);
   virtual bool      Fill(const CBufferFloat *obj);
   template<typename T>
   bool              Fill(const matrix<T> &obj);
   template<typename T>
   bool              Fill(const vector<T> &obj);
   virtual bool      Random(const float min_value, const float  max_value);
   //---
   virtual int       Type(void)                                    const { return defBufferDouble;      }///< Identificator of class.@return Type of class
   virtual void      BufferToCSV(const string file_name);   ///< Save buffer data to CSV file @param[in] file_name File name to write data
   //--- methods for working with files
   virtual bool      Save(const int file_handle);
   virtual bool      Load(const int file_handle);
   //---
   virtual COpenCL   *GetOpenCL(void) const { return OpenCL; }
   bool              operator=(const CBufferFloat *obj) {return AssignArray(obj);}
   float             operator[](const int index)  const
     {
      if(index < 0)
         return(At(m_data_total + index));
      return(At(index));
     }
  };
//+------------------------------------------------------------------+
///\ingroup neuron_base
///\class CNeuronBaseOCL
///\brief The base class of neuron for GPU calculation.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8435#para45">the link.</A>
//+------------------------------------------------------------------+
class CNeuronBaseOCL    :  public CObject
  {
protected:
   bool               bTrain;             ///< Training Mode Flag
   COpenCLMy         *OpenCL;             ///< Object for working with OpenCL
   CBufferFloat      *Output;             ///< Buffer of Output tensor
   CBufferFloat      *PrevOutput;         ///< Buffer of previous iteration Output tensor
   CBufferFloat      *Weights;            ///< Buffer of weights matrix
   CBufferFloat      *DeltaWeights;       ///< Buffer of last delta weights matrix (#SGD)
   CBufferFloat      *Gradient;           ///< Buffer of gradient tensor
   CBufferFloat      *FirstMomentum;      ///< Buffer of first momentum matrix (#ADAM)
   CBufferFloat      *SecondMomentum;     ///< Buffer of second momentum matrix (#ADAM)
   vector<float>      prev_output;
   //---
   //const float      lr;
   float             alpha;               ///< Multiplier to momentum in #SGD optimization
   uint              iBatch;              ///< Batch size used in LS optimization
   uint              t;                   ///< Count of iterations
   //---
   int               m_myIndex;           ///< Index of neuron in layer
   ENUM_ACTIVATION   activation;          ///< Activation type (#ENUM_ACTIVATION)
   ENUM_OPTIMIZATION optimization;        ///< Optimization method (#ENUM_OPTIMIZATION)
   //---
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { return feedForward(NeuronOCL); }               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   virtual bool      feedForwardSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *SecondNeuron) { return feedForward(NeuronOCL, (SecondNeuron ? SecondNeuron.getOutput() : NULL)); }

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)  {  return updateInputWeights(NeuronOCL); }        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      updateInputWeightsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *SecondNeuron) { return updateInputWeights(NeuronOCL, (SecondNeuron ? SecondNeuron.getOutput() : NULL)); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);        ///< Method to transfer gradients to previous layer @param[in] NeuronOCL Pointer to previous layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { return calcInputGradients(NeuronOCL); }
   virtual bool      calcInputGradientsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *SecondNeuron) { return calcInputGradients(NeuronOCL, (SecondNeuron ? SecondNeuron.getOutput() : NULL), (SecondNeuron ? SecondNeuron.getGradient() : NULL), (SecondNeuron ? (ENUM_ACTIVATION)SecondNeuron.Activation() : None)); }
   virtual bool      calcHiddenGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { ReturnFalse; }
   virtual bool      calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.
   virtual float     GenerateWeight(void);
   virtual bool      WeightsUpdateAdam(CNeuronBaseOCL *source, float tau);
   virtual bool      SumAndNormalize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f);
   virtual bool      Normalize(CBufferFloat *tensor, int dimension);
   virtual bool      Different(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 1.0f);
   virtual bool      DifferentGrad(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 1.0f);
   //---
   virtual bool      IdentSum(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f);
   virtual bool      IdentDifferent(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f);
   virtual bool      IdentDifferentGrad(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f);
   //---
   virtual bool      SumVecMatrix(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension, uint variables, uint shift_in1 = 0, uint shift_in2 = 0, uint shift_out = 0, float multiplyer = 1.0f);
   virtual bool      SumVecMatrixGrad(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension, uint variables, uint shift_in1 = 0, uint shift_in2 = 0, uint shift_out = 0, float multiplyer = 1.0f);
   virtual bool      MatMul(const CBufferFloat *matr1, const CBufferFloat *matr2, CBufferFloat *result, const int rows1, const int cols1, const int cols2, const int variables = 1, const bool mult_var_second = true);
   virtual bool      MatMulGrad(const CBufferFloat *matr1, CBufferFloat *matr1_gr, const CBufferFloat *matr2, CBufferFloat *matr2_gr, const CBufferFloat *result_gr, const int rows1, const int cols1, const int cols2, const int variables = 1, const bool mult_var_second = true);
   virtual bool      DiagMatMul(const CBufferFloat *diag, const CBufferFloat *matr, CBufferFloat *result, const int rows, const int cols, const int variables = 1, const int activation = None);
   virtual bool      DiagMatMulGrad(const CBufferFloat *diag, CBufferFloat *diag_gr, const CBufferFloat *matr, CBufferFloat *matr_gr, const CBufferFloat *result_gr, const int rows, const int cols, const int variables = 1);
   virtual bool      SparseMatMul(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, const CBufferFloat *full, CBufferFloat *result, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols);
   virtual bool      SparseMatMulGrad(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, CBufferFloat *sparse_grad, const CBufferFloat *full, CBufferFloat *full_gr, const CBufferFloat *grad, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols);
   virtual bool      SparseConcatenate(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, const CBufferFloat *full, CBufferFloat *result, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols);
   virtual bool      SparseConcatenateGrad(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, CBufferFloat *sparse_grad, const CBufferFloat *full, CBufferFloat *full_gr, const CBufferFloat *grad, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols);
   virtual bool      ScalarToVector(CBufferFloat *scalar, CBufferFloat *vector_in, CBufferFloat *vector_out, int dimension);
   virtual bool      ScalarToVectorGrad(CBufferFloat *scalar, CBufferFloat *scalar_gr, CBufferFloat *vector_in, CBufferFloat *vector_in_gr, CBufferFloat *vector_out_gr, int dimension);
   ///< \brief Method sum and normilize 2 tensors by calling 2 kernels ::SumMatrix() and ::Normalize().
   virtual bool      LoadInsideLayer(int file_handle, CNeuronBaseOCL *neuron);
   virtual bool      Activation(CBufferFloat *inputs, CBufferFloat *outputs, int activat);
   virtual bool      DeActivation(CBufferFloat *inputs, CBufferFloat *inputs_gr, CBufferFloat *output_gr, int activat);
   virtual bool      NLLGrad(const CBufferFloat *log_b, CBufferFloat *grad_in, const CBufferFloat *grad_out, CBufferFloat *grad_log_b);
   virtual bool      SparseSoftmaxHausdorff(const CBufferFloat *plan, const CBufferFloat *decis, CBufferFloat *output, CBufferFloat *top_index, const int dimension, const int top_k);
   virtual bool      SparseSoftmaxHausdorffGrad(const CBufferFloat *plan, CBufferFloat *grad_plan, const CBufferFloat *decis, CBufferFloat *grad_decis, const CBufferFloat *grad_out, const CBufferFloat *top_index, const int dimension, const int top_k);
   //---
   virtual bool      Concat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *output, int window1, int window2, int count);
   virtual bool      Concat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *output, int window1, int window2, int window3, int count);
   virtual bool      Concat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *input4, CBufferFloat *output, int window1, int window2, int window3, int window4, int count);
   virtual bool      Concat(int input1, int input2, int input3, int input4, int output, int window1, int window2, int window3, int window4, int count);
   virtual bool      DeConcat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *output, int window1, int window2, int count);
   virtual bool      DeConcat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *output, int window1, int window2, int window3, int count);
   virtual bool      DeConcat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *input4, CBufferFloat *output, int window1, int window2, int window3, int window4, int count);
   virtual bool      ConcatVecMatrix(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension_v, uint  rows, uint cols, uint variables, bool multvarsecond);
   virtual bool      ConcatVecMatrixGrad(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension_v, uint rows, uint  cols, uint variables, bool multvarsecond);
   //---
   virtual bool      ElementMult(const CBufferFloat *input1, const CBufferFloat *input2, CBufferFloat *output);
   virtual bool      ElementMultGrad(const CBufferFloat *input1, CBufferFloat *input1_gr, const CBufferFloat *input2, CBufferFloat *input2_gr, const CBufferFloat *output, const int activation1, const int activation2);
   virtual bool      GateElementMult(const CBufferFloat *input1, const CBufferFloat *input2, const CBufferFloat *gate, CBufferFloat *output);
   virtual bool      GateElementMultGrad(const CBufferFloat *input1, CBufferFloat *input1_gr, const CBufferFloat *input2, CBufferFloat *input2_gr, const CBufferFloat *gate, CBufferFloat *gate_gr, const CBufferFloat *output, const int activation1, const int activation2, const int activation_gate);
   int               RND(int total)   { xor128; return (int)((float)(total - 1) / UINT_MAX * rnd_w);  } ///< Generates a random neuron position to turn off
   //---
   virtual bool      DiversityLoss(CNeuronBaseOCL *neuron, const int units, const int dimension, const bool add = false);
   //---
   virtual bool      SwapBuffers(CBufferFloat *&buffer1, CBufferFloat *&buffer2, bool check_size = true);
   //---
   virtual bool      DilatedCorrelation(CNeuronBaseOCL* feature, CNeuronBaseOCL* correlations, uint shifts_index, uint dimension, uint units);
   virtual bool      DilatedCorrelationGrad(CNeuronBaseOCL* feature, CNeuronBaseOCL* correlations, uint shifts_index, uint dimension, uint units);
   virtual bool      DilatedDifference(CNeuronBaseOCL* feature, CNeuronBaseOCL* difference, uint shifts_index, uint dimension, uint units);
   virtual bool      DilatedDifferenceGrad(CNeuronBaseOCL* feature, CNeuronBaseOCL* difference, uint shifts_index, uint dimension, uint units);
   virtual bool      PerturbedMatrix(CNeuronBaseOCL* inputs, CNeuronBaseOCL* perturb, CNeuronBaseOCL* outputs, float perturb_mult, uint dimension, uint units);
   virtual bool      PerturbedMatrixGrad(CNeuronBaseOCL* inputs, CNeuronBaseOCL* perturb, CNeuronBaseOCL* outputs, float perturb_mult, uint dimension, uint units);
   virtual bool      LinearUpsample(CNeuronBaseOCL* data, CNeuronBaseOCL* upsample, uint upsampl_mult, uint units, uint variables);
   virtual bool      LinearUpsampleGrad(CNeuronBaseOCL* data, CNeuronBaseOCL* upsample, uint upsampl_mult, uint units, uint variables);
   //---
   virtual int       CreateShifts(const uint max_displacement, int &count);

public:
   /** Constructor */
                     CNeuronBaseOCL(void);
   /** Destructor */~CNeuronBaseOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) {  activation = value; }      ///< Set the type of activation function (#ENUM_ACTIVATION)
   //---
   virtual int       getOutputIndex(void)    const {  return Output.GetIndex();        }  ///< Get index of output buffer @return Index
   virtual int       getPrevOutIndex(void)   const {  return PrevOutput.GetIndex();    }  ///< Get index of previous iteration output buffer @return Index
   virtual int       getGradientIndex(void)  const {  return Gradient.GetIndex();      }  ///< Get index of gradient buffer @return Index
   virtual bool      SetGradient(CBufferFloat *buffer, bool delete_prev = true);
   virtual bool      SetOutput(CBufferFloat *buffer, bool delete_prev = true);
   virtual bool      SetGradientIndex(int index)   {  return Gradient.BufferSet(index);   }  ///< Method for change index of gradient buffer.@param[in] New index of buffer
   virtual int       getWeightsIndex(void)         const
     {
      if(!Weights)
         return INVALID_HANDLE;
      return Weights.GetIndex();
     }  ///< Get index of weights matrix buffer @return Index
   virtual int       getDeltaWeightsIndex(void)    const {  return DeltaWeights.GetIndex();  }  ///< Get index of delta weights matrix buffer (SGD)@return Index
   virtual int       getFirstMomentumIndex(void)   const {  return FirstMomentum.GetIndex(); }  ///< Get index of first momentum matrix buffer (Adam)@return Index
   virtual int       getSecondMomentumIndex(void)  const {  return SecondMomentum.GetIndex();}  ///< Get index of Second momentum matrix buffer (Adam)@return Index
   virtual int       getWeightsSAMIndex(void)      const {  return -1;                     }
   //---
   virtual int       getOutputVal(float &values[]) {  return Output.GetData(values);      }  ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getOutputVal(CArrayFloat *values)   {  return Output.GetData(values);  }  ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getOutputVal(vector<float> &values)   {  return Output.GetData(values);  }  ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getPrevVal(float &values[])   {  return PrevOutput.GetData(values);  }  ///< Get values of previous iteration output buffer @param[out] values Array of data @return number of items
   virtual int       getGradient(float &values[])  {  return Gradient.GetData(values);    }  ///< Get values of gradient buffer @param[out] values Array of data @return number of items
   virtual int       getGradient(vector<float> &values)  {  return Gradient.GetData(values);    }  ///< Get values of gradient buffer @param[out] values Array of data @return number of items
   virtual bool      calcAlphaGradients(CNeuronBaseOCL *NeuronOCL)    { return true; }
   virtual bool      GetAlphaLogProbs(vector<float> &log_probs)       { ReturnFalse; }
   virtual bool      ReCalcLogProbs(void)                             { ReturnFalse; }
   virtual CBufferFloat   *getOutput(void)      const    {  return Output;       }                 ///< Get pointer of output buffer @return Pointer to object
   virtual CBufferFloat   *getGradient(void)    const    {  return Gradient;     }                 ///< Get pointer of gradient buffer @return Pointer to object
   virtual CBufferFloat   *getPrevOutput(void)  const    {  return PrevOutput;   }                 ///< Get pointer of gradient buffer @return Pointer to object
   virtual int       getWeights(float &values[])   {  return Weights.GetData(values);     }  ///< Get values of weights matrix buffer @param[out] values Array of data @return number of items
   virtual CBufferFloat   *getWeights(void)        {  return Weights;     }                 ///< Get pointer of gradient buffer @return Pointer to object
   virtual int       Neurons(void) const           {  return Output.Total();              }  ///< Get number of neurons in layer @return Number of neurons
   virtual int       Activation(void) const        {  return (int)activation;             }  ///< Get type of activation function @return Type (#ENUM_ACTIVATION)
   virtual int       getConnections(void)          {  return (CheckPointer(Weights) != POINTER_INVALID ? Weights.Total() / (Output.Total() + 1) : 0);   } ///< Get number of connections 1 neuron to next layer @return Number of connections
   virtual ENUM_OPTIMIZATION Optimization(void)    {  return optimization; }
   //---
   virtual bool      FeedForward(CObject *SourceObject, CBufferFloat *SecondInput = NULL);                    ///< Dispatch method for defining the subroutine for feed forward process. @param SourceObject Pointer to the previous layer.
   virtual bool      FeedForwardSecond(CObject *SourceObject, CNeuronBaseOCL *SecondNeuron);
   virtual bool      UpdateInputWeights(CObject *SourceObject, CBufferFloat *SecondInput = NULL);             ///< Dispatch method for defining the subroutine for updating weights.@param SourceObject Pointer to previos layer.
   virtual bool      UpdateInputWeightsSecond(CObject *SourceObject, CNeuronBaseOCL *SecondNeuron);
   ///\ingroup neuron_base_gr
   ///@{
   virtual bool      CalcHiddenGradients(CObject *TargetObject, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, ENUM_ACTIVATION SecondActivation = None);        ///< Dispatch method for defining the subroutine for transferring the gradient to the previous layer. @param TargetObject Pointer to the next layer.
   virtual bool      CalcHiddenGradientsSecond(CObject *TargetObject, CNeuronBaseOCL *SecondNeuron);
   virtual bool      calcOutputGradients(CBufferFloat *Target, float &error);               ///< Method of output gradients calculation by calling kernel ::CalcOutputGradient().@param Target Traget value
   virtual bool      calcAlignmentGradient(const CNeuronBaseOCL *neuron, const bool add = false);
   ///@}
   //---
   virtual bool      Save(const int file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(const int file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronBaseOCL;                  }///< Identificator of class.@return Type of class
   virtual void      TrainMode(bool flag)                         {  bTrain = flag;                              } ///< Set Training Mode Flag
   virtual bool      TrainMode(void)                              {  return bTrain;                            }///< Get Training Mode Flag
   virtual CLayerDescription* GetLayerInfo(void);
   virtual bool      numOutputs(const uint outputs, ENUM_OPTIMIZATION optimization_type);
   virtual void      SetOpenCL(COpenCLMy *obj);
   //--- Load() allocates buffers from their serialized shapes.  It needs the
   //--- context before allocation, but SetOpenCL() is for already shaped
   //--- buffers and must not be used on a constructor-empty object.
   virtual void      AttachOpenCL(COpenCLMy *obj) { OpenCL=obj; }
   virtual CObject*  AsObject(void) {  return GetPointer(this);   }
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
   //---
   virtual bool      Clear(void) { return !(!Output || !Output.Fill(0) || !Gradient || !Gradient.Fill(0)); }
   //---
   virtual bool      SwapOutputs(void) { return SwapBuffers(Output, PrevOutput, true); }
  };
//+------------------------------------------------------------------+
///\class CNeuronProofOCL
/// Class of pooling layer GPU calculation
//+------------------------------------------------------------------+
class CNeuronProofOCL : public CNeuronBaseOCL
  {
protected:
   uint               iWindow;                                             ///< Input window size
   uint               iStep;                                               ///< Size of step

   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< Feed Forward method.@param NeuronOCL Pointer to previos layer.
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);        ///< Method to transfer gradients to previous layer @param[in] NeuronOCL Pointer to previous layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   { return true;};        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.

public:
   /** Constructor */
                     CNeuronProofOCL(void) : iWindow(2), iStep(1) {};
   /** Destructor */~CNeuronProofOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of input window @param[in] step Step size.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //--- methods for working with files
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual int       Type(void)   const   {  return defNeuronProofOCL;   }///< Identificator of class.@return Type of class
   virtual CLayerDescription* GetLayerInfo(void);
   virtual uint      GetWindow(void) const { return iWindow;   }
  };
//+------------------------------------------------------------------+
///\class CNeuronConvOCL
/// Class of convolution layer GPU calculation
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8765#para41">the link.</A>
//+------------------------------------------------------------------+
class CNeuronConvOCL    :  public CNeuronProofOCL
  {
protected:
   uint              iWindowOut;                                           ///< Size of out window
   uint              iVariables;                                           ///< Size of out window
   //---
   CBufferFloat      *WeightsConv;                                         ///< Matrix of weights to previous layer
   CBufferFloat      *DeltaWeightsConv;                                    ///< Matrix of delta weights to previous layer (#SGD)
   CBufferFloat      *FirstMomentumConv;                                   ///< Matrix of first momentum to previous layer (#ADAM)
   CBufferFloat      *SecondMomentumConv;                                  ///< Matrix of second momentum to previous layer (#ADAM)
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;      ///< Feed Forward method.@param NeuronOCL Pointer to previos layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;  ///< Method for updating weights.@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;  ///< Method to transfer gradients to previous layer @param[in] NeuronOCL Pointer to previous layer.

public:
   /** Constructor */
                     CNeuronConvOCL(void) :   iWindowOut(1), iVariables(1) {  activation = SoftPlus;   }
   /** Destructor */~CNeuronConvOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of input window @param[in] step Step size.@param[in] window_out Size of output window @param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual CBufferFloat* GetWeightsConv(void)   {  return WeightsConv;  }
   //--- Direct non-owning device-buffer input path.
   bool              FeedForwardBuffer(CBufferFloat *source);
   bool              UpdateInputWeightsBuffer(CBufferFloat *source);
   //---
   virtual int       Type(void) override const  {  return defNeuronConvOCL;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual CLayerDescription* GetLayerInfo(void) override;
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetFilters(void) const { return iWindowOut; }
   virtual uint      GetVariables(void) const { return iVariables; }
   virtual uint      GetUnits(void) const { return Output.Total() / (iWindowOut * iVariables); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConvSAMOCL    :  public CNeuronConvOCL
  {
protected:
   float             fRho;
   //---
   CBufferFloat      cWeightsSAM;
   CBufferFloat      cWeightsSAMConv;
   //---
   virtual bool      calcEpsilonWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForwardSAM(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      InitBufferLike(CBufferFloat *&buffer, CBufferFloat *master);
   virtual void      ReplaceBuffer(CBufferFloat *&buffer, CBufferFloat *master);

public:
                     CNeuronConvSAMOCL(void) {  activation = GELU;   }
                    ~CNeuronConvSAMOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, float rho, ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual bool      InitPS(CNeuronConvSAMOCL *master);
   //---
   virtual int       Type(void)   const         {  return defNeuronConvSAMOCL;   }
   virtual int       Activation(void)  const    {  return (fRho == 0 ? (int)None : (int)activation);   }
   virtual int       getWeightsSAMIndex(void) const override {  return cWeightsSAM.GetIndex();   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
  };
//+------------------------------------------------------------------+
///\class CNeuronAttentionOCL
/// Class of Self-Attention layer GPU calculation
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8765#para42">the link.</A>
//+------------------------------------------------------------------+
class CNeuronAttentionOCL : public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL    *Querys;             ///< Convolution layer for Querys
   CNeuronConvOCL    *Values;             ///< Convolution layer for Values
   CBufferFloat      *Scores;             ///< Buffer for Scores matrix
   CNeuronBaseOCL    *AttentionOut;       ///< Layer of Self-Attention Out
   CNeuronConvOCL    *FF1;                ///< Convolution layer for first layer of Feed Forward block
   CNeuronConvOCL    *FF2;                ///< Convolution layer for second layer of Feed Forward block
   //---
   uint              iWindow;             ///< Window size
   uint              iUnits;              ///< Number of units
   //---
   virtual bool      feedForward(CNeuronBaseOCL *prevLayer);                  ///< Feed Forward method.@param prevLayer Pointer to previos layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *prevLayer);           ///< Method for updating weights.@param prevLayer Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);  ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.

public:
   /** Constructor */
                     CNeuronAttentionOCL(void) : iWindow(1), iUnits(0) {};
   /** Destructor */~CNeuronAttentionOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual int       Type(void)   const   {  return defNeuronAttentionOCL;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual CLayerDescription* GetLayerInfo(void);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
///\ingroup neuron_base
///\class CNeuronMHAttentionOCL
///\brief Class of Multi-Head Self-Attention layer GPU calculation
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/8909#para52">the link.</A>
//+------------------------------------------------------------------+
class CNeuronMHAttentionOCL   :  public CNeuronAttentionOCL
  {
protected:
   CNeuronConvOCL    *Querys2;            ///< Convolution layer for Querys Head 2
   CNeuronConvOCL    *Querys3;            ///< Convolution layer for Querys Head 3
   CNeuronConvOCL    *Querys4;            ///< Convolution layer for Querys Head 4
   CNeuronConvOCL    *Keys2;              ///< Convolution layer for Keys Head 2
   CNeuronConvOCL    *Keys3;              ///< Convolution layer for Keys Head 3
   CNeuronConvOCL    *Keys4;              ///< Convolution layer for Keys Head 4
   CNeuronConvOCL    *Values2;            ///< Convolution layer for Values Head 2
   CNeuronConvOCL    *Values3;            ///< Convolution layer for Values Head 3
   CNeuronConvOCL    *Values4;            ///< Convolution layer for Values Head 4
   CBufferFloat      *Scores2;            ///< Buffer for Scores matrix Head 2
   CBufferFloat      *Scores3;            ///< Buffer for Scores matrix Head 3
   CBufferFloat      *Scores4;            ///< Buffer for Scores matrix Head 4
   CNeuronBaseOCL    *AttentionOut2;      ///< Layer of Self-Attention Out
   CNeuronBaseOCL    *AttentionOut3;      ///< Layer of Self-Attention Out
   CNeuronBaseOCL    *AttentionOut4;      ///< Layer of Self-Attention Out
   CNeuronBaseOCL    *AttentionConcatenate;///< Layer of Concatenate Self-Attention Out
   CNeuronConvOCL    *Weights0;           ///< Convolution layer for Weights0
   //---
   virtual bool      feedForward(CNeuronBaseOCL *prevLayer);                  ///< Feed Forward method.@param prevLayer Pointer to previos layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *prevLayer);            ///< Method for updating weights.@param prevLayer Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);  ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.
   /// Method to transfer gradients inside Head Self-Attention
   virtual bool      calcHeadGradient(CNeuronConvOCL *query, CNeuronConvOCL *value, CBufferFloat *score, CNeuronBaseOCL *attention, CNeuronBaseOCL *prevLayer);

public:
   /** Constructor */
                     CNeuronMHAttentionOCL(void) {};
   /** Destructor */~CNeuronMHAttentionOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual int       Type(void)   const   {  return defNeuronMHAttentionOCL;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);   ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);   ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual CLayerDescription* GetLayerInfo(void);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
///\class CCollection
///\brief Class of objects collection.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/9025#para41">the link.</A>
//+------------------------------------------------------------------+
class CCollection    :  public CArrayObj
  {
public:
                     CCollection(void) {};
                    ~CCollection(void) {};
   //---
   CObject*          operator[](const int i) const               { return(m_data[i]); }
   //---
   virtual bool      CreateElement(const int index)
     {
      if(index < 0)
         ReturnFalse;
      //---
      if(index >= m_data_max)
        {
         if(!Reserve(index - m_data_total))
            ReturnFalse;
        }
      m_data[index] = new CBufferFloat();
      return(CheckPointer(m_data[index]) != POINTER_INVALID);
     }
   //---
   virtual bool      SetOpenCL(COpenCLMy *open_cl)
     {
      if(!open_cl)
         ReturnFalse;
      bool result = true;
      for(int i = 0; (i < m_data_total && result); i++)
        {
         CBufferFloat* temp = m_data[i];
         if(!temp || !temp.BufferCreate(open_cl))
           {
            result = false;
            break;
           }
        }
      //---
      return result;
     }
   //---
  };
//+------------------------------------------------------------------+
///\ingroup neuron_base
///\class CNeuronMLMHAttentionOCL
///\brief Class of Multilayer multi-headed attention neuron.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/9025#para41">the link.</A>
//+------------------------------------------------------------------+
class CNeuronMLMHAttentionOCL       :  public CNeuronBaseOCL
  {
protected:
   uint              iLayers;                                     ///< Number of inner layers
   uint              iHeads;                                      ///< Number of heads
   uint              iWindow;                                     ///< Input window size
   uint              iUnits;                                      ///< Number of units
   uint              iWindowKey;                                  ///< Size of Key/Query window
   //---
   CCollection       *QKV_Tensors;                                ///< The collection of tensors of Queries, Keys and Values
   CCollection       *QKV_Weights;                                ///< The collection of Matrix of weights to previous layer
   CCollection       *S_Tensors;                                  ///< The collection of Scores tensors
   CCollection       *AO_Tensors;                                 ///< The collection of Attention Out tensors
   CCollection       *FF_Tensors;                                 ///< The collection of tensors of Feed Forward output
   CCollection       *FF_Weights;                                 ///< The collection of Matrix of Feed Forward weights

   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   virtual bool      ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0, uint variables = 1);
   ///< \brief Convolution Feed Forward method of calling kernel ::FeedForwardConv().
   virtual bool      AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false);
   ///< \brief Multi-heads attention scores method of calling kernel ::MHAttentionScore().
   virtual bool      AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out);
   ///< \brief Multi-heads attention out method of calling kernel ::MHAttentionOut().
   ///\ingroup neuron_base_opt
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);  ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *momentum1, CBufferFloat *momentum2, uint window, uint window_out, uint step = 0, uint heads = 0, uint variables = 1);
   virtual bool      ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *source, CBufferFloat *momentum1, CBufferFloat *momentum2, float tau);
   ///< Method for updating weights in convolution layer.\details Calling one of kernels ::UpdateWeightsConvMomentum() or ::UpdateWeightsConvAdam() in depends of optimization type (#ENUM_OPTIMIZATION).
   virtual bool      ConvolutionInputGradients(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0, uint variables = 1);
   ///< Method of passing gradients through a convolutional layer.
   virtual bool      AttentionInsideGradients(CBufferFloat *qkv, CBufferFloat *qkv_g, CBufferFloat *scores, CBufferFloat *gradient);
   ///< Method of passing gradients through attention layer.
public:
   /** Constructor */
                     CNeuronMLMHAttentionOCL(void);
   /** Destructor */~CNeuronMLMHAttentionOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual int       Type(void)   const   {  return defNeuronMLMHAttentionOCL;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual CLayerDescription* GetLayerInfo(void);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
///\ingroup neuron_base
///\class CNeuronDropoutOCL
///\brief The Dropout neuron for GPU calculation.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/9112#para2">the link.</A>
//+------------------------------------------------------------------+
class CNeuronDropoutOCL    :  public   CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    *PrevLayer;
   float             OutProbability;
   int               OutNumber;
   CBufferFloat      *DropOutMultiplier;
   float             dInitValue;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method.@param NeuronOCL Pointer to previos layer.
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) {return true;}        ///< Blank method for updating weights.@param NeuronOCL Pointer to previos layer.
   //---

public:
   /** Constructor */
                     CNeuronDropoutOCL(void);
   /** Destructor */
                    ~CNeuronDropoutOCL(void);
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, float out_prob, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer #param[in] out_prob Probability of neurons shutdown @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual int       getOutputIndex(void) const override  {  return (bTrain ? Output.GetIndex() : PrevLayer.getOutputIndex());             }  ///< Get index of output buffer @return Index
   virtual int       getGradientIndex(void) const override {  return (bTrain ? Gradient.GetIndex() : PrevLayer.getGradientIndex());          }  ///< Get index of gradient buffer @return Index
   //---
   virtual int       getOutputVal(float &values[]) override {  return (bTrain ? Output.GetData(values) : PrevLayer.getOutputVal(values)); }  ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getOutputVal(vector<float> &values) override {  return (bTrain ? Output.GetData(values) : PrevLayer.getOutputVal(values)); }  ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getOutputVal(CArrayFloat *values) override {  return (bTrain ? Output.GetData(values) : PrevLayer.getOutputVal(values)); }  ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getGradient(float &values[]) override {  return (bTrain ? Gradient.GetData(values) : PrevLayer.getGradient(values));    }  ///< Get values of gradient buffer @param[out] values Array of data @return number of items
   virtual int       getGradient(vector<float> &values) override {  return (bTrain ? Gradient.GetData(values) : PrevLayer.getGradient(values));    }  ///< Get values of gradient buffer @param[out] values Array of data @return number of items
   virtual CBufferFloat   *getOutput(void) const override {  return (bTrain ? Output : PrevLayer.getOutput());      }                 ///< Get pointer of output buffer @return Pointer to object
   virtual CBufferFloat   *getGradient(void) const override {  return (bTrain ? Gradient : PrevLayer.getGradient());  }                 ///< Get pointer of gradient buffer @return Pointer to object
   //---
   virtual bool      Save(int const file_handle) override ;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle) override ;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronDropoutOCL;                }///< Identificator of class.@return Type of class
   virtual CLayerDescription* GetLayerInfo(void) override;
  };
//+------------------------------------------------------------------+
///\class CNeuronBatchNormOCL
///\brief The class of Batch Normalization neuron for GPU calculation.
///\details Detailed description on <A HREF="https://www.mql5.com/ru/articles/9207#para4">the link.</A>
//+------------------------------------------------------------------+
class CNeuronBatchNormOCL  :  public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    *PrevLayer;       ///< Pointer to the object of the previous layer
   int               iBatchSize;       ///< Batch size
   int               iBatchCount;      ///< Batch count
   CBufferFloat      *BatchOptions;    ///< Container of method parameters

   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;               ///< \brief Feed Forward method of calling kernel ::BatchFeedForward().@param NeuronOCL Pointer to previos layer.

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;        ///< Method for updating weights.\details Calling one of kernels ::UpdateBatchOptionsMomentum() or ::UpdateBatchOptionsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradientBatch(). @param NeuronOCL Pointer to next layer.

public:
   /** Constructor */
                     CNeuronBatchNormOCL(void);
   /** Destructor */~CNeuronBatchNormOCL(void);
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual int       getOutputIndex(void)          {  return (iBatchSize > 1 || !PrevLayer ? Output.GetIndex() : PrevLayer.getOutputIndex());             } ///< Get index of output buffer @return Index
   virtual int       getGradientIndex(void)        {  return (iBatchSize >  1 || !PrevLayer ? Gradient.GetIndex() : PrevLayer.getGradientIndex());          } ///< Get index of gradient buffer @return Index
   //---
   virtual int       getOutputVal(float &values[])   {  return (iBatchSize > 1 || !PrevLayer ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getOutputVal(CArrayFloat *values)   {  return (iBatchSize > 1 || !PrevLayer ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getOutputVal(vector<float> &values)   {  return (iBatchSize > 1 || !PrevLayer ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items
   virtual int       getGradient(float &values[])    {  return (iBatchSize > 1 || !PrevLayer ? Gradient.GetData(values) : PrevLayer.getGradient(values));    } ///< Get values of gradient buffer @param[out] values Array of data @return number of items
   virtual CBufferFloat   *getOutput(void)           {  return (iBatchSize > 1 || !PrevLayer ? Output : PrevLayer.getOutput());      }               ///< Get pointer of output buffer @return Pointer to object
   virtual CBufferFloat   *getGradient(void)         {  return (iBatchSize > 1 || !PrevLayer ? Gradient : PrevLayer.getGradient());  }               ///< Get pointer of gradient buffer @return Pointer to object
   virtual CBufferFloat   *getBatchOptions(void)         {  return (iBatchSize > 1 ? BatchOptions : NULL);  }               ///< Get pointer of Batch Options buffer @return Pointer to object
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronBatchNormOCL;    }///< Identificator of class.@return Type of class
   virtual CLayerDescription* GetLayerInfo(void);
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual CBufferFloat   *getWeights(void) override { return BatchOptions; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLSTMOCL : public CNeuronBaseOCL
  {
protected:
   CBufferFloat      m_cWeightsLSTM;
   CBufferFloat      m_cFirstMomentumLSTM;      ///< Buffer of first momentum matrix (#ADAM) or last delta weights matrix (#SGD)
   CBufferFloat      m_cSecondMomentumLSTM;     ///< Buffer of second momentum matrix (#ADAM)

   int               m_iMemory;
   int               m_iConcatenated;
   int               m_iConcatenatedGradient;
   int               m_iHiddenState;
   int               m_iWeightsGradient;
   int               m_iInputs;
   int               m_iVariables;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;               ///< \brief Feed Forward method.@param NeuronOCL Pointer to previos layer.

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;        ///< Method for updating weights.
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronLSTMOCL(void);
                    ~CNeuronLSTMOCL(void);
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual bool      SetInputs(int count);
   ///@}
   //---
   virtual bool      Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void) override       const                      {  return defNeuronLSTMOCL;                  }///< Identificator of class.@return Type of class
   virtual bool      Clear(void);
   virtual CBufferFloat *getLSTMWeights(void) { return GetPointer(m_cWeightsLSTM);}
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#ifndef class_vae
#include "..\Unsupervised\AE\VAE.mqh"
#endif
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSoftMaxOCL    :  public CNeuronBaseOCL
  {
protected:
   uint              iHeads;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronSoftMaxOCL(void) : iHeads(1) { activation = None; }
                    ~CNeuronSoftMaxOCL(void) {};
   //---
   ///\ingroup neuron_base_gr
   ///@{
   virtual bool      calcOutputGradients(CBufferFloat *Target, float &error) override;               ///< Method of output gradients calculation by calling kernel ::CalcOutputGradient().@param Target Traget value
   virtual void      SetHeads(int heads)  { iHeads = heads; }
   ///@}
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual CLayerDescription* GetLayerInfo(void) override;
   virtual int       Type(void) override        const                      {  return defNeuronSoftMaxOCL;                  }///< Identificator of class.@return Type of class
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
   virtual uint      Heads(void) const {  return iHeads; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFQF : protected CNeuronBaseOCL
  {
protected:
   //--- Fractal Net
   CNeuronBaseOCL    cFraction;
   CNeuronSoftMaxOCL cSoftMax;
   //--- Cosine embeding
   CNeuronBaseOCL    cCosine;
   CNeuronBaseOCL    cCosineEmbeding;
   //--- Quantile Net
   CNeuronBaseOCL    cQuantile0;
   CNeuronBaseOCL    cQuantile1;
   CNeuronBaseOCL    cQuantile2;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronFQF();
                    ~CNeuronFQF();
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint actions, uint quantiles, uint numInputs, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual bool      Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void) override        const                      {  return defNeuronFQF; }///< Identificator of class.@return Type of class
   virtual CLayerDescription* GetLayerInfo(void) override;
   virtual CObject*  AsObject(void) {  return GetPointer(this);   }
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      SetActivationFunction(ENUM_ACTIVATION function)    {  CNeuronBaseOCL::SetActivationFunction(function); }
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMLMHSparseAttention  : public CNeuronMLMHAttentionOCL
  {
protected:
   float             m_dSparse;
   //---
   virtual bool      AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = true);
   ///< \brief Multi-heads attention scores method of calling kernel ::MHAttentionScore().
   virtual bool      AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out);
   ///< \brief Multi-heads attention out method of calling kernel ::MHAttentionOut().

public:
                     CNeuronMLMHSparseAttention(void)   :  m_dSparse(0.3f) {};
                    ~CNeuronMLMHSparseAttention(void) {};
   //---
   void              Sparse(float value)  { m_dSparse = value;}
   float             Sparse(void)         { return m_dSparse; }
   virtual int       Type(void)   const   {  return defNeuronMLMHSparseAttentionOCL;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiModel : public CNeuronBaseOCL
  {
protected:
   int               iModels;
   int               iUpdateModel;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.

public:
                     CNeuronMultiModel(void) {};
                    ~CNeuronMultiModel(void) {};
   virtual bool      Init(uint numInputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, int models);
   //---
   ///\ingroup neuron_base_gr
   ///@{
   virtual bool      calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.
   ///@}
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronMultiModels;                  }///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConcatenate   :  public CNeuronBaseOCL
  {
protected:
   int               i_SecondInputs;
   CBufferFloat      *ConcWeights;            ///< Buffer of weights matrix
   CBufferFloat      *ConcDeltaWeights;       ///< Buffer of last delta weights matrix (#SGD)
   CBufferFloat      *ConcFirstMomentum;      ///< Buffer of first momentum matrix (#ADAM)
   CBufferFloat      *ConcSecondMomentum;     ///< Buffer of second momentum matrix (#ADAM)
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;        ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronConcatenate(void);
   /** Destructor */~CNeuronConcatenate(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint inputs1, uint inputs2, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void) override       const                      {  return defNeuronConcatenate; }///< Identificator of class.@return Type of class
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual CBufferFloat   *getConcatWeights(void)        {  return ConcWeights;     }                 ///< Get pointer of gradient buffer @return Pointer to object
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSoftActorCritic  :  public CNeuronFQF
  {
protected:
   CNeuronConcatenate   cAlphas;
   CBufferFloat         cLogProbs;
   CBufferFloat         cRandomize;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronSoftActorCritic(void) {};
                    ~CNeuronSoftActorCritic(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint actions, uint quantiles, uint numInputs, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   virtual bool      calcAlphaGradients(CNeuronBaseOCL *NeuronOCL);
   virtual bool      GetAlphaLogProbs(vector<float> &log_probs)       { return (cLogProbs.GetData(log_probs) > 0); }
   virtual bool      CalcLogProbs(CBufferFloat *buffer);
   virtual bool      ReCalcLogProbs(void);
   //---
   virtual bool      Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void) override        const                      {  return defNeuronSoftActorCritic; }///< Identificator of class.@return Type of class
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual void      getCheckData(vector<float> &output, vector<float> &grad, vector<float> &quantiles, vector<float> &probability, vector<float> &alphas);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronEmbeddingOCL  :  public CNeuronBaseOCL
  {
protected:
   int               a_Windows[];
   int               i_WindowOut;
   int               i_StackSize;
   int               i_WindowsBuffer;
   int               i_STDBuffer;
   //---
   CBufferFloat      WeightsEmbedding;
   CBufferFloat      FirstMomentumEmbed;
   CBufferFloat      SecondMomentumEmbed;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   ///\ingroup neuron_base_gr
   ///@{
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.
   ///@}
   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
public:
                     CNeuronEmbeddingOCL(void);
                    ~CNeuronEmbeddingOCL(void);
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint window_out, uint &windows[]);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronEmbeddingOCL;                  }///< Identificator of class.@return Type of class
   virtual CLayerDescription* GetLayerInfo(void)   { return CNeuronBaseOCL::GetLayerInfo(); }
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual bool      Clear(void);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPositionEncoder  :  public CNeuronBaseOCL
  {
protected:
   CBufferFloat      PositionEncoder;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)         { return true; }        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)         { return true; }          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronPositionEncoder(void) {};
                    ~CNeuronPositionEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronPEOCL;                  }///< Identificator of class.@return Type of class
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTransposeOCL : public CNeuronBaseOCL
  {
protected:
   uint               iWindow;
   uint               iCount;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override       { return true; }        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;          ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronTransposeOCL(void) {};
                    ~CNeuronTransposeOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronTransposeOCL; }///< Identificator of class.@return Type of class
   //---
   virtual uint      GetWindow(void) const { return iWindow; }
   virtual uint      GetCount(void) const { return iCount; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMH2AttentionOCL       :  public CNeuronBaseOCL
  {
protected:
   uint              iHeads;                                      ///< Number of heads
   uint              iWindow;                                     ///< Input window size
   uint              iUnits;                                      ///< Number of units
   uint              iWindowKey;                                  ///< Size of Key/Query window
   //---
   CNeuronConvOCL    Q_Embedding;
   CNeuronConvOCL    KV_Embedding;
   CNeuronTransposeOCL Transpose;
   int               ScoreIndex;
   CNeuronBaseOCL    MHAttentionOut;
   CNeuronConvOCL    W0;
   CNeuronBaseOCL    AttentionOut;
   CNeuronConvOCL    FF[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      attentionOut(void);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentionInsideGradients(void);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
   /** Constructor */
                     CNeuronMH2AttentionOCL(void);
   /** Destructor */~CNeuronMH2AttentionOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint units_count, ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMH2AttentionOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCGConvOCL  :  public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cInputF;
   CNeuronBaseOCL    cInputS;
   CNeuronBaseOCL    cF;
   CNeuronBaseOCL    cS;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronCGConvOCL(void) {};
                    ~CNeuronCGConvOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint numNeurons,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   //---
   virtual int       Type(void)   const   {  return defNeuronCGConvOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMFTOCL  : public CNeuronMLMHAttentionOCL
  {
protected:
   //---
   CCollection       cTranspose;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      Transpose(CBufferFloat *in, CBufferFloat *out, int rows, int cols);
   virtual bool      MHCA(CBufferFloat *q, CBufferFloat *kv,
                          CBufferFloat *score, CBufferFloat *out);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      MHCAInsideGradients(CBufferFloat *q, CBufferFloat *qg,
                                         CBufferFloat *kv, CBufferFloat *kvg,
                                         CBufferFloat *score, CBufferFloat *aog);

public:
                     CNeuronMFTOCL(void) {};
                    ~CNeuronMFTOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint units_count, uint features,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   //---
   virtual int       Type(void)   const   {  return defNeuronMFTOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronXCiTOCL  : public CNeuronMLMHAttentionOCL
  {
protected:
   //---
   uint              iLPIWindow;
   uint              iLPIStep;
   uint              iBatchCount;
   //---
   CCollection       cLPI;
   CCollection       cLPI_Weights;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      XCiT(CBufferFloat *qkv, CBufferFloat *score, CBufferFloat *out);
   virtual bool      BatchNorm(CBufferFloat *inputs, CBufferFloat *options, CBufferFloat *out);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      XCiTInsideGradients(CBufferFloat *qkv, CBufferFloat *qkvg,
                                         CBufferFloat *score, CBufferFloat *aog);
   virtual bool      BatchNormInsideGradient(CBufferFloat *inputs, CBufferFloat *inputs_g,
         CBufferFloat *options, CBufferFloat *out,
         CBufferFloat *out_g, ENUM_ACTIVATION activation);
   virtual bool      BatchNormUpdateWeights(CBufferFloat *options, CBufferFloat *out_g);

public:
                     CNeuronXCiTOCL(void) {};
                    ~CNeuronXCiTOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint lpi_window, uint heads,
                          uint units_count, uint layers,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronXCiTOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDOTOCL     :  public CNeuronBaseOCL
  {
protected:
   uint              iWindowSize;
   uint              iPrevWindowSize;
   uint              iDimension;
   uint              iUnits;
   uint              iHeads;
   //---
   CNeuronConvOCL    cProjInput;
   CNeuronConvOCL    cQKV;
   int               iScoreBuffer;
   CNeuronBaseOCL    cRelativePositionsBias;
   CNeuronBaseOCL    MHAttentionOut;
   CNeuronConvOCL    cProj;
   CNeuronBaseOCL    AttentionOut;
   CNeuronConvOCL    cFF1;
   CNeuronConvOCL    cFF2;
   CNeuronBaseOCL    SAttenOut;
   CNeuronXCiTOCL    cCAtten;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      DOT(void);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateRelativePositionsBias(void);
   virtual bool      DOTInsideGradients(void);

public:
                     CNeuronDOTOCL(void) {};
                    ~CNeuronDOTOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint dimension, uint heads,
                          uint units_count, uint prev_window,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   //---
   virtual int       Type(void)   const   {  return defNeuronDOTOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFAQOCL  : public CNeuronBaseOCL
  {
protected:
   //---
   CNeuronConvOCL    cF;
   CNeuronBaseOCL    cWv;
   CNeuronBatchNormOCL cNormV;
   CNeuronBaseOCL    cQd;
   CNeuronXCiTOCL    cDQd;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronFAQOCL(void) {};
                    ~CNeuronFAQOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint heads,
                          uint units_count, uint input_units,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   //---
   virtual int       Type(void)   const   {  return defNeuronFAQOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCrossAttention : public CNeuronMH2AttentionOCL
  {
protected:
   uint              iWindow_K;
   uint              iUnits_K;
   CNeuronBaseOCL    *cContext;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context);
   virtual bool      attentionOut(void);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CNeuronBaseOCL *Context);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context);
   virtual bool      AttentionInsideGradients(void);

public:
                     CNeuronCrossAttention(void) : cContext(NULL) {};
                    ~CNeuronCrossAttention(void) { DeleteObj(cContext); }
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint units_count, uint window_k, uint units_k,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronCrossAttenOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGTE : public CNeuronBaseOCL
  {
protected:
   uint              iHeads;                                      ///< Number of heads
   uint              iWindow;                                     ///< Input window size
   uint              iUnits;                                      ///< Number of units
   uint              iWindowKey;                                  ///< Size of Key/Query window
   //---
   CNeuronConvOCL    cQKV;
   CNeuronSoftMaxOCL cSoftMax;
   int               ScoreIndex;
   CNeuronBaseOCL    cMHAttentionOut;
   CNeuronConvOCL    cW0;
   CNeuronBaseOCL    cAttentionOut;
   CNeuronCGConvOCL  cGraphConv[2];
   CNeuronConvOCL    cFF[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentionOut(void);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentionInsideGradients(void);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronGTE(void) {};
                    ~CNeuronGTE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint units_count,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronGTE;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);                ///< Set Training Mode Flag
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CResidualConv  :  public CNeuronBaseOCL
  {
protected:
   int               iWindowOut;
   //---
   CNeuronConvSAMOCL    cConvs[3];
   CNeuronBatchNormOCL cNorm[3];
   CNeuronBaseOCL    cTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CResidualConv(void) {};
                    ~CResidualConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint count,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defResidualConv;   }
   //--- methods for working with files
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;                ///< Set Training Mode Flag
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CCCMREncoder   :  public CNeuronBaseOCL
  {
protected:
   CResidualConv     cResidual[6];
   CNeuronConvOCL    cInput;
   CNeuronBatchNormOCL cNorm;
   CNeuronConvOCL    cOutput;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CCCMREncoder(void) {};
                    ~CCCMREncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint count,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defCCMREncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);                ///< Set Training Mode Flag
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCrossXCiTOCL  :  public CNeuronXCiTOCL
  {
protected:
   CCollection       cConcat;
   CCollection       cValue;
   CCollection       cV_Weights;
   CBufferFloat      TempBuffer;
   uint              iWindow2;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Motion);
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Motion);

public:
                     CNeuronCrossXCiTOCL(void) {};
                    ~CNeuronCrossXCiTOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window1, uint window2, uint lpi_window, uint heads,
                          uint units_count, uint layers,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CNeuronBaseOCL *Motion);
   virtual int       Type(void)   const   {  return defNeuronCrossXCiTOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCCMROCL    :  public CNeuronBaseOCL
  {
protected:
   CCCMREncoder      FeatureExtractor;
   CNeuronBaseOCL    PrevFeatures;
   CNeuronBaseOCL    Motion;
   CNeuronBaseOCL    Temp;
   CCCMREncoder      LocalContext;
   CNeuronXCiTOCL    GlobalContext;
   CNeuronCrossXCiTOCL  MotionContext;
   CNeuronLSTMOCL    RecurentUnit;
   CNeuronConvOCL    UpScale;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronCCMROCL(void) {};
                    ~CNeuronCCMROCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint count,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronCCMROCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);                ///< Set Training Mode Flag
   virtual bool      Clear(void);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronNODEOCL     :  public CNeuronBaseOCL
  {
protected:
   uint              iDimension;
   uint              iVariables;
   uint              iLenth;

   int               iBuffersK[];
   int               iInputsK[];
   int               iMeadl[];
   CBufferFloat      cAlpha;
   CBufferFloat      cTemp;
   CCollection       cBeta;
   CBufferFloat      cSolution;
   CCollection       cWeights;
   //---
   virtual bool      CalculateKBuffer(int k);
   virtual bool      CalculateInputK(CBufferFloat* inputs, int k);
   virtual bool      CalculateOutput(CBufferFloat* inputs);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      CalculateOutputGradient(CBufferFloat* inputs);
   virtual bool      CalculateInputKGradient(CBufferFloat* inputs, int k);
   virtual bool      CalculateKBufferGradient(int k);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronNODEOCL(void) {};
                    ~CNeuronNODEOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint variables, uint lenth,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronNODEOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConformer  :  public CNeuronBaseOCL
  {
protected:
   //---
   int               iWindow;
   int               iDimension;
   int               iHeads;
   int               iVariables;
   int               iCount;
   //---
   CNeuronConvOCL    cQKV;
   CNeuronBaseOCL    cdQKV;
   int               iScore;
   CNeuronBaseOCL    cAttentionOut;
   CNeuronConvOCL    cW0;
   CNeuronNODEOCL    cNODE[3];
   CNeuronConvOCL    cFF[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      attentionOut(void);
   //---
   virtual bool      AttentionInsideGradients(void);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronConformer(void)  :  iScore(-1) {};
                    ~CNeuronConformer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint variables, uint units_count,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronConformerOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRevINDenormOCL   :  public CNeuronBatchNormOCL
  {
protected:
   int               iBatchNormLayer;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   {  return true;   }
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronRevINDenormOCL(void)   :  iBatchNormLayer(-1) {};
                    ~CNeuronRevINDenormOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, int NormLayer, CNeuronBatchNormOCL *normLayer);
   virtual int       GetNormLayer(void)   {  return iBatchNormLayer; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronRevInDenormOCL;    }///< Identificator of class.@return Type of class
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override  {  return true;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronClientOCL  :  public CNeuronBaseOCL
  {
protected:
   //--- Attention
   CNeuronMLMHAttentionOCL cTransformerEncoder;
   CNeuronConvOCL          cProjection;
   //--- Linear model
   CNeuronConvOCL          cLinearModel[];
   //---
   CNeuronBaseOCL          cInput;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronClientOCL(void) {};
                    ~CNeuronClientOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint at_layers, uint count, uint &mlp[],
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronClientOCL;   }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLearnabledPE  :  public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cInputs;
   CNeuronBaseOCL    cPositionEncoder;
   ///\ingroup neuron_base_ff
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);               ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer.
   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);        ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer.

public:
                     CNeuronLearnabledPE(void) {};
                    ~CNeuronLearnabledPE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual int       Type(void)        const                      {  return defNeuronLearnabledPE;    }///< Identificator of class.@return Type of class
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual CBufferFloat* GetPE(void)   const { return cPositionEncoder.getOutput(); }
   virtual bool      AddPEGradient(CBufferFloat *grad)
     {
      if(!grad ||
         !SumAndNormalize(cPositionEncoder.getGradient(), grad, cPositionEncoder.getGradient(), 1, false, 0, 0, 0, 1))
         ReturnFalse;
      return true;
     }
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUShapeAttention  :  public CNeuronBaseOCL
  {
protected:
   CNeuronMLMHAttentionOCL    cAttention[2];
   CNeuronConvOCL             cMergeSplit[2];
   CNeuronBaseOCL             *cNeck;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   //---
public:
                     CNeuronUShapeAttention(void) { activation = None; }
                    ~CNeuronUShapeAttention(void) { DeleteObj(cNeck); }
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch);
   ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations.
   //---
   virtual int       Type(void)   const   {  return defNeuronUShapeAttention;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPatching   :  public CNeuronBaseOCL
  {
protected:
   uint              iWindowIn;
   uint              iStep;
   uint              iWindowOut;
   uint              iVariables;
   uint              iCount;
   //---
   CBufferFloat      cPatchWeights;
   CBufferFloat      cPatchFirstMomentum;
   CBufferFloat      cPatchSecondMomentum;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronPatching(void) {};
                    ~CNeuronPatching(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronPatchingOCL; }
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTiDEOCL :  public CNeuronBaseOCL
  {
protected:
   uint              iHistory;
   uint              iForecast;
   uint              iVariables;
   uint              iFeatures;
   //---
   CResidualConv     acEncoderDecoder[];
   CNeuronConvOCL    cGlobalResidual;
   CResidualConv     acFeatureProjection[2];
   CResidualConv     cTemporalDecoder;
   //---
   CNeuronBaseOCL    cHistoryInput;
   CNeuronBaseOCL    cFeatureInput;
   CNeuronBaseOCL    cEncoderInput;
   CNeuronBaseOCL    cTemporalDecoderInput;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None);

public:
                     CNeuronTiDEOCL(void) {};
                    ~CNeuronTiDEOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint history, uint forecast, uint variables, uint features, uint &encoder_decoder[], ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronTiDEOCL; }
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLegendreWavelets :  public CNeuronConvOCL
  {
protected:
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   { return true; }        ///< Method for updating weights.@param NeuronOCL Pointer to previos layer.

public:
                     CNeuronLegendreWavelets(void) {};
                    ~CNeuronLegendreWavelets(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronLegendreWavelets;   }///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFEDW : public CNeuronBaseOCL
  {
protected:
   //---
   uint                    iWindow;
   uint                    iCount;
   uint                    iFilters;
   //---
   CNeuronLegendreWavelets cWavlets;
   CNeuronBatchNormOCL     cNorm;
   CNeuronSoftMaxOCL       cSoftMax;
   CNeuronConvOCL          cFF[2];
   CNeuronBaseOCL          cReconstruct;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      Reconsruct(CBufferFloat* inputs, CBufferFloat *outputs);

public:
                     CNeuronFEDW(void) {};
                    ~CNeuronFEDW(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint count, uint filters,  ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronFEDW; }
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFITSOCL :  public CNeuronBaseOCL
  {
protected:
   //---
   uint              iWindow;
   uint              iWindowOut;
   uint              iCount;
   uint              iFFTin;
   uint              iIFFTin;
   //---
   CNeuronBaseOCL    cInputsRe;
   CNeuronBaseOCL    cInputsIm;
   CNeuronBaseOCL    cFFTRe;
   CNeuronBaseOCL    cFFTIm;
   CNeuronDropoutOCL cDropRe;
   CNeuronDropoutOCL cDropIm;
   CNeuronConvOCL    cInsideRe1;
   CNeuronConvOCL    cInsideIm1;
   CNeuronConvOCL    cInsideRe2;
   CNeuronConvOCL    cInsideIm2;
   CNeuronBaseOCL    cComplexRe;
   CNeuronBaseOCL    cComplexIm;
   CNeuronBaseOCL    cIFFTRe;
   CNeuronBaseOCL    cIFFTIm;
   CBufferFloat      cClear;
   //---
   virtual bool      FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false);
   virtual bool      ComplexLayerOut(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im);
   virtual bool      ComplexLayerGradient(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronFITSOCL(void) {};
                    ~CNeuronFITSOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint count, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronFITSOCL; }
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFreDFOCL   :  public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iCount;
   uint              iFFTin;
   bool              bTranspose;
   float             fAlpha;
   //---
   CBufferFloat      cForecastFreRe;
   CBufferFloat      cForecastFreIm;
   CBufferFloat      cTargetFreRe;
   CBufferFloat      cTargetFreIm;
   CBufferFloat      cLossFreRe;
   CBufferFloat      cLossFreIm;
   CBufferFloat      cGradientFreRe;
   CBufferFloat      cGradientFreIm;
   CBufferFloat      cTranspose;
   //---
   virtual bool      FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false);
   virtual bool      Transpose(CBufferFloat *inputs, CBufferFloat *outputs, uint rows, uint cols);
   virtual bool      FreqMSA(CBufferFloat *target, CBufferFloat *forecast, CBufferFloat *gradient);
   virtual bool      CumulativeGradient(CBufferFloat *gradient1, CBufferFloat *gradient2, CBufferFloat *cummulative, float alpha);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   {  return true;   }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronFreDFOCL(void)   {};
                    ~CNeuronFreDFOCL(void)   {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint count, float alpha, bool need_transpose = true, ENUM_OPTIMIZATION optimization_type = ADAM, uint batch = 1);
   virtual bool      calcOutputGradients(CArrayFloat *Target, float &error);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronFreDFOCL; }
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronComplexMLMHAttention :  public CNeuronMLMHAttentionOCL
  {
protected:
   virtual bool      ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0);
   virtual bool      AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false);
   virtual bool      AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out);
   virtual bool      ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *momentum1, CBufferFloat *momentum2, uint window, uint window_out, uint step = 0);
   virtual bool      ConvolutionInputGradients(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0);
   virtual bool      AttentionInsideGradients(CBufferFloat *qkv, CBufferFloat *qkv_g, CBufferFloat *scores, CBufferFloat *gradient);
   virtual bool      SumAndNormalize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f);

public:
                     CNeuronComplexMLMHAttention(void) {};
                    ~CNeuronComplexMLMHAttention(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronComplexMLMHAttentionOCL;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronATFNetOCL  :  public CNeuronBaseOCL
  {
protected:
   uint              iHistory;
   uint              iForecast;
   uint              iVariables;
   uint              iFFT;
   //--- T-Block
   CNeuronBatchNormOCL  cNorm;
   CNeuronTransposeOCL  cTranspose;
   CNeuronPositionEncoder  cPositionEncoder;
   CNeuronPatching      cPatching;
   CLayer               caAttention;
   CLayer               caProjection;
   CNeuronRevINDenormOCL cRevIN;
   //--- F-Block
   CBufferFloat*     cInputs;
   CBufferFloat      cInputFreqRe;
   CBufferFloat      cInputFreqIm;
   CNeuronBaseOCL    cInputFreqComplex;
   CBufferFloat      cMainFreqWeights;
   CNeuronBaseOCL    cNormFreqComplex;
   CBufferFloat      cMeans;
   CBufferFloat      cVariances;
   CNeuronComplexMLMHAttention cFreqAtteention;
   CNeuronBaseOCL    cUnNormFreqComplex;
   CBufferFloat      cOutputFreqRe;
   CBufferFloat      cOutputFreqIm;
   CBufferFloat      cOutputTimeSeriasRe;
   CBufferFloat      cOutputTimeSeriasIm;
   CBufferFloat      cOutputTimeSeriasReGrad;
   CBufferFloat      cReconstructInput;
   CBufferFloat      cForecast;
   CBufferFloat      cReconstructInputGrad;
   CBufferFloat      cForecastGrad;
   CBufferFloat      cZero;
   //---
   virtual bool      FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);
   virtual bool      ComplexNormalize(void);
   virtual bool      ComplexUnNormalize(void);
   virtual bool      ComplexNormalizeGradient(void);
   virtual bool      ComplexUnNormalizeGradient(void);
   virtual bool      MainFreqWeights(void);
   virtual bool      WeightedSum(void);
   virtual bool      WeightedSumGradient(void);
   virtual bool      calcReconstructGradient(void);

public:
                     CNeuronATFNetOCL(void) {};
                    ~CNeuronATFNetOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint history, uint forecast, uint variables, uint heads, uint layers, uint &patch[], ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronATFNetOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual CBufferFloat   *getWeights(void);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronS3   :  public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iSegments;
   //---
   CNeuronBaseOCL    cOne;
   CNeuronConvOCL    cShufle;
   CNeuronSoftMaxOCL cProbability;
   CNeuronConvOCL    cWeights;
   CBufferFloat      cPositions;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForwardS3(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradientsS3(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronS3(void)   {};
                    ~CNeuronS3(void)   {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronS3;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMLMHAttentionMLKV :  public CNeuronMLMHAttentionOCL
  {
protected:
   uint              iLayersToOneKV;                             ///< Number of inner layers
   uint              iHeadsKV;                                   ///< Number of heads
   CCollection       KV_Tensors;                                 ///< The collection of tensors of Keys and Values
   CCollection       KV_Weights;                                 ///< The collection of Matrix of weights to previous layer
   CBufferFloat      Temp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out);
   virtual bool      AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);  ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.

public:
                     CNeuronMLMHAttentionMLKV(void) {};
                    ~CNeuronMLMHAttentionMLKV(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMLMHAttentionMLKV;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMLCrossAttentionMLKV :  public CNeuronMLMHAttentionMLKV
  {
protected:
   uint              iWindowKV;
   uint              iUnitsKV;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out) override;
   virtual bool      AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv,
         CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }

public:
                     CNeuronMLCrossAttentionMLKV(void) {};
                    ~CNeuronMLCrossAttentionMLKV(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint window_kv, uint heads_kv, uint units_count, uint units_count_kv,
                          uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMLCrossAttentionMLKV;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual uint      GetSecondBufferSize(void)  {  return (iWindowKV * iUnitsKV); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCSCMOCL :  public CNeuronBaseOCL
  {
protected:
   uint              i_Count;
   uint              i_Variables;
   bool              b_NeedTranspose;
   //---
   CArrayInt         ia_Windows;
   CArrayObj         caTranspose;
   CArrayObj         caConvolutions;
   CArrayObj         caMLP;
   CArrayObj         caTemp;
   CArrayObj         caConvOutputs;
   CArrayObj         caConvGradients;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronCSCMOCL(void) {};
                    ~CNeuronCSCMOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &windows[], uint variables, uint inputs_count, bool need_transpose, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronCSCMOCL;   }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSPyrAttentionOCL :  public CNeuronBaseOCL
  {
protected:
   uint              iWindowIn;
   uint              iWindowKey;
   uint              iHeads;
   uint              iHeadsKV;
   uint              iCount;
   uint              iPAMLayers;
   //---
   CArrayObj         caS3;
   CArrayObj         caQuery;
   CArrayObj         caKV;
   CArrayInt         caScore;
   CArrayObj         caAttentionOut;
   CArrayObj         caW0;
   CNeuronConvOCL    cFF1;
   CNeuronConvOCL    cFF2;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, int scores, CBufferFloat *out, int window);
   virtual bool      AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, int scores, CBufferFloat *gradient);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);  ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer.
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.
   virtual void      ArraySetOpenCL(CArrayObj *array, COpenCLMy *obj);

public:
                     CNeuronSPyrAttentionOCL(void) {};
                    ~CNeuronSPyrAttentionOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_key, uint heads, uint heads_kv, uint units_count, uint pam_layers, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSPyrAttentionMLKV;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPLROCL  :  public CNeuronBaseOCL
  {
protected:
   bool              bTranspose;
   int               icIsTTP;
   int               iVariables;
   int               iCount;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  { return true; }

public:
                     CNeuronPLROCL(void)  : bTranspose(false) {};
                    ~CNeuronPLROCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint units_count, bool transpose, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronPLROCL;   }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTPMEncoder :  public CNeuronLSTMOCL
  {
protected:
   bool                 bTSinRow;
   //---
   CNeuronCSCMOCL       cFeatureExtraction;
   CNeuronBaseOCL       cMemAndHidden;
   CNeuronConcatenate   cConcatenated;
   CNeuronSoftMaxOCL    cSoftMax;
   CNeuronBaseOCL       cAttentionOut;
   CNeuronTransposeOCL  cTranspose;
   CBufferFloat         cTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTPMEncoder(void) {};
                    ~CNeuronTPMEncoder(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void) override       const                      {  return defNeuronTPMEncoder; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTPM :  public CNeuronLSTMOCL
  {
protected:
   CNeuronTPMEncoder    cEncoder;
   CNeuronPLROCL        cFeatureExtraction;
   CNeuronBaseOCL       cMemAndHidden;
   CNeuronConcatenate   cConcatenated;
   CNeuronSoftMaxOCL    cSoftMax;
   CNeuronBaseOCL       cAttentionOut;
   CNeuronConcatenate   cAttAndFeature;
   CBufferFloat         cTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTPM(void) {};
                    ~CNeuronTPM(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void) override       const                      {  return defNeuronTPM; }
   virtual void      SetOpenCL(COpenCLMy *obj);
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTNNEncoder  :  public CNeuronMLMHAttentionMLKV
  {
protected:
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTNNEncoder(void)  {};
                    ~CNeuronSTNNEncoder(void)  {};
   //---
   virtual int       Type(void)   override const   {  return defNeuronSTNNEncoder;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTNNDecoder   :  public CNeuronMLCrossAttentionMLKV
  {
protected:
   CNeuronSTNNEncoder      cEncoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;

public:
                     CNeuronSTNNDecoder(void) {};
                    ~CNeuronSTNNDecoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSTNNDecoder;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSparseTSF  :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL       cConvolution;
   CNeuronTransposeOCL  acTranspose[4];
   CNeuronConvOCL       cForecast;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSparseTSF(void) {};
                    ~CNeuronSparseTSF(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint sequence, uint variables, uint period, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSparseTSF;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTEMPOOCL   :  public CNeuronBaseOCL
  {
protected:
   //--- constants
   uint              iVariables;
   uint              iSequence;
   uint              iForecast;
   uint              iFFT;
   //--- Trend
   CNeuronPLROCL     cPLR;
   CNeuronBaseOCL    cTrend;
   //--- Seasons
   CNeuronBaseOCL    cInputSeasons;
   CNeuronTransposeOCL cTranspose[2];
   CBufferFloat      cInputFreqRe;
   CBufferFloat      cInputFreqIm;
   CNeuronBaseOCL    cInputFreqComplex;
   CNeuronBaseOCL    cNormFreqComplex;
   CBufferFloat      cMeans;
   CBufferFloat      cVariances;
   CNeuronComplexMLMHAttention cFreqAtteention;
   CNeuronBaseOCL    cUnNormFreqComplex;
   CBufferFloat      cOutputFreqRe;
   CBufferFloat      cOutputFreqIm;
   CNeuronBaseOCL    cOutputTimeSeriasRe;
   CBufferFloat      cOutputTimeSeriasIm;
   CBufferFloat      cZero;
   //--- Noise
   CNeuronBaseOCL    cResidual;
   //--- Forecast
   CNeuronBaseOCL    cConcatInput;
   CNeuronBatchNormOCL cNormalize;
   CNeuronPatching   cPatching;
   CNeuronBatchNormOCL cNormalizePLR;
   CNeuronPatching   cPatchingPLR;
   CNeuronPositionEncoder acPE[2];
   CNeuronMLCrossAttentionMLKV cAttention;
   CNeuronTransposeOCL  cTransposeAtt;
   CNeuronConvOCL    acForecast[2];
   CNeuronTransposeOCL  cTransposeFrc;
   CNeuronRevINDenormOCL cRevIn;
   CNeuronConvOCL    cSum;
   //--- Complex functions
   virtual bool      FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false);
   virtual bool      ComplexNormalize(void);
   virtual bool      ComplexUnNormalize(void);
   virtual bool      ComplexNormalizeGradient(void);
   virtual bool      ComplexUnNormalizeGradient(void);
   //---
   bool              CutTrendAndOther(CBufferFloat *inputs);
   bool              CutTrendAndOtherGradient(CBufferFloat *inputs_gr);
   bool              CutOneFromAnother(void);
   bool              CutOneFromAnotherGradient(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronTEMPOOCL(void)   {};
                    ~CNeuronTEMPOOCL(void)   {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint sequence, uint variables, uint forecast, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronTEMPOOCL;   }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual CBufferFloat   *getWeights(void)  override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMVMHAttentionMLKV :  public CNeuronMLMHAttentionOCL
  {
protected:
   uint              iLayersToOneKV;                              ///< Number of inner layers
   uint              iHeadsKV;                                    ///< Number of heads
   uint              iVariables;                                  ///< Number of variables
   CCollection       KV_Tensors;                                  ///< The collection of tensors of Keys and Values
   CCollection       K_Tensors;                                   ///< The collection of tensors of Keys
   CCollection       K_Weights;                                   ///< The collection of Matrix of weights to previous layer
   CCollection       V_Tensors;                                   ///< The collection of tensors of Values
   CCollection       V_Weights;                                   ///< The collection of Matrix of weights to previous layer
   CBufferFloat      Temp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out);
   virtual bool      AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMVMHAttentionMLKV(void) {};
                    ~CNeuronMVMHAttentionMLKV(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMVMHAttentionMLKV;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMVCrossAttentionMLKV :  public CNeuronMVMHAttentionMLKV
  {
protected:
   uint              iWindowKV;
   uint              iUnitsKV;
   uint              iVariablesKV;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context);
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out);
   virtual bool      AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context);        ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer.

public:
                     CNeuronMVCrossAttentionMLKV(void) {};
                    ~CNeuronMVCrossAttentionMLKV(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint window_kv,
                          uint heads_kv, uint units_count, uint units_count_kv,
                          uint layers, uint layers_to_one_kv, uint variables_q, uint variables_kv,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMVCrossMHAttentionMLKV;   }
   //---
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   virtual CBufferFloat* getWeights(void);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronInjectTST  :  public CNeuronBaseOCL
  {
protected:
   CNeuronPatching            cPatching;
   CNeuronLearnabledPE        cCIPosition;
   CNeuronLearnabledPE        cCMPosition;
   CNeuronMVMHAttentionMLKV   cChanelIndependentAttention;
   CNeuronMLMHAttentionMLKV   cChanelMixAttention;
   CNeuronMVCrossAttentionMLKV   cGlobalInjectionAttention;
   CBufferFloat               cTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronInjectTST(void) {};
                    ~CNeuronInjectTST(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronInjectTST; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual CBufferFloat   *getWeights(void)  override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSSMOCL  :  public CNeuronBaseOCL
  {
protected:
   uint              iWindowHidden;
   CNeuronBaseOCL    cHiddenStates;
   CNeuronConvOCL    cA;
   CNeuronConvOCL    cB;
   CNeuronBaseOCL    cAB;
   CNeuronConvOCL    cC;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronSSMOCL(void)  {};
                    ~CNeuronSSMOCL(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSSMOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMambaOCL   :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL    cXProject;
   CNeuronConvOCL    cZProject;
   CNeuronConvOCL    cInsideConv;
   CNeuronSSMOCL     cSSM;
   CNeuronBaseOCL    cZSSM;
   CNeuronConvOCL    cOutProject;
   CBufferFloat      Temp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronMambaOCL(void) {};
                    ~CNeuronMambaOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMambaOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMambaBlockOCL :  public   CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   CNeuronMambaOCL   cMamba;
   CNeuronBaseOCL    cMambaResidual;
   CNeuronConvOCL    cFF[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronMambaBlockOCL(void) {};
                    ~CNeuronMambaBlockOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override  const   {  return defNeuronMambaBlockOCL; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTrajLLMOCL :  public CNeuronBaseOCL
  {
protected:
   //--- State Encoder
   CNeuronLSTMOCL                cStateRNN;
   CNeuronConvOCL                cStateMLP[2];
   //--- Variables Encoder
   CNeuronTransposeOCL           cTranspose;
   CNeuronLSTMOCL                cVariablesRNN;
   CNeuronConvOCL                cVariablesMLP[2];
   //--- Context Encoder
   CNeuronLearnabledPE           cStatePE;
   CNeuronLearnabledPE           cVariablesPE;
   CNeuronMLMHAttentionMLKV      cStateToState;
   CNeuronMLCrossAttentionMLKV   cVariableToState;
   CNeuronMLCrossAttentionMLKV   cStateToVariable;
   CNeuronBaseOCL                cContext;
   CNeuronConvOCL                cContextMLP[2];
   //---
   CNeuronMLMHAttentionMLKV      cHighLevelInteraction;
   CNeuronMambaBlockOCL          caMamba[3];
   CNeuronMLCrossAttentionMLKV   cLaneAware;
   CNeuronConvOCL                caForecastMLP[2];
   CNeuronTransposeOCL           cTransposeOut;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronTrajLLMOCL(void) {};
                    ~CNeuronTrajLLMOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronTrajLLMOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUniTraj    :  public CNeuronBaseOCL
  {
protected:
   uint              iVariables;
   float             fDropout;
   //---
   CBufferFloat      cHistoryMask;
   CBufferFloat      cFutureMask;
   CNeuronBaseOCL    cData;
   CNeuronLearnabledPE cPE;
   CNeuronMVMHAttentionMLKV   cEncoder;
   CNeuronBaseOCL    cDForw;
   CNeuronBaseOCL    cDBakw;
   CNeuronConvOCL    cProjDForw;
   CNeuronConvOCL    cProjDBakw;
   CNeuronBaseOCL    cDataDForw;
   CNeuronBaseOCL    cDataDBakw;
   CNeuronBaseOCL    cConcatDataDForwBakw;
   CNeuronMambaBlockOCL cSSM[4];
   CNeuronConvOCL    cStat;
   CNeuronTransposeOCL cTranspStat;
   CVAE              cVAE;
   CNeuronTransposeOCL cTranspVAE;
   CNeuronConvOCL    cDecoder[2];
   CNeuronTransposeOCL cTranspResult;
   //---
   virtual bool      Prepare(const CBufferFloat* history, const CBufferFloat* future);
   virtual bool      PrepareGrad(CBufferFloat* history_gr, CBufferFloat* future_gr);
   virtual bool      BTS(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { return feedForward(NeuronOCL, NULL); }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return calcInputGradients(NeuronOCL, NULL, NULL, None); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return updateInputWeights(NeuronOCL, NULL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   //---

public:
                     CNeuronUniTraj(void) {};
                    ~CNeuronUniTraj(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronUniTrajOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHiVTAAEncoder : public CNeuronMVMHAttentionMLKV
  {
protected:
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHiVTAAEncoder(void) {};
                    ~CNeuronHiVTAAEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronHiVTAAEncoder;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTransposeRCDOCL  :  public CNeuronTransposeOCL
  {
protected:
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTransposeRCDOCL(void) {};
                    ~CNeuronTransposeRCDOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)        const                      {  return defNeuronTransposeRCDOCL; }///< Identificator of class.@return Type of class
   virtual uint      GetDimension(void) const { return Neurons() / (iCount * iWindow); }

  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHiVTOCL    :  public CNeuronBaseOCL
  {
protected:
   uint              iHistory;
   uint              iVariables;
   uint              iForecast;
   uint              iNumTraj;
   //---
   CNeuronBaseOCL    cDataTAD;
   CNeuronConvOCL    cEmbeddingTAD;
   CNeuronTransposeRCDOCL cTransposeATD;
   CNeuronHiVTAAEncoder cAAEncoder;
   CNeuronTransposeRCDOCL cTransposeTAD;
   CNeuronLearnabledPE  cPosEmbeddingTAD;
   CNeuronMVMHAttentionMLKV cTemporalEncoder;
   CNeuronLearnabledPE  cPosLineEmbeddingTAD;
   CNeuronPatching   cLineEmbeddibg;
   CNeuronMVCrossAttentionMLKV cALEncoder;
   CNeuronMLMHAttentionMLKV cGlobalEncoder;
   CNeuronTransposeOCL cTransposeADT;
   CNeuronConvOCL    cDecoder[3]; // Agent* Traj * Forecast
   CNeuronConvOCL    cProbProj;
   CNeuronSoftMaxOCL cProbability; // Agent* Traj
   CNeuronBaseOCL    cForecast;
   CNeuronTransposeOCL cTransposeTA;
   //---
   virtual bool      Prepare(const CNeuronBaseOCL *history);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHiVTOCL(void) {};
                    ~CNeuronHiVTOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, uint num_traj, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronHiVTOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPointNetOCL   :  public CNeuronBaseOCL
  {
protected:
   CNeuronPointNetOCL   *cTNet1;
   CNeuronBaseOCL       *cTurned1;
   CNeuronConvOCL       cPreNet[2];
   CNeuronBatchNormOCL  cPreNetNorm[2];
   CNeuronPointNetOCL   *cTNet2;
   CNeuronBaseOCL       *cTurned2;
   CNeuronConvOCL       cFeatureNet[3];
   CNeuronBatchNormOCL  cFeatureNetNorm[3];
   CNeuronTransposeOCL  cTranspose;
   CNeuronProofOCL      cMaxPool;
   CNeuronBaseOCL       cFinalMLP[2];
   //---
   virtual bool         OrthoganalLoss(CNeuronBaseOCL *NeuronOCL, bool add = false);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPointNetOCL(void) {};
                    ~CNeuronPointNetOCL(void);
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint output, bool use_tnets,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronPointNetOCL; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPointNet2Local   :  public CNeuronConvOCL
  {
protected:
   float                fRadius;
   uint                 iUnits;
   //---
   CBufferFloat         cDistance;
   CNeuronConvOCL       cFeatureNet[3];
   CNeuronBatchNormOCL  cFeatureNetNorm[3];
   CNeuronBaseOCL       cLocalMaxPool;
   CNeuronConvOCL       cFinalMLP;
   //---
   virtual bool         CalcDistance(CNeuronBaseOCL *NeuronOCL);
   virtual bool         LocalMaxPool(void);
   virtual bool         LocalMaxPoolGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
public:
                     CNeuronPointNet2Local(void) {};
                    ~CNeuronPointNet2Local(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint window_out, float radius,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronPointNet2LocalOCL; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPointNet2OCL   :  public CNeuronPointNetOCL
  {
protected:
   CNeuronPointNetOCL   *cTNetG;
   CNeuronBaseOCL       *cTurnedG;
   //---
   CNeuronPointNet2Local caLocalPointNet[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
public:
                     CNeuronPointNet2OCL(void) {};
                    ~CNeuronPointNet2OCL(void) ;
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint output, bool use_tnets,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronPointNet2OCL; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPointFormer   :  public CNeuronPointNet2OCL
  {
protected:
   CNeuronMLMHSparseAttention    caLocalAttention[2];
   CNeuronMLCrossAttentionMLKV   caLocalGlobalAttention[2];
   CNeuronMLMHAttentionMLKV      caGlobalAttention[2];
   CNeuronLearnabledPE           caLocalPE[2];
   CNeuronLearnabledPE           caGlobalPE[2];
   CNeuronBaseOCL                cConcatenate;
   CNeuronConvOCL                cScale;
   //---
   CBufferFloat                  *cbTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPointFormer(void) {};
                    ~CNeuronPointFormer(void) { DeleteObj(cbTemp); }
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint output, bool use_tnets,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronPointFormer; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSceneSpecific :  public CNeuronMLCrossAttentionMLKV
  {
protected:
   CNeuronBaseOCL    cOne;
   CNeuronBaseOCL    cSceneSpecificKnowledge;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return feedForward(NeuronOCL); }
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return updateInputWeights(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSceneSpecific(void) {};
                    ~CNeuronSceneSpecific(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override  const   {  return defNeuronSceneSpecific;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMLMHSceneConditionAttention  :  public CNeuronMLMHAttentionMLKV
  {
protected:
   CLayer            cSceneAgnostic;
   CLayer            cSceneSpecific;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMLMHSceneConditionAttention(void) {};
                    ~CNeuronMLMHSceneConditionAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override  const   {  return defNeuronMLMHSceneConditionAttention;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHyperDet   :  public CNeuronPointNet2OCL
  {
protected:
   CNeuronMLMHSparseAttention    caLocalAttention[2];
   CNeuronMLCrossAttentionMLKV   caLocalGlobalAttention[2];
   CNeuronMLMHSceneConditionAttention  caGlobalAttention[2];
   CNeuronLearnabledPE           caLocalPE[2];
   CNeuronLearnabledPE           caGlobalPE[2];
   CNeuronBaseOCL                cConcatenate;
   CNeuronConvOCL                cScale;
   //---
   CBufferFloat                  *cbTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHyperDet(void) {};
                    ~CNeuronHyperDet(void) { DeleteObj(cbTemp); }
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint output, bool use_tnets,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronHyperDet; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSEFormer   :    public CNeuronPointNet2OCL
  {
protected:
   uint              iUnits;
   uint              iPoints;
   //---
   CLayer            cQuery;
   CLayer            cKey;
   CLayer            cValue;
   CLayer            cKeyValue;
   CArrayInt         cScores;
   CLayer            cMHAttentionOut;
   CLayer            cAttentionOut;
   CLayer            cResidual;
   CLayer            cFeedForward;
   CLayer            cCenterPoints;
   CLayer            cFinalAttention;
   CNeuronMLCrossAttentionMLKV SEOut;
   CBufferFloat      cbTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      AttentionOut(CBufferFloat *q, CBufferFloat *kv, int scores, CBufferFloat *out);
   virtual bool      AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g,
         CBufferFloat *kv, CBufferFloat *kv_g,
         int scores, CBufferFloat *gradient);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   //---

public:
                     CNeuronSEFormer(void) {};
                    ~CNeuronSEFormer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint output, bool use_tnets,
                          uint center_points, uint center_window,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronSEFormer; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSPFormer   : public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iUnits;
   uint              iHeads;
   uint              iSPWindow;
   uint              iSPUnits;
   uint              iSPHeads;
   uint              iWindowKey;
   uint              iLayers;
   uint              iLayersSP;
   //---
   CLayer            cSuperPoints;
   CLayer            cQuery;
   CLayer            cSPKeyValue;
   CLayer            cMask;
   CArrayInt         cScores;
   CLayer            cMHCrossAttentionOut;
   CLayer            cCrossAttentionOut;
   CLayer            cResidual;
   CLayer            cQKeyValue;
   CLayer            cMHSelfAttentionOut;
   CLayer            cSelfAttentionOut;
   CLayer            cFeedForward;
   CBufferFloat      cTempSP;
   CBufferFloat      cTempQ;
   CBufferFloat      cTempSelfKV;
   CBufferFloat      cTempCrossKV;
   //---
   virtual bool      CreateBuffers(void);
   virtual bool      AttentionOut(CNeuronBaseOCL *q, CNeuronBaseOCL *kv, const int scores,
                                  CNeuronBaseOCL *out, CNeuronBaseOCL *mask,
                                  const int units,
                                  const int heads,
                                  const int units_kv,
                                  const int heads_kv,
                                  const int dimension,
                                  const float mask_level = 0.5f);
   virtual bool      AttentionInsideGradients(CNeuronBaseOCL *q, CNeuronBaseOCL *kv, const int scores,
         CNeuronBaseOCL *out, CNeuronBaseOCL *mask,
         const int units,
         const int heads,
         const int units_kv,
         const int heads_kv,
         const int dimension,
         const float mask_level = 0.5f);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSPFormer(void) {};
                    ~CNeuronSPFormer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint window_sp, uint units_sp, uint heads_sp,
                          uint layers, uint layers_to_sp,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronSPFormer; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMAFT   : public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iUnits;
   uint              iHeads;
   uint              iSPWindow;
   uint              iSPUnits;
   uint              iSPHeads;
   uint              iWindowKey;
   uint              iLayers;
   uint              iLayersSP;
   //---
   CLayer            cSuperPoints;
   CLayer            cQuery;
   CLayer            cQPosition;
   CLayer            cQKey;
   CLayer            cQValue;
   CLayer            cMHSelfAttentionOut;
   CLayer            cSelfAttentionOut;
   CLayer            cSPKey;
   CLayer            cSPValue;
   CArrayInt         cScores;
   CArrayInt         cPositionBias;
   CLayer            cMHCrossAttentionOut;
   CLayer            cCrossAttentionOut;
   CLayer            cResidual;
   CLayer            cFeedForward;
   CBufferFloat      cTempSP;
   CBufferFloat      cTempQ;
   CBufferFloat      cTempCrossK;
   CBufferFloat      cTempCrossV;
   //---
   virtual bool      CreateBuffers(void);
   virtual bool      CalcPositionBias(CBufferFloat *pos_q, CBufferFloat *pos_k, const int pos_bias,
                                      const int units,
                                      const int units_kv,
                                      const int dimension);
   virtual bool      AttentionOut(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v,
                                  const int scores, CNeuronBaseOCL *out, const int pos_bias,
                                  const int units,
                                  const int heads,
                                  const int units_kv,
                                  const int heads_kv,
                                  const int dimension,
                                  const bool use_pos_bias);
   virtual bool      AttentionInsideGradients(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v,
         const int scores, CNeuronBaseOCL *out,
         const int units, const int heads,
         const int units_kv, const int heads_kv,
         const int dimension);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMAFT(void) {};
                    ~CNeuronMAFT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint window_sp, uint units_sp, uint heads_sp,
                          uint layers, uint layers_to_sp,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMAFT; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGRES :  public CNeuronMAFT
  {
protected:
   CLayer            cReference;
   CLayer            cRefKey;
   CLayer            cRefValue;
   CLayer            cMHRefAttentionOut;
   CLayer            cRefAttentionOut;
   //---
   virtual bool      CreateBuffers(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronGRES(void) {};
                    ~CNeuronGRES(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint window_sp, uint units_sp, uint heads_sp,
                          uint ref_size, uint layers, uint layers_to_sp,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronGRES; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGEGWA   :  public CNeuronUShapeAttention
  {
protected:
   CNeuronBaseOCL    cResidual;
   CNeuronMLCrossAttentionMLKV   cCrossAttention;
   CBufferFloat      cTemp;
   bool              bAddNeckGradient;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronGEGWA(void)   :  bAddNeckGradient(false) {};
                    ~CNeuronGEGWA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint units_count,
                          uint window_kv, uint heads_kv, uint units_count_kv,
                          uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronGEGWA;   }///< Identificator of class.@return Type of class
   //--- methods for working with files
   virtual bool      Save(int const file_handle);  ///< Save method @param[in] file_handle handle of file @return logical result of operation
   virtual bool      Load(int const file_handle);  ///< Load method @param[in] file_handle handle of file @return logical result of operation
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual CNeuronBaseOCL* GetInsideLayer(const int layer) const;
   virtual void      AddNeckGradient(const bool flag) {  bAddNeckGradient = flag; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLPC  :  public CNeuronMLCrossAttentionMLKV
  {
protected:
   CNeuronBaseOCL    cOne;
   CNeuronBaseOCL    cPrimitives;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return feedForward(NeuronOCL); }
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return updateInputWeights(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronLPC(void) {};
                    ~CNeuronLPC(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override  const   {  return defNeuronLPC;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronOCM  :  public CNeuronBaseOCL
  {
protected:
   uint              iPrimWindow;
   uint              iPrimUnits;
   uint              iPrimHeads;
   uint              iContWindow;
   uint              iContUnits;
   uint              iContHeads;
   uint              iWindowKey;
   //---
   CLayer            cQuery;
   CLayer            cKey;
   CLayer            cValue;
   CLayer            cMHAttentionOut;
   CLayer            cAttentionOut;
   CArrayInt         cScores;
   CLayer            cResidual;
   CLayer            cFeedForward;
   //---
   virtual bool      CreateBuffers(void);
   virtual bool      AttentionOut(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v,
                                  const int scores, CNeuronBaseOCL *out,
                                  const int units,
                                  const int heads,
                                  const int units_kv,
                                  const int heads_kv,
                                  const int dimension);
   virtual bool      AttentionInsideGradients(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v,
         const int scores, CNeuronBaseOCL *out,
         const int units, const int heads,
         const int units_kv, const int heads_kv,
         const int dimension);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }

public:
                     CNeuronOCM(void) {};
                    ~CNeuronOCM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint prim_window, uint window_key, uint prim_units, uint prim_heads,
                          uint cont_window, uint cont_units, uint cont_heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronOCM; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *Primitives, CNeuronBaseOCL *Context);
   virtual bool      calcInputGradients(CNeuronBaseOCL *Primitives, CNeuronBaseOCL *Context);
   virtual bool      updateInputWeights(CNeuronBaseOCL *Primitives, CNeuronBaseOCL *Context);
   //---
   virtual uint      GetPrimitiveWindow(void)   const { return iPrimWindow; }
   virtual uint      GetContextWindow(void)     const { return iContWindow; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRefMask    :  public CNeuronBaseOCL
  {
protected:
   CNeuronGEGWA      cGEGWA;
   CLayer            cContentEncoder;
   CLayer            cBackGround;
   CNeuronLPC        cLPC;
   CLayer            cDecoder;
   CNeuronOCM        cOCM;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronRefMask(void) {};
                    ~CNeuronRefMask(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint content_size, uint content_units,
                          uint primitive_units, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronRefMask; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRelativeSelfAttention   :  public CNeuronBaseOCL
  {
protected:
   uint                    iWindow;
   uint                    iWindowKey;
   uint                    iHeads;
   uint                    iUnits;
   int                     iScore;
   //---
   CNeuronConvOCL       cQuery;
   CNeuronConvOCL       cKey;
   CNeuronConvOCL       cValue;
   CNeuronTransposeOCL     cTranspose;
   CNeuronBaseOCL          cDistance;
   CLayer                  cBKey;
   CLayer                  cBValue;
   CLayer                  cGlobalContentBias;
   CLayer                  cGlobalPositionalBias;
   CLayer                  cMHAttentionPooling;
   CLayer                  cScale;
   CBufferFloat            cTemp;
   //---
   virtual bool      AttentionOut(void);
   virtual bool      AttentionGradient(void);

   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronRelativeSelfAttention(void) : iScore(-1) {};
                    ~CNeuronRelativeSelfAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronRelativeSelfAttention; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint      GetWindow(void) const { return iWindow; }
   virtual uint      GetUnits(void) const { return iUnits; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRMAT :  public CNeuronBaseOCL
  {
protected:
   CLayer            cLayers;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronRMAT(void) {};
                    ~CNeuronRMAT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronRMAT; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override;
   //---
   virtual uint      GetWindow(void) const
     {
      CNeuronRelativeSelfAttention* neuron = cLayers[0];
      return (!neuron ? 0 : neuron.GetWindow());
     }
   virtual uint      GetUnits(void) const
     {
      CNeuronRelativeSelfAttention* neuron = cLayers[0];
      return (!neuron ? 0 : neuron.GetUnits());
     }
   virtual void      TrainMode(bool flag) override
     {
      bTrain = flag;
      CNeuronBaseOCL *neuron = NULL;
      for(int i = 0; i < cLayers.Total(); i++)
        {
         neuron = cLayers[i];
         if(!!neuron)
            neuron.TrainMode(bTrain);
        }
     }
   //---
   virtual bool      SetGradient(CBufferFloat *buffer, bool delete_prev = true)
     {
      CNeuronBaseOCL *last = cLayers[-1];
      if(!last ||
         !last.SetGradient(buffer, delete_prev) ||
         !CNeuronBaseOCL::SetGradient(last.getGradient(), delete_prev))
         ReturnFalse;
      return true;
     }
   virtual bool      SetOutput(CBufferFloat *buffer, bool delete_prev = true)
     {
      CNeuronBaseOCL *last = cLayers[-1];
      if(!last ||
         !last.SetOutput(buffer, delete_prev) ||
         !CNeuronBaseOCL::SetOutput(last.getOutput(), delete_prev))
         ReturnFalse;
      return true;
     }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHAttentionPooling  :  public   CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iHeads;
   uint              iUnits;
   CLayer            cNeurons;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMHAttentionPooling(void) {};
                    ~CNeuronMHAttentionPooling(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMHAttentionPooling; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMotifs    :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL    cMotifs;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMotifs(void) {};
                    ~CNeuronMotifs(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint window, uint step, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMotifs; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiScaleAttention :  public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iUnits;
   //---
   CNeuronBaseOCL    cWideInputs;
   CNeuronRelativeSelfAttention  cAttentions[4];
   CNeuronBaseOCL    cConcatAttentions;
   CNeuronMHAttentionPooling cPooling;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultiScaleAttention(void) {};
                    ~CNeuronMultiScaleAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMultiScaleAttention; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMolformer  :  public CNeuronRMAT
  {
public:
                     CNeuronMolformer(void) {};
                    ~CNeuronMolformer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads, uint layers,
                          uint motif_window, uint motif_step,
                          ENUM_OPTIMIZATION optimization_type, uint batch); //Molformer
   //---
   virtual int       Type(void) override   const   {  return defNeuronMolformer; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRelativeCrossAttention   :  public CNeuronRelativeSelfAttention
  {
protected:
   uint                    iUnitsKV;
   //---
   CLayer                  cKVProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronRelativeCrossAttention(void) {};
                    ~CNeuronRelativeCrossAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads,
                          uint window_kv, uint units_kv,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronRelativeCrossAttention; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint      GetUnitsKV(void) const { return iUnitsKV; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMotifEncoder  :  public CNeuronRMAT
  {
public:
                     CNeuronMotifEncoder(void) {};
                    ~CNeuronMotifEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronMotifEncoder; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPropertyAwareAttention    :  public CNeuronRMAT
  {
protected:
   CBufferFloat      cTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPropertyAwareAttention(void) {};
                    ~CNeuronPropertyAwareAttention(void) {};

   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint properties,
                          uint units_count, uint heads, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronPropertyAwareAttention; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAMCT    :  public CNeuronBaseOCL
  {
protected:
   CNeuronRMAT                   cAtomEncoder;
   CNeuronMotifEncoder           cMotifEncoder;
   CLayer                        cMotifProjection;
   CNeuronPropertyAwareAttention cPropertyDecoder;
   CLayer                        cPropertyProjection;
   CNeuronBaseOCL                cConcatenate;
   CNeuronMHAttentionPooling     cPooling;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAMCT(void) {};
                    ~CNeuronAMCT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint properties,
                          uint units_count, uint heads, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronAMCT; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronNAFS :  public CNeuronBaseOCL
  {
protected:
   uint                 iDimension;
   uint                 iSmoothing;
   uint                 iUnits;
   //---
   CNeuronBaseOCL       cFeatureSmoothing;
   CNeuronTransposeOCL  cTranspose;
   CNeuronBaseOCL       cDistance;
   CNeuronSoftMaxOCL    cAdaptation;
   //---
   virtual bool      FeatureSmoothing(const CNeuronBaseOCL *neuron, const CNeuronBaseOCL *smoothing);
   virtual bool      FeatureSmoothingGradient(const CNeuronBaseOCL *neuron, const CNeuronBaseOCL *smoothing);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }

public:
                     CNeuronNAFS(void) {};
                    ~CNeuronNAFS(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronNAFS; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBatchNormWithNoise  :  public CNeuronBatchNormOCL
  {
protected:
   CBufferFloat      cNoise;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronBatchNormWithNoise(void) {};
                    ~CNeuronBatchNormWithNoise(void) {};
   //---
   virtual int       Type(void)        const                      {  return defNeuronBatchNormWithNoise;    }///< Identificator of class.@return Type of class
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDiffusion  : public CNeuronUShapeAttention
  {
protected:
   CNeuronBatchNormWithNoise  cAddNoise;
   CNeuronBaseOCL             cResidual;
   CNeuronRevINDenormOCL      cRevIn;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronDiffusion(void) {};
                    ~CNeuronDiffusion(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronDiffusion;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag)                         {  bTrain = flag; cAddNoise.TrainMode(bTrain); } ///< Set Training Mode Flag
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHyperProjection   :  public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iUnits;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   { return true; }

public:
                     CNeuronHyperProjection(void) : iWindow(-1), iUnits(-1) {};
                    ~CNeuronHyperProjection(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronHyperProjection;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHyperboloids  :  public CNeuronBaseOCL
  {
protected:
   uint              iWindows;
   uint              iUnits;
   uint              iCentroids;
   //---
   CLayer            cHyperCentroids;
   CLayer            cHyperCurvatures;
   //---
   int               iProducts;
   int               iDistances;
   int               iNormes;
   //---
   virtual bool      LogMap(CNeuronBaseOCL *featers, CNeuronBaseOCL *centroids, CNeuronBaseOCL *curvatures, CNeuronBaseOCL *outputs);
   virtual bool      LogMapGrad(CNeuronBaseOCL *featers, CNeuronBaseOCL *centroids, CNeuronBaseOCL *curvatures, CNeuronBaseOCL *outputs);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronHyperboloids(void) : iWindows(0), iUnits(0), iCentroids(0), iProducts(-1), iDistances(-1), iNormes(-1) {};
                    ~CNeuronHyperboloids(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronHyperboloids;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      TrainMode(bool flag);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHypDiff :  public CNeuronRMAT
  {
public:
                     CNeuronHypDiff(void) {};
                    ~CNeuronHypDiff(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint centroids,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronHypDiff; }
   //---
   virtual uint      GetWindow(void) const override
     {
      CNeuronRMAT* neuron = cLayers[1];
      return (!neuron ? 0 : neuron.GetWindow() - 1);
     }
   virtual uint      GetUnits(void) const override
     {
      CNeuronRMAT* neuron = cLayers[1];
      return (!neuron ? 0 : neuron.GetUnits());
     }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBaseSAMOCL  :  public CNeuronBaseOCL
  {
protected:
   float             fRho;
   CBufferFloat      cWeightsSAM;
   //---
   virtual bool      calcEpsilonWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForwardSAM(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronBaseSAMOCL(void) {};
                    ~CNeuronBaseSAMOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, float rho, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle)   override;
   virtual bool      Load(int const file_handle)   override;
   //---
   virtual int       Type(void)        const override          {  return defNeuronBaseSAMOCL;      }
   virtual int       Activation(void)  const override          {  return (fRho == 0 ? (int)None : (int)activation);   }
   virtual int       getWeightsSAMIndex(void) const override   {  return cWeightsSAM.GetIndex();   }
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPSBlock :  public CNeuronConvSAMOCL
  {
protected:
   CNeuronConvSAMOCL acConvolution[2];
   CNeuronBaseOCL    cResidual;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPSBlock(void) {};
                    ~CNeuronPSBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, float rho, ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual bool      InitPS(CNeuronPSBlock *master);
   //---
   virtual int       Type(void)   const override   {  return defNeuronPSBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPSformer   :  public CNeuronBaseSAMOCL
  {
protected:
   CNeuronTransposeOCL           acTranspose[2];
   CNeuronPSBlock                acPSBlocks[3];
   CNeuronRelativeSelfAttention  acAttention[2];
   CNeuronBaseOCL                cResidual;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPSformer(void)   {};
                    ~CNeuronPSformer(void)   {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint segments, float rho,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const         {  return defNeuronPSformer;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMarketObserver   :  public CNeuronRMAT
  {
public:
                     CNeuronMarketObserver(void)   {};
                    ~CNeuronMarketObserver(void)   {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint forecast,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMarketObserver; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRLAgent :  public   CNeuronRMAT
  {
public:
                     CNeuronRLAgent(void) {};
                    ~CNeuronRLAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint segments, float rho,
                          uint layers, uint n_actions,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   override {  return defNeuronRLAgent;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronControlAgent  :  public CNeuronRMAT
  {
protected:
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronControlAgent(void) {};
                    ~CNeuronControlAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint window_kv, uint units_kv, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronControlAgent; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMASA    :  public CNeuronBaseSAMOCL
  {
protected:
   CNeuronMarketObserver   cMarketObserver;
   CNeuronRevINDenormOCL   cRevIN;
   CNeuronRLAgent          cRLAgent;
   CNeuronControlAgent     cControlAgent;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMASA(void) {};
                    ~CNeuronMASA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers_mo, uint forecast,
                          uint segments_rl, float rho,
                          uint layers_rl, uint n_actions,
                          uint heads_contr,
                          uint layers_contr,
                          int NormLayer, CNeuronBatchNormOCL *normLayer,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMASA; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override;
   //---
   virtual int       GetNormLayer(void)   {  return cRevIN.GetNormLayer(); }
   virtual bool      SetNormLayer(int NormLayer, CNeuronBatchNormOCL *normLayer);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPLRMultiAgentsOCL  :  public CNeuronPLROCL
  {
protected:
   int               iAgents;
   CBufferFloat      cMinDistance;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronPLRMultiAgentsOCL(void)  : iAgents(1) {};
                    ~CNeuronPLRMultiAgentsOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint units_count, bool transpose,
                          vector<float> &min_distance,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronPLRMultiAgentsOCL;   }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCrossSectionalAnalysis :  public CNeuronMVMHAttentionMLKV
  {
protected:
   CNeuronConvOCL          cEmbeding;
   CNeuronTransposeRCDOCL  cTransposeRCD;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCrossSectionalAnalysis(void) {};
                    ~CNeuronCrossSectionalAnalysis(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint heads_kv,
                          uint units_count, uint layers, uint layers_to_one_kv,
                          uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void)   const override   {  return defNeuronCrossSectionalAnalysis;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTransposeVRCOCL  :  public CNeuronTransposeOCL
  {
protected:
   CNeuronTransposeOCL     cTranspose;
   CNeuronTransposeRCDOCL  cTransposeRCD;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;

public:
                     CNeuronTransposeVRCOCL(void) {};
                    ~CNeuronTransposeVRCOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)  const override    {  return defNeuronTransposeVRCOCL; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint      GetWindow(void) override const { return cTransposeRCD.GetWindow(); }
   virtual uint      GetCount(void) override const { return cTransposeRCD.GetCount(); }
   virtual uint      GetVariables(void) const { return iWindow; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTemporalAnalysis :  public CNeuronCrossSectionalAnalysis
  {
protected:
   CNeuronTransposeVRCOCL  cTranspose;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTemporalAnalysis(void) {};
                    ~CNeuronTemporalAnalysis(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint heads_kv,
                          uint units_count, uint layers, uint layers_to_one_kv,
                          uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void)   const override   {  return defNeuronTemporalAnalysis;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPortfolioGenerator  :  public CNeuronBaseOCL
  {
protected:
   uint              iAssets;
   uint              iTimePoints;
   uint              iAgents;
   uint              iDimension;
   //---
   CNeuronBaseOCL          cAssetTime[2];
   CNeuronTransposeVRCOCL  cTransposeVRC;
   CNeuronSoftMaxOCL       cSoftMax;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPortfolioGenerator(void) {};
                    ~CNeuronPortfolioGenerator(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint assets, uint time_points, uint dimension,
                          uint agents,  uint projection,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const override   {  return defNeuronPortfolioGenerator;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMASAAT  :  public CNeuronPortfolioGenerator
  {
protected:
   CNeuronTransposeOCL              cTranspose;
   CNeuronPLRMultiAgentsOCL         cPLR;
   CNeuronBaseOCL                   cConcat;
   CNeuronCrossSectionalAnalysis    cCrossSectionalAnalysis;
   CNeuronTemporalAnalysis          cTemporalAnalysis;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)  override
     { return        feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMASAAT(void)  {};
                    ~CNeuronMASAAT(void)  {};
   //---
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint units_cout,
                          uint layers, vector<float> &min_distance, uint projection,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const override   {  return defNeuronMASAAT;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHConvOCL  :  public CNeuronConvOCL
  {
protected:
   uint              iHeads;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMHConvOCL(void)  :  iHeads(1)   {};
                    ~CNeuronMHConvOCL(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const override   {  return defNeuronMHConvOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHFeedForward   :  public CNeuronBaseOCL
  {
protected:
   CNeuronMHConvOCL  acConvolutions[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMHFeedForward(void) {};
                    ~CNeuronMHFeedForward(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out,
                          uint units_count, uint variables, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const override   {  return defNeuronMHFeedForward;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      SetGradient(CBufferFloat *buffer, bool delete_prev = true)
     {
      CNeuronBaseOCL *last = GetPointer(acConvolutions[acConvolutions.Size() - 1]);
      if(!last ||
         !last.SetGradient(buffer, delete_prev) ||
         !CNeuronBaseOCL::SetGradient(last.getGradient(), delete_prev))
         ReturnFalse;
      return true;
     }
   virtual bool      SetOutput(CBufferFloat *buffer, bool delete_prev = true)
     {
      CNeuronBaseOCL *last = GetPointer(acConvolutions[acConvolutions.Size() - 1]);
      if(!last ||
         !last.SetOutput(buffer, delete_prev) ||
         !CNeuronBaseOCL::SetOutput(last.getOutput(), delete_prev))
         ReturnFalse;
      return true;
     }
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDMHAttention  :  public CNeuronRMAT
  {
public:
                     CNeuronDMHAttention(void) {};
                    ~CNeuronDMHAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   virtual int       Type(void) override   const   {  return defNeuronDMHAttention; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCrossDMHAttention   :  public CNeuronRMAT
  {
protected:
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronCrossDMHAttention(void)  {};
                    ~CNeuronCrossDMHAttention(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint window_cross, uint units_cross,
                          uint heads, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronCrossDMHAttention; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLegendreWaveletsHL :  public CNeuronConvOCL
  {
protected:
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   { return true; }        ///< Method for updating weights.@param NeuronOCL Pointer to previos layer.

public:
                     CNeuronLegendreWaveletsHL(void) {};
                    ~CNeuronLegendreWaveletsHL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, uint filters, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void)   const   {  return defNeuronLegendreWaveletsHL;   }///< Identificator of class.@return Type of class
   //---
   virtual uint      GetFilters(void)  const {return (iWindowOut / 2); }
   virtual uint      GetVariables(void)  const {return (iVariables); }
   virtual uint      GetUnits(void)  const {return (Neurons() / (iVariables * iWindowOut)); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDecouplingFlow   :  public   CNeuronLegendreWaveletsHL
  {
protected:
   CNeuronTransposeOCL  cTranspose;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronDecouplingFlow(void) {};
                    ~CNeuronDecouplingFlow(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint units_count,
                          uint filters, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void)   const   {  return defNeuronDecouplingFlow;   }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDilatedCasualConv   :  public CNeuronRMAT
  {
public:
                     CNeuronDilatedCasualConv(void) {};
                    ~CNeuronDilatedCasualConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint dimension,
                          uint units_count, uint variables, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronDilatedCasualConv; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultitaskStockformer   :  public CNeuronBaseOCL
  {
protected:
   CNeuronDecouplingFlow      cDecouplingFlow;
   CNeuronBaseOCL             cLowFreqSignal;
   CNeuronBaseOCL             cHighFreqSignal;
   CNeuronRMAT                cTemporalAttention;
   CNeuronDilatedCasualConv   cDilatedCasualConvolution;
   CNeuronLearnabledPE        cLowFreqPE;
   CNeuronLearnabledPE        cHighFreqPE;
   CNeuronNAFS                cLowFreqGraphAttention;
   CNeuronNAFS                cHighFreqGraphAttention;
   CNeuronDMHAttention        cLowFreqFusionDecoder;
   CNeuronCrossDMHAttention   cLowHighFreqFusionDecoder;
   CNeuronBaseOCL             cLowHigh;
   CNeuronConvOCL             cProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultitaskStockformer(void) {};
                    ~CNeuronMultitaskStockformer(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint neurons_out, uint filters,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMultitaskStockformer; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMemory  :  public CNeuronRelativeCrossAttention
  {
protected:
   CNeuronLSTMOCL    cLSTM;
   CNeuronMambaOCL   cMamba;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override  { return feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        updateInputWeights(NeuronOCL); }

public:
                     CNeuronMemory(void) {};
                    ~CNeuronMemory(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronMemory; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFinMem   :  public CNeuronRelativeCrossAttention
  {
protected:
   CNeuronTransposeOCL  cTransposeState;
   CNeuronMemory        cMemory[2];
   CNeuronRelativeCrossAttention cCrossMemory;
   CNeuronRelativeCrossAttention cMemoryToAccount;
   CNeuronRelativeCrossAttention cActionToAccount;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override   { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override   { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override   { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronFinMem(void) {};
                    ~CNeuronFinMem(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint account_descr, uint nactions,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronFinMem; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLowLevelReflection :   public CNeuronMemory
  {
protected:
   CNeuronLSTMOCL    cChangeLSTM;
   CNeuronMambaOCL   cChangeMamba;
   CNeuronRelativeCrossAttention cCrossAttention[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronLowLevelReflection(void) {};
                    ~CNeuronLowLevelReflection(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronLowLevelReflection; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHighLevelReflection :  public CNeuronMemory
  {
protected:
   CNeuronBaseOCL    cAccount;
   CNeuronLSTMOCL    cHistoryAccount;
   CNeuronRelativeCrossAttention cActionReason;
   CNeuronLSTMOCL    cHistoryActions;
   CNeuronRelativeCrossAttention cActionResult;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronHighLevelReflection(void) {};
                    ~CNeuronHighLevelReflection(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint desc_account, uint actions_state,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override   const   {  return defNeuronHighLevelReflection; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMoreLessEqual :  public CNeuronBaseOCL
  {
protected:
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override {return true; }

public:
                     CNeuronMoreLessEqual(void) {};
                    ~CNeuronMoreLessEqual(void) {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFinAgent   :  public CNeuronRelativeCrossAttention
  {
protected:
   CNeuronTransposeOCL  cTransposeState;
   CNeuronLowLevelReflection  cLowLevelReflection[2];
   CNeuronHighLevelReflection cHighLevelReflection;
   CNeuronMoreLessEqual cTools;
   CNeuronPSformer      cMarketIntelligence;
   CNeuronMemory        cMarketMemory;
   CNeuronRelativeCrossAttention cCrossLowLevel;
   CNeuronRelativeCrossAttention cMarketToLowLevel;
   CNeuronRelativeCrossAttention cMarketToTools;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronFinAgent(void) {};
                    ~CNeuronFinAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint account_descr, uint nactions, uint segments,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronFinAgent; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMemoryDistil  :  public CNeuronMemory
  {
protected:
   CNeuronBaseOCL       cConcatenated;
   CNeuronTransposeOCL  cTransposeConc;
   CNeuronConvOCL       cConvolution;
   CNeuronEmbeddingOCL  cDistilMemory;
   CNeuronRelativeCrossAttention cCrossAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMemoryDistil(void) {};
                    ~CNeuronMemoryDistil(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint stack_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMemoryDistil; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFinConAgent   :  public CNeuronRelativeCrossAttention
  {
protected:
   CNeuronMemoryDistil  cStatesMemory;
   CNeuronMemoryDistil  cActionsMemory;
   CNeuronBaseOCL       caRole[2];
   CNeuronRelativeCrossAttention cStateToRole;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override  { return feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        updateInputWeights(NeuronOCL); }

public:
                     CNeuronFinConAgent(void) {};
                    ~CNeuronFinConAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint stack_size, uint action_space,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronFinConAgent; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFinConManager   :  public CNeuronFinConAgent
  {
protected:
   CNeuronTransposeOCL  cTransposeState;
   CNeuronFinConAgent   caAgents[3];
   CNeuronFinConAgent   caTrAgents[3];
   CNeuronFinConAgent   cRiskAgent;
   CNeuronBaseOCL       cConcatenatedAgents;
   CNeuronBaseOCL       cAccount;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override  {ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronFinConManager(void) {};
                    ~CNeuronFinConManager(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint stack_size, uint account_descr, uint action_space,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronFinConManager; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
   virtual void      TrainMode(bool flag);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMacroHFTHyperAgent  :  public CNeuronSoftMaxOCL
  {
protected:
   CNeuronMemoryDistil  cMemory;
   CNeuronRMAT          cStatePrepare;
   CNeuronTransposeOCL  cTranspose;
   CNeuronConvOCL       cScale;
   CNeuronBaseOCL       cMLP[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMacroHFTHyperAgent(void) {};
                    ~CNeuronMacroHFTHyperAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint agents, uint stack_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMacroHFTHyperAgent; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMacroHFT   :  public CNeuronBaseOCL
  {
protected:
   CNeuronTransposeOCL  cTranspose;
   CNeuronFinConAgent   caAgents[6];
   CNeuronMacroHFTHyperAgent  cHyperAgent;
   CNeuronBaseOCL       cConcatenated;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMacroHFT(void) {};
                    ~CNeuronMacroHFT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint stack_size, uint nactions,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMacroHFT; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMacroHFTvsRiskManager  :  public CResidualConv
  {
protected:
   CNeuronBaseOCL       caAccountProjection[2];
   CNeuronMemoryDistil  cMemoryAccount;
   CNeuronMemoryDistil  cMemoryAction;
   CNeuronRelativeCrossAttention cCrossAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;

public:
                     CNeuronMacroHFTvsRiskManager(void) {};
                    ~CNeuronMacroHFTvsRiskManager(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint stack_size, uint nactions, uint account_decr,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMacroHFTvsRiskManager; }
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiScaleRelativeSelfAttention   :  public CNeuronRelativeSelfAttention
  {
protected:
   //---
   virtual bool      AttentionOut(void);

public:
                     CNeuronMultiScaleRelativeSelfAttention(void) {};
                    ~CNeuronMultiScaleRelativeSelfAttention(void) {};
   //---
   virtual int       Type(void) override   const   {  return defNeuronMultiScaleRelativeSelfAttention; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiScaleRelativeCrossAttention   :  public CNeuronRelativeCrossAttention
  {
protected:
   virtual bool      AttentionOut(void);

public:
                     CNeuronMultiScaleRelativeCrossAttention(void) {};
                    ~CNeuronMultiScaleRelativeCrossAttention(void) {};
   //---
   virtual int       Type(void) override   const   {  return defNeuronMultiScaleRelativeCrossAttention; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRecursiveAttention  :  public CNeuronMultiScaleRelativeCrossAttention
  {
protected:
   CNeuronMultiScaleRelativeSelfAttention cSelfAttention;
   CNeuronTransposeOCL  cTransposeSA;
   CNeuronConvOCL       cConvolution;
   CNeuronEmbeddingOCL  cHistory;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
   override  { return feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None)
   override  { return calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
   override  { return updateInputWeights(NeuronOCL); }

public:
                     CNeuronRecursiveAttention(void) {};
                    ~CNeuronRecursiveAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint history_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronRecursiveAttention; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLinerAttention   :  public CNeuronBaseOCL
  {
protected:
   uint                 iWindow;
   uint                 iWindowKey;
   uint                 iUnits;
   uint                 iVariables;
   //---
   CNeuronConvOCL       cQuery;
   CNeuronConvOCL       cKey;
   CNeuronTransposeVRCOCL  cKeyT;
   CNeuronBaseOCL       cKeyValue;
   CNeuronBaseOCL       cAttentionOut;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronLinerAttention(void) {};
                    ~CNeuronLinerAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key,
                          uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronLinerAttention; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHidformerTSAgent    :  public CResidualConv
  {
protected:
   CNeuronBaseOCL                caRole[2];
   CNeuronRelativeCrossAttention cStateToRole;
   CNeuronS3                     cShuffle;
   CNeuronRecursiveAttention     cRecursiveState;
   CResidualConv                 cResidualState;
   CNeuronRecursiveAttention     cRecursiveAction;
   CNeuronMultiScaleRelativeCrossAttention   cActionToState;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHidformerTSAgent(void) {};
                    ~CNeuronHidformerTSAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint stack_size, uint action_space,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronHidformerTSAgent; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHidformerFreqAgent    :  public CResidualConv
  {
protected:
   CNeuronTransposeOCL           cTranspose;
   CNeuronLegendreWaveletsHL     cLegendre;
   CNeuronTransposeRCDOCL        cHLState;
   CNeuronLinerAttention         cAttentionState;
   CResidualConv                 cResidualState;
   CNeuronS3                     cShuffle;
   CNeuronRecursiveAttention     cRecursiveAction;
   CNeuronMultiScaleRelativeCrossAttention cActionToState;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHidformerFreqAgent(void) {};
                    ~CNeuronHidformerFreqAgent(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint filters, uint units_count,
                          uint heads, uint stack_size, uint action_space,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronHidformerFreqAgent; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHidformer  :  public CNeuronBaseOCL
  {
protected:
   CNeuronTransposeOCL  cTranspose;
   CNeuronHidformerTSAgent caTSAgents[4];
   CNeuronHidformerFreqAgent  caFreqAgents[2];
   CNeuronMacroHFTHyperAgent  cHyperAgent;
   CNeuronBaseOCL       cConcatenated;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHidformer(void) {};
                    ~CNeuronHidformer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint stack_size, uint nactions,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronHidformer; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronResNeXtBottleneck :  public CNeuronConvOCL
  {
protected:
   CNeuronConvOCL          cProjectionIn;
   CNeuronBatchNormOCL     cNormalizeIn;
   CNeuronTransposeRCDOCL  cTransposeIn;
   CNeuronConvOCL          cFeatureExtraction;
   CNeuronBatchNormOCL     cNormalizeFeature;
   CNeuronTransposeRCDOCL  cTransposeOut;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;


public:
                     CNeuronResNeXtBottleneck(void) {};
                    ~CNeuronResNeXtBottleneck(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out, uint window,
                          uint step, uint units_count, uint group_size, uint groups,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronResNeXtBottleneck;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronResNeXtResidual:  public CNeuronConvOCL
  {
protected:
   CNeuronTransposeOCL     cTransposeIn;
   CNeuronConvOCL          cProjectionTime;
   CNeuronBatchNormOCL     cNormalizeTime;
   CNeuronTransposeOCL     cTransposeOut;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronResNeXtResidual(void) {};
                    ~CNeuronResNeXtResidual(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out,
                          uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronResNeXtResidual;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronResNeXtBlock :  public CNeuronBaseOCL
  {
protected:
   uint                       iChanelsOut;
   CNeuronResNeXtBottleneck   cBottleneck;
   CNeuronResNeXtResidual     cResidual;
   CBufferFloat               cBuffer;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronResNeXtBlock(void) {};
                    ~CNeuronResNeXtBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out, uint window,
                          uint step, uint units_count, uint group_size, uint groups,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronResNeXtBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuron2DSSMOCL  :  public CNeuronBaseOCL
  {
protected:
   uint                 iWindowOut;
   uint                 iUnitsOut;
   CNeuronBaseOCL       cHiddenStates;
   CLayer               cProjectionX_Time;
   CLayer               cProjectionX_Variable;
   CNeuronConvOCL       cA;
   CNeuronConvOCL       cB_Time;
   CNeuronConvOCL       cB_Variable;
   CNeuronConvOCL       cC_Time;
   CNeuronConvOCL       cC_Variable;
   CNeuronConvOCL       cDelta_Time;
   CNeuronConvOCL       cDelta_Variable;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForwardSSM2D(void);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradientsSSM2D(void);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuron2DSSMOCL(void)  {};
                    ~CNeuron2DSSMOCL(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint window_out, uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuron2DSSMOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronChimera    :  public CNeuronBaseOCL
  {
protected:
   CNeuron2DSSMOCL    caSSM[2];
   CNeuronConvOCL     cDiscretization;
   CLayer             cResidual;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronChimera(void) {};
                    ~CNeuronChimera(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint window_out, uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override  const   {  return defNeuronChimera; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMoT  :  public CNeuronMHAttentionPooling
  {
protected:
   CNeuronConvOCL             cNodesTokenizer;
   CNeuronConvOCL             cEdgesTokenizer;
   CNeuronConvOCL             cSubGraphsTokenizer;
   CLayer                     cUnitarSubGraphsTokenizer;
   CNeuronBaseOCL             cConcatenate;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMoT(void) {};
                    ~CNeuronMoT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMoT; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronChimeraPlus    :  public CNeuronChimera
  {
protected:
   CNeuron2DSSMOCL    cSSMPlus;
   CLayer             cDiscretizationPlus;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronChimeraPlus(void) {};
                    ~CNeuronChimeraPlus(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint window_out, uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override  const   {  return defNeuronChimeraPlus; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHypridDecoder :  public CNeuronHidformer
  {
protected:
   CNeuronChimeraPlus   cChimera;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHypridDecoder(void) {};
                    ~CNeuronHypridDecoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint heads, uint layers, uint stack_size, uint nactions,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronHypridDecoder; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAttraos :  public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cOne;
   CNeuronBaseOCL    cX_norm;
   CNeuronConvOCL    cA;
   CNeuronConvOCL    cX_proj;
   CNeuronBaseOCL    cDelta;
   CNeuronBaseOCL    cB;
   CNeuronBaseOCL    cC;
   CNeuronConvOCL    cD;
   CNeuronBaseOCL    cH;
   CNeuronConvOCL    cDelta_proj;
   CNeuronBaseOCL    cDeltaA;
   CNeuronBaseOCL    cDeltaB;
   CNeuronBaseOCL    cDeltaBX;
   CNeuronBaseOCL    cDeltaH;
   CNeuronBaseOCL    cHS;
   //---
   virtual bool      PScan(void);
   virtual bool      PScanCalcGradient(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAttraos(void) {};
                    ~CNeuronAttraos(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronAttraos; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool       WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTopKGates  :  public CNeuronSoftMaxOCL
  {
protected:
   int               iK;
   CBufferFloat      cbNoise;
   CNeuronConvOCL    cProjection;
   CNeuronBaseOCL    cGates;
   //---
   virtual bool      TopKgates(void);
   virtual bool      TopKgatesGradient(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTopKGates(void) {};
                    ~CNeuronTopKGates(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint gates, uint top_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronTopKGates; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint      GetGates(void) const { return cProjection.GetFilters() / 2; }
   virtual uint      GetUnits(void) const { return cProjection.GetUnits(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMoE  :  public CNeuronBaseOCL
  {
protected:
   CNeuronTopKGates     cGates;
   CNeuronConvOCL       cXProj;
   CLayer               cExperts;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMoE(void) {};
                    ~CNeuronMoE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint units_count,
                          uint experts, uint top_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronMoE; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual void      TrainMode(bool flag)
     {  bTrain = flag;  cGates.TrainMode(bTrain); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronChanelMask :  public CNeuronBaseOCL
  {
   //---
protected:
   uint              iUnits;
   uint              iFFTdimension;
   CBufferFloat      cbFFTReal;
   CBufferFloat      cbFFTImag;
   //---
   virtual bool      FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false);
   virtual bool      Mask(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) { return true; }

public:
                     CNeuronChanelMask(void) {};
                    ~CNeuronChanelMask(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronChanelMask; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool       WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDUET :  public CNeuronTransposeOCL
  {
protected:
   uint                       iWindowKey;
   uint                       iHeads;
   //---
   CNeuronTransposeOCL        cTranspose;
   CNeuronMoE                 cExperts;
   CNeuronConvOCL             cQKV;
   CNeuronBaseOCL             cQ;
   CNeuronBaseOCL             cKV;
   CNeuronChanelMask          cMask;
   CBufferFloat               cbScores;
   CNeuronBaseOCL             cMHAttentionOut;
   CNeuronConvOCL             cPooling;
   CNeuronBaseOCL             cResidual;
   CNeuronMHFeedForward       cFeedForward;
   //---
   virtual bool      AttentionOut(void);
   virtual bool      AttentionInsideGradients(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer);

public:
                     CNeuronDUET(void) {};
                    ~CNeuronDUET(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count, uint heads,
                          uint units_out, uint experts, uint top_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronDUET; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   //virtual bool       WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual void      TrainMode(bool flag)
     {  bTrain = flag; cExperts.TrainMode(bTrain); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiWindowsConvOCL    :  public CNeuronConvOCL
  {
protected:
   int               aiWindows[];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronMultiWindowsConvOCL(void) {  activation = SoftPlus;  iWindow = -1; }
                    ~CNeuronMultiWindowsConvOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &windows[], uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMultiWindowsConvOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual int       GetWindow(uint index)
     {
      if(index >= aiWindows.Size())
         return 0;
      return aiWindows[index];
     }
   virtual uint      GetUnits(void) const override { return CNeuronConvOCL::GetUnits() / aiWindows.Size(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAdaBN   :  public CNeuronMoE
  {
public:
                     CNeuronAdaBN(void) {};
                    ~CNeuronAdaBN(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint units_count,
                          uint &bottlenecks[], uint top_k, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronAdaBN; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronComplexConvOCL    :  public CNeuronConvOCL
  {
protected:
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronComplexConvOCL(void)   {  activation = None;   }
                    ~CNeuronComplexConvOCL(void)   {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint window_out,
                          uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void)   const   {  return defNeuronComplexConvOCL;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronComplexMVMHMaskAttention :  public CNeuronBaseOCL
  {
protected:
   uint                       iWindow;
   uint                       iWindowKey;
   uint                       iHeads;
   uint                       iUnits;
   uint                       iVariables;
   //---
   CNeuronComplexConvOCL      cQKV;
   CNeuronBaseOCL             cQ;
   CNeuronBaseOCL             cKV;
   CNeuronConvOCL             cMask;
   CNeuronBaseOCL             cMHAttentionOut;
   CNeuronComplexConvOCL      cPooling;
   CNeuronBaseOCL             cResidual;
   CNeuronComplexConvOCL      cFeedForward[2];
   //---
   virtual bool               AttentionOut(void);
   virtual bool               AttentionInsideGradients(void);
   virtual bool               SumAndNormalize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) override;
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *prevLayer) override;

public:
                     CNeuronComplexMVMHMaskAttention(void) {};
                    ~CNeuronComplexMVMHMaskAttention(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void)   const   {  return defNeuronComplexMVMHMaskAttention;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCATCH   :  public CNeuronTransposeOCL
  {
protected:
   CNeuronTransposeOCL        cTranspose;
   CNeuronBaseOCL             caFreqIn[2];
   CNeuronBaseOCL             cFreqConcat;
   CNeuronComplexConvOCL      caProjection[2];
   CNeuronComplexMVMHMaskAttention  caChannelFusion[2];
   CNeuronComplexConvOCL      caLinearHead[2];
   CNeuronBaseOCL             caFreqOut[2];
   //---
   virtual bool               FFT(CBufferFloat *inp_re, CBufferFloat *inp_im,
                                  CBufferFloat *out_re, CBufferFloat *out_im,
                                  uint variables, bool reverse = false);
   virtual bool               ComplexNormalize(CNeuronBaseOCL *cInputFreqComplex,
         CNeuronBaseOCL *cNormFreqComplex,
         CBufferFloat   *cMeans,
         CBufferFloat   *cVariances,
         int dimension, int variables);
   virtual bool               ComplexNormalizeGradient(CNeuronBaseOCL *cInputFreqComplex,
         CNeuronBaseOCL *cNormFreqComplex,
         CBufferFloat   *cVariances,
         int dimension, int variables);
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *prevLayer) override;


public:
                     CNeuronCATCH(void) {};
                    ~CNeuronCATCH(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                   uint time_step, uint variables,
                                   uint window, uint step,
                                   uint window_key, uint heads,
                                   ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void) override const   {  return defNeuronCATCH;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSkillsEncoder   :  public CNeuronSoftMaxOCL
  {
protected:
   CNeuronCATCH               cCrossObservAttention;
   CNeuronTransposeOCL        cTranspose;
   CNeuronConvOCL             cSkillsProjection[2];
   CNeuronBaseOCL             cPrevSkillsConcat;
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *prevLayer) override;

public:
                     CNeuronSkillsEncoder(void) {};
                    ~CNeuronSkillsEncoder(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                   uint time_step, uint variables, uint skills,
                                   uint window, uint step, uint window_key, uint heads,
                                   ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void) override const   {  return defNeuronSkillsEncoder;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHiSSDLowLevelControler:  public CNeuronConvOCL
  {
protected:
   uint                       iTaskSkills;
   uint                       iCommonSkills;
   //---
   CNeuronSkillsEncoder       cTaskSpecificSkillsEncoder;
   CNeuronTransposeOCL        cTranspose;
   CNeuronBaseOCL             cObservAndSkillsConcat;
   CNeuronBatchNormOCL        cNormalizarion;
   CNeuronConvOCL             cActionDecoder[2];
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool               calcInputGradients(CNeuronBaseOCL *NeuronOCL,
         CBufferFloat *SecondInput,
         CBufferFloat *SecondGradient,
         ENUM_ACTIVATION SecondActivation = None
                                                ) override;

public:
                     CNeuronHiSSDLowLevelControler(void) {};
                    ~CNeuronHiSSDLowLevelControler(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                   uint time_step, uint variables,
                                   uint task_skills, uint common_skills, uint n_actions,
                                   uint window, uint step, uint window_key, uint heads,
                                   ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void) override const   {  return defNeuronHiSSDLowLevelControler;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCGLSTMOCL : public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cConcatenateInputs;
   CNeuronConvOCL    cProjection;
   //---
   virtual bool      CSLSTM_feedForward(void);
   virtual bool      CSLSTM_CalcHiddenGradient(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCGLSTMOCL(void) {};
                    ~CNeuronCGLSTMOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void) override  const    {  return defNeuronCGLSTMOCL; }
   virtual bool      Clear(void) override;
   virtual CBufferFloat *getLSTMWeights(void) { return cProjection.GetWeightsConv(); }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHProbAttention :  public CResidualConv
  {
protected:
   uint                       iWindow;
   uint                       iWindowKey;
   uint                       iHeads;
   uint                       iUnits;
   uint                       iTopKQuerys;
   uint                       iRandomKeys;
   int                        ibScore;
   //---
   CNeuronConvOCL             cQKV;
   CNeuronBaseOCL             cQ;
   CNeuronBaseOCL             cKV;
   CNeuronBaseOCL             cRandomK;
   CNeuronBaseOCL             cMHAttentionOut;
   CNeuronConvOCL             cPooling;
   CNeuronTransposeOCL        cTranspose[2];
   CNeuronConvOCL             cScaling;
   //---
   virtual bool               RandomKeys(CBufferFloat* indexes, int random, int units, int heads);
   virtual bool               QueryImportance(void);
   virtual bool               TopKIndexes(void);
   //---
   virtual bool               AttentionOut(void);
   virtual bool               AttentionInsideGradients(void);
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *prevLayer) override;

public:
                     CNeuronMHProbAttention(void) {};
                    ~CNeuronMHProbAttention(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void)   const   {  return defNeuronMHProbAttention;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTSPositionEncoder   :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL             cProjection;
   CNeuronBatchNormOCL        cNorm;
   CBufferFloat               cPeriods;
   //---
   virtual bool               AddPE(CBufferFloat *time);
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTSPositionEncoder(void) {};
                    ~CNeuronTSPositionEncoder(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                   uint window, uint units_count, uint &periods[], uint freqs,
                                   ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void) override  const   {  return defNeuronTSPositionEncoder;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint               GetWindow(void) const { return cProjection.GetWindow(); }
   virtual uint               GetWindowOut(void) const { return cProjection.GetFilters(); }
   virtual uint               GetUnits(void) const { return cProjection.GetUnits(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CMamba4CastEmbeding :   public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL             cProjection;
   CNeuronBatchNormOCL        cNorm;
   CNeuronTSPositionEncoder   cProjectionWithTE;
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CMamba4CastEmbeding(void) {};
                    ~CMamba4CastEmbeding(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                   uint window, uint window_out, uint units_count, uint &periods[],
                                   ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int                Type(void) override  const   {  return defMamba4CastEmbeding;   }
   //---
   virtual bool               Save(int const file_handle) override;
   virtual bool               Load(int const file_handle) override;
   //---
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void               SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiWindowsConvWPadOCL    :  public CNeuronConvOCL
  {
protected:
   int               aiWindows[];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronMultiWindowsConvWPadOCL(void) {  activation = SoftPlus;  iWindow = -1; }
                    ~CNeuronMultiWindowsConvWPadOCL(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &windows[],
                          uint step, uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMultiWindowsConvWPadOCL;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual uint      GetWindow(void) const { return aiWindows[0]; }
   virtual uint      GetWindowsSize(void) const { return aiWindows.Size(); }
   virtual uint      GetWindowOut(void) const { return iWindowOut; }
   virtual uint      GetUnits(void) const { return Neurons() / (iVariables * GetWindowsSize() * iWindowOut); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConcatDiff:  public CNeuronBaseOCL
  {
protected:
   uint              iUnits;
   uint              iVariables;
   uint              iStep;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override   { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronConcatDiff(void) :  iUnits(0), iVariables(1), iStep(1) {  activation = None; }
                    ~CNeuronConcatDiff(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint step, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override  const   {  return defNeuronConcatDiff;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMantisPatching   :  public CNeuronTransposeOCL
  {
protected:
   CNeuronTransposeOCL     cToVarSeq;
   CNeuronConvOCL          cProjecting;
   CNeuronTransposeVRCOCL  cToVarProjSeq;
   CNeuronConvOCL          cPatchingProj;
   CNeuronProofOCL         cProof;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronMantisPatching(void) {};
                    ~CNeuronMantisPatching(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint count, uint patchs, uint variables,
                          uint embedding_size, uint patch_filters,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void)  override const {  return defNeuronMantisPatching; }
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMantisAttentionUnit :  public CNeuronSoftMaxOCL
  {
protected:
   CNeuronBaseOCL                cClassToken[2];
   CNeuronMVCrossAttentionMLKV   cAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronMantisAttentionUnit(void) {};
                    ~CNeuronMantisAttentionUnit(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint token_size, uint window, uint window_key, uint heads,
                          uint units_count, uint layers, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void)  override const {  return defNeuronMantisAttentionUnit; }
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTimeFoundPatching   :  public CNeuronConvOCL
  {
protected:
   CNeuronTransposeOCL              cToVarSeq;
   CNeuronMultiWindowsConvWPadOCL   cProjecting;
   CNeuronTransposeVRCOCL           cToVarProjSeq;
   CNeuronMultiWindowsConvWPadOCL   cPatchingProj;
   CNeuronMHFeedForward             cPatchsFeedForward;
   CNeuronConvOCL                   cPatchingAgr;
   CNeuronProofOCL                  cProof;
   CNeuronTransposeOCL              cToPatchVarEmb;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronTimeFoundPatching(void) {};
                    ~CNeuronTimeFoundPatching(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint count, uint patchs, uint variables,
                          uint embedding_size, uint patch_filters,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void)  override const {  return defNeuronTimeFoundPatching; }
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTimeFoundTransformerUnit  :  public CNeuronCrossDMHAttention
  {
protected:
   CNeuronMVMHAttentionMLKV   cSelfAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        updateInputWeights(NeuronOCL); }

public:
                     CNeuronTimeFoundTransformerUnit(void) {};
                    ~CNeuronTimeFoundTransformerUnit(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window,
                          uint window_key, uint heads, uint heads_kv, uint units_count,
                          uint layers, uint layers_to_one_kv, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const { return defNeuronTimeFoundTransformerUnit; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSwiGLUOCL  :  public   CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL    caProjections[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL)          override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)   override;

public:
                     CNeuronSwiGLUOCL(void) {};
                    ~CNeuronSwiGLUOCL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint window_out, uint units_count,
                          uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const {   return defNeuronSwiGLUOCL; }
   //--- methods for working with files
   virtual bool      Save(int const file_handle)   override;
   virtual bool      Load(int const file_handle)   override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)     override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual uint      GetWindow(void) const      { return caProjections[0].GetWindow(); }
   virtual uint      GetWindowOut(void) const   { return caProjections[0].GetFilters();}
   virtual uint      GetVariables(void) const   { return caProjections[0].GetVariables(); }
   virtual uint      GetUnits(void) const       { return caProjections[0].GetUnits(); }
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMaskMultiWinConv    :  public CNeuronConvOCL
  {
protected:
   uint              iWindowsTotal;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronMaskMultiWinConv(void) {};
                    ~CNeuronMaskMultiWinConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window,
                          uint windows_total, uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMaskMultiWinConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTimeMoESparseExperts   :  public CNeuronBaseOCL
  {
protected:
   CNeuronSwiGLUOCL        cExpertsIn;
   CNeuronSwiGLUOCL        cSharedIn;
   CNeuronMaskMultiWinConv cExpertsOut;
   CNeuronConvOCL          cSharedOut;
   CNeuronTopKGates        cMasks;
   CNeuronConvOCL          cSharedGates;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL)          override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)   override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)   override;

public:
                     CNeuronTimeMoESparseExperts(void) {};
                    ~CNeuronTimeMoESparseExperts(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window,
                          uint window_out, uint units_count, uint variables, uint experts,
                          uint topK, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const {   return defNeuronTimeMoESparseExperts; }
   //--- methods for working with files
   virtual bool      Save(int const file_handle)   override;
   virtual bool      Load(int const file_handle)   override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)     override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual uint      GetWindow(void) const      { return cSharedIn.GetWindow();     }
   virtual uint      GetVariables(void) const   { return cSharedIn.GetVariables();  }
   virtual uint      GetUnits(void) const       { return cSharedIn.GetUnits(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTimeMoEAttention   :  public CNeuronCrossDMHAttention
  {
protected:
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        updateInputWeights(NeuronOCL); }

public:
                     CNeuronTimeMoEAttention(void)  {};
                    ~CNeuronTimeMoEAttention(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_count,
                          uint window_cross, uint units_cross,
                          uint heads, uint layers,
                          uint experts, uint experts_dimension, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronTimeMoEAttention; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAdaptConv    :  public CNeuronConvOCL
  {
protected:
   CBufferFloat      bMainFreq;
   //---
   virtual bool      FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, uint variables, bool reverse = false);
   virtual bool      PeriodsFinding(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *main_freq, uint variables);
   virtual bool      AdaptiveConvolution(CNeuronBaseOCL *NeuronOCL, CBufferFloat *main_freq);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAdaptConv(void) {};
                    ~CNeuronAdaptConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window,
                          uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronAdaptConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRoPE  :  public CNeuronPositionEncoder
  {
protected:
   uint              iWindow;
   uint              iVariables;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)override;

public:
                     CNeuronRoPE(void) : iWindow(0), iVariables(0) {};
                    ~CNeuronRoPE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint count, uint window, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual int       Type(void) override const {  return defNeuronRoPE; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CParams  :  public   CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cOne;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override          { return FeedForward(); }

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override   { return UpdateInputWeights(); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override   { return true; }

public:
                     CParams(void) {};
                    ~CParams(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override;
   virtual bool      Identity(const int rows, const int cols);
   //---
   virtual bool      FeedForward(void);
   virtual bool      UpdateInputWeights(void);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defParams; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual CBufferFloat   *getWeightsParams(void)        {  return cOne.getWeights();  }                 ///< Get pointer of gradient buffer @return Pointer to object
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronK2VAEEncoder   :  public CNeuronBaseOCL
  {
protected:
   //--- Koopman
   CNeuronTimeMoESparseExperts   cKoopman;
   CNeuronBaseOCL                cKoopmanPred;
   CNeuronBaseOCL                cKoopmanRest;
   //---
   CNeuronTimeMoEAttention cAuxiliaryNet;
   //--- Kalman Filter
   CParams              cB;      // Control input matrix B
   CParams              cF;      // State transition matrix F
   CParams              cH;      // Observation matrix H
   CParams              cQ;      // Learnable covariance matrices Q
   CParams              cR;      // Learnable covariance matrices R
   CNeuronBaseOCL       cP;      // Covariance matrices P
   //---
   CNeuronTransposeOCL  cFT;
   CNeuronTransposeOCL  cHT;
   CNeuronTransposeOCL  cQT;
   CNeuronTransposeOCL  cRT;
   CNeuronTransposeOCL  cPT;
   //---
   CNeuronBaseOCL       cQ_QT;
   CNeuronBaseOCL       cR_RT;
   CNeuronBaseOCL       cXPred;
   CNeuronBaseOCL       cF_P;
   CNeuronBaseOCL       cPPred;
   //---
   CNeuronBaseOCL       cP_HT;
   CNeuronBaseOCL       cH_P_HT;
   matrix<double>       mS, mSGrad;
   CNeuronBaseOCL       cSInv;
   CNeuronBaseOCL       cK;
   CNeuronTransposeOCL  cKT;
   CNeuronBaseOCL       cYPred;
   CNeuronBaseOCL       cDeltY;
   CNeuronBaseOCL       cX;
   CNeuronBaseOCL       cK_H;
   CNeuronBaseOCL       cIdifK_H;
   CNeuronTransposeOCL  cIdifK_HT;
   matrix<double>       mP;
   matrix<double>       mPGrad;
   matrix<double>       mNoise;
   matrix<double>       mKL;
   matrix<double>       mGrad;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronK2VAEEncoder(void) {};
                    ~CNeuronK2VAEEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units_cross,
                          uint heads, uint layers, uint scenarios,
                          uint experts, uint experts_dimension, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void)        const                      {  return defNeuronK2VAEEncoder; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronInterpolationAttention :  public CNeuronBaseOCL
  {
protected:
   //---
   uint                 iCount;
   uint                 iDimension;
   //---
   CParams              cW;
   CParams              cA;
   CParams              cGL;
   //---
   CNeuronBaseOCL       cH;
   CNeuronBaseOCL       cAdj;
   CNeuronBaseOCL       cAttention;
   //---
   virtual bool      InterpolationAttention(CNeuronBaseOCL *NeuronOCL);
   virtual bool      InterpolationAttentionGrad(CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronInterpolationAttention(void) { activation = None; }
                    ~CNeuronInterpolationAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const {  return defNeuronInterpolationAttention; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAGCN    :  public CNeuronBaseOCL
  {
protected:
   CParams              cEa;
   CNeuronSwiGLUOCL     cWx;
   CNeuronSwiGLUOCL     cWe;
   CNeuronBaseOCL       cWconcat_ex;
   CNeuronConvOCL       cEn;
   CNeuronTransposeOCL  cEnT;
   CNeuronBaseOCL       cEnEnT;
   CNeuronSoftMaxOCL    cAadapt;
   CNeuronBaseOCL       cAadaptX;
   CNeuronBaseOCL       cApreX;
   CNeuronSwiGLUOCL     cWadapt;
   CNeuronSwiGLUOCL     cWpre;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronAGCN(void) {activation = None;}
                    ~CNeuronAGCN(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronAGCN; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
   //---
   virtual uint      GetWindow(void) const      { return cWx.GetWindow(); }
   virtual uint      GetUnits(void) const       { return cWx.GetUnits(); }
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGinARCell  :  public CNeuronBaseOCL
  {
protected:
   CNeuronInterpolationAttention cX_IA;
   CNeuronAGCN                   cX_AGCN;
   CNeuronAGCN                   cForgetGate;
   CNeuronAGCN                   cResetGate;
   CNeuronBaseOCL                cContext;
   CBufferFloat                  bTemp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronGinARCell(void) { activation = None;}
                    ~CNeuronGinARCell(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronGinARCell; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGinAR   :  public CNeuronSwiGLUOCL
  {
protected:
   CParams              cEa;
   CNeuronSwiGLUOCL     cWx;
   CNeuronSwiGLUOCL     cWe;
   CNeuronBaseOCL       cWconcat_ex;
   CNeuronConvOCL       cEn;
   CNeuronTransposeOCL  cEnT;
   CNeuronBaseOCL       cEnEnT;
   CNeuronSoftMaxOCL    cApre;
   CNeuronGinARCell     caCells[4];
   CNeuronBaseOCL       cConcat;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGinAR(void) {};
                    ~CNeuronGinAR(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronGinAR; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPeriodNorm :  public CNeuronBaseOCL
  {
protected:
   uint                    iPeriod;
   uint                    iVariables;
   uint                    iCount;
   CNeuronBaseOCL          cMeanSTDevs;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPeriodNorm(void) : iPeriod(-1), iVariables(1) {};
                    ~CNeuronPeriodNorm(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint period, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronPeriodNorm; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   CNeuronBaseOCL*   GetMeanSTDevs(void) { return cMeanSTDevs.AsObject(); }
   virtual uint      GetPeriod(void) const { return iPeriod; }
   virtual uint      GetVariables(void) const { return iVariables; }
   virtual uint      GetUnits(void) const { return iCount; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAdaptSpatialNorm :  public CNeuronBaseOCL
  {
protected:
   uint                    iVariables;
   uint                    iCount;
   //---
   CParams                 cEn;
   CNeuronTransposeOCL     cEnT;
   CNeuronBaseOCL          cEnEnT;
   CNeuronSoftMaxOCL       cAttan;
   CNeuronBaseOCL          cMeanSTDevs;
   //---
   virtual bool      AdaptSpatialNorm(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AdaptSpatialNormGrad(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAdaptSpatialNorm(void) : iCount(0), iVariables(1) {};
                    ~CNeuronAdaptSpatialNorm(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronAdaptSpatialNorm; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   CNeuronBaseOCL*   GetMeanSTDevs(void) { return cMeanSTDevs.AsObject(); }
   virtual uint      GetVariables(void) const { return iVariables; }
   virtual uint      GetUnits(void) const { return iCount; }
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSCNNEncoder   :  public CNeuronTransposeOCL
  {
protected:
   CNeuronPeriodNorm       cLongNorm;
   CNeuronTransposeVRCOCL  cSeasonTransp;
   CNeuronPeriodNorm       cSeasonNorm;
   CNeuronTransposeVRCOCL  cUnSeasonTransp;
   CNeuronPeriodNorm       cShortNorm;
   CNeuronAdaptSpatialNorm cAdaptSpatNorm;
   CNeuronBaseOCL          cConcatenated;
   CNeuronSwiGLUOCL        cProjection;
   CNeuronTransposeOCL     cTranspose;
   CNeuronConvOCL          caFusion[2];
   CNeuronBaseOCL          cFusionOut;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronSCNNEncoder(void) {};
                    ~CNeuronSCNNEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint variables, uint forecast,
                          uint season_period, uint short_period,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSCNNEncoder; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSCNN    :  public   CNeuronBaseOCL
  {
protected:
   CLayer            cLayers;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronSCNN(void) {};
                    ~CNeuronSCNN(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint variables, uint forecast,
                          uint season_period, uint short_period, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSCNN; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAttentNorm :  public CNeuronBaseOCL
  {
protected:
   uint                    iPeriod;
   uint                    iVariables;
   uint                    iCount;
   //---
   CParams                 cAttention;
   CNeuronSoftMaxOCL       cSoftMax;
   CNeuronBaseOCL          cMeans;
   CNeuronBaseOCL          cSTDevs;
   //---
   virtual bool      AttentNorm(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentNormGrad(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAttentNorm(void) :  iPeriod(0), iVariables(0), iCount(0) {};
                    ~CNeuronAttentNorm(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint period, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronAttentNorm; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   CNeuronBaseOCL*   GetMeans(void) { return cMeans.AsObject(); }
   CNeuronBaseOCL*   GetSTDevs(void) { return cSTDevs.AsObject(); }
   virtual uint      GetPeriod(void) const { return iPeriod; }
   virtual uint      GetVariables(void) const { return iVariables; }
   virtual uint      GetUnits(void) const { return iCount; }
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {  activation = None; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSAttentNorm :  public CNeuronTransposeOCL
  {
protected:
   CNeuronTransposeOCL     cTranspose;
   CNeuronBaseOCL          cAttention;
   CNeuronSoftMaxOCL       cSoftMax;
   CNeuronBaseOCL          cMeans;
   CNeuronBaseOCL          cSTDevs;
   CNeuronBaseOCL          cNorm;
   //---
   virtual bool      AttentNorm(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AttentNormGrad(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSAttentNorm(void) {};
                    ~CNeuronSAttentNorm(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSAttentNorm; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   CNeuronBaseOCL*   GetMeans(void) { return cMeans.AsObject(); }
   CNeuronBaseOCL*   GetSTDevs(void) { return cSTDevs.AsObject(); }
   virtual uint      GetVariables(void) const { return iWindow; }
   virtual uint      GetUnits(void) const { return iCount; }
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {  activation = None; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPolynomialRegression   :  public CNeuronBaseOCL
  {
protected:
   CNeuronSwiGLUOCL     cProjection;
   CNeuronConvOCL       cConvolution;
   CNeuronConvOCL       cResidual;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPolynomialRegression(void) {};
                    ~CNeuronPolynomialRegression(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint window, uint window_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronPolynomialRegression; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint      GetWindowIn(void) const { return cProjection.GetWindow(); }
   virtual uint      GetWindowOut(void) const { return cResidual.GetFilters(); }
   virtual uint      GetUnits(void) const { return cProjection.GetUnits(); }
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSSCNNEncoder   :  public CNeuronTransposeOCL
  {
protected:
   CNeuronPeriodNorm       cLongNorm;
   CNeuronConvOCL          cLongExtrapolate;
   CNeuronTransposeOCL     cLongMeanSTDevTransp;
   CNeuronBaseOCL          cLongMeanExtrapolate;
   CNeuronTransposeVRCOCL  cSeasonTransp;
   CNeuronAttentNorm       cSeasonNorm;
   CNeuronTransposeVRCOCL  cUnSeasonTransp;
   CNeuronConvOCL          cSeasonExtrapolate;
   CNeuronConvOCL          cSeasonMeanExtrapolate;
   CNeuronAttentNorm       cShortNorm;
   CNeuronConvOCL          cShortExtrapolate;
   CNeuronConvOCL          cShortMeanExtrapolate;
   CNeuronSAttentNorm      cSpatialNorm;
   CNeuronConvOCL          cSpatialExtrapolate;
   CNeuronConvOCL          cSpatialMeanExtrapolate;
   CNeuronBaseOCL          cConcatenated;
   CNeuronTransposeOCL     cTranspose;
   CNeuronPolynomialRegression   cFusion;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL)  override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronSSCNNEncoder(void) {};
                    ~CNeuronSSCNNEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint variables, uint forecast,
                          uint season_period, uint short_period,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSSCNNEncoder; }
   virtual void      TrainMode(bool flag) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSSCNN    :  public   CNeuronSCNN
  {
protected:
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL)  override;

public:
                     CNeuronSSCNN(void) {};
                    ~CNeuronSSCNN(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units_count, uint variables, uint forecast,
                          uint season_period, uint short_period, uint layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSSCNN; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CCircleParams  :  public CNeuronBaseOCL
  {
protected:
   CLayer            cOnes;
   CNeuronBaseOCL    *cCurrent;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override          { return FeedForward(); }

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override   { return UpdateInputWeights(); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override   { return true; }

public:
                     CCircleParams(void) : cCurrent(NULL) {};
                    ~CCircleParams(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint period, ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual bool      Identity(const int rows, const int cols);
   virtual bool      Zeros(void);
   //---
   virtual bool      FeedForward(void);
   virtual bool      UpdateInputWeights(void);
   virtual bool      SetPosition(int position);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defCircleParams; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual CBufferFloat   *getWeightsParams(void);
   virtual int       GetPeriod(void) const { return cOnes.Total(); }
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTQMHA   :  public CNeuronCrossAttention
  {
protected:
   uint              iTimeframe;
   CCircleParams     cParams;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput);
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput);

public:
                     CNeuronTQMHA(void) {};
                    ~CNeuronTQMHA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint units_in, uint period, uint timeframe,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronTQMHA;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CChebPolinom   :  public CNeuronBaseOCL
  {
protected:
   uint              iDimension;
   uint              iSteps;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CChebPolinom(void)   {};
                    ~CChebPolinom(void)   {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint steps, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defChebPolinom; }
   virtual uint      GetDimension(void) const { return iDimension; }
   virtual uint      GetSteps(void) const { return iSteps; }
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHimNetGrapConv   :  public CNeuronTransposeRCDOCL
  {
protected:
   CNeuronBaseOCL    cX_G;

public:
                     CNeuronHimNetGrapConv(void) {};
                    ~CNeuronHimNetGrapConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint count, uint window, uint cheb_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual bool      FeedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support);
   //---
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void)        const                      {  return defNeuronHimNetGrapConv; }
   virtual void      SetOpenCL(COpenCLMy *obj);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual uint      GetCount(void) override const { return iWindow; }
   virtual uint      GetWindow(void) override const { return GetDimension(); }
   virtual uint      GetChebK(void) const { return iCount; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHimNetGCRU    :  public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL          cInpAndHidden;
   CNeuronHimNetGrapConv   cZ_R;
   CNeuronConvOCL          cZ_R_emb;
   CNeuronBaseOCL          cZe_Re;
   CNeuronBaseOCL          cZ;
   CNeuronBaseOCL          cR;
   CNeuronBaseOCL          cCandidate;
   CNeuronHimNetGrapConv   cHC;
   CNeuronConvOCL          cHC_emb;
   CNeuronBaseOCL          cHCe;
   CBufferFloat            bSupportAccum;

public:
                     CNeuronHimNetGCRU(void) {};
                    ~CNeuronHimNetGCRU(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint window_out,
                          uint cheb_k, uint embed_dim,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL,
                                 CChebPolinom *Support,
                                 CNeuronBaseOCL *Embedding);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CChebPolinom *Support,
                                        CNeuronBaseOCL *Embedding);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL,
                                        CChebPolinom *Support,
                                        CNeuronBaseOCL *Embedding);
   //---
   virtual int       Type(void)   const   {  return defNeuronHimNetGCRU;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual uint      GetCount(void) const { return cZ_R.GetCount(); }
   virtual uint      GetWindowIn(void) const { return cZ_R.GetWindow() - GetWindowOut(); }
   virtual uint      GetWindowOut(void) const { return Neurons() / GetCount(); }
   virtual uint      GetChebK(void) const { return cZ_R.GetChebK(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHimNetTempEncoder   : public CNeuronBaseOCL
  {
protected:
   uint              aTimeframes[2];
   CCircleParams     caEmbeddings[2];
   CNeuronBaseOCL    cConcatEmbeddings;
   CLayer            cGRCUs;
   CBufferFloat      bSupportAccum;

public:
                     CNeuronHimNetTempEncoder(void) {};
                    ~CNeuronHimNetTempEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint window_out, uint cheb_k,
                          uint layers, uint embed_dim, uint period1,
                          uint timeframe1, uint period2, uint timeframe2,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL,
                                 CChebPolinom *Support,
                                 int label);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CChebPolinom *Support);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL,
                                        CChebPolinom *Support);
   //---
   virtual int       Type(void)   const   {  return defNeuronHimNetTempEncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual bool      Clear(void) override;
   //---
   virtual uint      GetCount(void) const;
   virtual uint      GetWindowIn(void) const;
   virtual uint      GetWindowOut(void) const;
   virtual uint      GetChebK(void) const;
   //---
   virtual bool      SetGradient(CBufferFloat *buffer, bool delete_prev = true);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHimNetSpatEncoder   :     public CNeuronBaseOCL
  {
protected:
   CParams           cEmbedding;
   CLayer            cGRCUs;
   CBufferFloat      bSupportAccum;
   CBufferFloat      bEmbeddingAccum;

public:
                     CNeuronHimNetSpatEncoder(void) {};
                    ~CNeuronHimNetSpatEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint window_out, uint cheb_k,
                          uint layers, uint embed_dim,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL,
                                 CChebPolinom *Support);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CChebPolinom *Support);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL,
                                        CChebPolinom *Support);
   //---
   virtual int       Type(void)   const   {  return defNeuronHimNetSpatEncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual bool      Clear(void) override;
   //---
   virtual uint      GetCount(void) const;
   virtual uint      GetWindowIn(void) const;
   virtual uint      GetWindowOut(void) const;
   virtual uint      GetChebK(void) const;
   //---
   virtual bool      SetGradient(CBufferFloat *buffer, bool delete_prev = true);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHimNetEncoder    :  public CNeuronBaseOCL
  {
protected:
   CParams                    cEmbedding;
   CNeuronTransposeOCL        cEmbeddingT;
   CNeuronBaseOCL             cSupport;
   CNeuronSoftMaxOCL          cNormSupport;
   CChebPolinom               cPolinomSupport;
   CNeuronHimNetTempEncoder   cTempEncoder;
   CNeuronHimNetSpatEncoder   cSpatEncoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHimNetEncoder(void) {};
                    ~CNeuronHimNetEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint window_out, uint cheb_k,
                          uint layers, uint embed_dim, uint period1,
                          uint timeframe1, uint period2, uint timeframe2,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronHimNetEncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual bool      Clear(void) override;
   virtual bool      SetGradient(CBufferFloat *buffer, bool delete_prev = true);
  };

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronHimNetDecoder    :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL             cProjection;
   CNeuronTransposeOCL        cProjectionT;
   CNeuronConvOCL             cEmbedding;
   CNeuronBaseOCL             cSupport;
   CNeuronSoftMaxOCL          cNormSupport;
   CChebPolinom               cPolinomSupport;
   CLayer                     cGRCUs;
   CBufferFloat               bSupportAccum;
   CBufferFloat               bEmbeddingAccum;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronHimNetDecoder(void) {};
                    ~CNeuronHimNetDecoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint window_out,
                          uint cheb_k, uint layers, uint embed_dim,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronHimNetDecoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSNSMHAttention   :  public CNeuronConvOCL
  {
   float             fSparse;
   //---
   CNeuronBaseOCL    cNeighbors;
   CNeuronBaseOCL    cRamdomCandidates;
   CNeuronConvOCL    cProjection[2];
   CNeuronBaseOCL    cScores;

   //---
   virtual bool      SignificantNeighborsSampling(CNeuronBaseOCL *NeuronOCL);
   virtual bool      SparseMHScores(void);
   virtual bool      SparseMHScoresGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSNSMHAttention(void) {};
                    ~CNeuronSNSMHAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint heads,
                          uint m_units, float sparse,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSNSMHAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual CBufferFloat* GetIndexes(void) { return cNeighbors.getOutput(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFastGConv   :     public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL          cInpAndHidden;
   CNeuronBaseOCL          cNormAttention;
   CNeuronBaseOCL          cInvDiag;
   CNeuronBaseOCL          cAX;
   CNeuronBaseOCL          cAXplusX;
   CNeuronBaseOCL          cNormAXplusX;
   CNeuronConvOCL          cZ_R;
   CNeuronBaseOCL          cZ;
   CNeuronBaseOCL          cR;
   CNeuronBaseOCL          cCandidate;
   CNeuronConvOCL          cHC;
   //---
   virtual bool      RandomWalk(CBufferFloat* data,
                                CBufferFloat* normal,
                                CBufferFloat* inv_diag,
                                const int rows,
                                const int cols
                               );
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)  override { ReturnFalse; };
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None
                                       ) override { ReturnFalse; }
   //---
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronFastGConv(void) {};
                    ~CNeuronFastGConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint sparse_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      FeedForward(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent);
   virtual bool      CalcInputGradients(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent);
   //---
   virtual int       Type(void)   const   {  return defNeuronFastGConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual uint      GetCount(void) const { return (uint)cInvDiag.Neurons(); }
   virtual uint      GetSparseDimension(void) const { return (uint)cNormAttention.Neurons() / GetCount(); }
   virtual uint      GetWindow(void) const { return (uint)Neurons() / GetCount(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSAGDFN  :  public CNeuronTransposeOCL
  {
protected:
   CNeuronTransposeOCL     cTranspose;
   CLayer                  cEmbedding;
   CNeuronSNSMHAttention   cAttention;
   CLayer                  cGCRU;
   CLayer                  cProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSAGDFN(void) {};
                    ~CNeuronSAGDFN(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint time_steps, uint variables, uint embedding_dim,
                          uint emb_layers, uint sparse_dimension, uint heads,
                          float sparse, uint gcru_layers,
                          uint forecast, uint forec_layers,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSAGDFN;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpatialEmbedding   :     public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iUnits;
   uint              iEmbeddingDim;
   CParams           cEmbedding;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpatialEmbedding(void) {};
                    ~CNeuronSpatialEmbedding(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint embed_dim,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSpatialEmbedding;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTempEmbedding    : public CNeuronBaseOCL
  {
protected:
   uint              iWindow;
   uint              iUnits;
   uint              aiEmbeddingDim[2];
   uint              aiFrames[2];
   uint              aiPeriod[2];
   CParams           caEmbeddings[2];
   //---
   virtual bool      ConcatByLabel(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput);
   virtual bool      ConcatByLabelGrad(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;}
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;}
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None
                                       ) override;

public:
                     CNeuronTempEmbedding(void) {};
                    ~CNeuronTempEmbedding(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window,
                          uint embed_dim1, uint period1, uint frame1,
                          uint embed_dim2, uint period2, uint frame2,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronTempEmbedding;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGlobalLocalAttention   :  public CNeuronMHFeedForward
  {
protected:
   CNeuronSNSMHAttention   cMask;
   CNeuronConvOCL          cQ;
   CNeuronConvOCL          cKV;
   CNeuronBaseOCL          cScore;
   CNeuronBaseOCL          cMHAttention;
   CNeuronConvOCL          cW0;
   CNeuronBaseOCL          cResidual;
   //---
   virtual bool      GlobalLocalAttention(void);
   virtual bool      GlobalLocalAttentionGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGlobalLocalAttention(void) {};
                    ~CNeuronGlobalLocalAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint dimension_k,
                          uint heads, uint m_units, float sparse,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronGlobalLocalAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronExtralonger   :  public CNeuronMHAttentionPooling
  {
protected:
   CLayer               cProjectionT;
   CLayer               cTimeModule;
   CLayer               cSpatialModule;
   CLayer               cMixModule;
   CNeuronBaseOCL       cConcatResults;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;}
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;}
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None
                                       ) override;

public:
                     CNeuronExtralonger(void) {};
                    ~CNeuronExtralonger(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint time_steps_in, uint time_steps_out, uint variables,
                          uint dimension, uint emb_dimension, uint period1,
                          uint frame1, uint period2, uint frame2, uint layers,
                          uint heads, uint dimension_k, uint m_units, float sparse,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronExtralonger;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGraphons   :  public CNeuronBaseOCL
  {
protected:
   CLayer            acProbability;
   CLayer            acDataEmb;
   CParams           cExpertsEmb;
   CNeuronBaseOCL    cGraphs;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGraphons(void) {};
                    ~CNeuronGraphons(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint emb_dimension,
                          uint experts, float dropout,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronGraphons;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGraphAttention   :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL    cValue;
   CNeuronGraphons   cGraphs;
   CNeuronSoftMaxOCL cScores;
   CNeuronBaseOCL    cAttention;
   CNeuronBaseOCL    cResidual;
   CNeuronConvOCL    cFeedForward[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGraphAttention(void) {};
                    ~CNeuronGraphAttention(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint emb_dimension,
                          uint experts, float dropout,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronGraphAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSparseSoftMax    :  public CNeuronSoftMaxOCL
  {
protected:
   uint              iDimensionIn;
   CBufferFloat      cIndexes;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSparseSoftMax(void) {};
                    ~CNeuronSparseSoftMax(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint dimension_in, uint dimension_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual int       Type(void) override const { return defNeuronSparseSoftMax; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual CBufferFloat* GetIndexes(void)  { return GetPointer(cIndexes); }
   virtual uint      DimensionOut(void) const {  return uint(Neurons() / iHeads); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSparseGraphAttention   :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL       cValue;
   CNeuronGraphons      cGraphs;
   CNeuronSparseSoftMax cScores;
   CNeuronBaseOCL       cAttention;
   CNeuronBaseOCL       cResidual;
   CNeuronConvOCL       cFeedForward[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSparseGraphAttention(void) {};
                    ~CNeuronSparseGraphAttention(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint experts, float dropout,
                          uint emb_dimension, uint sparse_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronSparseGraphAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGlobLocGraphAtt  :  public CNeuronMHAttentionPooling
  {
protected:
   CNeuronGraphAttention         cGlobal;
   CNeuronSparseGraphAttention   cLocal;
   CNeuronBaseOCL                cConcat;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGlobLocGraphAtt(void) {};
                    ~CNeuronGlobLocGraphAtt(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint experts, float dropout,
                          uint emb_dimension, uint sparse_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronGlobLocGraphAtt;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronExtralongerGraph   :  public CNeuronExtralonger
  {
public:
                     CNeuronExtralongerGraph(void) {};
                    ~CNeuronExtralongerGraph(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint time_steps_in, uint time_steps_out, uint variables,
                          uint dimension, uint emb_dimension, uint period1,
                          uint frame1, uint period2, uint frame2, uint layers,
                          uint experts, uint m_units, float sparse,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronExtralongerGraph;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeConv  :  public CNeuronBaseOCL
  {
protected:
   CNeuronConvOCL    cConv;
   CParams           cLevels;
   //---
   virtual bool      FloatToSpike(void);
   virtual bool      FloatToSpikeGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeConv(void) {};
                    ~CNeuronSpikeConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint window_out, uint units_count,
                          uint variables, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual CBufferFloat* GetWeightsConv(void)   {  return cConv.GetWeightsConv();  }
   virtual int       Type(void) override const  {  return defNeuronSpikeConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cConv.GetWindow(); }
   virtual uint      GetFilters(void) const { return cConv.GetFilters(); }
   virtual uint      GetVariables(void) const { return cConv.GetVariables(); }
   virtual uint      GetUnits(void) const { return cConv.GetUnits(); }
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGateLineAttention  :  public CNeuronSpikeConv
  {
protected:
   uint              iDimensionK;
   uint              iVariables;
   uint              iHeads;
   //---
   CNeuronConvOCL    cQKV;
   CNeuronBaseOCL    cQ;
   CNeuronBaseOCL    cK;
   CNeuronBaseOCL    cV;
   CNeuronBaseOCL    cKV;
   CNeuronConvOCL    cGate;
   CNeuronBaseOCL    cState;
   CNeuronBaseOCL    cMHAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGateLineAttention(void) {};
                    ~CNeuronGateLineAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint variables, uint dimension, uint dimension_k, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronGateLineAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual bool      Clear(void) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetVariables(void) const { return iVariables; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeSparseAttention   :  public CNeuronSpikeConv
  {
protected:
   CNeuronConvOCL       cValue;
   CNeuronGraphons      cGraphs;
   CNeuronSparseSoftMax cScores;
   CNeuronBaseOCL       cAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeSparseAttention(void) {};
                    ~CNeuronSpikeSparseAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint experts, float dropout,
                          uint emb_dimension, uint sparse_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronSpikeSparseAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMoEConv   :  public CNeuronBaseOCL
  {
protected:
   CLayer            acProbability;
   CParams           cExperts;
   CNeuronBaseOCL    cWeightsConv;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMoEConv(void) {};
                    ~CNeuronMoEConv(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint window_out,
                          uint experts, float dropout, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronMoEConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikingBrain  :  public CNeuronBaseOCL
  {
protected:
   CNeuronSpikeSparseAttention   cTimeAttention;
   CLayer                        caSpatialAttention;
   CNeuronBaseOCL                cResidual;
   CLayer                        caFFN;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikingBrain(void) {};
                    ~CNeuronSpikingBrain(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint experts, float dropout,
                          uint emb_dimension, uint sparse_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const   {  return defNeuronSpikingBrain;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeActivation  :  public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cHV;
   CParams           cLevels;
   CParams           cGates;
   //---
   virtual bool      FloatToSpike(void);
   virtual bool      FloatToSpikeGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeActivation(void) {};
                    ~CNeuronSpikeActivation(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint neurons, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeActivation;   }
   //--- methods for working with files
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeResNeXtBottleneck :  public CNeuronSpikeActivation
  {
protected:
   CLayer            cBottleneck;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;


public:
                     CNeuronSpikeResNeXtBottleneck(void) {};
                    ~CNeuronSpikeResNeXtBottleneck(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out, uint units_in,
                          uint units_out, uint group_size, uint groups,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeResNeXtBottleneck;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeResNeXtResidual :  public CNeuronSpikeResNeXtBottleneck
  {
public:
                     CNeuronSpikeResNeXtResidual(void) {};
                    ~CNeuronSpikeResNeXtResidual(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out,
                          uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeResNeXtResidual;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeResNeXtBlock   :  public CNeuronSpikeActivation
  {
protected:
   CNeuronTransposeOCL           caTranspose[2];
   CNeuronSpikeResNeXtBottleneck caBottleneck[2];
   CNeuronSpikeResNeXtResidual   cResidual;
   CLayer                        cG;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeResNeXtBlock(void) {};
                    ~CNeuronSpikeResNeXtBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out, uint units_in,
                          uint units_out, uint group_size, uint groups,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeResNeXtBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   //---
   virtual uint      GetChannelsOut(void) const { return caTranspose[1].GetCount(); }
   virtual uint      GetUnitsOut(void) const { return caTranspose[1].GetWindow(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSSAM    :  public CNeuronSpikeActivation
  {
protected:
   uint                 iUnits;
   uint                 iHeads;
   uint                 iDimension;
   bool                 bMaskFuture;
   CNeuronSpikeConv     cQKV;
   CParams              cDiaganalBias;
   int                  ciScore;
   CNeuronBaseOCL       cMHAttentionOut;
   CNeuronSpikeConv     cW0;
   CNeuronBaseOCL       cResidual;
   CNeuronBatchNormOCL  cNorm;
   //---
   virtual bool      SpikeMHAttention(void);
   virtual bool      SpikeMHAttentionGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSSAM(void) : ciScore(INVALID_HANDLE) {};
                    ~CNeuronSSAM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint heads, uint dimension_k,
                          bool mask_future, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSSAM;   }
   //--- methods for working with files
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSSAMResNeXtBlock   :  public CNeuronSpikeResNeXtBlock
  {
protected:
   CNeuronSSAM        caSSAM[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSSAMResNeXtBlock(void) {};
                    ~CNeuronSSAMResNeXtBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out, uint units_in,
                          uint units_out, uint group_size, uint groups,
                          uint heads, uint dimension_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSSAMResNeXtBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTFS    :  public   CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL    cMaskTime;
   CNeuronBaseOCL    cMaskSpatial;
   CNeuronBaseOCL    cConcatenated;
   CNeuronConvOCL    cSpatialDScale;
   CNeuronConvOCL    cTimeDScale;
   CNeuronTransposeVRCOCL caTranspose[2];
   CBufferFloat      cOne;
   //---
   virtual bool      STFS(CNeuronBaseOCL *NeuronOCL);
   virtual bool      STFSGrad(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTFS(void) {};
                    ~CNeuronSTFS(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out, uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSTFS;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeConvGRU  :  public CNeuronSpikeActivation
  {
protected:
   CNeuronBaseOCL       cConcatenated[2];
   CNeuronConvOCL       cZR;
   CNeuronBaseOCL       cZ;
   CNeuronBaseOCL       cR;
   CNeuronConvOCL       cHt;
   CNeuronBaseOCL       cH;
   CNeuronBatchNormOCL  cNorm;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeConvGRU(void) {};
                    ~CNeuronSpikeConvGRU(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint chanels_in, uint chanels_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeConvGRU;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      Clear(void) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual uint      GetChanelsIn(void) const { return cHt.GetWindow() - cHt.GetFilters(); }
   virtual uint      GetChanelsOut(void) const { return cHt.GetFilters(); }
   virtual uint      GetUnits(void) const { return cHt.GetUnits(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAddToStack :  public CNeuronBaseOCL
  {
protected:
   uint              iStackSize;
   uint              iDimension;
   uint              iVariables;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAddToStack(void) : iStackSize(0), iDimension(0), iVariables(0) {};
                    ~CNeuronAddToStack(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronAddToStack;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual uint      GetStackSize(void) const { return iStackSize; }
   virtual uint      GetDimension(void) const { return iDimension; }
   virtual uint      GetVariables(void) const { return iVariables; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronStackCorrelation :  public CNeuronBaseOCL
  {
protected:
   CNeuronAddToStack cStack;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronStackCorrelation(void) {};
                    ~CNeuronStackCorrelation(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronStackCorrelation;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      Clear(void) override;
   //---
   virtual uint      GetStackSize(void) const { return cStack.GetStackSize(); }
   virtual uint      GetDimension(void) const { return cStack.GetDimension(); }
   virtual uint      GetVariables(void) const { return cStack.GetVariables(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTEFlowNetEncoder   :  public CNeuronSpikeConvGRU
  {
protected:
   CLayer            cRecurrentLine;
   CLayer            cFlow;
   CLayer            cCorrelation;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTEFlowNetEncoder(void) {};
                    ~CNeuronSTEFlowNetEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &chanels[], uint &units[], uint group_size,
                          uint groups, uint heads, uint dimension_k, uint stack_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual CNeuronBaseOCL* GetLayer(uint layer);
   virtual uint      Layers(void)   const { return (cRecurrentLine.Total() + 1) / 3; }
   //--
   virtual int       Type(void)   override const   {  return defNeuronSTEFlowNetEncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTEFlowNetResidualBlock :  public CNeuronBaseOCL
  {
protected:
   CLayer            cBlock;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTEFlowNetResidualBlock(void) {};
                    ~CNeuronSTEFlowNetResidualBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels, uint units, uint group_size, uint groups,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //--
   virtual int       Type(void)   override const   {  return defNeuronSTEFlowNetResidualBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTEFlowNetDecoder   :  public CNeuronSpikeActivation
  {
protected:
   CLayer                        cMainLine;
   CLayer                        cFlow;
   CNeuronSTEFlowNetEncoder*     cEncoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTEFlowNetDecoder(void) {};
                    ~CNeuronSTEFlowNetDecoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &chanels[], uint &units[], uint group_size,
                          uint groups, uint heads, uint dimension_k,
                          CNeuronSTEFlowNetEncoder* encoder,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //--
   virtual int       Type(void)   override const   {  return defNeuronSTEFlowNetDecoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetEncoder(CNeuronSTEFlowNetEncoder* encoder) { cEncoder = encoder; }
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTEFlowNet  :  public CNeuronSTEFlowNetDecoder
  {
   //---
protected:
   CNeuronSTEFlowNetEncoder         cSTEEncoder;
   CNeuronSTEFlowNetResidualBlock   cSTEResidualBlock;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTEFlowNet(void) {};
                    ~CNeuronSTEFlowNet(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &chanels[], uint &units[], uint group_size,
                          uint groups, uint heads, uint dimension_k, uint stack_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //--
   virtual int       Type(void)   override const   {  return defNeuronSTEFlowNet;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;

  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiScaleStackCorrelation :  public CNeuronStackCorrelation
  {
protected:
   uint                 iLevels;
   CNeuronBaseOCL       cScales;
   //---
   virtual bool      AggregationByTime(CNeuronBaseOCL *NeuronOCL);
   virtual bool      AggregationByTimeGrad(CNeuronBaseOCL *NeuronOCL);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultiScaleStackCorrelation(void) {};
                    ~CNeuronMultiScaleStackCorrelation(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension,
                          uint variables, uint levels,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronMultiScaleStackCorrelation;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual uint      GetLevels(void) const { return iLevels; }
   virtual uint      GetVariables(void) override const { return cStack.GetVariables() / iLevels; }

  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeConvGRU2D  :  public CNeuronSpikeConvGRU
  {
protected:
   CNeuronConvOCL       cProjectionX;
   CNeuronBaseOCL       cConcatenated2[2];
   CNeuronConvOCL       cZR2;
   CNeuronBaseOCL       cZ2;
   CNeuronBaseOCL       cR2;
   CNeuronConvOCL       cHt2;
   CNeuronBaseOCL       cH2;
   CNeuronTransposeOCL  cTranspose[3];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeConvGRU2D(void) {};
                    ~CNeuronSpikeConvGRU2D(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint chanels_in, uint chanels_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeConvGRU2D;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronRAFT :  public CNeuronSpikeActivation
  {
protected:
   CLayer                              cEncoder;
   CLayer                              cContext;
   CLayer                              cCorrelation;
   CNeuronBaseOCL                      cConcatenated;
   CNeuronSpikeConvGRU2D               cGRU;
   CNeuronSpikeResNeXtResidual         cResidual;
   CLayer                              cG;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronRAFT(void) {};
                    ~CNeuronRAFT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &units[], uint &chanels[], uint group_size,
                          uint groups, uint stack_size, uint levels,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronRAFT;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultScalStackCorrelBySegments :  public CNeuronAddToStack
  {
protected:
   CNeuronMultiScaleStackCorrelation cCorrelation;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultScalStackCorrelBySegments(void) {};
                    ~CNeuronMultScalStackCorrelBySegments(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint segments, uint segment_size, uint dimension,
                          uint variables, uint levels,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronMultScalStackCorrelBySegments;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override;
   virtual bool      Clear(void) override;
   //---
   virtual uint      GetLevels(void) const { return cCorrelation.GetLevels(); }
   virtual uint      SetSegmentSize(void) const { return cCorrelation.GetStackSize(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTMAMFE  :  public CNeuronConvOCL
  {
protected:
   CLayer            cEncoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTMAMFE(void) {};
                    ~CNeuronTMAMFE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &windows[], uint window_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronTMAMFE;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTMAMPA  :  public CNeuronBaseOCL
  {
protected:
   CNeuronRelativeCrossAttention cCrossAttention;
   CLayer                        cProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTMAMPA(void) {};
                    ~CNeuronTMAMPA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint units,
                          uint segments, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronTMAMPA; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronTMA  :  public CNeuronRAFT
  {
protected:
   CNeuronAddToStack cStatesStack;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronTMA(void) {};
                    ~CNeuronTMA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &units[], uint &chanels[], uint group_size,
                          uint groups, uint segments, uint segment_size, uint levels,
                          uint window_key, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronTMA;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeSTConvGRU  :  public CNeuronSpikeActivation
  {
protected:
   CNeuronBaseOCL             cConcatenated[2];
   CNeuronMultiWindowsConvOCL cFSM;
   CNeuronBaseOCL             cSM;
   CNeuronConvOCL             cOs;
   CNeuronConvOCL             cOt;
   CNeuronBaseOCL             cFo;
   CNeuronTransposeRCDOCL     cS;
   CNeuronBatchNormOCL        cNorm;
   //---
   virtual bool      GRU(void);
   virtual bool      GRU_grad(void);

   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronSpikeSTConvGRU(void) {};
                    ~CNeuronSpikeSTConvGRU(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &chanels_in[], uint chanels_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeSTConvGRU;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      Clear(void) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual uint      GetChanels(void) const { return cOt.GetFilters(); }
   virtual uint      GetUnits(void) const { return cOt.GetUnits(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFERENet :  public CNeuronSpikeConvGRU
  {
protected:
   CLayer            cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronFERENet(void) {};
                    ~CNeuronFERENet(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &chanels[],
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronFERENet;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      Clear(void) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual CNeuronBaseOCL* GetLayer(int index) const { return cFlow[index]; }
   virtual CNeuronBaseOCL* operator[](const int index) const { return GetLayer(index); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFGDModule  :  public CNeuronBaseOCL
  {
protected:
   CNeuronBatchNormOCL        cNorm;
   CNeuronConvOCL             cD;
   CNeuronBaseOCL             cFlowIn;
   CNeuronConvOCL             cFlow;
   CNeuronBaseOCL             cConcatenated;
   CNeuronMultiWindowsConvOCL cXH;
   //---
   virtual bool      GRU(void);
   virtual bool      GRU_grad(void);

   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronFGDModule(void) {};
                    ~CNeuronFGDModule(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units,
                          uint main_dimension, uint flow_dimension, uint dimension_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronFGDModule;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual CNeuronConvOCL* GetD(void)  { return cD.AsObject(); }
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronEVMGRFlowNet  :  public CNeuronSpikeActivation
  {
protected:
   CNeuronFERENet                   cEncoder;
   CNeuronSTEFlowNetResidualBlock   cNeck;
   CLayer                           cDecoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronEVMGRFlowNet(void) {};
                    ~CNeuronEVMGRFlowNet(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &chanels[],
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronEVMGRFlowNet;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual bool      Clear(void) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeConvBlock   :  public CNeuronBatchNormOCL
  {
protected:
   CNeuronSpikeActivation  cActivation;
   CNeuronConvOCL          cConv;
   CNeuronAddToStack       cStack;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeConvBlock(void) {};
                    ~CNeuronSpikeConvBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint step, uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeConvBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cConv.GetWindow();   }
   virtual uint      GetFilters(void) const { return cConv.GetFilters(); }
   virtual uint      GetVariables(void) const { return cConv.GetVariables(); }
   virtual uint      GetUnits(void) const { return cConv.GetUnits(); }
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMSRes   :  public CNeuronBaseOCL
  {
protected:
   CLayer            cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMSRes(void) {};
                    ~CNeuronMSRes(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMSRes;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   virtual uint      GetWindow(void) const { return ((CNeuronSpikeConvBlock*)cFlow[0]).GetWindow();   }
   virtual uint      GetVariables(void) const { return ((CNeuronSpikeConvBlock*)cFlow[0]).GetVariables(); }
   virtual uint      GetUnits(void) const { return ((CNeuronSpikeConvBlock*)cFlow[0]).GetUnits(); }
   virtual CBufferFloat   *getWeights(void) override { return (!!cFlow[-1] ? cFlow[-1].getWeights() : NULL); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeQKAttention :  public CNeuronBaseOCL
  {
protected:
   CNeuronSpikeConvBlock      cQK;
   CNeuronSpikeActivation     cSpikeQK;
   CNeuronBaseOCL             cQ;
   CNeuronBaseOCL             cK;
   CNeuronMultiWindowsConvOCL cScore;
   CNeuronSpikeActivation     cSpikeScore;
   CLayer                     cProject;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeQKAttention(void) {};
                    ~CNeuronSpikeQKAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint heads, uint dimension_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeQKAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSDSA :  public CNeuronMSRes
  {
protected:
   CNeuronSpikeQKAttention cAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSDSA(void) {};
                    ~CNeuronSDSA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint heads, uint dimension_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSDSA;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSDformerUBlock   :   public CNeuronSpikeConvBlock
  {
protected:
   CLayer            cEncoder;
   CNeuronBaseOCL*   cNeck;
   CNeuronBaseOCL    cConcat;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSDformerUBlock(void) {};
                    ~CNeuronSDformerUBlock(void) { DeleteObj(cNeck) };
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &windows[], uint &heads[],
                          uint &layers[], uint dimension_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSDformerUBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   virtual CBufferFloat*   GetStack(void);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSDformerFlow   :   public CNeuronSDformerUBlock
  {
protected:
   CLayer            cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSDformerFlow(void) {};
                    ~CNeuronSDformerFlow(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &windows[], uint &heads[],
                          uint &layers[], uint dimension_k,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSDformerFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikePatchStak   :  public CNeuronAddToStack
  {
protected:
   CNeuronSpikeConvBlock   cBlock;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikePatchStak(void) {};
                    ~CNeuronSpikePatchStak(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension,
                          uint patch_dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikePatchStak;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpike2DSSMOCL  :  public CNeuronBaseOCL
  {
protected:
   uint                    iWindowOut;
   uint                    iUnitsOut;
   //---
   CNeuronBaseOCL          cHiddenStates;
   CLayer                  cProjectionX_Time;
   CLayer                  cProjectionX_Variable;
   CNeuronSpikeConvBlock   cA;
   CNeuronSpikeConvBlock   cBC_Time;
   CNeuronSpikeConvBlock   cBC_Variable;
   CNeuronBaseOCL          cB_Time;
   CNeuronBaseOCL          cB_Variable;
   CNeuronBaseOCL          cC_Time;
   CNeuronBaseOCL          cC_Variable;
   CNeuronSpikeConvBlock   cDelta_Time;
   CNeuronSpikeConvBlock   cDelta_Variable;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForwardSSM2D(void);
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradientsSSM2D(void);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpike2DSSMOCL(void)  {};
                    ~CNeuronSpike2DSSMOCL(void)  {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint window_out, uint units_in, uint units_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSpike2DSSMOCL; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeMamba2D   :  public CNeuronSpikeConvBlock
  {
protected:
   CNeuronSpikeConvBlock   cXZProject;
   CNeuronBaseOCL          cX;
   CNeuronBaseOCL          cZ;
   CNeuronSpike2DSSMOCL    cSSM;
   CNeuronBaseOCL          cZSSM;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeMamba2D(void) {};
                    ~CNeuronSpikeMamba2D(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint inside_dimension, uint units_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSpikeMamba2D; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj);
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeSTSSM   :  public CNeuronSpikeConvBlock
  {
protected:
   CNeuronSpikePatchStak   cEmbeddings;
   CLayer                  cProjection;
   CLayer                  cSSM;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeSTSSM(void) {};
                    ~CNeuronSpikeSTSSM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension, uint patch_dimension,
                          uint ssm_dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeSTSSM;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      Clear(void) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronESTMEncoder   :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer            cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronESTMEncoder(void) {};
                    ~CNeuronESTMEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &stack_size[], uint &dimension[], uint patch_dimension,
                          uint ssm_dimension, uint &variables[],
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronESTMEncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      Clear(void) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCreateFlow :  public CNeuronSpikePatchStak
  {
protected:
   CNeuronBaseOCL    cState;
   //---
   virtual bool      CalcFlow(CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCreateFlow(void) {};
                    ~CNeuronCreateFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension,
                          uint patch_dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronCreateFlow;   }
   //--- methods for working with files
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      Clear(void) override;
   //---
   virtual CNeuronBaseOCL* GetState(void) { return cState.AsObject(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBiDirectCorrelation :  public CNeuronTMAMFE
  {
protected:
   CNeuronBatchNormOCL        cInpNorm;
   CNeuronBaseOCL             cRefFrame;
   CNeuronTransposeOCL        cRefFrameT;
   CNeuronMultiWindowsConvOCL cWarp;
   CNeuronBatchNormOCL        cWarpNorm;
   CNeuronBaseOCL             cCorrelation;
   CNeuronBaseOCL             cConcat;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronBiDirectCorrelation(void) {};
                    ~CNeuronBiDirectCorrelation(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint variables, uint window_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronBiDirectCorrelation;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSATMA  :  public CNeuronBaseOCL
  {
protected:
   CLayer                        cWarp;
   CNeuronBaseOCL                cRefFrame;
   CNeuronRelativeCrossAttention cCrossAttention;
   CNeuronConvOCL                cSpatialAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSATMA(void) {};
                    ~CNeuronSATMA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint variables,
                          uint window_key, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override   const   {  return defNeuronSATMA; }
   //---
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value)  override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBAT :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer                              cPrepare;
   CLayer                              cEncoder;
   CLayer                              cContext;
   CNeuronBaseOCL                      cConcatenated;
   CNeuronSpikeConvGRU2D               cGRU;
   CNeuronTransposeRCDOCL              cTranspose;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronBAT(void) {};
                    ~CNeuronBAT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint embed_size,
                          uint stack_size, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronBAT;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   virtual CBufferFloat   *getWeights(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeSCM   :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer            cScale;
   CNeuronBaseOCL    cConcat;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeSCM(void) {};
                    ~CNeuronSpikeSCM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeSCM;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return (!cScale[0] ? 0 : ((CNeuronSpikeConvBlock*)cScale[0]).GetWindow());   }
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeDepthWiseConv  :  public CNeuronSpikeConvBlock
  {
protected:
   CNeuronTransposeVRCOCL           cTranspose[2];
   CNeuronMultiWindowsConvWPadOCL   cDepthConv;
   CNeuronBatchNormOCL              cNorm;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeDepthWiseConv(void) {};
                    ~CNeuronSpikeDepthWiseConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out,
                          uint depth_window, uint depth_step,
                          uint units, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeDepthWiseConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeFAM   :  public CNeuronBatchNormOCL
  {
protected:
   CNeuronBaseOCL                   cProduct;
   CNeuronSpikeActivation           cActivation;
   CNeuronSpikeDepthWiseConv        cConv;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronSpikeFAM(void) {};
                    ~CNeuronSpikeFAM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeFAM;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cConv.GetWindow();   }
   virtual uint      GetVariables(void) const { return cConv.GetVariables(); }
   virtual uint      GetUnits(void) const { return cConv.GetUnits(); }
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeMDC   :  public CNeuronBatchNormOCL
  {
protected:
   CLayer            cSCMs;
   CLayer            cFAMs;
   CLayer            cAFFs;
   CLayer            cProjects;
   CLayer            cProjectsOut;
   CLayer            cFeatExtracts;
   CNeuronBaseOCL    cConcatOut;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeMDC(void) {};
                    ~CNeuronSpikeMDC(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out,
                          uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeMDC;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMDS   :  public CNeuronBaseOCL
  {
protected:
   uint                    iDimension;
   uint                    iUnits;
   uint                    iVariables;
   CNeuronBaseOCL          cConcat;
   CLayer                  cScores;
   CNeuronTransposeVRCOCL  cTranspose;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronMDS(void) {};
                    ~CNeuronMDS(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMDS;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeADM : public CNeuronMDS
  {
protected:
   CNeuronSpikeConvBlock   cConv;
   CNeuronSpikeMDC         cMDC;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override {  return updateInputWeights(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); }

public:
                     CNeuronSpikeADM(void) {};
                    ~CNeuronSpikeADM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeADM;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBATvADM :  public CNeuronBAT
  {
public:
                     CNeuronBATvADM(void) {};
                    ~CNeuronBATvADM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint embed_size,
                          uint stack_size, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronBATvADM;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCDCSelfCorrector :  public CNeuronBatchNormOCL
  {
protected:
   uint              iUnits;
   uint              iVariables;
   CLayer            cEstimator;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCDCSelfCorrector(void) {};
                    ~CNeuronCDCSelfCorrector(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint &inside_dimensions[],
                          uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronCDCSelfCorrector;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeCDC  :  public CNeuronSpikeConvBlock
  {
protected:
   CNeuronCDCSelfCorrector       cEsimator;
   CNeuronBaseOCL                cFlowCorrector;
   CNeuronRelativeSelfAttention  cFlowCorrelation;
   CNeuronBaseOCL                cFlowMask;
   CNeuronBaseOCL                cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeCDC(void) {};
                    ~CNeuronSpikeCDC(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint &inside_dimensions[],
                          uint window_out, uint units_count, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeCDC;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeEEMFLow  :   public CNeuronSpikeCDC
  {
protected:
   uint                    iMaxDisplacement;
   int                     ibShifts;
   CLayer                  cPrepare;
   CLayer                  cEncoder[3];
   CNeuronBaseOCL          cCorrelations[3];
   CNeuronBaseOCL          cConcat[3];
   CLayer                  cDecoder[4];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeEEMFLow(void) : ibShifts(INVALID_HANDLE) {};
                    ~CNeuronSpikeEEMFLow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint embed_size,
                          uint stack_size, uint variables,
                          uint max_displacement,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeEEMFLow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   virtual CBufferFloat   *getWeights(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiScaleExcitation   :  public CNeuronBaseOCL
  {
protected:
   CLayer            cExcitation;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultiScaleExcitation(void) {};
                    ~CNeuronMultiScaleExcitation(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint scales, uint bottleneck,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMultiScaleExcitation;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiScaleDifference :  public CNeuronSpikeConvBlock
  {
protected:
   uint              iaShifts[];
   int               ibShifts;
   //---
   CLayer            cBottleneck;
   CLayer            cExcitation;
   CLayer            cProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultiScaleDifference(void) : ibShifts(INVALID_HANDLE) {};
                    ~CNeuronMultiScaleDifference(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint bottleneck, uint &shifts[],
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMultiScaleDifference;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCorrelationEncoder  :  public CNeuronSpikeConvBlock
  {
protected:
   uint                    iMaxDisplacement;
   int                     ibShifts;
   CNeuronBaseOCL          cCorrelation;
   CNeuronSpikeConvBlock   cWarpCorrs;
   CNeuronSpikeConvBlock   cWarpInps;
   CLayer                  cProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCorrelationEncoder(void) : ibShifts(INVALID_HANDLE) {};
                    ~CNeuronCorrelationEncoder(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units,
                          uint bottleneck, uint max_displacement,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronCorrelationEncoder;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronEDCFlow :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer                        cPrepare;
   CLayer                        cFeatureEncoder[2];
   CLayer                        cConextEncoder;
   CNeuronMultiScaleDifference   cDiffEncoder;
   CNeuronCorrelationEncoder     cCorrelation;
   CLayer                        cCorrelationUpsampling;
   CLayer                        cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronEDCFlow(void) {};
                    ~CNeuronEDCFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_in, uint units_in, uint bottleneck,
                          uint dimension_out, uint units_out,
                          uint &shifts[], uint max_displacement,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronEDCFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual CBufferFloat   *getWeights(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCreateICEFlow :  public CNeuronCreateFlow
  {
protected:
   CNeuronBaseOCL       cConcat;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCreateICEFlow(void) {};
                    ~CNeuronCreateICEFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension,
                          uint patch_dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronCreateICEFlow;   }
   //--- methods for working with files
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeMFE  :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer            cEncoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeMFE(void) {};
                    ~CNeuronSpikeMFE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint &windows[], uint window_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronSpikeMFE;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeMHCrossAttention :  public CNeuronMSRes
  {
protected:
   uint                    iHeads;
   //---
   CNeuronSpikeConvBlock   cQ_Embedding;
   CNeuronSpikeConvBlock   cKV_Embedding;
   int                     ibScoreIndex;
   CNeuronBaseOCL          cMHAttentionOut;
   CNeuronSpikeConvBlock   cW0;
   CNeuronBaseOCL          cAttentionOut;
   CNeuronBaseOCL          cContext;
   //---
   virtual bool            Attention(void);
   virtual bool            AttentionGradients(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;

public:
                     CNeuronSpikeMHCrossAttention(void) : ibScoreIndex(INVALID_HANDLE) {};
                    ~CNeuronSpikeMHCrossAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads,
                          uint units_count, uint window_k, uint units_k,
                          ENUM_OPTIMIZATION optimization_type,
                          uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronSpikeMHCrossAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeMixFusion   :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer            cMain;
   CNeuronConvOCL    cResidual;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeMixFusion(void) {};
                    ~CNeuronSpikeMixFusion(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_in, uint dimension_out,
                          uint units_count, uint depth,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeMixFusion;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cResidual.GetWindow();   }
   //---
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTFlow :  public CNeuronSpikeConvBlock
  {
protected:
   uint                    iMaxDisplacement;
   int                     ibShifts;
   //---
   CNeuronBatchNormOCL           cNorm;
   CNeuronCreateICEFlow          cICE;
   CNeuronSpikePatchStak         cEvents;
   CLayer                        cEncoder;
   CNeuronBaseOCL                cICEEmbedding;
   CNeuronBaseOCL                cEventsEmbedding;
   CNeuronBaseOCL                cContext;
   CNeuronBaseOCL                cICECorrelation;
   CNeuronBaseOCL                cEventsCorrelation;
   CNeuronBaseOCL                cFeatureVsCorrelation;
   CNeuronSpikeMFE               cMoutionEncoder;
   CNeuronTransposeRCDOCL        cTranspose;
   CNeuronSpikeMHCrossAttention  cCrossModalAggregation;
   CNeuronSpikeMixFusion         cMixFusion;
   CLayer                        cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTFlow(void) {};
                    ~CNeuronSTFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_in, uint units_in, uint bottleneck,
                          uint dimension_out, uint units_out, uint max_displacement,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSTFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPSSE :  public CNeuronAddToStack
  {
protected:
   CLayer                  cPrepare;
   //---
   CParams                 cA;
   CNeuronSpikeConvBlock   cBCDxE;
   //---
   CNeuronBaseOCL          cB;
   CNeuronBaseOCL          cC;
   CNeuronBaseOCL          cDx;
   CNeuronBaseOCL          cE;
   CNeuronBaseOCL          cH;
   //---
   CNeuronBaseOCL          cAE;
   CNeuronTransposeVRCOCL  cAET;
   CNeuronBaseOCL          cAED;
   //---
   CNeuronBaseOCL          cAH;
   CNeuronBaseOCL          cBX;
   CNeuronBaseOCL          cHc;
   CNeuronBaseOCL          cCHc;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPSSE(void) {};
                    ~CNeuronPSSE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension,
                          uint patch_dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronPSSE;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeDepthWiseResidual :  public CNeuronSpikeDepthWiseConv
  {
protected:
   CNeuronSpikeConvBlock         cResidual;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeDepthWiseResidual(void) {};
                    ~CNeuronSpikeDepthWiseResidual(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out,
                          uint depth_window, uint depth_step,
                          uint units, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeDepthWiseResidual;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeSuperKernelBlock  :  public CNeuronMSRes
  {
protected:
   CLayer            cSKFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeSuperKernelBlock(void) {};
                    ~CNeuronSpikeSuperKernelBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels_in, uint chanels_out,
                          uint &depth_window[], uint &depth_step[],
                          uint units, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeSuperKernelBlock;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeGMA   :  public CNeuronBaseOCL
  {
protected:
   uint              iChanels;
   uint              iUnits;
   uint              iVariables;
   //---
   CLayer            cQKV;
   CNeuronBaseOCL    cQ;
   CNeuronBaseOCL    cK;
   CNeuronTransposeVRCOCL  cKT;
   CNeuronBaseOCL    cV;
   CLayer            cScore;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeGMA(void) {};
                    ~CNeuronSpikeGMA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint chanels, uint units, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeGMA;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronETROF   :  public   CNeuronAddToStack
  {
protected:
   CNeuronPSSE                   cPSSE;
   CNeuronBaseOCL                cRefFrame;
   CNeuronTransposeOCL           cRefFrameT;
   CNeuronBaseOCL                cForwardBackward;
   CNeuronBaseOCL                cCorrelation;
   CLayer                        cMixFusion;
   CNeuronSpikeSuperKernelBlock  cContext;
   CLayer                        cMotionEncoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;


public:
                     CNeuronETROF(void) {};
                    ~CNeuronETROF(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension_in, uint dimension_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronETROF;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPSSEFlow   :  public CNeuronSpikeSuperKernelBlock
  {
protected:
   CNeuronETROF                  cETROF;
   CNeuronSpikeSuperKernelBlock  cLocal;
   CLayer                        cGlobal;
   CLayer                        cUpdateFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPSSEFlow(void) {};
                    ~CNeuronPSSEFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint forecast, uint dimension, uint hidden_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronPSSEFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeUSR   :  public CNeuronSpikeADM
  {
protected:
   CNeuronCreateICEFlow    cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSpikeUSR(void) {};
                    ~CNeuronSpikeUSR(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint stack_size, uint dimension,
                          uint patch_dimension, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeUSR;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronInitialFlow   :  public CNeuronBaseOCL
  {
protected:
   CParams           cCandidates;
   CLayer            cScores;
   CNeuronAddToStack cStack;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronInitialFlow(void) {};
                    ~CNeuronInitialFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units, uint window_out,
                          uint candidates, uint stack_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronInitialFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSpikeSMR   :  public CNeuronMSRes
  {
protected:
   CNeuronSpikeSuperKernelBlock  cUpsampleFlow;
   CNeuronBaseOCL                cConcat;
   CNeuronSpikeConvGRU2D         cConvGRU;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronSpikeSMR(void) {};
                    ~CNeuronSpikeSMR(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint window, uint second_window, uint window_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronSpikeSMR;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   virtual CBufferFloat* GetHidden() const { return cConvGRU.getOutput(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronEVAFlow :  public   CNeuronSpikeSMR
  {
protected:
   CLayer               cPrepare;
   CLayer               cEncoder;
   CNeuronInitialFlow   cInitFlow;
   CLayer               cDecoder;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override
     { return        feedForward(NeuronOCL); }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override
     { return        updateInputWeights(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(NeuronOCL); }

public:
                     CNeuronEVAFlow(void) {};
                    ~CNeuronEVAFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint forecast, uint dimension, uint hidden_dimension,
                          uint layers, uint candidates,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronEVAFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronResFlowHTR :  public   CNeuronMSRes
  {
protected:
   CLayer                  cContextProjection;
   CLayer                  cMFProjection;
   CNeuronBaseOCL          cContextVsMF;
   CNeuronSpikeConvGRU2D   cGRU;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronResFlowHTR(void) {};
                    ~CNeuronResFlowHTR(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint units, uint mf_dimension, uint context_size,
                          uint inside_dimension, uint chanels_out,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   override const   {  return defNeuronResFlowHTR;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronResFlow :  public CNeuronSpikeSuperKernelBlock
  {
protected:
   CNeuronPSSE          cPrepare;
   CNeuronEVAFlow       cLTR;
   CLayer               cContext;
   CLayer               cMFE;
   CNeuronBaseOCL       cUpsampleLTR;
   CNeuronBaseOCL       cMFEvsUpLTR;
   CNeuronResFlowHTR    cHTR;
   CNeuronBaseOCL       cUpdatedFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronResFlow(void) {};
                    ~CNeuronResFlow(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint hidden_dimension,
                          uint ltr_forecast, uint htr_points_to_segment,
                          uint ltr_layers, uint ltr_candidates,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronResFlow;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConfGateTailAwareMoE   :  public CNeuronBaseOCL
  {
protected:
   CLayer            cConfidenceGating;
   CLayer            cScore;
   CParams           cTailAwareMoE;
   CLayer            cCGScore;
   CNeuronAddToStack cCGStack;
   CNeuronAddToStack cScoreStack;
   CNeuronBaseOCL    cDensity;
   //---
   virtual bool      MixExpertsPredict(void);
   virtual bool      MixExpertsPredictGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronConfGateTailAwareMoE(void) {};
                    ~CNeuronConfGateTailAwareMoE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units,
                          uint forecast, uint experts,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronConfGateTailAwareMoE;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronResFlowLattice :  public CNeuronResFlow
  {
protected:
   CNeuronConfGateTailAwareMoE   cTailAwareMoE;
   CNeuronBaseOCL                cConcat;
   CNeuronSpikeSuperKernelBlock  cAdaptFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronResFlowLattice(void) {};
                    ~CNeuronResFlowLattice(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint hidden_dimension,
                          uint ltr_forecast, uint htr_points_to_segment,
                          uint ltr_layers, uint ltr_candidates,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronResFlowLattice;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CFieldAwareParams    :  public CNeuronBaseOCL
  {
protected:
   uint              iFields;
   CParams           cCandidates;
   CParams           cFieldEmbeddings;
   CLayer            cScore;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override          { return FeedForward(); }

   ///\ingroup neuron_base_opt
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override   { return UpdateInputWeights(); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override   { return true; }

public:
                     CFieldAwareParams(void) {};
                    ~CFieldAwareParams(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint params_to_field, uint fields, uint embed_size,
                          uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      FeedForward(void);
   virtual bool      UpdateInputWeights(void);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defFieldAwareParams; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetFields(void) const { return iFields; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFieldAwareConv   :  public CNeuronBaseOCL
  {
protected:
   CFieldAwareParams    cParams;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronFieldAwareConv(void) {};
                    ~CNeuronFieldAwareConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint window_out, uint fields,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronFieldAwareConv; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cParams.Neurons() / Neurons(); }
   virtual uint      GetFields(void) const { return cParams.GetFields(); }
   virtual uint      GetFilters(void) const { return Neurons() / GetFields(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHFAT :  public CNeuronMSRes
  {
protected:
   uint                    iHeads;
   //---
   CNeuronFieldAwareConv   cQ_Embedding;
   CNeuronFieldAwareConv   cKV_Embedding;
   CParams                 cScale;
   int                     ibScoreIndex;
   CNeuronBaseOCL          cMHAttentionOut;
   CNeuronSpikeConvBlock   cW0;
   CNeuronBaseOCL          cAttentionOut;
   //---
   virtual bool            Attention(void);
   virtual bool            AttentionGradients(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override ;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMHFAT(void) {};
                    ~CNeuronMHFAT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint fields,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMHFAT;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHCrossFAT :  public CNeuronMHFAT
  {
protected:
   CNeuronBaseOCL    cContext;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   //---
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;

public:
                     CNeuronMHCrossFAT(void) {};
                    ~CNeuronMHCrossFAT(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_key, uint heads, uint fields,
                          uint window_cross, uint fields_cross,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defNeuronMHCrossFAT;   }
   //--- methods for working with files
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFieldPatternEmbedding    :  public CNeuronBaseOCL
  {
protected:
   uint              iFields;
   uint              iWindow;
   CParams           cCandidates;
   CLayer            cScore;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronFieldPatternEmbedding(void) {};
                    ~CNeuronFieldPatternEmbedding(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint fields, uint embed_size,
                          uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronFieldPatternEmbedding; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetFields(void) const { return iFields; }
   virtual CBufferFloat   *getWeights(void) override { return (!!cScore[-2] ? ((CNeuronConvOCL*)cScore[-2]).GetWeightsConv() : NULL); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronAutoToken  :  public CNeuronBaseOCL
  {
protected:
   uint              iEmbedingSize;
   uint              iTopK;
   uint              iUnits;
   uint              iGroups;
   //---
   CLayer            cDNN;
   CLayer            cW;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronAutoToken(void) {};
                    ~CNeuronAutoToken(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint embed_size,
                          uint units_in, uint groups,
                          uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronAutoToken; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMultiMixAttention   :  public CNeuronBaseOCL
  {
protected:
   CFieldAwareParams    cParams;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMultiMixAttention(void) {};
                    ~CNeuronMultiMixAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint heads,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronMultiMixAttention; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual uint      GetHeads(void) { return cParams.GetFields(); }
   virtual uint      GetUnits(void) { return uint(MathSqrt(cParams.Neurons() / GetHeads())); }
   virtual uint      GetDimension(void) { return Neurons() / GetUnits(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSparseMoEConv  :  public CNeuronBaseOCL
  {
protected:
   CNeuronFieldPatternEmbedding  cParams;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSparseMoEConv(void) {};
                    ~CNeuronSparseMoEConv(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint filters, uint units,
                          uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSparseMoEConv; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cParams.Neurons() / Neurons(); }
   virtual uint      GetFields(void) const { return cParams.GetFields(); }
   virtual uint      GetFilters(void) const { return Neurons() / GetFields(); }
   virtual CBufferFloat   *getWeights(void) override { return cParams.getWeights(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMTmixAttBlock :  public CNeuronBaseOCL
  {
protected:
   CNeuronMultiMixAttention   cAttention;
   CLayer                     cSharedExpert;
   CLayer                     cFieldMoE;
   CLayer                     cSceneMoE;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMTmixAttBlock(void) {};
                    ~CNeuronMTmixAttBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint heads,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronMTmixAttBlock; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CCrossMHFlashAttention   :  public CNeuronBaseOCL
  {
protected:
   uint              iQUnits;
   uint              iKVUnits;
   uint              iHeads;
   uint              iDimension;
   bool              bMask;
   CBufferFloat      cLogSumExp;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override  { return true; }

public:
                     CCrossMHFlashAttention(void) {};
                    ~CCrossMHFlashAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint heads, bool mask_future,
                          uint q_units, uint kv_units,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void)   const   {  return defCrossMHFlashAttention;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMixFeedForward :  public CNeuronBaseOCL
  {
protected:
   CLayer                     cSharedExpert;
   CLayer                     cFieldMoE;
   CLayer                     cSceneMoE;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMixFeedForward(void) {};
                    ~CNeuronMixFeedForward(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronMixFeedForward; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {};
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual CBufferFloat   *getWeights(void) override { return (!!cSceneMoE[-1] ? cSceneMoE[-1].getWeights() : NULL); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronOneTrans   :  public CNeuronMixFeedForward
  {
protected:
   CLayer               cPrepare;
   CNeuronBaseOCL       cSequenceLast;
   CNeuronSparseMoEConv cKVSequenceLast;
   CNeuronAddToStack    cStackSequence;
   CNeuronAddToStack    cStackKVSequence;
   CLayer               cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronOneTrans(void) {};
                    ~CNeuronOneTrans(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint units_out,
                          uint heads, uint stack_size, uint layers,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronOneTrans; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMHTHCrossAttention :  public CNeuronMTmixAttBlock
  {
protected:
   uint              iQUnits;
   uint              iXUnits;
   uint              iHeads;
   uint              iXDimension;
   bool              bMask;
   //---
   CBufferFloat      cLogSumExp;
   CLayer            cPrepareQ;
   CLayer            cW0;
   //---
   virtual bool      AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context);
   virtual bool      AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
         CBufferFloat *SecondGradient,
         ENUM_ACTIVATION SecondActivation = None);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;

public:
                     CNeuronMHTHCrossAttention(void) {};
                    ~CNeuronMHTHCrossAttention(void) {};
   //---
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_q, uint units_q, uint heads,
                          uint dimension_x, uint unit_x,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronMHTHCrossAttention; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSTCA :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer               cPrepare;
   CNeuronBaseOCL       cLastSequence;
   CNeuronBaseOCL       cLastNonSequence;
   CNeuronAddToStack    cStackSequence;
   CFieldAwareParams    cScenarios;
   CLayer               cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSTCA(void) {};
                    ~CNeuronSTCA(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint units_out,
                          uint heads, uint scenarios, uint stack_size, uint layers,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSTCA; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronWeightGenerator :  public CNeuronConvOCL
  {
protected:
   CNeuronConvOCL    cInputProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronWeightGenerator(void) {};
                    ~CNeuronWeightGenerator(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint scenarios,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronWeightGenerator; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPCGR :  public CNeuronTransposeRCDOCL
  {
protected:
   uint                    iWindowIn;
   uint                    iInsideDimension;
   uint                    iWindowOut;
   uint                    iQuerys;
   uint                    iScenarios;
   //---
   CNeuronFieldAwareConv   cShared;
   CFieldAwareParams       cScenarios;
   CNeuronBaseOCL          cQueryVsScenarios;
   CNeuronWeightGenerator  cWQscenarios;
   CNeuronBaseOCL          cGeneratedQuery;
   CNeuronTransposeRCDOCL  cQScInToScQIn;
   CFieldAwareParams       cWKshared;
   CNeuronWeightGenerator  cWKscenarios;
   CNeuronTransposeOCL     cWKscT;
   CNeuronBaseOCL          cWKT;
   CNeuronTransposeOCL     cWK;
   CNeuronBaseOCL          cQueryMod;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPCGR(void) {};
                    ~CNeuronPCGR(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint querys, uint scenarios,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronPCGR; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronADS :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer               cPrepare;
   CNeuronBaseOCL       cLastSequence;
   CNeuronBaseOCL       cLastNonSequence;
   CNeuronAddToStack    cStackSequence;
   CNeuronPCGR          cQuerys;
   CLayer               cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronADS(void) {};
                    ~CNeuronADS(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint units_out,
                          uint heads, uint scenarios, uint stack_size, uint layers,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronADS; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPerTokenFFN   :  public CNeuronFieldAwareConv
  {
protected:
   CNeuronFieldAwareConv   cProj;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPerTokenFFN(void) {};
                    ~CNeuronPerTokenFFN(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronPerTokenFFN; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual uint      GetWindow(void) const override { return cProj.GetWindow(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronScenariosToken :  public CNeuronFieldPatternEmbedding
  {
protected:
   CNeuron2DSSMOCL   cScenariosGenerator;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronScenariosToken(void) {};
                    ~CNeuronScenariosToken(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint fields, uint window_out, uint scenarios,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronScenariosToken; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUnifiedTokenizer :  public CNeuronFieldPatternEmbedding
  {
protected:
   CNeuronBatchNormOCL        cNorm;
   CNeuronMultiWindowsConvOCL cProj;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronUnifiedTokenizer(void) {};
                    ~CNeuronUnifiedTokenizer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronUnifiedTokenizer; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDomainAwareAttention :  public CNeuronPerTokenFFN
  {
protected:
   uint              iQUnits;
   uint              iXUnits;
   uint              iHeads;
   uint              iXDimension;
   //---
   CBufferFloat      cLogSumExp;
   CLayer            cPrepareQ;
   CLayer            cW0;
   //---
   virtual bool      AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context);
   virtual bool      AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
         CBufferFloat *SecondGradient,
         ENUM_ACTIVATION SecondActivation = None);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;

public:
                     CNeuronDomainAwareAttention(void) {};
                    ~CNeuronDomainAwareAttention(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_q, uint units_q, uint heads,
                          uint dimension_x, uint unit_x,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronDomainAwareAttention; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronFeatureSelfIteration   :  public CNeuronDomainAwareAttention
  {
protected:
   CNeuronAddToStack          cStack;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override
     { return        feedForward(NeuronOCL); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override
     { return        calcInputGradients(prevLayer); }
   virtual bool      calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override
     { return        updateInputWeights(NeuronOCL); }

public:
                     CNeuronFeatureSelfIteration(void) {};
                    ~CNeuronFeatureSelfIteration(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_q, uint units_q, uint heads, uint stack_size,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronFeatureSelfIteration; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual CNeuronAddToStack*   GetStack(void) { return cStack.AsObject(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDomainFused  :  public CNeuronPerTokenFFN
  {
protected:
   CNeuronBaseOCL       cTasks;
   CNeuronBaseOCL       cScenarios;
   CLayer               cSelector;
   CNeuronBaseOCL       cScenariosToTask;
   CNeuronBaseOCL       cConcat;
   //---
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronDomainFused(void) {};
                    ~CNeuronDomainFused(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint tasks, uint scenarios,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronDomainFused; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMDL :  public CNeuronPerTokenFFN
  {
protected:
   CNeuronUnifiedTokenizer    cPrepare;
   CNeuronBaseOCL             cLastNonSequence;
   CLayer                     cFeatures;
   CNeuronScenariosToken      cScenarios;
   CLayer                     cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronMDL(void) {};
                    ~CNeuronMDL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads,
                          uint scenarios, uint stack_size, uint layers,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronMDL; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUniMixerTokenizer   :  public CNeuronFieldAwareConv
  {
protected:
   CNeuronUnifiedTokenizer cTokenizer;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronUniMixerTokenizer(void) {};
                    ~CNeuronUniMixerTokenizer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units, uint embed_size,
                          uint blocks, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronUnifiedTokenizer; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPerTokenSwiGLU   :  public CNeuronPerTokenFFN
  {
protected:
   CNeuronFieldAwareConv   cGate;
   CNeuronBaseOCL          cAfterGate;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronPerTokenSwiGLU(void) {};
                    ~CNeuronPerTokenSwiGLU(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronPerTokenSwiGLU; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUniMixer  :  public CNeuronPerTokenSwiGLU
  {
protected:
   uint              iBlocks;
   //---
   CFieldAwareParams cWb;
   CNeuronBaseOCL    cLocalMixing;
   CParams           cG[2];
   CNeuronBaseOCL    cGlobalMixing[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronUniMixer(void) {};
                    ~CNeuronUniMixer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units, uint blocks,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronUniMixer; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSiameseNorm   : public CNeuronBaseOCL
  {
protected:
   CNeuronBaseOCL       cNormYIn;
   CNeuronBaseOCL       cSumXYIn;
   CNeuronUniMixer      cMixer;
   CNeuronBaseOCL       cYOut;
   //---
   virtual bool         feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool         feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;
   virtual bool         calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput,
                                           CBufferFloat *SecondGradient,
                                           ENUM_ACTIVATION SecondActivation = None) override;
   virtual bool         calcInputGradients(CNeuronBaseOCL *prevLayer) override;
   virtual bool         updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool         updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override;

public:
                     CNeuronSiameseNorm(void) {};
                    ~CNeuronSiameseNorm(void) {};
   //---
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units, uint blocks,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSiameseNorm; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual CNeuronBaseOCL*  GetYOut(void) { return cYOut.AsObject(); }
   //---
   virtual uint      GetWindow(void) const { return cMixer.GetWindow(); }
   virtual uint      GetFields(void) const { return cMixer.GetFields(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUniMixerBlock :  public CNeuronPerTokenSwiGLU
  {
protected:
   CNeuronUniMixerTokenizer   cPrepare;
   CNeuronBaseOCL             cLastNonSequence;
   CLayer                     cFeatures;
   CNeuronScenariosToken      cScenarios;
   CLayer                     cFlow;
   CLayer                     cOutputProj;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronUniMixerBlock(void) {};
                    ~CNeuronUniMixerBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint blocks,
                          uint scenarios, uint stack_size, uint layers,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronUniMixerBlock; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronINFNetScenarios  :  public CNeuronBaseOCL
  {
protected:
   CLayer            cSequence;
   CLayer            cContext;
   CLayer            cGlobal;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronINFNetScenarios(void) {};
                    ~CNeuronINFNetScenarios(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window_in, uint &fields[], uint window_out,
                          uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronINFNetScenarios; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override {  activation = None; }

   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronINFNetHUBSequence  :   public CNeuronSpikeConvBlock
  {
protected:
   CNeuronSpikeConvBlock   cProjection;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronINFNetHUBSequence(void) {};
                    ~CNeuronINFNetHUBSequence(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units_count, uint hub_size, uint variables,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronINFNetHUBSequence;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   virtual uint      GetWindow(void) const { return cProjection.GetWindow();   }
   virtual uint      GetUnits(void) const { return cProjection.GetUnits(); }
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronINFNetBlock :  public CNeuronPerTokenSwiGLU
  {
protected:
   uint                       iDimension;
   uint                       iSequenceUnits;
   uint                       iSequenceVar;
   uint                       iContextUnits;
   uint                       iScenariosUnits;
   CNeuronUniMixerTokenizer   cPrepare;
   CLayer                     cSequence;
   CLayer                     cContext;
   CLayer                     cScenarios;
   CLayer                     cHUBs;
   //---
   virtual bool      INFNetBGU(CNeuronBaseOCL* scale_and_shift, CNeuronBaseOCL* sequence_in, CNeuronBaseOCL* sequence_out,
                               CNeuronBaseOCL* context_in, CNeuronBaseOCL* context_out,
                               CNeuronBaseOCL* scenarios_in, CNeuronBaseOCL* scenarios_out);
   virtual bool      INFNetBGUGrad(CNeuronBaseOCL* scale_and_shift, CNeuronBaseOCL* sequence_in, CNeuronBaseOCL* sequence_out,
                                   CNeuronBaseOCL* context_in, CNeuronBaseOCL* context_out,
                                   CNeuronBaseOCL* scenarios_in, CNeuronBaseOCL* scenarios_out);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronINFNetBlock(void) {};
                    ~CNeuronINFNetBlock(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads, uint stack_size,
                          uint layers, uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronINFNetBlock; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronSequenceTokenizer :  public CNeuronBaseOCL
  {
protected:
   CNeuronBatchNormOCL        cNorm;
   CNeuronSpikeConvBlock      cProj[2];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronSequenceTokenizer(void) {};
                    ~CNeuronSequenceTokenizer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units,  uint embed_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronSequenceTokenizer; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual CNeuronBatchNormOCL*    GetNorm(void) { return cNorm.AsObject(); }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDifferenceTokenizer :  public CNeuronBaseOCL
  {
protected:
   CLayer            cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronDifferenceTokenizer(void) {};
                    ~CNeuronDifferenceTokenizer(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint inpSize,
                          uint tokenSize, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronDifferenceTokenizer; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronReGENTADBackbone :  public CNeuronBaseOCL
  {
protected:
   CLayer                     cSeqToken;
   CLayer                     cTransformer;
   CLayer                     cRecurrent;
   CNeuronDifferenceTokenizer cDifference;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronReGENTADBackbone(void) {};
                    ~CNeuronReGENTADBackbone(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads, uint stack_size,
                          uint layers, uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronReGENTADBackbone; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   virtual CNeuronBatchNormOCL*    GetNorm(void);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronReGENTADHead  :  public CNeuronRevINDenormOCL
  {
protected:
   CLayer            cFlow;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronReGENTADHead(void) {};
                    ~CNeuronReGENTADHead(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension_in, uint units_in, uint dimension_out,
                          uint units_out, CNeuronBatchNormOCL* norm,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronReGENTADHead; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   virtual bool      SetNorm(CNeuronBatchNormOCL* norm);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronReGENTAD   :  public CNeuronBaseOCL
  {
protected:
   CNeuronReGENTADBackbone*   cBackbone;
   CLayer                     cHeads[3];
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual CNeuronReGENTADBackbone* CreateBackbone(void) { return new CNeuronReGENTADBackbone(); }

public:
                     CNeuronReGENTAD(void) : cBackbone(NULL) {};
                    ~CNeuronReGENTAD(void) { DeleteObj(cBackbone); }
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads, uint stack_size,
                          uint layers, uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronReGENTAD; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) override { };
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronDictionaryCrossAtt   :  public CNeuronSpikeConvBlock
  {
protected:
   uint                    iDictionarySize;
   uint                    iHeads;
   uint                    iHiddenDimension;
   int                     bufLogSumExp;
   CParams                 cDictionaryKeys;
   CParams                 cDictionaryValues;
   CParams                 cPrototypes;
   CParams                 cLevels;
   CNeuronSpikeConvBlock   cQuerys;
   CNeuronBaseOCL          cAttentionOut;
   CNeuronBaseOCL          cSimilarity;
   CNeuronSpikeConvBlock   cW0;
   CNeuronSpikeConv        cProjSimilarity;
   CNeuronSpikeConvBlock   cFeedForward0;
   //---
   virtual bool      MHCrossAttvsSim(void);
   virtual bool      MHCrossAttvsSimGrad(void);
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronDictionaryCrossAtt(void)   : bufLogSumExp(INVALID_HANDLE) {};
                    ~CNeuronDictionaryCrossAtt(void) { if(bufLogSumExp >= 0 && !!OpenCL) OpenCL.BufferFree(bufLogSumExp); }
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units, uint heads, uint Dictionary_size,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronDictionaryCrossAtt;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual uint      GetWindow(void) const { return cQuerys.GetWindow();   }
   //---
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
   //---
   virtual CNeuronBaseOCL* GetSimilarity(void)  { return cSimilarity.AsObject(); }
  };
//+------------------------------------------------------------------+
//| Immutable scenario/action centroid memory for RAG inference.    |
//+------------------------------------------------------------------+
union URAGFloatBits
  {
   float             value;
   uint              bits;
  };

struct SRAGTerminalOutcome
  {
   float             relative_return;
   float             max_drawdown;
   float             costs;
   float             risk_usage;
   float             duration;
   float             reward;
  };

struct SRAGActionCentroid
  {
   bool              used;
   uint              scenario_id;
   ulong             hit_count;
   float             mean_reward;
   float             centroid[];
   SRAGTerminalOutcome mean_terminal;
  };

struct SRAGScenarioCentroid
  {
   bool              used;
   uint              scenario_id;
   ulong             hit_count;
   float             centroid[];
   SRAGActionCentroid actions[];
  };

// Completed facts are staged on the host until the explicit publication
// boundary.  They never expose the internal centroid snapshot to an Expert.
struct SRAGCompletedRecord
  {
   float             scenario_embedding[];
   float             action[];
   SRAGTerminalOutcome terminal;
  };

// A pending fact is causal host state only.  The Expert supplies execution
// identities and the terminal outcome; the library never queries trade state.
struct SRAGPendingEvent
  {
   bool              confirmed;
   ulong             deal;
   ulong             position;
   float             action[];
  };

struct SRAGPendingFact
  {
   ulong             id;
   float             scenario_embedding[];
   SRAGPendingEvent  events[];
  };

// Complete immutable device image.  CNeuronRAGMemory builds a candidate first
// and replaces the active pointer only after every buffer is valid.
class CRAGInferenceSet : public CObject
  {
public:
   COpenCLMy         *opencl;
   CBufferFloat      *centroids;
   CBufferFloat      *used;
   CBufferFloat      *scores;
   CBufferFloat      *partial;
   CBufferFloat      *merge;
   CBufferFloat      *top;
   CBufferFloat      *actions;
   CBufferFloat      *action_used;
   CBufferFloat      *action_mean_reward;
   CBufferFloat      *action_hit_count;
   CBufferFloat      *absolute_relevance;
   CBufferFloat      *tokens;
   ulong              fingerprint;

                     CRAGInferenceSet(void);
                    ~CRAGInferenceSet(void);
  };

class CNeuronRAGMemory : public CNeuronBaseOCL
  {
protected:
   uint              iScenarioEmbedding;
   uint              iActionDimension;
   uint              iTokenWidth;
   uint              iScenarioCapacity;
   uint              iActionCapacity;
   uint              iTopK;
   uint              iOnlineMemorySize;
   uint              iScenarioCount;
   uint              iFormatVersion;
   SRAGScenarioCentroid scenarios[];
   SRAGCompletedRecord pending_records[];
   SRAGPendingFact   pending_facts[];
   CRAGInferenceSet  *m_inference;
   // Reused by bounded-memory publication.  BufferInit releases a device
   // allocation only when its shape changes; same-shape rounds become uploads.
   CBufferFloat      cNearestCentroids;
   CBufferFloat      cNearestUsed;
   CBufferFloat      cNearestResult;
   ulong              iPublicationGeneration;
   ulong              iNextPendingId;

   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return true; }
   bool              Configure(CLayerDescription &description);
   bool              InitializeAction(SRAGActionCentroid &action,const uint scenario_id) const;
   bool              InitializeScenario(SRAGScenarioCentroid &scenario,const uint scenario_id) const;
   void              ClearTerminal(SRAGTerminalOutcome &terminal) const;
   bool              CopyAction(const SRAGActionCentroid &source,SRAGActionCentroid &destination) const;
   bool              CopyScenario(const SRAGScenarioCentroid &source,SRAGScenarioCentroid &destination) const;
   bool              CopyScenarioFlexible(const SRAGScenarioCentroid &source,
                                            SRAGScenarioCentroid &destination) const;
   bool              AppendActionCandidate(const uint scenario_id,SRAGScenarioCentroid &scenario,
                                             const SRAGActionCentroid &incoming) const;
   bool              CompactActionCandidates(SRAGScenarioCentroid &scenario) const;
   bool              ConsolidateActionCandidates(SRAGScenarioCentroid &candidate_snapshot[]);
   bool              NormalizeL2(const float &source[],float &destination[]) const;
   float             SquaredDistance(const float &left[],const float &right[]) const;
   bool              ValidAction(const float &action[]) const;
   bool              ValidTerminal(const SRAGTerminalOutcome &terminal) const;
   void              MergeTerminal(SRAGTerminalOutcome &destination,const ulong destination_hits,
                                     const SRAGTerminalOutcome &source,const ulong source_hits) const;
   int               FirstFreeScenario() const;
   int               FirstFreeAction(const uint scenario_id) const;
   bool              RebindScenarioActions(SRAGScenarioCentroid &scenario,const uint destination_scenario_id) const;
   bool              MergeActionCandidates(const uint scenario_id,SRAGActionCentroid &destination,
                                             const SRAGActionCentroid &source) const;
   bool              FindNearestActionPair(const uint scenario_id,const SRAGActionCentroid &action_candidates[],
                                            int &destination,int &source);
   bool              PairCountSafe(const uint candidate_count,int &pair_count) const;
   bool              FindNearestPairsGPU(const float &candidate_centroids[],const float &candidate_used[],
                                          const uint candidate_count,const uint dimension,const uint batch_count,
                                          int &destinations[],int &sources[]);
   bool              FindNearestPairGPU(const float &candidate_centroids[],const float &candidate_used[],
                                         const uint candidate_count,const uint dimension,const uint batch_count,
                                          int &destination,int &source);
   bool              MergeScenarioCandidates(SRAGScenarioCentroid &destination,
                                               const SRAGScenarioCentroid &source);
   bool              FindNearestScenarioPair(const SRAGScenarioCentroid &scenario_candidates[],
                                               int &destination,int &source);
   bool              StageScenarioCandidates(const uint source_scenario_id,const float &scenario_embedding[],
                                               const float &action[],const SRAGTerminalOutcome &terminal,
                                               SRAGScenarioCentroid &scenario_candidates[]);
   bool              ValidateSnapshot(const SRAGScenarioCentroid &snapshot[],const uint scenario_count,
                                        const uint scenario_embedding,const uint action_dimension,
                                        const uint scenario_capacity,const uint action_capacity) const;
   bool              CommitSnapshot(const SRAGScenarioCentroid &snapshot[],const uint scenario_count);
   bool              CopyCompletedRecord(const SRAGCompletedRecord &source,SRAGCompletedRecord &destination) const;
   bool              CopyPendingEvent(const SRAGPendingEvent &source,SRAGPendingEvent &destination) const;
   bool              CopyPendingFact(const SRAGPendingFact &source,SRAGPendingFact &destination) const;
   int               FindPendingFact(const ulong pending_id) const;
   bool              BuildRemainingPendingFacts(const int excluded,SRAGPendingFact &remaining[]) const;
   bool              CopySnapshot(const SRAGScenarioCentroid &source[],SRAGScenarioCentroid &destination[]) const;
   bool              ApplyCompletedRecordToSnapshot(SRAGScenarioCentroid &candidate_snapshot[],
                                                      uint &candidate_count,const uint source_scenario_id,
                                                      const float &scenario_embedding[],const float &action[],
                                                      const SRAGTerminalOutcome &terminal);
   bool              BuildInferenceSet(const SRAGScenarioCentroid &snapshot[],const ulong fingerprint,
                                        CRAGInferenceSet *&result) const;
   ulong             HashValue(const ulong hash,const ulong value) const;
   ulong             HashFloatBits(const ulong hash,const float value) const;
   ulong             SnapshotFingerprint(const SRAGScenarioCentroid &snapshot[],const uint scenario_count) const;
   bool              WritePublishedSnapshot(const int file_handle,const ulong fingerprint) const;
   bool              ReadPublishedSnapshot(const int file_handle,SRAGScenarioCentroid &snapshot[],
                                             uint &scenario_count,ulong &fingerprint) const;
   bool              HasBytes(const int file_handle,const ulong bytes) const;
   bool              WriteUInt(const int file_handle,const uint value) const;
   bool              WriteULong(const int file_handle,const ulong value) const;
   bool              WriteFloat(const int file_handle,const float value) const;
   bool              ReadUInt(const int file_handle,uint &value) const;
   bool              ReadULong(const int file_handle,ulong &value) const;
   bool              ReadFloat(const int file_handle,float &value) const;
   bool              SnapshotPayloadBytes(const uint scenario_embedding,const uint action_dimension,
                                           const uint scenario_capacity,const uint action_capacity,
                                           ulong &bytes) const;

public:
                     CNeuronRAGMemory(void);
                     CNeuronRAGMemory(CLayerDescription &description);
                    ~CNeuronRAGMemory(void);
   bool              Init(uint numOutputs,uint myIndex,COpenCLMy *open_cl,
                           CLayerDescription &description);
   bool              SetOnlineMemorySize(const uint value);
   uint              OnlineMemorySize(void) const { return iOnlineMemorySize; }
   uint              ScenarioCount(void) const { return iScenarioCount; }
   uint              ScenarioCapacity(void) const { return iScenarioCapacity; }
   uint              ActionCapacity(void) const { return iActionCapacity; }
   uint              TopK(void) const { return iTopK; }
   uint              TokenCount(void) const { return iTopK*iActionCapacity; }
   uint              TokenWidth(void) const { return iTokenWidth; }
   COpenCLMy        *GetOpenCL(void) const { return OpenCL; }
   // Canonical six-component execution descriptor normalisation.  The
   // caller supplies every domain scale; this layer never reads terminal,
   // account, symbol or trade state.
   bool              NormalizeActionDescriptor(const float &source[],const double equity,
                                                const double risk_budget,const double point_cost,
                                                const double point,const double volatility,
                                                const double volatility_range,
                                                const double maximum_take_profit,
                                                const double maximum_stop_loss,
                                                float &destination[]) const;
   // Converts raw completed-episode facts into the bounded memory contract.
   // Duration and its scale are numeric facts supplied by the caller.
   void              NormalizeTerminalOutcome(const double reward,const double max_drawdown,
                                              const double costs,const double risk_usage,
                                              const double duration,const double starting_equity,
                                              const double risk_budget,const double duration_scale,
                                              SRAGTerminalOutcome &result) const;
   bool              AddCompletedScenario(const uint source_scenario_id,const float &scenario_embedding[],
                                            const float &action[],const SRAGTerminalOutcome &terminal);
   bool              BindInferenceSnapshot(const string path);
   bool              BindNullInference(void);
   bool              Retrieve(CNet *scenario_model,const uint layer); ///< Retrieve from a model-owned device output without exposing its buffer to an Expert.
   bool              Retrieve(CBufferFloat *scenario_embedding);
   CBufferFloat     *Tokens(void) const { return (m_inference==NULL ? NULL : m_inference.tokens); }
   CBufferFloat     *AbsoluteRelevance(void) const { return (m_inference==NULL ? NULL : m_inference.absolute_relevance); }
   bool              AddCompletedRecord(const float &scenario_embedding[],const float &action[],
                                          const SRAGTerminalOutcome &terminal);
   uint              PendingRecordCount(void) const { return (uint)ArraySize(pending_records); }
   ulong             OpenPending(const float &scenario_embedding[],const float &action[]);
   bool              ConfirmPendingExecution(const ulong pending_id,const ulong deal,const ulong position);
   bool              CorrectPending(const ulong pending_id,const float &action[]);
   bool              ConfirmPendingCorrection(const ulong pending_id,const ulong position);
   bool              CancelPendingCorrection(const ulong pending_id);
   bool              ClosePending(const ulong pending_id,const SRAGTerminalOutcome &terminal);
   bool              CancelPending(const ulong pending_id);
   bool              Publish(const bool episode_closed=false);
   bool              SavePublishedSnapshot(const string path) const;
   bool              LoadPublishedSnapshot(const string path);
   bool              ScenarioCentroid(const uint scenario_id,float &destination[]) const;
   bool              ActionCentroid(const uint scenario_id,const uint action_id,float &destination[]) const;
   ulong             ScenarioHitCount(const uint scenario_id) const;
   ulong             ActionHitCount(const uint scenario_id,const uint action_id) const;
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual CLayerDescription* GetLayerInfo(void) override;
   virtual int       Type(void) const override { return defNeuronRAGMemory; }
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override { CNeuronBaseOCL::TrainMode(false); }
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source,float tau) override { return true; }
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGDformerBackbone :  public CNeuronReGENTADBackbone
  {
protected:
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronGDformerBackbone(void) {};
                    ~CNeuronGDformerBackbone(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads, uint stack_size,
                          uint layers, uint embed_size, uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronGDformerBackbone; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronGDformer   :  public CNeuronReGENTAD
  {
protected:
   virtual CNeuronReGENTADBackbone* CreateBackbone(void) override { return new CNeuronGDformerBackbone(); }

public:
                     CNeuronGDformer(void) {};
                    ~CNeuronGDformer(void) {};
   //---
   virtual int       Type(void) override const  {  return defNeuronGDformer; }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMomADTTM   :  public CNeuronBaseOCL
  {
protected:
   uint                 iCandidates;
   uint                 iDecisions;
   uint                 iDimension;
   uint                 iTopK;
   CNeuronBaseOCL       cDistance;
   CNeuronBaseOCL       cMeanDistance;
   CNeuronSparseSoftMax cScores;
   CBufferFloat         cCandidatesGradient;
   //---
   virtual bool      MeanScores(void);
   virtual bool      MeanScoresGradient(void);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override  { return true; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronMomADTTM(void) : iCandidates(0),
                     iDecisions(0),
                     iDimension(0),
                     iTopK(0) {};
                    ~CNeuronMomADTTM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint latent_dim, uint candidates, uint decisions,
                          uint topK, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMomADTTM; }
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMomADMPI  :  public CNeuronCrossAttention
  {
protected:
   CNeuronAddToStack   cStack;
   CNeuronMomADTTM     cTTM;
   CNeuronCrossAttention cPolicyMarket;
   CNeuronCrossAttention cTTMHistory;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronMomADMPI(void) { cContext = NULL; };
                    ~CNeuronMomADMPI(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint latent_dim, uint candidates, uint market_dim,
                          uint market_units, uint stack_size, uint topK,
                          uint heads, uint key_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronMomADMPI; }
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//| VLADriver sequential market/RAG MPI extension                    |
//+------------------------------------------------------------------+
class CNeuronVLADriverRAGMPI : public CNeuronMomADMPI
  {
protected:
   CBufferFloat      *pRAGTokens;
   CBufferFloat      *pRAGAbsoluteRelevance;
   uint               iRAGTokenCount;
   uint               iRAGTokenWidth;
   //--- Public second input is a frozen ScenarioEmbedding.  These two
   //--- neurons are Actor-owned and expand it into the MPI market K/V tokens.
   uint               iMarketEmbeddingSize;
   CNeuronBaseOCL     cMarketEmbedding;
   CNeuronBaseOCL     cMarketProjection;
   CNeuronBaseOCL     cMarketState;
   //--- One-input convolution supplies a single non-normalized trainable bias.
   //--- Its bias is the Actor-owned null-scenario threshold; the fixed input is zero.
   CNeuronConvOCL     cNullScenarioThreshold;
   CNeuronBaseOCL     cNullThresholdInput;
   CNeuronCrossAttention cPolicyRAG;
   CBufferFloat       cGatedRAGTokens;
   CBufferFloat       cRAGBranchActivity;
   CBufferFloat       cRAGBranchActivityGradient;
   CBufferFloat       cRAGBranchDelta;
   CBufferFloat       cRAGBranchOutput;
   CBufferFloat       cRAGResidualGradient;
   CBufferFloat       cRAGGradient;
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL,
                                        CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronVLADriverRAGMPI(void) : pRAGTokens(NULL), pRAGAbsoluteRelevance(NULL), iRAGTokenCount(0), iRAGTokenWidth(0), iMarketEmbeddingSize(0) {};
                    ~CNeuronVLADriverRAGMPI(void) {};
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint latent_dim, uint candidates, uint market_dim,
                          uint market_units, uint stack_size, uint topK,
                          uint heads, uint key_dimension,
                          uint rag_token_count, uint rag_token_width,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   virtual int       Type(void) override const { return defNeuronVLADriverRAGMPI; }
   bool              SetRAGTokens(CBufferFloat *tokens,CBufferFloat *absolute_relevance,
                                   const uint token_width);
   CBufferFloat      *GetRAGTokens(void) const { return pRAGTokens; }
   uint              RAGTokenCount(void) const { return iRAGTokenCount; }
   uint              RAGTokenWidth(void) const { return iRAGTokenWidth; }
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUncAD_MSD  :  public CNeuronBaseOCL
  {
protected:
   uint              iDimension;
   uint              iUnits;
   uint              iVariables;
   uint              iQuantiles;
   float             fQuantSize;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return (!!NeuronOCL && NeuronOCL.getGradient().Fill(0)); };

public:
                     CNeuronUncAD_MSD(void) {};
                    ~CNeuronUncAD_MSD(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint units, uint variables, uint quantiles,
                          float range, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronUncAD_MSD;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCogDriverData  :  public CNeuronSpikeConvBlock
  {
protected:
   uint              iWindow;
   uint              iVariables;
   uint              iStateStackSize;
   uint              iQuantStackSize;
   uint              iQuantiles;
   float             fRange;
   CNeuronBatchNormOCL cNorm;
   CNeuronAddToStack cStateStack;
   CNeuronUncAD_MSD  cMSD;
   CNeuronConvOCL    cAffine;
   CNeuronAddToStack cQuantStack;
   CNeuronBaseOCL    cPrevQuantStack;
   //---
   virtual bool      AffineStack(void);
   virtual bool      AffineStackGradient(void);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCogDriverData(void) : iWindow(0),
                     iVariables(0),
                     iStateStackSize(0),
                     iQuantStackSize(0),
                     iQuantiles(0),
                     fRange(0) {};
                    ~CNeuronCogDriverData(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint variables, uint state_stack_size,
                          uint quant_stack_size, uint quantiles, float range,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronCogDriverData;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCogDriverRankTCM  :  public CNeuronBaseOCL
  {
protected:
   uint              iInputDim;
   uint              iQueryDim;
   uint              iVariables;
   uint              iHeads;
   uint              iHeadDim;
   uint              iTotalQueryDim;
   uint              iTotalHeads;
   uint              iMemoryLen;
   uint              iMemoryCount;
   bool              bLastQueryValid;
   CNeuronSpikeConvBlock cQueryProjection;
   CNeuronBaseOCL    cGateInput;
   CNeuronBaseOCL    cGateProjection;
   CNeuronBaseOCL    cMemoryContext;
   CNeuronBaseOCL    cSelfContext;
   CBufferFloat      cMemoryQuery;
   CBufferFloat      cMemoryAge;
   CBufferFloat      cMemoryScore;
   CBufferFloat      cMemoryValid;
   CBufferFloat      cMemoryKV;
   CBufferFloat      cMemoryKVGradient;
   CBufferFloat      cSelfKV;
   CBufferFloat      cSelfKVGradient;
   CBufferFloat      cLogSumExp;
   CBufferFloat      cSelfLogSumExp;
   CBufferFloat      cQueryGateGradient;
   CBufferFloat      cGateQueryGradient;
   CBufferFloat      cGateContextGradient;
   //---
   virtual bool      InitStateBuffers(void);
   virtual bool      InsertPendingQuery(void);
   virtual bool      BuildMemoryKV(void);
   virtual bool      FlashAttentionForward(CBufferFloat *query, CBufferFloat *out);
   virtual bool      FlashAttentionBackward(CBufferFloat *query, CBufferFloat *query_gr,
         CBufferFloat *out, CBufferFloat *out_gr);
   virtual bool      BuildSelfKV(CBufferFloat *source);
   virtual bool      SelfAttentionForward(CBufferFloat *query, CBufferFloat *out);
   virtual bool      SelfAttentionBackward(CBufferFloat *query, CBufferFloat *query_gr,
                                           CBufferFloat *out, CBufferFloat *out_gr);
   virtual bool      MeanFlashScores(CBufferFloat *query);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCogDriverRankTCM(void) :  iInputDim(0),
                     iQueryDim(0),
                     iVariables(0),
                     iHeads(0),
                     iHeadDim(0),
                     iTotalQueryDim(0),
                     iTotalHeads(0),
                     iMemoryLen(0),
                     iMemoryCount(0),
                     bLastQueryValid(false) {};
                    ~CNeuronCogDriverRankTCM(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint input_dim, uint query_dim, uint variables, uint memory_len, uint heads,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronCogDriverRankTCM;   }
   virtual uint      MemoryCount(void) const { return iMemoryCount; }
   virtual bool      LastQueryValid(void) const { return bLastQueryValid; }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCogDriverForecastHead  :  public CNeuronMLMHAttentionOCL
  {
protected:
   uint              iVariables;
   CNeuronBaseOCL    cPlanInput;
   CNeuronConvOCL    cContextProjection;
   CNeuronLearnabledPE cQueryPE;
   CLayer            cCrossAttention;
   CLayer            cCrossResidual;
   CLayer            cCrossFF1;
   CLayer            cCrossFF2;
   CLayer            cCrossFFOutput;
   CLayer            cSelfOutput;
   CNeuronBaseOCL    cErrorInput;
   CNeuronConvOCL    cErrorHead;
   CBufferFloat      cContextKV;
   CBufferFloat      cContextKVGradient;
   CBufferFloat      cShiftedConfidence;
   CBufferFloat      cDeltaMask;
   CBufferFloat      cPlanDelta;
   CBufferFloat      cPlanDeltaNorm;
   CBufferFloat      cConfidence;
   CBufferFloat      cErrorTarget;
   //---
   virtual bool      ShiftState(void);
   virtual bool      BuildContextKV(void);
   virtual bool      BuildContextProjectionGradient(void);
   virtual bool      BuildErrorInput(CNeuronBaseOCL *NeuronOCL);
   virtual bool      BuildErrorTarget(void);
   virtual bool      BuildConfidence(void);
   virtual bool      BuildDeltaNorm(void);
   virtual bool      CrossLayerForward(uint layer, CNeuronBaseOCL *plan, CNeuronBaseOCL *context);
   virtual bool      CrossLayerBackward(uint layer, CNeuronBaseOCL *plan, CNeuronBaseOCL *context, CBufferFloat *out_grad);
   virtual bool      CrossLayerUpdateWeights(uint layer, CNeuronBaseOCL *plan, CNeuronBaseOCL *context);
   virtual bool      SelfLayerForward(uint layer, CNeuronBaseOCL *NeuronOCL);
   virtual bool      SelfLayerBackward(uint layer, CNeuronBaseOCL *NeuronOCL, CBufferFloat *out_grad);
   virtual bool      SelfLayerUpdateWeights(uint layer, CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCogDriverForecastHead(void) : iVariables(0) {};
                    ~CNeuronCogDriverForecastHead(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint horizon, uint variables, uint forecast_blocks,
                          uint heads, ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronCogDriverForecastHead;   }
   virtual CBufferFloat *GetExpectedError(void) { return cErrorHead.getOutput(); }
   virtual CBufferFloat *GetConfidence(void) { return GetPointer(cConfidence); }
   virtual CBufferFloat *GetPlanDeltaNorm(void) { return GetPointer(cPlanDeltaNorm); }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronCogDriverForecastToken  :  public CNeuronBaseOCL
  {
protected:
   uint              iDimension;
   uint              iHorizon;
   CNeuronRoPE       cRoPE;
   CBufferFloat      cConfidenceGradient;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; }
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronCogDriverForecastToken(void) : iDimension(0), iHorizon(0) {};
                    ~CNeuronCogDriverForecastToken(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint dimension, uint horizon,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronCogDriverForecastToken;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override { return true; }
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUncAD_MUE  :  public CNeuronSpikeConvBlock
  {
protected:
   CLayer            cMSDFlow;
   CLayer            cUncertainty;
   CNeuronBaseOCL    cConcatenated;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;

public:
                     CNeuronUncAD_MUE(void) {};
                    ~CNeuronUncAD_MUE(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint window_out, uint variables,
                          uint stack_size, uint quantiles, float range,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronUncAD_MUE;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj)   override;
   virtual void      TrainMode(bool flag) override;
   //---
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUncAD_UGP  :  public CNeuronSpikeMHCrossAttention
  {
protected:
   CNeuronMultiMixAttention      cSelfAttention;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronUncAD_UGP(void) {};
                    ~CNeuronUncAD_UGP(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units, uint context_window,
                          uint context_units, uint heads, uint key_dimension,
                          uint candidates, uint topK,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronUncAD_UGP;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUncAD_UGPL :  public CNeuronSpikeMHCrossAttention
  {
protected:
   CParams                     cMode;
   CNeuronSpikeMHCrossAttention cModeFusion;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput,
                                        CBufferFloat *SecondGradient,
                                        ENUM_ACTIVATION SecondActivation = None) override;

public:
                     CNeuronUncAD_UGPL(void) {};
                    ~CNeuronUncAD_UGPL(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint window, uint units, uint context_window,
                          uint context_units, uint mode_units, uint heads,
                          uint key_dimension,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronUncAD_UGPL;  }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronUncAD : public CNeuronSpikeConvBlock
  {
protected:
   uint                         iInputDim;
   uint                         iAgentUnits;
   uint                         iContextUnits;
   uint                         iEmbedSize;
   uint                         iQuantiles;
   uint                         iCandidates;
   uint                         iTopK;
   //---
   CNeuronUniMixerTokenizer     cTokenizer;
   CNeuronBaseOCL               cAgents;
   CNeuronBaseOCL               cContext;
   CNeuronUncAD_MUE             cMUE;
   CNeuronUncAD_UGP             cUGP;
   CNeuronUncAD_UGPL            cUGPL;
   CNeuronSpikeMHCrossAttention cMapInteraction;
   //--- tau heads
   CNeuronConvOCL               cTauAValues;
   CNeuronConvOCL               cTauAScores;
   CNeuronSparseSoftMax         cTauASoftMax;
   CNeuronBaseOCL               cTauAWeighted;
   CNeuronConvOCL               cTauEValues;
   CNeuronConvOCL               cTauEScores;
   CNeuronSparseSoftMax         cTauESoftMax;
   CNeuronBaseOCL               cTauEWeighted;
   //--- selection/fusion
   CNeuronBaseOCL               cFusionInput;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      TauAForward(void);
   virtual bool      TauEForward(void);
   virtual bool      TauAGradients(void);
   virtual bool      TauEGradients(void);

public:
                     CNeuronUncAD(void) : iInputDim(0),
                     iAgentUnits(0),
                     iContextUnits(0),
                     iEmbedSize(0),
                     iQuantiles(0),
                     iCandidates(0),
                     iTopK(0) {};
                    ~CNeuronUncAD(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads,
                          uint stack_size, uint quantiles, uint embed_size,
                          uint candidates, uint topK, uint latent_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch);
   //---
   virtual int       Type(void) override const  {  return defNeuronUncAD;  }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronMomAD : public CNeuronUncAD
  {
protected:
   CNeuronMomADMPI              cMPI;
   bool                         bTauELinked;
   //---
   virtual bool      TauEForward(void) override;
   virtual bool      TauEGradients(void) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   bool              LinkTauEToMPI(void);
   bool              DetachTauEFromMPI(void);

public:
                     CNeuronMomAD(void) : bTauELinked(false) {};
                    ~CNeuronMomAD(void) { DetachTauEFromMPI(); };
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint &dimensions[], uint units_s, uint heads,
                          uint stack_size, uint quantiles, uint embed_size,
                          uint candidates, uint topK, uint latent_count,
                          ENUM_OPTIMIZATION optimization_type, uint batch) override;
   //---
   virtual int       Type(void) override const  {  return defNeuronMomAD;  }
   //--- methods for working with files
   virtual bool      Save(int const file_handle) override;
   virtual bool      Load(int const file_handle) override;
   //---
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CScenarioCodebook : public CObject
  {
protected:
   uint              iScenarios;
   uint              iDimension;
   COpenCLMy         *OpenCL;
   CBufferFloat      cPrototypes;
   CBufferFloat      cEMASums;
   CBufferFloat      cEMACounts;
   CBufferFloat      cUsage;
   CBufferFloat      cInactive;
   CBufferFloat      cInactivityAge;

public:
                     CScenarioCodebook(void) : iScenarios(0), iDimension(0), OpenCL(NULL) {};
                    ~CScenarioCodebook(void) { OpenCL = NULL; };
   //---
   bool              Init(uint scenarios, uint dimension, COpenCLMy *open_cl);
   bool              Save(const int file_handle);
   bool              Load(const int file_handle, COpenCLMy *open_cl);
   bool              ValidateContract(uint scenarios, uint dimension);
   void              SetOpenCL(COpenCLMy *open_cl);
   uint              Scenarios(void) const { return iScenarios; }
   uint              Dimension(void) const { return iDimension; }
   CBufferFloat      *GetPrototypes(void) { return GetPointer(cPrototypes); }
   CBufferFloat      *GetEMASums(void) { return GetPointer(cEMASums); }
   CBufferFloat      *GetEMACounts(void) { return GetPointer(cEMACounts); }
   CBufferFloat      *GetUsage(void) { return GetPointer(cUsage); }
   CBufferFloat      *GetInactive(void) { return GetPointer(cInactive); }
   CBufferFloat      *GetInactivityAge(void) { return GetPointer(cInactivityAge); }
  };
//+------------------------------------------------------------------+
//| Shared scenario-major forecast foundation                        |
//+------------------------------------------------------------------+
class CNeuronScenarioForecast : public CNeuronBaseOCL
  {
protected:
   uint              iVariables;
   uint              iScenarios;
   uint              iTopK;
   uint              iHorizon;
   uint              iDimension;
   uint              iRecoveryAge;
   bool              bLastTargetAccepted;
   bool              bLastControlValid;
   //---
   CScenarioCodebook cCodebook;
   CNeuronBaseOCL    cGeneratorInput;
   CNeuronBaseOCL    cRouterInput;
   CNeuronBaseOCL    cConfidenceInput;
   CNeuronConvOCL    cGenerator;
   CNeuronConvOCL    cRouter;
   CNeuronConvOCL    cConfidenceHead;
   CNeuronSoftMaxOCL cRouterSoftmax;
   CNeuronBaseOCL    cResponsibilityScores;
   CNeuronSparseSoftMax cTopKSelector;
   //---
   CBufferFloat      cConfidence;
   CBufferFloat      cLogConfidence;
   CBufferFloat      cLogProbabilities;
   CBufferFloat      cResponsibilities;
   CBufferFloat      cTarget;
   CBufferFloat      cBatchControl;
   CBufferFloat      cRecovery;
   CBufferFloat      cRecoveryEvents;
   CBufferFloat      cAgeZeroEvents;
   CBufferFloat      cForwardValidity;
   CBufferFloat      cDiagnosticPartials;
   CBufferFloat      cEpochDiagnostics;
   CBufferFloat      cDiagnosticScratch;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL) override;
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL) override;
   //---
   bool              ValidateLaunchConfiguration(COpenCLMy *open_cl, uint scenarios, uint horizon,
         uint latent, uint variables);
   bool              BindResponsibilitiesKernel(CBufferFloat *target_source,
         CBufferFloat *observation_by_scenario);
   bool              UpdateCodebook(CBufferFloat *delta_target,float decay = 0.99f);

public:
                     CNeuronScenarioForecast(void) : iVariables(0), iScenarios(0), iTopK(0), iHorizon(0), iDimension(0), iRecoveryAge(256),
                     bLastTargetAccepted(false), bLastControlValid(false) {};
                    ~CNeuronScenarioForecast(void) {};
   //---
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                          uint scenarios = 21, uint top_k = 3, uint horizon = 12,
                          uint latent = 16, ENUM_OPTIMIZATION optimization_type = ADAM,
                          uint batch = 1, uint variables = 1);
   //---
   bool              BuildResponsibilities(CBufferFloat *target);
   bool              BuildResponsibilities(CBufferFloat *target,CBufferFloat *delta_target);
   bool              ResetEpochDiagnostics(void);
   bool              CommitBatchDiagnostics(void);
   bool              BuildEpochCodebookDiagnostics(void);
   bool              ReadEpochDiagnostics(double &mix_loss, double &router_loss,
                                          double &trajectory_loss, double &confidence_loss,
                                          double &latent_error, double &observation_error,
                                          double &valid_batches, double &invalid_batches,
                                          double &usage_entropy, double &pairwise_codebook,
                                          double &inactive_count, double &recovered);
   bool              ReadEpochRecoveryEvents(uint &events[]);
   bool              ReadEpochAgeZeroEvents(uint &events[]);
   bool              SetRecoveryAge(const uint age);
   //---
   virtual int       Type(void) override const { return defNeuronScenarioForecast; }
   virtual bool      Save(const int file_handle) override;
   virtual bool      Load(const int file_handle) override;
   bool              Load(const int file_handle, COpenCLMy *open_cl);
   virtual void      SetOpenCL(COpenCLMy *obj) override;
   virtual void      TrainMode(bool flag) override;
   virtual bool      WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool      Clear(void) override;
   //---
   uint              Variables(void) const { return iVariables; }
   uint              Scenarios(void) const { return iScenarios; }
   uint              GetTopK(void) const { return iTopK; }
   uint              Horizon(void) const { return iHorizon; }
   uint              Dimension(void) const { return iDimension; }
   uint              RecoveryAge(void) const { return iRecoveryAge; }
   //---
   CBufferFloat      *GetZ(void) { return getOutput(); }
   CBufferFloat      *GetU(void) { return GetPointer(cConfidence); }
   CBufferFloat      *GetLogU(void) { return GetPointer(cLogConfidence); }
   CBufferFloat      *GetPi(void) { return cRouterSoftmax.getOutput(); }
   CBufferFloat      *GetLogPi(void) { return GetPointer(cLogProbabilities); }
   CBufferFloat      *GetResponsibilities(void) { return GetPointer(cResponsibilities); }
   //---
   CScenarioCodebook *GetCodebook(void) { return GetPointer(cCodebook); }
   CNeuronConvOCL    *GetGenerator(void) { return GetPointer(cGenerator); }
   CNeuronConvOCL    *GetRouter(void) { return GetPointer(cRouter); }
   CNeuronConvOCL    *GetConfidenceHead(void) { return GetPointer(cConfidenceHead); }
   CBufferFloat      *GetConfidence(void) { return GetPointer(cConfidence); }
   CBufferFloat      *GetTarget(void) { return GetPointer(cTarget); }
   CNeuronSoftMaxOCL *GetRouterSoftmax(void) { return GetPointer(cRouterSoftmax); }
   CNeuronSparseSoftMax *GetTopKSelector(void) { return GetPointer(cTopKSelector); }
   //---
   ulong             ContractSignature(void);
   bool              ExportDetached(CBufferFloat *z, CBufferFloat *u, CBufferFloat *log_u,
                                    CBufferFloat *pi, CBufferFloat *log_pi);
   uint              TrainableWeights(void);
   bool              LastTargetAccepted(void) const { return bLastTargetAccepted; }
   bool              LastBatchControlValid(void) const { return bLastControlValid; }
   uint              ActiveTrajectories(void);
   uint              DenseRouterGradientCount(void);
  };
//+------------------------------------------------------------------+
//| Scenario-local Flash cross-attention for ORION Actor/Critic      |
//+------------------------------------------------------------------+
class CNeuronScenarioCrossAttention : public CNeuronMSRes
  {
protected:
   uint                       iVariables;
   uint                       iScenarios;
   uint                       iHorizon;
   uint                       iDimension;
   uint                       iQueries;
   uint                       iStackSize;
   uint                       iTopK;
   bool                       bCritic;
   CNeuronMultiWindowsConvOCL cQueryProjection;
   CNeuronBaseOCL             cQueryGradient;
   CNeuronConvOCL             cWz;
   CNeuronConvOCL             cWu;
   CNeuronBaseOCL             cTokenSum;
   CNeuronLearnabledPE        cTokenPE;
   CNeuronTransposeRCDOCL     cTokensTranspose;
   CNeuronBaseOCL             cKV;
   CNeuronBaseOCL             cKVGradK;
   CNeuronBaseOCL             cKVGradV;
   CCrossMHFlashAttention     cAttention;
   CNeuronTransposeRCDOCL     cAttentionTranspose;
   CNeuronBaseOCL             cHeadGate;
   CNeuronTransposeRCDOCL     cHeadGateTranspose;
   CNeuronConvOCL             cOutputProjection;
   CNeuronConvOCL             cQueryResidual;
   CNeuronBaseOCL             cMHAOutput;
//--- Actor-only MPI continuity.  The scenario axis remains intact through
//--- cScenarioHistory; Pi is applied only afterwards in cHeadGate.
   CNeuronAddToStack          cHistoryStack;
   CNeuronMomADTTM            cHistoryTTM;
   CNeuronConvOCL             cHistoryTokenProjection;
   CCrossMHFlashAttention     cHistoryMarket;
   CNeuronCrossAttention      cHistoryContext;
   CNeuronCrossAttention      cScenarioHistory;
   //---
   virtual bool               feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               feedForwardSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) override;
   virtual bool               calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               calcInputGradientsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) override;
   virtual bool               updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; }
   virtual bool               updateInputWeightsSecond(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Forecast) override;
   bool                       InitHolders(uint myIndex, ENUM_OPTIMIZATION optimization_type, uint batch);

public:
                     CNeuronScenarioCrossAttention(void) : iVariables(0), iScenarios(0), iHorizon(0), iDimension(0),
                     iQueries(0), iStackSize(0), iTopK(0), bCritic(false) {};
                    ~CNeuronScenarioCrossAttention(void) {};
   //---
   virtual bool               Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                   uint scenarios = 21, uint horizon = 12, uint latent = 16,
                                   bool critic = false, ENUM_OPTIMIZATION optimization_type = ADAM,
                                   uint batch = 1, uint variables = 1, uint stack_size = 1,
                                   uint top_k = 1);
   virtual int                Type(void) override const { return defNeuronScenarioCrossAttention; }
   virtual bool               Save(const int file_handle) override;
   virtual bool               Load(const int file_handle) override;
   bool                       Load(const int file_handle, COpenCLMy *open_cl);
   virtual void               SetOpenCL(COpenCLMy *obj) override;
   virtual void               TrainMode(bool flag) override;
   virtual bool               WeightsUpdate(CNeuronBaseOCL *source, float tau) override;
   virtual bool               Clear(void) override;
   uint                       Variables(void) const { return iVariables; }
   uint                       Scenarios(void) const { return iScenarios; }
   uint                       Horizon(void) const { return iHorizon; }
   uint                       Latent(void) const { return iDimension; }
   uint                       Queries(void) const { return iQueries; }
   uint                       HistorySize(void) const { return iStackSize; }
   uint                       HistoryTopK(void) const { return iTopK; }
   bool                       IsCritic(void) const { return bCritic; }
   CNeuronConvOCL            *GetWz(void) { return GetPointer(cWz); }
   CNeuronConvOCL            *GetWu(void) { return GetPointer(cWu); }
   CBufferFloat              *GetHeadContexts(void) { return (bCritic ? cAttention.getOutput() : cScenarioHistory.getOutput()); }
   bool                       HasHistory(void) const { return !bCritic; }
   CNeuronAddToStack         *GetHistoryStack(void) { return (bCritic ? NULL : GetPointer(cHistoryStack)); }
  };
//+------------------------------------------------------------------+
#endif // NEURONET_DECLARATIONS_MQH

#ifndef NEURONET_BUILDING_FACADE
#include "NeuroNet.mqh"
#endif // NEURONET_BUILDING_FACADE
