#ifndef NEURONET_BUILDING_FACADE
#include "NeuroNet.mqh"
#endif // NEURONET_BUILDING_FACADE
//+------------------------------------------------------------------+
//| Logical method implementations generated from NeuroNet.mqh: Misc
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::Init(uint numInputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, int models)
  {
   if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
      ReturnFalse;
   OpenCL = open_cl;
   optimization = ADAM;
   iBatch = 1;
   iModels = models;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferFloat();
      if(CheckPointer(Output) == POINTER_INVALID)
         ReturnFalse;
     }
   if(!Output.BufferInit(numNeurons * models, 0.0))
      ReturnFalse;
   if(!Output.BufferCreate(OpenCL))
      ReturnFalse;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferFloat();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         ReturnFalse;
     }
   if(!PrevOutput.BufferInit(numNeurons * models, 1.0))
      ReturnFalse;
   if(!PrevOutput.BufferCreate(OpenCL))
      ReturnFalse;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferFloat();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         ReturnFalse;
     }
   if(!Gradient.BufferInit((numNeurons + 1)*models, 0.0))
      ReturnFalse;
   if(!Gradient.BufferCreate(OpenCL))
      ReturnFalse;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      Weights = new CBufferFloat();
      if(CheckPointer(Weights) == POINTER_INVALID)
         ReturnFalse;
     }
   int count = (int)((numInputs + 1) * numNeurons * models);
   if(!Weights.Reserve(count))
      ReturnFalse;
   float k = (float)(1 / sqrt(numInputs + 1));
   for(int i = 0; i < count; i++)
     {
      if(!Weights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier))
         ReturnFalse;
     }
   if(!Weights.BufferCreate(OpenCL))
      ReturnFalse;
//---
   DeleteObj(DeltaWeights);
//---
   if(CheckPointer(FirstMomentum) == POINTER_INVALID)
     {
      FirstMomentum = new CBufferFloat();
      if(CheckPointer(FirstMomentum) == POINTER_INVALID)
         ReturnFalse;
     }
   if(!FirstMomentum.BufferInit(count, 0))
      ReturnFalse;
   if(!FirstMomentum.BufferCreate(OpenCL))
      ReturnFalse;
//---
   if(CheckPointer(SecondMomentum) == POINTER_INVALID)
     {
      SecondMomentum = new CBufferFloat();
      if(CheckPointer(SecondMomentum) == POINTER_INVALID)
         ReturnFalse;
     }
   if(!SecondMomentum.BufferInit(count, 0))
      ReturnFalse;
   if(!SecondMomentum.BufferCreate(OpenCL))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::feedForward(CNeuronBaseOCL* NeuronOCL)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID)
      ReturnFalse;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = Output.Total() / iModels;
   global_work_size[1] = iModels;
   setBuffer(def_k_FFMultiModels, def_k_ff_matrix_w, getWeightsIndex())
   setBuffer(def_k_FFMultiModels, def_k_ff_matrix_i, NeuronOCL.getOutputIndex())
   setBuffer(def_k_FFMultiModels, def_k_ff_matrix_o, Output.GetIndex())
   setArgument(def_k_FFMultiModels, def_k_ff_inputs, NeuronOCL.Neurons() / iModels)
   setArgument(def_k_FFMultiModels, def_k_ff_activation, (int)activation)
   kernelExecute(def_k_FFMultiModels, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!Output.BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::calcHiddenGradients(CNeuronBaseOCL* NeuronOCL)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID)
      ReturnFalse;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = NeuronOCL.Neurons() / iModels;
   global_work_size[1] = iModels;
   setBuffer(def_k_HGMultiModels, def_k_chg_matrix_w, getWeightsIndex())
   setBuffer(def_k_HGMultiModels, def_k_chg_matrix_g, getGradientIndex())
   setBuffer(def_k_HGMultiModels, def_k_chg_matrix_o, NeuronOCL.getOutputIndex())
   setBuffer(def_k_HGMultiModels, def_k_chg_matrix_ig, NeuronOCL.getGradientIndex())
   setArgument(def_k_HGMultiModels, def_k_chg_outputs, Neurons() / iModels)
   setArgument(def_k_HGMultiModels, def_k_chg_activation, NeuronOCL.Activation())
   iUpdateModel = (int)MathRound(MathRand() / 32767.0 * (iModels - 1));
   setArgument(def_k_HGMultiModels, def_k_chg_model, iUpdateModel)
//Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__);
   kernelExecute(def_k_HGMultiModels, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!NeuronOCL.getGradient().BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID)
      ReturnFalse;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = Neurons() / iModels;
   global_work_size[1] = NeuronOCL.Neurons() / iModels + 1;
   uint rest = 0;
   float lt = lr;
   setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_w, getWeightsIndex())
   setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_g, getGradientIndex())
   setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex())
   setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_m, getFirstMomentumIndex())
   setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_v, getSecondMomentumIndex())
   lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t)));
   setArgument(def_k_UWMultiModels, def_k_uwa_inputs, NeuronOCL.Neurons() / iModels)
   setArgument(def_k_UWMultiModels, def_k_uwa_l, lt)
   setArgument(def_k_UWMultiModels, def_k_uwa_b1, b1)
   setArgument(def_k_UWMultiModels, def_k_uwa_b2, b2)
   setArgument(def_k_UWMultiModels, def_k_uwa_model, iUpdateModel)
   global_work_size[1] = (global_work_size[1] + 3) / 4;
   ResetLastError();
   kernelExecute(def_k_UWMultiModels, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!Weights.BufferRead())
      ReturnFalse;
#endif
   t++;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
   if(FileWriteInteger(file_handle, iModels) <= 0)
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiModel::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
   iModels = FileReadInteger(file_handle);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronConcatenate::CNeuronConcatenate(void) : i_SecondInputs(0)
  {
   ConcWeights = new CBufferFloat();
   ConcDeltaWeights = new CBufferFloat();
   ConcFirstMomentum = new CBufferFloat();
   ConcSecondMomentum = new CBufferFloat;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronConcatenate::~CNeuronConcatenate()
  {
   DeleteObj(ConcWeights);
   DeleteObj(ConcDeltaWeights);
   DeleteObj(ConcFirstMomentum);
   DeleteObj(ConcSecondMomentum);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint numInputs1, uint numInputs2, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch))
      ReturnFalse;
//---
   i_SecondInputs = (int)numInputs2;
   if(!ConcWeights)
     {
      ConcWeights = new CBufferFloat();
      if(!ConcWeights)
         ReturnFalse;
     }
   int count = (int)((numInputs1 + numInputs2 + 1) * numNeurons);
   if(!ConcWeights.Reserve(count))
      ReturnFalse;
   float k = (float)(1 / sqrt(numNeurons + 1));
   for(int i = 0; i < count; i++)
     {
      if(!ConcWeights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier))
         ReturnFalse;
     }
   if(!ConcWeights.BufferCreate(OpenCL))
      ReturnFalse;
//---
   if(optimization == SGD)
     {
      if(!ConcDeltaWeights)
        {
         ConcDeltaWeights = new CBufferFloat();
         if(!ConcDeltaWeights)
            ReturnFalse;
        }
      if(!ConcDeltaWeights.BufferInit(count, 0))
         ReturnFalse;
      if(!ConcDeltaWeights.BufferCreate(OpenCL))
         ReturnFalse;
      DeleteObj(ConcFirstMomentum);
      DeleteObj(ConcSecondMomentum);
     }
   else
     {
      DeleteObj(ConcDeltaWeights);
      //---
      if(!ConcFirstMomentum)
        {
         ConcFirstMomentum = new CBufferFloat();
         if(CheckPointer(ConcFirstMomentum) == POINTER_INVALID)
            ReturnFalse;
        }
      if(!ConcFirstMomentum.BufferInit(count, 0))
         ReturnFalse;
      if(!ConcFirstMomentum.BufferCreate(OpenCL))
         ReturnFalse;
      //---
      if(!ConcSecondMomentum)
        {
         ConcSecondMomentum = new CBufferFloat();
         if(!ConcSecondMomentum)
            ReturnFalse;
        }
      if(!ConcSecondMomentum.BufferInit(count, 0))
         ReturnFalse;
      if(!ConcSecondMomentum.BufferCreate(OpenCL))
         ReturnFalse;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput)
  {
   if(!OpenCL || !NeuronOCL || !SecondInput)
      ReturnFalse;
   if(SecondInput.Total() < i_SecondInputs)
      ReturnFalse;
   if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL))
      ReturnFalse;
//---
   setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_w, ConcWeights.GetIndex())
   setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i1, NeuronOCL.getOutputIndex())
   setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i2, SecondInput.GetIndex())
   setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_o, Output.GetIndex())
   setArgument(def_k_ConcatFeedForward, def_k_cff_inputs1, (int)NeuronOCL.Neurons())
   setArgument(def_k_ConcatFeedForward, def_k_cff_inputs2, (int)i_SecondInputs)
   setArgument(def_k_ConcatFeedForward, def_k_cff_activation, (int)activation)
//---
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = Output.Total();
   kernelExecute(def_k_ConcatFeedForward, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!Output.BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None)
  {
   if(!OpenCL || !NeuronOCL || !SecondInput || !SecondGradient)
      ReturnFalse;
   if(SecondInput.Total() < i_SecondInputs || SecondGradient.Total() < i_SecondInputs)
      ReturnFalse;
   if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL))
      ReturnFalse;
   if(SecondGradient.GetIndex() < 0 && !SecondGradient.BufferCreate(OpenCL))
      ReturnFalse;
//---
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = NeuronOCL.Neurons() + i_SecondInputs;
   setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_w, ConcWeights.GetIndex())
   setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_g, Gradient.GetIndex())
   setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig1, NeuronOCL.getGradientIndex())
   setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig2, SecondGradient.GetIndex())
   setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o1, NeuronOCL.getOutputIndex())
   setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o2, SecondInput.GetIndex())
   setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs1, NeuronOCL.Neurons())
   setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs2, i_SecondInputs)
   setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_outputs, Neurons())
   setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation1, NeuronOCL.Activation())
   setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation2, (int)SecondActivation)
   kernelExecute(def_k_ConcatCalcHiddenGradient, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!ConcWeights.BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput)
  {
   if(!OpenCL || !NeuronOCL || !SecondInput)
      ReturnFalse;
   if(SecondInput.Total() < i_SecondInputs)
      ReturnFalse;
   if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL))
      ReturnFalse;
//---
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = Neurons();
   global_work_size[1] = NeuronOCL.Neurons() + i_SecondInputs + 1;
   float lt = lr;
   ResetLastError();
   switch(NeuronOCL.Optimization())
     {
      case SGD:
         setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_w, ConcWeights.GetIndex())
         setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_g, getGradientIndex())
         setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i1, NeuronOCL.getOutputIndex())
         setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i2, SecondInput.GetIndex())
         setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_dw, ConcDeltaWeights.GetIndex())
         setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, NeuronOCL.Neurons())
         setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, i_SecondInputs)
         setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_learning_rates, lr)
         setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_momentum, alpha)
         ResetLastError();
         kernelExecute(def_k_ConcatUpdWeightsMomentum, global_work_offset, global_work_size)
         break;
      case ADAM:
      case ADAM_MINI:
         setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_w, ConcWeights.GetIndex())
         setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_g, getGradientIndex())
         setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i1, NeuronOCL.getOutputIndex())
         setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i2, SecondInput.GetIndex())
         setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_m, ConcFirstMomentum.GetIndex())
         setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_v, ConcSecondMomentum.GetIndex())
         lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t)));
         setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs1, NeuronOCL.Neurons())
         setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs2, i_SecondInputs)
         setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_l, lt)
         setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b1, b1)
         setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b2, b2)
         kernelExecute(def_k_ConcatUpdWeightsAdam, global_work_offset, global_work_size)
         t++;
         break;
      default:
         ReturnFalse;
         break;
     }
#ifdef  _DEBUG
   if(!ConcWeights.BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
//---
   if(FileWriteInteger(file_handle, i_SecondInputs) < INT_VALUE)
      ReturnFalse;
   if(!ConcWeights.Save(file_handle))
      ReturnFalse;
   if(optimization == SGD)
     {
      if(!ConcDeltaWeights.Save(file_handle))
         ReturnFalse;
     }
   else
     {
      if(!ConcFirstMomentum.Save(file_handle))
         ReturnFalse;
      if(!ConcSecondMomentum.Save(file_handle))
         ReturnFalse;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
//---
   i_SecondInputs = FileReadInteger(file_handle);
   if(!ConcWeights.Load(file_handle))
      ReturnFalse;
   ConcWeights.BufferCreate(OpenCL);
   if(optimization == SGD)
     {
      if(!ConcDeltaWeights.Load(file_handle))
         ReturnFalse;
      ConcDeltaWeights.BufferCreate(OpenCL);
     }
   else
     {
      if(!ConcFirstMomentum.Load(file_handle))
         ReturnFalse;
      if(!ConcSecondMomentum.Load(file_handle))
         ReturnFalse;
      ConcFirstMomentum.BufferCreate(OpenCL);
      ConcSecondMomentum.BufferCreate(OpenCL);
     }
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronConcatenate::SetOpenCL(COpenCLMy * obj)
  {
   CNeuronBaseOCL::SetOpenCL(obj);
   if(!ConcWeights)
      return;
   ConcWeights.BufferCreate(OpenCL);
   if(optimization == SGD)
     {
      if(!ConcDeltaWeights)
         return;
      ConcDeltaWeights.BufferCreate(OpenCL);
     }
   else
     {
      if(!ConcFirstMomentum || !ConcSecondMomentum)
         return;
      ConcFirstMomentum.BufferCreate(OpenCL);
      ConcSecondMomentum.BufferCreate(OpenCL);
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConcatenate::WeightsUpdate(CNeuronBaseOCL* source, float tau)
  {
   if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
      ReturnFalse;
//---
   CNeuronConcatenate *temp = source;
   if(ConcWeights.Total() != temp.ConcWeights.Total())
      ReturnFalse;
//---
   uint global_work_offset[1] = {0};
   uint global_work_size[1] = {ConcWeights.Total()};
   ResetLastError();
   if(tau != 1.0f && optimization == ADAM)
     {
      setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, ConcWeights.GetIndex())
      setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, temp.ConcWeights.GetIndex())
      setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, ConcFirstMomentum.GetIndex())
      setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, ConcSecondMomentum.GetIndex())
      setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau)
      setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1)
      setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2)
      kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size)
#ifdef  _DEBUG
      if(!ConcFirstMomentum.BufferRead())
         ReturnFalse;
#endif
     }
   else
     {
      setBuffer(def_k_SoftUpdate, def_k_su_target, ConcWeights.GetIndex())
      setBuffer(def_k_SoftUpdate, def_k_su_source, temp.ConcWeights.GetIndex())
      setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau)
      kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size)
#ifdef  _DEBUG
      if(!ConcWeights.BufferRead())
         ReturnFalse;
#endif
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint & windows[], uint variables, uint inputs_count, bool need_transpose, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   const uint layers = windows.Size();
   if(layers <= 0)
      ReturnFalse;
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs_count * variables, optimization_type, batch))
      ReturnFalse;
//---
   ia_Windows.Clear();
   ia_Windows.Reserve(layers);
   for(uint i = 0; i < layers; i++)
      if(!ia_Windows.Add((int)windows[i]))
         ReturnFalse;
   i_Variables = variables;
   i_Count = inputs_count / ia_Windows[0];
   b_NeedTranspose = need_transpose;
//---
   if(b_NeedTranspose)
     {
      CNeuronTransposeOCL *transp = new CNeuronTransposeOCL();
      if(!transp)
         ReturnFalse;
      if(!transp.Init(0, 0, OpenCL, inputs_count, i_Variables, optimization, iBatch))
        {
         DeleteObj(transp);
         ReturnFalse;
        }
      if(!caTranspose.Add(transp))
        {
         DeleteObj(transp);
         ReturnFalse;
        }
      transp = new CNeuronTransposeOCL();
      if(!transp)
         ReturnFalse;
      if(!transp.Init(0, 1, OpenCL, i_Variables, inputs_count, optimization, iBatch))
        {
         DeleteObj(transp);
         ReturnFalse;
        }
      if(!caTranspose.Add(transp))
        {
         DeleteObj(transp);
         ReturnFalse;
        }
      if(!SetOutput(transp.getOutput()) ||
         !SetGradient(transp.getGradient())
        )
         ReturnFalse;
     }
//---
   uint total = ia_Windows[0] * i_Count;
   CNeuronConvOCL *conv = new CNeuronConvOCL();
   if(!conv.Init(0, 0, OpenCL, inputs_count, inputs_count, total, 1, i_Variables, optimization, iBatch))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(SIGMOID);
   if(!caConvolutions.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
//---
   total = 0;
   for(uint i = 0; i < layers; i++)
     {
      conv = new CNeuronConvOCL();
      if(!conv.Init(0, i + 1, OpenCL, ia_Windows[i], ia_Windows[i], (i < (layers - 1) ? ia_Windows[i + 1] : 1), i_Count, i_Variables, optimization, iBatch))
        {
         DeleteObj(conv);
         ReturnFalse;
        }
      conv.SetActivationFunction(SIGMOID);
      if(!caConvolutions.Add(conv))
        {
         DeleteObj(conv);
         ReturnFalse;
        }
      if(!caConvOutputs.Add(conv.getOutput()) ||
         !caConvGradients.Add(conv.getGradient())
        )
         ReturnFalse;
      total += conv.Neurons();
     }
//---
   CNeuronBaseOCL *comul = new CNeuronBaseOCL();
   if(!comul.Init(0, 0, OpenCL, total, optimization, iBatch))
     {
      DeleteObj(comul);
      ReturnFalse;
     }
   if(!caMLP.Add(comul))
     {
      DeleteObj(comul);
      ReturnFalse;
     }
   if(layers == 1)
     {
      comul.SetOutput(conv.getOutput());
      comul.SetGradient(conv.getGradient());
     }
   conv = new CNeuronConvOCL();
   if(!conv.Init(0, 0, OpenCL, total / i_Variables, total / i_Variables, inputs_count, 1, i_Variables, optimization, iBatch))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   if(!caMLP.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   if(!b_NeedTranspose)
     {
      if(!SetOutput(conv.getOutput()) ||
         !SetGradient(conv.getGradient())
        )
         ReturnFalse;
     }
//---
   CBufferFloat *buf = new CBufferFloat();
   if(!buf)
      ReturnFalse;
   if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) ||
      !caTemp.Add(buf))
     {
      DeleteObj(buf);
      ReturnFalse;
     }
   buf = new CBufferFloat();
   if(!buf)
      ReturnFalse;
   if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) ||
      !caTemp.Add(buf))
     {
      DeleteObj(buf);
      ReturnFalse;
     }
   buf = new CBufferFloat();
   if(!buf)
      ReturnFalse;
   if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) ||
      !caTemp.Add(buf))
     {
      DeleteObj(buf);
      ReturnFalse;
     }
//---
   caConvOutputs.FreeMode(false);
   caConvGradients.FreeMode(false);
   caTranspose.FreeMode(true);
   caConvolutions.FreeMode(true);
   caMLP.FreeMode(true);
   caTemp.FreeMode(true);
   SetActivationFunction(None);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::feedForward(CNeuronBaseOCL* NeuronOCL)
  {
   CNeuronBaseOCL *inp = NeuronOCL;
   CNeuronBaseOCL *current = NULL;
//---
   if(b_NeedTranspose)
     {
      current = caTranspose.At(0);
      if(!current ||
         !current.FeedForward(inp))
         ReturnFalse;
      inp = current;
     }
//---
   int layers = caConvolutions.Total() - 1;
   for(int l = 0; l <= layers; l++)
     {
      current = caConvolutions.At(l);
      if(!current ||
         !current.FeedForward(inp))
         ReturnFalse;
      inp = current;
     }
//---
   current = caMLP.At(0);
   if(!current)
      ReturnFalse;
   switch(layers)
     {
      case 0:
         ReturnFalse;
      case 1:
         break;
      case 2:
         if(!Concat(caConvOutputs.At(0),
                    caConvOutputs.At(1),
                    current.getOutput(),
                    ia_Windows[1], 1,
                    i_Variables * i_Count))
            ReturnFalse;
         break;
      case 3:
         if(!Concat(caConvOutputs.At(0),
                    caConvOutputs.At(1),
                    caConvOutputs.At(2),
                    current.getOutput(),
                    ia_Windows[1], ia_Windows[2], 1,
                    i_Variables * i_Count))
            ReturnFalse;
         break;
      case 4:
         if(!Concat(caConvOutputs.At(0),
                    caConvOutputs.At(1),
                    caConvOutputs.At(2),
                    caConvOutputs.At(3),
                    current.getOutput(),
                    ia_Windows[1], ia_Windows[2], ia_Windows[3], 1,
                    i_Variables * i_Count))
            ReturnFalse;
         break;
      default:
         if(!Concat(caConvOutputs.At(0),
                    caConvOutputs.At(1),
                    caConvOutputs.At(2),
                    caConvOutputs.At(3),
                    caTemp.At(0),
                    ia_Windows[1], ia_Windows[2], ia_Windows[3], ia_Windows[4],
                    i_Variables * i_Count))
            ReturnFalse;
         break;
     }
   uint last_buf = 0;
   for(int i = 4; i < layers; i += 3)
     {
      uint buf_size = 0;
      for(int j = 1; j <= i; j++)
         buf_size += ia_Windows[j];
      switch(layers - i)
        {
         case 1:
            if(!Concat(caTemp.At(last_buf),
                       caConvOutputs.At(i),
                       current.getOutput(),
                       buf_size, 1,
                       i_Variables * i_Count))
               ReturnFalse;
            break;
         case 2:
            if(!Concat(caTemp.At(last_buf),
                       caConvOutputs.At(i),
                       caConvOutputs.At(i + 1),
                       current.getOutput(),
                       buf_size, ia_Windows[i + 1], 1,
                       i_Variables * i_Count))
               ReturnFalse;
            break;
         case 3:
            if(!Concat(caTemp.At(last_buf),
                       caConvOutputs.At(i),
                       caConvOutputs.At(i + 1),
                       caConvOutputs.At(i + 2),
                       current.getOutput(),
                       buf_size, ia_Windows[i + 1], ia_Windows[i + 2], 1,
                       i_Variables * i_Count))
               ReturnFalse;
            break;
         default:
            if(!Concat(caTemp.At(last_buf),
                       caConvOutputs.At(i),
                       caConvOutputs.At(i + 1),
                       caConvOutputs.At(i + 2),
                       caTemp.At((last_buf + 1) % 2),
                       buf_size, ia_Windows[i + 1], ia_Windows[i + 2], ia_Windows[i + 3],
                       i_Variables * i_Count))
               ReturnFalse;
            break;
        }
      last_buf = (last_buf + 1) % 2;
     }
//---
   inp = current;
   current = caMLP.At(1);
   if(!current ||
      !current.FeedForward(inp))
      ReturnFalse;
//---
   if(b_NeedTranspose)
     {
      inp = current;
      current = caTranspose.At(1);
      if(!current ||
         !current.FeedForward(inp))
         ReturnFalse;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::calcInputGradients(CNeuronBaseOCL* prevLayer)
  {
   if(!prevLayer)
      ReturnFalse;
//---
   CNeuronBaseOCL *current = caMLP.At(0);
   CNeuronBaseOCL *next = caMLP.At(1);
   if(b_NeedTranspose)
     {
      if(!next.CalcHiddenGradients(caTranspose.At(1)))
         ReturnFalse;
     }
   if(!current.CalcHiddenGradients(next.AsObject()))
      ReturnFalse;
   next = current;
//---
   int layers = caConvGradients.Total();
   if(layers == 1)
     {
      next = caConvolutions.At(1);
      if(next.Activation() != None)
        {
         if(!DeActivation(next.getOutput(), next.getGradient(), next.getGradient(), next.Activation()))
            ReturnFalse;
        }
     }
   else
     {
      int prev_window = 0;
      for(int i = 1; i < layers; i++)
         prev_window += int(ia_Windows[i]);
      if(!DeConcat(caTemp.At(0), caConvGradients.At(layers - 1), next.getGradient(), prev_window, 1, i_Variables * i_Count))
         ReturnFalse;
      next = caConvolutions.At(layers);
      int current_buf = 0;
      for(int l = layers; l > 1; l--)
        {
         current = caConvolutions.At(l - 1);
         if(!current.CalcHiddenGradients(next.AsObject()))
            ReturnFalse;
         int window = int(ia_Windows[l - 1]);
         prev_window -= window;
         if(!DeConcat(caTemp.At((current_buf + 1) % 2), caTemp.At(2), caTemp.At(current_buf), prev_window, window, i_Variables * i_Count))
            ReturnFalse;
         if(current.Activation() != None)
           {
            if(!DeActivation(current.getOutput(), caTemp.At(2), caTemp.At(2), current.Activation()))
               ReturnFalse;
           }
         if(!SumAndNormalize(current.getGradient(), caTemp.At(2), current.getGradient(), 1, false, 0, 0, 0, 1))
            ReturnFalse;
         next = current;
         current_buf = (current_buf + 1) % 2;
        }
     }
   current = caConvolutions.At(0);
   if(!current.CalcHiddenGradients(next.AsObject()))
      ReturnFalse;
   next = current;
//---
   if(b_NeedTranspose)
     {
      current = caTranspose.At(0);
      if(!current.CalcHiddenGradients(next.AsObject()))
         ReturnFalse;
      next = current;
     }
//---
   if(!prevLayer.CalcHiddenGradients(next.AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
  {
   CObject *prev = (b_NeedTranspose ? caTranspose.At(0) : NeuronOCL);
   CNeuronBaseOCL *current = NULL;
   for(int i = 0; i < caConvolutions.Total(); i++)
     {
      current = caConvolutions.At(1);
      if(!current ||
         !current.UpdateInputWeights(prev)
        )
         ReturnFalse;
      prev = current;
     }
   current = caMLP.At(1);
   if(!current ||
      !current.UpdateInputWeights(caMLP.At(0))
     )
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
//--- Save constants
   if(FileWriteInteger(file_handle, int(i_Variables)) < INT_VALUE)
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(i_Count)) < INT_VALUE)
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(b_NeedTranspose)) < INT_VALUE)
      ReturnFalse;
//--- Objects
   if(!ia_Windows.Save(file_handle))
      ReturnFalse;
   if(b_NeedTranspose)
     {
      for(int i = 0; i < 2; i++)
         if(!caTranspose.At(i).Save(file_handle))
            ReturnFalse;
     }
   int layers = caConvolutions.Total();
   if(FileWriteInteger(file_handle, layers) < INT_VALUE)
      ReturnFalse;
   for(int i = 0; i < layers; i++)
     {
      if(!caConvolutions.At(i).Save(file_handle))
         ReturnFalse;
     }
   for(int i = 0; i < 2; i++)
      if(!caMLP.At(i).Save(file_handle))
         ReturnFalse;
   for(int i = 0; i < 3; i++)
      if(!caTemp.At(i).Save(file_handle))
         ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::Load(const int file_handle)
  {
   CBufferFloat *temp = new CBufferFloat();
   if(!temp)
      ReturnFalse;
   if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) ||
      !SetOutput(temp, false))
     {
      DeleteObj(temp);
      ReturnFalse;
     }
   temp = new CBufferFloat();
   if(!temp)
      ReturnFalse;
   if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) ||
      !SetGradient(temp, false))
     {
      DeleteObj(temp);
      ReturnFalse;
     }
//---
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
//--- Load constants
   if(FileIsEnding(file_handle))
      ReturnFalse;
   i_Variables = uint(FileReadInteger(file_handle));
   if(FileIsEnding(file_handle))
      ReturnFalse;
   i_Count = uint(FileReadInteger(file_handle));
   if(FileIsEnding(file_handle))
      ReturnFalse;
   b_NeedTranspose = bool(FileReadInteger(file_handle));
   if(FileIsEnding(file_handle))
      ReturnFalse;
//--- Objects
   if(!ia_Windows.Load(file_handle))
      ReturnFalse;
   if(b_NeedTranspose)
     {
      caTranspose.Clear();
      CNeuronTransposeOCL *transp = NULL;
      for(int i = 0; i < 2; i++)
        {
         transp = new CNeuronTransposeOCL();
         if(!transp)
            ReturnFalse;
         if(!transp.Init(0, i, OpenCL, 1, 1, optimization, iBatch) ||
            !LoadInsideLayer(file_handle, transp) ||
            !caTranspose.Add(transp))
           {
            DeleteObj(transp);
            ReturnFalse;
           }
        }
     }
   int layers = FileReadInteger(file_handle);
   caConvolutions.Clear();
   caConvOutputs.Clear();
   caConvGradients.Clear();
   CNeuronConvOCL *conv = NULL;
   for(int i = 0; i < layers; i++)
     {
      conv = new CNeuronConvOCL();
      if(!conv)
         ReturnFalse;
      if(!conv.Init(0, i, OpenCL, 1, 1, 1, 1, 1, optimization, iBatch) ||
         !LoadInsideLayer(file_handle, conv) ||
         !caConvolutions.Add(conv)
        )
        {
         DeleteObj(conv);
         ReturnFalse;
        }
      if(i > 0)
        {
         if(!caConvOutputs.Add(conv.getOutput()) ||
            !caConvGradients.Add(conv.getGradient())
           )
            ReturnFalse;
        }
     }
   for(int i = 0; i < 2; i++)
      if(!LoadInsideLayer(file_handle, caMLP.At(i)))
         ReturnFalse;
   for(int i = 0; i < 3; i++)
      if(!caTemp.At(i).Load(file_handle))
         ReturnFalse;
//---
   CNeuronBaseOCL *neuron = (b_NeedTranspose ? caTranspose.At(1) : caMLP.At(1));
   if(!SetOutput(neuron.getOutput()) ||
      !SetGradient(neuron.getGradient())
     )
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronCSCMOCL::SetOpenCL(COpenCLMy * obj)
  {
   CNeuronBaseOCL::SetOpenCL(obj);
   CNeuronBaseOCL *current = NULL;
   if(b_NeedTranspose)
      for(int i = 0; i < 2; i++)
        {
         current = caTranspose.At(i);
         current.SetOpenCL(OpenCL);
        }
//---
   for(int i = 0; i < caConvolutions.Total(); i++)
     {
      current = caConvolutions.At(i);
      current.SetOpenCL(OpenCL);
     }
//---
   for(int i = 0; i < 2; i++)
     {
      current = caMLP.At(i);
      current.SetOpenCL(OpenCL);
     }
   for(int i = 0; i < 3; i++)
     {
      CBufferFloat *buf = caTemp.At(i);
      buf.BufferCreate(OpenCL);
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronCSCMOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau)
  {
   if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
      ReturnFalse;
   CNeuronCSCMOCL *Source = source;
   if(caConvolutions.Total() != Source.caConvolutions.Total())
      ReturnFalse;
   CNeuronBaseOCL *current = NULL;
   for(int i = 0; i < caConvolutions.Total(); i++)
     {
      current = caConvolutions.At(i);
      if(!current ||
         !current.WeightsUpdate(Source.caConvolutions.At(i), tau)
        )
         ReturnFalse;
     }
   current = caMLP.At(1);
   if(!current ||
      !current.WeightsUpdate(Source.caMLP.At(1), tau)
     )
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch))
      ReturnFalse;
   if(!cAddNoise.Init(0, 0, OpenCL, window * units_count, iBatch, optimization))
      ReturnFalse;
   cAddNoise.SetActivationFunction(None);
   if(!cAttention[0].Init(0, 1, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch))
      ReturnFalse;
   if(!cMergeSplit[0].Init(0, 2, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch))
      ReturnFalse;
   if(inside_block > 0)
     {
      CNeuronDiffusion *temp = new CNeuronDiffusion();
      if(!temp)
         ReturnFalse;
      if(!temp.Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, inside_block - 1, optimization, iBatch))
        {
         DeleteObj(temp);
         ReturnFalse;
        }
      cNeck = temp;
     }
   else
     {
      CNeuronConvOCL *temp = new CNeuronConvOCL();
      if(!temp)
         ReturnFalse;
      if(!temp.Init(0, 3, OpenCL, window, window, window, (units_count + 1) / 2, optimization, iBatch))
        {
         DeleteObj(temp);
         ReturnFalse;
        }
      cNeck = temp;
     }
   if(!cAttention[1].Init(0, 4, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch))
      ReturnFalse;
   if(!cMergeSplit[1].Init(0, 5, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch))
      ReturnFalse;
   if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch))
      ReturnFalse;
   if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true))
      ReturnFalse;
   cResidual.SetActivationFunction((ENUM_ACTIVATION)cMergeSplit[1].Activation());
//---
   if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject()))
      ReturnFalse;
//---
   if(!SetOutput(cRevIn.getOutput(), true))
      ReturnFalse;
//---
   cRevIn.SetActivationFunction(None);
   SetActivationFunction(None);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::feedForward(CNeuronBaseOCL* NeuronOCL)
  {
   if(!cAddNoise.FeedForward(NeuronOCL))
      ReturnFalse;
   if(!cAttention[0].FeedForward(cAddNoise.AsObject()))
      ReturnFalse;
   if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject()))
      ReturnFalse;
   if(!cNeck.FeedForward(cMergeSplit[0].AsObject()))
      ReturnFalse;
   if(!cAttention[1].FeedForward(cNeck))
      ReturnFalse;
   if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject()))
      ReturnFalse;
   if(!SumAndNormalize(cAddNoise.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, true, 0, 0, 0, 1))
      ReturnFalse;
   if(!cRevIn.FeedForward(cResidual.AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::calcInputGradients(CNeuronBaseOCL* prevLayer)
  {
   if(!prevLayer)
      ReturnFalse;
//---
   float error = 1;
   if(!cRevIn.calcOutputGradients(prevLayer.getOutput(), error) ||
      !SumAndNormalize(cRevIn.getGradient(), Gradient, cRevIn.getGradient(), 1, false, 0, 0, 0, 1))
      ReturnFalse;
   if(!cResidual.CalcHiddenGradients(cRevIn.AsObject()))
      ReturnFalse;
   if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject()))
      ReturnFalse;
   if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject()))
      ReturnFalse;
   if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject()))
      ReturnFalse;
   if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject()))
      ReturnFalse;
   if(!cAddNoise.CalcHiddenGradients(cAttention[0].AsObject()))
      ReturnFalse;
   if(!SumAndNormalize(cAddNoise.getGradient(), cResidual.getGradient(), cAddNoise.getGradient(), 1, false, 0, 0, 0, 1))
      ReturnFalse;
   if(!prevLayer.CalcHiddenGradients(cAddNoise.AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
  {
   if(!cAddNoise.UpdateInputWeights(NeuronOCL))
      ReturnFalse;
   if(!cAttention[0].UpdateInputWeights(cAddNoise.AsObject()))
      ReturnFalse;
   if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject()))
      ReturnFalse;
   if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject()))
      ReturnFalse;
   if(!cAttention[1].UpdateInputWeights(cNeck))
      ReturnFalse;
   if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronDiffusion::SetOpenCL(COpenCLMy * obj)
  {
   CNeuronUShapeAttention::SetOpenCL(obj);
   cAddNoise.SetOpenCL(OpenCL);
   cResidual.SetOpenCL(OpenCL);
   cRevIn.SetOpenCL(OpenCL);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::Save(const int file_handle)
  {
   if(!CNeuronUShapeAttention::Save(file_handle))
      ReturnFalse;
   if(!cAddNoise.Save(file_handle))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronDiffusion::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
   for(int i = 0; i < 2; i++)
     {
      if(!LoadInsideLayer(file_handle, cAttention[i].AsObject()))
         ReturnFalse;
      if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject()))
         ReturnFalse;
     }
//---
   int type = FileReadInteger(file_handle);
   if(!!cNeck)
     {
      if(cNeck.Type() != type)
         DeleteObj(cNeck);
     }
//---
   if(!cNeck)
     {
      switch(type)
        {
         case defNeuronUShapeAttention:
            cNeck = new CNeuronUShapeAttention();
            if(!cNeck)
               ReturnFalse;
            break;
         case defNeuronDiffusion:
            cNeck = new CNeuronDiffusion();
            if(!cNeck ||
               !((CNeuronDiffusion*)cNeck).Init(0, 0, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1))
               ReturnFalse;
            break;
         case defNeuronConvOCL:
            cNeck = new CNeuronConvOCL();
            if(!cNeck)
               ReturnFalse;
            break;
         default:
            ReturnFalse;
        }
     }
   cNeck.SetOpenCL(OpenCL);
   if(!cNeck.Load(file_handle))
      ReturnFalse;
//---
   if(!cAddNoise.Init(0, 0, OpenCL, Neurons(), iBatch, optimization) ||
      !LoadInsideLayer(file_handle, cAddNoise.AsObject()))
      ReturnFalse;
//---
   if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch))
      ReturnFalse;
   if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true))
      ReturnFalse;
   if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject()))
      ReturnFalse;
//---
   if(!SetOutput(cRevIn.getOutput(), true))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + 1)*units_count, optimization_type, batch))
      ReturnFalse;
   iWindow = window;
   iUnits = units_count;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::feedForward(CNeuronBaseOCL* NeuronOCL)
  {
   if(!NeuronOCL)
      ReturnFalse;
//---
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2] = {iUnits, iWindow};
   uint local_work_size[2] = {1, iWindow};
   int kernel = def_k_HyperProjection;
   ResetLastError();
   setBuffer(kernel, def_k_lp_inputs, NeuronOCL.getOutputIndex())
   setBuffer(kernel, def_k_lp_outputs, getOutputIndex())
//---
   kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
#ifdef  _DEBUG
   if(!NeuronOCL.getOutput().BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::calcInputGradients(CNeuronBaseOCL* prevLayer)
  {
   if(!prevLayer)
      ReturnFalse;
//---
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2] = {iUnits, iWindow};
   int kernel = def_k_HyperProjectionGrad;
   ResetLastError();
   setBuffer(kernel, def_k_lpg_inputs, prevLayer.getOutputIndex())
   setBuffer(kernel, def_k_lpg_inputs_gr, prevLayer.getGradientIndex())
   setBuffer(kernel, def_k_lpg_outputs_gr, getGradientIndex())
//---
   kernelExecute(kernel, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!prevLayer.getGradient().BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE)
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE)
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperProjection::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iWindow = (int)FileReadInteger(file_handle);
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iUnits = (int)FileReadInteger(file_handle);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * centroids, optimization_type, batch))
      ReturnFalse;
//---
   iWindows = window;
   iUnits = units_count;
   iCentroids = centroids;
//---
   cHyperCentroids.Clear();
   cHyperCurvatures.Clear();
   cHyperCentroids.SetOpenCL(OpenCL);
   cHyperCurvatures.SetOpenCL(OpenCL);
//---
   CNeuronTransposeOCL *transp = new CNeuronTransposeOCL();
   if(!transp ||
      !transp.Init(0, 0, OpenCL, iUnits, iWindows, optimization, iBatch) ||
      !cHyperCentroids.Add(transp))
     {
      DeleteObj(transp);
      ReturnFalse;
     }
   transp.SetActivationFunction(None);
//---
   CNeuronConvOCL *conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, 1, OpenCL, iUnits, iUnits, iCentroids, iWindows, 1, optimization, iBatch) ||
      !cHyperCentroids.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(TANH);
//---
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, 2, OpenCL, iCentroids, iCentroids, iCentroids, 1, iWindows, optimization, iBatch) ||
      !cHyperCentroids.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(None);
//---
   transp = new CNeuronTransposeOCL();
   if(!transp ||
      !transp.Init(0, 3, OpenCL, iWindows, iCentroids, optimization, iBatch) ||
      !cHyperCentroids.Add(transp))
     {
      DeleteObj(transp);
      ReturnFalse;
     }
   transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
//---
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, 4, OpenCL, iWindows, iWindows, iWindows, iCentroids, 1, optimization, iBatch) ||
      !cHyperCurvatures.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(TANH);
//---
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, 5, OpenCL, iWindows, iWindows, 1, 1, iCentroids, optimization, iBatch) ||
      !cHyperCurvatures.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(None);
//---
   uint size = iCentroids * iUnits * sizeof(float);
   iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
   if(iProducts < 0)
      ReturnFalse;
   iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
   if(iDistances < 0)
      ReturnFalse;
   iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
   if(iNormes < 0)
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::LogMap(CNeuronBaseOCL* featers, CNeuronBaseOCL* centroids, CNeuronBaseOCL* curvatures, CNeuronBaseOCL* outputs)
  {
   if(!featers || !centroids || !curvatures || !outputs)
      ReturnFalse;
//---
   uint global_work_offset[3] = {0, 0, 0};
   uint global_work_size[3] = {iUnits, iCentroids, iWindows};
   uint local_work_size[3] = {1, 1, iWindows};
   int kernel = def_k_LogMap;
   ResetLastError();
   setBuffer(kernel, def_k_logmap_centroids, centroids.getOutputIndex())
   setBuffer(kernel, def_k_logmap_curvatures, curvatures.getOutputIndex())
   setBuffer(kernel, def_k_logmap_features, featers.getOutputIndex())
   setBuffer(kernel, def_k_logmap_outputs, outputs.getOutputIndex())
   setBuffer(kernel, def_k_logmap_distance, iDistances)
   setBuffer(kernel, def_k_logmap_product, iProducts)
   setBuffer(kernel, def_k_logmap_norma, iNormes)
//---
   kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
#ifdef  _DEBUG
   if(!outputs.getOutput().BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::LogMapGrad(CNeuronBaseOCL* featers, CNeuronBaseOCL* centroids, CNeuronBaseOCL* curvatures, CNeuronBaseOCL* outputs)
  {
   if(!featers || !centroids || !curvatures || !outputs)
      ReturnFalse;
//---
//---
   CBufferFloat *temp = featers.getGradient();
   if(!temp ||
      !temp.Fill(0) ||
      !temp.BufferWrite())
      ReturnFalse;
   temp = centroids.getGradient();
   if(!temp ||
      !temp.Fill(0) ||
      !temp.BufferWrite())
      ReturnFalse;
   temp = curvatures.getGradient();
   if(!temp ||
      !temp.Fill(0) ||
      !temp.BufferWrite())
      ReturnFalse;
//---
   uint global_work_offset[3] = {0, 0, 0};
   uint global_work_size[3] = {iUnits, iCentroids, iWindows};
   uint local_work_size[3] = {1, 1, iWindows};
   int kernel = def_k_LogMapGrad;
   ResetLastError();
   setBuffer(kernel, def_k_logmapgr_centroids, centroids.getOutputIndex())
   setBuffer(kernel, def_k_logmapgr_centroids_gr, centroids.getGradientIndex())
   setBuffer(kernel, def_k_logmapgr_curvatures, curvatures.getOutputIndex())
   setBuffer(kernel, def_k_logmapgr_curvatures_gr, curvatures.getGradientIndex())
   setBuffer(kernel, def_k_logmapgr_features, featers.getOutputIndex())
   setBuffer(kernel, def_k_logmapgr_features_gr, featers.getGradientIndex())
   setBuffer(kernel, def_k_logmapgr_outputs, outputs.getOutputIndex())
   setBuffer(kernel, def_k_logmapgr_outputs_gr, outputs.getGradientIndex())
   setBuffer(kernel, def_k_logmapgr_distance, iDistances)
   setBuffer(kernel, def_k_logmapgr_product, iProducts)
   setBuffer(kernel, def_k_logmapgr_norma, iNormes)
//---
   kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size)
#ifdef  _DEBUG
   if(!featers.getGradient().BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::feedForward(CNeuronBaseOCL* NeuronOCL)
  {
   CNeuronBaseOCL *prev = NeuronOCL;
   CNeuronBaseOCL *centroids = NULL;
   CNeuronBaseOCL *curvatures = NULL;
//--- Centroids
   for(int i = 0; i < cHyperCentroids.Total(); i++)
     {
      centroids = cHyperCentroids[i];
      if(!centroids ||
         !centroids.FeedForward(prev))
         ReturnFalse;
      prev = centroids;
     }
//--- Curvatures
   for(int i = 0; i < cHyperCurvatures.Total(); i++)
     {
      curvatures = cHyperCurvatures[i];
      if(!curvatures ||
         !curvatures.FeedForward(prev))
         ReturnFalse;
      prev = curvatures;
     }
   if(!LogMap(NeuronOCL, centroids, curvatures, AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::calcInputGradients(CNeuronBaseOCL* prevLayer)
  {
   if(!prevLayer)
      ReturnFalse;
//---
   CObject *next = NULL;
   CNeuronBaseOCL *centroids = cHyperCentroids[-1];
   CNeuronBaseOCL *curvatures = cHyperCurvatures[-1];
//---
   if(!LogMapGrad(prevLayer, centroids, curvatures, AsObject()))
      ReturnFalse;
//--- Curvatures
   for(int i = cHyperCurvatures.Total() - 2; i >= 0; i--)
     {
      next = curvatures;
      curvatures = cHyperCurvatures[i];
      if(!curvatures ||
         !curvatures.CalcHiddenGradients(next))
         ReturnFalse;
     }
   CBufferFloat *temp = centroids.getGradient();
   if(centroids.Activation() != None)
      if(!DeActivation(centroids.getOutput(), temp, temp, centroids.Activation()))
         ReturnFalse;
   if(!centroids.SetGradient(centroids.getPrevOutput(), false) ||
      !centroids.CalcHiddenGradients(curvatures.AsObject()) ||
      !SumAndNormalize(temp, centroids.getGradient(), temp, iWindows, false, 0, 0, 0, 1) ||
      !centroids.SetGradient(temp, false)
     )
      ReturnFalse;
//--- Centroids
   for(int i = cHyperCentroids.Total() - 2; i >= 0; i--)
     {
      next = centroids;
      centroids = cHyperCentroids[i];
      if(!centroids ||
         !centroids.CalcHiddenGradients(next))
         ReturnFalse;
     }
//---
   temp = prevLayer.getGradient();
   if(prevLayer.Activation() != None)
      if(!DeActivation(prevLayer.getOutput(), temp, temp, prevLayer.Activation()))
         ReturnFalse;
   if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) ||
      !prevLayer.CalcHiddenGradients(centroids.AsObject()) ||
      !SumAndNormalize(temp, prevLayer.getGradient(), temp, iWindows, false, 0, 0, 0, 1) ||
      !prevLayer.SetGradient(temp, false)
     )
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
  {
   CNeuronBaseOCL *prev = NeuronOCL;
   CNeuronBaseOCL *centroids = NULL;
   CNeuronBaseOCL *curvatures = NULL;
//--- Centroids
   for(int i = 0; i < cHyperCentroids.Total(); i++)
     {
      centroids = cHyperCentroids[i];
      if(!centroids ||
         !centroids.UpdateInputWeights(prev))
         ReturnFalse;
      prev = centroids;
     }
//--- Curvatures
   for(int i = 0; i < cHyperCurvatures.Total(); i++)
     {
      curvatures = cHyperCurvatures[i];
      if(!curvatures ||
         !curvatures.UpdateInputWeights(prev))
         ReturnFalse;
      prev = curvatures;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronHyperboloids::SetOpenCL(COpenCLMy * obj)
  {
   if(!!OpenCL)
     {
      OpenCL.BufferFree(iProducts);
      OpenCL.BufferFree(iDistances);
      OpenCL.BufferFree(iNormes);
     }
//---
   CNeuronBaseOCL::SetOpenCL(obj);
   cHyperCentroids.SetOpenCL(OpenCL);
   cHyperCurvatures.SetOpenCL(OpenCL);
//---
   uint size = iCentroids * iUnits * sizeof(float);
   iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
   iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
   iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
   if(!cHyperCentroids.Save(file_handle))
      ReturnFalse;
   if(!cHyperCurvatures.Save(file_handle))
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(iWindows)) < INT_VALUE)
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE)
      ReturnFalse;
   if(FileWriteInteger(file_handle, int(iCentroids)) < INT_VALUE)
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHyperboloids::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
   if(!cHyperCentroids.Load(file_handle))
      ReturnFalse;
   if(!cHyperCurvatures.Load(file_handle))
      ReturnFalse;
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iWindows = (int)FileReadInteger(file_handle);
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iUnits = (int)FileReadInteger(file_handle);
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iCentroids = (int)FileReadInteger(file_handle);
   SetOpenCL(OpenCL);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronHyperboloids::TrainMode(bool flag)
  {
   CNeuronBaseOCL::TrainMode(flag);
   CNeuronBaseOCL *neuron = NULL;
//---
   for(int i = 0; i < cHyperCentroids.Total(); i++)
     {
      neuron = cHyperCentroids[i];
      if(!neuron)
         continue;
      neuron.TrainMode(bTrain);
     }
//---
   for(int i = 0; i < cHyperCurvatures.Total(); i++)
     {
      neuron = cHyperCurvatures[i];
      if(!neuron)
         continue;
      neuron.TrainMode(bTrain);
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronHypDiff::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch))
      ReturnFalse;
//---
   cLayers.Clear();
   cLayers.SetOpenCL(OpenCL);
   int layer = 0;
//--- Projection
   CNeuronHyperProjection *lorenz = new CNeuronHyperProjection();
   if(!lorenz ||
      !lorenz.Init(0, layer, OpenCL, window, units_count, optimization, iBatch) ||
      !cLayers.Add(lorenz))
     {
      DeleteObj(lorenz);
      ReturnFalse;
     }
   layer++;
//--- Encoder
   CNeuronRMAT *rmat = new CNeuronRMAT();
   if(!rmat ||
      !rmat.Init(0, layer, OpenCL, window + 1, window_key, units_count, heads, layers, optimization, iBatch) ||
      !cLayers.Add(rmat))
     {
      DeleteObj(rmat);
      ReturnFalse;
     }
   layer++;
//---
   CNeuronConvOCL *conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, layer, OpenCL, window + 1, window + 1, 2 * window, units_count, 1, optimization, iBatch) ||
      !cLayers.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   layer++;
   conv.SetActivationFunction(TANH);
//---
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, layer, OpenCL, 2 * window, 2 * window, 3, units_count, 1, optimization, iBatch) ||
      !cLayers.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   layer++;
//--- LogMap projecction
   CNeuronHyperboloids *logmap = new CNeuronHyperboloids();
   if(!logmap ||
      !logmap.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) ||
      !cLayers.Add(logmap))
     {
      DeleteObj(logmap);
      ReturnFalse;
     }
   layer++;
//--- Diffusion model
   CNeuronDiffusion *diff = new CNeuronDiffusion();
   if(!diff ||
      !diff.Init(0, layer, OpenCL, 3, window_key, heads, units_count * centroids, 2, layers, optimization, iBatch) ||
      !cLayers.Add(diff))
     {
      DeleteObj(diff);
      ReturnFalse;
     }
   layer++;
//--- Pooling
   CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling();
   if(!pooling ||
      !pooling.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) ||
      !cLayers.Add(pooling))
     {
      DeleteObj(pooling);
      ReturnFalse;
     }
   layer++;
//--- Resize to source size
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, layer, OpenCL, 3, 3, window, units_count, 1, optimization, iBatch) ||
      !cLayers.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
//---
   if(!SetOutput(conv.getOutput(), true) ||
      !SetGradient(conv.getGradient(), true))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPSformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint segments, float rho, ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if((window * units_count) % segments > 0)
      ReturnFalse;
   if(!CNeuronBaseSAMOCL::Init(numOutputs, myIndex, open_cl, window * units_count, rho, optimization_type, batch))
      ReturnFalse;
//---
   uint count = Neurons() / segments;
//---
   if(!acTranspose[0].Init(0, 0, OpenCL, segments, count, optimization, iBatch))
      ReturnFalse;
   acTranspose[0].SetActivationFunction(None);
//---
   if(!acPSBlocks[0].Init(0, 1, OpenCL, segments, segments, units_count / segments, 1, fRho, optimization, iBatch))
      ReturnFalse;
   for(int i = 0; i < 2; i++)
     {
      if(!acAttention[i].Init(0, i + 2, OpenCL, segments, segments, units_count / segments, 2, optimization, iBatch))
         ReturnFalse;
      if(!acPSBlocks[i + 1].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject()))
         ReturnFalse;
     }
//---
   if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch))
      ReturnFalse;
   if(!cResidual.SetGradient(acAttention[1].getGradient(), true))
      ReturnFalse;
   cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation());
//---
   if(!acTranspose[1].Init(0, 5, OpenCL, count, segments, optimization, iBatch))
      ReturnFalse;
   acTranspose[1].SetActivationFunction((ENUM_ACTIVATION)acPSBlocks[2].Activation());
//---
   if(!SetOutput(acTranspose[1].getOutput(), true) ||
      !SetGradient(acTranspose[1].getGradient(), true))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPSformer::feedForward(CNeuronBaseOCL* NeuronOCL)
  {
//--- Dimension Transformation
   if(!acTranspose[0].FeedForward(NeuronOCL))
      ReturnFalse;
//--- Segment Attention
   CObject* prev = acTranspose[0].AsObject();
   for(int i = 0; i < 2; i++)
     {
      if(!acPSBlocks[i].FeedForward(prev))
         ReturnFalse;
      if(!acAttention[i].FeedForward(acPSBlocks[i].AsObject()))
         ReturnFalse;
      prev = acAttention[i].AsObject();
     }
//--- Residual Add
   if(!SumAndNormalize(acTranspose[0].getOutput(), acAttention[1].getOutput(), cResidual.getOutput(), acAttention[1].GetWindow(), false, 0, 0, 0, 1))
      ReturnFalse;
//--- PS Block
   if(!acPSBlocks[2].FeedForward(cResidual.AsObject()))
      ReturnFalse;
//--- Inverse Transformation
   if(!acTranspose[1].FeedForward(acPSBlocks[2].AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPSformer::calcInputGradients(CNeuronBaseOCL* NeuronOCL)
  {
   if(!NeuronOCL)
      ReturnFalse;
//---
   if(!acPSBlocks[2].CalcHiddenGradients(acTranspose[1].AsObject()))
      ReturnFalse;
//---
   if(!cResidual.CalcHiddenGradients(acPSBlocks[2].AsObject()))
      ReturnFalse;
//---
   if(!acPSBlocks[1].CalcHiddenGradients(acAttention[1].AsObject()))
      ReturnFalse;
   if(!acAttention[0].CalcHiddenGradients(acPSBlocks[1].AsObject()))
      ReturnFalse;
   if(!acPSBlocks[0].CalcHiddenGradients(acAttention[0].AsObject()))
      ReturnFalse;
//---
   if(!acTranspose[0].CalcHiddenGradients(acPSBlocks[0].AsObject()))
      ReturnFalse;
   if(acTranspose[0].Activation() == None)
     {
      if(!SumAndNormalize(acTranspose[0].getGradient(), cResidual.getGradient(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1))
         ReturnFalse;
     }
   else
     {
      if(!DeActivation(acTranspose[0].getOutput(), cResidual.getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].Activation()) ||
         !SumAndNormalize(acTranspose[0].getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1))
         ReturnFalse;
     }
   if(!NeuronOCL.CalcHiddenGradients(acTranspose[0].AsObject()))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPSformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL)
  {
   if(!acPSBlocks[2].UpdateInputWeights(cResidual.AsObject()))
      ReturnFalse;
//---
   CObject* prev = acAttention[0].AsObject();
   for(int i = 1; i >= 0; i--)
     {
      if(!acAttention[i].UpdateInputWeights(acPSBlocks[i].AsObject()))
         ReturnFalse;
      if(!acPSBlocks[i].UpdateInputWeights(prev))
         ReturnFalse;
      prev = acTranspose[0].AsObject();
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronPSformer::SetOpenCL(COpenCLMy * obj)
  {
   CNeuronBaseSAMOCL::SetOpenCL(obj);
   for(int i = 0; i < 2; i++)
     {
      acTranspose[i].SetOpenCL(OpenCL);
      acPSBlocks[i].SetOpenCL(OpenCL);
      acAttention[i].SetOpenCL(OpenCL);
     }
   acPSBlocks[2].SetOpenCL(OpenCL);
   cResidual.SetOpenCL(OpenCL);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPSformer::Save(const int file_handle)
  {
   if(!CNeuronBaseSAMOCL::Save(file_handle))
      ReturnFalse;
   if(!acPSBlocks[0].Save(file_handle))
      ReturnFalse;
   for(int i = 0; i < 2; i++)
      if(!acTranspose[i].Save(file_handle) ||
         !acAttention[i].Save(file_handle))
         ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPSformer::Load(const int file_handle)
  {
   if(!CNeuronBaseSAMOCL::Load(file_handle))
      ReturnFalse;
//---
   if(!LoadInsideLayer(file_handle, acPSBlocks[0].AsObject()))
      ReturnFalse;
   for(int i = 0; i < 2; i++)
      if(!LoadInsideLayer(file_handle, acTranspose[i].AsObject()) ||
         !LoadInsideLayer(file_handle, acAttention[i].AsObject()))
         ReturnFalse;
//---
   for(int i = 1; i < 3; i++)
      if(!acPSBlocks[i].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject()))
         ReturnFalse;
//---
   if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch))
      ReturnFalse;
   if(!cResidual.SetGradient(acAttention[1].getGradient(), true))
      ReturnFalse;
   cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation());
//---
   if(!SetOutput(acTranspose[1].getOutput(), true) ||
      !SetGradient(acTranspose[1].getGradient(), true))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronAdaBN::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl,
                        uint window, uint window_out, uint units_count,
                        uint & bottlenecks[], uint top_k, uint variables,
                        ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch))
      ReturnFalse;
   int index = 0;
   if(!cGates.Init(0, index, OpenCL, window, units_count * variables, bottlenecks.Size(), top_k, optimization, iBatch))
      ReturnFalse;
//---
   index++;
   if(!cXProj.Init(0, index, OpenCL, window, window, window, units_count, optimization, iBatch))
      ReturnFalse;
   cXProj.SetActivationFunction(None);
//---
   cExperts.Clear();
   cExperts.SetOpenCL(OpenCL);
   CNeuronConvOCL *conv = NULL;
   CNeuronMultiWindowsConvOCL *mwconv = NULL;
   CNeuronTransposeRCDOCL *transp = NULL;
//---
   uint bn_size = 0;
   for(uint i = 0; i < bottlenecks.Size(); i++)
      bn_size += bottlenecks[i];
//---
   index++;
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, index, OpenCL, window, window, bn_size, units_count, variables, optimization, iBatch) ||
      !cExperts.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(LReLU);
//---
   index++;
   mwconv = new CNeuronMultiWindowsConvOCL();
   if(!mwconv ||
      !mwconv.Init(0, index, OpenCL, bottlenecks, window_out, units_count, variables, optimization, iBatch) ||
      !cExperts.Add(mwconv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   mwconv.SetActivationFunction(LReLU);
   transp = new CNeuronTransposeRCDOCL();
   index++;
   if(!transp ||
      !transp.Init(0, index, OpenCL, units_count * variables, bottlenecks.Size(), window_out, optimization, iBatch) ||
      !cExperts.Add(transp))
     {
      DeleteObj(transp);
      ReturnFalse;
     }
   transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
   index++;
   conv = new CNeuronConvOCL();
   if(!conv ||
      !conv.Init(0, index, OpenCL, window_out, window_out, window, units_count * variables, bottlenecks.Size(), optimization, iBatch) ||
      !cExperts.Add(conv))
     {
      DeleteObj(conv);
      ReturnFalse;
     }
   conv.SetActivationFunction(None);
   transp = new CNeuronTransposeRCDOCL();
   index++;
   if(!transp ||
      !transp.Init(0, index, OpenCL, bottlenecks.Size(), units_count * variables, window, optimization, iBatch) ||
      !cExperts.Add(transp))
     {
      DeleteObj(transp);
      ReturnFalse;
     }
   transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation());
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                uint units, uint window,
                                uint embed_dim1, uint period1, uint frame1,
                                uint embed_dim2, uint period2, uint frame2,
                                ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl,
                            (window + embed_dim1 + embed_dim2)*units, optimization_type, batch))
      ReturnFalse;
//---
   if(!caEmbeddings[0].Init(0, 0, OpenCL, embed_dim1 * period1, optimization, iBatch))
      ReturnFalse;
   if(!caEmbeddings[1].Init(0, 1, OpenCL, embed_dim2 * period2, optimization, iBatch))
      ReturnFalse;
//---
   iWindow = window;
   iUnits = units;
   aiEmbeddingDim[0] = embed_dim1;
   aiEmbeddingDim[1] = embed_dim2;
   aiFrames[0] = MathMax(1, frame1);
   aiFrames[1] = MathMax(1, frame2);
   aiPeriod[0] = period1;
   aiPeriod[1] = period2;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::ConcatByLabel(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
  {
   if(!OpenCL || !NeuronOCL || !SecondInput)
      ReturnFalse;
   if(NeuronOCL.Neurons() < int(iUnits * iWindow) ||
      SecondInput.Total() < (int)iUnits)
      ReturnFalse;
//---
   uint global_work_offset[3] = {0, 0, 0};
   uint global_work_size[3] = {iUnits,
                               MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]),
                               3
                              };
   uint kernel = def_k_ConcatByLabel;
   setBuffer(kernel, def_k_cbl_data, NeuronOCL.getOutputIndex())
   setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex())
   setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getOutputIndex())
   setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getOutputIndex())
   setBuffer(kernel, def_k_cbl_output, getOutputIndex())
   setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow)
   setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0])
   setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1])
   setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0])
   setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1])
   setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0])
   setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1])
//---
   kernelExecute(kernel, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!Output.BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::ConcatByLabelGrad(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
  {
   if(!OpenCL || !NeuronOCL || !SecondInput)
      ReturnFalse;
   if(NeuronOCL.Neurons() < int(iUnits * iWindow) ||
      SecondInput.Total() < (int)iUnits)
      ReturnFalse;
//---
   uint global_work_offset[3] = {0, 0, 0};
   uint global_work_size[3] = {MathMax(iUnits, aiPeriod[ArrayMaximum(aiPeriod)]),
                               MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]),
                               3
                              };
   uint kernel = def_k_ConcatByLabelGrad;
   setBuffer(kernel, def_k_cbl_data, NeuronOCL.getGradientIndex())
   setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex())
   setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getGradientIndex())
   setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getGradientIndex())
   setBuffer(kernel, def_k_cbl_output, getGradientIndex())
   setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow)
   setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0])
   setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1])
   setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0])
   setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1])
   setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0])
   setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1])
   setArgument(kernel, def_k_cbl_gr_units, (int)iUnits)
//---
   kernelExecute(kernel, global_work_offset, global_work_size)
#ifdef  _DEBUG
   if(!NeuronOCL.getGradient().BufferRead())
      ReturnFalse;
#endif
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)
  {
   if(bTrain)
      for(uint i = 0; i < caEmbeddings.Size(); i++)
         if(!caEmbeddings[i].FeedForward())
            ReturnFalse;
//---
   return ConcatByLabel(NeuronOCL, SecondInput);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL,
      CBufferFloat *SecondInput,
      CBufferFloat *SecondGradient,
      ENUM_ACTIVATION SecondActivation = None)
  {
   if(!ConcatByLabelGrad(NeuronOCL, SecondInput))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   for(uint i = 0; i < caEmbeddings.Size(); i++)
      if(!caEmbeddings[i].UpdateInputWeights())
         ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau)
  {
   if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
      ReturnFalse;
//---
   CNeuronTempEmbedding* Source = source;
   for(uint i = 0; i < caEmbeddings.Size(); i++)
      if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau))
         ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
//---
   for(uint i = 0; i < caEmbeddings.Size(); i++)
      if(!caEmbeddings[i].Save(file_handle))
         ReturnFalse;
//---
   if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE)
      ReturnFalse;
   if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE)
      ReturnFalse;
   for(uint i = 0; i < caEmbeddings.Size(); i++)
     {
      if(FileWriteInteger(file_handle, (int)aiEmbeddingDim[i]) < INT_VALUE)
         ReturnFalse;
      if(FileWriteInteger(file_handle, (int)aiFrames[i]) < INT_VALUE)
         ReturnFalse;
      if(FileWriteInteger(file_handle, (int)aiPeriod[i]) < INT_VALUE)
         ReturnFalse;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronTempEmbedding::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
//---
   for(uint i = 0; i < caEmbeddings.Size(); i++)
      if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject()))
         ReturnFalse;
//---
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iWindow = (uint)FileReadInteger(file_handle);
   if(FileIsEnding(file_handle))
      ReturnFalse;
   iUnits = (uint)FileReadInteger(file_handle);
   for(uint i = 0; i < caEmbeddings.Size(); i++)
     {
      if(FileIsEnding(file_handle))
         ReturnFalse;
      aiEmbeddingDim[i] = (uint)FileReadInteger(file_handle);
      if(FileIsEnding(file_handle))
         ReturnFalse;
      aiFrames[i] = (uint)FileReadInteger(file_handle);
      if(FileIsEnding(file_handle))
         ReturnFalse;
      aiPeriod[i] = (uint)FileReadInteger(file_handle);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronTempEmbedding::SetOpenCL(COpenCLMy *obj)
  {
   CNeuronBaseOCL::SetOpenCL(obj);
//---
   for(uint i = 0; i < caEmbeddings.Size(); i++)
      caEmbeddings[i].SetOpenCL(OpenCL);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl,
                                       uint dimension, uint units, uint scales, uint bottleneck,
                                       ENUM_OPTIMIZATION optimization_type, uint batch)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units * scales, optimization_type, batch))
      ReturnFalse;
//---
   cExcitation.Clear();
   cExcitation.SetOpenCL(OpenCL);
   CNeuronSpikeConvBlock*  conv = NULL;
   CNeuronTransposeOCL*    transp = NULL;
   CNeuronSoftMaxOCL*      softmax = NULL;
   uint index = 0;
//--- [Scale, Unit, Dimension]
   transp = new CNeuronTransposeOCL();
   if(!transp ||
      !transp.Init(0, index, OpenCL, scales, units * dimension, optimization, iBatch) ||
      !cExcitation.Add(transp))
     {
      DeleteObj(transp)
      ReturnFalse;
     }
   index++;
//--- [Unit, Dimension, Scale]
   conv = new CNeuronSpikeConvBlock();
   if(!conv ||
      !conv.Init(0, index, OpenCL, scales, scales, 1, dimension * units, 1, optimization, iBatch) ||
      !cExcitation.Add(conv))
     {
      DeleteObj(conv)
      ReturnFalse;
     }
   index++;
//--- [Unit, Dimension]
   conv = new CNeuronSpikeConvBlock();
   if(!conv ||
      !conv.Init(0, index, OpenCL, dimension, dimension, bottleneck, units, 1, optimization, iBatch) ||
      !cExcitation.Add(conv))
     {
      DeleteObj(conv)
      ReturnFalse;
     }
   index++;
//--- [Unit, Bottleneck]
   conv = new CNeuronSpikeConvBlock();
   if(!conv ||
      !conv.Init(0, index, OpenCL, bottleneck, bottleneck, scales * dimension, units, 1, optimization, iBatch) ||
      !cExcitation.Add(conv))
     {
      DeleteObj(conv)
      ReturnFalse;
     }
   index++;
//--- [Unit, Dimension, Scale]
   softmax = new CNeuronSoftMaxOCL();
   if(!softmax ||
      !softmax.Init(0, index, OpenCL, conv.Neurons(), optimization, iBatch) ||
      !cExcitation.Add(softmax))
     {
      DeleteObj(softmax)
      ReturnFalse;
     }
   softmax.SetHeads(dimension * units);
   index++;
//--- [Unit, Dimension, Scale]
   transp = new CNeuronTransposeOCL();
   if(!transp ||
      !transp.Init(0, index, OpenCL, units * dimension, scales, optimization, iBatch) ||
      !cExcitation.Add(transp))
     {
      DeleteObj(transp)
      ReturnFalse;
     }
   index++;
//--- [Scale, Unit, Dimension]
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   CNeuronBaseOCL* prev = NeuronOCL;
   CNeuronBaseOCL* curr = NULL;
   for(int i = 0; i < cExcitation.Total(); i++)
     {
      curr = cExcitation[i];
      if(!curr ||
         !curr.FeedForward(prev))
         ReturnFalse;
      prev = curr;
     }
   if(!ElementMult(NeuronOCL.getOutput(), prev.getOutput(), Output))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
  {
   CNeuronBaseOCL* next = cExcitation[-1];
   CNeuronBaseOCL* curr = NULL;
//---
   if(!NeuronOCL || !next)
      ReturnFalse;
   if(!ElementMultGrad(NeuronOCL.getOutput(), PrevOutput,
                       next.getOutput(), next.getGradient(),
                       Gradient, NeuronOCL.Activation(), next.Activation()))
      ReturnFalse;
//---
   for(int i = cExcitation.Total() - 2; i >= 0; i--)
     {
      curr = cExcitation[i];
      if(!curr ||
         !curr.CalcHiddenGradients(next))
         ReturnFalse;
      next = curr;
     }
//---
   if(!NeuronOCL.CalcHiddenGradients(next) ||
      !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput,
                       NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   CNeuronBaseOCL* prev = NeuronOCL;
   CNeuronBaseOCL* curr = NULL;
   for(int i = 0; i < cExcitation.Total(); i++)
     {
      curr = cExcitation[i];
      if(!curr ||
         !curr.UpdateInputWeights(prev))
         ReturnFalse;
      prev = curr;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::WeightsUpdate(CNeuronBaseOCL *source, float tau)
  {
   if(!CNeuronBaseOCL::WeightsUpdate(source, tau))
      ReturnFalse;
//---
   CNeuronMultiScaleExcitation* Source = source;
   if(!cExcitation.WeightsUpdate(Source.cExcitation.AsObject(), tau))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      ReturnFalse;
   if(!cExcitation.Save(file_handle))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      ReturnFalse;
   if(!cExcitation.Load(file_handle))
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronMultiScaleExcitation::Clear(void)
  {
   if(!CNeuronBaseOCL::Clear())
      ReturnFalse;
   if(!cExcitation.ClearStates())
      ReturnFalse;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronMultiScaleExcitation::SetOpenCL(COpenCLMy *obj)
  {
   CNeuronBaseOCL::SetOpenCL(obj);
   cExcitation.SetOpenCL(OpenCL);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNeuronMultiScaleExcitation::TrainMode(bool flag)
  {
   CNeuronBaseOCL::TrainMode(flag);
   cExcitation.TrainMode(bTrain);
  }
