17 #property copyright "Copyright 2019, DNG"
18 #property link "https://www.mql5.com"
19 #property version "1.00"
23 #include <Arrays\ArrayDouble.mqh>
24 #include <Arrays\ArrayInt.mqh>
25 #include <Arrays\ArrayObj.mqh>
26 #include <OpenCL\OpenCL.mqh>
52 #define defArrayConnects 0x7782
53 #define defLayer 0x7787
54 #define defArrayLayer 0x7788
56 #define defConnect 0x7781
61 #define defNeuronBase 0x7783
62 #define defNeuron 0x7784
63 #define defNeuronConv 0x7785
64 #define defNeuronProof 0x7786
65 #define defNeuronLSTM 0x7791
66 #define defBufferDouble 0x7882
71 #define defNeuronBaseOCL 0x7883
72 #define defNeuronConvOCL 0x7885
73 #define defNeuronProofOCL 0x7886
74 #define defNeuronAttentionOCL 0x7887
75 //#define defNeuronLSTMOCL 0x7884
90 #define def_k_FeedForward 0
91 #define def_k_ff_matrix_w 0
92 #define def_k_ff_matrix_i 1
93 #define def_k_ff_matrix_o 2
94 #define def_k_ff_inputs 3
95 #define def_k_ff_activation 4
96 #define def_k_CalcOutputGradient 1
102 #define def_k_cog_matrix_t 0
103 #define def_k_cog_matrix_o 1
104 #define def_k_cog_matrix_ig 2
105 #define def_k_cog_activation 3
107 #define def_k_CalcHiddenGradient 2
108 #define def_k_chg_matrix_w 0
109 #define def_k_chg_matrix_g 1
110 #define def_k_chg_matrix_o 2
111 #define def_k_chg_matrix_ig 3
112 #define def_k_chg_outputs 4
113 #define def_k_chg_activation 5
114 #define def_k_UpdateWeightsMomentum 3
121 #define def_k_uwm_matrix_w 0
122 #define def_k_uwm_matrix_g 1
123 #define def_k_uwm_matrix_i 2
124 #define def_k_uwm_matrix_dw 3
125 #define def_k_uwm_inputs 4
126 #define def_k_uwm_learning_rates 5
127 #define def_k_uwm_momentum 6
129 #define def_k_UpdateWeightsAdam 4
130 #define def_k_uwa_matrix_w 0
131 #define def_k_uwa_matrix_g 1
132 #define def_k_uwa_matrix_i 2
133 #define def_k_uwa_matrix_m 3
134 #define def_k_uwa_matrix_v 4
135 #define def_k_uwa_inputs 5
136 #define def_k_uwa_l 6
137 #define def_k_uwa_b1 7
138 #define def_k_uwa_b2 8
142 #define def_k_FeedForwardProof 5
149 #define def_k_ffp_matrix_i 0
150 #define def_k_ffp_matrix_o 1
151 #define def_k_ffp_inputs 2
152 #define def_k_ffp_window 3
153 #define def_k_ffp_step 4
156 #define def_k_CalcInputGradientProof 6
160 #define def_k_cigp_matrix_i 0
161 #define def_k_cigp_matrix_g 1
162 #define def_k_cigp_matrix_o 2
163 #define def_k_cigp_matrix_ig 3
164 #define def_k_cigp_outputs 4
165 #define def_k_cigp_window 5
166 #define def_k_cigp_step 6
170 #define def_k_FeedForwardConv 7
177 #define def_k_ffc_matrix_w 0
178 #define def_k_ffc_matrix_i 1
179 #define def_k_ffc_matrix_o 2
180 #define def_k_ffc_inputs 3
181 #define def_k_ffc_step 4
182 #define def_k_ffc_window_in 5
183 #define def_k_ffс_window_out 6
184 #define def_k_ffc_activation 7
187 #define def_k_CalcHiddenGradientConv 8
191 #define def_k_chgc_matrix_w 0
192 #define def_k_chgc_matrix_g 1
193 #define def_k_chgc_matrix_o 2
194 #define def_k_chgc_matrix_ig 3
195 #define def_k_chgc_outputs 4
196 #define def_k_chgc_step 5
197 #define def_k_chgc_window_in 6
198 #define def_k_chgc_window_out 7
199 #define def_k_chgc_activation 8
202 #define def_k_UpdateWeightsConvMomentum 9
206 #define def_k_uwcm_matrix_w 0
207 #define def_k_uwcm_matrix_g 1
208 #define def_k_uwcm_matrix_i 2
209 #define def_k_uwcm_matrix_dw 3
210 #define def_k_uwcm_inputs 4
211 #define def_k_uwcm_learning_rates 5
212 #define def_k_uwcm_momentum 6
213 #define def_k_uwcm_window_in 7
214 #define def_k_uwcm_window_out 8
215 #define def_k_uwcm_step 9
217 #define def_k_UpdateWeightsConvAdam 10
218 #define def_k_uwca_matrix_w 0
219 #define def_k_uwca_matrix_g 1
220 #define def_k_uwca_matrix_i 2
221 #define def_k_uwca_matrix_m 3
222 #define def_k_uwca_matrix_v 4
223 #define def_k_uwca_inputs 5
224 #define def_k_uwca_l 6
225 #define def_k_uwca_b1 7
226 #define def_k_uwca_b2 8
227 #define def_k_uwca_window_in 9
228 #define def_k_uwca_window_out 10
229 #define def_k_uwca_step 11
233 #define def_k_AttentionScore 11
242 #define def_k_as_querys 0
243 #define def_k_as_keys 1
244 #define def_k_as_score 2
245 #define def_k_as_dimension 3
247 #define def_k_AttentionOut 12
248 #define def_k_aout_scores 0
249 #define def_k_aout_values 1
250 #define def_k_aout_inputs 2
251 #define def_k_aout_out 3
253 #define def_k_MatrixSum 13
254 #define def_k_sum_matrix1 0
255 #define def_k_sum_matrix2 1
256 #define def_k_sum_matrix_out 2
257 #define def_k_sum_dimension 3
258 #define def_k_sum_multiplyer 4
261 #define def_k_AttentionGradients 14
266 #define def_k_ag_querys 0
267 #define def_k_ag_querys_g 1
268 #define def_k_ag_keys 2
269 #define def_k_ag_keys_g 3
270 #define def_k_ag_values 4
271 #define def_k_ag_values_g 5
272 #define def_k_ag_scores 6
273 #define def_k_ag_gradient 7
277 #define def_k_Normilize 15
282 #define def_k_norm_buffer 0
283 #define def_k_norm_dimension 1
285 #define def_k_NormilizeWeights 16
286 //+------------------------------------------------------------------+
291 #resource "NeuroNet.cl" as string cl_program
329 virtual bool Save(
int const file_handle);
330 virtual bool Load(
int const file_handle);
338 if(file_handle==INVALID_HANDLE)
341 if(FileWriteDouble(file_handle,
weight)<=0)
345 if(FileWriteDouble(file_handle,
mt)<=0)
347 if(FileWriteDouble(file_handle,
vt)<=0)
357 if(file_handle==INVALID_HANDLE)
360 weight=FileReadDouble(file_handle);
362 mt=FileReadDouble(file_handle);
363 vt=FileReadDouble(file_handle);
387 if(index<0 || index>=m_data_max)
390 double weigh=(MathRand()+1)/32768.0-0.5;
394 if(!CheckPointer(m_data[index])!=POINTER_INVALID)
452 virtual bool Save(
int const file_handle);
453 virtual bool Load(
int const file_handle)
498 for(uint c=0; c<numOutputs; c++)
517 virtual bool feedForward(
CLayer *prevLayer);
518 virtual bool calcHiddenGradients(
CLayer *&nextLayer);
519 virtual bool updateInputWeights(
CLayer *&prevLayer);
532 bool CNeuron::updateInputWeights(
CLayer *&prevLayer)
534 if(CheckPointer(prevLayer)==POINTER_INVALID)
537 double lt=
eta*sqrt(1-pow(
b2,
t))/(1-pow(
b1,
t));
538 int total=prevLayer.Total();
539 for(
int n=0; n<total && !IsStopped(); n++)
541 CNeuron *neuron= prevLayer.At(n);
543 if(CheckPointer(con)==POINTER_INVALID)
563 int total=nextLayer.Total()-1;
564 for(
int n=0; n<total; n++)
567 if(CheckPointer(con)==POINTER_INVALID)
572 CNeuron *neuron=nextLayer.At(n);
582 bool CNeuron::calcHiddenGradients(
CLayer *&nextLayer)
592 double delta=(targetVal>1 ? 1 : targetVal<-1 ? -1 : targetVal)-
outputVal;
599 bool CNeuron::feedForward(
CLayer *prevLayer)
601 if(CheckPointer(prevLayer)==POINTER_INVALID || prevLayer.
Type()!=
defLayer)
606 int total=prevLayer.Total();
607 for(
int n=0; n<total && !IsStopped(); n++)
614 if(CheckPointer(con)==POINTER_INVALID)
633 int AddBufferFromArray(T &data[],
const uint data_array_offset,
const uint data_array_count,
const uint flags);
647 public:
CLayer(uint outputs=0,
int handle=INVALID_HANDLE,
COpenCLMy *OpenCL=NULL);
655 virtual bool Load(
const int file_handle);
662 if(index>=m_data_max)
674 if(CheckPointer(temp)==POINTER_INVALID || !temp.
Init(iOutputs,index,
SGD))
680 int type=FileReadInteger(iFileHandle);
685 if(CheckPointer(temp)==POINTER_INVALID)
687 result=temp.
Init(iOutputs,index,
ADAM);
691 if(CheckPointer(temp_p)==POINTER_INVALID)
693 if(temp_p.
Init(iOutputs,index,1,1,1,
ADAM))
701 if(CheckPointer(temp_p)==POINTER_INVALID)
703 if(temp_p.
Init(iOutputs,index,1,1,1,
ADAM))
711 if(CheckPointer(temp_p)==POINTER_INVALID)
713 if(temp_p.
Init(iOutputs,index,1,1,1,
ADAM))
720 if(CheckPointer(OpenCL)==POINTER_INVALID)
723 if(CheckPointer(temp_ocl)==POINTER_INVALID)
725 if(temp_ocl.
Init(iOutputs,index,OpenCL,1,
ADAM))
727 m_data[index]=temp_ocl;
732 if(CheckPointer(OpenCL)==POINTER_INVALID)
735 if(CheckPointer(temp_con_ocl)==POINTER_INVALID)
737 if(temp_con_ocl.
Init(iOutputs,index,OpenCL,1,1,1,1,
ADAM))
739 m_data[index]=temp_con_ocl;
744 if(CheckPointer(OpenCL)==POINTER_INVALID)
747 if(CheckPointer(temp_at_ocl)==POINTER_INVALID)
749 if(temp_at_ocl.
Init(iOutputs,index,OpenCL,1,1,
ADAM))
751 m_data[index]=temp_at_ocl;
788 if(!CheckPointer(layer)!=POINTER_INVALID)
791 if(!layer.Reserve(neurons+1))
793 for(uint i=0; i<=neurons; i++)
820 virtual bool Init(uint numOutputs,uint myIndex,
int window,
int step,
int units_count,
ENUM_OPTIMIZATION optimization_type);
827 virtual bool Save(
int const file_handle);
828 virtual bool Load(
int const file_handle);
853 virtual bool Save(
int const file_handle);
854 virtual bool Load(
int const file_handle);
863 if(CheckPointer(SourceObject)==POINTER_INVALID)
868 switch(SourceObject.Type())
891 if(CheckPointer(SourceObject)==POINTER_INVALID)
896 switch(SourceObject.Type())
920 if(CheckPointer(prevLayer)==POINTER_INVALID)
923 int total=prevLayer.Total()-
iWindow+1;
927 for(
int i=0; (i<total && result); i+=
iStep)
930 for(
int j=0; (j<
iWindow && result); j++)
932 temp=prevLayer.At(i+j);
934 if(CheckPointer(temp)==POINTER_INVALID || CheckPointer(con)==POINTER_INVALID)
940 if(CheckPointer(temp)==POINTER_INVALID)
963 if(CheckPointer(TargetObject)==POINTER_INVALID)
968 switch(TargetObject.Type())
1010 if(CheckPointer(nextLayer)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID ||
OutputLayer.Total()<=0)
1016 for(
int i=0; i<total; i++)
1019 if(CheckPointer(temp)==POINTER_INVALID)
1039 if(CheckPointer(prevLayer)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID)
1043 double lt=
eta*sqrt(1-pow(
b2,
t))/(1-pow(
b1,
t));
1044 for(
int n=0; n<
iWindow && !IsStopped(); n++)
1047 if(CheckPointer(con)==POINTER_INVALID)
1052 for(
int i=0; i<total_i; i++)
1054 prev=prevLayer.At(n*
iStep+i);
1056 if(CheckPointer(prev)==POINTER_INVALID || CheckPointer(out)==POINTER_INVALID)
1065 con.
vt=
b2*con.
vt+(1-
b2)*pow(delta,2)+0.00000001;
1086 for(
int i=0; i<units_count; i++)
1113 if(CheckPointer(prevLayer)==POINTER_INVALID)
1116 int total=prevLayer.Total()-
iWindow+1;
1118 for(
int i=0; i<=total; i+=
iStep)
1123 temp=prevLayer.At(i+j);
1124 if(CheckPointer(temp)==POINTER_INVALID)
1129 if(CheckPointer(temp)==POINTER_INVALID)
1141 if(CheckPointer(nextLayer)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID ||
OutputLayer.Total()<=0)
1147 for(
int i=0; i<total; i++)
1150 if(CheckPointer(temp)==POINTER_INVALID)
1162 if(CheckPointer(prevLayer)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID || CheckPointer(prevLayer.At(0))==POINTER_INVALID)
1168 if(CheckPointer(temp)==POINTER_INVALID)
1171 if(CheckPointer(prevLayer)==POINTER_INVALID)
1176 int total=prevLayer.Total();
1177 for(
int i=0; i<total; i++)
1179 prevNeuron=prevLayer.At(i);
1180 if(CheckPointer(prevNeuron)==POINTER_INVALID)
1182 double prev_gradient=0;
1186 for(
int out=(
int)fmax(0,start); out<(int)fmin(
OutputLayer.Total(),stop); out++)
1189 if(CheckPointer(outputNeuron)==POINTER_INVALID)
1203 if(CheckPointer(prevNeuron)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID)
1213 double prev_gradient=0;
1217 for(
int out=(
int)fmax(0,start); out<(int)fmin(
OutputLayer.Total(),stop); out++)
1220 if(CheckPointer(outputNeuron)==POINTER_INVALID)
1233 if(CheckPointer(prevLayer)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID)
1239 if(CheckPointer(temp)==POINTER_INVALID)
1242 if(CheckPointer(prevLayer)==POINTER_INVALID)
1248 int total=prevLayer.Total();
1249 for(
int i=0; i<total; i++)
1251 prevNeuron=prevLayer.At(i);
1252 if(CheckPointer(prevNeuron)==POINTER_INVALID)
1254 double prev_gradient=0;
1258 for(
int out=(
int)fmax(0,start); out<(int)fmin(
OutputLayer.Total(),stop); out++)
1263 if(CheckPointer(outputNeuron)==POINTER_INVALID || CheckPointer(con)==POINTER_INVALID)
1277 if(CheckPointer(prevNeuron)==POINTER_INVALID || CheckPointer(
OutputLayer)==POINTER_INVALID)
1288 double prev_gradient=0;
1292 for(
int out=(
int)fmax(0,start); out<(int)fmin(
OutputLayer.Total(),stop); out++)
1297 if(CheckPointer(outputNeuron)==POINTER_INVALID || CheckPointer(con)==POINTER_INVALID)
1310 if(file_handle==INVALID_HANDLE)
1312 if(FileWriteInteger(file_handle,
Type())<INT_VALUE)
1315 if(FileWriteInteger(file_handle,(
int)
activation,INT_VALUE)<INT_VALUE)
1318 if(FileWriteInteger(file_handle,(
int)
optimization,INT_VALUE)<INT_VALUE)
1321 if(FileWriteInteger(file_handle,
t,INT_VALUE)<INT_VALUE)
1364 public:
CNet(CArrayObj *Description);
~CNet(
void);
1368 void backProp(CArrayDouble *targetVals);
1371 bool Save(
string file_name,
double error,
double undefine,
double forecast, datetime time,
bool common=
true);
1373 bool Load(
string file_name,
double &error,
double &undefine,
double &forecast, datetime &time,
bool common=
true);
1382 double recentAverageError;
1393 if(CheckPointer(Description)==POINTER_INVALID)
1396 int total=Description.Total();
1401 if(CheckPointer(layers)==POINTER_INVALID)
1411 next=Description.At(1);
1415 if(CheckPointer(opencl)!=POINTER_INVALID && !opencl.Initialize(cl_program,
true))
1420 if(CheckPointer(opencl)!=POINTER_INVALID)
1424 for(
int i=0; i<total; i++)
1427 desc=Description.At(i);
1430 next=Description.At(i+1);
1431 if(CheckPointer(next)==POINTER_INVALID)
1437 temp=
new CLayer(outputs);
1439 if(CheckPointer(opencl)!=POINTER_INVALID)
1449 if(CheckPointer(neuron_ocl)==POINTER_INVALID)
1461 if(!temp.Add(neuron_ocl))
1471 if(CheckPointer(neuron_conv_ocl)==POINTER_INVALID)
1478 delete neuron_conv_ocl;
1483 if(!temp.Add(neuron_conv_ocl))
1485 delete neuron_conv_ocl;
1489 neuron_conv_ocl=NULL;
1493 if(CheckPointer(neuron_attention_ocl)==POINTER_INVALID)
1500 delete neuron_attention_ocl;
1505 if(!temp.Add(neuron_attention_ocl))
1507 delete neuron_attention_ocl;
1511 neuron_attention_ocl=NULL;
1519 for(
int n=0; n<neurons; n++)
1525 if(CheckPointer(neuron)==POINTER_INVALID)
1536 if(CheckPointer(neuron_p)==POINTER_INVALID)
1542 if(CheckPointer(prev)!=POINTER_INVALID)
1546 temp_count=(int)((prev.count-desc.
window)%desc.
step);
1547 output_count=(int)((prev.count-desc.
window-temp_count)/desc.
step+(temp_count==0 ? 1 : 2));
1552 temp_count=(int)((output_count-desc.
window)%desc.
step);
1553 output_count=(int)((output_count-desc.
window-temp_count)/desc.
step+(temp_count==0 ? 1 : 2));
1561 if(CheckPointer(neuron_p)==POINTER_INVALID)
1567 if(CheckPointer(prev)!=POINTER_INVALID)
1571 temp_count=(int)((prev.count-desc.
window)%desc.
step);
1572 output_count=(int)((prev.count-desc.
window-temp_count)/desc.
step+(temp_count==0 ? 1 : 2));
1577 temp_count=(int)((output_count-desc.
window)%desc.
step);
1578 output_count=(int)((output_count-desc.
window-temp_count)/desc.
step+(temp_count==0 ? 1 : 2));
1586 if(CheckPointer(neuron_p)==POINTER_INVALID)
1592 output_count=(next!=NULL ? next.window : desc.
step);
1597 if(!temp.Add(neuron))
1605 if(!layers.Add(temp))
1613 if(CheckPointer(opencl)==POINTER_INVALID)
1616 opencl.SetKernelsCount(17);
1642 if(CheckPointer(layers)==POINTER_INVALID || CheckPointer(inputVals)==POINTER_INVALID || layers.Total()<=1)
1646 CLayer *current=layers.At(0);
1647 int total=MathMin(current.Total(),inputVals.Total());
1649 if(CheckPointer(opencl)==POINTER_INVALID)
1651 for(
int i=0; i<total; i++)
1653 neuron=current.At(i);
1654 if(CheckPointer(neuron)==POINTER_INVALID)
1656 neuron.
setOutputVal(inputVals.At(i)+(i%2==0 ? sin(i) : cos(i)) );
1663 int total_data=inputVals.Total();
1664 if(ArrayResize(array,total_data)<0)
1666 for(
int d=0;d<total_data;d++)
1667 array[d]=inputVals.At(d)+(d%2==0 ? sin(d) : cos(d));
1668 if(!opencl.BufferWrite(neuron_ocl.
getOutputIndex(),array,0,0,total_data))
1673 for(
int l=1; l<layers.Total(); l++)
1676 current=layers.At(l);
1677 if(CheckPointer(current)==POINTER_INVALID)
1680 if(CheckPointer(opencl)!=POINTER_INVALID)
1688 total=current.Total();
1692 for(
int n=0; n<total; n++)
1694 neuron=current.At(n);
1695 if(CheckPointer(neuron)==POINTER_INVALID)
1709 if(CheckPointer(temp_l)==POINTER_INVALID)
1712 for(
int p=0; p<previous.Total(); p++)
1714 proof=previous.At(p);
1715 if(CheckPointer(proof)==POINTER_INVALID)
1726 temp_l.FreeMode(
false);
1732 temp=previous.At(n);
1733 if(CheckPointer(temp)==POINTER_INVALID)
1747 if(CheckPointer(targetVals)==POINTER_INVALID || CheckPointer(layers)==POINTER_INVALID)
1749 if(CheckPointer(opencl)!=POINTER_INVALID)
1755 CLayer *outputLayer=layers.At(layers.Total()-1);
1756 if(CheckPointer(outputLayer)==POINTER_INVALID)
1760 int total=outputLayer.Total()-1;
1761 for(
int n=0; n<total && !IsStopped(); n++)
1763 CNeuron *neuron=outputLayer.At(n);
1764 double target=targetVals.At(n);
1765 double delta=(target>1 ? 1 : target<-1 ? -1 : target)-neuron.
getOutputVal();
1770 error = sqrt(error);
1776 for(
int layerNum=layers.Total()-2; layerNum>0; layerNum--)
1778 CLayer *hiddenLayer=layers.At(layerNum);
1779 CLayer *nextLayer=layers.At(layerNum+1);
1780 total=hiddenLayer.Total();
1781 for(
int n=0; n<total && !IsStopped(); ++n)
1783 neuron=hiddenLayer.At(n);
1793 for(
int i=0; i<nextLayer.Total(); i++)
1795 temp=nextLayer.At(i);
1802 temp=nextLayer.At(n);
1807 for(
int layerNum=layers.Total()-1; layerNum>0; layerNum--)
1809 CLayer *layer=layers.At(layerNum);
1810 CLayer *prevLayer=layers.At(layerNum-1);
1811 total=layer.Total()-(layer.At(0).
Type()==
defNeuron ? 1 : 0);
1813 for(
int n=0; n<total && !IsStopped(); n++)
1816 if(CheckPointer(neuron)==POINTER_INVALID)
1820 switch(prevLayer.At(0).
Type())
1831 for(n_conv=0; n_conv<prevLayer.Total(); n_conv++)
1833 temp=prevLayer.At(n_conv);
1839 temp=prevLayer.At(n);
1855 if(CheckPointer(targetVals)==POINTER_INVALID || CheckPointer(layers)==POINTER_INVALID || CheckPointer(opencl)==POINTER_INVALID)
1857 CLayer *currentLayer=layers.At(layers.Total()-1);
1858 if(CheckPointer(currentLayer)==POINTER_INVALID)
1862 int total=targetVals.Total();
1867 for(
int n=0; n<total && !IsStopped(); n++)
1869 double target=targetVals.At(n);
1870 double delta=(target==0 ? 0 : (target>1 ? 1 : target<-1 ? -1 : target)-result[n]);
1871 error+=MathPow(delta,2);
1874 error = sqrt(error);
1881 total=layers.Total();
1882 for(
int layerNum=total-2; layerNum>0; layerNum--)
1884 CLayer *nextLayer=currentLayer;
1885 currentLayer=layers.At(layerNum);
1886 neuron=currentLayer.At(0);
1891 CLayer *prevLayer=layers.At(total-1);
1892 for(
int layerNum=total-1; layerNum>0; layerNum--)
1894 currentLayer=prevLayer;
1895 prevLayer=layers.At(layerNum-1);
1896 neuron=currentLayer.At(0);
1906 if(CheckPointer(resultVals)==POINTER_INVALID)
1908 resultVals=
new CArrayDouble();
1909 if(CheckPointer(resultVals)==POINTER_INVALID)
1914 if(CheckPointer(layers)==POINTER_INVALID || layers.Total()<=0)
1917 CLayer *output=layers.At(layers.Total()-1);
1918 if(CheckPointer(output)==POINTER_INVALID)
1929 int total=output.Total();
1933 for(
int i=0; i<total; i++)
1935 neuron=output.At(i);
1936 if(CheckPointer(neuron)==POINTER_INVALID)
1945 for(
int ii=0; ii<temp.Total(); ii++)
1948 if(CheckPointer(neuron)==POINTER_INVALID)
1957 bool CNet::Save(
string file_name,
double error,
double undefine,
double forecast,datetime time,
bool common=
true)
1959 if(MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD) || MQLInfoInteger(MQL_OPTIMIZATION))
1964 int handle=FileOpen(file_name,(common ? FILE_COMMON : 0)|FILE_BIN|FILE_WRITE);
1965 if(handle==INVALID_HANDLE)
1968 if(FileWriteDouble(handle,error)<=0 || FileWriteDouble(handle,undefine)<=0 || FileWriteDouble(handle,forecast)<=0 || FileWriteLong(handle,(
long)time)<=0)
1973 bool result=layers.Save(handle);
1982 bool CNet::Load(
string file_name,
double &error,
double &undefine,
double &forecast,datetime &time,
bool common=
true)
1984 if(MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD) || MQLInfoInteger(MQL_OPTIMIZATION))
1991 int handle=FileOpen(file_name,(common ? FILE_COMMON : 0)|FILE_BIN|FILE_READ);
1992 if(handle==INVALID_HANDLE)
1995 error=FileReadDouble(handle);
1996 undefine=FileReadDouble(handle);
1997 forecast=FileReadDouble(handle);
1998 time=(datetime)FileReadLong(handle);
2000 if(CheckPointer(layers)!=POINTER_INVALID)
2006 if(CheckPointer(opencl)==POINTER_INVALID)
2009 if(CheckPointer(opencl)!=POINTER_INVALID && !opencl.Initialize(cl_program,
true))
2014 opencl.SetKernelsCount(17);
2036 long temp=FileReadLong(handle);
2040 if(FileReadInteger(handle,INT_VALUE)!=layers.
Type())
2052 num=FileReadInteger(handle,INT_VALUE);
2056 for(i=0; i<num; i++)
2060 if(!Layer.
Load(handle))
2062 if(!layers.Add(Layer))
2068 return (layers.Total()==num);
2077 if(FileWriteInteger(file_handle,
iWindow,INT_VALUE)<INT_VALUE)
2079 if(FileWriteInteger(file_handle,
iStep,INT_VALUE)<INT_VALUE)
2091 iWindow=FileReadInteger(file_handle,INT_VALUE);
2092 iStep=FileReadInteger(file_handle,INT_VALUE);
2103 if(FileWriteDouble(file_handle,
param)<8)
2115 param=FileReadDouble(file_handle);
2147 virtual bool Init(uint numOutputs,uint myIndex,
int window,
int step,
int units_count,
ENUM_OPTIMIZATION optimization_type);
2154 virtual bool Save(
int const file_handle);
2155 virtual bool Load(
int const file_handle);
2167 Memory =
new CArrayDouble();
2169 Input =
new CArrayDouble();
2177 if(CheckPointer(
ForgetGate)!=POINTER_INVALID)
2179 if(CheckPointer(
InputGate)!=POINTER_INVALID)
2181 if(CheckPointer(
OutputGate)!=POINTER_INVALID)
2183 if(CheckPointer(
NewContent)!=POINTER_INVALID)
2185 if(CheckPointer(
Memory)!=POINTER_INVALID)
2187 if(CheckPointer(
PrevMemory)!=POINTER_INVALID)
2189 if(CheckPointer(
Input)!=POINTER_INVALID)
2202 if(!
CNeuronProof::Init(numOutputs,myIndex,window,step,units_count,optimization_type))
2212 if(!
Memory.Reserve(units_count))
2217 for(
int i=0; i<units_count; i++)
2234 if(CheckPointer(layer)==POINTER_INVALID)
2236 layer=
new CLayer(numOutputs);
2237 if(CheckPointer(layer)==POINTER_INVALID)
2242 if(!layer.Reserve(numUnits))
2246 for(
int i=0; i<numUnits; i++)
2249 if(CheckPointer(temp)==POINTER_INVALID)
2251 if(!temp.
Init(numOutputs+1,i,optimization_type))
2253 if(!layer.Add(temp))
2264 if(CheckPointer(prevLayer)==POINTER_INVALID || prevLayer.Total()<=0)
2268 if(CheckPointer(
Input)==POINTER_INVALID)
2270 Input=
new CArrayDouble();
2271 if(CheckPointer(
Input)==POINTER_INVALID)
2277 int total=prevLayer.Total();
2280 for(
int i=0; i<total; i++)
2282 temp=prevLayer.At(i);
2287 for(
int i=0; i<total; i++)
2293 int total_data=
Input.Total();
2296 if(CheckPointer(forget_gate)==POINTER_INVALID)
2300 if(CheckPointer(input_gate)==POINTER_INVALID)
2304 if(CheckPointer(output_gate)==POINTER_INVALID)
2307 CArrayDouble *new_content=
new CArrayDouble();
2308 if(CheckPointer(new_content)==POINTER_INVALID)
2311 for(
int i=0; i<total; i++)
2314 if(CheckPointer(temp)==POINTER_INVALID)
2317 for(
int c=0; c<total_data; c++)
2320 if(CheckPointer(temp_con)==POINTER_INVALID)
2326 if(!new_content.Add(val))
2330 for(
int i=0; i<total; i++)
2336 double value=
Memory.At(i)*forget_gate.At(i)+new_content.At(i)*input_gate.At(i);
2337 if(!
Memory.Update(i,value))
2358 CArrayDouble *result=
new CArrayDouble();
2359 if(CheckPointer(gate)==POINTER_INVALID)
2361 int total=gate.Total();
2362 int total_data=sequence.Total();
2363 for(
int i=0; i<total; i++)
2366 if(CheckPointer(temp)==POINTER_INVALID)
2372 for(
int c=0; c<total_data; c++)
2375 if(CheckPointer(temp_con)==POINTER_INVALID)
2380 val+=temp_con.
weight*(sequence.At(c)==DBL_MAX ? 1 : sequence.At(c));
2384 if(!result.Add(val))
2409 CArrayDouble *MemoryGradient=
new CArrayDouble();
2414 for(
int i=0; i<total; i++)
2417 if(CheckPointer(temp)==POINTER_INVALID)
2422 if(CheckPointer(MemoryGradient)==POINTER_INVALID)
2424 if(!MemoryGradient.Reserve(total))
2426 for(
int i=0; i<total; i++)
2430 if(CheckPointer(gate)==POINTER_INVALID)
2434 if(i>=MemoryGradient.Total())
2436 if(!MemoryGradient.Add(value))
2441 value=MemoryGradient.At(i)+value;
2442 if(!MemoryGradient.Update(i,value))
2448 if(CheckPointer(gate)==POINTER_INVALID)
2453 if(CheckPointer(gate)==POINTER_INVALID)
2460 for(
int n=0; n<total_inp; n++)
2463 for(
int i=0; i<total; i++)
2491 int shift=total_inp-total;
2492 for(
int i=0; i<total; i++)
2495 if(CheckPointer(temp)==POINTER_INVALID)
2500 for(
int i=0; i<total; i++)
2504 if(CheckPointer(gate)==POINTER_INVALID)
2508 if(!MemoryGradient.Update(i,value))
2513 if(CheckPointer(gate)==POINTER_INVALID)
2518 if(CheckPointer(gate)==POINTER_INVALID)
2524 delete MemoryGradient;
2533 if(CheckPointer(prevLayer)==POINTER_INVALID || CheckPointer(
Input)==POINTER_INVALID)
2549 if(CheckPointer(gate)==POINTER_INVALID || CheckPointer(input_data)==POINTER_INVALID)
2553 int total_n=gate.Total();
2554 int total_data=input_data.Total();
2555 double lt=
eta*sqrt(1-pow(
b2,
t))/(1-pow(
b1,
t));
2556 for(
int n=0; n<total_n; n++)
2559 if(CheckPointer(neuron)==POINTER_INVALID)
2561 for(
int i=0; i<total_data; i++)
2564 if(CheckPointer(con)==POINTER_INVALID)
2566 double data=input_data.At(i);
2573 con.
vt=
b2*con.
vt+(1-
b2)*pow(g,2)+0.00000001;
2587 if(CheckPointer(prevNeuron)==POINTER_INVALID || CheckPointer(
InputGradient)==POINTER_INVALID ||
InputGradient.Total()<=(
int)index)
2599 if(CheckPointer(prevLayer)==POINTER_INVALID)
2602 int total=prevLayer.Total();
2607 for(
int i=0; (i<total && result); i++)
2609 neuron=prevLayer.At(i);
2610 if(CheckPointer(neuron)==POINTER_INVALID)
2635 if(!
Memory.Save(file_handle))
2655 if(!
Memory.Load(file_handle))
2697 template<
typename T>
2701 for(
int i=0; i<m_buffers_total; i++)
2703 if(m_buffers[i]!=INVALID_HANDLE)
2711 if(ArrayResize(m_buffers,m_buffers_total+1)>0)
2713 m_buffers_total=ArraySize(m_buffers);
2714 result=m_buffers_total-1;
2715 m_buffers[result]=INVALID_HANDLE;
2721 if(!BufferFromArray(result,data,data_array_offset,data_array_count,flags))
2740 iFileHandle=file_handle;
2741 if(!CArrayObj::Load(file_handle))
2743 if(CheckPointer(m_data[0])==POINTER_INVALID)
2764 if(CheckPointer(layers)!=POINTER_INVALID)
2766 if(CheckPointer(opencl)!=POINTER_INVALID)
2786 virtual bool BufferInit(uint count,
double value);
2792 virtual int GetData(
double &values[]);
2793 virtual int GetData(CArrayDouble *values);
2836 if(CheckPointer(opencl)==POINTER_INVALID)
2870 for(
int i=0;i<m_data_total;i++)
2872 if(!MathIsValidNumber(m_data[i]))
2895 m_data_total=(int)fmin(ArrayInitialize(m_data,value),count);
2897 return m_data_total==count;
2906 return ArrayCopy(values,m_data,0,0,m_data_total);
2916 if(!values.AddArray(GetPointer(
this)))
2918 return m_data_total;
2985 virtual bool Save(
int const file_handle);
2986 virtual bool Load(
int const file_handle);
3012 if(CheckPointer(
Output)!=POINTER_INVALID)
3014 if(CheckPointer(
PrevOutput)!=POINTER_INVALID)
3016 if(CheckPointer(
Weights)!=POINTER_INVALID)
3020 if(CheckPointer(
Gradient)!=POINTER_INVALID)
3033 if(CheckPointer(open_cl)==POINTER_INVALID || numNeurons<=0)
3038 if(CheckPointer(
Output)==POINTER_INVALID)
3041 if(CheckPointer(
Output)==POINTER_INVALID)
3049 if(CheckPointer(
PrevOutput)==POINTER_INVALID)
3052 if(CheckPointer(
PrevOutput)==POINTER_INVALID)
3060 if(CheckPointer(
Gradient)==POINTER_INVALID)
3063 if(CheckPointer(
Gradient)==POINTER_INVALID)
3073 if(CheckPointer(
Weights)==POINTER_INVALID)
3076 if(CheckPointer(
Weights)==POINTER_INVALID)
3079 int count=(int)((numNeurons+1)*numOutputs);
3082 for(
int i=0;i<count;i++)
3084 double weigh=(MathRand()+1)/32768.0-0.5;
3140 if(CheckPointer(
Weights)!=POINTER_INVALID)
3153 if(CheckPointer(SourceObject)==POINTER_INVALID)
3157 switch(SourceObject.Type())
3174 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3176 uint global_work_offset[1]={0};
3177 uint global_work_size[1];
3178 global_work_size[0]=
Output.Total();
3187 printf(
"Error of execution kernel FeedForward: %d",GetLastError());
3206 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3208 uint global_work_offset[1]={0};
3209 uint global_work_size[1];
3210 global_work_size[0]=
Neurons()+1;
3220 printf(
"Error of execution kernel CalcHiddenGradient: %d",GetLastError());
3239 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(Target)==POINTER_INVALID)
3241 uint global_work_offset[1]={0};
3242 uint global_work_size[1];
3243 global_work_size[0]=Target.Total();
3244 for(uint i=0;i<global_work_size[0];i++)
3245 if(!
Gradient.Update(i,Target.At(i)))
3256 printf(
"Error of execution kernel CalcOutputGradient: %d",GetLastError());
3268 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3270 uint global_work_offset[2]={0,0};
3271 uint global_work_size[2];
3272 global_work_size[0]=
Neurons();
3273 global_work_size[1]=NeuronOCL.
Neurons();
3286 printf(
"Error of execution kernel UpdateWeightsMomentum: %d",GetLastError());
3302 double lt=
eta*sqrt(1-pow(
b2,
t))/(1-pow(
b1,
t));
3311 uint rest=global_work_size[1]%4;
3312 global_work_size[1]=(global_work_size[1]-rest)/4 + (rest>0 ? 1 : 0);
3317 printf(
"Error of execution kernel UpdateWeightsAdam: %d",GetLastError());
3330 if(CheckPointer(TargetObject)==POINTER_INVALID)
3336 switch(TargetObject.Type())
3344 temp=GetPointer(
this);
3349 temp=GetPointer(
this);
3361 if(CheckPointer(SourceObject)==POINTER_INVALID)
3365 switch(SourceObject.Type())
3382 if(file_handle==INVALID_HANDLE)
3384 if(FileWriteInteger(file_handle,
Type())<INT_VALUE)
3387 if(FileWriteInteger(file_handle,(
int)
activation,INT_VALUE)<INT_VALUE)
3389 if(FileWriteInteger(file_handle,(
int)
optimization,INT_VALUE)<INT_VALUE)
3391 if(FileWriteInteger(file_handle,(
int)
t,INT_VALUE)<INT_VALUE)
3401 if(CheckPointer(
Weights)==POINTER_INVALID)
3403 FileWriteInteger(file_handle,0);
3407 FileWriteInteger(file_handle,1);
3431 if(file_handle==INVALID_HANDLE)
3436 t=FileReadInteger(file_handle,INT_VALUE);
3437 if(CheckPointer(
Output)==POINTER_INVALID)
3440 if(CheckPointer(
Output)==POINTER_INVALID)
3445 if(!
Output.Load(file_handle))
3450 if(CheckPointer(
PrevOutput)==POINTER_INVALID)
3453 if(CheckPointer(
PrevOutput)==POINTER_INVALID)
3463 if(CheckPointer(
Gradient)==POINTER_INVALID)
3466 if(CheckPointer(
Gradient)==POINTER_INVALID)
3476 if(FileReadInteger(file_handle)==0)
3479 if(CheckPointer(
Weights)==POINTER_INVALID)
3482 if(CheckPointer(
Weights)==POINTER_INVALID)
3487 if(!
Weights.Load(file_handle))
3559 virtual bool Save(
int const file_handle);
3560 virtual bool Load(
int const file_handle);
3586 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3588 uint global_work_offset[1]={0};
3589 uint global_work_size[1];
3590 global_work_size[0]=
Output.Total();
3599 printf(
"Error of execution kernel FeedForwardProof: %d",GetLastError());
3610 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3612 uint global_work_offset[1]={0};
3613 uint global_work_size[1];
3614 global_work_size[0]=NeuronOCL.
Neurons();
3625 printf(
"Error of execution kernel CalcInputGradientProof: %d",GetLastError());
3653 virtual bool Init(uint numOutputs,uint myIndex,
COpenCLMy *open_cl,uint window, uint step, uint window_out, uint units_count,
ENUM_OPTIMIZATION optimization_type);
3661 virtual bool Save(
int const file_handle);
3662 virtual bool Load(
int const file_handle);
3685 if(!
CNeuronProofOCL::Init(numOutputs,myIndex,open_cl,window_in,step,units_count*window_out,optimization_type))
3699 for(
int i=0;i<count;i++)
3701 double weigh=(MathRand()+1)/32768.0-0.5;
3756 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3758 uint global_work_offset[1]={0};
3759 uint global_work_size[1];
3771 printf(
"Error of execution kernel FeedForwardProof: %d",GetLastError());
3782 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3784 uint global_work_offset[1]={0};
3785 uint global_work_size[1];
3786 global_work_size[0]=NeuronOCL.
Neurons();
3798 printf(
"Error of execution kernel CalcHiddenGradientConv: %d",GetLastError());
3810 if(CheckPointer(
OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID)
3812 uint global_work_offset[1]={0};
3813 uint global_work_size[1];
3830 printf(
"Error of execution kernel UpdateWeightsConvMomentum: %d",GetLastError());
3846 double lt=
eta*sqrt(1-pow(
b2,
t))/(1-pow(
b1,
t));
3861 printf(
"Error of execution kernel UpdateWeightsConvAdam: %d",GetLastError());
3870 global_work_size[0]=1;
3875 printf(
"Error of execution kernel Normalize: %d",GetLastError());
3916 virtual bool Save(
int const file_handle);
3917 virtual bool Load(
int const file_handle);
3924 if(CheckPointer(
Querys)==POINTER_INVALID)
3926 if(CheckPointer(
Keys)==POINTER_INVALID)
3928 if(CheckPointer(
Values)==POINTER_INVALID)
3930 if(CheckPointer(
Scores)==POINTER_INVALID)
3934 if(CheckPointer(
FF1)==POINTER_INVALID)
3936 if(CheckPointer(
FF2)==POINTER_INVALID)
3947 if(CheckPointer(
Querys)==POINTER_INVALID)
3950 if(CheckPointer(
Querys)==POINTER_INVALID)
3952 if(!
Querys.
Init(0,0,open_cl,window,window,window,units_count,optimization_type))
3957 if(CheckPointer(
Keys)==POINTER_INVALID)
3960 if(CheckPointer(
Keys)==POINTER_INVALID)
3962 if(!
Keys.
Init(0,1,open_cl,window,window,window,units_count,optimization_type))
3967 if(CheckPointer(
Values)==POINTER_INVALID)
3970 if(CheckPointer(
Values)==POINTER_INVALID)
3972 if(!
Values.
Init(0,2,open_cl,window,window,window,units_count,optimization_type))
3977 if(CheckPointer(
Scores)==POINTER_INVALID)
3980 if(CheckPointer(
Scores)==POINTER_INVALID)
3998 if(CheckPointer(
FF1)==POINTER_INVALID)
4001 if(CheckPointer(
FF1)==POINTER_INVALID)
4003 if(!
FF1.
Init(0,4,open_cl,window,window,window*2,units_count,optimization_type))
4008 if(CheckPointer(
FF2)==POINTER_INVALID)
4011 if(CheckPointer(
FF2)==POINTER_INVALID)
4013 if(!
FF2.
Init(0,5,open_cl,window*2,window*2,window,units_count,optimization_type))
4030 if(CheckPointer(prevLayer)==POINTER_INVALID)
4034 uint global_work_offset[1]={0};
4035 uint global_work_size[1];
4036 global_work_size[0]=1;
4041 printf(
"Error of execution kernel Normalize: %d",GetLastError());
4056 uint global_work_offset[1]={0};
4057 uint global_work_size[1];
4058 global_work_size[0]=
iUnits;
4065 printf(
"Error of execution kernel AttentionScore: %d",GetLastError());
4073 uint global_work_offset[2]={0,0};
4074 uint global_work_size[2];
4075 global_work_size[0]=
iUnits;
4084 printf(
"Error of execution kernel Attention Out: %d",GetLastError());
4093 uint global_work_offset[1]={0};
4094 uint global_work_size[1];
4095 global_work_size[0]=1;
4100 printf(
"Error of execution kernel Normalize: %d",GetLastError());
4114 uint global_work_offset[1]={0};
4115 uint global_work_size[1];
4116 global_work_size[0]=
iUnits;
4124 printf(
"Error of execution kernel MatrixSum: %d",GetLastError());
4138 if(CheckPointer(prevLayer)==POINTER_INVALID)
4147 uint global_work_offset[1]={0};
4148 uint global_work_size[1];
4149 global_work_size[0]=
iUnits;
4157 printf(
"Error of execution kernel MatrixSum: %d",GetLastError());
4166 uint global_work_offset[2]={0,0};
4167 uint global_work_size[2];
4168 global_work_size[0]=
iUnits;
4180 printf(
"Error of execution kernel AttentionGradients: %d",GetLastError());
4192 uint global_work_offset[1]={0};
4193 uint global_work_size[1];
4194 global_work_size[0]=
iUnits;
4202 printf(
"Error of execution kernel MatrixSum: %d",GetLastError());
4214 uint global_work_offset[1]={0};
4215 uint global_work_size[1];
4216 global_work_size[0]=
iUnits;
4224 printf(
"Error of execution kernel MatrixSum: %d",GetLastError());
4236 uint global_work_offset[1]={0};
4237 uint global_work_size[1];
4238 global_work_size[0]=
iUnits;
4246 printf(
"Error of execution kernel MatrixSum: %d",GetLastError());
4255 uint global_work_offset[1]={0};
4256 uint global_work_size[1];
4257 global_work_size[0]=1;
4262 printf(
"Error of execution kernel Normalize: %d",GetLastError());
4296 int h=FileOpen(file_name,FILE_CSV|FILE_WRITE);
4297 if(h==INVALID_HANDLE)
4300 for(
int i=0;i<m_data_total;i++)
4301 FileWrite(h,m_data[i]);
4314 if(CheckPointer(
Keys)==POINTER_INVALID || !
Keys.
Save(file_handle))
4318 if(CheckPointer(
Scores)==POINTER_INVALID || !
Scores.Save(file_handle))
4322 if(CheckPointer(
FF1)==POINTER_INVALID || !
FF1.
Save(file_handle))
4324 if(CheckPointer(
FF2)==POINTER_INVALID || !
FF2.
Save(file_handle))
4326 if(FileWriteInteger(file_handle,
iWindow,INT_VALUE)<INT_VALUE)
4328 if(FileWriteInteger(file_handle,
iUnits,INT_VALUE)<INT_VALUE)
4341 if(CheckPointer(
Querys)==POINTER_INVALID)
4346 if(CheckPointer(
Keys)==POINTER_INVALID)
4351 if(CheckPointer(
Values)==POINTER_INVALID)
4356 if(CheckPointer(
Scores)==POINTER_INVALID)
4360 if(!
Scores.Load(file_handle))
4369 if(CheckPointer(
FF1)==POINTER_INVALID)
4373 if(CheckPointer(
FF2)==POINTER_INVALID)
4377 iWindow=FileReadInteger(file_handle);
4378 iUnits=FileReadInteger(file_handle);
4450 if(FileWriteInteger(file_handle,
iWindowOut,INT_VALUE)<INT_VALUE)
4462 if(FileWriteInteger(file_handle,
iWindow,INT_VALUE)<INT_VALUE)
4464 if(FileWriteInteger(file_handle,
iStep,INT_VALUE)<INT_VALUE)
4476 iWindow=FileReadInteger(file_handle);
4477 iStep=FileReadInteger(file_handle);