Discussing the article: "Neural networks made easy (Part 51): Behavior-Guided Actor-Critic (BAC)"
#Enjoy! <3
Thanks Sir @Dmitriy
//+------------------------------------------------------------------+
Thanks Sir @Dmitriy
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CreateDescriptions(CArrayObj *actor, CArrayObj *critic, CArrayObj *autoencoder)
{
--- Smart print statement for debugging
Print("Creating layer descriptions...");
CLayerDescription *descr;
int layerNumber = 0; To keep track of layer numbers
--- Check and initialize input arrays if they are null
if (!actor)
{
actor = new CArrayObj();
if (!actor)
return false;
}
if (!critic)
{
critic = new CArrayObj();
if (!critic)
return false;
}
if (!autoencoder)
{
autoencoder = new CArrayObj();
if (!autoencoder)
return false;
}
--- Actor
actor. Clear();
--- Layer 1: Input layer
layerNumber++;
Print("Creating actor - Input Layer ", layerNumber, ": Input layer");
if (!( descr = new CLayerDescription())))
return false;
descr.type = defNeuronBaseOCL;
int prev_count = descr.count = (HistoryBars * BarDescr);
descr.activation = None;
descr.optimisation = ADAM;
if (!actor. Add(descr))
{
delete descr;
return false;
}
--- Layer 2
layerNumber++;
Print("Creating actor - Batch Normalisation Layer ", layerNumber);
if (!( descr = new CLayerDescription())))
return false;
descr.type = defNeuronBatchNormOCL;
descr.count = prev_count;
descr.batch = 1000;
descr.activation = None;
descr.optimisation = ADAM;
if (!actor. Add(descr))
{
delete descr;
return false;
}
--- Layer 3
layerNumber++;
Print("Creating actor - Convolutional Layer ", layerNumber);
if (!( descr = new CLayerDescription())))
return false;
descr.type = defNeuronConvOCL;
prev_count = descr.count = prev_count - 1;
descr.window = 2;
descr.step = 1;
descr.window_out = 8;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!actor. Add(descr))
{
delete descr;
return false;
}
--- Layer 4
layerNumber++;
Print("Creating actor - Convolutional Layer ", layerNumber);
if (!( descr = new CLayerDescription())))
return false;
descr.type = defNeuronConvOCL;
prev_count = descr.count = prev_count;
descr.window = 8;
descr.step = 8;
descr.window_out = 8;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!actor. Add(descr))
{
delete descr;
return false;
}
--- Layer 5
layerNumber++;
Print("Creating actor - Dense/Base Layer ", layerNumber);
if (!( descr = new CLayerDescription())))
return false;
descr.type = defNeuronBaseOCL;
descr.count = 2048;
descr.optimisation = ADAM;
descr.activation = LReLU;
if (!actor. Add(descr))
{
delete descr;
return false;
}
--- Layer 6
layerNumber++;
Print("Creating actor - Dense/Base Layer ", layerNumber);
if (!( descr = new CLayerDescription())))
return false;
descr.type = defNeuronBaseOCL;
prev_count = descr.count = 1024;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 7
layerNumber++;
Print("Creating actor - SoftMax Layer ", layerNumber);
if(!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronSoftMaxOCL;
prev_count = descr.count = prev_count / 16;
descr.step = 16;
descr.optimisation = ADAM;
descr.activation = None;
if(!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 8 Multilayer Multi-Head Attention Layer
layerNumber++;
Print("Creating actor - Multilayer Multi-Head Attention Layer ", layerNumber);
if(!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronMLMHAttentionOCL;
descr.count = prev_count;
descr.window = 16;
descr.window_out = 8;
descr.step = 4;
descr.layers = 3;
descr.optimisation = ADAM;
descr.activation = None;
if(!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 9
layerNumber++;
Print("Creating actor - Concatenate Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronConcatenate;
descr.count = LatentCount;
descr.window = prev_count;
descr.step = AccountDescr;
descr.optimisation = ADAM;
descr.activation = SIGMOID;
if (!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 10
layerNumber++;
Print("Creating actor - SoftMax Layer ", layerNumber);
if(!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronSoftMaxOCL;
prev_count = descr.count = prev_count / 16;
descr.step = 16;
descr.optimisation = ADAM;
descr.activation = None;
if(!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 11 Multilayer Multi-Head Attention Layer
layerNumber++;
Print("Creating actor - Multilayer Multi-Head Attention Layer ", layerNumber);
if(!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronMLMHAttentionOCL;
descr.count = prev_count;
descr.window = 16;
descr.window_out = 8;
descr.step = 4;
descr.layers = 3;
descr.optimisation = ADAM;
descr.activation = None;
if(!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 12
layerNumber++;
Print("Creating actor - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
descr.count = 2048;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 13
layerNumber++;
Print("Creating actor - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
descr.count = 2048;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 14
layerNumber++;
Print("Creating actor - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
descr.count = 2 * NActions;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!actor.Add(descr))
{
delete descr;
return false;
}
//--- Layer 15
layerNumber++;
Print("Creating actor - VAE Output Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronVAEOCL;
descr.count = NActions;
descr.optimise = ADAM;
if (!actor.Add(descr))
{
delete descr;
return false;
}
//--- Critic
critic.Clear();
//--- Layer 1: Input layer
layerNumber++;
Print("Creating critic - Input Layer ", layerNumber, ": Input layer");
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
prev_count = descr.count = LatentCount;
descr.activation = None;
descr.optimisation = ADAM;
if (!critic.Add(descr))
{
delete descr;
return false;
}
//--- Layer 2
layerNumber++;
Print("Creating critic - Concatenate Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronConcatenate;
descr.count = LatentCount;
descr.window = prev_count;
descr.step = NActions;
descr.optimisation = ADAM;
descr.activation = LReLU;
if (!critic.Add(descr))
{
delete descr;
return false;
}
//--- Layer 3
layerNumber++;
Print("Creating critic - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
descr.count = LatentCount;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!critic.Add(descr))
{
delete descr;
return false;
}
//--- Layer 4
layerNumber++;
Print("Creating critic - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
descr.count = LatentCount;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!critic.Add(descr))
{
delete descr;
return false;
}
//--- Layer 5
layerNumber++;
Print("Creating critic - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
descr.count = 1;
descr.optimisation = ADAM;
descr.activation = None;
if (!critic.Add(descr))
{
delete descr;
return false;
}
//--- Autoencoder
autoencoder.Clear();
//--- Layer 1: Input layer
layerNumber++;
Print("Creating autoencoder - Dense/Base Layer ", layerNumber, ": Input layer");
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
prev_count = descr.count = LatentCount;
descr.activation = None;
descr.optimisation = ADAM;
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Layer 2
layerNumber++;
Print("Creating autoencoder - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
prev_count = descr.count = prev_count / 2;
descr.optimisation = ADAM;
descr.activation = LReLU;
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Layer 3
layerNumber++;
Print("Creating autoencoder - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
prev_count = descr.count = prev_count / 2;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Layer 4
layerNumber++;
Print("Creating autoencoder - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
descr.type = defNeuronBaseOCL;
prev_count = descr.count = 20;
descr.count = LatentCount;
descr.activation = LReLU;
descr.optimisation = ADAM;
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Layer 5
layerNumber++;
Print("Creating autoencoder - Dense/Base Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
if (!(descr.Copy(autoencoder.At(2))))
{
delete descr;
return false;
}
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Layer 6
layerNumber++;
Print("Creating autoencoder - Output Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
if (!(descr.Copy(autoencoder.At(1))))
{
delete descr;
return false;
}
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Layer 7
layerNumber++;
Print("Creating autoencoder - Output Layer ", layerNumber);
if (!(descr = new CLayerDescription()))
return false;
if (!(descr.Copy(autoencoder.At(0))))
{
delete descr;
return false;
}
if (!autoencoder.Add(descr))
{
delete descr;
return false;
}
//--- Smart print statement for debugging
Print("Layer descriptions created successfully!");
return true;
}
Hello. Research is again failing to get into the green zone. Negative MinProfit does not help. I have brought the value to -10000. In general, it is strange. If the initial values of parameters are chosen randomly, then at least some of them should have led to a plus. But this is not even close to being observed.
Hello everyone. I have this version after about 3-4 cycles (database collection - training - test) began to give just a straight line on tests. Deals do not open. Training was done all times by 500 000 iterations. Another interesting point - at a certain moment the error of one of the critics became very large at first, and then gradually the errors of both critics decreased to 0. And for 2-3 cycles the errors of both critics are at 0. And on the tests Test.mqh gives a straight line and no deals. In Research.mqh passes there are passes with negative profit and deals. There are also passes with no deals and zero outcome. There were only 5 passes with a positive outcome in one of the cycles.
In general, it is strange. I have been training strictly according to Dmitry's instructions in all articles, and I have not been able to get a result from any article. I do not understand what I do wrong....
MetaQuotes:
New article Neural networks made easy (Part 51): behavioural actor-criticism (BAC) has been published:
Author: Dmitriy Gizlyk
I downloaded the zipped folder, but there were many other folders inside.
If possible I would like you to explain how to deploy and train.
Congratulations on a great job!
Thank you very much
You are missing trading opportunities:
- Free trading apps
- Over 8,000 signals for copying
- Economic news for exploring financial markets
Registration
Log in
You agree to website policy and terms of use
If you do not have an account, please register
Check out the new article: Neural networks made easy (Part 51): Behavior-Guided Actor-Critic (BAC).
The last two articles considered the Soft Actor-Critic algorithm, which incorporates entropy regularization into the reward function. This approach balances environmental exploration and model exploitation, but it is only applicable to stochastic models. The current article proposes an alternative approach that is applicable to both stochastic and deterministic models.
First, let's talk about the need to study the environment in general. I think everyone agrees that this process is necessary. But for what exactly and at what stage?
Let's start with a simple example. Suppose that we find ourselves in a room with three identical doors and we need to get into the street. What shall we do? We open the doors one by one until we find the one we need. When we enter the same room again, we will no longer open all the doors to get outside, but instead we will immediately head to the already known exit. If we have a different task, then some options are possible. We can again open all the doors, except for the exit we already know, and look for the right one. Or we can first remember which doors we opened earlier when looking for a way out and whether the one we need was among them. If we remember the right door, we head towards it. Otherwise, we check the doors we have not tried before.
Conclusion: We need to study the environment in an unfamiliar situation to choose the right action. After finding the required route, additional exploration of the environment can only get in the way.
However, when the task changes in a known state, we may need to additionally study the environment. This may include searching for a more optimal route. In the example above, this may happen if we needed to go through several more rooms or we found ourselves at the wrong side of the building.
Therefore, we need an algorithm that allows us to enhance environmental exploration in unexplored states and minimize it in previously explored states.
Author: Dmitriy Gizlyk