Neuron Net
Macros | Functions
Attention layer's neuron Feed Forward

Describes the feed forward process for the Neuron of attention layer. More...

Macros

#define def_k_AttentionScore   11
 Index of the kernel of the attention neuron to calculate score matrix (AttentionScore) More...
 
#define def_k_as_querys   0
 Matrix of Querys. More...
 
#define def_k_as_keys   1
 Matriz of Keys. More...
 
#define def_k_as_score   2
 Matrix of Scores. More...
 
#define def_k_as_dimension   3
 Dimension of Key. More...
 
#define def_k_AttentionOut   12
 Index of the Attention Neuron Output calculation kernel (AttentionOut) More...
 
#define def_k_aout_scores   0
 Matrix of Scores. More...
 
#define def_k_aout_values   1
 Matrix of Values. More...
 
#define def_k_aout_inputs   2
 Inputs tesor. More...
 
#define def_k_aout_out   3
 Output tesor. More...
 
#define def_k_MatrixSum   13
 Index of the kernel for calculation Sum of 2 matrix with multiplyer (SumMatrix) More...
 
#define def_k_sum_matrix1   0
 First matrix. More...
 
#define def_k_sum_matrix2   1
 Second matrix. More...
 
#define def_k_sum_matrix_out   2
 Output matrix. More...
 
#define def_k_sum_dimension   3
 Dimension of matrix. More...
 
#define def_k_sum_multiplyer   4
 Multiplyer for output. More...
 

Functions

__kernel void AttentionScore (__global double *querys, __global double *keys, __global double *score, int dimension)
 | Describes the Score calculation process for the Neuron of attention layer (CNeuronAttentionOCL). More...
 
__kernel void AttentionOut (__global double *scores, __global double *values, __global double *inputs, __global double *out)
 Describes the Attention out calculation process for the Neuron of attention layer (CNeuronAttentionOCL). More...
 
__kernel void SumMatrix (__global double *matrix1, __global double *matrix2, __global double *matrix_out, int dimension, double multiplyer)
 2 . Describes the calculation Sum of 2 matrixs. More...
 

Detailed Description

Describes the feed forward process for the Neuron of attention layer.

Detailed description on the link.

Macro Definition Documentation

◆ def_k_aout_inputs

#define def_k_aout_inputs   2

Inputs tesor.

Definition at line 250 of file NeuroNet.mqh.

◆ def_k_aout_out

#define def_k_aout_out   3

Output tesor.

Definition at line 251 of file NeuroNet.mqh.

◆ def_k_aout_scores

#define def_k_aout_scores   0

Matrix of Scores.

Definition at line 248 of file NeuroNet.mqh.

◆ def_k_aout_values

#define def_k_aout_values   1

Matrix of Values.

Definition at line 249 of file NeuroNet.mqh.

◆ def_k_as_dimension

#define def_k_as_dimension   3

Dimension of Key.

Definition at line 245 of file NeuroNet.mqh.

◆ def_k_as_keys

#define def_k_as_keys   1

Matriz of Keys.

Definition at line 243 of file NeuroNet.mqh.

◆ def_k_as_querys

#define def_k_as_querys   0

Matrix of Querys.

Definition at line 242 of file NeuroNet.mqh.

◆ def_k_as_score

#define def_k_as_score   2

Matrix of Scores.

Definition at line 244 of file NeuroNet.mqh.

◆ def_k_AttentionOut

#define def_k_AttentionOut   12

Index of the Attention Neuron Output calculation kernel (AttentionOut)

Definition at line 247 of file NeuroNet.mqh.

◆ def_k_AttentionScore

#define def_k_AttentionScore   11

Index of the kernel of the attention neuron to calculate score matrix (AttentionScore)

Definition at line 241 of file NeuroNet.mqh.

◆ def_k_MatrixSum

#define def_k_MatrixSum   13

Index of the kernel for calculation Sum of 2 matrix with multiplyer (SumMatrix)

Definition at line 253 of file NeuroNet.mqh.

◆ def_k_sum_dimension

#define def_k_sum_dimension   3

Dimension of matrix.

Definition at line 257 of file NeuroNet.mqh.

◆ def_k_sum_matrix1

#define def_k_sum_matrix1   0

First matrix.

Definition at line 254 of file NeuroNet.mqh.

◆ def_k_sum_matrix2

#define def_k_sum_matrix2   1

Second matrix.

Definition at line 255 of file NeuroNet.mqh.

◆ def_k_sum_matrix_out

#define def_k_sum_matrix_out   2

Output matrix.

Definition at line 256 of file NeuroNet.mqh.

◆ def_k_sum_multiplyer

#define def_k_sum_multiplyer   4

Multiplyer for output.

Definition at line 258 of file NeuroNet.mqh.

Function Documentation

◆ AttentionOut()

__kernel void AttentionOut ( __global double *  scores,
__global double *  values,
__global double *  inputs,
__global double *  out 
)

Describes the Attention out calculation process for the Neuron of attention layer (CNeuronAttentionOCL).

Detailed description on the link.

Parameters
[in]scoresMatrix of Scores
[in]valuesMatrix of Values
[in]inputsInputs tesor
[out]outOutput tesor

Definition at line 551 of file NeuroNet.cl.

556  {
557  int units=get_global_size(0);
558  int u=get_global_id(0);
559  int d=get_global_id(1);
560  int dimension=get_global_size(1);
561  int shift=u*dimension+d;
562  double result=0;
563  for(int i=0;i<units;i++)
564  result+=scores[u*units+i]*values[i*dimension+d];
565  out[shift]=result+inputs[shift];
566  }

◆ AttentionScore()

__kernel void AttentionScore ( __global double *  querys,
__global double *  keys,
__global double *  score,
int  dimension 
)

| Describes the Score calculation process for the Neuron of attention layer (CNeuronAttentionOCL).

Detailed description on the link.

Parameters
[in]querysMatrix of Querys
[in]keysMatriz of Keys
[out]scoreMatrix of Scores
dimensionDimension of Key

Definition at line 519 of file NeuroNet.cl.

524  {
525  int q=get_global_id(0);
526  int shift_q=q*dimension;
527  int units=get_global_size(0);
528  int shift_s=q*units;
529  double koef=sqrt((double)(units*dimension));
530  if(koef<1)
531  koef=1;
532  double sum=0;
533  for(int k=0;k<units;k++)
534  {
535  double result=0;
536  int shift_k=k*dimension;
537  for(int i=0;i<dimension;i++)
538  result+=(querys[shift_q+i]*keys[shift_k+i]);
539  result=exp(clamp(result/koef,-500.0,500.0));
540  score[shift_s+k]=result;
541  sum+=result;
542  }
543  for(int k=0;k<units;k++)
544  score[shift_s+k]/=sum;
545  }

◆ SumMatrix()

__kernel void SumMatrix ( __global double *  matrix1,
__global double *  matrix2,
__global double *  matrix_out,
int  dimension,
double  multiplyer 
)

2 . Describes the calculation Sum of 2 matrixs.

Detailed description on the link.

Parameters
[in]matrix1First matrix
[in]matrix2Second matrix
[out]matrix_outOutput matrix
dimensionDimension of matrix
multiplyerMultiplyer for output

Definition at line 572 of file NeuroNet.cl.

578  {
579  const int i=get_global_id(0)*dimension;
580  for(int k=0;k<dimension;k++)
581  matrix_out[i+k]=(matrix1[i+k]+matrix2[i+k])*multiplyer;
582  }