//+------------------------------------------------------------------+
//|                                                     DMTest.mqh   |
//|                                       https://www.mql5.com       |
//+------------------------------------------------------------------+
#property copyright "Olamide Adebayo"
#property strict

//--- Feature contract constant — the ONE place this is defined. DM_Test_EA.mq5
//--- and DM_RollingMonitor.mq5 both include this header rather than redeclaring
//--- their own copy, so the EA and the indicator can never drift onto two
//--- different feature widths without the self-test in SelfTestFeatureContract()
//--- catching it immediately.
#define NUM_FEATURES 8

//--- Loss function types
enum ENUM_DM_LOSS
  {
   DM_LOSS_SQUARED = 0,   // (e)^2
   DM_LOSS_ABSOLUTE = 1   // |e|
  };

//--- Result container for a single DM test
struct DMResult
  {
   double            mean_diff;      // d-bar
   double            hac_variance;   // long-run variance estimate
   double            dm_stat;        // raw DM statistic
   double            hln_stat;       // Harvey-Leybourne-Newbold corrected statistic
   double            pvalue;         // two-tailed p-value (Student-t)
   int               n;              // sample size used
   int               horizon;        // forecast horizon h
   string            verdict;        // "A_BETTER", "B_BETTER", "NO_SIGNIFICANT_DIFF", "INSUFFICIENT_SAMPLE"
  };

//+------------------------------------------------------------------+
//| Compute per-bar loss differential d_t = L(e1_t) - L(e2_t)        |
//+------------------------------------------------------------------+
void DM_LossDifferential(const double &errA[], const double &errB[], const ENUM_DM_LOSS loss_type,
                          double &d_out[])
  {
   int n = MathMin(ArraySize(errA), ArraySize(errB));
   ArrayResize(d_out, n);
   for(int i = 0; i < n; i++)
     {
      double la, lb;
      if(loss_type == DM_LOSS_SQUARED)
        {
         la = errA[i] * errA[i];
         lb = errB[i] * errB[i];
        }
      else
        {
         la = MathAbs(errA[i]);
         lb = MathAbs(errB[i]);
        }
      d_out[i] = la - lb;
     }
  }

//+------------------------------------------------------------------+
//| Sample autocovariance at lag k for a zero-lag-demeaned series    |
//+------------------------------------------------------------------+
double DM_Autocovariance(const double &d[], const double mean, const int k)
  {
   int n = ArraySize(d);
   if(k >= n)
      return(0.0);
   double sum = 0.0;
   for(int t = k; t < n; t++)
      sum += (d[t] - mean) * (d[t - k] - mean);
   return(sum / n);
  }

//+------------------------------------------------------------------+
//| Bartlett-kernel HAC (Newey-West) long-run variance of d-bar      |
//| lag_max = h - 1, per Diebold & Mariano (1995)                    |
//+------------------------------------------------------------------+
double DM_HACVariance(const double &d[], const int horizon)
  {
   int n = ArraySize(d);
   if(n <= 1)
      return(0.0);

   double mean = 0.0;
   for(int i = 0; i < n; i++)
      mean += d[i];
   mean /= n;

   int lag_max = MathMax(0, horizon - 1);
   //--- guard: lag window cannot exceed available sample
   lag_max = MathMin(lag_max, n - 1);

   double gamma0 = DM_Autocovariance(d, mean, 0);
   double var_sum = gamma0;

   //--- h == 1 collapses to plain sample variance, no Bartlett terms needed
   for(int k = 1; k <= lag_max; k++)
     {
      double w = 1.0 - (double)k / (double)horizon;   // Bartlett kernel weight
      double gamma_k = DM_Autocovariance(d, mean, k);
      var_sum += 2.0 * w * gamma_k;
     }
   return(var_sum);
  }

//+------------------------------------------------------------------+
//| Self-contained Student-t CDF and quantile                        |
//| Not every MetaTrader install ships Math\Stat\Student.mqh from    |
//| the Standard Library, so the distribution math lives here        |
//| instead of depending on it - one less thing that can be missing. |
//| DM_StudentCDF() is the standard Numerical-Recipes-style route    |
//| through the regularized incomplete beta function; DM_Student-    |
//| Quantile() inverts it by bisection, which is robust for any      |
//| degrees-of-freedom value this EA will ever see.                  |
//+------------------------------------------------------------------+
double DM_LogGamma(const double x)
  {
   double cof[6] =
     {
      76.18009172947146, -86.50532032941677, 24.01409824083091,
      -1.231739572450155, 0.1208650973866179e-2, -0.5395239384953e-5
     };
   double y = x;
   double tmp = x + 5.5;
   tmp -= (x + 0.5) * MathLog(tmp);
   double ser = 1.000000000190015;
   for(int j = 0; j < 6; j++)
     {
      y += 1.0;
      ser += cof[j] / y;
     }
   return(-tmp + MathLog(2.5066282746310005 * ser / x));
  }

double DM_BetaCF(const double x, const double a, const double b)
  {
   const int max_iter = 200;
   const double eps = 3.0e-12;
   const double fpmin = 1.0e-300;

   double qab = a + b;
   double qap = a + 1.0;
   double qam = a - 1.0;
   double c = 1.0;
   double d = 1.0 - qab * x / qap;
   if(MathAbs(d) < fpmin)
      d = fpmin;
   d = 1.0 / d;
   double h = d;

   for(int m = 1; m <= max_iter; m++)
     {
      double m2 = 2.0 * m;
      double aa = m * (b - m) * x / ((qam + m2) * (a + m2));
      d = 1.0 + aa * d;
      if(MathAbs(d) < fpmin)
         d = fpmin;
      c = 1.0 + aa / c;
      if(MathAbs(c) < fpmin)
         c = fpmin;
      d = 1.0 / d;
      h *= d * c;

      aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2));
      d = 1.0 + aa * d;
      if(MathAbs(d) < fpmin)
         d = fpmin;
      c = 1.0 + aa / c;
      if(MathAbs(c) < fpmin)
         c = fpmin;
      d = 1.0 / d;
      double del = d * c;
      h *= del;

      if(MathAbs(del - 1.0) < eps)
         break;
     }
   return(h);
  }

double DM_IncompleteBeta(const double x, const double a, const double b)
  {
   if(x <= 0.0)
      return(0.0);
   if(x >= 1.0)
      return(1.0);

   double bt = MathExp(DM_LogGamma(a + b) - DM_LogGamma(a) - DM_LogGamma(b) +
                        a * MathLog(x) + b * MathLog(1.0 - x));

   if(x < (a + 1.0) / (a + b + 2.0))
      return(bt * DM_BetaCF(x, a, b) / a);
   else
      return(1.0 - bt * DM_BetaCF(1.0 - x, b, a) / b);
  }

//+------------------------------------------------------------------+
//| P(T <= t) for a Student-t distribution with the given df         |
//+------------------------------------------------------------------+
double DM_StudentCDF(const double t, const double df)
  {
   double x  = df / (df + t * t);
   double ib = DM_IncompleteBeta(x, df / 2.0, 0.5);
   return(t > 0.0 ? 1.0 - 0.5 * ib : 0.5 * ib);
  }

//+------------------------------------------------------------------+
//| Inverse CDF via bisection - monotone in t, so this is robust for |
//| any df without needing a closed-form quantile formula            |
//+------------------------------------------------------------------+
double DM_StudentQuantile(const double p, const double df)
  {
   double lo = -100.0, hi = 100.0;
   for(int i = 0; i < 200; i++)
     {
      double mid = 0.5 * (lo + hi);
      if(DM_StudentCDF(mid, df) < p)
         lo = mid;
      else
         hi = mid;
     }
   return(0.5 * (lo + hi));
  }

//+------------------------------------------------------------------+
//| Full Diebold-Mariano test with HLN small-sample correction       |
//+------------------------------------------------------------------+
bool DM_RunTest(const double &errA[], const double &errB[], const int horizon,
                 const ENUM_DM_LOSS loss_type, DMResult &result,
                 const int min_sample_size = 30, const double significance_alpha = 0.10)
  {
   result.horizon = horizon;

   //--- edge case: a horizon of zero or less makes the HLN correction and
   //--- the truncation lag (h-1) meaningless, so reject it up front
   if(horizon <= 0)
     {
      result.n = 0;
      result.verdict = "INVALID_HORIZON";
      result.mean_diff = 0.0;
      result.hac_variance = 0.0;
      result.dm_stat = 0.0;
      result.hln_stat = 0.0;
      result.pvalue = 1.0;
      return(false);
     }

   double d[];
   DM_LossDifferential(errA, errB, loss_type, d);
   int n = ArraySize(d);
   result.n = n;

   //--- edge case: insufficient sample size for asymptotic validity, driven
   //--- by the manifest's min_sample_size rather than a hardcoded literal
   if(n < min_sample_size)
     {
      result.verdict = "INSUFFICIENT_SAMPLE";
      result.mean_diff = 0.0;
      result.hac_variance = 0.0;
      result.dm_stat = 0.0;
      result.hln_stat = 0.0;
      result.pvalue = 1.0;
      return(false);
     }

   double mean = 0.0;
   for(int i = 0; i < n; i++)
      mean += d[i];
   mean /= n;

   double long_run_var = DM_HACVariance(d, horizon);
   result.mean_diff    = mean;
   result.hac_variance = long_run_var;

   //--- a negative long-run variance is a numerically invalid HAC estimate
   //--- (Bartlett-kernel truncation isn't guaranteed positive semi-definite
   //--- in finite samples), and it must not be silently folded into a
   //--- "no significant difference" verdict - that's a distinct failure mode
   if(long_run_var < 0.0)
     {
      result.dm_stat  = 0.0;
      result.hln_stat = 0.0;
      result.pvalue   = 1.0;
      result.verdict  = "INVALID_VARIANCE";
      return(false);
     }

   //--- guard against near-zero denominator (degenerate/identical forecasters);
   //--- this is a legitimate case, unlike the negative-variance case above
   double denom = long_run_var / (double)n;
   if(denom < 1e-12)
     {
      result.dm_stat  = 0.0;
      result.hln_stat = 0.0;
      result.pvalue   = 1.0;
      result.verdict  = "NO_SIGNIFICANT_DIFF";
      return(true);
     }

   double dm_raw = mean / MathSqrt(denom);

   //--- Harvey, Leybourne & Newbold (1997) small-sample correction:
   //--- S* = S * sqrt[ (n + 1 - 2h + n^-1 h(h-1)) / n ]
   //--- the h(h-1) term is divided by n BEFORE being summed with (n+1-2h)
   double correction_numerator = (double)(n + 1 - 2 * horizon) +
                                  (double)(horizon * (horizon - 1)) / (double)n;

   //--- for a horizon large relative to n, correction_numerator can go negative,
   //--- which would hand MathSqrt() a negative argument and silently return NaN
   //--- straight into dm_hln, pvalue and verdict - that must surface as an
   //--- explicit failure, not a NaN-poisoned "result"
   if(correction_numerator <= 0.0)
     {
      result.dm_stat  = dm_raw;
      result.hln_stat = 0.0;
      result.pvalue   = 1.0;
      result.verdict  = "INVALID_HORIZON";
      return(false);
     }

   double hln_factor = MathSqrt(correction_numerator / (double)n);
   double dm_hln = dm_raw * hln_factor;

   result.dm_stat  = dm_raw;
   result.hln_stat = dm_hln;

   //--- two-tailed p-value against Student-t(n-1), self-contained CDF above
   double cdf = DM_StudentCDF(MathAbs(dm_hln), (double)(n - 1));
   result.pvalue = 2.0 * (1.0 - cdf);

   if(result.pvalue > significance_alpha)
      result.verdict = "NO_SIGNIFICANT_DIFF";
   else
      result.verdict = (mean < 0.0) ? "A_BETTER" : "B_BETTER";   // d = L(A) - L(B); negative mean => A has lower loss

   return(true);
  }

//+------------------------------------------------------------------+
//| Rolling-window wrapper: slices the tail of a vector-backed       |
//| history buffer and runs DM_RunTest on just that window           |
//+------------------------------------------------------------------+
bool DM_RunRolling(const double &errA_full[], const double &errB_full[], const int window,
                    const int horizon, const ENUM_DM_LOSS loss_type, DMResult &result,
                    const int min_sample_size = 30, const double significance_alpha = 0.10)
  {
   int total = MathMin(ArraySize(errA_full), ArraySize(errB_full));
   int start = MathMax(0, total - window);
   int len   = total - start;

   double slice_a[], slice_b[];
   ArrayResize(slice_a, len);
   ArrayResize(slice_b, len);
   for(int i = 0; i < len; i++)
     {
      slice_a[i] = errA_full[start + i];
      slice_b[i] = errB_full[start + i];
     }
   return(DM_RunTest(slice_a, slice_b, horizon, loss_type, result, min_sample_size, significance_alpha));
  }

//+------------------------------------------------------------------+
//| Shared 8-feature builder for the ONNX forecaster (Model A)       |
//| Used identically by the EA and the indicator so both ever run    |
//| the same model on the same feature contract — no separate,       |
//| drifting copy of this logic in each file.                        |
//+------------------------------------------------------------------+
bool DM_BuildFeatureVector(const string symbol, const int h_ema, const int h_atr,
                            const int h_rsi, const int h_macd, const int shift,
                            float &features[])
  {
   double close_arr[], atr_arr[], rsi_arr[], macd_main[], macd_signal[], ema_arr[];

   if(CopyClose(symbol, PERIOD_CURRENT, shift, 5, close_arr) < 5)
      return(false);
   if(CopyBuffer(h_atr, 0, shift, 2, atr_arr) < 2)
      return(false);
   if(CopyBuffer(h_rsi, 0, shift, 2, rsi_arr) < 2)
      return(false);
   if(CopyBuffer(h_macd, 0, shift, 2, macd_main) < 2)
      return(false);
   if(CopyBuffer(h_macd, 1, shift, 2, macd_signal) < 2)
      return(false);
   if(CopyBuffer(h_ema, 0, shift, 3, ema_arr) < 3)
      return(false);

   //--- volume_real, not tick_volume: falls back to tick volume only when
   //--- the broker feed genuinely has no real-volume data for this bar
   MqlRates rates[];
   if(CopyRates(symbol, PERIOD_CURRENT, shift, 2, rates) < 2)
      return(false);
   double vol_real = (rates[0].tick_volume > 0) ? (double)rates[0].tick_volume : 1.0;
   if(rates[0].real_volume > 0)
      vol_real = (double)rates[0].real_volume;

   double atr_now = MathMax(atr_arr[1], 1e-8);   // divide-by-zero guard

   ArrayResize(features, NUM_FEATURES);
   features[0] = (float)((close_arr[4] - close_arr[3]) / atr_now);   // return lag1, ATR-normalized
   features[1] = (float)((close_arr[3] - close_arr[2]) / atr_now);   // return lag2
   features[2] = (float)((close_arr[2] - close_arr[1]) / atr_now);   // return lag3
   features[3] = (float)(atr_now / close_arr[4]);                    // ATR normalized by price
   features[4] = (float)(rsi_arr[1] / 100.0);                        // RSI scaled 0-1
   features[5] = (float)((macd_main[1] - macd_signal[1]) / atr_now); // MACD histogram, ATR-normalized
   features[6] = (float)((ema_arr[2] - ema_arr[0]) / (2.0 * atr_now)); // EMA slope, ATR-normalized
   features[7] = (float)MathLog(1.0 + vol_real / 1000.0);            // log-scaled real volume

   //--- reject a feature vector carrying NaN/Inf before it ever reaches the
   //--- ONNX model - a bad upstream indicator tick is more useful as a
   //--- skipped bar than as silently-propagated garbage in the error history
   for(int i = 0; i < NUM_FEATURES; i++)
      if(!MathIsValidNumber(features[i]))
         return(false);

   return(true);
  }

//+------------------------------------------------------------------+
//| Shared Model B forecast: raw EMA slope, used directly as the     |
//| next-bar return prediction (no ML, no rescaling) — this is the   |
//| deliberately simple momentum-continuation baseline               |
//+------------------------------------------------------------------+
double DM_ForecastModelB(const int h_ema, const int shift)
  {
   double ema_arr[];
   if(CopyBuffer(h_ema, 0, shift, 3, ema_arr) < 3)
      return(0.0);
   return((ema_arr[2] - ema_arr[0]) / 2.0);
  }
//+------------------------------------------------------------------+
