//+------------------------------------------------------------------+
//|                                     VolatilityForecastCheck.mq5  |
//| Runs the volatility forecast once and prints it, so the          |
//| numbers can be read before anything is automated.                |
//| Two modes. The self test builds a deterministic series from      |
//| the generator written out below and prints the fitted            |
//| coefficients. The identical generator exists in the C++          |
//| reference and in the Python checker, so all three see the        |
//| same input and any disagreement is a real one rather than a      |
//| difference in market data. The live mode reads the published     |
//| feed, forecasts the symbol of the chart it runs on, reports how  |
//| its daily bars line up with the index close, and tests the model |
//| out of sample on this broker's own bars.                         |
//| Reads only. Never trades.                                        |
//+------------------------------------------------------------------+
#property copyright "Copyright 2026, Eugenio G."
#property link      "https://www.mql5.com"
#property version   "1.10"
#property script_show_inputs

#include <VolatilityForecast\VolatilityForecastEngine.mqh>

//--- input parameters
input bool   InpSelfTest    = true;              // Run the self test instead of the feed
input string InpFeedFile    = "vol_spy.json";    // Feed file in MQL5\Files
input int    InpHorizon     = 5;                 // Forecast horizon in sessions
input int    InpHistory     = 3000;              // Daily bars to request
input int    InpImpliedLag  = -1;                // Implied lag: -1 auto, 0 same date, 1 previous bar
input bool   InpOutOfSample = true;              // Also test out of sample on this symbol

//+------------------------------------------------------------------+
//| Deterministic generator, mirrored in the C++ and Python versions |
//+------------------------------------------------------------------+
class CLcg
  {
private:
   uint              m_state;
public:
   void              Seed(const uint s) { m_state = s & 0x7fffffff; }
   double            Next(void);
   double            Normal(void);
  };
//+------------------------------------------------------------------+
//| Next uniform variate                                             |
//+------------------------------------------------------------------+
double CLcg::Next(void)
  {
   m_state = (1103515245 * m_state + 12345) & 0x7fffffff;
   return((double)m_state / 2147483648.0);
  }
//+------------------------------------------------------------------+
//| Next normal variate, Box-Muller, first value only                |
//+------------------------------------------------------------------+
double CLcg::Normal(void)
  {
//--- two uniforms are drawn and only the first variate kept, which
//--- wastes half of them and keeps the three implementations aligned
   double u1 = Next();
   double u2 = Next();
   if(u1 < 1e-12)
      u1 = 1e-12;
   return(MathSqrt(-2.0 * MathLog(u1)) * MathCos(6.283185307179586 * u2));
  }

//+------------------------------------------------------------------+
//| Build the shared synthetic series and fit the engine to it       |
//+------------------------------------------------------------------+
void SelfTest(void)
  {
//--- 601 rather than 600, and the extra bar is the whole point. Handed a
//--- series, the engine discards its newest bar unless told otherwise, as
//--- it does in live use while that bar is still forming. The C++
//--- reference has no such notion: it is handed a finished series and uses
//--- all of it. Feeding 601 here leaves exactly 600 after the discard, so
//--- both implementations fit the same 573 rows built from the same bars.
//--- The generator draws four numbers per bar in sequence, so appending a
//--- bar at the end cannot disturb any bar before it
   const int N = 601;
   CLcg rng;
   rng.Seed(11);

   MqlRates rates[];
   datetime times[];
   double   values[];
   ArrayResize(rates, N);
   ArrayResize(times, N);
   ArrayResize(values, N);

   double price = 100.0;
   double sigma = 0.011;
   datetime day = D'2020.01.01 00:00';

   for(int i = 0; i < N; i++)
     {
      sigma = MathExp(0.97 * MathLog(sigma) + 0.03 * MathLog(0.011)
                      + 0.10 * rng.Normal());
      double gap  = sigma * 0.6 * rng.Normal();
      double path = sigma * rng.Normal();
      double open = price * MathExp(gap);
      double clos = open * MathExp(path);
      double wick = MathAbs(sigma * rng.Normal());

      rates[i].time  = day + i * 86400;
      rates[i].open  = open;
      rates[i].close = clos;
      rates[i].high  = MathMax(open, clos) * MathExp(wick);
      rates[i].low   = MathMin(open, clos) * MathExp(-wick);
      rates[i].tick_volume = 0;
      rates[i].spread      = 0;
      rates[i].real_volume = 0;

      times[i]  = rates[i].time;
      values[i] = sigma * sigma * HARIV_TRADING * 1.35;
      price     = clos;
     }

   CVolatilityForecast engine;
   engine.Init("SELFTEST", PERIOD_D1, 5, N);
   if(!engine.SetImplied(times, values, N))
     {
      Print("Self test: the implied history was rejected.");
      return;
     }
   if(!engine.CalculateFrom(rates, N))
     {
      Print("Self test failed: ", engine.StateText());
      return;
     }

   PrintFormat("Self test on the shared series, %d observations", engine.Rows());
   PrintFormat("  beta %.10f %.10f %.10f %.10f %.10f",
               engine.Beta(0), engine.Beta(1), engine.Beta(2),
               engine.Beta(3), engine.Beta(4));
   PrintFormat("  forecast %.6f%% annualized, in-sample R2 %.4f",
               engine.ForecastVol(), engine.RSquared());
   Print("  The five coefficients above are what the C++ reference and the");
   Print("  Python checker also print. Any difference past the tenth decimal");
   Print("  is a bug, and all three should read 573 observations.");
   Print("  The forecast is NOT comparable and is not meant to be. This engine");
   Print("  forecasts from the newest bar it holds, which is the only useful");
   Print("  answer in live use. The reference prints the fitted value at the");
   Print("  last row of the design matrix, five sessions earlier, because that");
   Print("  is all a batch checker needs. Same model, two evaluation points.");
  }

//+------------------------------------------------------------------+
//| Read one flat text field out of the feed                         |
//+------------------------------------------------------------------+
string FieldOf(const string json, const string key)
  {
//--- This reads the feed this article publishes and nothing more general.
//--- It assumes one flat object: no escaped quotes inside strings, no
//--- nested objects and no arrays. The feed is written to that shape on
//--- purpose, and a document outside it needs a real JSON parser
   int at = StringFind(json, "\"" + key + "\"");
   if(at < 0)
      return("");
   int colon = StringFind(json, ":", at);
   if(colon < 0)
      return("");
   int i = colon + 1;
   int n = StringLen(json);
   while(i < n && StringGetCharacter(json, i) == ' ')
      i++;
   if(i >= n)
      return("");

//--- A quoted value ends at its closing quote and nowhere else. An earlier
//--- version stopped at the first comma whatever the value looked like,
//--- which is right for a bare number and wrong for any string holding a
//--- list. The history arrives as one quoted field of comma-separated
//--- dates, so that version returned its first date and discarded the rest.
   if(StringGetCharacter(json, i) == '\"')
     {
      i++;
      int start = i;
      while(i < n && StringGetCharacter(json, i) != '\"')
         i++;
      return(StringSubstr(json, start, i - start));
     }

//--- an unquoted value: a number, true, false or null. Any of the usual
//--- JSON separators ends it
   int begin = i;
   while(i < n)
     {
      ushort ch = StringGetCharacter(json, i);
      if(ch == ',' || ch == '}' || ch == '\n' || ch == '\r')
         break;
      i++;
     }
   return(StringSubstr(json, begin, i - begin));
  }

//+------------------------------------------------------------------+
//| Load the published implied history into the engine               |
//+------------------------------------------------------------------+
bool LoadFeed(const string file, CVolatilityForecast &engine, string &symbol,
              string &asof)
  {
   int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI);
   if(h == INVALID_HANDLE)
     {
      PrintFormat("Cannot open %s in MQL5\\Files. Run the feed first.", file);
      return(false);
     }
   string json = "";
   while(!FileIsEnding(h))
      json += FileReadString(h);
   FileClose(h);

   symbol = FieldOf(json, "symbol");
   asof   = FieldOf(json, "asof");
   string dates  = FieldOf(json, "dates");
   string values = FieldOf(json, "values");
   if(dates == "" || values == "")
     {
      Print("The feed carries no history. Expected flat \"dates\" and \"values\".");
      return(false);
     }

   string dparts[], vparts[];
   int nd = StringSplit(dates, ',', dparts);
   int nv = StringSplit(values, ',', vparts);
   if(nd != nv || nd < HARIV_MIN_ROWS)
     {
      PrintFormat("Feed history is unusable: %d dates against %d values.", nd, nv);
      return(false);
     }

   datetime times[];
   double   vals[];
   ArrayResize(times, nd);
   ArrayResize(vals, nd);
   int kept = 0;
   for(int i = 0; i < nd; i++)
     {
      datetime t = StringToTime(dparts[i]);
      double   v = StringToDouble(vparts[i]);
      if(t <= 0 || v <= 0.0)
         continue;
      times[kept] = t;
      vals[kept]  = v;
      kept++;
     }
   if(kept < HARIV_MIN_ROWS)
     {
      PrintFormat("Only %d usable rows in the feed history.", kept);
      return(false);
     }
   return(engine.SetImplied(times, vals, kept));
  }

//+------------------------------------------------------------------+
//| Script program start function                                    |
//+------------------------------------------------------------------+
void OnStart(void)
  {
   if(InpSelfTest)
     {
      SelfTest();
      return;
     }

   CVolatilityForecast engine;
   if(!engine.Init(_Symbol, PERIOD_D1, InpHorizon, InpHistory))
     {
      Print("Init rejected the settings.");
      return;
     }

   string feed_symbol = "", asof = "";
   if(!LoadFeed(InpFeedFile, engine, feed_symbol, asof))
      return;

//--- The index for a date is final at 16:15 New York time. If this server's
//--- daily bar closes earlier than that, each bar takes the quote of the
//--- bar before it, so no row uses a number published after it closed
   int lag = (InpImpliedLag < 0 ? CVolatilityForecast::SuggestedLag()
              : (InpImpliedLag > 0 ? 1 : 0));
   engine.SetImpliedLag(lag);
   PrintFormat("Server clock UTC%+.2f hours, so a daily bar closes at %.2f hours New York time.",
               CVolatilityForecast::ServerOffsetMinutes() / 60.0,
               CVolatilityForecast::BarCloseNewYork());
   PrintFormat("  The index is final at 16.25 hours, so each bar uses the %s quote%s.",
               (lag == 0 ? "same-date" : "previous bar's"),
               (InpImpliedLag < 0 ? ", chosen automatically" : ""));

   if(!engine.Calculate())
     {
      Print("No forecast: ", engine.StateText());
      return;
     }

   PrintFormat("%s, %d-session forecast from %d observations, origin %s",
               _Symbol, InpHorizon, engine.Rows(),
               TimeToString(engine.OriginTime(), TIME_DATE));
   PrintFormat("  implied volatility  %6.2f%%   from %s as of %s%s",
               engine.ImpliedVol(), feed_symbol, asof,
               (engine.Carried() ? ", carried over a missing quote" : ""));
   PrintFormat("  forecast volatility %6.2f%%", engine.ForecastVol());
   PrintFormat("  realized 1 / 5 / 22 %6.2f%% %6.2f%% %6.2f%%",
               engine.RealizedVol(1), engine.RealizedVol(5),
               engine.RealizedVol(22));
   PrintFormat("  implied over forecast volatility %.2f, above %.0f%% of this history",
               engine.VolRatio(), 100.0 * engine.RatioRank());
   PrintFormat("  in-sample R2 %.3f, typical error %.3f in log variance",
               engine.RSquared(), engine.ResidualSD());
   PrintFormat("  bars matched to an implied quote: %d", engine.Matched());
   PrintFormat("  overnight share of daily variance: %.1f%%",
               100.0 * engine.OvernightShare());
   Print("  Implied volatility is risk-neutral. It enters the model as a");
   Print("  predictor. The forecast above is the physical estimate.");

   if(!InpOutOfSample)
      return;
//--- The in-sample R2 above is not evidence of forecast accuracy. This is:
//--- the last sixty percent of rows, each forecast by models refitted only
//--- on rows whose forward windows had closed, as in the research tests
   double r2_har = 0.0, r2_hariv = 0.0, cw_t = 0.0;
   int scored = 0;
   if(!engine.OutOfSample(0.40, r2_har, r2_hariv, cw_t, scored))
     {
      Print("Out of sample: too few rows on this history to score.");
      return;
     }
   PrintFormat("Out of sample on %s, last 60%% of rows, %d forecasts", _Symbol, scored);
   PrintFormat("  HAR    R2 %+6.1f%%", 100.0 * r2_har);
   PrintFormat("  HAR-IV R2 %+6.1f%%   gain %+.1f points", 100.0 * r2_hariv,
               100.0 * (r2_hariv - r2_har));
   PrintFormat("  Clark-West t %.2f, Newey-West errors over %d lags",
               cw_t, 2 * (InpHorizon - 1));
  }
//+------------------------------------------------------------------+