//+------------------------------------------------------------------+
//| VolatilityForecastEngine.mqh                                     |
//| CVolatilityForecast: forecasts realized volatility from the      |
//| terminal's own bars and a history of option-implied variance.    |
//| The model is HAR, three trailing averages of realized variance   |
//| at one, five and twenty-two sessions, extended with the implied  |
//| variance known at the close of each of those sessions.           |
//| Coefficients are fitted here, on the reader's own symbol and     |
//| history, not shipped as constants estimated elsewhere. Two       |
//| limits travel with the number: implied variance is risk-neutral, |
//| so it enters as a predictor and is not treated as a forecast,    |
//| and the published index is built on a related instrument, so it  |
//| is a proxy and not an identity.                                  |
//| Reads only. Never trades.                                        |
//+------------------------------------------------------------------+
#ifndef VOLATILITY_FORECAST_ENGINE_MQH
#define VOLATILITY_FORECAST_ENGINE_MQH

//--- model dimensions
#define HARIV_NPARAM      5       // intercept, three HAR terms, implied
#define HARIV_TRADING   252       // sessions per year, for annualizing
#define HARIV_RELTOL    1e-10     // a pivot this small against the largest is singular
#define HARIV_MIN_ROWS  100       // fewer observations than this is not a fit
#define HARIV_WARMUP     22       // longest lookback, so the first usable index
#define HARIV_CARRY_DAYS  4       // oldest quote the forecast origin may borrow
#define HARIV_CBOE_CLOSE 58500    // 16:15 New York, in seconds after midnight

//+------------------------------------------------------------------+
//| Why the fit can fail, so the caller can say which                |
//+------------------------------------------------------------------+
enum ENUM_HARIV_STATE
  {
   HARIV_OK,                  // a forecast is available
   HARIV_NO_BARS,             // the terminal returned too little history
   HARIV_NO_IMPLIED,          // the implied history is missing or unusable
   HARIV_TOO_FEW_ROWS,        // not enough aligned observations to fit
   HARIV_SINGULAR             // the least-squares problem could not be solved
  };

//+------------------------------------------------------------------+
//| Volatility forecast engine                                       |
//+------------------------------------------------------------------+
class CVolatilityForecast
  {
private:
   //--- configuration
   string            m_symbol;
   ENUM_TIMEFRAMES   m_period;
   int               m_horizon;         // sessions ahead being forecast
   int               m_history;         // bars requested from the terminal
   int               m_lag;             // 0 same-date quote, 1 previous bar's quote

   //--- the implied history as supplied, sorted ascending by date
   datetime          m_iv_time[];
   double            m_iv_value[];

   //--- working series, index 0 is the oldest bar
   datetime          m_day[];           // calendar day of each bar
   double            m_daily[];         // variance of each session
   double            m_iv[];            // implied variance known at each bar
   int               m_usable;          // bars in the working series

   //--- design matrix of every complete row, built once per calculation
   int               m_row_at[];        // bar index of each row
   double            m_x[];             // rows x HARIV_NPARAM, row-major
   double            m_y[];             // log forward variance of each row
   int               m_nrows;

   //--- results
   double            m_beta[HARIV_NPARAM];
   ENUM_HARIV_STATE  m_state;
   int               m_rows;            // observations the fit actually used
   int               m_matched;         // bars that found an implied value
   double            m_forecast;        // annualized variance, physical
   double            m_implied;         // annualized variance, risk neutral
   bool              m_carried;         // origin borrowed an earlier quote
   datetime          m_origin;          // bar the forecast is made from
   double            m_r1, m_r5, m_r22; // the trailing terms, annualized
   double            m_r2;              // in-sample fit quality
   double            m_resid_sd;        // typical error of the fit, in logs
   double            m_ratio_rank;      // share of history below today's ratio
   double            m_overnight_share; // diagnostic, see the note on the estimator

   //--- internals
   double            DailyVariance(const double prev_close, const MqlRates &r);
   double            Trailing(const int i, const int n);
   bool              Prepare(const MqlRates &rates[], const int count,
                             const bool drop_newest);
   bool              Align(void);
   void              BuildRows(const int h);
   bool              LeastSquares(const int nrows, const int p, double &beta[],
                                  double &ssr);
   double            QuoteBefore(const int i);
   bool              Origin(void);

public:
                     CVolatilityForecast(void);
                    ~CVolatilityForecast(void) {}

   bool              Init(const string symbol, const ENUM_TIMEFRAMES period,
                          const int horizon, const int history);
   bool              SetImplied(const datetime &times[], const double &values[],
                                const int count);
   void              SetImpliedLag(const int lag) { m_lag = (lag > 0 ? 1 : 0); }
   int               ImpliedLag(void)     const { return(m_lag); }
   static int        SuggestedLag(void);
   static int        ServerOffsetMinutes(void);
   static double     BarCloseNewYork(void);
   bool              Calculate(void);
   bool              CalculateFrom(const MqlRates &rates[], const int count,
                                   const bool drop_newest = true);
   bool              OutOfSample(const double score_from, double &r2_har,
                                 double &r2_hariv, double &cw_t, int &scored);

   //--- results, annualized variances unless the name says otherwise
   ENUM_HARIV_STATE  State(void)          const { return(m_state);    }
   string            StateText(void)      const;
   double            Forecast(void)       const { return(m_forecast); }
   double            ForecastVol(void)    const { return(m_forecast > 0.0 ? 100.0 * MathSqrt(m_forecast) : 0.0); }
   double            ImpliedVol(void)     const { return(m_implied  > 0.0 ? 100.0 * MathSqrt(m_implied)  : 0.0); }
   double            RealizedVol(const int n) const;
   double            VolRatio(void)       const { return(m_forecast > 0.0 ? MathSqrt(m_implied / m_forecast) : 0.0); }
   double            RatioRank(void)      const { return(m_ratio_rank); }
   double            ResidualSD(void)     const { return(m_resid_sd); }
   bool              Carried(void)        const { return(m_carried);  }
   datetime          OriginTime(void)     const { return(m_origin);   }
   double            Beta(const int i)    const { return(i >= 0 && i < HARIV_NPARAM ? m_beta[i] : 0.0); }
   int               Rows(void)           const { return(m_rows);     }
   int               Matched(void)        const { return(m_matched);  }
   double            RSquared(void)       const { return(m_r2);       }
   double            OvernightShare(void) const { return(m_overnight_share); }
  };

//+------------------------------------------------------------------+
//| Constructor                                                      |
//+------------------------------------------------------------------+
CVolatilityForecast::CVolatilityForecast(void) : m_symbol(""),
                                                 m_period(PERIOD_D1),
                                                 m_horizon(5),
                                                 m_history(3000),
                                                 m_lag(0),
                                                 m_usable(0),
                                                 m_nrows(0),
                                                 m_state(HARIV_NO_BARS),
                                                 m_rows(0),
                                                 m_matched(0),
                                                 m_forecast(0.0),
                                                 m_implied(0.0),
                                                 m_carried(false),
                                                 m_origin(0),
                                                 m_r1(0.0), m_r5(0.0), m_r22(0.0),
                                                 m_r2(0.0),
                                                 m_resid_sd(0.0),
                                                 m_ratio_rank(-1.0),
                                                 m_overnight_share(0.0)
  {
   ArrayInitialize(m_beta, 0.0);
  }

//+------------------------------------------------------------------+
//| Set the instrument and the shape of the problem                  |
//+------------------------------------------------------------------+
bool CVolatilityForecast::Init(const string symbol, const ENUM_TIMEFRAMES period,
                               const int horizon, const int history)
  {
   if(symbol == "" || horizon < 1 || history < 300)
      return(false);
   m_symbol  = symbol;
   m_period  = period;
   m_horizon = horizon;
   m_history = history;
   return(true);
  }

//+------------------------------------------------------------------+
//| Supply the implied variance history, oldest first                |
//+------------------------------------------------------------------+
bool CVolatilityForecast::SetImplied(const datetime &times[], const double &values[],
                                     const int count)
  {
   if(count < HARIV_MIN_ROWS)
      return(false);
   ArrayResize(m_iv_time, count);
   ArrayResize(m_iv_value, count);
   for(int i = 0; i < count; i++)
     {
      m_iv_time[i]  = times[i];
      m_iv_value[i] = values[i];
     }
   return(true);
  }

//+------------------------------------------------------------------+
//| Day of the week of a date, 0 = Sunday                            |
//+------------------------------------------------------------------+
int HarivDayOfWeek(const int year, const int month, const int day)
  {
   MqlDateTime s;
   ZeroMemory(s);
   s.year = year;
   s.mon  = month;
   s.day  = day;
   MqlDateTime back;
   TimeToStruct(StructToTime(s), back);
   return(back.day_of_week);
  }

//+------------------------------------------------------------------+
//| Day of the year of a date, 0 = the first of January              |
//+------------------------------------------------------------------+
int HarivDayOfYear(const int year, const int month, const int day)
  {
   MqlDateTime s;
   ZeroMemory(s);
   s.year = year;
   s.mon  = month;
   s.day  = day;
   MqlDateTime back;
   TimeToStruct(StructToTime(s), back);
   return(back.day_of_year);
  }

//+------------------------------------------------------------------+
//| Server clock minus UTC, rounded to a quarter hour                |
//+------------------------------------------------------------------+
int CVolatilityForecast::ServerOffsetMinutes(void)
  {
   long diff = (long)TimeTradeServer() - (long)TimeGMT();
   return((int)(MathRound(diff / 900.0) * 15));
  }

//+------------------------------------------------------------------+
//| When a daily bar closes, in hours after midnight New York time   |
//+------------------------------------------------------------------+
double CVolatilityForecast::BarCloseNewYork(void)
  {
//--- A daily bar dated D closes at the server's midnight. The volatility
//--- index for D is final at 16:15 New York time. Both clocks are placed
//--- on New York time for today's date, using the US daylight-saving rule
//--- that starts on the second Sunday of March and ends on the first
//--- Sunday of November
   MqlDateTime t;
   TimeToStruct(TimeGMT(), t);
   int first_mar = (7 - HarivDayOfWeek(t.year, 3, 1)) % 7 + 1;
   int first_nov = (7 - HarivDayOfWeek(t.year, 11, 1)) % 7 + 1;
   int doy_start = HarivDayOfYear(t.year, 3, first_mar + 7);
   int doy_end   = HarivDayOfYear(t.year, 11, first_nov);
   bool dst = (t.day_of_year >= doy_start && t.day_of_year < doy_end);
   int  ny  = (dst ? -4 : -5) * 3600;
   return((86400.0 - ServerOffsetMinutes() * 60.0 + ny) / 3600.0);
  }

//+------------------------------------------------------------------+
//| Lag the implied series needs on this server, 0 or 1              |
//+------------------------------------------------------------------+
int CVolatilityForecast::SuggestedLag(void)
  {
//--- If this server's daily bar closes before the index is final, the
//--- quote dated D was not yet known when bar D closed, and pairing them
//--- would use information from the future. Each bar then takes the
//--- quote of the bar before it. Servers that close at 17:00 New York,
//--- or later, need no lag
   return(BarCloseNewYork() * 3600.0 < HARIV_CBOE_CLOSE ? 1 : 0);
  }

//+------------------------------------------------------------------+
//| Variance of one session, in daily units                          |
//+------------------------------------------------------------------+
double CVolatilityForecast::DailyVariance(const double prev_close, const MqlRates &r)
  {
//--- The overnight jump plus the intraday path. Garman-Klass on its own
//--- is quieter, but it is built from a single session's open, high, low
//--- and close, so it cannot see what happened while the market was shut.
//--- On an instrument trading six and a half hours that gap carries close
//--- to forty percent of total variance. On one trading around the clock
//--- the open sits on the previous close and the first term falls away by
//--- itself, so the same expression serves both without a special case
   if(prev_close <= 0.0 || r.open <= 0.0 || r.high <= 0.0 ||
      r.low <= 0.0 || r.close <= 0.0)
      return(-1.0);

   double overnight = MathLog(r.open / prev_close);
   double ln_hl     = MathLog(r.high / r.low);
   double ln_co     = MathLog(r.close / r.open);
   double garman    = 0.5 * ln_hl * ln_hl
                      - (2.0 * MathLog(2.0) - 1.0) * ln_co * ln_co;
   return(overnight * overnight + garman);
  }

//+------------------------------------------------------------------+
//| Annualized mean of the last n daily variances ending at index i  |
//+------------------------------------------------------------------+
double CVolatilityForecast::Trailing(const int i, const int n)
  {
   if(i + 1 < n || i < 0)
      return(-1.0);

   double sum = 0.0;
   for(int k = i - n + 1; k <= i; k++)
     {
      if(m_daily[k] < 0.0)
         return(-1.0);
      sum += m_daily[k];
     }
   return(sum / n * HARIV_TRADING);
  }

//+------------------------------------------------------------------+
//| Daily variances, overnight share and implied alignment           |
//+------------------------------------------------------------------+
bool CVolatilityForecast::Prepare(const MqlRates &rates[], const int count,
                                  const bool drop_newest)
  {
   if(count < 300)
     {
      m_state = HARIV_NO_BARS;
      return(false);
     }
//--- A bar that may still be forming has no final high, low or close, so
//--- the caller can ask for it to be dropped. A bar whose day has ended,
//--- which is the newest one over a weekend or a holiday, is kept, so the
//--- forecast is not made from a session older than it needs to be
   m_usable = (drop_newest ? count - 1 : count);
   ArrayResize(m_daily, m_usable);
   ArrayResize(m_day, m_usable);
   m_daily[0] = -1.0;

   double on_sum = 0.0, tot_sum = 0.0;
   for(int i = 0; i < m_usable; i++)
     {
      //--- a daily bar opens at the server's midnight, so the whole number
      //--- of days in its open time is its trading date on the server clock
      m_day[i] = rates[i].time / 86400 * 86400;
      if(i == 0)
         continue;
      m_daily[i] = DailyVariance(rates[i - 1].close, rates[i]);
      if(m_daily[i] >= 0.0)
        {
         double on = MathLog(rates[i].open / rates[i - 1].close);
         on_sum  += on * on;
         tot_sum += m_daily[i];
        }
     }
   m_overnight_share = (tot_sum > 0.0 ? on_sum / tot_sum : 0.0);

   if(!Align())
     {
      m_state = HARIV_NO_IMPLIED;
      return(false);
     }
   return(true);
  }

//+------------------------------------------------------------------+
//| Give each bar the implied figure that was known when it closed   |
//+------------------------------------------------------------------+
bool CVolatilityForecast::Align(void)
  {
//--- Both series are sorted ascending, so one pass over each is enough.
//--- A bar is first matched to the quote carrying its own date. A bar with
//--- no quote of its own stays unmatched and drops out of the fit rather
//--- than borrowing a neighbor's value, because carrying a figure forward
//--- makes the column artificially persistent and flatters its
//--- coefficient. With a lag of one, each bar then takes the match of the
//--- bar before it, which is the quote already final when this bar closed
   int have = ArraySize(m_iv_time);
   if(have < HARIV_MIN_ROWS)
      return(false);

   double exact[];
   ArrayResize(exact, m_usable);
   ArrayResize(m_iv, m_usable);
   ArrayInitialize(exact, -1.0);
   ArrayInitialize(m_iv, -1.0);

   int j = 0;
   for(int i = 0; i < m_usable; i++)
     {
      while(j < have && (m_iv_time[j] / 86400 * 86400) < m_day[i])
         j++;
      if(j < have && (m_iv_time[j] / 86400 * 86400) == m_day[i] && m_iv_value[j] > 0.0)
         exact[i] = m_iv_value[j];
     }

   m_matched = 0;
   for(int i = m_lag; i < m_usable; i++)
     {
      m_iv[i] = exact[i - m_lag];
      if(m_iv[i] > 0.0)
         m_matched++;
     }
   return(m_matched >= HARIV_MIN_ROWS);
  }

//+------------------------------------------------------------------+
//| Rows with trailing terms, a quote and a closed forward window    |
//+------------------------------------------------------------------+
void CVolatilityForecast::BuildRows(const int h)
  {
//--- A row needs a complete set of trailing terms, an implied quote of its
//--- own and a forward window that has already closed. A session whose
//--- forward window runs past the last bar is not a row, which is what
//--- keeps any fit from seeing anything it should not. Rows are kept in
//--- bar order, so a fit on the first k rows is a fit on the past only
   ArrayResize(m_row_at, m_usable);
   ArrayResize(m_x, m_usable * HARIV_NPARAM);
   ArrayResize(m_y, m_usable);
   m_nrows = 0;

   for(int i = HARIV_WARMUP; i + h < m_usable; i++)
     {
      if(m_iv[i] <= 0.0)
         continue;
      double r1  = Trailing(i, 1);
      double r5  = Trailing(i, 5);
      double r22 = Trailing(i, 22);
      if(r1 <= 0.0 || r5 <= 0.0 || r22 <= 0.0)
         continue;

      double fwd = 0.0;
      bool   bad = false;
      for(int k = i + 1; k <= i + h; k++)
        {
         if(m_daily[k] < 0.0)
           {
            bad = true;
            break;
           }
         fwd += m_daily[k];
        }
      fwd = fwd / h * HARIV_TRADING;
      if(bad || fwd <= 0.0)
         continue;

      int at = m_nrows * HARIV_NPARAM;
      m_x[at]     = 1.0;
      m_x[at + 1] = MathLog(r1);
      m_x[at + 2] = MathLog(r5);
      m_x[at + 3] = MathLog(r22);
      m_x[at + 4] = MathLog(m_iv[i]);
      m_y[m_nrows]      = MathLog(fwd);
      m_row_at[m_nrows] = i;
      m_nrows++;
     }
  }

//+------------------------------------------------------------------+
//| Least squares on the first rows, by Householder QR               |
//+------------------------------------------------------------------+
bool CVolatilityForecast::LeastSquares(const int nrows, const int p, double &beta[],
                                       double &ssr)
  {
//--- The three HAR terms are averages of one series over overlapping
//--- windows, so they are strongly correlated with each other and with
//--- the implied term. Forming the normal equations squares the condition
//--- number of that problem. Householder reflections work on the design
//--- matrix itself and never form it, so they lose about half as many
//--- digits. The first p columns are used, so p = 4 fits plain HAR on
//--- exactly the rows the full model sees
   double a[], b[], v[];
   ArrayResize(a, nrows * p);
   ArrayResize(b, nrows);
   ArrayResize(v, nrows);
   for(int i = 0; i < nrows; i++)
     {
      for(int j = 0; j < p; j++)
         a[i * p + j] = m_x[i * HARIV_NPARAM + j];
      b[i] = m_y[i];
     }

   double rmax = 0.0;
   for(int k = 0; k < p; k++)
     {
      double norm = 0.0;
      for(int i = k; i < nrows; i++)
         norm += a[i * p + k] * a[i * p + k];
      norm = MathSqrt(norm);
      if(norm == 0.0)
         return(false);
      double alpha = (a[k * p + k] > 0.0 ? -norm : norm);

      double vv = 0.0;
      for(int i = k; i < nrows; i++)
        {
         v[i] = a[i * p + k];
         if(i == k)
            v[i] -= alpha;
         vv += v[i] * v[i];
        }
      if(vv > 0.0)
        {
         for(int j = k; j < p; j++)
           {
            double s = 0.0;
            for(int i = k; i < nrows; i++)
               s += v[i] * a[i * p + j];
            double f = 2.0 * s / vv;
            for(int i = k; i < nrows; i++)
               a[i * p + j] -= f * v[i];
           }
         double s = 0.0;
         for(int i = k; i < nrows; i++)
            s += v[i] * b[i];
         double f = 2.0 * s / vv;
         for(int i = k; i < nrows; i++)
            b[i] -= f * v[i];
        }
      rmax = MathMax(rmax, MathAbs(a[k * p + k]));
     }

//--- a diagonal entry this small against the largest one means a column
//--- is a combination of the others, and the coefficients are not defined
   for(int k = 0; k < p; k++)
      if(MathAbs(a[k * p + k]) <= HARIV_RELTOL * rmax)
         return(false);

   for(int i = p - 1; i >= 0; i--)
     {
      double s = b[i];
      for(int j = i + 1; j < p; j++)
         s -= a[i * p + j] * beta[j];
      beta[i] = s / a[i * p + i];
     }
   for(int j = p; j < HARIV_NPARAM; j++)
      beta[j] = 0.0;

//--- what the reflections leave below the first p entries is the residual
   ssr = 0.0;
   for(int i = p; i < nrows; i++)
      ssr += b[i] * b[i];
   return(true);
  }

//+------------------------------------------------------------------+
//| Latest quote already final when bar i closed, if recent enough   |
//+------------------------------------------------------------------+
double CVolatilityForecast::QuoteBefore(const int i)
  {
//--- Used for the forecast origin only, never for the fit. When a US
//--- holiday leaves the newest bar without a quote of its own, the origin
//--- takes the last quote that was already known, provided it is no more
//--- than a few days old, instead of stepping back to an older bar
   datetime limit = m_day[i] - HARIV_CARRY_DAYS * 86400;
   for(int j = ArraySize(m_iv_time) - 1; j >= 0; j--)
     {
      datetime d = m_iv_time[j] / 86400 * 86400;
      bool known = (m_lag == 0 ? d <= m_day[i] : d < m_day[i]);
      if(!known)
         continue;
      if(d < limit)
         return(-1.0);
      return(m_iv_value[j] > 0.0 ? m_iv_value[j] : -1.0);
     }
   return(-1.0);
  }

//+------------------------------------------------------------------+
//| Forecast from the newest complete bar                            |
//+------------------------------------------------------------------+
bool CVolatilityForecast::Origin(void)
  {
   int last = m_usable - 1;
   while(last >= HARIV_WARMUP && Trailing(last, 22) <= 0.0)
      last--;
   if(last < HARIV_WARMUP)
     {
      m_state = HARIV_TOO_FEW_ROWS;
      return(false);
     }

   m_carried = false;
   m_implied = m_iv[last];
   if(m_implied <= 0.0)
     {
      m_implied = QuoteBefore(last);
      m_carried = true;
     }
   if(m_implied <= 0.0)
     {
      m_state = HARIV_NO_IMPLIED;
      return(false);
     }

   m_origin = m_day[last];
   m_r1  = Trailing(last, 1);
   m_r5  = Trailing(last, 5);
   m_r22 = Trailing(last, 22);
   if(m_r1 <= 0.0 || m_r5 <= 0.0)
     {
      m_state = HARIV_TOO_FEW_ROWS;
      return(false);
     }

   double logf = m_beta[0]
                 + m_beta[1] * MathLog(m_r1)
                 + m_beta[2] * MathLog(m_r5)
                 + m_beta[3] * MathLog(m_r22)
                 + m_beta[4] * MathLog(m_implied);
   m_forecast = MathExp(logf);
   return(true);
  }

//+------------------------------------------------------------------+
//| Refit on current history and produce a forecast                  |
//+------------------------------------------------------------------+
bool CVolatilityForecast::Calculate(void)
  {
   MqlRates rates[];
   ArraySetAsSeries(rates, false);
   int got = CopyRates(m_symbol, m_period, 0, m_history, rates);
   if(got < 300)
     {
      m_state = HARIV_NO_BARS;
      return(false);
     }
//--- the newest bar is dropped only until its day ends on the server clock
   bool forming = ((long)rates[got - 1].time + PeriodSeconds(m_period)
                   > (long)TimeTradeServer());
   return(CalculateFrom(rates, got, forming));
  }

//+------------------------------------------------------------------+
//| The same calculation on bars supplied by the caller              |
//+------------------------------------------------------------------+
bool CVolatilityForecast::CalculateFrom(const MqlRates &rates[], const int count,
                                        const bool drop_newest)
  {
//--- Separated from Calculate so the arithmetic can be run on a series
//--- built in the test rather than on whatever the terminal happens to
//--- hold. That is what makes the self test in the check script possible,
//--- and the self test is how this port is compared against independent
//--- C++ and Python implementations, coefficient by coefficient
   m_forecast = 0.0;
   m_implied  = 0.0;
   m_r1 = m_r5 = m_r22 = 0.0;
   m_ratio_rank = -1.0;

   if(!Prepare(rates, count, drop_newest))
      return(false);
   BuildRows(m_horizon);
   m_rows = m_nrows;
   if(m_rows < HARIV_MIN_ROWS)
     {
      m_state = HARIV_TOO_FEW_ROWS;
      return(false);
     }

   double ssr = 0.0;
   if(!LeastSquares(m_rows, HARIV_NPARAM, m_beta, ssr))
     {
      m_state = HARIV_SINGULAR;
      return(false);
     }

   double mean = 0.0, tss = 0.0;
   for(int i = 0; i < m_rows; i++)
      mean += m_y[i];
   mean /= m_rows;
   for(int i = 0; i < m_rows; i++)
      tss += (m_y[i] - mean) * (m_y[i] - mean);
   m_r2       = (tss > 0.0 ? 1.0 - ssr / tss : 0.0);
   m_resid_sd = MathSqrt(ssr / (m_rows - HARIV_NPARAM));

   if(!Origin())
      return(false);

//--- Where today's gap between option prices and the forecast sits in
//--- this symbol's own history. Each past row gets the same ratio from
//--- its fitted value, so the comparison is like with like, and the panel
//--- can call options cheap or expensive against what this market has
//--- actually shown rather than against a threshold picked by hand
   double now = VolRatio();
   int below = 0;
   for(int i = 0; i < m_rows; i++)
     {
      double fit = 0.0;
      for(int j = 0; j < HARIV_NPARAM; j++)
         fit += m_x[i * HARIV_NPARAM + j] * m_beta[j];
      double ratio = MathSqrt(MathExp(m_x[i * HARIV_NPARAM + 4]) / MathExp(fit));
      if(ratio < now)
         below++;
     }
   m_ratio_rank = (double)below / m_rows;
   m_state = HARIV_OK;
   return(true);
  }

//+------------------------------------------------------------------+
//| Expanding-window test on the bars already prepared               |
//+------------------------------------------------------------------+
bool CVolatilityForecast::OutOfSample(const double score_from, double &r2_har,
                                      double &r2_hariv, double &cw_t, int &scored)
  {
//--- Call after a successful Calculate. Every row from score_from of the
//--- sample onward is forecast twice, by plain HAR and by HAR with the
//--- implied term, each refitted only on rows whose forward windows had
//--- closed by that row's date. The benchmark is the mean of those same
//--- training rows. Nothing about the scored row enters its own forecast
   scored = 0;
   r2_har = r2_hariv = cw_t = 0.0;
   int h = m_horizon;
   int first = (int)(m_nrows * score_from);
   if(m_nrows < HARIV_MIN_ROWS || first < HARIV_MIN_ROWS)
      return(false);

   double f[];
   ArrayResize(f, m_nrows);
   double sse_b = 0.0, sse_f = 0.0, sst = 0.0;
   double bh[HARIV_NPARAM], bf[HARIV_NPARAM];

   for(int k = first; k < m_nrows; k++)
     {
      //--- rows are in bar order, so the training set is a prefix
      int n = 0;
      while(n < k && m_row_at[n] + h <= m_row_at[k])
         n++;
      if(n < HARIV_MIN_ROWS)
         continue;
      double s1 = 0.0, s2 = 0.0;
      if(!LeastSquares(n, HARIV_NPARAM - 1, bh, s1) ||
         !LeastSquares(n, HARIV_NPARAM, bf, s2))
         continue;

      double mean = 0.0;
      for(int i = 0; i < n; i++)
         mean += m_y[i];
      mean /= n;
      double ph = 0.0, pf = 0.0;
      for(int j = 0; j < HARIV_NPARAM; j++)
        {
         ph += m_x[k * HARIV_NPARAM + j] * bh[j];
         pf += m_x[k * HARIV_NPARAM + j] * bf[j];
        }
      double a = m_y[k];
      sse_b += (a - ph) * (a - ph);
      sse_f += (a - pf) * (a - pf);
      sst   += (a - mean) * (a - mean);
      //--- Clark-West: the plain difference in squared errors, plus the
      //--- penalty the larger model pays for estimating a useless term
      f[scored] = (a - ph) * (a - ph) - (a - pf) * (a - pf) + (ph - pf) * (ph - pf);
      scored++;
     }
   if(scored < 30 || sst <= 0.0)
      return(false);
   r2_har   = 1.0 - sse_b / sst;
   r2_hariv = 1.0 - sse_f / sst;

//--- neighboring forecast windows overlap by h - 1 sessions, so the
//--- standard error uses Newey-West weights over 2(h - 1) lags
   double mf = 0.0;
   for(int i = 0; i < scored; i++)
      mf += f[i];
   mf /= scored;
   double s = 0.0;
   for(int i = 0; i < scored; i++)
      s += (f[i] - mf) * (f[i] - mf);
   int lags = 2 * (h - 1);
   for(int j = 1; j <= lags && j < scored; j++)
     {
      double g = 0.0;
      for(int i = j; i < scored; i++)
         g += (f[i] - mf) * (f[i - j] - mf);
      s += 2.0 * (1.0 - j / (lags + 1.0)) * g;
     }
   double se = (s > 0.0 ? MathSqrt(s) / scored : 0.0);
   cw_t = (se > 0.0 ? mf / se : 0.0);
   return(true);
  }

//+------------------------------------------------------------------+
//| Trailing realized volatility in percent, for display             |
//+------------------------------------------------------------------+
double CVolatilityForecast::RealizedVol(const int n) const
  {
   double v = 0.0;
   if(n == 1)
      v = m_r1;
   else
      if(n == 5)
         v = m_r5;
      else
         if(n == 22)
            v = m_r22;
   return(v > 0.0 ? 100.0 * MathSqrt(v) : 0.0);
  }

//+------------------------------------------------------------------+
//| Why the last call failed, in words                               |
//+------------------------------------------------------------------+
string CVolatilityForecast::StateText(void) const
  {
   switch(m_state)
     {
      case HARIV_OK:
         return("ok");
      case HARIV_NO_BARS:
         return("not enough history from the terminal");
      case HARIV_NO_IMPLIED:
         return("implied history missing or unaligned");
      case HARIV_TOO_FEW_ROWS:
         return("too few usable observations");
      case HARIV_SINGULAR:
         return("the least-squares problem is singular");
     }
   return("unknown");
  }
//+------------------------------------------------------------------+
#endif
//+------------------------------------------------------------------+
