"""Recompute the runs test report outside MQL5, starting from the raw deals.

Usage:
  python runs_crosscheck.py LOG DEALS.csv [LOG DEALS.csv ...] [--shuffles N] [--power]

LOG is a tester log or journal file with the lines RunsTestPrintReport() writes.
DEALS.csv is the file RunsTestExportDeals() writes into the common Files folder.

For every "=== Runs test: ... ===" block in LOG the script reads the filter line, rebuilds
the trades from DEALS.csv with its own code (one trade per position, entry costs included,
netting reversals split, sorted by close time in ms then position ID), and recomputes:
  - wins, losses, breakeven trades, runs, expected runs and Z
  - the exact two-sided p from integer binomial coefficients, and the normal p
  - the longest losing streak, the loss rate after k or more losses, and the loss rate
    of the first trade after exactly k losses
  - the win rate of each block and the chi-square drift test
These must equal the printed report. The shuffle results are then repeated with Python's
own random numbers, so they agree only within Monte Carlo error.

--power adds a simulation: how often each check would flag a sequence in which a loss
raises the chance of another loss by 5 or 10 percentage points (a two-state Markov
chain with the same overall loss rate as the real trades).
Only the Python 3 standard library is used.
"""
import csv, random, re, sys
from datetime import datetime, timezone
from math import comb, erf, exp, lgamma, log, sqrt

HEAD = re.compile(r"=== Runs test: (.+?) ===")
FILTER = re.compile(r"Filter: symbol (\S+), magic (-?\d+) to (-?\d+), closed (\S+ \S+) to (\S+ \S+)")
TRADES = re.compile(r"Trades (\d+) \(wins (\d+), losses (\d+)\), breakeven (\d+) \((\w+), \|net\| <= ([\d.]+)\)")
RUNS = re.compile(r"Runs (\d+), expected ([\d.]+), Z (-?[\d.]+)")
EXACT = re.compile(r"two-sided ([\d.]+) \(normal ([\d.]+)\)")
STREAK = re.compile(r"Longest losing streak (\d+), shuffles with one as long: p ([\d.]+)")
AFTER = re.compile(r"After (\d+) or more losses in a row: loss rate ([\d.]+)% \((\d+) trades\), "
                   r"shuffled ([\d.]+)% \(95% range ([\d.]+)-([\d.]+)%\), p ([\d.]+)")
FIRST = re.compile(r"First trade after exactly (\d+) losses: loss rate ([\d.]+)% \((\d+) streaks\), "
                   r"shuffled ([\d.]+)% \(95% range ([\d.]+)-([\d.]+)%\), p ([\d.]+)")
BLOCKS = re.compile(r"Win rate in (\d+) blocks: ([\d.% ]+), chi-square ([\d.]+), p ([\d.]+)")
INSIDE = re.compile(r"Within-block shuffles \((\d+) blocks\): runs p ([\d.]+), streak p ([\d.]+), "
                    r"after p ([\d.]+), first p ([\d.]+)")
ENTRY_IN, ENTRY_OUT, ENTRY_INOUT = 0, 1, 2


# ---------------------------------------------------------------- reading the files
def read_text(path):
    raw = open(path, "rb").read()
    if raw[:2] in (b"\xff\xfe", b"\xfe\xff"):
        return raw.decode("utf-16")
    return raw.decode("utf-8", errors="ignore")


def blocks(path):
    """One dict per report block: the regex groups of each line."""
    out, cur = [], None
    for line in read_text(path).splitlines():
        m = HEAD.search(line)
        if m:
            cur = {"name": m.group(1)}
            out.append(cur)
            continue
        if cur is None:
            continue
        for key, rx in (("filter", FILTER), ("trades", TRADES), ("runs", RUNS), ("exact", EXACT),
                        ("streak", STREAK), ("after", AFTER), ("first", FIRST),
                        ("blocks", BLOCKS), ("inside", INSIDE)):
            m = rx.search(line)
            if m and key not in cur:
                cur[key] = m.groups()
    return out


def read_deals(path):
    with open(path, newline="", encoding="ascii", errors="ignore") as f:
        return [r for r in csv.DictReader(f) if r["type"] in ("0", "1")]   # buy and sell deals only


def epoch(text):
    return int(datetime.strptime(text, "%Y.%m.%d %H:%M:%S").replace(tzinfo=timezone.utc).timestamp())


# ---------------------------------------------------------------- deals to trades
def rebuild(deals, flt):
    """Trades closed inside the filter window, as (close_msc, position, net), in close order."""
    symbol, mfrom, mto, t_from, t_to = flt
    mfrom, mto = int(mfrom), int(mto)
    from_msc, to_msc = epoch(t_from) * 1000, epoch(t_to) * 1000 + 999
    trades, latest = [], {}
    for d in deals:
        pos, msc, entry = int(d["position"]), int(d["time_msc"]), int(d["entry"])
        vol = float(d["volume"])
        pnl = float(d["profit"]) + float(d["swap"])
        costs = float(d["commission"]) + float(d["fee"])
        t = latest.get(pos)
        if t is None:
            t = {"pos": pos, "open": msc, "close": 0, "net": 0.0, "vin": 0.0, "vout": 0.0,
                 "magic": int(d["magic"]), "symbol": d["symbol"], "entry_seen": False}
            trades.append(t)
            latest[pos] = t
        if entry == ENTRY_IN:
            t["net"] += pnl + costs
            t["vin"] += vol
            if not t["entry_seen"]:
                t["magic"], t["entry_seen"] = int(d["magic"]), True
        elif entry == ENTRY_INOUT:                 # netting reversal: close, then open the rest
            closing = max(t["vin"] - t["vout"], 0.0)
            share = min(closing / vol, 1.0) if vol > 0 else 1.0
            t["net"] += pnl + costs * share
            t["vout"] += closing
            t["close"] = msc
            n = {"pos": pos, "open": msc, "close": 0, "net": costs * (1 - share), "vin": vol - closing,
                 "vout": 0.0, "magic": int(d["magic"]), "symbol": d["symbol"], "entry_seen": True}
            trades.append(n)
            latest[pos] = n
        else:                                      # DEAL_ENTRY_OUT and DEAL_ENTRY_OUT_BY
            t["net"] += pnl + costs
            t["vout"] += vol
            t["close"] = max(t["close"], msc)
    kept, still_open = [], 0
    for t in trades:
        if symbol != "any" and t["symbol"] != symbol:
            continue
        if (mfrom >= 0 and t["magic"] < mfrom) or (mto >= 0 and t["magic"] > mto):
            continue
        if t["open"] > to_msc:
            continue
        if t["close"] == 0 or t["vout"] < t["vin"] - 1e-8 or t["close"] > to_msc:
            still_open += 1
            continue
        if t["close"] < from_msc:
            continue
        kept.append((t["close"], t["pos"], t["net"]))
    kept.sort()
    return kept, still_open


# ---------------------------------------------------------------- statistics
def runs_of(seq):
    return 1 + sum(seq[i] != seq[i - 1] for i in range(1, len(seq))) if seq else 0


def longest_loss(seq):
    best = cur = 0
    for w in seq:
        cur = 0 if w else cur + 1
        best = max(best, cur)
    return best


def rate_after(seq, k, exactly):
    """Loss rate of trades after k or more losses (exactly=False) or after exactly k (True)."""
    run = n = lost = 0
    for w in seq:
        if (run == k) if exactly else (run >= k):
            n += 1
            lost += not w
        run = 0 if w else run + 1
    return (lost / n if n else None), n


def pmf_num(n1, n2, r):
    k = r // 2
    if r % 2 == 0:
        return 2 * comb(n1 - 1, k - 1) * comb(n2 - 1, k - 1)
    return comb(n1 - 1, k) * comb(n2 - 1, k - 1) + comb(n1 - 1, k - 1) * comb(n2 - 1, k)


def exact_two_sided(n1, n2, runs):
    top = 2 * min(n1, n2) + (0 if n1 == n2 else 1)
    lo = sum(pmf_num(n1, n2, r) for r in range(2, runs + 1))
    hi = sum(pmf_num(n1, n2, r) for r in range(runs, top + 1))
    return min(1.0, 2 * min(lo, hi) / comb(n1 + n2, n1))


def moments(n1, n2):
    n, p = n1 + n2, 2 * n1 * n2
    return p / n + 1, sqrt(p * (p - n) / (n * n * (n - 1)))


def phi(z):
    return 0.5 * (1 + erf(z / sqrt(2)))


def normal_two_sided(n1, n2, runs):
    mean, sd = moments(n1, n2)
    return min(1.0, 2 * min(phi((runs + 0.5 - mean) / sd), 1 - phi((runs - 0.5 - mean) / sd)))


def block_start(b, n, blocks):
    return b * n // blocks


def chi2_upper(x, df):
    """Upper tail of the chi-square distribution: regularized gamma Q(df/2, x/2)."""
    a, x = df / 2.0, x / 2.0
    if x <= 0:
        return 1.0
    if x < a + 1:                                  # series for P, then Q = 1 - P
        term = total = 1.0 / a
        ap = a
        while abs(term) > 1e-15 * abs(total):
            ap += 1
            term *= x / ap
            total += term
        return 1.0 - total * exp(-x + a * log(x) - lgamma(a))
    b, c, d = x + 1 - a, 1e300, 1 / (x + 1 - a)    # continued fraction for Q
    h, i = d, 1
    while True:
        an = -i * (i - a)
        b += 2
        d = an * d + b
        d = 1e-300 if abs(d) < 1e-300 else d
        c = b + an / c
        c = 1e-300 if abs(c) < 1e-300 else c
        d = 1 / d
        h *= d * c
        if abs(d * c - 1) < 1e-15:
            break
        i += 1
    return exp(-x + a * log(x) - lgamma(a)) * h


def drift(seq, blocks):
    n = len(seq)
    pool = sum(seq) / n
    rates, chi2 = [], 0.0
    for b in range(blocks):
        part = seq[block_start(b, n, blocks):block_start(b + 1, n, blocks)]
        w = sum(part)
        rates.append(w / len(part))
        chi2 += (w - len(part) * pool) ** 2 / (len(part) * pool * (1 - pool))
    return rates, chi2, chi2_upper(chi2, blocks - 1)


def shuffled(seq, k, count, blocks, rnd):
    """Python's own shuffles: streak p, after/first mean and p, then the within-block p values."""
    longest = longest_loss(seq)
    a_obs, _ = rate_after(seq, k, False)
    f_obs, _ = rate_after(seq, k, True)
    work, n = list(seq), len(seq)
    hits = {"streak": 0, "after": 0, "first": 0}
    sums, cnt = {"after": 0.0, "first": 0.0}, {"after": 0, "first": 0}
    for _ in range(count):
        rnd.shuffle(work)
        hits["streak"] += longest_loss(work) >= longest
        for key, obs, ex in (("after", a_obs, False), ("first", f_obs, True)):
            r, m = rate_after(work, k, ex)
            if obs is not None and m:
                sums[key] += r
                cnt[key] += 1
                hits[key] += r >= obs
    res = {"streak p": (hits["streak"] + 1) / (count + 1)}
    for key in ("after", "first"):
        res[key + " shuffled"] = 100 * sums[key] / cnt[key] if cnt[key] else 0.0
        res[key + " p"] = (hits[key] + 1) / (cnt[key] + 1) if cnt[key] else 1.0
    obs_runs, lo = runs_of(seq), 0
    hi = b_streak = 0
    work = list(seq)
    for _ in range(count):
        for b in range(blocks):
            s, e = block_start(b, n, blocks), block_start(b + 1, n, blocks)
            part = work[s:e]
            rnd.shuffle(part)
            work[s:e] = part
        r = runs_of(work)
        lo += r <= obs_runs
        hi += r >= obs_runs
        b_streak += longest_loss(work) >= longest
    res["block runs p"] = min(1.0, 2 * (min(lo, hi) + 1) / (count + 1))
    res["block streak p"] = (b_streak + 1) / (count + 1)
    return res


# ---------------------------------------------------------------- power simulation
def markov(n, q, d, rnd):
    """n outcomes (True = win) with loss rate q, where a loss raises P(next loss) to q + d."""
    a = q + d                                      # P(loss | loss)
    b = q * (1 - a) / (1 - q)                      # P(loss | win), keeps the loss rate at q
    loss = rnd.random() < q
    out = []
    for _ in range(n):
        out.append(not loss)
        loss = rnd.random() < (a if loss else b)
    return out


def power(n1, n2, k, rnd, sims=2000):
    """Share of Markov sequences each check flags at 5%, with critical values from shuffles."""
    seq = [True] * n1 + [False] * n2
    n, q = n1 + n2, n2 / (n1 + n2)
    runs, longest, after, first = [], [], [], []
    for _ in range(sims):
        rnd.shuffle(seq)
        runs.append(runs_of(seq))
        longest.append(longest_loss(seq))
        after.append(rate_after(seq, k, False)[0] or 0.0)
        first.append(rate_after(seq, k, True)[0] or 0.0)
    pick = lambda v, qq: sorted(v)[int(qq * (len(v) - 1))]
    crit = (pick(runs, 0.025), pick(longest, 0.95), pick(after, 0.95), pick(first, 0.95))
    rows = {}
    for d in (0.05, 0.10):
        flag = [0, 0, 0, 0]
        for _ in range(sims):
            s = markov(n, q, d, rnd)
            flag[0] += runs_of(s) < crit[0]
            flag[1] += longest_loss(s) > crit[1]
            flag[2] += (rate_after(s, k, False)[0] or 0.0) > crit[2]
            flag[3] += (rate_after(s, k, True)[0] or 0.0) > crit[3]
        rows[d] = [100.0 * f / sims for f in flag]
    return rows


# ---------------------------------------------------------------- main
def main(argv):
    count, do_power = 10000, False
    if "--shuffles" in argv:
        i = argv.index("--shuffles")
        count = int(argv[i + 1])
        del argv[i:i + 2]
    if "--power" in argv:
        argv.remove("--power")
        do_power = True
    pairs = list(zip(argv[0::2], argv[1::2]))
    checked = differ = 0
    samples = []
    for log_path, deals_path in pairs:
        deals = read_deals(deals_path)
        name = lambda p: p.replace(chr(92), "/").split("/")[-1]
        print(f"{name(log_path)} + {name(deals_path)}: {len(deals)} buy and sell deals")
        for b in blocks(log_path):
            if "filter" not in b or "trades" not in b:
                continue
            kept, still_open = rebuild(deals, b["filter"])
            eps = float(b["trades"][5])
            mode = b["trades"][4]
            seq, be = [], 0
            for _, _, net in kept:
                if abs(net) <= eps:
                    be += 1
                    if mode == "BE_SKIP":
                        continue
                    seq.append(mode == "BE_AS_WIN")
                    continue
                seq.append(net > 0)
            n1, n2 = sum(seq), len(seq) - sum(seq)
            r = runs_of(seq)
            mean, sd = moments(n1, n2)
            k = int(b["after"][0])
            a_rate, a_n = rate_after(seq, k, False)
            f_rate, f_n = rate_after(seq, k, True)
            nb = int(b["blocks"][0])
            rates, chi2, dp = drift(seq, nb)
            mine = [len(seq), n1, n2, be, r, f"{mean:.1f}", f"{(r - mean) / sd:.2f}",
                    f"{exact_two_sided(n1, n2, r):.4f}", f"{normal_two_sided(n1, n2, r):.4f}",
                    longest_loss(seq), f"{100 * a_rate:.1f}", a_n, f"{100 * f_rate:.1f}", f_n,
                    " ".join(f"{100 * x:.1f}%" for x in rates), f"{chi2:.2f}", f"{dp:.3f}"]
            printed = [int(b["trades"][0]), int(b["trades"][1]), int(b["trades"][2]), int(b["trades"][3]),
                       int(b["runs"][0]), b["runs"][1], b["runs"][2], b["exact"][0], b["exact"][1],
                       int(b["streak"][0]), b["after"][1], int(b["after"][2]), b["first"][1],
                       int(b["first"][2]), b["blocks"][1].strip(), b["blocks"][2], b["blocks"][3]]
            bad = [i for i, (x, y) in enumerate(zip(mine, printed)) if str(x) != str(y)]
            checked += len(mine)
            differ += len(bad)
            sh = shuffled(seq, k, count, nb, random.Random(20261003))
            print(f"  {b['name']}: {len(mine) - len(bad)} of {len(mine)} values match the report"
                  + ("" if not bad else f", DIFFERENT: {bad}"))
            print(f"    {len(seq)} trades ({n1} W, {n2} L), runs {r}, exact p {exact_two_sided(n1, n2, r):.4f}, "
                  f"longest {longest_loss(seq)}, after {100 * a_rate:.1f}%, first {100 * f_rate:.1f}%")
            print(f"    shuffles Python/MQL5: streak p {sh['streak p']:.3f}/{b['streak'][1]}, "
                  f"after p {sh['after p']:.3f}/{b['after'][6]}, first p {sh['first p']:.3f}/{b['first'][6]}")
            print(f"    within blocks Python/MQL5: runs p {sh['block runs p']:.3f}/{b['inside'][1]}, "
                  f"streak p {sh['block streak p']:.3f}/{b['inside'][2]}")
            stem = name(log_path).rsplit(".", 1)[0]
            samples.append((f"{stem}, {b['name'].split(', ')[-1]}", n1, n2, k, sd))
    print(f"values rebuilt from the deals and compared: {checked}, different: {differ}")
    print(f"shuffle p values use {count} Python shuffles, so they match MQL5 only within Monte Carlo error")
    if do_power:
        rnd = random.Random(7)
        print("power at 5%: % of 2000 Markov sequences flagged by the runs, streak, after and first")
        print("checks, when a loss raises the chance of another loss by 5 or by 10 points")
        print(f"  {'sample':<30}{'trades':>7}   {'+5 points':<26}{'+10 points'}")
        for name, n1, n2, k, sd in samples:
            rows = power(n1, n2, k, rnd)
            print(f"  {name[:30]:<30}{n1 + n2:>7}   " + "".join(f"{x:>6.0f}" for x in rows[0.05])
                  + "  " + "".join(f"{x:>6.0f}" for x in rows[0.10]))

if __name__ == "__main__":
    if len(sys.argv) < 3:
        sys.exit(__doc__)
    main(sys.argv[1:])
