from __future__ import annotations

import argparse
import csv
import hashlib
import json
import math
import os
import re
import shlex
import subprocess
import time
import ast
import calendar
import itertools
from datetime import datetime, timedelta
from dataclasses import dataclass, field, asdict
from pathlib import Path
from statistics import median
from typing import Any
import xml.etree.ElementTree as ET

Range = tuple[Any, Any, Any]


class SaveAndExit(Exception):
    """Raised when the user chooses to save the current state and exit."""


MQL_ENUM_VALUES = {
    "PERIOD_M1": 1,
    "PERIOD_M2": 2,
    "PERIOD_M3": 3,
    "PERIOD_M4": 4,
    "PERIOD_M5": 5,
    "PERIOD_M6": 6,
    "PERIOD_M10": 10,
    "PERIOD_M12": 12,
    "PERIOD_M15": 15,
    "PERIOD_M20": 20,
    "PERIOD_M30": 30,
    "PERIOD_H1": 16385,
    "PERIOD_H2": 16386,
    "PERIOD_H3": 16387,
    "PERIOD_H4": 16388,
    "PERIOD_H6": 16390,
    "PERIOD_H8": 16392,
    "PERIOD_H12": 16396,
    "PERIOD_D1": 16408,
    "PERIOD_W1": 32769,
    "PERIOD_MN1": 49153,
}


@dataclass
class Subgroup:
    tag: str
    title: str
    group: str = "Default"
    ranges: dict[str, Range] = field(default_factory=dict)
    fixed: dict[str, Any] = field(default_factory=dict)
    wfe: dict[str, str] = field(default_factory=dict)


@dataclass
class OptimizerConfig:
    terminal_exe: str
    data_path: str
    expert_name: str
    csv_prefix: str
    run_dir: str
    symbols: list[str]
    from_date: str
    to_date: str
    period: str
    model: int = 1
    optimization_criterion: int = 6
    deposit: float = 10000
    currency: str = "USD"
    leverage: str = "1:100"
    genetic_threshold_passes: int = 500
    max_boundary_expansions_per_subgroup: int = 1
    timeout_seconds: int = 7200
    close_terminal_before_run: bool = True
    selection_mode: str = "semi_auto"  # auto | semi_auto | manual
    top_sets_to_show: int = 20
    pause_after_each_subgroup: bool = True
    allow_user_override: bool = True
    base: dict[str, Any] = field(default_factory=dict)
    subgroups: list[dict[str, Any]] = field(default_factory=list)
    expand_limits: dict[str, tuple[float, float]] = field(default_factory=dict)
    wfe_insample_months: int = 6
    wfe_outsample_months: int = 3
    wfe_step_months: int = 3
    wfe_top_n: int = 5
    wfe_is_top_n: int = 20
    wfe_frequency_top_n: int = 5
    wfe_min_efficiency: float = 0.40
    wfe_max_efficiency: float = 1.60
    wfe_metric: str = "global_score"
    enable_wfe: bool = False
    wfe_from_date: str = ""
    wfe_to_date: str = ""
    wfe_use_top_candidates: bool = True
    wfe_max_candidates: int = 30


INPUT_RE = re.compile(
    r"^\s*input\s+(?P<type>bool|int|long|double|float|string|ENUM_[A-Za-z0-9_]+|[A-Za-z0-9_]+)\s+"
    r"(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?P<value>[^;]+);"
)


def parse_value(value: str) -> Any:
    v = str(value).strip()
    if len(v) >= 2 and ((v[0] == v[-1] == '"') or (v[0] == v[-1] == "'")):
        return v[1:-1]
    lv = v.lower()
    if lv == "true":
        return True
    if lv == "false":
        return False
    try:
        if "." not in v and "," not in v:
            return int(v)
        return float(v.replace(",", "."))
    except ValueError:
        arithmetic = re.fullmatch(r"[0-9\s\+\-\*\/\(\)\.]+", v)
        if arithmetic:
            try:
                return eval_arithmetic(v)
            except Exception:
                pass
        return v


def eval_arithmetic(expr: str) -> int | float:
    allowed = (
        ast.Expression,
        ast.BinOp,
        ast.UnaryOp,
        ast.Constant,
        ast.Add,
        ast.Sub,
        ast.Mult,
        ast.Div,
        ast.USub,
        ast.UAdd,
    )
    tree = ast.parse(expr, mode="eval")
    if not all(isinstance(node, allowed) for node in ast.walk(tree)):
        raise ValueError(expr)
    value = eval(compile(tree, "<expr>", "eval"), {"__builtins__": {}}, {})
    if abs(value - round(value)) < 1e-9:
        return int(round(value))
    return float(value)


def mt5_value(value: Any) -> str:
    if isinstance(value, bool):
        return "true" if value else "false"
    if isinstance(value, float):
        return f"{value:.8g}"
    if isinstance(value, str) and value in MQL_ENUM_VALUES:
        return str(MQL_ENUM_VALUES[value])
    return str(value)


def as_float(value: Any, default: float = 0.0) -> float:
    try:
        return float(str(value).replace(",", "."))
    except (TypeError, ValueError):
        return default


def parse_kv_line(line: str) -> dict[str, Any]:
    clean = line.strip().replace("//", "", 1).strip()
    for prefix in ("@opt", "@fixed", "@system"):
        if clean.startswith(prefix):
            clean = clean[len(prefix):].strip()
            break
    lexer = shlex.shlex(clean, posix=True)
    lexer.whitespace_split = True
    lexer.commenters = ""
    result: dict[str, Any] = {}
    for token in lexer:
        if "=" not in token:
            continue
        k, v = token.split("=", 1)
        result[k.strip()] = parse_value(v.strip())
    return result


INCLUDE_RE = re.compile(r'^\s*#include\s+[<"](?P<file>[^>"]+)[>"]')


def collect_mq5_lines(path: Path, visited: set[Path] | None = None) -> list[tuple[Path, str]]:
    """
    Reads the main .mq5 file and locally included #include files, when they exist.
    """
    visited = visited or set()
    path = path.resolve()

    if path in visited:
        return []
    visited.add(path)

    if not path.exists():
        return []

    try:
        raw_lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
    except Exception:
        raw_lines = path.read_text(encoding="latin1", errors="ignore").splitlines()

    result: list[tuple[Path, str]] = []

    for raw in raw_lines:
        result.append((path, raw))

        match = INCLUDE_RE.match(raw.strip())
        if not match:
            continue

        include_name = match.group("file").replace("\\", "/")
        candidates = [
            path.parent / include_name,
            path.parent / Path(include_name).name,
            path.parent.parent / include_name,
            path.parent.parent / Path(include_name).name,
        ]

        for candidate in candidates:
            if candidate.exists():
                result.extend(collect_mq5_lines(candidate, visited))
                break

    return result


ENUM_DECL_RE = re.compile(r"\benum\s+[A-Za-z_][A-Za-z0-9_]*\s*\{")
ENUM_ITEM_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:=\s*([-+]?\d+))?\s*,?\s*(?://.*)?$")


def update_enum_values_from_lines(lines: list[tuple[Path, str]]) -> None:
    """
    Reads simple MQL5 enums and registers name -> value mappings for numeric .set output.
    """
    in_enum = False
    next_value = 0
    for _, raw in lines:
        line = raw.strip()
        if not in_enum:
            if ENUM_DECL_RE.search(line):
                in_enum = True
                next_value = 0
                after_brace = line.split("{", 1)[1]
                if after_brace:
                    line = after_brace
                else:
                    continue
            else:
                continue

        if "}" in line:
            before = line.split("}", 1)[0].strip()
            if before:
                for part in before.split(","):
                    registered = register_enum_item(part, next_value)
                    if registered is not None:
                        next_value = registered + 1
            in_enum = False
            continue

        for part in line.split(","):
            registered = register_enum_item(part, next_value)
            if registered is not None:
                next_value = registered + 1


def register_enum_item(text: str, default_value: int) -> int | None:
    clean = text.strip()
    if not clean:
        return None
    match = ENUM_ITEM_RE.match(clean)
    if not match:
        return None
    name = match.group(1)
    explicit = match.group(2)
    value = int(explicit) if explicit is not None else default_value
    MQL_ENUM_VALUES[name] = value
    return value


def parse_simple_opt_values(text: str) -> tuple[Any, Any, Any] | None:
    """
    Reads compact formats:
      // @opt 10 2 40
      // @opt false true

    For bool values with two entries, step is assumed as 1:
      false, 1, true
    """
    clean = text.strip()
    clean = clean.replace("//", "", 1).strip()
    if not clean.startswith("@opt"):
        return None

    clean = clean[len("@opt"):].strip()
    if not clean:
        return None

    parts = clean.split()
    if len(parts) == 2:
        start_v = parse_value(parts[0])
        stop_v = parse_value(parts[1])
        return (start_v, 1, stop_v)

    if len(parts) >= 3:
        start_v = parse_value(parts[0])
        step_v = parse_value(parts[1])
        stop_v = parse_value(parts[2])
        return (start_v, step_v, stop_v)

    return None


def parse_mq5_annotations(mq5_path: Path) -> tuple[dict[str, Any], list[Subgroup]]:
    """
    Flexible parser for two annotation styles.

    Style 1, explicit:
      // @opt group=Regime subgroup=G1A_EMAS label="EMAs" start=10 step=2 stop=40
      input int fastEMAPeriod = 24;

    Style 2, compact:
      // @group G1_REGIME
      // @subgroup G1B_EMAS
      // @opt 10 2 40
      input int fastEMAPeriod = 24;

      // @fixed
      input ulong magicBuy = 1003;

    Rules:
    - @group defines the current group.
    - @subgroup defines the current subgroup.
    - @opt with 3 values becomes start/step/stop.
    - @opt with 2 values becomes start/1/stop, useful for bool and simple enums.
    - @fixed immediately above an input adds that input to the current subgroup fixed values.
    - @opt in key=value format remains supported.
    - @wfe immediately above an @opt input marks it for individual WFE.
    - @wfe1, @wfe2, @wfe3 mark slots in one grouped WFE set inside the same subgroup.
      A grouped set must have at least @wfe1 and @wfe2. Do not repeat the same slot.
    """
    if not mq5_path.exists():
        raise FileNotFoundError(mq5_path)

    base: dict[str, Any] = {}
    subgroup_map: dict[str, Subgroup] = {}

    current_group = "Default"
    current_subgroup = "DEFAULT"
    current_title = "DEFAULT"

    pending_opt: dict[str, Any] | None = None
    pending_fixed = False
    pending_wfe: str | None = None

    lines = collect_mq5_lines(mq5_path)
    update_enum_values_from_lines(lines)

    for source_path, raw in lines:
        stripped = raw.strip()

        if not stripped:
            continue

        # ------------------------------------------------------------
        # Compact format: // @group G1_REGIME
        # ------------------------------------------------------------
        if stripped.startswith("// @group"):
            clean = stripped.replace("//", "", 1).strip()
            parts = clean.split(maxsplit=1)
            if len(parts) == 2:
                current_group = parts[1].strip()
            continue

        # ------------------------------------------------------------
        # Compact format: // @subgroup G1B_EMAS
        # ------------------------------------------------------------
        if stripped.startswith("// @subgroup"):
            clean = stripped.replace("//", "", 1).strip()
            parts = clean.split(maxsplit=1)
            if len(parts) == 2:
                current_subgroup = parts[1].strip()
                current_title = current_subgroup
                subgroup_map.setdefault(
                    current_subgroup,
                    Subgroup(tag=current_subgroup, title=current_title, group=current_group)
                )
            continue

        # ------------------------------------------------------------
        # Compact @wfe:
        # // @wfe
        # // @wfe1
        # input Type parameter = value;
        # ------------------------------------------------------------
        if stripped.startswith("// @wfe"):
            clean = stripped.replace("//", "", 1).strip().split()[0]
            if clean == "@wfe":
                pending_wfe = "individual"
            else:
                match_wfe = re.fullmatch(r"@wfe([1-3])", clean)
                pending_wfe = f"group{match_wfe.group(1)}" if match_wfe else None
            continue

        # ------------------------------------------------------------
        # Full @fixed with key=value:
        # // @fixed subgroup=G1A timeframeRegime=5
        # ------------------------------------------------------------
        if stripped.startswith("// @fixed") and "=" in stripped:
            fixed = parse_kv_line(stripped)
            subgroup_tag = str(fixed.pop("subgroup", current_subgroup)).strip()
            group = str(fixed.pop("group", current_group)).strip()
            title = str(fixed.pop("label", subgroup_tag)).strip()
            if subgroup_tag:
                sg = subgroup_map.setdefault(
                    subgroup_tag,
                    Subgroup(tag=subgroup_tag, title=title, group=group)
                )
                if sg.group == "Default" and group != "Default":
                    sg.group = group
                if not sg.title or sg.title == subgroup_tag:
                    sg.title = title
                sg.fixed.update(fixed)
            pending_fixed = False
            pending_wfe = None
            continue

        # ------------------------------------------------------------
        # Compact @fixed:
        # // @fixed
        # input Type parameter = value;
        # ------------------------------------------------------------
        if stripped == "// @fixed" or stripped.startswith("// @fixed"):
            pending_fixed = True
            pending_opt = None
            pending_wfe = None
            continue

        # ------------------------------------------------------------
        # @opt: accepts both key=value and compact formats
        # ------------------------------------------------------------
        if stripped.startswith("// @opt"):
            if "=" in stripped:
                parsed = parse_kv_line(stripped)
                pending_opt = parsed
            else:
                simple = parse_simple_opt_values(stripped)
                if simple is not None:
                    start_v, step_v, stop_v = simple
                    pending_opt = {
                        "group": current_group,
                        "subgroup": current_subgroup,
                        "label": current_subgroup,
                        "start": start_v,
                        "step": step_v,
                        "stop": stop_v,
                    }
            pending_fixed = False
            continue

        match = INPUT_RE.match(raw)
        if match:
            name = match.group("name")
            default_value = parse_value(match.group("value").strip())
            base[name] = default_value

            # Optimizable input
            if pending_opt:
                subgroup_tag = str(pending_opt.get("subgroup", current_subgroup)).strip()
                group = str(pending_opt.get("group", current_group)).strip()
                title = str(pending_opt.get("label", subgroup_tag)).strip()
                start_v = pending_opt.get("start", default_value)
                step_v = pending_opt.get("step", 1)
                stop_v = pending_opt.get("stop", default_value)

                sg = subgroup_map.setdefault(
                    subgroup_tag,
                    Subgroup(tag=subgroup_tag, title=title, group=group)
                )
                if sg.group == "Default" and group != "Default":
                    sg.group = group
                if not sg.title or sg.title == subgroup_tag:
                    sg.title = title
                sg.ranges[name] = (start_v, step_v, stop_v)
                if pending_wfe:
                    sg.wfe[name] = pending_wfe

                pending_opt = None
                pending_fixed = False
                pending_wfe = None
                continue

            # Fixed input for the current subgroup
            if pending_fixed:
                sg = subgroup_map.setdefault(
                    current_subgroup,
                    Subgroup(tag=current_subgroup, title=current_title, group=current_group)
                )
                sg.fixed[name] = default_value
                pending_fixed = False
                pending_wfe = None
                continue

            pending_wfe = None
            continue

        # If a pending @opt exists and another code line appears before the input,
        # discard it to avoid assigning the metadata to the wrong input.
        if pending_opt and not stripped.startswith("//"):
            pending_opt = None
        if pending_fixed and not stripped.startswith("//"):
            pending_fixed = False
        if pending_wfe and not stripped.startswith("//"):
            pending_wfe = None

    # Remove empty subgroups that have neither ranges nor fixed values.
    subgroups = [
        sg for sg in subgroup_map.values()
        if sg.ranges or sg.fixed
    ]

    return base, subgroups


def parse_set_file(set_path: Path) -> dict[str, Any]:
    defaults: dict[str, Any] = {}
    if not set_path or not set_path.exists():
        return defaults
    for enc in ("utf-16", "utf-8", "ascii", "latin1"):
        try:
            lines = set_path.read_text(encoding=enc, errors="ignore").splitlines()
            local: dict[str, Any] = {}
            for line in lines:
                if not line or line.strip().startswith(";") or "=" not in line:
                    continue
                key, rest = line.split("=", 1)
                value = rest.split("||", 1)[0]
                local[key.strip()] = parse_value(value.strip())
            if local:
                defaults.update(local)
                break
        except Exception:
            continue
    return defaults


def ask(prompt: str, default: Any | None = None) -> str:
    suffix = f" [{default}]" if default not in (None, "") else ""
    value = input(f"{prompt}{suffix}: ").strip()
    if not value and default is not None:
        return str(default)
    return value


def ask_with_hint(prompt: str, default: Any | None = None, hint: str | None = None) -> str:
    """
    Shows a generic example in the wizard while still using the real inferred
    default when the user presses ENTER.
    """
    shown = hint if hint not in (None, "") else default
    suffix = f" [{shown}]" if shown not in (None, "") else ""
    value = input(f"{prompt}{suffix}: ").strip()
    if not value and default is not None:
        return str(default)
    return value


def ask_bool(prompt: str, default: bool = False) -> bool:
    d = "y" if default else "n"
    value = ask(prompt + " (y/n)", d).lower()
    return value in {"y", "yes", "1", "true"}



def detect_csv_prefix_from_base(base: dict[str, Any], expert_name: str) -> str:
    """
    Detects the CSV prefix from the csvExportPrefix input read from the .mq5 file.
    If the input does not exist or is empty, use the EA name.
    """
    value = base.get("csvExportPrefix", "")
    if isinstance(value, str) and value.strip():
        return value.strip()
    return expert_name


def validate_required_optimizer_inputs(base: dict[str, Any]) -> list[str]:
    """
    Checks whether the EA has the minimum inputs expected by the automation.
    """
    required = ["optimizationTag", "saveOptimizationPasses", "dummyOptimizationPass", "csvExportPrefix"]
    return [name for name in required if name not in base]


def detect_hardcoded_csv_prefix(mq5_path: Path) -> str | None:
    """
    Detects the prefix in EAs that write CSV files with this literal pattern:
    "PREFIX_" + optimizationTag + "_frames.csv".
    """
    if not mq5_path.exists():
        return None
    text = mq5_path.read_text(encoding="utf-8", errors="ignore")
    match = re.search(
        r'"(?P<prefix>[A-Za-z0-9_\-]+)_"\s*\+\s*optimizationTag\s*\+\s*"_frames\.csv"',
        text,
    )
    if match:
        return match.group("prefix")
    return None


def optimizer_dir_from_config_path(config_path: Path | None, cfg: OptimizerConfig) -> Path:
    if config_path:
        return config_path.resolve().parent
    return Path(cfg.data_path) / "MQL5" / "Experts" / "MT5_Global_Optimizer"


def infer_data_path_from_mq5(mq5_path: Path) -> str | None:
    for parent in mq5_path.resolve().parents:
        if parent.name.upper() == "MQL5":
            return str(parent.parent)
    return None


def infer_data_path_from_config(config_path: Path | None, expert_name: str) -> str | None:
    if not config_path:
        return None
    for parent in config_path.resolve().parents:
        if parent.name.upper() == "MQL5":
            data_path = parent.parent
            mq5_path = data_path / "MQL5" / "Experts" / f"{expert_name}.mq5"
            ex5_path = data_path / "MQL5" / "Experts" / f"{expert_name}.ex5"
            if mq5_path.exists() or ex5_path.exists():
                return str(data_path)
    return None


def default_terminal_exe() -> str:
    candidates = [
        Path(r"C:\Program Files\MetaTrader 5\terminal64.exe"),
        Path(r"C:\Program Files\MetaTrader 5 Terminal\terminal64.exe"),
    ]
    for candidate in candidates:
        if candidate.exists():
            return str(candidate)
    return str(candidates[0])


def default_data_path_hint() -> str:
    appdata = os.environ.get("APPDATA")
    if appdata:
        return str(Path(appdata) / "MetaQuotes" / "Terminal" / "YOUR_TERMINAL_ID")
    return str(Path.home() / "AppData" / "Roaming" / "MetaQuotes" / "Terminal" / "YOUR_TERMINAL_ID")


def normalize_config(cfg: OptimizerConfig, config_path: Path | None = None) -> OptimizerConfig:
    """
    Cleans manually created or copy-pasted configs and keeps the .set compatible
    only with real EA inputs.
    """
    inferred_data_path = infer_data_path_from_config(config_path, cfg.expert_name)
    if inferred_data_path and Path(inferred_data_path).resolve() != Path(cfg.data_path).resolve():
        print(f"Warning: config data_path points to another terminal. Using: {inferred_data_path}", flush=True)
        cfg.data_path = inferred_data_path

    data_path = Path(cfg.data_path)
    mq5_path = data_path / "MQL5" / "Experts" / f"{cfg.expert_name}.mq5"
    if mq5_path.exists():
        parsed_base, parsed_subgroups = parse_mq5_annotations(mq5_path)
        valid_inputs = set(parsed_base)

        clean_base = {
            k: parse_value(v) if isinstance(v, str) else v
            for k, v in cfg.base.items()
            if k in valid_inputs
        }
        for key, value in parsed_base.items():
            clean_base.setdefault(key, value)
        cfg.base = clean_base

        clean_subgroups: list[dict[str, Any]] = []
        source_subgroups = cfg.subgroups or [asdict(sg) for sg in parsed_subgroups]
        parsed_wfe_by_tag = {sg.tag: sg.wfe for sg in parsed_subgroups}
        for sg in source_subgroups:
            ranges = {k: v for k, v in sg.get("ranges", {}).items() if k in valid_inputs}
            fixed = {k: v for k, v in sg.get("fixed", {}).items() if k in valid_inputs}
            source_wfe = sg.get("wfe", {}) or parsed_wfe_by_tag.get(sg.get("tag"), {})
            wfe = {k: v for k, v in source_wfe.items() if k in valid_inputs and k in ranges}
            if ranges or fixed:
                item = dict(sg)
                item["ranges"] = ranges
                item["fixed"] = fixed
                item["wfe"] = wfe
                clean_subgroups.append(item)
        if clean_subgroups:
            cfg.subgroups = clean_subgroups

        hardcoded_prefix = detect_hardcoded_csv_prefix(mq5_path)
        if hardcoded_prefix:
            cfg.csv_prefix = hardcoded_prefix
        else:
            cfg.csv_prefix = detect_csv_prefix_from_base(cfg.base, cfg.expert_name)

    run_dir = Path(cfg.run_dir)
    if not run_dir.is_absolute() or run_dir.suffix.lower() == ".csv":
        cfg.run_dir = str(optimizer_dir_from_config_path(config_path, cfg) / "runs" / f"{cfg.expert_name}_global")

    return cfg


def wizard() -> None:
    print("\n=== MT5 Global Optimizer Wizard ===")
    print("Creates a config.json manually or by reading @opt/@fixed comments from the EA.\n")

    mq5_file = ask_with_hint(
        "Path to the .mq5 file for automatic @opt/@fixed reading, or press ENTER to skip",
        "",
        r"C:\Path\To\Experts\MyExpertAdvisor.mq5",
    )
    inferred_data_path = infer_data_path_from_mq5(Path(mq5_file)) if mq5_file else None
    terminal_exe = ask_with_hint(
        "Path to terminal64.exe",
        default_terminal_exe(),
        r"C:\Program Files\MetaTrader 5\terminal64.exe",
    )
    data_path = ask_with_hint(
        "MT5 Data Path",
        inferred_data_path or default_data_path_hint(),
        r"C:\Users\YourUser\AppData\Roaming\MetaQuotes\Terminal\YOUR_TERMINAL_ID",
    )

    base: dict[str, Any] = {}
    subgroups: list[Subgroup] = []

    if mq5_file:
        parsed_base, parsed_subgroups = parse_mq5_annotations(Path(mq5_file))
        base.update(parsed_base)
        subgroups.extend(parsed_subgroups)
        print(f"OK: {len(base)} inputs read and {len(subgroups)} subgroups detected in the .mq5 file.")

        missing_required = validate_required_optimizer_inputs(base)
        if missing_required:
            print("Warning: required inputs were not found in the EA:", ", ".join(missing_required))
            print("Add these inputs for full optimizer integration.")

    set_file = ask_with_hint(
        "Path to the base .set file to override defaults, or press ENTER to skip",
        "",
        r"C:\Path\To\Presets\base.set",
    )
    if set_file:
        set_defaults = parse_set_file(Path(set_file))
        base.update(set_defaults)
        print(f"OK: {len(set_defaults)} defaults imported from the .set file.")

    expert_name = ask_with_hint(
        "Expert Advisor name in the Strategy Tester",
        Path(mq5_file).stem if mq5_file else "MyExpertAdvisor",
        "MyExpertAdvisor",
    )
    csv_prefix = (detect_hardcoded_csv_prefix(Path(mq5_file)) if mq5_file else None) or detect_csv_prefix_from_base(base, expert_name)
    run_dir = str(Path.cwd() / "runs" / f"{expert_name}_global")
    print("CSV prefix automatically detected from the EA or config.")
    print("Expected CSV pattern: <csvExportPrefix>_<optimizationTag>_frames.csv")
    print("Run folder automatically set under: ./runs/<expert_name>_global")

    symbols = [x.strip() for x in ask_with_hint("Symbols separated by commas", "EURUSD,GBPUSD,USDJPY", "SYMBOL1,SYMBOL2,SYMBOL3").split(",") if x.strip()]
    from_date = ask("Start date YYYY.MM.DD", "2022.01.01")
    to_date = ask("End date YYYY.MM.DD", "2025.12.31")
    period = ask("Test timeframe", "M5")
    model = int(ask("Tester model: 0=Every tick, 1=1min OHLC, 2=Open prices", "1"))
    currency = ask("Tester account currency", "USD").strip().upper() or "USD"
    timeout_seconds = int(ask("Timeout per run in seconds", "7200"))

    print("\nSet selection mode:")
    print("  auto      = apply the recommended set automatically")
    print("  manual    = always show TOP sets and ask")
    print("  semi_auto = ask only when the decision is sensitive")
    selection_mode = ask("Selection mode", "semi_auto").strip().lower()
    if selection_mode not in {"auto", "manual", "semi_auto"}:
        selection_mode = "semi_auto"
    top_sets_to_show = int(ask("How many TOP sets to show when selection is required", "20"))

    print("\nWalk-forward validation:")
    enable_wfe = ask_bool("Enable WFE stage in this config", False)
    wfe_from_date = from_date
    wfe_to_date = to_date
    wfe_insample_months = 6
    wfe_outsample_months = 3
    wfe_step_months = 3
    wfe_top_n = 5
    wfe_is_top_n = 1
    wfe_frequency_top_n = 1
    wfe_min_efficiency = 0.40
    wfe_max_efficiency = 1.60
    wfe_use_top_candidates = True
    wfe_max_candidates = 30
    if enable_wfe:
        wfe_from_date = ask("WFE start date YYYY.MM.DD", from_date)
        wfe_to_date = ask("WFE end date YYYY.MM.DD", to_date)
        wfe_insample_months = int(ask("WFE in-sample window in months", "6"))
        wfe_outsample_months = int(ask("WFE out-of-sample window in months", "3"))
        wfe_step_months = int(ask("WFE rolling step in months", str(wfe_outsample_months)))
        wfe_top_n = int(ask("How many TOP rows per subgroup to include as WFE candidates", "5"))
        wfe_use_top_candidates = ask_bool("Use TOP parameter candidates from optimizer results", True)
        wfe_max_candidates = int(ask("Maximum WFE candidates to test", "30"))

    if not subgroups or ask_bool("Do you also want to add subgroups manually", False):
        n = int(ask("How many subgroups do you want to register manually", "0"))
        for i in range(n):
            print(f"\nManual subgroup {i+1}/{n}")
            group = ask_with_hint("Main group", "Manual", "GroupName")
            tag = ask("Subgroup tag", f"G{i+1}")
            title = ask("Title", tag)
            ranges: dict[str, Range] = {}
            fixed: dict[str, Any] = {}
            pcount = int(ask("How many optimizable parameters in this subgroup", "1"))
            for _ in range(pcount):
                name = ask_with_hint("Input name", "", "parameterName")
                start = parse_value(ask("Start"))
                step = parse_value(ask("Step"))
                stop = parse_value(ask("Stop"))
                ranges[name] = (start, step, stop)
            fixed_text = ask_with_hint(
                "Fixed values for this subgroup in key=value format, or press ENTER",
                "",
                "parameterA=1 parameterB=true",
            )
            if fixed_text:
                lexer = shlex.shlex(fixed_text, posix=True)
                lexer.whitespace_split = True
                lexer.commenters = ""
                for token in lexer:
                    if "=" in token:
                        k, v = token.split("=", 1)
                        fixed[k] = parse_value(v)
            subgroups.append(Subgroup(tag=tag, title=title, group=group, ranges=ranges, fixed=fixed))

    base.setdefault("optimizationTag", "manual")
    base.setdefault("saveOptimizationPasses", True)
    base.setdefault("dummyOptimizationPass", 0)
    if "csvExportPrefix" in base:
        base.setdefault("csvExportPrefix", csv_prefix)

    config = OptimizerConfig(
        terminal_exe=terminal_exe,
        data_path=data_path,
        expert_name=expert_name,
        csv_prefix=csv_prefix,
        run_dir=run_dir,
        symbols=symbols,
        from_date=from_date,
        to_date=to_date,
        period=period,
        model=model,
        currency=currency,
        timeout_seconds=timeout_seconds,
        selection_mode=selection_mode,
        top_sets_to_show=top_sets_to_show,
        enable_wfe=enable_wfe,
        wfe_from_date=wfe_from_date,
        wfe_to_date=wfe_to_date,
        wfe_insample_months=wfe_insample_months,
        wfe_outsample_months=wfe_outsample_months,
        wfe_step_months=wfe_step_months,
        wfe_top_n=wfe_top_n,
        wfe_is_top_n=wfe_is_top_n,
        wfe_frequency_top_n=wfe_frequency_top_n,
        wfe_min_efficiency=wfe_min_efficiency,
        wfe_max_efficiency=wfe_max_efficiency,
        wfe_use_top_candidates=wfe_use_top_candidates,
        wfe_max_candidates=wfe_max_candidates,
        base=base,
        subgroups=[asdict(sg) for sg in subgroups],
        expand_limits={},
    )

    out = Path(ask_with_hint("Output config.json name", f"config_{expert_name}_global.json", "config_my_ea_global.json"))
    config = normalize_config(config, out)
    config_data = asdict(config)
    for deprecated_key in (
        "wfe_is_top_n",
        "wfe_frequency_top_n",
        "wfe_min_efficiency",
        "wfe_max_efficiency",
        "wfe_metric",
    ):
        config_data.pop(deprecated_key, None)
    out.write_text(json.dumps(config_data, indent=2, ensure_ascii=False), encoding="utf-8")
    print("\nOK: configuration saved to the selected config file.")


def load_config(path: Path) -> OptimizerConfig:
    data = json.loads(path.read_text(encoding="utf-8-sig"))
    return normalize_config(OptimizerConfig(**data), path)


def subgroup_from_dict(data: dict[str, Any]) -> Subgroup:
    ranges = {k: tuple(v) for k, v in data.get("ranges", {}).items()}
    return Subgroup(
        tag=data["tag"],
        title=data.get("title", data["tag"]),
        group=data.get("group", "Default"),
        ranges=ranges,
        fixed=data.get("fixed", {}),
        wfe=data.get("wfe", {}),
    )


def estimate_passes(ranges: dict[str, Range]) -> int:
    total = 1
    for start, step, stop in ranges.values():
        if isinstance(start, bool) or isinstance(stop, bool):
            total *= 2
            continue
        step_f = abs(as_float(step, 1.0)) or 1.0
        total *= max(1, int(math.floor((as_float(stop) - as_float(start)) / step_f + 1.000001)))
    return total


def paths(cfg: OptimizerConfig) -> dict[str, Path]:
    data_path = Path(cfg.data_path)
    return {
        "data_path": data_path,
        "tester_dir": data_path / "MQL5" / "Profiles" / "Tester",
        "files_dir": data_path / "MQL5" / "Files",
        "cache_dir": data_path / "Tester" / "cache",
        "run_dir": Path(cfg.run_dir),
        "terminal_exe": Path(cfg.terminal_exe),
    }


def write_set(cfg: OptimizerConfig, tag: str, params: dict[str, Any], subgroup: Subgroup, baseline: bool = False) -> Path:
    p = paths(cfg)
    p["tester_dir"].mkdir(parents=True, exist_ok=True)
    merged = {**params, **subgroup.fixed, "optimizationTag": tag, "saveOptimizationPasses": True}
    set_path = p["tester_dir"] / f"{cfg.expert_name}_{tag}.set"
    lines: list[str] = []
    for key, value in merged.items():
        if key == "optimizationTag":
            lines.append(f"{key}={value}")
        elif baseline and key == "dummyOptimizationPass":
            lines.append(f"{key}=0||0||1||1||Y")
        elif not baseline and key in subgroup.ranges:
            start, step, stop = subgroup.ranges[key]
            lines.append(f"{key}={mt5_value(value)}||{mt5_value(start)}||{mt5_value(step)}||{mt5_value(stop)}||Y")
        else:
            lines.append(f"{key}={mt5_value(value)}||0||0||0||N")
    set_path.write_text("\n".join(lines) + "\n", encoding="ascii", errors="ignore")
    return set_path


def write_ini(cfg: OptimizerConfig, symbol: str, tag: str, set_path: Path, genetic: bool) -> Path:
    p = paths(cfg)
    p["run_dir"].mkdir(parents=True, exist_ok=True)
    ini_path = p["run_dir"] / f"{tag}.ini"
    optimization = 2 if genetic else 1
    report_path = p["run_dir"] / tag
    ini = f"""[Experts]
AllowLiveTrading=0
AllowDllImport=0
Enabled=1

[Tester]
Expert={cfg.expert_name}
ExpertParameters={set_path.name}
Symbol={symbol}
Period={cfg.period}
Login=
Model={cfg.model}
ExecutionMode=0
Optimization={optimization}
OptimizationCriterion={cfg.optimization_criterion}
FromDate={cfg.from_date}
ToDate={cfg.to_date}
ForwardMode=0
Report={report_path}
ReplaceReport=1
ShutdownTerminal=1
Deposit={cfg.deposit}
Currency={cfg.currency}
Leverage={cfg.leverage}
UseLocal=1
UseRemote=0
UseCloud=0
Visual=0
"""
    ini_path.write_text(ini, encoding="ascii", errors="ignore")
    return ini_path


def clean_cache(cfg: OptimizerConfig, symbol: str) -> None:
    p = paths(cfg)
    pattern = f"{cfg.expert_name}.{symbol}.{cfg.period}.{cfg.from_date.replace('.', '')}.{cfg.to_date.replace('.', '')}*.opt"
    for file in p["cache_dir"].glob(pattern):
        try:
            file.unlink()
        except OSError:
            pass


def run_mt5(cfg: OptimizerConfig, ini_path: Path) -> None:
    terminal = Path(cfg.terminal_exe)
    if not terminal.exists():
        raise FileNotFoundError(f"terminal64.exe not found: {terminal}")
    if cfg.close_terminal_before_run:
        close_existing_terminal(terminal)
    proc = subprocess.Popen([str(terminal), f"/config:{ini_path}"], cwd=str(terminal.parent))
    try:
        if cfg.timeout_seconds and cfg.timeout_seconds > 0:
            proc.wait(timeout=cfg.timeout_seconds)
        else:
            proc.wait()
    except subprocess.TimeoutExpired:
        proc.terminate()
        raise RuntimeError(f"MT5 timeout: {ini_path.name}")
    if proc.returncode not in (0, None):
        raise RuntimeError(f"MT5 exited with code {proc.returncode}: {ini_path.name}")


def close_existing_terminal(terminal: Path) -> None:
    escaped = str(terminal).replace("'", "''")
    command = (
        "$target = '" + escaped + "'; "
        "Get-Process -Name terminal64 -ErrorAction SilentlyContinue | "
        "Where-Object { $_.Path -eq $target } | "
        "Stop-Process -Force"
    )
    subprocess.run(
        ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", command],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
        check=False,
    )
    time.sleep(1.0)


def read_rows(path: Path) -> list[dict[str, Any]]:
    if not path.exists():
        raise FileNotFoundError(path)

    rows: list[dict[str, Any]] = []
    for enc in ("utf-8", "utf-8-sig", "latin1"):
        try:
            with path.open("r", encoding=enc, newline="") as fh:
                rows = list(csv.DictReader(fh, delimiter=";"))
            break
        except Exception:
            rows = []

    for row in rows:
        if "optimizationTag" not in row and "tag" in row:
            row["optimizationTag"] = row.get("tag")

        for k, v in list(row.items()):
            if k in {"tag", "optimizationTag", "symbol", "worst_asset", "best_asset"}:
                continue
            try:
                row[k] = float(str(v).replace(",", "."))
            except Exception:
                pass

    return rows


def top_rows(rows: list[dict[str, Any]], min_trades: int = 30) -> list[dict[str, Any]]:
    filtered = [
        r for r in rows
        if as_float(r.get("profit")) > 0
        and as_float(r.get("pf")) >= 1.01
        and as_float(r.get("trades")) >= min_trades
        and as_float(r.get("ddRel"), 999) <= 50
        and as_float(r.get("recovery")) > 0
    ]
    return sorted(filtered, key=lambda r: as_float(r.get("score")), reverse=True)[:20]


def param_key(row: dict[str, Any], subgroup: Subgroup) -> tuple[Any, ...]:
    values = []
    for param in subgroup.ranges:
        value = row.get(param, "")
        number = as_float(value)
        if abs(number - round(number)) < 1e-9:
            values.append(int(round(number)))
        else:
            values.append(round(number, 6))
    return tuple(values)


def score_global_metrics(cfg: OptimizerConfig, metrics: dict[str, Any]) -> float:
    n = max(1, len(cfg.symbols))
    total_profit = as_float(metrics.get("total_profit"))
    avg_pf = as_float(metrics.get("avg_pf"))
    median_pf = as_float(metrics.get("median_pf"))
    positive_assets = as_float(metrics.get("positive_assets"))
    pf_gt_105 = as_float(metrics.get("pf_gt_105"))
    concentration = as_float(metrics.get("profit_concentration_best_asset"), 1.0)
    max_dd = as_float(metrics.get("max_ddRel"), 999)
    total_trades = as_float(metrics.get("total_trades"))
    tradable_assets = as_float(metrics.get("tradable_assets"))
    low_trade_assets = as_float(metrics.get("low_trade_assets"))
    avg_recovery = as_float(metrics.get("avg_recovery"))
    profit_score = max(total_profit, 0.0)
    breadth_score = (positive_assets / n) ** 2.0
    pf_breadth_score = (pf_gt_105 / n) ** 1.4
    median_pf_score = max(0.0, min(1.8, median_pf)) ** 1.8
    avg_pf_score = max(0.0, min(1.8, avg_pf))
    concentration_penalty = max(0.08, 1.0 - max(0.0, concentration - 0.40) * 1.7)
    dd_penalty = 1.0 / (1.0 + (max_dd / 10.0) ** 2)
    trade_score = min(1.0, total_trades / (n * 300.0))
    trade_breadth_score = (tradable_assets / n) ** 1.5
    low_trade_penalty = 0.65 ** low_trade_assets
    recovery_score = max(0.25, min(1.35, avg_recovery / 3.0))
    return profit_score * breadth_score * pf_breadth_score * median_pf_score * avg_pf_score * concentration_penalty * dd_penalty * trade_score * trade_breadth_score * low_trade_penalty * recovery_score


def aggregate_metric_rows(cfg: OptimizerConfig, by_symbol: dict[str, dict[str, Any]]) -> dict[str, Any]:
    symbols = cfg.symbols
    profits = [as_float(by_symbol.get(s, {}).get("profit")) for s in symbols]
    pfs = [as_float(by_symbol.get(s, {}).get("pf")) for s in symbols]
    trades = [as_float(by_symbol.get(s, {}).get("trades")) for s in symbols]
    dds = [as_float(by_symbol.get(s, {}).get("ddRel"), 0) for s in symbols]
    payoffs = [as_float(by_symbol.get(s, {}).get("payoff")) for s in symbols]
    recoveries = [as_float(by_symbol.get(s, {}).get("recovery")) for s in symbols]
    total_profit = sum(profits)
    positive_assets = sum(1 for p in profits if p > 0)
    pf_gt_105 = sum(1 for pf in pfs if pf > 1.05)
    median_pf = median(pfs) if pfs else 0
    avg_pf = sum(pfs) / len(pfs) if pfs else 0
    best_symbol = max(symbols, key=lambda s: as_float(by_symbol.get(s, {}).get("profit")))
    worst_symbol = min(symbols, key=lambda s: as_float(by_symbol.get(s, {}).get("profit")))
    positive_profit = sum(p for p in profits if p > 0)
    concentration = as_float(by_symbol.get(best_symbol, {}).get("profit")) / positive_profit if positive_profit > 0 else 1.0
    metrics = {
        "total_profit": total_profit,
        "avg_pf": avg_pf,
        "median_pf": median_pf,
        "positive_assets": positive_assets,
        "pf_gt_105": pf_gt_105,
        "best_asset": best_symbol,
        "worst_asset": worst_symbol,
        "best_profit": as_float(by_symbol.get(best_symbol, {}).get("profit")),
        "worst_profit": as_float(by_symbol.get(worst_symbol, {}).get("profit")),
        "profit_concentration_best_asset": concentration,
        "max_ddRel": max(dds) if dds else 999,
        "total_trades": sum(trades),
        "tradable_assets": sum(1 for t in trades if t >= 30),
        "low_trade_assets": sum(1 for t in trades if t < 30),
        "avg_payoff": sum(payoffs) / len(payoffs) if payoffs else 0,
        "avg_recovery": sum(recoveries) / len(recoveries) if recoveries else 0,
    }
    metrics["global_score"] = score_global_metrics(cfg, metrics)
    return metrics


def aggregate_global_rows(cfg: OptimizerConfig, symbol_rows: dict[str, list[dict[str, Any]]], subgroup: Subgroup) -> list[dict[str, Any]]:
    grouped: dict[tuple[Any, ...], dict[str, dict[str, Any]]] = {}
    for symbol, rows in symbol_rows.items():
        for row in rows:
            grouped.setdefault(param_key(row, subgroup), {})[symbol] = row
    aggregates = []
    for key, by_symbol in grouped.items():
        metrics = aggregate_metric_rows(cfg, by_symbol)
        if as_float(metrics["total_trades"]) <= 0:
            continue
        row = dict(metrics)
        for i, param in enumerate(subgroup.ranges):
            row[param] = key[i]
        for symbol in cfg.symbols:
            src = by_symbol.get(symbol, {})
            row[f"{symbol}_profit"] = as_float(src.get("profit"))
            row[f"{symbol}_pf"] = as_float(src.get("pf"))
            row[f"{symbol}_trades"] = as_float(src.get("trades"))
        aggregates.append(row)
    return sorted(aggregates, key=lambda r: as_float(r.get("global_score")), reverse=True)


def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
    if not rows:
        return
    path.parent.mkdir(parents=True, exist_ok=True)
    fields: list[str] = []
    for row in rows:
        for key in row.keys():
            if key not in fields:
                fields.append(key)
    with path.open("w", encoding="utf-8", newline="") as fh:
        writer = csv.DictWriter(fh, fieldnames=fields, delimiter=";")
        writer.writeheader()
        writer.writerows(rows)


def collect_symbol_rows(
    cfg: OptimizerConfig,
    subgroup: Subgroup,
    params: dict[str, Any],
    baseline: bool = False,
    run_label: str = "",
    force_run: bool = False,
) -> dict[str, list[dict[str, Any]]]:
    p = paths(cfg)
    p["files_dir"].mkdir(parents=True, exist_ok=True)
    p["run_dir"].mkdir(parents=True, exist_ok=True)
    passes = 1 if baseline else estimate_passes(subgroup.ranges)
    genetic = False if baseline else passes > cfg.genetic_threshold_passes
    rows_by_symbol: dict[str, list[dict[str, Any]]] = {}
    for symbol in cfg.symbols:
        prefix = "BASE" if baseline else "MSG"
        label = f"_{run_label}" if run_label else ""
        tag = f"{prefix}{label}_{subgroup.tag}_{symbol}"
        cache_all = p["run_dir"] / f"{tag}_all.csv"
        if force_run and cache_all.exists():
            cache_all.unlink()
        if cache_all.exists():
            rows_by_symbol[symbol] = read_rows(cache_all)
            continue
        csv_path = p["files_dir"] / f"{cfg.csv_prefix}_{tag}_frames.csv"
        if csv_path.exists():
            csv_path.unlink()
        clean_cache(cfg, symbol)
        set_path = write_set(cfg, tag, params, subgroup, baseline=baseline)
        ini_path = write_ini(cfg, symbol, tag, set_path, genetic=genetic)
        mode = "baseline" if baseline else ("genetic" if genetic else "complete")
        print(
            f"  {symbol}: {mode}, estimated passes={passes}, "
            f"threshold={cfg.genetic_threshold_passes}, ini={ini_path}",
            flush=True,
        )
        run_mt5(cfg, ini_path)

        if not csv_path.exists():
            raise FileNotFoundError(
                f"CSV not found: {csv_path}\n"
                f"Check whether the EA has csvExportPrefix='{cfg.csv_prefix}', "
                f"optimizationTag, saveOptimizationPasses and Frames-based export "
                f"with a file following the pattern csvExportPrefix + '_' + optimizationTag + '_frames.csv'."
            )

        rows = read_rows(csv_path)
        write_csv(cache_all, rows)
        write_csv(p["run_dir"] / f"{tag}_top20.csv", top_rows(rows))
        rows_by_symbol[symbol] = rows
    return rows_by_symbol


def choose_cluster(cfg: OptimizerConfig, symbol_rows: dict[str, list[dict[str, Any]]], subgroup: Subgroup) -> tuple[dict[str, Any] | None, dict[str, Any], list[dict[str, Any]]]:
    p = paths(cfg)
    global_rows = aggregate_global_rows(cfg, symbol_rows, subgroup)
    global_top = global_rows[: max(1, cfg.top_sets_to_show)]
    write_csv(p["run_dir"] / f"{subgroup.tag}_global_all.csv", global_rows)
    write_csv(p["run_dir"] / f"{subgroup.tag}_global_top{cfg.top_sets_to_show}.csv", global_top)
    best = global_top[0] if global_top else None
    stats = {
        "tag": subgroup.tag,
        "title": subgroup.title,
        "group": subgroup.group,
        "passes_estimated": estimate_passes(subgroup.ranges),
        "global_top_count": len(global_top),
        "best_global": best or {},
        "parameters": {},
        "risk_flags": [],
    }
    if not best:
        return None, stats, global_top
    for param in subgroup.ranges:
        values = [as_float(row.get(param)) for row in global_top if row.get(param) not in ("", None)]
        if not values:
            continue
        counts: dict[str, int] = {}
        for v in values:
            key = str(int(round(v))) if abs(v - round(v)) < 1e-9 else str(round(v, 6))
            counts[key] = counts.get(key, 0) + 1
        mode, mode_count = sorted(counts.items(), key=lambda item: (-item[1], item[0]))[0]
        recommended: Any = parse_value(mode)
        if mode_count < 4:
            recommended = median(values)
            if abs(as_float(recommended) - round(as_float(recommended))) < 1e-9:
                recommended = int(round(as_float(recommended)))
        stats["parameters"][param] = {
            "recommended": recommended,
            "median": median(values),
            "mode": mode,
            "mode_count": mode_count,
            "min": min(values),
            "max": max(values),
            "unique_count": len(counts),
            "sensitive": mode_count < 4,
        }
    stats["risk_flags"] = build_risk_flags(cfg, stats, global_top)
    return best, stats, global_top


def build_risk_flags(cfg: OptimizerConfig, stats: dict[str, Any], global_top: list[dict[str, Any]]) -> list[str]:
    if not global_top:
        return ["no_global_top"]
    best = global_top[0]
    flags: list[str] = []
    if as_float(best.get("profit_concentration_best_asset"), 1.0) > 0.55:
        flags.append("high_concentration_in_best_asset")
    if as_float(best.get("median_pf")) < 1.05:
        flags.append("low_median_pf")
    if as_float(best.get("max_ddRel")) > 20:
        flags.append("high_drawdown")
    if as_float(best.get("positive_assets")) < len(cfg.symbols):
        flags.append("not_all_assets_positive")
    if as_float(best.get("total_trades")) < len(cfg.symbols) * 100:
        flags.append("few_trades")
    sensitive = [p for p, info in stats.get("parameters", {}).items() if info.get("sensitive")]
    if sensitive:
        flags.append("sensitive_parameters:" + ",".join(sensitive[:5]))
    if len(global_top) >= 2:
        s1 = as_float(global_top[0].get("global_score"))
        s2 = as_float(global_top[1].get("global_score"))
        if s1 > 0 and (s1 - s2) / s1 < 0.03:
            flags.append("top1_too_close_to_top2")
    return flags


def accept_cluster(cfg: OptimizerConfig, baseline: dict[str, Any], best: dict[str, Any] | None) -> tuple[bool, str]:
    if not best:
        return False, "no valid global cluster"
    base_score = as_float(baseline.get("global_score"))
    opt_score = as_float(best.get("global_score"))
    base_positive = as_float(baseline.get("positive_assets"))
    opt_positive = as_float(best.get("positive_assets"))
    base_pf = as_float(baseline.get("median_pf"))
    opt_pf = as_float(best.get("median_pf"))
    base_dd = max(0.01, as_float(baseline.get("max_ddRel")))
    opt_dd = as_float(best.get("max_ddRel"))
    opt_profit = as_float(best.get("total_profit"))
    if opt_profit <= 0:
        return False, "aggregate profit is negative or zero"
    if opt_pf < 1.0:
        return False, "median PF below 1.0"
    if opt_positive < max(1, math.ceil(len(cfg.symbols) * 0.60)):
        return False, "too few positive assets"
    delta = (opt_score - base_score) / abs(base_score) if abs(base_score) > 1e-9 else 1.0
    if delta >= 0.05 and opt_positive >= base_positive and opt_pf >= base_pf - 0.03 and opt_dd <= base_dd * 1.30:
        return True, f"score improved {delta:.1%} without worsening stability"
    if opt_positive > base_positive:
        return True, "structural improvement: more positive assets"
    if opt_pf >= base_pf + 0.05 and opt_profit > 0:
        return True, "structural improvement: higher median PF"
    return False, f"did not beat baseline: score delta {delta:.1%}"


def decision_is_sensitive(cfg: OptimizerConfig, baseline: dict[str, Any], best: dict[str, Any] | None, stats: dict[str, Any], accepted: bool) -> bool:
    if cfg.selection_mode == "manual":
        return True
    if cfg.selection_mode == "auto":
        return False
    if not best or not accepted:
        return True
    flags = stats.get("risk_flags", [])
    base_score = as_float(baseline.get("global_score"))
    opt_score = as_float(best.get("global_score"))
    delta = (opt_score - base_score) / abs(base_score) if abs(base_score) > 1e-9 else 1.0
    if flags:
        return True
    if delta < 0.15:
        return True
    return False


def format_float(v: Any, nd: int = 2) -> str:
    return f"{as_float(v):.{nd}f}"


def serialize_range(value: Range) -> dict[str, Any]:
    start, step, stop = value
    return {"start": start, "step": step, "stop": stop}


def format_range(value: Range) -> str:
    start, step, stop = value
    return f"{start} .. {stop} step {step}"


def subgroup_defaults(params: dict[str, Any], subgroup: Subgroup) -> dict[str, Any]:
    return {param: params.get(param, "") for param in subgroup.ranges}


def enrich_stats_with_parameter_context(stats: dict[str, Any], params: dict[str, Any], subgroup: Subgroup) -> None:
    stats["optimization_ranges"] = {
        param: serialize_range(rng)
        for param, rng in subgroup.ranges.items()
    }
    stats["defaults_before"] = subgroup_defaults(params, subgroup)
    stats["fixed_values"] = dict(subgroup.fixed)


def row_parameter_text(row: dict[str, Any], subgroup: Subgroup) -> str:
    parts = []
    for param in subgroup.ranges:
        parts.append(f"{param}={row.get(param, '')}")
    return ", ".join(parts)


def print_top_sets(cfg: OptimizerConfig, subgroup: Subgroup, baseline: dict[str, Any], global_top: list[dict[str, Any]], stats: dict[str, Any]) -> None:
    print("\n" + "=" * 95)
    print(f"Subgroup: {subgroup.tag} - {subgroup.title}")
    if subgroup.ranges:
        print("\nCurrent defaults:")
        defaults = stats.get("defaults_before", {})
        print("  " + ", ".join(f"{p}={defaults.get(p, '')}" for p in subgroup.ranges))
        print("\nOptimization ranges:")
        for param, rng in subgroup.ranges.items():
            print(f"  {param}: {format_range(rng)}")
    if subgroup.fixed:
        print("\nFixed values:")
        print("  " + ", ".join(f"{k}={v}" for k, v in subgroup.fixed.items()))
    print("Baseline:")
    print(
        f"  Score={format_float(baseline.get('global_score'), 4)} | "
        f"Profit={format_float(baseline.get('total_profit'))} | "
        f"PFmed={format_float(baseline.get('median_pf'), 3)} | "
        f"Positive assets={int(as_float(baseline.get('positive_assets')))}/{len(cfg.symbols)} | "
        f"Trades={int(as_float(baseline.get('total_trades')))} | "
        f"DDmax={format_float(baseline.get('max_ddRel'), 2)}%"
    )
    if stats.get("risk_flags"):
        print("Warnings:", ", ".join(stats["risk_flags"]))
    print("\nTOP sets:")
    print("#  Score       Profit      PFmed  Pos  Trades  DDmax   Conc.  Best/Worst     Parameters")
    print("-" * 140)
    for i, row in enumerate(global_top[: cfg.top_sets_to_show], 1):
        print(
            f"{i:<2} {as_float(row.get('global_score')):<11.2f} "
            f"{as_float(row.get('total_profit')):<10.2f} "
            f"{as_float(row.get('median_pf')):<6.3f} "
            f"{int(as_float(row.get('positive_assets'))):>2}/{len(cfg.symbols):<2} "
            f"{int(as_float(row.get('total_trades'))):<7} "
            f"{as_float(row.get('max_ddRel')):<6.2f} "
            f"{as_float(row.get('profit_concentration_best_asset')):<6.2f} "
            f"{str(row.get('best_asset')) + '/' + str(row.get('worst_asset')):<14} "
            f"{row_parameter_text(row, subgroup)}"
        )
    print("=" * 95)


def row_to_params(params: dict[str, Any], subgroup: Subgroup, selected: dict[str, Any]) -> dict[str, Any]:
    out = dict(params)
    out.update(subgroup.fixed)
    for param in subgroup.ranges:
        if param in selected:
            value = selected[param]
            if abs(as_float(value) - round(as_float(value))) < 1e-9:
                out[param] = int(round(as_float(value)))
            else:
                out[param] = as_float(value)
    return out


def update_params_from_stats(params: dict[str, Any], subgroup: Subgroup, stats: dict[str, Any]) -> dict[str, Any]:
    out = dict(params)
    out.update(subgroup.fixed)
    for param, info in stats.get("parameters", {}).items():
        out[param] = info["recommended"]
    return out


def write_subgroup_parameter_review(
    cfg: OptimizerConfig,
    subgroup: Subgroup,
    baseline: dict[str, Any],
    global_top: list[dict[str, Any]],
    stats: dict[str, Any],
) -> None:
    p = paths(cfg)
    lines = [
        f"# Parameter Review - {subgroup.tag}",
        "",
        f"- Group: {subgroup.group}",
        f"- Title: {subgroup.title}",
        f"- Estimated passes: {stats.get('passes_estimated', 0)}",
        "",
        "## Current Defaults",
    ]
    defaults = stats.get("defaults_before", {})
    for param in subgroup.ranges:
        lines.append(f"- {param}: `{defaults.get(param, '')}`")
    lines += ["", "## Optimization Ranges"]
    for param, rng in subgroup.ranges.items():
        lines.append(f"- {param}: `{format_range(rng)}`")
    lines += ["", "## Fixed Values"]
    if subgroup.fixed:
        for key, value in subgroup.fixed.items():
            lines.append(f"- {key}: `{value}`")
    else:
        lines.append("- None")
    lines += [
        "",
        "## Baseline",
        f"- Score: `{format_float(baseline.get('global_score'), 4)}`",
        f"- Profit: `{format_float(baseline.get('total_profit'))}`",
        f"- Median PF: `{format_float(baseline.get('median_pf'), 3)}`",
        f"- Total trades: `{int(as_float(baseline.get('total_trades')))}`",
        f"- Max DD relative: `{format_float(baseline.get('max_ddRel'), 2)}%`",
        "",
        "## TOP Sets",
    ]
    if not global_top:
        lines.append("- No valid TOP rows.")
    else:
        for i, row in enumerate(global_top[: cfg.top_sets_to_show], 1):
            lines += [
                "",
                f"### TOP {i}",
                f"- Score: `{format_float(row.get('global_score'), 4)}`",
                f"- Profit: `{format_float(row.get('total_profit'))}`",
                f"- Median PF: `{format_float(row.get('median_pf'), 3)}`",
                f"- Total trades: `{int(as_float(row.get('total_trades')))}`",
                f"- Max DD relative: `{format_float(row.get('max_ddRel'), 2)}%`",
                f"- Parameters: `{row_parameter_text(row, subgroup)}`",
            ]
    lines += ["", "## Recommended Cluster"]
    for param, info in stats.get("parameters", {}).items():
        lines.append(
            f"- {param}: recommended=`{info.get('recommended')}`, "
            f"median=`{info.get('median')}`, mode=`{info.get('mode')}`, "
            f"mode_count=`{info.get('mode_count')}`, range=`{info.get('min')}..{info.get('max')}`, "
            f"sensitive=`{info.get('sensitive')}`"
        )
    if stats.get("risk_flags"):
        lines += ["", "## Risk Flags"]
        for flag in stats["risk_flags"]:
            lines.append(f"- {flag}")
    (p["run_dir"] / f"{subgroup.tag}_PARAMETER_REVIEW.md").write_text("\n".join(lines) + "\n", encoding="utf-8")


def choose_set_interactively(
    cfg: OptimizerConfig,
    subgroup: Subgroup,
    params: dict[str, Any],
    baseline: dict[str, Any],
    global_top: list[dict[str, Any]],
    stats: dict[str, Any],
    accepted_auto: bool,
    auto_reason: str,
) -> tuple[str, dict[str, Any], str, int | None]:
    if not cfg.allow_user_override:
        if accepted_auto:
            return "ACCEPT", update_params_from_stats(params, subgroup, stats), auto_reason, 1
        return "KEEP_BASELINE", params, auto_reason, None

    sensitive = decision_is_sensitive(cfg, baseline, global_top[0] if global_top else None, stats, accepted_auto)
    if not sensitive:
        if accepted_auto:
            return "ACCEPT_AUTO", update_params_from_stats(params, subgroup, stats), auto_reason, 1
        return "KEEP_BASELINE_AUTO", params, auto_reason, None

    print_top_sets(cfg, subgroup, baseline, global_top, stats)
    print("Choose:")
    print("  A = accept recommended")
    print("  B = keep baseline")
    print(f"  1-{min(len(global_top), cfg.top_sets_to_show)} = apply selected set")
    print("  P = skip subgroup without changes")
    print("  S = save and exit")
    print("  R = mark for manual boundary reprocessing later")
    default_choice = "A" if accepted_auto else "B"
    if cfg.selection_mode == "semi_auto" and not accepted_auto and global_top:
        default_choice = "R"
    choice = ask("Your choice", default_choice).strip().upper()

    if choice == "S":
        raise SaveAndExit("User chose to save and exit")
    if choice in {"B", "P"}:
        return "KEEP_BASELINE", params, "user kept baseline", None
    if choice == "R":
        return "REPROCESS_BOUNDARIES", params, "user requested boundary reprocessing", None
    if choice.isdigit():
        idx = int(choice)
        if 1 <= idx <= len(global_top):
            selected = global_top[idx - 1]
            return "USER_SELECTED", row_to_params(params, subgroup, selected), f"user selected TOP {idx}", idx
        print("Choice outside the valid range; applying recommended set.")
    if accepted_auto:
        return "ACCEPT", update_params_from_stats(params, subgroup, stats), "user accepted recommended: " + auto_reason, 1
    return "KEEP_BASELINE", params, "user accepted the recommendation to keep baseline", None


def save_final_set(cfg: OptimizerConfig, params: dict[str, Any]) -> Path:
    p = paths(cfg)
    p["tester_dir"].mkdir(parents=True, exist_ok=True)
    final = p["tester_dir"] / f"{cfg.expert_name}_GLOBAL_FINAL.set"
    lines = []
    for key, value in params.items():
        if key == "optimizationTag":
            lines.append(f"{key}=global_final")
        else:
            lines.append(f"{key}={mt5_value(value)}||0||0||0||N")
    final.write_text("\n".join(lines) + "\n", encoding="ascii", errors="ignore")
    return final


def write_report(cfg: OptimizerConfig, manifest: list[dict[str, Any]], params: dict[str, Any]) -> None:
    p = paths(cfg)
    final_set = save_final_set(cfg, params)
    lines = [
        f"# Report - {cfg.expert_name} Global Optimizer",
        "",
        f"- EA: {cfg.expert_name}",
        f"- Symbols: {', '.join(cfg.symbols)}",
        f"- Period: {cfg.from_date} to {cfg.to_date}",
        f"- Timeframe: {cfg.period}",
        f"- Selection mode: {cfg.selection_mode}",
        f"- Final set: {final_set}",
        f"- WFE enabled: {cfg.enable_wfe}",
        "",
        "## Executed sequence",
    ]
    for item in manifest:
        rank = item.get("selected_rank")
        rank_text = f" | rank={rank}" if rank else ""
        lines.append(
            f"- {item['tag']} | {item['decision']}{rank_text} | "
            f"baseline={item.get('baseline_score', 0):.4g} | "
            f"optimized={item.get('optimized_score', 0):.4g} | "
            f"reason={item.get('reason', '')}"
        )
    lines += ["", "## Final defaults"]
    for key, value in params.items():
        lines.append(f"- {key} = {value}")
    (p["run_dir"] / "GLOBAL_OPTIMIZER_REPORT.md").write_text("\n".join(lines) + "\n", encoding="utf-8")


def build_rewind_plan(params: dict[str, Any], original: list[Subgroup]) -> list[Subgroup]:
    rewinds: list[Subgroup] = []
    for sg in original:
        new_ranges: dict[str, Range] = {}
        for param, (start, step, stop) in sg.ranges.items():
            center = params.get(param, start)
            if isinstance(start, bool) or isinstance(stop, bool):
                new_ranges[param] = (False, 1, True)
                continue
            step_f = abs(as_float(step, 1.0)) or 1.0
            width = max(step_f * 2.0, abs(as_float(stop) - as_float(start)) * 0.20)
            c = as_float(center)
            new_start = round(c - width, 6)
            new_stop = round(c + width, 6)
            if isinstance(start, int) and isinstance(stop, int):
                new_ranges[param] = (int(round(new_start)), int(round(step_f)), int(round(new_stop)))
            else:
                new_ranges[param] = (new_start, step, new_stop)
        rewinds.append(Subgroup(tag=f"R_{sg.tag}", title=f"Fine rewind - {sg.title}", group=sg.group, ranges=new_ranges, fixed=sg.fixed, wfe=sg.wfe))
    return rewinds


def parse_date(value: str) -> datetime:
    return datetime.strptime(value, "%Y.%m.%d")


def format_date(value: datetime) -> str:
    return value.strftime("%Y.%m.%d")


def add_months(value: datetime, months: int) -> datetime:
    month = value.month - 1 + months
    year = value.year + month // 12
    month = month % 12 + 1
    day = min(value.day, calendar.monthrange(year, month)[1])
    return value.replace(year=year, month=month, day=day)


def inclusive_month_count(start_value: str, end_value: str) -> int:
    try:
        start = parse_date(start_value)
        end = parse_date(end_value)
    except Exception:
        return 1
    months = (end.year - start.year) * 12 + (end.month - start.month) + 1
    return max(1, months)


def build_wfe_windows(cfg: OptimizerConfig) -> list[dict[str, str]]:
    start = parse_date(cfg.wfe_from_date or cfg.from_date)
    end = parse_date(cfg.wfe_to_date or cfg.to_date)
    windows: list[dict[str, str]] = []
    current = start
    idx = 1
    while current < end:
        is_from = current
        is_to = add_months(is_from, max(1, cfg.wfe_insample_months)) - timedelta(days=1)
        oos_from = is_to + timedelta(days=1)
        oos_to = add_months(oos_from, max(1, cfg.wfe_outsample_months)) - timedelta(days=1)
        if oos_from > end:
            break
        if oos_to > end:
            oos_to = end
        windows.append({
            "window": idx,
            "is_from": format_date(is_from),
            "is_to": format_date(is_to),
            "oos_from": format_date(oos_from),
            "oos_to": format_date(oos_to),
        })
        current = add_months(current, max(1, cfg.wfe_step_months))
        idx += 1
    return windows


def candidate_key(params: dict[str, Any]) -> str:
    return json.dumps(params, sort_keys=True, default=str)


def parse_row_value(value: Any) -> Any:
    parsed = parse_value(value) if isinstance(value, str) else value
    if isinstance(parsed, float) and abs(parsed - round(parsed)) < 1e-9:
        return int(round(parsed))
    return parsed


def range_values(rng: Range, max_values: int = 200) -> list[Any]:
    start, step, stop = rng
    if isinstance(start, bool) or isinstance(stop, bool):
        return [False, True]

    step_f = abs(as_float(step, 1.0)) or 1.0
    start_f = as_float(start)
    stop_f = as_float(stop)
    values: list[Any] = []
    current = start_f
    guard = 0
    while current <= stop_f + 1e-9 and guard < max_values:
        if isinstance(start, int) and isinstance(stop, int) and not isinstance(start, bool):
            values.append(int(round(current)))
        else:
            values.append(round(current, 10))
        current += step_f
        guard += 1
    return values


def get_wfe_specs(cfg: OptimizerConfig) -> list[dict[str, Any]]:
    specs: list[dict[str, Any]] = []
    for subgroup in [subgroup_from_dict(x) for x in cfg.subgroups]:
        for param, marker in subgroup.wfe.items():
            if param not in subgroup.ranges:
                continue
            if marker == "individual":
                specs.append({
                    "mode": "individual",
                    "group": "",
                    "subgroup": subgroup.tag,
                    "title": subgroup.title,
                    "params": [param],
                    "ranges": {param: subgroup.ranges[param]},
                })

        grouped_slots: dict[int, list[str]] = {}
        for param, marker in subgroup.wfe.items():
            if param in subgroup.ranges and marker.startswith("group"):
                try:
                    slot = int(marker.replace("group", ""))
                except ValueError:
                    continue
                if 1 <= slot <= 3:
                    grouped_slots.setdefault(slot, []).append(param)

        if grouped_slots:
            warnings: list[str] = []
            selected: list[str] = []
            for slot in (1, 2, 3):
                params = grouped_slots.get(slot, [])
                if len(params) > 1:
                    warnings.append(f"duplicate @wfe{slot}: ignored {', '.join(params[1:])}")
                if params:
                    selected.append(params[0])

            if 1 in grouped_slots and 2 in grouped_slots:
                ignored = []
                for params in grouped_slots.values():
                    ignored.extend(params[1:])
                if len(selected) > 3:
                    ignored.extend(selected[3:])
                    selected = selected[:3]
                group_label = "wfe" + "".join(str(i) for i in range(1, len(selected) + 1))
                specs.append({
                    "mode": "grouped",
                    "group": group_label,
                    "subgroup": subgroup.tag,
                    "title": subgroup.title,
                    "params": selected,
                    "ranges": {param: subgroup.ranges[param] for param in selected},
                    "ignored_params": ignored,
                    "warnings": warnings,
                })
            else:
                specs.append({
                    "mode": "invalid_group",
                    "group": "wfe_group_incomplete",
                    "subgroup": subgroup.tag,
                    "title": subgroup.title,
                    "params": [],
                    "ranges": {},
                    "ignored_params": [p for params in grouped_slots.values() for p in params],
                    "warnings": warnings + ["grouped WFE requires at least @wfe1 and @wfe2 in the same subgroup"],
                })
    return [spec for spec in specs if spec["mode"] != "invalid_group"]


def candidate_values_from_top_rows(
    cfg: OptimizerConfig,
    spec: dict[str, Any],
    final_params: dict[str, Any],
) -> list[dict[str, Any]]:
    p = paths(cfg)
    top_file = p["run_dir"] / f"{spec['subgroup']}_global_top20.csv"
    if not top_file.exists():
        matches = sorted(p["run_dir"].glob(f"{spec['subgroup']}_global_top*.csv"))
        top_file = matches[0] if matches else top_file

    rows = read_rows(top_file)[: max(1, cfg.wfe_top_n)] if top_file.exists() and cfg.wfe_use_top_candidates else []
    candidates: list[dict[str, Any]] = []
    seen_values: set[str] = set()

    if rows:
        for rank, row in enumerate(rows, 1):
            values: dict[str, Any] = {}
            missing = False
            for param in spec["params"]:
                if param not in row or row[param] in ("", None):
                    missing = True
                    break
                values[param] = parse_row_value(row[param])
            if missing:
                continue
            key = candidate_key(values)
            if key in seen_values:
                continue
            seen_values.add(key)
            candidates.append({"rank": rank, "values": values})
            if len(candidates) >= max(1, cfg.wfe_max_candidates):
                break

    if candidates:
        return candidates

    if spec["mode"] == "individual":
        param = spec["params"][0]
        return [{"rank": i + 1, "values": {param: value}} for i, value in enumerate(range_values(spec["ranges"][param]))]

    # Fallback for grouped WFE: generate true permutations/combinations for the
    # marked slots, capped by wfe_max_candidates to avoid runaway WFE runs.
    values_by_param = {param: range_values(spec["ranges"][param]) for param in spec["params"]}
    grouped_candidates: list[dict[str, Any]] = []
    param_order = list(spec["params"])
    product_values = [values_by_param[param] for param in param_order]
    for i, combo in enumerate(itertools.product(*product_values), 1):
        values = {param: combo[idx] for idx, param in enumerate(param_order)}
        grouped_candidates.append({"rank": i, "values": values})
        if len(grouped_candidates) >= max(1, cfg.wfe_max_candidates):
            break
    return grouped_candidates


def build_wfe_candidates(cfg: OptimizerConfig) -> list[dict[str, Any]]:
    p = paths(cfg)
    params_path = p["run_dir"] / "current_defaults.json"
    final_params = dict(cfg.base)
    if params_path.exists():
        final_params.update(json.loads(params_path.read_text(encoding="utf-8")))

    specs = get_wfe_specs(cfg)
    if not specs:
        return []

    candidates: list[dict[str, Any]] = []
    seen: set[str] = set()

    for spec in specs:
        value_candidates = candidate_values_from_top_rows(cfg, spec, final_params)
        for item in value_candidates:
            params = dict(final_params)
            params.update(item["values"])
            key = candidate_key(params)
            if key in seen:
                continue
            seen.add(key)
            value_label = "_".join(f"{k}_{v}" for k, v in item["values"].items())
            prefix = "IND" if spec["mode"] == "individual" else spec["group"].upper()
            raw_candidate_id = re.sub(
                r"[^A-Za-z0-9_]+",
                "_",
                f"{prefix}_{spec['subgroup']}_{value_label}",
            )
            value_hash = hashlib.sha1(
                json.dumps(item["values"], sort_keys=True, ensure_ascii=False).encode("utf-8")
            ).hexdigest()[:8]
            candidate_id = f"{raw_candidate_id[:96]}_{value_hash}"
            candidates.append({
                "candidate_id": candidate_id,
                "source": spec["subgroup"],
                "rank": item["rank"],
                "params": params,
                "candidate_mode": spec["mode"],
                "wfe_group": spec.get("group", ""),
                "subgroup": spec["subgroup"],
                "tested_params": list(spec["params"]),
                "tested_values": dict(item["values"]),
            })
            if len(candidates) >= max(1, cfg.wfe_max_candidates):
                return candidates
    return candidates[: max(1, cfg.wfe_max_candidates)]


def run_wfe_single_test(cfg: OptimizerConfig, candidate: dict[str, Any], window: dict[str, str], symbol: str, phase: str) -> dict[str, Any]:
    p = paths(cfg)
    candidate_id = re.sub(r"[^A-Za-z0-9_]+", "_", str(candidate["candidate_id"]))[:80]
    phase = phase.upper()
    if phase == "IS":
        phase = "ISS"
    if phase == "OOS":
        phase = "OSS"
    if phase not in {"ISS", "OSS"}:
        raise ValueError(f"Invalid WFE phase: {phase}")
    from_key = "is_from" if phase == "ISS" else "oos_from"
    to_key = "is_to" if phase == "ISS" else "oos_to"
    tag = f"WFE_{phase}_{candidate_id}_W{window['window']}_{symbol}"
    cache_all = p["run_dir"] / f"{tag}_all.csv"
    if cache_all.exists():
        rows = read_rows(cache_all)
    else:
        test_cfg = OptimizerConfig(**{**asdict(cfg), "from_date": window[from_key], "to_date": window[to_key]})
        subgroup = Subgroup(tag=f"WFE_{phase}_SINGLE_TEST", title=f"WFE {phase} single test")
        set_path = write_set(test_cfg, tag, candidate["params"], subgroup, baseline=True)
        ini_path = write_ini(test_cfg, symbol, tag, set_path, genetic=False)
        csv_path = paths(test_cfg)["files_dir"] / f"{cfg.csv_prefix}_{tag}_frames.csv"
        if csv_path.exists():
            csv_path.unlink()
        clean_cache(test_cfg, symbol)
        print(
            f"  WFE {phase} {candidate['candidate_id']} | window {window['window']} | {symbol}: "
            f"{window[from_key]} to {window[to_key]}",
            flush=True,
        )
        run_mt5(test_cfg, ini_path)
        if not csv_path.exists():
            raise FileNotFoundError(f"WFE CSV not found: {csv_path}")
        rows = read_rows(csv_path)
        write_csv(cache_all, rows)
    best = sorted(rows, key=lambda r: as_float(r.get("score")), reverse=True)[0] if rows else {}
    result = {
        "candidate_id": candidate["candidate_id"],
        "selected_candidate_id": candidate.get("selected_candidate_id", candidate["candidate_id"]),
        "wfe_path_id": candidate.get("wfe_path_id", candidate["candidate_id"]),
        "selection_key": candidate.get("selection_key", wfe_selection_key(candidate)),
        "source": candidate["source"],
        "rank": candidate["rank"],
        "candidate_mode": candidate.get("candidate_mode", ""),
        "wfe_group": candidate.get("wfe_group", ""),
        "subgroup": candidate.get("subgroup", ""),
        "tested_params": "|".join(candidate.get("tested_params", [])),
        "tested_values": json.dumps(candidate.get("tested_values", {}), ensure_ascii=False, sort_keys=True),
        "phase": phase,
        "window": window["window"],
        "is_from": window["is_from"],
        "is_to": window["is_to"],
        "oos_from": window["oos_from"],
        "oos_to": window["oos_to"],
        "symbol": symbol,
        "profit": as_float(best.get("profit")),
        "pf": as_float(best.get("pf")),
        "trades": as_float(best.get("trades")),
        "ddRel": as_float(best.get("ddRel"), 0),
        "payoff": as_float(best.get("payoff")),
        "recovery": as_float(best.get("recovery")),
        "score": as_float(best.get("score")),
    }
    for key, value in candidate["params"].items():
        if isinstance(value, (bool, int, float, str)):
            result[f"param_{key}"] = value
    return result


def wfe_spec_label(spec: dict[str, Any]) -> str:
    prefix = "IND" if spec["mode"] == "individual" else spec.get("group", "wfe").upper()
    params = "_".join(spec.get("params", []))
    return re.sub(r"[^A-Za-z0-9_]+", "_", f"{prefix}_{spec['subgroup']}_{params}")[:72]


def wfe_candidate_from_global_row(
    spec: dict[str, Any],
    final_params: dict[str, Any],
    row: dict[str, Any],
    window_id: int,
) -> dict[str, Any]:
    values = {param: parse_row_value(row.get(param, final_params.get(param))) for param in spec["params"]}
    params = dict(final_params)
    params.update(values)
    value_label = "_".join(f"{k}_{v}" for k, v in values.items())
    prefix = "IND" if spec["mode"] == "individual" else spec.get("group", "wfe").upper()
    raw_candidate_id = re.sub(r"[^A-Za-z0-9_]+", "_", f"{prefix}_{spec['subgroup']}_{value_label}")
    value_hash = hashlib.sha1(
        json.dumps(values, sort_keys=True, ensure_ascii=False).encode("utf-8")
    ).hexdigest()[:8]
    candidate_id = f"{raw_candidate_id[:96]}_{value_hash}"
    selection_key = wfe_selection_key_from_values(
        spec["mode"],
        spec["subgroup"],
        spec.get("group", ""),
        "|".join(spec["params"]),
    )
    path_id = re.sub(
        r"[^A-Za-z0-9_]+",
        "_",
        f"WF_{spec['subgroup']}_{'|'.join(spec['params'])}",
    )[:90]
    return {
        "candidate_id": candidate_id,
        "selected_candidate_id": candidate_id,
        "wfe_path_id": path_id,
        "selection_key": selection_key,
        "source": spec["subgroup"],
        "rank": 1,
        "params": params,
        "candidate_mode": spec["mode"],
        "wfe_group": spec.get("group", ""),
        "subgroup": spec["subgroup"],
        "tested_params": list(spec["params"]),
        "tested_values": values,
        "window": window_id,
    }


def wfe_result_row_from_global_metrics(
    candidate: dict[str, Any],
    window: dict[str, str],
    phase: str,
    metrics: dict[str, Any],
) -> dict[str, Any]:
    phase = phase.upper()
    result = {
        "candidate_id": candidate["candidate_id"],
        "selected_candidate_id": candidate.get("selected_candidate_id", candidate["candidate_id"]),
        "wfe_path_id": candidate.get("wfe_path_id", candidate["candidate_id"]),
        "selection_key": candidate.get("selection_key", wfe_selection_key(candidate)),
        "source": candidate["source"],
        "rank": candidate["rank"],
        "candidate_mode": candidate.get("candidate_mode", ""),
        "wfe_group": candidate.get("wfe_group", ""),
        "subgroup": candidate.get("subgroup", ""),
        "tested_params": "|".join(candidate.get("tested_params", [])),
        "tested_values": json.dumps(candidate.get("tested_values", {}), ensure_ascii=False, sort_keys=True),
        "phase": phase,
        "window": window["window"],
        "is_from": window["is_from"],
        "is_to": window["is_to"],
        "oos_from": window["oos_from"],
        "oos_to": window["oos_to"],
        "symbol": "GLOBAL",
        "profit": as_float(metrics.get("total_profit")),
        "pf": as_float(metrics.get("median_pf")),
        "trades": as_float(metrics.get("total_trades")),
        "ddRel": as_float(metrics.get("max_ddRel")),
        "payoff": as_float(metrics.get("avg_payoff")),
        "recovery": as_float(metrics.get("avg_recovery")),
        "score": as_float(metrics.get("global_score")),
    }
    for key, value in candidate["params"].items():
        if isinstance(value, (bool, int, float, str)):
            result[f"param_{key}"] = value
    return result


def summarize_wfe_results(cfg: OptimizerConfig, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    grouped: dict[str, list[dict[str, Any]]] = {}
    for row in rows:
        grouped.setdefault(str(row.get("wfe_path_id") or row["candidate_id"]), []).append(row)

    def phase_metrics(items: list[dict[str, Any]]) -> dict[str, Any]:
        profits = [as_float(x.get("profit")) for x in items]
        pfs = [as_float(x.get("pf")) for x in items]
        trades = [as_float(x.get("trades")) for x in items]
        dds = [as_float(x.get("ddRel")) for x in items]
        recoveries = [as_float(x.get("recovery")) for x in items]
        total_tests = max(1, len(items))
        positive_tests = sum(1 for x in profits if x > 0)
        pf_gt_105 = sum(1 for x in pfs if x > 1.05)
        return {
            "total_profit": sum(profits),
            "positive_tests": positive_tests,
            "total_tests": total_tests,
            "consistency": positive_tests / total_tests,
            "pf_gt_105": pf_gt_105,
            "pf_consistency": pf_gt_105 / total_tests,
            "median_pf": median(pfs) if pfs else 0,
            "max_ddRel": max(dds) if dds else 0,
            "total_trades": sum(trades),
            "avg_recovery": sum(recoveries) / len(recoveries) if recoveries else 0,
        }

    summary: list[dict[str, Any]] = []
    for candidate_id, items in grouped.items():
        is_items = [x for x in items if str(x.get("phase", "")).upper() in {"IS", "ISS"}]
        oos_items = [x for x in items if str(x.get("phase", "")).upper() in {"OOS", "OSS"}]
        active_items = oos_items if oos_items else is_items if is_items else items

        is_m = phase_metrics(is_items)
        oos_m = phase_metrics(oos_items)
        active_m = phase_metrics(active_items)

        is_profit = is_m["total_profit"]
        oos_profit = oos_m["total_profit"]
        is_months = sum(
            inclusive_month_count(str(x.get("is_from", "")), str(x.get("is_to", "")))
            for x in is_items
        )
        oos_months = sum(
            inclusive_month_count(str(x.get("oos_from", "")), str(x.get("oos_to", "")))
            for x in oos_items
        )
        is_profit_per_month = is_profit / max(1, is_months)
        oos_profit_per_month = oos_profit / max(1, oos_months)
        if is_profit_per_month > 0 and oos_items:
            wfe_efficiency = oos_profit_per_month / is_profit_per_month
        else:
            wfe_efficiency = 0.0

        efficiency_ok = (bool(oos_items) and oos_profit > 0)
        efficiency_multiplier = 1.0

        dd_penalty = 1.0 / (1.0 + (active_m["max_ddRel"] / 10.0) ** 2)
        trade_score = min(1.0, active_m["total_trades"] / max(1.0, active_m["total_tests"] * 30.0))
        wfe_score = (
            max(0.0, active_m["total_profit"])
            * active_m["consistency"]
            * active_m["pf_consistency"]
            * max(0.0, active_m["median_pf"])
            * dd_penalty
            * trade_score
        )
        summary.append({
            "candidate_id": candidate_id,
            "source": items[0].get("source"),
            "rank": items[0].get("rank"),
            "candidate_mode": items[0].get("candidate_mode", ""),
            "wfe_group": items[0].get("wfe_group", ""),
            "subgroup": items[0].get("subgroup", ""),
            "tested_params": items[0].get("tested_params", ""),
            "tested_values": items[0].get("tested_values", ""),
            "wfe_score": wfe_score,
            "total_profit": active_m["total_profit"],
            "positive_tests": active_m["positive_tests"],
            "total_tests": active_m["total_tests"],
            "consistency": active_m["consistency"],
            "pf_gt_105": active_m["pf_gt_105"],
            "median_pf": active_m["median_pf"],
            "max_ddRel": active_m["max_ddRel"],
            "total_trades": active_m["total_trades"],
            "avg_recovery": active_m["avg_recovery"],
            "is_total_profit": is_m["total_profit"],
            "is_months": is_months,
            "is_profit_per_month": is_profit_per_month,
            "is_positive_tests": is_m["positive_tests"],
            "is_total_tests": is_m["total_tests"],
            "is_median_pf": is_m["median_pf"],
            "is_max_ddRel": is_m["max_ddRel"],
            "is_total_trades": is_m["total_trades"],
            "oos_total_profit": oos_m["total_profit"],
            "oos_months": oos_months,
            "oos_profit_per_month": oos_profit_per_month,
            "oos_positive_tests": oos_m["positive_tests"],
            "oos_total_tests": oos_m["total_tests"],
            "oos_median_pf": oos_m["median_pf"],
            "oos_max_ddRel": oos_m["max_ddRel"],
            "oos_total_trades": oos_m["total_trades"],
            "wfe_efficiency": wfe_efficiency,
            "wfe_efficiency_pct": wfe_efficiency * 100.0,
            "wfe_efficiency_ok": efficiency_ok,
            "wfe_efficiency_multiplier": efficiency_multiplier,
        })
    return sorted(summary, key=lambda r: as_float(r.get("wfe_score")), reverse=True)


def decode_json_dict(value: Any) -> dict[str, Any]:
    if isinstance(value, dict):
        return value
    try:
        decoded = json.loads(str(value or "{}"))
        return decoded if isinstance(decoded, dict) else {}
    except Exception:
        return {}


def wfe_selection_key_from_values(mode: str, subgroup: str, wfe_group: str, tested_params: str) -> str:
    return "|".join([
        str(mode or ""),
        str(subgroup or ""),
        str(wfe_group or ""),
        str(tested_params or ""),
    ])


def wfe_selection_key(row_or_candidate: dict[str, Any]) -> str:
    tested_params = row_or_candidate.get("tested_params", "")
    if isinstance(tested_params, list):
        tested_params = "|".join(tested_params)
    return wfe_selection_key_from_values(
        str(row_or_candidate.get("candidate_mode", "")),
        str(row_or_candidate.get("subgroup", "")),
        str(row_or_candidate.get("wfe_group", "")),
        str(tested_params),
    )


def wfe_basic_metrics(items: list[dict[str, Any]]) -> dict[str, Any]:
    profits = [as_float(x.get("profit")) for x in items]
    pfs = [as_float(x.get("pf")) for x in items]
    trades = [as_float(x.get("trades")) for x in items]
    dds = [as_float(x.get("ddRel")) for x in items]
    recoveries = [as_float(x.get("recovery")) for x in items]
    total_tests = max(1, len(items))
    positive_tests = sum(1 for x in profits if x > 0)
    pf_gt_105 = sum(1 for x in pfs if x > 1.05)
    max_dd = max(dds) if dds else 0.0
    total_trades = sum(trades)
    median_pf = median(pfs) if pfs else 0.0
    dd_penalty = 1.0 / (1.0 + (max_dd / 10.0) ** 2)
    trade_score = min(1.0, total_trades / max(1.0, total_tests * 30.0))
    score = (
        max(0.0, sum(profits))
        * (positive_tests / total_tests)
        * (pf_gt_105 / total_tests)
        * max(0.0, median_pf)
        * dd_penalty
        * trade_score
    )
    return {
        "score": score,
        "total_profit": sum(profits),
        "positive_tests": positive_tests,
        "total_tests": total_tests,
        "median_pf": median_pf,
        "max_ddRel": max_dd,
        "total_trades": total_trades,
        "avg_recovery": sum(recoveries) / len(recoveries) if recoveries else 0.0,
    }


def wfe_value_label(values: dict[str, Any], params: list[str]) -> str:
    return " + ".join(f"{param}={values.get(param, '')}" for param in params)


def wfe_parameter_sequence_labels(params: list[str]) -> list[list[str]]:
    ordered = [str(param) for param in params if str(param)]
    if not ordered:
        return []
    sequences: list[list[str]] = []
    for size in range(1, min(3, len(ordered)) + 1):
        for combo in itertools.combinations(ordered, size):
            sequences.append(list(combo))
    return sequences


def aggregate_iss_candidate_windows(iss_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    grouped: dict[tuple[int, str, str], list[dict[str, Any]]] = {}
    for row in iss_rows:
        phase = str(row.get("phase", "")).upper()
        if phase not in {"ISS", "IS"}:
            continue
        window = int(as_float(row.get("window"), 0))
        key = wfe_selection_key(row)
        candidate_id = str(row.get("candidate_id", ""))
        grouped.setdefault((window, key, candidate_id), []).append(row)

    output: list[dict[str, Any]] = []
    for (window, key, candidate_id), items in grouped.items():
        m = wfe_basic_metrics(items)
        first = items[0]
        values = decode_json_dict(first.get("tested_values"))
        tested_params = [param for param in str(first.get("tested_params", "")).split("|") if param]
        output.append({
            "window": window,
            "selection_key": key,
            "candidate_id": candidate_id,
            "candidate_mode": first.get("candidate_mode", ""),
            "wfe_group": first.get("wfe_group", ""),
            "subgroup": first.get("subgroup", ""),
            "tested_params": "|".join(tested_params),
            "tested_values": json.dumps(values, ensure_ascii=False, sort_keys=True),
            "values_dict": values,
            "iss_score": m["score"],
            "iss_profit": m["total_profit"],
            "iss_median_pf": m["median_pf"],
            "iss_max_ddRel": m["max_ddRel"],
            "iss_trades": m["total_trades"],
        })
    return sorted(output, key=lambda r: (r["selection_key"], int(r["window"]), str(r["candidate_id"])))


def build_iss_frequency_analysis(iss_rows: list[dict[str, Any]], max_top_n: int = 5) -> list[dict[str, Any]]:
    aggregated = aggregate_iss_candidate_windows(iss_rows)
    if not aggregated:
        return []

    max_top_n = max(1, int(max_top_n))
    by_key_window: dict[tuple[str, int], list[dict[str, Any]]] = {}
    all_by_analysis: dict[tuple[str, str, str, str], list[dict[str, Any]]] = {}

    for row in aggregated:
        by_key_window.setdefault((row["selection_key"], int(row["window"])), []).append(row)
        params = [param for param in str(row.get("tested_params", "")).split("|") if param]
        values = row.get("values_dict", {})
        for combo in wfe_parameter_sequence_labels(params):
            analysis_label = " + ".join(combo)
            value_label = wfe_value_label(values, combo)
            all_by_analysis.setdefault(
                (row["selection_key"], analysis_label, "|".join(combo), value_label),
                [],
            ).append(row)

    rows: list[dict[str, Any]] = []
    total_windows_by_key: dict[str, int] = {
        key: len({window for (item_key, window) in by_key_window if item_key == key})
        for key, _window in by_key_window
    }

    for top_n in range(1, max_top_n + 1):
        top_by_analysis: dict[tuple[str, str, str, str], list[dict[str, Any]]] = {}
        for (_key, _window), items in by_key_window.items():
            ranked = sorted(
                items,
                key=lambda r: (as_float(r.get("iss_score")), as_float(r.get("iss_profit"))),
                reverse=True,
            )[:top_n]
            for row in ranked:
                params = [param for param in str(row.get("tested_params", "")).split("|") if param]
                values = row.get("values_dict", {})
                for combo in wfe_parameter_sequence_labels(params):
                    analysis_label = " + ".join(combo)
                    value_label = wfe_value_label(values, combo)
                    top_by_analysis.setdefault(
                        (row["selection_key"], analysis_label, "|".join(combo), value_label),
                        [],
                    ).append(row)

        for key in sorted(set(all_by_analysis) | set(top_by_analysis)):
            top_items = top_by_analysis.get(key, [])
            selection_key, analysis_label, parameters, value_label = key
            all_items = all_by_analysis.get(key, [])
            total_windows = max(1, total_windows_by_key.get(selection_key, 0))
            avg_score_all = sum(as_float(x.get("iss_score")) for x in all_items) / max(1, len(all_items))
            avg_profit_all = sum(as_float(x.get("iss_profit")) for x in all_items) / max(1, len(all_items))
            avg_pf_all = sum(as_float(x.get("iss_median_pf")) for x in all_items) / max(1, len(all_items))
            avg_trades_all = sum(as_float(x.get("iss_trades")) for x in all_items) / max(1, len(all_items))
            max_dd_all = max([as_float(x.get("iss_max_ddRel")) for x in all_items] or [0.0])
            first = (top_items or all_items)[0]
            rows.append({
                "analysis_label": analysis_label,
                "combo_size": len(parameters.split("|")) if parameters else 0,
                "selection_key": selection_key,
                "candidate_mode": first.get("candidate_mode", ""),
                "wfe_group": first.get("wfe_group", ""),
                "subgroup": first.get("subgroup", ""),
                "parameters": parameters.replace("|", " + "),
                "values": value_label,
                "top_n": top_n,
                "frequency_count": len(top_items),
                "total_windows": total_windows,
                "frequency": len(top_items) / total_windows,
                "avg_metric_all": avg_score_all,
                "avg_profit_all": avg_profit_all,
                "avg_pf_all": avg_pf_all,
                "avg_trades_all": avg_trades_all,
                "max_ddRel_all": max_dd_all,
            })

    return sorted(
        rows,
        key=lambda r: (
            str(r["selection_key"]),
            str(r["analysis_label"]),
            int(r["top_n"]),
            -as_float(r["frequency"]),
            -as_float(r["avg_metric_all"]),
            str(r["values"]),
        ),
    )


def candidate_matches_wfe_values(candidate: dict[str, Any], parameters: str, value_label: str) -> bool:
    values = candidate.get("tested_values", {})
    params = [param.strip() for param in str(parameters).replace(" + ", "|").split("|") if param.strip()]
    return wfe_value_label(values, params) == value_label


def iss_window_winner_rows(iss_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    grouped: dict[tuple[int, str, str], list[dict[str, Any]]] = {}
    for row in iss_rows:
        window = int(as_float(row.get("window"), 0))
        key = wfe_selection_key(row)
        candidate_id = str(row.get("candidate_id", ""))
        grouped.setdefault((window, key, candidate_id), []).append(row)

    ranked_by_window_key: dict[tuple[int, str], list[dict[str, Any]]] = {}
    for (window, key, candidate_id), items in grouped.items():
        m = wfe_basic_metrics(items)
        first = items[0]
        ranked_by_window_key.setdefault((window, key), []).append({
            "window": window,
            "selection_key": key,
            "candidate_id": candidate_id,
            "candidate_mode": first.get("candidate_mode", ""),
            "wfe_group": first.get("wfe_group", ""),
            "subgroup": first.get("subgroup", ""),
            "tested_params": first.get("tested_params", ""),
            "tested_values": first.get("tested_values", ""),
            "iss_score": m["score"],
            "iss_profit": m["total_profit"],
            "iss_median_pf": m["median_pf"],
            "iss_max_ddRel": m["max_ddRel"],
            "iss_trades": m["total_trades"],
        })

    winners: list[dict[str, Any]] = []
    for (_window, _key), items in ranked_by_window_key.items():
        winners.append(sorted(items, key=lambda r: (as_float(r["iss_score"]), as_float(r["iss_profit"])), reverse=True)[0])
    return sorted(winners, key=lambda r: (r["selection_key"], int(r["window"])))


def build_iss_selection_table(iss_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    winners = iss_window_winner_rows(iss_rows)
    by_candidate: dict[tuple[str, str], list[dict[str, Any]]] = {}
    total_windows_by_key: dict[str, set[int]] = {}
    for row in winners:
        key = row["selection_key"]
        total_windows_by_key.setdefault(key, set()).add(int(row["window"]))
        by_candidate.setdefault((key, row["candidate_id"]), []).append(row)

    table: list[dict[str, Any]] = []
    for (key, candidate_id), items in by_candidate.items():
        total_windows = max(1, len(total_windows_by_key.get(key, set())))
        first = items[0]
        table.append({
            "selected": False,
            "selection_key": key,
            "candidate_id": candidate_id,
            "candidate_mode": first.get("candidate_mode", ""),
            "wfe_group": first.get("wfe_group", ""),
            "subgroup": first.get("subgroup", ""),
            "tested_params": first.get("tested_params", ""),
            "tested_values": first.get("tested_values", ""),
            "wins": len(items),
            "total_windows": total_windows,
            "frequency": len(items) / total_windows,
            "avg_iss_score": sum(as_float(x.get("iss_score")) for x in items) / len(items),
            "avg_iss_profit": sum(as_float(x.get("iss_profit")) for x in items) / len(items),
            "avg_iss_pf": sum(as_float(x.get("iss_median_pf")) for x in items) / len(items),
            "max_iss_ddRel": max(as_float(x.get("iss_max_ddRel")) for x in items),
            "avg_iss_trades": sum(as_float(x.get("iss_trades")) for x in items) / len(items),
        })
    return sorted(table, key=lambda r: (r["selection_key"], -as_float(r["frequency"]), -as_float(r["avg_iss_score"])))


def choose_wfe_candidates_from_iss(
    cfg: OptimizerConfig,
    candidates: list[dict[str, Any]],
    iss_rows: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
    frequency_rows = build_iss_frequency_analysis(iss_rows, max(1, cfg.wfe_frequency_top_n))
    aggregated = aggregate_iss_candidate_windows(iss_rows)
    candidate_by_id = {str(c["candidate_id"]): c for c in candidates}
    selected_ids: set[str] = set()
    selection_table: list[dict[str, Any]] = []

    keys = sorted({wfe_selection_key(candidate) for candidate in candidates})
    for key in keys:
        candidates_for_key = [candidate for candidate in candidates if wfe_selection_key(candidate) == key]
        if not candidates_for_key:
            continue
        first_candidate = candidates_for_key[0]
        full_params = list(first_candidate.get("tested_params", []))
        full_label = " + ".join(full_params)
        selection_top_n = min(max(1, cfg.wfe_top_n), max(1, cfg.wfe_frequency_top_n))
        rows = [
            row for row in frequency_rows
            if row.get("selection_key") == key
            and int(as_float(row.get("top_n"), 0)) == selection_top_n
            and str(row.get("parameters", "")) == full_label
        ]

        if not rows:
            fallback = build_iss_selection_table(iss_rows)
            rows = [
                {
                    **row,
                    "parameters": str(row.get("tested_params", "")).replace("|", " + "),
                    "values": row.get("tested_values", ""),
                    "top_n": 1,
                    "frequency_count": row.get("wins", 0),
                    "avg_metric_all": row.get("avg_iss_score", 0),
                    "avg_profit_all": row.get("avg_iss_profit", 0),
                    "avg_pf_all": row.get("avg_iss_pf", 0),
                    "avg_trades_all": row.get("avg_iss_trades", 0),
                }
                for row in fallback
                if row.get("selection_key") == key
            ]
        if not rows:
            continue

        candidate_options: list[dict[str, Any]] = []
        for row in sorted(rows, key=lambda r: (-as_float(r.get("frequency")), -as_float(r.get("avg_metric_all")))):
            matching_candidates = [
                candidate for candidate in candidates_for_key
                if candidate_matches_wfe_values(candidate, str(row.get("parameters", "")), str(row.get("values", "")))
            ]
            if not matching_candidates:
                continue
            candidate_scores: list[tuple[float, float, dict[str, Any]]] = []
            for candidate in matching_candidates:
                candidate_rows = [
                    item for item in aggregated
                    if item.get("selection_key") == key and item.get("candidate_id") == candidate.get("candidate_id")
                ]
                score = sum(as_float(item.get("iss_score")) for item in candidate_rows) / max(1, len(candidate_rows))
                profit = sum(as_float(item.get("iss_profit")) for item in candidate_rows) / max(1, len(candidate_rows))
                candidate_scores.append((score, profit, candidate))
            candidate_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
            if not candidate_scores:
                continue
            candidate = candidate_scores[0][2]
            candidate_options.append({
                "selected": False,
                "selection_key": key,
                "candidate_id": candidate["candidate_id"],
                "candidate_mode": candidate.get("candidate_mode", ""),
                "wfe_group": candidate.get("wfe_group", ""),
                "subgroup": candidate.get("subgroup", ""),
                "tested_params": "|".join(candidate.get("tested_params", [])),
                "tested_values": json.dumps(candidate.get("tested_values", {}), ensure_ascii=False, sort_keys=True),
                "parameters": row.get("parameters", ""),
                "values": row.get("values", ""),
                "top_n": row.get("top_n", selection_top_n),
                "wins": row.get("frequency_count", 0),
                "total_windows": row.get("total_windows", 0),
                "frequency": row.get("frequency", 0),
                "avg_iss_score": row.get("avg_metric_all", 0),
                "avg_iss_profit": row.get("avg_profit_all", 0),
                "avg_iss_pf": row.get("avg_pf_all", 0),
                "max_iss_ddRel": row.get("max_ddRel_all", 0),
                "avg_iss_trades": row.get("avg_trades_all", 0),
            })

        deduped: list[dict[str, Any]] = []
        seen_options: set[str] = set()
        for row in candidate_options:
            cid = str(row["candidate_id"])
            if cid in seen_options:
                continue
            seen_options.add(cid)
            deduped.append(row)
        rows = sorted(
            deduped,
            key=lambda r: (-as_float(r.get("frequency")), -as_float(r.get("avg_iss_score")), -as_float(r.get("avg_iss_profit"))),
        )
        if not rows:
            continue

        best_freq = as_float(rows[0].get("frequency"))
        tied = [row for row in rows if abs(as_float(row.get("frequency")) - best_freq) < 1e-12]
        tied = sorted(tied, key=lambda r: (-as_float(r.get("avg_iss_score")), -as_float(r.get("avg_iss_profit"))))

        selected = tied[0]
        options = rows[:20] if cfg.selection_mode == "manual" else tied[:20]
        should_ask = cfg.selection_mode == "manual" or (cfg.selection_mode == "semi_auto" and len(tied) > 1)
        if should_ask:
            print("\nISS WFE selection", flush=True)
            print(f"Selection key: {key}", flush=True)
            for idx, row in enumerate(options, 1):
                print(
                    f"  {idx}. {row['candidate_id']} | wins={row['wins']}/{row['total_windows']} "
                    f"freq={as_float(row['frequency']):.2%} score={as_float(row['avg_iss_score']):.4g} "
                    f"profit={as_float(row['avg_iss_profit']):.2f} values={row['tested_values']}",
                    flush=True,
                )
            choice = input("Choose candidate number or ENTER for recommended: ").strip()
            if choice.isdigit() and 1 <= int(choice) <= len(options):
                selected = options[int(choice) - 1]

        selected_ids.add(str(selected["candidate_id"]))
        selection_table.extend(rows)

    for row in selection_table:
        row["selected"] = str(row["candidate_id"]) in selected_ids

    selected_candidates = [candidate_by_id[cid] for cid in selected_ids if cid in candidate_by_id]
    selected_candidates = sorted(selected_candidates, key=lambda c: str(c.get("candidate_id")))
    return selected_candidates, selection_table, frequency_rows


def choose_wfe_top1_by_iss_window(
    candidates: list[dict[str, Any]],
    iss_rows: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    candidate_by_id = {str(candidate["candidate_id"]): candidate for candidate in candidates}
    aggregated = aggregate_iss_candidate_windows(iss_rows)
    grouped: dict[tuple[int, str], list[dict[str, Any]]] = {}
    for row in aggregated:
        grouped.setdefault((int(as_float(row.get("window"), 0)), str(row.get("selection_key", ""))), []).append(row)

    selected: list[dict[str, Any]] = []
    table: list[dict[str, Any]] = []
    for (window, key), items in sorted(grouped.items()):
        ranked = sorted(
            items,
            key=lambda r: (as_float(r.get("iss_score")), as_float(r.get("iss_profit"))),
            reverse=True,
        )
        if not ranked:
            continue
        winner = ranked[0]
        base_candidate = candidate_by_id.get(str(winner.get("candidate_id")))
        if not base_candidate:
            continue
        path_id = re.sub(
            r"[^A-Za-z0-9_]+",
            "_",
            f"WF_{winner.get('subgroup', '')}_{winner.get('tested_params', '')}",
        )[:90]
        candidate = dict(base_candidate)
        candidate["wfe_path_id"] = path_id
        candidate["selected_candidate_id"] = base_candidate["candidate_id"]
        candidate["selection_key"] = key
        candidate["window"] = window
        selected.append(candidate)
        table.append({
            "selected": True,
            "window": window,
            "selection_key": key,
            "wfe_path_id": path_id,
            "candidate_id": base_candidate["candidate_id"],
            "candidate_mode": winner.get("candidate_mode", ""),
            "wfe_group": winner.get("wfe_group", ""),
            "subgroup": winner.get("subgroup", ""),
            "tested_params": winner.get("tested_params", ""),
            "tested_values": winner.get("tested_values", ""),
            "iss_rank": 1,
            "iss_score": winner.get("iss_score", 0),
            "iss_profit": winner.get("iss_profit", 0),
            "iss_median_pf": winner.get("iss_median_pf", 0),
            "iss_max_ddRel": winner.get("iss_max_ddRel", 0),
            "iss_trades": winner.get("iss_trades", 0),
        })
    return selected, table


def wfe_parameter_rows(cfg: OptimizerConfig) -> list[dict[str, Any]]:
    rows: list[dict[str, Any]] = []
    for spec in get_wfe_specs(cfg):
        ignored = ",".join(spec.get("ignored_params", []))
        for param in spec["params"]:
            start, step, stop = spec["ranges"][param]
            rows.append({
                "parameter": param,
                "mode": spec["mode"],
                "wfe_group": spec.get("group", ""),
                "subgroup": spec["subgroup"],
                "subgroup_title": spec["title"],
                "default": cfg.base.get(param, ""),
                "start": start,
                "step": step,
                "stop": stop,
                "ignored_extra_group_params": ignored,
            })
    return rows


def build_frequency_rows(summary: list[dict[str, Any]], top_n: int = 20) -> list[dict[str, Any]]:
    ranked = [row for row in summary if row.get("candidate_mode") != "baseline"]
    top10 = ranked[: min(10, len(ranked))]
    topx = ranked[: min(top_n, len(ranked))]

    def count_values(items: list[dict[str, Any]]) -> dict[tuple[str, str], int]:
        counts: dict[tuple[str, str], int] = {}
        for row in items:
            values = decode_json_dict(row.get("tested_values"))
            for param, value in values.items():
                key = (param, str(value))
                counts[key] = counts.get(key, 0) + 1
        return counts

    c10 = count_values(top10)
    cx = count_values(topx)
    keys = sorted(set(c10) | set(cx))
    rows: list[dict[str, Any]] = []
    for param, value in keys:
        count10 = c10.get((param, value), 0)
        countx = cx.get((param, value), 0)
        rows.append({
            "label": f"{param}={value}",
            "parameter": param,
            "value": value,
            "count_top10": count10,
            "freq_top10": count10 / max(1, len(top10)),
            f"count_oss_top{top_n}": countx,
            f"freq_oss_top{top_n}": countx / max(1, len(topx)),
        })
    return sorted(rows, key=lambda r: (r["parameter"], -as_float(r.get(f"freq_oss_top{top_n}")), -as_float(r.get("freq_top10")), str(r["value"])))


def build_recommended_default_rows(cfg: OptimizerConfig, summary: list[dict[str, Any]]) -> list[dict[str, Any]]:
    freq_top_n = max(1, cfg.wfe_frequency_top_n)
    freq_rows = build_frequency_rows(summary, freq_top_n)
    best_by_param: dict[str, dict[str, Any]] = {}
    for row in freq_rows:
        param = row["parameter"]
        if param not in best_by_param:
            best_by_param[param] = row

    rows: list[dict[str, Any]] = []
    for param, row in sorted(best_by_param.items()):
        rows.append({
            "parameter": param,
            "current_default": cfg.base.get(param, ""),
            "recommended": row["value"],
            "freq_top10": row["freq_top10"],
            f"freq_oss_top{freq_top_n}": row[f"freq_oss_top{freq_top_n}"],
            "reason": f"highest frequency among ranked TOP{freq_top_n} WFE candidates, using TOP10 as tie-break context",
        })
    return rows


def build_group_permutation_frequency_rows(summary: list[dict[str, Any]], top_n: int = 20) -> list[dict[str, Any]]:
    ranked = [
        row for row in summary
        if row.get("candidate_mode") == "grouped" and decode_json_dict(row.get("tested_values"))
    ]
    top10 = ranked[: min(10, len(ranked))]
    topx = ranked[: min(top_n, len(ranked))]

    def count_combos(items: list[dict[str, Any]]) -> dict[tuple[str, str, str, str], int]:
        counts: dict[tuple[str, str, str, str], int] = {}
        for row in items:
            values = decode_json_dict(row.get("tested_values"))
            params = list(values.keys())[:3]
            for size in range(1, len(params) + 1):
                for combo in itertools.combinations(params, size):
                    combo_values = tuple((param, values[param]) for param in combo)
                    parameters = " + ".join(param for param, _ in combo_values)
                    value_label = " + ".join(f"{param}={value}" for param, value in combo_values)
                    key = (
                        str(row.get("subgroup", "")),
                        str(row.get("wfe_group", "")),
                        parameters,
                        value_label,
                    )
                    counts[key] = counts.get(key, 0) + 1
        return counts

    c10 = count_combos(top10)
    cx = count_combos(topx)
    keys = sorted(set(c10) | set(cx))
    rows: list[dict[str, Any]] = []
    for subgroup, wfe_group, parameters, value_label in keys:
        count10 = c10.get((subgroup, wfe_group, parameters, value_label), 0)
        countx = cx.get((subgroup, wfe_group, parameters, value_label), 0)
        combo_size = parameters.count("+") + 1
        rows.append({
            "label": value_label,
            "subgroup": subgroup,
            "wfe_group": wfe_group,
            "combo_size": combo_size,
            "parameters": parameters,
            "values": value_label,
            "count_top10": count10,
            "freq_top10": count10 / max(1, len(top10)),
            f"count_oss_top{top_n}": countx,
            f"freq_oss_top{top_n}": countx / max(1, len(topx)),
        })
    return sorted(
        rows,
        key=lambda r: (
            str(r["subgroup"]),
            int(r["combo_size"]),
            -as_float(r.get(f"freq_oss_top{top_n}")),
            str(r["values"]),
        ),
    )


def build_wfe_window_result_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    grouped: dict[tuple[str, int, str], list[dict[str, Any]]] = {}
    for row in rows:
        phase = str(row.get("phase", "")).upper()
        if phase not in {"ISS", "OSS", "IS", "OOS"}:
            continue
        grouped.setdefault((str(row.get("candidate_id")), int(as_float(row.get("window"), 0)), phase), []).append(row)

    result: list[dict[str, Any]] = []
    for (candidate_id, window, phase), items in grouped.items():
        m = wfe_basic_metrics(items)
        first = items[0]
        result.append({
            "candidate_id": candidate_id,
            "phase": "ISS" if phase == "IS" else "OSS" if phase == "OOS" else phase,
            "window": window,
            "candidate_mode": first.get("candidate_mode", ""),
            "wfe_group": first.get("wfe_group", ""),
            "subgroup": first.get("subgroup", ""),
            "tested_params": first.get("tested_params", ""),
            "tested_values": first.get("tested_values", ""),
            "from": first.get("is_from") if phase in {"ISS", "IS"} else first.get("oos_from"),
            "to": first.get("is_to") if phase in {"ISS", "IS"} else first.get("oos_to"),
            "profit": m["total_profit"],
            "median_pf": m["median_pf"],
            "max_ddRel": m["max_ddRel"],
            "trades": m["total_trades"],
            "positive_tests": m["positive_tests"],
            "total_tests": m["total_tests"],
            "score": m["score"],
        })

    final_by_candidate: dict[str, list[dict[str, Any]]] = {}
    for row in result:
        if row["phase"] == "OSS":
            final_by_candidate.setdefault(row["candidate_id"], []).append(row)
    for candidate_id, items in final_by_candidate.items():
        result.append({
            "candidate_id": candidate_id,
            "phase": "FINAL_OSS",
            "window": "ALL",
            "candidate_mode": items[0].get("candidate_mode", ""),
            "wfe_group": items[0].get("wfe_group", ""),
            "subgroup": items[0].get("subgroup", ""),
            "tested_params": items[0].get("tested_params", ""),
            "tested_values": items[0].get("tested_values", ""),
            "from": min(str(x.get("from")) for x in items),
            "to": max(str(x.get("to")) for x in items),
            "profit": sum(as_float(x.get("profit")) for x in items),
            "median_pf": median([as_float(x.get("median_pf")) for x in items]),
            "max_ddRel": max(as_float(x.get("max_ddRel")) for x in items),
            "trades": sum(as_float(x.get("trades")) for x in items),
            "positive_tests": sum(as_float(x.get("positive_tests")) for x in items),
            "total_tests": sum(as_float(x.get("total_tests")) for x in items),
            "score": sum(as_float(x.get("score")) for x in items),
        })

    return sorted(result, key=lambda r: (str(r["candidate_id"]), str(r["phase"]), str(r["window"])))


def build_wfe_didactic_window_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    by_candidate_window_phase: dict[tuple[str, int, str], list[dict[str, Any]]] = {}
    for row in rows:
        phase = str(row.get("phase", "")).upper()
        if phase == "IS":
            phase = "ISS"
        if phase == "OOS":
            phase = "OSS"
        if phase not in {"ISS", "OSS"}:
            continue
        path_id = str(row.get("wfe_path_id") or row.get("candidate_id"))
        by_candidate_window_phase.setdefault(
            (path_id, int(as_float(row.get("window"), 0)), phase),
            []
        ).append(row)

    candidate_windows = sorted({(cid, window) for cid, window, _phase in by_candidate_window_phase})
    output: list[dict[str, Any]] = []
    totals: dict[str, dict[str, Any]] = {}

    for candidate_id, window in candidate_windows:
        iss_items = by_candidate_window_phase.get((candidate_id, window, "ISS"), [])
        oss_items = by_candidate_window_phase.get((candidate_id, window, "OSS"), [])
        if not iss_items and not oss_items:
            continue

        first = (oss_items or iss_items)[0]
        iss = wfe_basic_metrics(iss_items)
        oss = wfe_basic_metrics(oss_items)
        iss_months = inclusive_month_count(str(first.get("is_from", "")), str(first.get("is_to", "")))
        oss_months = inclusive_month_count(str(first.get("oos_from", "")), str(first.get("oos_to", "")))
        iss_monthly = iss["total_profit"] / max(1, iss_months)
        oss_monthly = oss["total_profit"] / max(1, oss_months)
        wfe_pct = (oss_monthly / iss_monthly * 100.0) if iss_monthly > 0 and oss_items else 0.0

        totals.setdefault(candidate_id, {
            "candidate_id": candidate_id,
            "candidate_mode": first.get("candidate_mode", ""),
            "wfe_group": first.get("wfe_group", ""),
            "subgroup": first.get("subgroup", ""),
            "tested_params": first.get("tested_params", ""),
            "tested_values": first.get("tested_values", ""),
            "iss_profit": 0.0,
            "oss_profit": 0.0,
            "iss_months": 0,
            "oss_months": 0,
            "oss_trades": 0.0,
            "oss_dd": 0.0,
            "oss_pf_values": [],
        })
        totals[candidate_id]["iss_profit"] += iss["total_profit"]
        totals[candidate_id]["oss_profit"] += oss["total_profit"]
        totals[candidate_id]["iss_months"] += iss_months
        totals[candidate_id]["oss_months"] += oss_months
        totals[candidate_id]["oss_trades"] += oss["total_trades"]
        totals[candidate_id]["oss_dd"] = max(totals[candidate_id]["oss_dd"], oss["max_ddRel"])
        if oss_items:
            totals[candidate_id]["oss_pf_values"].append(oss["median_pf"])

        output.append({
            "candidate_id": candidate_id,
            "selected_candidate_id": first.get("selected_candidate_id", first.get("candidate_id", "")),
            "window": window,
            "candidate_mode": first.get("candidate_mode", ""),
            "wfe_group": first.get("wfe_group", ""),
            "subgroup": first.get("subgroup", ""),
            "tested_params": first.get("tested_params", ""),
            "tested_values": first.get("tested_values", ""),
            "iss_period": f"{first.get('is_from')} to {first.get('is_to')}",
            "iss_months": iss_months,
            "iss_profit": iss["total_profit"],
            "iss_profit_per_month": iss_monthly,
            "iss_pf": iss["median_pf"],
            "iss_ddRel": iss["max_ddRel"],
            "oss_period": f"{first.get('oos_from')} to {first.get('oos_to')}",
            "oss_months": oss_months,
            "oss_profit": oss["total_profit"],
            "oss_profit_per_month": oss_monthly,
            "oss_pf": oss["median_pf"],
            "oss_ddRel": oss["max_ddRel"],
            "oss_trades": oss["total_trades"],
            "wfe_pct": wfe_pct,
        })

    for candidate_id, total in totals.items():
        iss_monthly = total["iss_profit"] / max(1, total["iss_months"])
        oss_monthly = total["oss_profit"] / max(1, total["oss_months"])
        output.append({
            "candidate_id": candidate_id,
            "selected_candidate_id": "",
            "window": "FINAL",
            "candidate_mode": total.get("candidate_mode", ""),
            "wfe_group": total.get("wfe_group", ""),
            "subgroup": total.get("subgroup", ""),
            "tested_params": total.get("tested_params", ""),
            "tested_values": total.get("tested_values", ""),
            "iss_period": "ALL ISS WINDOWS",
            "iss_months": total["iss_months"],
            "iss_profit": total["iss_profit"],
            "iss_profit_per_month": iss_monthly,
            "iss_pf": "",
            "iss_ddRel": "",
            "oss_period": "ALL OSS WINDOWS",
            "oss_months": total["oss_months"],
            "oss_profit": total["oss_profit"],
            "oss_profit_per_month": oss_monthly,
            "oss_pf": median(total["oss_pf_values"]) if total["oss_pf_values"] else 0.0,
            "oss_ddRel": total["oss_dd"],
            "oss_trades": total["oss_trades"],
            "wfe_pct": (oss_monthly / iss_monthly * 100.0) if iss_monthly > 0 else 0.0,
        })

    return sorted(output, key=lambda r: (str(r["candidate_id"]), 999999 if str(r["window"]) == "FINAL" else int(r["window"])))


def flatten_summary_for_excel(summary: list[dict[str, Any]]) -> list[dict[str, Any]]:
    rows: list[dict[str, Any]] = []
    for row in summary:
        item = dict(row)
        values = decode_json_dict(item.get("tested_values"))
        for param, value in values.items():
            item[f"tested_{param}"] = value
        rows.append(item)
    return rows


def write_wfe_excel_report(
    cfg: OptimizerConfig,
    windows: list[dict[str, str]],
    summary: list[dict[str, Any]],
    rows: list[dict[str, Any]],
    iss_selection: list[dict[str, Any]] | None = None,
    iss_frequency_rows: list[dict[str, Any]] | None = None,
) -> Path | None:
    try:
        from openpyxl import Workbook
        from openpyxl.styles import Font, PatternFill, Alignment, Border, Side
        from openpyxl.formatting.rule import DataBarRule, ColorScaleRule
        from openpyxl.utils import get_column_letter
        from openpyxl.chart import BarChart, Reference
    except Exception as exc:
        print(f"WARNING: openpyxl is not available; Excel WFE report skipped: {exc}", flush=True)
        return None

    p = paths(cfg)
    workbook = Workbook()
    workbook.remove(workbook.active)
    workbook.calculation.calcMode = "auto"
    workbook.calculation.fullCalcOnLoad = True
    workbook.calculation.forceFullCalc = True

    header_fill = PatternFill("solid", fgColor="1F4E78")
    header_font = Font(color="FFFFFF", bold=True)
    section_fill = PatternFill("solid", fgColor="D9EAF7")
    light_fill = PatternFill("solid", fgColor="F3F6FA")
    wfe_fill = PatternFill("solid", fgColor="E2F0D9")
    thin_gray = Side(style="thin", color="B7B7B7")
    table_border = Border(left=thin_gray, right=thin_gray, top=thin_gray, bottom=thin_gray)

    def set_page_layout(ws: Any) -> None:
        ws.sheet_view.showGridLines = False
        ws.page_margins.left = 0.35
        ws.page_margins.right = 0.35
        ws.page_margins.top = 0.45
        ws.page_margins.bottom = 0.45

    def style_range(ws: Any, min_row: int, max_row: int, min_col: int, max_col: int, fill: Any | None = None, bold: bool = False) -> None:
        for row in ws.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col):
            for cell in row:
                cell.border = table_border
                cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
                if fill is not None:
                    cell.fill = fill
                if bold:
                    cell.font = Font(bold=True, color="FFFFFF" if fill == header_fill else "000000")

    def add_summary_sheet(name: str, data: list[dict[str, Any]]) -> None:
        ws = workbook.create_sheet(name[:31])
        set_page_layout(ws)
        ws.column_dimensions["A"].width = 4
        ws.column_dimensions["B"].width = 32
        ws.column_dimensions["C"].width = 78
        ws.row_dimensions[2].height = 28
        ws.merge_cells("B2:C2")
        ws["B2"] = "Walk Forward Summary"
        ws["B2"].fill = header_fill
        ws["B2"].font = Font(color="FFFFFF", bold=True, size=14)
        ws["B2"].alignment = Alignment(horizontal="center", vertical="center")
        ws["B3"] = ""
        ws["C3"] = ""
        style_range(ws, 3, 3, 2, 3, light_fill)

        row_idx = 4
        for item in data:
            ws.cell(row_idx, 2, item.get("item", ""))
            ws.cell(row_idx, 3, item.get("value", ""))
            ws.cell(row_idx, 2).fill = section_fill
            ws.cell(row_idx, 2).font = Font(bold=True)
            ws.cell(row_idx, 3).fill = light_fill
            style_range(ws, row_idx, row_idx, 2, 3)
            row_idx += 1
        ws.freeze_panes = "B4"

    def add_oss_wfe_reference_sheet(name: str, data: list[dict[str, Any]]) -> None:
        ws = workbook.create_sheet(name[:31])
        set_page_layout(ws)
        if not data:
            ws["B2"] = "No OSS WFE data"
            return

        windows_by_candidate: dict[str, list[dict[str, Any]]] = {}
        for row in data:
            candidate_id = str(row.get("candidate_id", ""))
            if str(row.get("window", "")) != "FINAL":
                windows_by_candidate.setdefault(candidate_id, []).append(row)

        gap = 1
        start_col = 1
        next_col = start_col
        max_row_used = 1
        separator_cols: list[int] = []

        for _block_index, (candidate_id, items) in enumerate(sorted(windows_by_candidate.items())):
            col = next_col
            items = sorted(items, key=lambda r: int(as_float(r.get("window"), 0)))
            first = items[0]
            param_headers = [param for param in str(first.get("tested_params", "")).split("|") if param]
            if not param_headers:
                param_headers = list(decode_json_dict(first.get("tested_values")).keys())
            block_width = 2 + len(param_headers) + 2
            end_col = col + block_width - 1
            pf_col = col + 2 + len(param_headers)
            dd_col = pf_col + 1

            ws.merge_cells(start_row=1, start_column=col, end_row=1, end_column=end_col)
            title = ws.cell(1, col, candidate_id)
            title.fill = header_fill
            title.font = Font(color="FFFFFF", bold=True, size=12)
            title.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)

            headers = ["Window", "Result / WFE", *param_headers, "PF", "DD"]
            for offset, header in enumerate(headers):
                cell = ws.cell(2, col + offset, header)
                cell.fill = section_fill
                cell.font = Font(bold=True)
                cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
                cell.border = table_border

            row_idx = 4
            wfe_values: list[float] = []

            for item in items:
                window = int(as_float(item.get("window"), 0))
                train_row = row_idx
                train_month_row = row_idx + 1
                test_row = row_idx + 2
                test_month_row = row_idx + 3
                wfe_row = row_idx + 4

                iss_months = max(1, int(as_float(item.get("iss_months"), 1)))
                oss_months = max(1, int(as_float(item.get("oss_months"), 1)))
                train_label = f"Training{window}"
                test_label = f"Test{window}"
                iss_period = str(item.get("iss_period", ""))
                oss_period = str(item.get("oss_period", ""))
                item_values = decode_json_dict(item.get("tested_values"))

                ws.cell(train_row, col, train_label)
                ws.cell(train_row, col + 1, as_float(item.get("iss_profit")))
                for param_offset, param in enumerate(param_headers):
                    ws.cell(train_row, col + 2 + param_offset, item_values.get(param, ""))
                ws.cell(train_row, pf_col, as_float(item.get("iss_pf")))
                ws.cell(train_row, dd_col, as_float(item.get("iss_ddRel")))

                ws.cell(train_month_row, col, iss_period)

                ws.cell(test_row, col, test_label)
                ws.cell(test_row, col + 1, as_float(item.get("oss_profit")))
                for param_offset, param in enumerate(param_headers):
                    ws.cell(test_row, col + 2 + param_offset, item_values.get(param, ""))
                ws.cell(test_row, pf_col, as_float(item.get("oss_pf")))
                ws.cell(test_row, dd_col, as_float(item.get("oss_ddRel")))

                ws.cell(test_month_row, col, oss_period)

                iss_monthly = as_float(item.get("iss_profit_per_month"))
                if iss_monthly == 0.0:
                    iss_monthly = as_float(item.get("iss_profit")) / max(1, iss_months)
                oss_monthly = as_float(item.get("oss_profit_per_month"))
                if oss_monthly == 0.0:
                    oss_monthly = as_float(item.get("oss_profit")) / max(1, oss_months)
                wfe_value = (oss_monthly / iss_monthly * 100.0) if iss_monthly > 0 else 0.0
                ws.cell(wfe_row, col, f"WFE{window}")
                ws.cell(wfe_row, col + 1, wfe_value)
                wfe_values.append(wfe_value)

                style_range(ws, train_row, test_month_row, col, end_col, light_fill)
                style_range(ws, wfe_row, wfe_row, col, end_col, wfe_fill, bold=True)
                row_idx += 5

            avg_row = row_idx + 1
            ws.cell(avg_row, col, "Average WFE")
            ws.cell(avg_row, col + 1, sum(wfe_values) / len(wfe_values) if wfe_values else 0.0)
            style_range(ws, avg_row, avg_row, col, end_col, wfe_fill, bold=True)
            max_row_used = max(max_row_used, avg_row)

            widths = [20, 14] + [14 for _ in param_headers] + [10, 10]
            for width_offset, width in enumerate(widths):
                ws.column_dimensions[get_column_letter(col + width_offset)].width = width
            if gap:
                separator_col = end_col + 1
                separator_cols.append(separator_col)
                ws.column_dimensions[get_column_letter(separator_col)].width = 3
            next_col = end_col + gap + 1

        for row in range(1, max_row_used + 1):
            ws.row_dimensions[row].height = 22
        for separator_col in separator_cols:
            for separator_row in range(1, max_row_used + 1):
                cell = ws.cell(separator_row, separator_col)
                cell.fill = PatternFill("solid", fgColor="D9D9D9")
                cell.border = table_border
        for row in ws.iter_rows(min_row=1, max_row=max_row_used):
            for cell in row:
                if isinstance(cell.value, (int, float)) or (isinstance(cell.value, str) and cell.value.startswith("=")):
                    cell.number_format = "0.00"
                cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
                cell.border = table_border
        ws.freeze_panes = "A4"

    def add_sheet(name: str, data: list[dict[str, Any]], chart_frequency: bool = False) -> None:
        ws = workbook.create_sheet(name[:31])
        if not data:
            ws.append(["message"])
            ws.append(["no data"])
            return

        headers = list(data[0].keys())
        ws.append(headers)
        for cell in ws[1]:
            cell.fill = header_fill
            cell.font = header_font
            cell.alignment = Alignment(horizontal="center")

        for row in data:
            ws.append([row.get(header, "") for header in headers])

        ws.freeze_panes = "A2"
        ws.auto_filter.ref = ws.dimensions
        for idx, header in enumerate(headers, 1):
            letter = get_column_letter(idx)
            max_len = max(len(str(header)), *(len(str(ws.cell(r, idx).value or "")) for r in range(2, min(ws.max_row, 200) + 1)))
            ws.column_dimensions[letter].width = min(max_len + 2, 45)
            if "freq_" in str(header).lower():
                for r in range(2, ws.max_row + 1):
                    ws.cell(r, idx).number_format = "0.00%"
                ws.conditional_formatting.add(
                    f"{letter}2:{letter}{ws.max_row}",
                    DataBarRule(start_type="num", start_value=0, end_type="num", end_value=1, color="63C384"),
                )
            if str(header).lower() == "wfe_pct":
                for r in range(2, ws.max_row + 1):
                    ws.cell(r, idx).number_format = "0.00"
                ws.conditional_formatting.add(
                    f"{letter}2:{letter}{ws.max_row}",
                    DataBarRule(start_type="num", start_value=0, end_type="num", end_value=150, color="63C384"),
                )
            if any(token in str(header).lower() for token in ("profit", "score", "trades", "pf", "dd", "recovery")):
                for r in range(2, ws.max_row + 1):
                    ws.cell(r, idx).number_format = "0.00"

        if chart_frequency and ws.max_row > 1:
            headers_lower = [str(h).lower() for h in headers]
            value_col = None
            for idx, header in enumerate(headers_lower, 1):
                if header == "frequency" or header.startswith("freq_oss_top") or header == "wfe_pct":
                    value_col = idx
                    break
            label_col = headers_lower.index("label") + 1 if "label" in headers_lower else 1
            if value_col:
                end_row = min(ws.max_row, 16)
                chart = BarChart()
                chart.type = "bar"
                chart.style = 10
                chart.title = f"{name[:24]} - bars"
                chart.y_axis.title = "Parameter/value"
                chart.x_axis.title = "Frequency"
                data_ref = Reference(ws, min_col=value_col, min_row=1, max_row=end_row)
                cats_ref = Reference(ws, min_col=label_col, min_row=2, max_row=end_row)
                chart.add_data(data_ref, titles_from_data=True)
                chart.set_categories(cats_ref)
                chart.height = 9
                chart.width = 18
                ws.add_chart(chart, f"{get_column_letter(ws.max_column + 2)}2")

    def write_table(ws: Any, start_row: int, title: str, data: list[dict[str, Any]]) -> int:
        ws.cell(start_row, 1, title)
        ws.cell(start_row, 1).font = Font(bold=True, size=13, color="1F4E78")
        row_idx = start_row + 1
        if not data:
            ws.cell(row_idx, 1, "no data")
            return row_idx + 2

        headers = list(data[0].keys())
        for col_idx, header in enumerate(headers, 1):
            cell = ws.cell(row_idx, col_idx, header)
            cell.fill = header_fill
            cell.font = header_font
            cell.alignment = Alignment(horizontal="center")

        for item in data:
            row_idx += 1
            for col_idx, header in enumerate(headers, 1):
                ws.cell(row_idx, col_idx, item.get(header, ""))

        for col_idx, header in enumerate(headers, 1):
            letter = get_column_letter(col_idx)
            max_len = max(
                len(str(header)),
                *(len(str(ws.cell(r, col_idx).value or "")) for r in range(start_row + 2, row_idx + 1)),
            )
            ws.column_dimensions[letter].width = min(max_len + 2, 45)
            if "frequency" in str(header).lower() or str(header).lower().startswith("freq_"):
                for r in range(start_row + 2, row_idx + 1):
                    ws.cell(r, col_idx).number_format = "0.00%"
                ws.conditional_formatting.add(
                    f"{letter}{start_row + 2}:{letter}{row_idx}",
                    DataBarRule(start_type="num", start_value=0, end_type="num", end_value=1, color="63C384"),
                )
            if str(header).lower() == "wfe_pct":
                for r in range(start_row + 2, row_idx + 1):
                    ws.cell(r, col_idx).number_format = "0.00"
                ws.conditional_formatting.add(
                    f"{letter}{start_row + 2}:{letter}{row_idx}",
                    DataBarRule(start_type="num", start_value=0, end_type="num", end_value=150, color="63C384"),
                )
            if any(token in str(header).lower() for token in ("profit", "score", "trades", "pf", "dd", "recovery")):
                for r in range(start_row + 2, row_idx + 1):
                    ws.cell(r, col_idx).number_format = "0.00"

        return row_idx + 2

    def add_iss_sheet(name: str, parameters: list[dict[str, Any]], selection: list[dict[str, Any]], frequency: list[dict[str, Any]]) -> None:
        ws = workbook.create_sheet(name[:31])
        set_page_layout(ws)

        def parse_frequency_row(row: dict[str, Any]) -> dict[str, Any]:
            params = [item.strip() for item in str(row.get("parameters", "")).split("+") if item.strip()]
            value_map: dict[str, Any] = {}
            for part in str(row.get("values", "")).split("+"):
                if "=" not in part:
                    continue
                key, value = part.split("=", 1)
                value_map[key.strip()] = parse_value(str(value).strip())
            return {
                **row,
                "params": params,
                "value_map": value_map,
                "media": as_float(row.get("avg_profit_all")),
                "frequency_count": as_float(row.get("frequency_count")),
            }

        def percentile_rank(values: list[float], value: float) -> float:
            if not values:
                return 0.0
            ordered = sorted(values)
            less_or_equal = sum(1 for item in ordered if item <= value)
            return less_or_equal / len(ordered)

        def robustness_rows(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
            numeric_items: list[dict[str, Any]] = []
            for item in items:
                coords: list[float] = []
                ok = True
                for param in item["params"][:3]:
                    value = item["value_map"].get(param)
                    if not isinstance(value, (int, float)):
                        ok = False
                        break
                    coords.append(float(value))
                if ok and len(coords) == 3:
                    numeric_items.append({**item, "_coords": coords})
            if not numeric_items:
                return items

            diffs: list[float] = []
            for i, item in enumerate(numeric_items):
                distances: list[tuple[float, int]] = []
                for j, other in enumerate(numeric_items):
                    if i == j:
                        continue
                    dist = math.sqrt(sum((item["_coords"][k] - other["_coords"][k]) ** 2 for k in range(3)))
                    distances.append((dist, j))
                nearest = [idx for _dist, idx in sorted(distances)[:5]]
                if nearest:
                    diffs.append(sum(abs(item["media"] - numeric_items[idx]["media"]) for idx in nearest) / len(nearest))
                else:
                    diffs.append(0.0)
            max_diff = max(diffs) if diffs else 0.0
            media_values = [item["media"] for item in numeric_items]
            freq_values = [as_float(item.get("frequency_count")) for item in numeric_items]
            robust_values = [max_diff - diff for diff in diffs]
            for item, robust in zip(numeric_items, robust_values):
                item["robustness"] = robust
                item["score_final"] = (
                    percentile_rank(media_values, item["media"]) * 0.4
                    + percentile_rank(freq_values, as_float(item.get("frequency_count"))) * 0.4
                    + percentile_rank(robust_values, robust) * 0.2
                )
            by_values = {str(item.get("values")): item for item in numeric_items}
            return [by_values.get(str(item.get("values")), item) for item in items]

        def write_headers(row_idx: int, col_idx: int, headers: list[str]) -> None:
            for offset, header in enumerate(headers):
                cell = ws.cell(row_idx, col_idx + offset, header)
                cell.fill = header_fill
                cell.font = header_font
                cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
                cell.border = table_border

        def write_title(row_idx: int, text: str, width: int = 8) -> None:
            ws.merge_cells(start_row=row_idx, start_column=1, end_row=row_idx, end_column=width)
            cell = ws.cell(row_idx, 1, text)
            cell.fill = section_fill
            cell.font = Font(bold=True, size=12, color="1F4E78")
            cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
            style_range(ws, row_idx, row_idx, 1, width, section_fill, bold=True)

        def add_color_scale(range_text: str) -> None:
            ws.conditional_formatting.add(
                range_text,
                ColorScaleRule(
                    start_type="min",
                    start_color="7030A0",
                    mid_type="percentile",
                    mid_value=50,
                    mid_color="F8696B",
                    end_type="max",
                    end_color="FFFF00",
                ),
            )

        parsed = [parse_frequency_row(row) for row in frequency]
        ws.column_dimensions["A"].width = 20
        for col_idx in range(2, 18):
            ws.column_dimensions[get_column_letter(col_idx)].width = 14

        ws.merge_cells("A1:H1")
        ws["A1"] = "ISS Frequency Analysis"
        ws["A1"].fill = header_fill
        ws["A1"].font = Font(color="FFFFFF", bold=True, size=14)
        ws["A1"].alignment = Alignment(horizontal="center", vertical="center")
        ws["A2"] = "Use the filters in the table below to select one parameter, a pair, or a 3-parameter sequence."
        ws["A2"].alignment = Alignment(wrap_text=True)

        filter_rows: list[dict[str, Any]] = []
        for item in parsed:
            params = item["params"]
            values = item["value_map"]
            filter_rows.append({
                "analysis_label": item.get("analysis_label", ""),
                "combo_size": item.get("combo_size", ""),
                "top_n": item.get("top_n", ""),
                "parameter_1": params[0] if len(params) > 0 else "",
                "value_1": values.get(params[0], "") if len(params) > 0 else "",
                "parameter_2": params[1] if len(params) > 1 else "",
                "value_2": values.get(params[1], "") if len(params) > 1 else "",
                "parameter_3": params[2] if len(params) > 2 else "",
                "value_3": values.get(params[2], "") if len(params) > 2 else "",
                "Frequency": item.get("frequency_count", 0),
                "MediaMetrica": item.get("media", 0),
                "PF": item.get("avg_pf_all", 0),
                "DD": item.get("max_ddRel_all", 0),
            })
        next_row = write_table(ws, 4, "Filterable ISS frequency data", filter_rows[: max(1, len(filter_rows))])
        if filter_rows:
            ws.auto_filter.ref = f"A5:{get_column_letter(len(filter_rows[0]))}{next_row - 2}"

        grouped: dict[tuple[str, int, int], list[dict[str, Any]]] = {}
        for item in parsed:
            grouped.setdefault((str(item.get("analysis_label")), int(as_float(item.get("top_n"), 0)), int(as_float(item.get("combo_size"), 0))), []).append(item)

        row_cursor = next_row + 1
        chart_anchor_col = 8
        for (analysis_label, top_n, combo_size), items in sorted(grouped.items(), key=lambda x: (x[0][2], x[0][0], x[0][1])):
            items = sorted(items, key=lambda r: (-as_float(r.get("frequency_count")), -as_float(r.get("media"))))
            if combo_size == 1:
                param = items[0]["params"][0] if items and items[0]["params"] else "parameter"
                write_title(row_cursor, f"Top {top_n} ranking - {param}", 6)
                header_row = row_cursor + 1
                write_headers(header_row, 1, [param, "Frequencia", "MediaMetrica"])
                for idx, item in enumerate(items, header_row + 1):
                    ws.cell(idx, 1, item["value_map"].get(param, ""))
                    ws.cell(idx, 2, item.get("frequency_count", 0))
                    ws.cell(idx, 3, item.get("media", 0))
                    style_range(ws, idx, idx, 1, 3, light_fill if (idx - header_row) % 2 == 0 else None)
                data_end = header_row + len(items)
                if items:
                    chart = BarChart()
                    chart.type = "bar"
                    chart.style = 10
                    chart.title = f"Ranking de {param} - Top {top_n}"
                    chart.y_axis.title = param
                    chart.x_axis.title = "Frequencia"
                    data_ref = Reference(ws, min_col=2, min_row=header_row, max_row=min(data_end, header_row + 15))
                    cats_ref = Reference(ws, min_col=1, min_row=header_row + 1, max_row=min(data_end, header_row + 15))
                    chart.add_data(data_ref, titles_from_data=True)
                    chart.set_categories(cats_ref)
                    chart.height = 8
                    chart.width = 16
                    ws.add_chart(chart, f"{get_column_letter(chart_anchor_col)}{row_cursor}")
                row_cursor = max(data_end + 3, row_cursor + 18)
                continue

            if combo_size == 2:
                params = items[0]["params"][:2] if items else ["param1", "param2"]
                p1, p2 = params[0], params[1]
                write_title(row_cursor, f"Top {top_n} ranking combinations - {p1}, {p2}", 7)
                header_row = row_cursor + 1
                write_headers(header_row, 1, [p1, p2, "Frequencia", "Media"])
                for idx, item in enumerate(items, header_row + 1):
                    ws.cell(idx, 1, item["value_map"].get(p1, ""))
                    ws.cell(idx, 2, item["value_map"].get(p2, ""))
                    ws.cell(idx, 3, item.get("frequency_count", 0))
                    ws.cell(idx, 4, item.get("media", 0))
                    style_range(ws, idx, idx, 1, 4, light_fill if (idx - header_row) % 2 == 0 else None)
                data_end = header_row + len(items)

                heat_col = 7
                for heat_idx, metric in enumerate(("frequency_count", "media")):
                    title_row = row_cursor + heat_idx * (len(set(item["value_map"].get(p1, "") for item in items)) + 5)
                    ws.cell(title_row, heat_col, f"Heatmap - {'Frequencia' if metric == 'frequency_count' else 'Media'} - Top {top_n}")
                    ws.cell(title_row, heat_col).font = Font(bold=True, color="1F4E78")
                    x_values = sorted({item["value_map"].get(p2, "") for item in items}, key=lambda x: str(x))
                    y_values = sorted({item["value_map"].get(p1, "") for item in items}, key=lambda x: str(x))
                    for offset, value in enumerate(x_values, 1):
                        ws.cell(title_row + 1, heat_col + offset, value)
                    for y_offset, y_value in enumerate(y_values, 1):
                        ws.cell(title_row + 1 + y_offset, heat_col, y_value)
                        for x_offset, x_value in enumerate(x_values, 1):
                            match = next((item for item in items if item["value_map"].get(p1) == y_value and item["value_map"].get(p2) == x_value), None)
                            ws.cell(title_row + 1 + y_offset, heat_col + x_offset, as_float(match.get(metric)) if match else 0.0)
                    style_range(ws, title_row + 1, title_row + 1 + len(y_values), heat_col, heat_col + len(x_values), None)
                    if x_values and y_values:
                        add_color_scale(f"{get_column_letter(heat_col + 1)}{title_row + 2}:{get_column_letter(heat_col + len(x_values))}{title_row + 1 + len(y_values)}")
                row_cursor = max(data_end + 3, row_cursor + 2 * (len({item["value_map"].get(p1, "") for item in items}) + 5) + 2)
                continue

            if combo_size == 3:
                params = items[0]["params"][:3] if items else ["param1", "param2", "param3"]
                p1, p2, p3 = params[0], params[1], params[2]
                robust_items = sorted(
                    robustness_rows(items),
                    key=lambda r: (-as_float(r.get("score_final")), -as_float(r.get("frequency_count")), -as_float(r.get("media"))),
                )
                write_title(row_cursor, f"Top {top_n} robust 3-parameter combinations - {p1}, {p2}, {p3}", 9)
                header_row = row_cursor + 1
                write_headers(header_row, 1, [p1, p2, p3, "Frequencia", "Media", "Robustez", "Score_Final"])
                for idx, item in enumerate(robust_items[:30], header_row + 1):
                    ws.cell(idx, 1, item["value_map"].get(p1, ""))
                    ws.cell(idx, 2, item["value_map"].get(p2, ""))
                    ws.cell(idx, 3, item["value_map"].get(p3, ""))
                    ws.cell(idx, 4, item.get("frequency_count", 0))
                    ws.cell(idx, 5, item.get("media", 0))
                    ws.cell(idx, 6, item.get("robustness", 0))
                    ws.cell(idx, 7, item.get("score_final", 0))
                    style_range(ws, idx, idx, 1, 7, light_fill if (idx - header_row) % 2 == 0 else None)
                data_end = header_row + min(30, len(robust_items))

                heat_col = 10
                heat_row = row_cursor
                p3_values = sorted({item["value_map"].get(p3, "") for item in robust_items}, key=lambda x: str(x))[:5]
                for p3_value in p3_values:
                    slice_items = [item for item in robust_items if item["value_map"].get(p3) == p3_value]
                    ws.cell(heat_row, heat_col, f"Heatmap Media - {p3}={p3_value}")
                    ws.cell(heat_row, heat_col).font = Font(bold=True, color="1F4E78")
                    x_values = sorted({item["value_map"].get(p2, "") for item in slice_items}, key=lambda x: str(x))
                    y_values = sorted({item["value_map"].get(p1, "") for item in slice_items}, key=lambda x: str(x))
                    for offset, value in enumerate(x_values, 1):
                        ws.cell(heat_row + 1, heat_col + offset, value)
                    for y_offset, y_value in enumerate(y_values, 1):
                        ws.cell(heat_row + 1 + y_offset, heat_col, y_value)
                        for x_offset, x_value in enumerate(x_values, 1):
                            match = next((item for item in slice_items if item["value_map"].get(p1) == y_value and item["value_map"].get(p2) == x_value), None)
                            ws.cell(heat_row + 1 + y_offset, heat_col + x_offset, as_float(match.get("media")) if match else 0.0)
                    style_range(ws, heat_row + 1, heat_row + 1 + len(y_values), heat_col, heat_col + len(x_values), None)
                    if x_values and y_values:
                        add_color_scale(f"{get_column_letter(heat_col + 1)}{heat_row + 2}:{get_column_letter(heat_col + len(x_values))}{heat_row + 1 + len(y_values)}")
                    heat_row += len(y_values) + 4
                row_cursor = max(data_end + 3, heat_row + 1)

        ws.freeze_panes = "A6"

    summary_rows = [
        {"item": "EA", "value": cfg.expert_name},
        {"item": "symbols", "value": ", ".join(cfg.symbols)},
        {"item": "period", "value": f"{cfg.wfe_from_date or cfg.from_date} to {cfg.wfe_to_date or cfg.to_date}"},
        {"item": "model", "value": cfg.model},
        {"item": "selection_mode", "value": cfg.selection_mode},
        {"item": "wfe_windows", "value": len(windows)},
        {"item": "iss_selection_rule", "value": "run ISS candidates in each window and select the ISS TOP 1 by score"},
        {"item": "oss_validation_rule", "value": "run OSS only with the ISS TOP 1 selected for the same window"},
        {"item": "wfe_formula", "value": "(OSS profit / OSS months) / (ISS profit / ISS months) * 100"},
    ]

    iss_selection = iss_selection or []
    iss_frequency_rows = iss_frequency_rows or []

    add_summary_sheet("SUMMARY", summary_rows)
    add_oss_wfe_reference_sheet("WFE_ANALYSIS", build_wfe_didactic_window_rows(rows))

    out = p["run_dir"] / "WFE_ANALYSIS.xlsx"
    workbook.save(out)
    return out


def write_wfe_report(
    cfg: OptimizerConfig,
    windows: list[dict[str, str]],
    summary: list[dict[str, Any]],
    rows: list[dict[str, Any]],
    iss_selection: list[dict[str, Any]] | None = None,
    iss_frequency_rows: list[dict[str, Any]] | None = None,
) -> None:
    p = paths(cfg)
    excel_path = write_wfe_excel_report(cfg, windows, summary, rows, iss_selection, iss_frequency_rows)
    lines = [
        f"# WFE Report - {cfg.expert_name}",
        "",
        f"- Symbols: {', '.join(cfg.symbols)}",
        f"- WFE period: {cfg.wfe_from_date or cfg.from_date} to {cfg.wfe_to_date or cfg.to_date}",
        f"- In-sample months: {cfg.wfe_insample_months}",
        f"- Out-of-sample months: {cfg.wfe_outsample_months}",
        f"- Step months: {cfg.wfe_step_months}",
        f"- Selection rule: run ISS candidates per window, select the ISS TOP 1 by score, then run OSS for the matching window",
        f"- Selection mode: {cfg.selection_mode}",
        f"- WFE efficiency: calculated as OSS/ISS profit for diagnosis; not used as a hard min/max filter",
        f"- Candidates tested: {len(summary)}",
        f"- Excel analysis: {excel_path if excel_path else 'not generated'}",
        "",
        "## Windows",
    ]
    for w in windows:
        lines.append(
            f"- W{w['window']}: ISS {w['is_from']} to {w['is_to']} | "
            f"OSS {w['oos_from']} to {w['oos_to']}"
        )
    lines += ["", "## Selected WFE Candidates"]
    for i, row in enumerate(summary, 1):
        lines.append(
            f"- {i}: {row['candidate_id']} | score={row['wfe_score']:.4g} | "
            f"OSS_profit={row.get('oos_total_profit', row['total_profit']):.2f} | "
            f"ISS_profit={row.get('is_total_profit', 0):.2f} | "
            f"WFE%={row.get('wfe_efficiency_pct', 0):.2f} | "
            f"consistency={row['positive_tests']}/{row['total_tests']} | "
            f"median_pf={row['median_pf']:.3f} | trades={int(row['total_trades'])} | "
            f"max_dd={row['max_ddRel']:.2f}%"
        )
    lines += ["", "## ISS TOP 1 Selection"]
    for row in (iss_selection or []):
        lines.append(
            f"- W{row.get('window')}: {row.get('candidate_id')} | "
            f"ISS_score={as_float(row.get('iss_score')):.4g} | "
            f"ISS_profit={as_float(row.get('iss_profit')):.2f} | values={row.get('tested_values')}"
        )
    (p["run_dir"] / "WFE_REPORT.md").write_text("\n".join(lines) + "\n", encoding="utf-8")


def run_wfe(cfg: OptimizerConfig) -> None:
    if not cfg.enable_wfe:
        print("WFE is disabled in the config.")
        return
    p = paths(cfg)
    p["run_dir"].mkdir(parents=True, exist_ok=True)
    windows = build_wfe_windows(cfg)
    if not windows:
        raise RuntimeError("No WFE windows could be built. Check WFE dates and window sizes.")
    specs = get_wfe_specs(cfg)
    if not specs:
        print("WFE skipped: no @wfe, @wfe1, @wfe2 or @wfe3 parameters were found in the config.", flush=True)
        (p["run_dir"] / "WFE_REPORT.md").write_text(
            "# WFE Report\n\nWFE skipped: no @wfe parameters were found.\n",
            encoding="utf-8",
        )
        return
    params_path = p["run_dir"] / "current_defaults.json"
    final_params = dict(cfg.base)
    if params_path.exists():
        final_params.update(json.loads(params_path.read_text(encoding="utf-8")))

    print(f"\n=== WFE STAGE ===")
    print(f"Windows: {len(windows)} | WFE specs: {len(specs)} | Symbols: {len(cfg.symbols)}")
    rows: list[dict[str, Any]] = []
    iss_rows_all: list[dict[str, Any]] = []
    oss_rows_all: list[dict[str, Any]] = []
    selected_candidates: list[dict[str, Any]] = []
    iss_selection: list[dict[str, Any]] = []

    print("\n=== WFE ISS PASS: optimizing marked ranges and selecting TOP 1 per window/spec ===", flush=True)
    for window in windows:
        window_id = int(as_float(window.get("window"), 0))
        print(
            f"\nWFE window {window['window']}: "
            f"ISS {window['is_from']} to {window['is_to']} | "
            f"OSS {window['oos_from']} to {window['oos_to']}",
            flush=True,
        )
        for spec in specs:
            label = wfe_spec_label(spec)
            subgroup = Subgroup(
                tag=f"WFEISS_{label}_W{window_id}",
                title=f"WFE ISS {spec['title']}",
                group="WFE",
                ranges=dict(spec["ranges"]),
                fixed={},
            )
            passes = estimate_passes(subgroup.ranges)
            test_cfg = OptimizerConfig(**{
                **asdict(cfg),
                "from_date": window["is_from"],
                "to_date": window["is_to"],
                # WFE ISS is intentionally complete/slow over the marked range.
                "genetic_threshold_passes": max(cfg.genetic_threshold_passes, passes),
            })
            print(
                f"  ISS optimize {spec['subgroup']} ({' + '.join(spec['params'])}) | "
                f"passes={passes} | mode=complete",
                flush=True,
            )
            symbol_rows = collect_symbol_rows(
                test_cfg,
                subgroup,
                final_params,
                baseline=False,
                run_label="",
                force_run=False,
            )
            global_rows = aggregate_global_rows(test_cfg, symbol_rows, subgroup)
            write_csv(p["run_dir"] / f"WFE_ISS_{label}_W{window_id}_global_all.csv", global_rows)
            write_csv(p["run_dir"] / f"WFE_ISS_{label}_W{window_id}_global_top20.csv", global_rows[:20])
            if not global_rows:
                continue
            best = global_rows[0]
            candidate = wfe_candidate_from_global_row(spec, final_params, best, window_id)
            selected_candidates.append(candidate)
            iss_row = wfe_result_row_from_global_metrics(candidate, window, "ISS", best)
            iss_rows_all.append(iss_row)
            rows.append(iss_row)
            iss_selection.append({
                "selected": True,
                "window": window_id,
                "selection_key": candidate["selection_key"],
                "wfe_path_id": candidate["wfe_path_id"],
                "candidate_id": candidate["candidate_id"],
                "candidate_mode": candidate.get("candidate_mode", ""),
                "wfe_group": candidate.get("wfe_group", ""),
                "subgroup": candidate.get("subgroup", ""),
                "tested_params": "|".join(candidate.get("tested_params", [])),
                "tested_values": json.dumps(candidate.get("tested_values", {}), ensure_ascii=False, sort_keys=True),
                "iss_rank": 1,
                "iss_score": best.get("global_score", 0),
                "iss_profit": best.get("total_profit", 0),
                "iss_median_pf": best.get("median_pf", 0),
                "iss_max_ddRel": best.get("max_ddRel", 0),
                "iss_trades": best.get("total_trades", 0),
                "passes": passes,
            })
            write_csv(p["run_dir"] / "WFE_ISS_RESULTS.csv", iss_rows_all)
            write_csv(p["run_dir"] / "WFE_RESULTS.csv", rows)
    write_csv(p["run_dir"] / "WFE_ISS_TOP1_SELECTION.csv", iss_selection)

    print("\n=== WFE ISS SELECTION ===", flush=True)
    if not selected_candidates:
        print("No ISS candidate was selected. WFE stopped after ISS.", flush=True)
    for candidate in selected_candidates:
        print(
            f"  W{candidate.get('window')}: selected for OSS: "
            f"{candidate['candidate_id']} values={candidate.get('tested_values')}",
            flush=True,
        )

    selected_by_window: dict[int, list[dict[str, Any]]] = {}
    for candidate in selected_candidates:
        selected_by_window.setdefault(int(as_float(candidate.get("window"), 0)), []).append(candidate)

    print("\n=== WFE OSS PASS: running each ISS TOP 1 on its matching OSS window ===", flush=True)
    for window in windows:
        window_id = int(as_float(window.get("window"), 0))
        print(
            f"\nWFE window {window['window']}: OSS {window['oos_from']} to {window['oos_to']}",
            flush=True,
        )
        for candidate in selected_by_window.get(window_id, []):
            for symbol in cfg.symbols:
                row = run_wfe_single_test(cfg, candidate, window, symbol, "OSS")
                oss_rows_all.append(row)
                rows.append(row)
                write_csv(p["run_dir"] / "WFE_OSS_RESULTS.csv", oss_rows_all)
                write_csv(p["run_dir"] / "WFE_RESULTS.csv", rows)

    selected_keys = {
        (int(as_float(candidate.get("window"), 0)), str(candidate.get("selection_key", "")), str(candidate["candidate_id"]))
        for candidate in selected_candidates
    }
    selected_meta = {
        (int(as_float(candidate.get("window"), 0)), str(candidate.get("selection_key", "")), str(candidate["candidate_id"])): candidate
        for candidate in selected_candidates
    }
    final_rows: list[dict[str, Any]] = []
    for row in rows:
        row_key = (
            int(as_float(row.get("window"), 0)),
            str(row.get("selection_key", "")),
            str(row.get("candidate_id")),
        )
        if row_key not in selected_keys:
            continue
        normalized = dict(row)
        meta = selected_meta[row_key]
        normalized["wfe_path_id"] = meta.get("wfe_path_id", normalized.get("wfe_path_id"))
        normalized["selected_candidate_id"] = meta.get("selected_candidate_id", normalized.get("candidate_id"))
        final_rows.append(normalized)
    summary = summarize_wfe_results(cfg, final_rows)
    write_csv(p["run_dir"] / "WFE_SUMMARY.csv", summary)
    write_wfe_report(cfg, windows, summary, final_rows, iss_selection, [])
    print(f"OK: WFE finished. Report: {p['run_dir'] / 'WFE_REPORT.md'}")


def run_stage(cfg: OptimizerConfig, stage: str = "main", start_at: str | None = None, stop_after: str | None = None) -> None:
    p = paths(cfg)
    p["run_dir"].mkdir(parents=True, exist_ok=True)
    params_path = p["run_dir"] / "current_defaults.json"
    manifest_path = p["run_dir"] / "manifest.json"
    params = dict(cfg.base)
    if params_path.exists():
        params.update(json.loads(params_path.read_text(encoding="utf-8")))
    manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else []
    subgroups = [subgroup_from_dict(x) for x in cfg.subgroups]
    if stage == "rewind":
        subgroups = build_rewind_plan(params, subgroups)
    if start_at:
        while subgroups and subgroups[0].tag != start_at:
            subgroups.pop(0)
    reprocess_queue: list[Subgroup] = []
    reprocess_seen: set[str] = set()

    def mark_for_reprocess(subgroup: Subgroup) -> None:
        if subgroup.tag in reprocess_seen:
            return
        reprocess_seen.add(subgroup.tag)
        reprocess_queue.append(subgroup)
        queue_path = p["run_dir"] / f"{stage}_reprocess_queue.json"
        queue_path.write_text(
            json.dumps([asdict(item) for item in reprocess_queue], indent=2, ensure_ascii=False),
            encoding="utf-8",
        )

    def run_one_subgroup(subgroup: Subgroup, reprocess_pass: bool = False) -> bool:
        nonlocal params, manifest
        label = "REPROCESS " if reprocess_pass else ""
        print(f"\n[{time.strftime('%H:%M:%S')}] {label}{subgroup.tag} - {subgroup.title}", flush=True)
        if not subgroup.ranges:
            params.update(subgroup.fixed)
            manifest_item = {
                "tag": subgroup.tag,
                "title": subgroup.title,
                "group": subgroup.group,
                "decision": "FIXED_APPLIED",
                "reason": "subgroup without optimizable parameters; fixed values applied without calling MT5",
                "selected_rank": None,
                "baseline_score": 0.0,
                "optimized_score": 0.0,
                "risk_flags": [],
                "updated_params": list(subgroup.fixed.keys()),
                "reprocess_pending": False,
            }
            manifest = [x for x in manifest if x.get("tag") != subgroup.tag] + [manifest_item]
            params_path.write_text(json.dumps(params, indent=2, ensure_ascii=False), encoding="utf-8")
            manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
            write_report(cfg, manifest, params)
            print("  decision: FIXED_APPLIED | subgroup without optimizable ranges", flush=True)
            return False

        run_label = "REPROCESS" if reprocess_pass else ""
        baseline_rows = collect_symbol_rows(
            cfg,
            subgroup,
            params,
            baseline=True,
            run_label=run_label,
            force_run=reprocess_pass,
        )
        baseline_by_symbol = {}
        for sym, rows in baseline_rows.items():
            if rows:
                baseline_by_symbol[sym] = sorted(rows, key=lambda r: as_float(r.get("score")), reverse=True)[0]
        baseline_metrics = aggregate_metric_rows(cfg, baseline_by_symbol)
        opt_rows = collect_symbol_rows(
            cfg,
            subgroup,
            params,
            baseline=False,
            run_label=run_label,
            force_run=reprocess_pass,
        )
        best, stats, global_top = choose_cluster(cfg, opt_rows, subgroup)
        enrich_stats_with_parameter_context(stats, params, subgroup)
        write_subgroup_parameter_review(cfg, subgroup, baseline_metrics, global_top, stats)
        accepted_auto, auto_reason = accept_cluster(cfg, baseline_metrics, best)
        decision, new_params, reason, selected_rank = choose_set_interactively(
            cfg, subgroup, params, baseline_metrics, global_top, stats, accepted_auto, auto_reason
        )
        if decision in {"ACCEPT", "ACCEPT_AUTO", "USER_SELECTED"}:
            params = new_params
        elif decision == "REPROCESS_BOUNDARIES":
            if reprocess_pass:
                decision = "REPROCESS_SKIPPED"
                reason = "user requested another reprocess during the reprocess pass; skipped to avoid an endless loop"
            else:
                mark_for_reprocess(subgroup)
        manifest_item = {
            "tag": subgroup.tag,
            "title": subgroup.title,
            "group": subgroup.group,
            "decision": decision,
            "reason": reason,
            "selected_rank": selected_rank,
            "baseline_score": as_float(baseline_metrics.get("global_score")),
            "optimized_score": as_float((best or {}).get("global_score")),
            "risk_flags": stats.get("risk_flags", []),
            "updated_params": list(stats.get("parameters", {}).keys()) if decision in {"ACCEPT", "ACCEPT_AUTO", "USER_SELECTED"} else [],
            "reprocess_pending": decision == "REPROCESS_BOUNDARIES",
            "reprocess_pass": reprocess_pass,
        }
        manifest = [x for x in manifest if x.get("tag") != subgroup.tag] + [manifest_item]
        params_path.write_text(json.dumps(params, indent=2, ensure_ascii=False), encoding="utf-8")
        manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
        (p["run_dir"] / f"{subgroup.tag}_stats.json").write_text(json.dumps(stats, indent=2, ensure_ascii=False), encoding="utf-8")
        write_report(cfg, manifest, params)
        print(f"  decision: {decision} | {reason}", flush=True)
        if decision == "REPROCESS_BOUNDARIES":
            print("  queued: this subgroup will be reprocessed at the end of the current stage", flush=True)
        return decision == "REPROCESS_BOUNDARIES"

    try:
        for subgroup in subgroups:
            run_one_subgroup(subgroup)
            if stop_after and subgroup.tag == stop_after:
                break
        if reprocess_queue and not stop_after:
            print(f"\n=== REPROCESS QUEUE: {len(reprocess_queue)} subgroup(s) marked with R ===", flush=True)
            for subgroup in list(reprocess_queue):
                run_one_subgroup(subgroup, reprocess_pass=True)
            queue_path = p["run_dir"] / f"{stage}_reprocess_queue.json"
            if queue_path.exists():
                queue_path.unlink()
        elif reprocess_queue and stop_after:
            print("\nReprocess queue was saved, but --stop-after prevented the end-of-stage reprocess pass.", flush=True)
    except SaveAndExit:
        print("\nSave and exit requested. Saving current state...")
        params_path.write_text(json.dumps(params, indent=2, ensure_ascii=False), encoding="utf-8")
        manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
        write_report(cfg, manifest, params)
        final_set = save_final_set(cfg, params)
        print(f"\nOK: state saved. Final set: {final_set}")
        raise SystemExit(0)
    except KeyboardInterrupt:
        print("\nExecution interrupted by the user. Saving current state...")
        params_path.write_text(json.dumps(params, indent=2, ensure_ascii=False), encoding="utf-8")
        manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
        write_report(cfg, manifest, params)
    final_set = save_final_set(cfg, params)
    print(f"\nOK: stage finished. Final set: {final_set}")


def main() -> None:
    parser = argparse.ArgumentParser(description="MT5 Global Optimizer Wizard V3")
    parser.add_argument("--wizard", action="store_true", help="Create interactive config.json")
    parser.add_argument("--config", type=str, help="Path to config.json")
    parser.add_argument("--stage", choices=["main", "rewind", "wfe", "all"], default="main")
    parser.add_argument("--start-at", type=str, default=None)
    parser.add_argument("--stop-after", type=str, default=None)
    parser.add_argument("--parse-mq5", type=str, help="Only read @opt/@fixed comments from an .mq5 file and print JSON")
    args = parser.parse_args()
    if args.parse_mq5:
        mq5_path = Path(args.parse_mq5)
        base, subgroups = parse_mq5_annotations(mq5_path)
        csv_prefix = detect_hardcoded_csv_prefix(mq5_path) or detect_csv_prefix_from_base(base, mq5_path.stem)
        print(json.dumps({
            "detected_csv_prefix": csv_prefix,
            "expected_csv_file": f"{csv_prefix}_<optimizationTag>_frames.csv",
            "inferred_data_path": infer_data_path_from_mq5(mq5_path),
            "missing_required_inputs": validate_required_optimizer_inputs(base),
            "base": base,
            "subgroups": [asdict(x) for x in subgroups]
        }, indent=2, ensure_ascii=False))
        return
    if args.wizard:
        wizard()
        return
    if not args.config:
        raise SystemExit("Use --wizard or provide --config file.json")
    cfg = load_config(Path(args.config))

    if args.stage == "all":
        print("\n=== STAGE 1/2: MAIN ===")
        run_stage(cfg, stage="main", start_at=args.start_at, stop_after=args.stop_after)

        # The rewind stage should start after the full main stage.
        # Therefore, by default, start_at/stop_after are not reused in rewind.
        # To run a partial rewind, use --stage rewind --start-at R_TAG.
        if args.stop_after:
            print("\nWarning: --stop-after was used in main. Automatic rewind will be skipped to avoid refining an incomplete sequence.")
            return

        print("\n=== STAGE 2/2: REWIND ===")
        run_stage(cfg, stage="rewind")
        if cfg.enable_wfe:
            print("\n=== STAGE 3/3: WFE ===")
            run_wfe(cfg)
    elif args.stage == "wfe":
        run_wfe(cfg)
    else:
        run_stage(cfg, stage=args.stage, start_at=args.start_at, stop_after=args.stop_after)


if __name__ == "__main__":
    main()