"""Train the example classifier on the terminal's own features and export everything the
MQL5 side checks against. For your own model, use make_check_files.py instead.

Usage:  python export_model.py <folder with features.csv>

Needs numpy, scikit-learn, skl2onnx, onnx and onnxruntime. Writes into the same folder:
  model.onnx            StandardScaler + RandomForest, zipmap off (label + probability tensors)
  model_contract.csv    the inputs and outputs, feature names, class labels and SHA-256 (make_check_files.py)
  golden.csv            out-of-sample rows with the onnxruntime and scikit-learn answers
  forest.json           scaler and trees as plain numbers, for the standard-library cross-check
  model_zipmap.onnx     the same pipeline exported with the skl2onnx default (zipmap on)
  model_5features.onnx  a pipeline trained on five features
  model_double.onnx     the same pipeline with a double input
  model_label_only.onnx model.onnx with the probability output removed
  model_retrained.onnx  the same pipeline trained again with another seed: same signature,
                        other numbers

Periods: the model trains on rows before TRAIN_END, minus the last HORIZON rows, whose labels
look into the golden period. The golden rows are TRAIN_END <= time < GOLDEN_END. Rows after
GOLDEN_END are not used here, they are left for the tester run of the EA.
"""
import csv, json, os, sys
import numpy as np
import onnx
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import DoubleTensorType, FloatTensorType

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import make_check_files as mcf

TRAIN_END = "2026.01.01"                   # first golden row
GOLDEN_END = "2026.05.01"                  # first row after the golden period
HORIZON = 6                                # bars ahead of the label, as in ExportFeatures.mq5
NAMES = ["rsi14", "atr14_norm", "ret10", "bbw20"]


def load(folder):
    with open(os.path.join(folder, "features.csv"), newline="") as f:
        rows = list(csv.DictReader(f))
    t = [r["time"] for r in rows]
    x = np.array([[float(r[n]) for n in NAMES] for r in rows], dtype=np.float32)
    y = np.array([int(r["label"]) for r in rows], dtype=np.int64)
    return t, x, y


def pipeline(seed=7):
    return Pipeline([("scaler", StandardScaler()),
                     ("forest", RandomForestClassifier(n_estimators=50, max_depth=6, min_samples_leaf=20,
                                                       random_state=seed, n_jobs=1))])


def export(pipe, nfeat, path, zipmap=False, double=False):
    kind = DoubleTensorType if double else FloatTensorType
    opts = None if zipmap else {id(pipe.steps[-1][1]): {"zipmap": False}}
    model = convert_sklearn(pipe, initial_types=[("float_input", kind([None, nfeat]))],
                            options=opts, target_opset={"": 17, "ai.onnx.ml": 3})
    onnx.save(model, path)
    return model


def write_forest(pipe, path):
    sc, rf = pipe.named_steps["scaler"], pipe.named_steps["forest"]
    trees = []
    for est in rf.estimators_:
        tr = est.tree_
        value = tr.value[:, 0, :]
        value = value / value.sum(axis=1, keepdims=True)
        trees.append({"feature": tr.feature.tolist(), "threshold": tr.threshold.tolist(),
                      "left": tr.children_left.tolist(), "right": tr.children_right.tolist(),
                      "value": value.tolist()})
    with open(path, "w") as f:
        json.dump({"features": NAMES, "mean": sc.mean_.tolist(), "scale": sc.scale_.tolist(),
                   "classes": rf.classes_.tolist(), "trees": trees}, f)


def check_thresholds(model_path, forest_path):
    """how skl2onnx stored the split thresholds: nearest float32, or the largest float32 not above"""
    model = onnx.load(model_path)
    with open(forest_path) as f:
        trees = json.load(f)["trees"]
    node = next(n for n in model.graph.node if n.op_type == "TreeEnsembleClassifier")
    a = {x.name: x for x in node.attribute}
    near = down = total = 0
    for t, k, v, mode in zip(a["nodes_treeids"].ints, a["nodes_nodeids"].ints, a["nodes_values"].floats, a["nodes_modes"].strings):
        if mode != b"BRANCH_LEQ":
            continue
        thr = trees[t]["threshold"][k]
        n = np.float32(thr)
        d = n if float(n) <= thr else np.nextafter(n, np.float32(-np.inf))
        total, near, down = total + 1, near + (np.float32(v) == n), down + (np.float32(v) == d)
    print("graph name %s; split nodes %d, threshold equal to the nearest float32 %d, to the largest float32 not above it %d"
          % (model.graph.name, total, near, down))


def main(folder):
    t, x, y = load(folder)
    first_gold = next(i for i, s in enumerate(t) if s >= TRAIN_END)
    idx = np.arange(len(t))
    #--- a row trains only if the bar its label looks at is still before the golden period
    train = idx < first_gold - HORIZON
    gold = np.array([TRAIN_END <= s < GOLDEN_END for s in t])
    pipe = pipeline().fit(x[train], y[train])
    print("rows %d, train %d (the last %d rows before %s left out), golden %d (%s to %s), after the golden period %d"
          % (len(t), train.sum(), HORIZON, TRAIN_END, gold.sum(), t[first_gold], t[np.where(gold)[0][-1]],
             sum(s >= GOLDEN_END for s in t)))
    print("golden-period accuracy %.4f, share of the largest class %.4f"
          % ((pipe.predict(x[gold]) == y[gold]).mean(), np.bincount(y[gold]).max() / gold.sum()))

    path = os.path.join(folder, "model.onnx")
    export(pipe, 4, path)
    mcf.write_contract(path, os.path.join(folder, "model_contract.csv"), NAMES)
    write_forest(pipe, os.path.join(folder, "forest.json"))
    times = [t[k] for k in np.where(gold)[0]]
    s_prob = pipe.predict_proba(x[gold])
    s_label = pipe.classes_[s_prob.argmax(axis=1)]
    o_label, o_prob = mcf.write_golden(path, os.path.join(folder, "golden.csv"), times, x[gold], NAMES,
                                       skl=(s_label, s_prob))
    print("onnxruntime vs scikit-learn: %d of %d labels differ, largest probability difference %.3g"
          % ((o_label != s_label).sum(), gold.sum(), np.abs(o_prob - s_prob).max()))
    check_thresholds(path, os.path.join(folder, "forest.json"))

    #--- variants for the unit tests
    export(pipe, 4, os.path.join(folder, "model_zipmap.onnx"), zipmap=True)
    export(pipe, 4, os.path.join(folder, "model_double.onnx"), double=True)
    x5 = np.hstack([x, x[:, :1] * 0.5]).astype(np.float32)
    export(pipeline().fit(x5[train], y[train]), 5, os.path.join(folder, "model_5features.onnx"))
    export(pipeline(8).fit(x[train], y[train]), 4, os.path.join(folder, "model_retrained.onnx"))
    lo = onnx.load(path)
    lo.graph.output.pop()                  # drop the probability output, keep the label
    onnx.save(lo, os.path.join(folder, "model_label_only.onnx"))
    for name in ("model.onnx", "model_zipmap.onnx", "model_double.onnx", "model_5features.onnx", "model_label_only.onnx",
                 "model_retrained.onnx"):
        sess = mcf.ort.InferenceSession(os.path.join(folder, name), providers=["CPUExecutionProvider"])
        sig = [mcf.value_row("in", i, v)[3:] for i, v in enumerate(sess.get_inputs())]
        sig += [mcf.value_row("out", i, v)[3:] for i, v in enumerate(sess.get_outputs())]
        print("%-22s %s  sha256 %s" % (name, " | ".join(" ".join(s).strip() for s in sig),
                                       mcf.sha256(os.path.join(folder, name))[:16]))


if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    main(sys.argv[1])
