"""
tick_reader.py
Read a binary tick export file produced by TickDataExporter.mq5.

Header layout (64 bytes, little-endian):
  uint32   magic           = 0x4B54514D ("MQTK")
  uint16   version         = 1
  uint16   digits          = symbol decimal places
  char[20] symbol          = null-padded ASCII name
  int64    start_time_msc  = ms timestamp of first tick
  int64    end_time_msc    = ms timestamp of last tick
  uint64   tick_count      = number of tick records
  byte[12] reserved        = zeros

Tick record layout (48 bytes, little-endian):
  int64    time_msc   millisecond timestamp
  float64  bid        bid price
  float64  ask        ask price
  float64  last       last trade price (0 for forex)
  uint64   volume     tick volume
  uint32   flags      ENUM_TICK_FLAG bitmask
  uint32   padding    always zero

Requirements: Python 3.7+, numpy
Install:  pip install numpy
Usage:    python tick_reader.py ticks.bin
"""

import struct
import sys
import datetime
import numpy as np

# Header format string for struct.unpack (little-endian)
# < = little-endian
# I  = uint32  magic
# H  = uint16  version
# H  = uint16  digits
# 20s = char[20] symbol
# q  = int64   start_time_msc
# q  = int64   end_time_msc
# Q  = uint64  tick_count
# 12s = char[12] reserved
HEADER_FORMAT = '<IHH20sqqQ12s'
HEADER_SIZE   = struct.calcsize(HEADER_FORMAT)  # must equal 64

EXPECTED_MAGIC   = 0x4B54514D  # "MQTK" in little-endian
EXPECTED_VERSION = 1

# NumPy dtype matching the 48-byte CTickRecord binary layout
TICK_DTYPE = np.dtype([
    ('time_msc', '<i8'),   # 8 bytes: millisecond timestamp (signed)
    ('bid',      '<f8'),   # 8 bytes: bid price
    ('ask',      '<f8'),   # 8 bytes: ask price
    ('last',     '<f8'),   # 8 bytes: last trade price
    ('volume',   '<u8'),   # 8 bytes: tick volume (unsigned)
    ('flags',    '<u4'),   # 4 bytes: ENUM_TICK_FLAG bitmask
    ('padding',  '<u4'),   # 4 bytes: always zero
])  # total: 48 bytes per record


def ms_to_datetime(ms):
    """Convert a millisecond Unix timestamp to a UTC datetime string."""
    ts_sec = ms / 1000.0
    dt = datetime.datetime.utcfromtimestamp(ts_sec)
    return dt.strftime('%Y-%m-%d %H:%M:%S.') + f'{ms % 1000:03d}'


def read_tick_file(path):
    """
    Read a tick export binary file and return (symbol, digits, ticks).

    Parameters
    ----------
    path : str
        Path to the .bin file produced by TickDataExporter.mq5.

    Returns
    -------
    symbol : str
        Instrument name stored in the header.
    digits : int
        Decimal places for the instrument.
    ticks : numpy.ndarray
        Structured array with fields: time_msc, bid, ask, last, volume, flags, padding.
    """
    if HEADER_SIZE != 64:
        raise RuntimeError(f"Header format calculates to {HEADER_SIZE} bytes, expected 64")

    with open(path, 'rb') as f:
        raw_header = f.read(HEADER_SIZE)
        if len(raw_header) < HEADER_SIZE:
            raise ValueError(
                f"File too short: expected at least {HEADER_SIZE} header bytes, "
                f"got {len(raw_header)}"
            )

        (magic, version, digits, symbol_bytes,
         start_msc, end_msc, tick_count, reserved) = struct.unpack(
            HEADER_FORMAT, raw_header
        )

        if magic != EXPECTED_MAGIC:
            raise ValueError(
                f"Invalid magic number 0x{magic:08X}, expected 0x{EXPECTED_MAGIC:08X}. "
                "This is not a valid tick export file."
            )

        if version != EXPECTED_VERSION:
            raise ValueError(
                f"Unsupported format version {version}, expected {EXPECTED_VERSION}."
            )

        symbol = symbol_bytes.rstrip(b'\x00').decode('ascii')

        print(f"Symbol     : {symbol}")
        print(f"Digits     : {digits}")
        print(f"Version    : {version}")
        print(f"Tick count : {tick_count}")
        print(f"Start      : {ms_to_datetime(start_msc)} UTC")
        print(f"End        : {ms_to_datetime(end_msc)} UTC")
        print(f"File size  : {HEADER_SIZE + tick_count * 48} bytes (expected)")

        raw_records = f.read()

    # Load all records in a single call; no Python-level loop required
    ticks = np.frombuffer(raw_records, dtype=TICK_DTYPE)

    if len(ticks) != tick_count:
        print(
            f"Warning: header declares {tick_count} ticks but "
            f"file contains {len(ticks)} records."
        )

    return symbol, digits, ticks


def print_sample(ticks, digits, n=5):
    """Print the first n tick records in a readable format."""
    print(f"\nFirst {min(n, len(ticks))} tick(s):")
    for t in ticks[:n]:
        fmt = f"  {ms_to_datetime(t['time_msc'])} UTC" \
              f"  bid={t['bid']:.{digits}f}" \
              f"  ask={t['ask']:.{digits}f}" \
              f"  flags={t['flags']:02d}"
        print(fmt)


def print_spread_stats(ticks, digits):
    """Compute and print spread statistics in points."""
    pip = 10 ** -digits
    # Only use ticks where both bid and ask are non-zero
    valid = ticks[(ticks['bid'] > 0) & (ticks['ask'] > 0)]
    if len(valid) == 0:
        print("\nNo valid bid/ask ticks for spread analysis.")
        return

    spread_points = (valid['ask'] - valid['bid']) / pip
    print(f"\nSpread statistics ({len(valid)} ticks with bid and ask, in points):")
    print(f"  min    = {spread_points.min():.1f}")
    print(f"  p25    = {np.percentile(spread_points, 25):.1f}")
    print(f"  median = {np.median(spread_points):.1f}")
    print(f"  p75    = {np.percentile(spread_points, 75):.1f}")
    print(f"  max    = {spread_points.max():.1f}")
    print(f"  mean   = {spread_points.mean():.2f}")


if __name__ == '__main__':
    file_path = sys.argv[1] if len(sys.argv) > 1 else 'ticks.bin'

    print(f"Reading: {file_path}\n")
    try:
        symbol, digits, ticks = read_tick_file(file_path)
        print_sample(ticks, digits)
        print_spread_stats(ticks, digits)
    except (ValueError, RuntimeError, OSError) as exc:
        print(f"Error: {exc}")
        sys.exit(1)