Derivatives market quotes in MT5
Please help to find a bug in the code, or confirm that CopyTicks() is not working correctly.
Problem: Tick skipping, when copying with CopyTicks()
sequence of operations:
Code
//+------------------------------------------------------------------+ //| ABL_Collector.mq5 | //| Copyright 2021, prostotrader | //| https://www.mql5.com | //+------------------------------------------------------------------+ #property copyright "Copyright 2021, prostotrader" #property link "https://www.mql5.com" #property version "1.00" //--- input string StTime = "07:00:00"; //Начало сбора тиков input string EndTime = "23:50:00"; //Конец сбора тиков //--- struct MARKET_DATA { int cnt; datetime time[]; ulong time_msc[]; double ask[]; double bid[]; double last[]; long volume[]; string flags[]; ulong mem_time; int skip_cnt; MqlTick ticks[]; } m_data; int f_handle; datetime start_time, end_time; //+------------------------------------------------------------------+ //| Expert initialization function | //+------------------------------------------------------------------+ int OnInit() { m_data.cnt = 0; m_data.mem_time = 0; ArrayResize(m_data.time, 5000000, 5000000); ArrayResize(m_data.time_msc, 5000000, 5000000); ArrayResize(m_data.ask, 5000000, 5000000); ArrayResize(m_data.bid, 5000000, 5000000); ArrayResize(m_data.last, 5000000, 5000000); ArrayResize(m_data.volume, 5000000, 5000000); ArrayResize(m_data.flags, 5000000, 5000000); f_handle = FileOpen("ABL_Colletor.csv", FILE_WRITE|FILE_CSV); if(f_handle == INVALID_HANDLE) { Alert("Не создан *.CSV файл!"); return(INIT_FAILED); } else { FileWrite(f_handle, "Symbol: ", Symbol()); FileWrite(f_handle, "Tick num", "Date", "Tick time", "ASK", "BID", "LAST", "VOLUME", "Tick Flags"); } start_time = StringToTime(StTime); end_time = StringToTime(EndTime); return(INIT_SUCCEEDED); } //+------------------------------------------------------------------+ //| Expert deinitialization function | //+------------------------------------------------------------------+ void OnDeinit(const int reason) { if(f_handle != INVALID_HANDLE) { ulong a_time = m_data.time_msc[0]; for(int i = 0; i<m_data.cnt; i++) { if(a_time != m_data.time_msc[i]) { FileWrite(f_handle, ""); a_time = m_data.time_msc[i]; } FileWrite(f_handle, IntegerToString(i + 1), TimeToString(m_data.time[i], TIME_DATE), TimeToString(m_data.time[i], TIME_SECONDS) + "." + StringFormat("%03i", m_data.time_msc[i]%1000), DoubleToString(m_data.ask[i], Digits()), DoubleToString(m_data.bid[i], Digits()), DoubleToString(m_data.last[i], Digits()), string(m_data.volume[i]), m_data.flags[i]); } } ArrayResize(m_data.time, 0, -1); ArrayResize(m_data.time_msc, 0, -1); ArrayResize(m_data.ask, 0, -1); ArrayResize(m_data.bid, 0, -1); ArrayResize(m_data.last, 0, -1); ArrayResize(m_data.volume, 0, -1); ArrayResize(m_data.flags, 0, -1); } //+------------------------------------------------------------------+ //| Expert fill data function | //+------------------------------------------------------------------+ void FillData(const int skip, const int t_cnt) { string c_flags; m_data.skip_cnt = 0; //Обнуляем счетчик тиков с одним временем (последним) for(int i = skip; i < t_cnt; i++) { c_flags = ""; m_data.ask[m_data.cnt] = m_data.ticks[i].ask; //Сохраняем значения m_data.bid[m_data.cnt] = m_data.ticks[i].bid; m_data.last[m_data.cnt] = m_data.ticks[i].last; m_data.volume[m_data.cnt] = long(m_data.ticks[i].volume_real); m_data.time[m_data.cnt] = m_data.ticks[i].time; m_data.time_msc[m_data.cnt] = m_data.ticks[i].time_msc; //Собираем все флаги if((m_data.ticks[i].flags&TICK_FLAG_ASK) == TICK_FLAG_ASK) c_flags += " TICK_FLAG_ASK,"; if((m_data.ticks[i].flags&TICK_FLAG_BID) == TICK_FLAG_BID) c_flags += " TICK_FLAG_BID,"; if((m_data.ticks[i].flags&TICK_FLAG_LAST) == TICK_FLAG_LAST) c_flags += " TICK_FLAG_LAST,"; if((m_data.ticks[i].flags&TICK_FLAG_BUY) == TICK_FLAG_BUY) c_flags += " TICK_FLAG_BUY,"; if((m_data.ticks[i].flags&TICK_FLAG_SELL) == TICK_FLAG_SELL) c_flags += " TICK_FLAG_SELL,"; if((m_data.ticks[i].flags&TICK_FLAG_VOLUME) == TICK_FLAG_VOLUME) c_flags += " TICK_FLAG_VOLUME,"; int f_len = StringLen(c_flags); //Кастрируем последнюю запятую в строке if(f_len > 1) { StringSetCharacter(c_flags, f_len - 1, ushort(" ")); StringTrimRight(c_flags); } m_data.flags[m_data.cnt] = c_flags + " (" + string(m_data.ticks[i].flags) + ")"; //Записываем флаги m_data.cnt++; //Увеличиваем счетчик всех записей if(m_data.mem_time == ulong(m_data.ticks[i].time_msc)) m_data.skip_cnt++; //Увеличиваем счетчик тиков с последним (одинаковым) временем } } //+------------------------------------------------------------------+ //| Expert tick function | //+------------------------------------------------------------------+ void OnTick() { datetime cur_time = TimeTradeServer(); //Берем текущее время сервера if((cur_time >=start_time)&&(cur_time<end_time)) //Проверка времени торговли { int result = 0; if(m_data.mem_time == 0) //Проверка на 1-ю запись { result = CopyTicks(Symbol(), m_data.ticks, COPY_TICKS_ALL, 0, 20); //Копируем последние 20 тиков if(result >= 1) { m_data.mem_time = ulong(m_data.ticks[result-1].time_msc); //Запоминаем время последнего тика FillData(0, result); //Сохраняем данные } } else //Последующие записи { result = CopyTicks(Symbol(), m_data.ticks, COPY_TICKS_ALL, m_data.mem_time, 1000); //Копируем тики с запомненного времени if(result >= 1) //плюс последующие тики (если есть) { if(m_data.mem_time < ulong(m_data.ticks[result-1].time_msc)) //Проверяем, изменилось ли время последнего тика { m_data.mem_time = ulong(m_data.ticks[result-1].time_msc); //Запоминаем время последнего тика FillData(m_data.skip_cnt, result); //Сохраняем данные, минус уже сохраненные с } //предыдущим последнем временем (m_data.skip_cnt) } } } } //+------------------------------------------------------------------+
After the Expert Advisor works, we get the file ABL_Colletor.csv (in the zip file)
I save ticks from the terminal
I get GAZR-12.21.csv ( in zip)
Compare files in GAZR-12.21_mix.csv.xlsx ( in zip file)
Yes indeed, with the COPY_TICKS_TRADE flag, no skips, but I need both trades and bids asks,
again a crutch?
result = CopyTicks(Symbol(), m_data.ticks, COPY_TICKS_TRADE, m_data.mem_time, 1000); //плюс result = CopyTicks(Symbol(), m_data.ticks, COPY_TICKS_INFO, m_data.mem_time, 1000);
But 2 big problems arise immediately.
1. How to "synchronize" the time
2. How to check ASK and BID for correctness
Yes indeed, with the COPY_TICKS_TRADE flag, no skips, but I need both trades and asks with bids,
Forum on trading, automated trading systems and testing trading strategies
New MetaTrader 5 build 3081: Improvements in MQL5 Services and Updated Design
fxsaber, 2021.10.19 16:30
Two different threads (INFO and LAST) are artificially combined into one ALL-flow.
At the same millisecond the exchange gave in the INFO-flow a bid-price of 36800. And in the LAST-flow there were a lot of deals. It is clear that if time was measured in nanoseconds, then INFO-price would be later than deals.
MT5 spends time merging and synchronizing streams. Because of this, there can be very decent lags in real time when building the current tick history. Milliseconds may be lost.
Forum on trading, automated trading systems and strategy testing
New MetaTrader 5 build 3081: Improvements in MQL5 Services and Updated Design
fxsaber, 2021.10.19 16:38
I think if you monitor the Book-stream in the terminal and compare it with the fresh CopyTicks, there will be a lot of discrepancies. That said, in hindsight it will be pretty decent.
The Exchange trades go in a separate stream, that's right,
as do ask and bid.
COPY_TICKS_ALL is sampling from these two streams, it's just not done correctly.
And that sample cannot take milliseconds, because both threads have "straight through" time, not in separate ones!
Made a crutch, seems to work.
//+------------------------------------------------------------------+ //| ABL_Collector.mq5 | //| Copyright 2021, prostotrader | //| https://www.mql5.com | //+------------------------------------------------------------------+ #property copyright "Copyright 2021, prostotrader" #property link "https://www.mql5.com" #property version "1.00" //--- input string StTime = "07:00:00"; //Начало сбора тиков input string EndTime = "23:50:00"; //Конец сбора тиков //--- struct MARKET_DATA { datetime time; ulong time_msc; double ask; double bid; double last; long volume; string flags; }; struct A_DATA { int cnt; ulong m_time_info; ulong m_time_trade; int s_cnt_info; int s_cnt_trade; MqlTick ticks_trade[]; MqlTick ticks_info[]; MARKET_DATA m_data[]; }a_data; int f_handle; datetime start_time, end_time; //+------------------------------------------------------------------+ //| Expert Array fast sort function | //+------------------------------------------------------------------+ void ArrayFastSort(MARKET_DATA &array[], const int size) { ulong msc_val; MARKET_DATA temp; MARKET_DATA tmp_arr[]; ArrayResize(tmp_arr, size, size); for(int i = 0; i < size; i++) { tmp_arr[i] = array[i]; } int n[] = {9,5,3,2,1}; int i, j, z, y; for(z = 0;z < 5;z++) { y = n[z]; for(i = y;i < size;i++) { msc_val = tmp_arr[i].time_msc; temp = tmp_arr[i]; for(j = i - y; j >= 0 && msc_val < tmp_arr[j].time_msc; j -= y) { tmp_arr[j + y] = tmp_arr[j]; } tmp_arr[j + y] = temp; } } for(i = 0; i < size; i++) { msc_val = tmp_arr[i].time_msc; for(j = 0; j < size; j++) { if(array[j].time_msc == 0) continue; if(msc_val == array[j].time_msc) { tmp_arr[i] = array[j]; array[j].time_msc = 0; break; } } } for(i = 0; i < size; i++) { array[i] = tmp_arr[i]; } } //+------------------------------------------------------------------+ //| Expert initialization function | //+------------------------------------------------------------------+ int OnInit() { a_data.cnt = 0; a_data.m_time_info = 0; a_data.m_time_trade = 0; ArrayResize(a_data.m_data, 5000000, 5000000); f_handle = FileOpen("ABL_Colletor.csv", FILE_WRITE|FILE_CSV); if(f_handle == INVALID_HANDLE) { Alert("Не создан *.CSV файл!"); return(INIT_FAILED); } else { FileWrite(f_handle, "Symbol: ", Symbol()); FileWrite(f_handle, "Tick num", "Date", "Tick time", "ASK", "BID", "LAST", "VOLUME", "Tick Flags"); } start_time = StringToTime(StTime); end_time = StringToTime(EndTime); int result = CopyTicks(Symbol(), a_data.ticks_info, COPY_TICKS_ALL, 0, 1); if(result >= 1) { a_data.m_time_info = a_data.ticks_info[0].time_msc; a_data.m_time_trade = a_data.ticks_info[0].time_msc; } return(INIT_SUCCEEDED); } //+------------------------------------------------------------------+ //| Expert deinitialization function | //+------------------------------------------------------------------+ void OnDeinit(const int reason) { if(f_handle != INVALID_HANDLE) { ArrayFastSort(a_data.m_data, a_data.cnt); ulong a_time = a_data.m_data[0].time_msc; for(int i = 0; i < a_data.cnt; i++) { if(a_time != a_data.m_data[i].time_msc) { FileWrite(f_handle, ""); a_time = a_data.m_data[i].time_msc; } FileWrite(f_handle, IntegerToString(i + 1), TimeToString(a_data.m_data[i].time, TIME_DATE), TimeToString(a_data.m_data[i].time, TIME_SECONDS) + "." + StringFormat("%03i", a_data.m_data[i].time_msc%1000), DoubleToString(a_data.m_data[i].ask, Digits()), DoubleToString(a_data.m_data[i].bid, Digits()), DoubleToString(a_data.m_data[i].last, Digits()), string(a_data.m_data[i].volume), a_data.m_data[i].flags); } } ArrayResize(a_data.m_data, 0, -1); Print("Collect ticks DONE!"); } //+------------------------------------------------------------------+ //| Expert fill data function | //+------------------------------------------------------------------+ void FillData(const int skip, const int t_cnt, ulong &mem_time, int &skip_cnt, MqlTick &ticks[]) { string c_flags; skip_cnt = 0; for(int i = skip; i < t_cnt; i++) { c_flags = ""; a_data.m_data[a_data.cnt].ask = ticks[i].ask; a_data.m_data[a_data.cnt].bid = ticks[i].bid; a_data.m_data[a_data.cnt].last = ticks[i].last; a_data.m_data[a_data.cnt].volume = long(ticks[i].volume_real); a_data.m_data[a_data.cnt].time = ticks[i].time; a_data.m_data[a_data.cnt].time_msc = ticks[i].time_msc; //--- if((ticks[i].flags&TICK_FLAG_ASK) == TICK_FLAG_ASK) c_flags += " TICK_FLAG_ASK,"; if((ticks[i].flags&TICK_FLAG_BID) == TICK_FLAG_BID) c_flags += " TICK_FLAG_BID,"; if((ticks[i].flags&TICK_FLAG_LAST) == TICK_FLAG_LAST) c_flags += " TICK_FLAG_LAST,"; if((ticks[i].flags&TICK_FLAG_BUY) == TICK_FLAG_BUY) c_flags += " TICK_FLAG_BUY,"; if((ticks[i].flags&TICK_FLAG_SELL) == TICK_FLAG_SELL) c_flags += " TICK_FLAG_SELL,"; if((ticks[i].flags&TICK_FLAG_VOLUME) == TICK_FLAG_VOLUME) c_flags += " TICK_FLAG_VOLUME,"; int f_len = StringLen(c_flags); if(f_len > 1) { StringSetCharacter(c_flags, f_len - 1, ushort(" ")); StringTrimRight(c_flags); } a_data.m_data[a_data.cnt].flags = c_flags + " (" + string(ticks[i].flags) + ")"; a_data.cnt++; if(mem_time == ulong(ticks[i].time_msc)) skip_cnt++; } } //+------------------------------------------------------------------+ //| Expert tick function | //+------------------------------------------------------------------+ void OnTick() { datetime cur_time = TimeTradeServer(); if((cur_time >=start_time)&&(cur_time<end_time)) { int result = 0; result = CopyTicks(Symbol(), a_data.ticks_info, COPY_TICKS_INFO, a_data.m_time_info, 1000); if(result >= 1) { if(a_data.m_time_info < ulong(a_data.ticks_info[result -1].time_msc)) { a_data.m_time_info = a_data.ticks_info[result -1].time_msc; FillData(a_data.s_cnt_info, result, a_data.m_time_info, a_data.s_cnt_info, a_data.ticks_info); } } result = CopyTicks(Symbol(), a_data.ticks_trade, COPY_TICKS_TRADE, a_data.m_time_trade, 1000); if(result >= 1) { if(a_data.m_time_trade < ulong(a_data.ticks_trade[result -1].time_msc)) { a_data.m_time_trade = a_data.ticks_trade[result -1].time_msc; FillData(a_data.s_cnt_trade, result, a_data.m_time_trade, a_data.s_cnt_trade, a_data.ticks_trade); } } } } //+------------------------------------------------------------------+
You can use it if someone needs to collect quotes.
Too bad that Asks and Bids can't be checked against the Exchange
To summarise investigation of CopyTicks() CopyTicksRange()
Bild 3101, real, Open
1. If the COPY_TICKS_ALL flag is used - there are skips of transactions,
whenusingCOPY_TICKS_TRADE flag - are displayed correctly.
2. Something unimaginable happens with quotes (ASK/BID)
The trades are executed at prices that do not exist
Transactions are executed at opposite price (instead of ASK they use BID)
Skipping of quotes
There is a big question:
If there is a huge price discrepancy, what is in the terminal?
What prices are we trading at?
Source files in ZIP archive
2. There is something unimaginable going on with quotes (ASK/BID)
The deals are made at the wrong prices
Forum on trading, automated trading systems and strategy testing
New MetaTrader 5 build 3081: Improvements in MQL5 Services and Updated Design
fxsaber, 2021.10.19 16:57
Well then poke MQ with proof that they are missing the data being broadcast by the exchange!
Do it in a thread that MQs don't ignore, and in the manner of the first post in the thread.
Forum on trading, automated trading systems and testing trading strategies
Dmitriy Skub, 2015.03.24 09:17
Let's start with a simple one - volume. Below is a picture of the detected glitch. Periodically there are "doubles" in volumes.
- Free trading apps
- Over 8,000 signals for copying
- Economic news for exploring financial markets
You agree to website policy and terms of use
Dear moderators!
Please move the posts from the topic "Clearing on the merits????*
not related to clearing, here.