What is the fastest way to get the last quote for about 10,000 symbols and keep them updated?

New comment
 

Hi,

What is the fastest way to get the last quote for about 10,000 symbols and keep them updated?

I want to make kind of a server to communicate to delphi.

I tried

   MqlTick ticks_array[];

   CopyTicks(symbol_name, ticks_array, COPY_TICKS_ALL, 0, 1);

To get the quotes, but its slow and does not update them....

Any advices?

Thanks in advance.

 
Your topic has been moved to the section: Expert Advisors and Automated Trading
Please consider which section is most appropriate — https://www.mql5.com/en/forum/172166/page6#comment_49114893
 
jrussi: What is the fastest way to get the last quote for about 10,000 symbols and keep them updated?

I don't know of any broker offering that many symbols. That is an absurd amount! Even 1000 symbols would be too many.

As for processing tick data, read the following ...

Forum on trading, automated trading systems and testing trading strategies

30s Timeframe MA Cross EA - how do you pull tick data when you cant choose seconds in the enum_timeframes?

Fernando Carreiro, 2023.09.04 15:57

It is somewhat complex and can easily be a couple hundred lines of code.

Here is a skeleton code example for the tick processing and timing of a seconds based time-frame. It does however, not build the OHLC bar data. In simply uses the open mid price "(Ask + Bid)/2".

You will have to extend the functionality if you want to build the OHLC data as well.

//---------------------------------------------------------------------------------------------------------------------
#property   copyright      "Copyright \x00A9 2022, Fernando M. I. Carreiro, All rights reserved"
#property   link           "https://www.mql5.com/en/users/FMIC"
//---------------------------------------------------------------------------------------------------------------------

//--- Parameter settings

   input group "Parameters: Tick data processing"
      input uint  i_nTimeframeSeconds  = 15,    // Time-frame in seconds
                  i_nTickHistoryDays   = 5;     // Days to backtrack tick history
                                                // enough to cover long weekends
//--- Macro definitions

   // Define macro for invalid parameter values
      #define MCheckParameter( _condition, _text ) if( _condition ) \
         { Print( "Error: Invalid parameter - ", _text ); return INIT_PARAMETERS_INCORRECT; }

   // Define macros for time masking
      #define MDayMinutes ( 24 * 60      )
      #define MDaySeconds ( 24 * 60 * 60 )
      #define MTimeOnly( _vardatetime )       ( int(      ( _vardatetime ) % MDaySeconds ) )
      #define MDateOnly( _vardatetime )       ( int(      ( _vardatetime ) / MDaySeconds ) )
      #define MDateTime( _vardate, _vartime ) ( datetime( ( _vardate )     * MDaySeconds + ( _vartime ) ) )

//--- Structures and classes

   // Tick data processor structure
      struct STickProcessor {
         // Declare member variables
            long     m_nTicksTimeMsc;     // Next search time (milliseconds)
            int      m_nTicksTimeCount,   // Next search duplicate time counter
                     m_iTicksIndex,       // Index of current tick data
                     m_iTicksLast,        // Index of last valid tick data
                     m_nTicksCount;       // Count of available ticks in the cached data
            MqlTick  m_oTicks[];          // Tick data cache
            string   m_sSymbol;           // Symbol name for tick data
            bool     m_bRealtime;         // Real-time processing flag

         // Default deconstructor
            ~STickProcessor( void ) {
               ArrayFree( m_oTicks );
            };

         // Default constructor
            STickProcessor( void ) :
               m_nTicksTimeMsc(   WRONG_VALUE ),
               m_nTicksTimeCount( WRONG_VALUE ),
               m_iTicksIndex(     WRONG_VALUE ),
               m_iTicksLast(      WRONG_VALUE ),
               m_nTicksCount(     WRONG_VALUE ),
               m_bRealtime(       false       ),
               m_sSymbol(         ""          ) {};

         // Initialiser
            bool Init( string sSymbol, long nTicksTimeMsc ) {
               if( ( nTicksTimeMsc > 0 ) && ( sSymbol.Length() > 0 ) ) {
                  m_nTicksTimeMsc   = nTicksTimeMsc;
                  m_nTicksTimeCount = 0;
                  m_iTicksIndex     =
                  m_iTicksLast      =
                  m_nTicksCount     = WRONG_VALUE;
                  m_bRealtime       = false;
                  m_sSymbol         = sSymbol;
                  return true;
               };
               return false; // Failed to initialise
            };

         // Verify validaty of current tick data state
            bool IsValid( void ) {
               return ( m_nTicksCount >  0            ) &&
                      ( m_iTicksIndex >= 0            ) &&
                      ( m_iTicksIndex <= m_iTicksLast );
            };

         // Update Real-time status
            void RealTimeUpdate( void ) {
               if( m_iTicksIndex == m_iTicksLast )
                  m_bRealtime = m_oTicks[ m_iTicksIndex ].time >= TimeCurrent();
            };

         // Advance to next tick
            bool Next( void ) {
               if( ( m_nTicksCount > 0 ) && ( m_iTicksIndex < m_iTicksLast ) ) {
                  m_iTicksIndex++;                       // Next tick
                  if( !m_bRealtime ) RealTimeUpdate(); // Update Real-time status
                  return true;
               };
               if( m_nTicksTimeMsc > 0 ) {
                  // Acquire new data for the cache
                     ResetLastError();
                     m_nTicksCount = _StopFlag ? 0
                                   : CopyTicks( m_sSymbol, m_oTicks, COPY_TICKS_ALL, m_nTicksTimeMsc );
                     if( m_nTicksCount > m_nTicksTimeCount ) {
                        // Count final ticks as there may be more in the future with the same timestamp
                           int iTicksFinal = m_nTicksCount - 1;
                           m_iTicksLast    = iTicksFinal;
                           m_nTicksTimeMsc = m_oTicks[ m_iTicksLast ].time_msc;
                           while( ( iTicksFinal >= 0 ) && ( m_oTicks[ iTicksFinal ].time_msc >= m_nTicksTimeMsc ) )
                              iTicksFinal--;
                           m_iTicksIndex     = m_nTicksTimeCount; // Dismiss initial duplicate ticks
                           m_nTicksTimeCount = m_iTicksLast - iTicksFinal;
                           if( !m_bRealtime ) RealTimeUpdate(); // Update Real-time status
                           return m_iTicksIndex <= m_iTicksLast;  // Succeeded in acquiring tick data
                     };
                  // Report any errors while acquiring the tick data
                     if( ( m_nTicksCount == WRONG_VALUE ) || ( _LastError != ERR_SUCCESS ) )
                        PrintFormat( "Error %d: Unable to acquire tick data for symbol %s",
                           _LastError, m_sSymbol );
                  // Reset everything
                     m_iTicksIndex =
                     m_iTicksLast  =
                     m_nTicksCount = WRONG_VALUE;
               };
               return false; // Failed to acquire tick data
            };

         // Get current tick time - Attention! For the sake of efficiency, validity is not verified
            datetime Time( void ) { return m_oTicks[ m_iTicksIndex ].time; };

         // Get current mid price - Attention! For the sake of efficiency, validity is not verified
            double MidPrice( void ) {
               return 0.5 * ( m_oTicks[ m_iTicksIndex ].ask + m_oTicks[ m_iTicksIndex ].bid );
            };
      };

//--- Global variable declarations

      STickProcessor g_oTickProcessor;                // Tick data processor structure object

//--- Event handling functions

   // Initialisation event handler
      int OnInit( void ) {
         // Validate inputs and calculate parameter variables
            MCheckParameter( i_nTickHistoryDays  < 1, "days to backtrack tick history" );
            MCheckParameter( i_nTimeframeSeconds < 1, "time-frame in seconds"          );

         return INIT_SUCCEEDED;  // Successful initialisation of indicator
      };

   // Tick event handler
      void OnTick( void ) {
         // Detect first tick
            static bool bFirstTick = true;
            if( bFirstTick ) {
               // Initialise tick processing object with symbol and backtracking time
                  g_oTickProcessor.Init( _Symbol, ( TimeCurrent() - i_nTickHistoryDays * MDaySeconds ) * 1000 );
                  bFirstTick = false; // First tick condition complete
            };

         // Process tick data
            while( g_oTickProcessor.Next() ) {
               // Detect a change in timestamp (new bar in specified time-frame)
                  static datetime dtTimeStampCurrent  = WRONG_VALUE;
                         datetime dtTimeStampPrevious = dtTimeStampCurrent;
                                  dtTimeStampCurrent  = g_oTickProcessor.Time() / i_nTimeframeSeconds;
                         bool     bTimeStampNew       = dtTimeStampCurrent != dtTimeStampPrevious;
               // Process opening state for new time (second based time-frame)
                  if( bTimeStampNew ) {
                     // Build data ...
                        // ... do something with data ...

                     // Check if we processing data in real-time
                        if( g_oTickProcessor.m_bRealtime ) {
                           // ... carry out trading operations ...
                        };
                  };
            };
      };

 //---------------------------------------------------------------------------------------------------------------------
 
Fernando Carreiro #:
I don't know of any broker offering that many symbols. That is an absurd amount! Even 1000 symbols would be too many.

I am interested in stock options. A single symbol can have over 1,500 options in brazilian B3!


The problem is that when I rerun the code it does not update the values.

//+------------------------------------------------------------------+
//|                                                       testes.mq5 |
//|                                  Copyright 2022, MetaQuotes Ltd. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, MetaQuotes Ltd."
#property link      "https://www.mql5.com"
#property version   "1.00"



#include <Json.mqh>
#include <Math\Stat\Math.mqh>


bool debug = true;



//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool SaveTextToFile(string path, string content) {
//Open file/////////////////////////////////////////////////////////////////////////////////////////////////////////
   int handle = FileOpen(path, FILE_CSV | FILE_READ | FILE_WRITE, ',');
   if(handle < 1) {
      Comment("File " + path + " not found, the last error is ", GetLastError());
      return(false);
   } else {
      Comment("Ok");
      FileWrite(handle, content);
      Print("Salvando " + path);
      Comment("1");
      FileClose(handle);
      return( true);
   }

//return(handle);
}



//+------------------------------------------------------------------+
//|  listar opções de um ativo                                                                |
//+------------------------------------------------------------------+
string getAllSeries(string stockName, bool fast=false) {
   int c=0;
   CJAVal info;
   MqlTick ticks_array[];

// Get the total number of available symbols
   int totalSymbols = SymbolsTotal(false);

   CopyTicks(stockName, ticks_array, COPY_TICKS_ALL, 0, 1);
   double valorAtivo = ticks_array[0].last;
   info["spot_price"] = ticks_array[0].last;

   datetime currentTime = TimeCurrent();

// Loop through the symbols
   CJAVal options;
   for(int i = 0; i < totalSymbols; i++)   {
      string symbol = SymbolName(i, false) ;

      if (SymbolInfoString(symbol, SYMBOL_BASIS)==stockName) {
         // Print(i, symbol);

         double strike = SymbolInfoDouble(symbol, SYMBOL_OPTION_STRIKE);
         long expiration1 = SymbolInfoInteger(symbol, SYMBOL_EXPIRATION_TIME);

         if ((strike>0.8*valorAtivo)&&(strike<1.2*valorAtivo)) {
            // Request the market data
            double spotPrice = SymbolInfoDouble(symbol, SYMBOL_BID);

            if (fast) {
               info["bid"] = 0;
               info["ask"] = 0;
            } else {
               CopyTicks(symbol, ticks_array, COPY_TICKS_ALL, 0, 1);
               if (ArraySize(ticks_array)>0) {
                  info["bid"] = ticks_array[0].bid;
                  info["ask"] = ticks_array[0].ask;
               } else {
                  info["bid"] = 0;
                  info["ask"] = 0;
               }
            }

            info["symbol"] = symbol;
            info["strike"] = strike;

            long tipo = SymbolInfoInteger(symbol, SYMBOL_OPTION_RIGHT);
            if (tipo==SYMBOL_OPTION_RIGHT_CALL) {
               info["type"] = "CALL";
            } else {
               info["type"] = "PUT";
            }

            long expiration = SymbolInfoInteger(symbol, SYMBOL_EXPIRATION_TIME);
            info["expiration"] = SymbolInfoInteger(symbol, SYMBOL_EXPIRATION_TIME);

            int days_to_maturity = (int(expiration1)-int(currentTime))/(60*60*24);
            info["days_to_maturity"] = days_to_maturity;
            Print("Info: ", info.Serialize());

            options["options"].Add(info);
            c++;
         }
      }
   }
   string t = options.Serialize();

   if(debug) {
      Print(t);
   }
   return t;
}




//+------------------------------------------------------------------+
//| Script program start function                                    |
//+------------------------------------------------------------------+
void OnStart() {
//---
   string symbol = "PETR4";
   int expiration = 35;
   string t;

   string symbols[] = {"ABEV3", "B3SA3", "BBAS3", "BBDC4", "BOVA11", "CIEL3", "CMIG4", "COGN3", "CSNA3", "GGBR3",
                       "GOAU4", "ITSA4", "ITUB4", "PETR4", "PRIO3", "RAIL3", "RENT3", "SANB11", "SUZB3", "USIM5", "VALE3", "WEGE3"
                      };

   for (int ii; ii<ArraySize(symbols); ii++) {
      t = getAllSeries(symbols[ii], false);
      StringReplace(t, "},", "},\n");

      string currentDate = TimeToString(TimeCurrent(), TIME_DATE);
      StringReplace(currentDate,".","_");
      string fileName = "OptionsLog" + "\\" + currentDate + "\\"+ "Results " + symbols[ii] + ".txt";

      SaveTextToFile(fileName, t);
   }
}
//+------------------------------------------------------------------+
New comment
Reason: