Can't scrape vix from can

[Deleted]  

Hello Guys,


hope u can help...

trying to scrape vix level from cnn.


Unfortunately this is the result:




I do not understand why nothing is found.


Thank you for your help!


This is my code:



//+------------------------------------------------------------------+


//|                                          Fear & Greed Webscraper |


//|                                         Copyright 2022, |


//|                                       |


//+------------------------------------------------------------------+










//------------------------------importing wininet.dll and prototyping the functions that will be called


#import "wininet.dll"


int InternetCheckConnectionW(string &lpszUrl,uint dwFlags,uint dwReserved);


int InternetOpenW(string &lpszAgent,uint dwAccessType,string &lpszProxyName,string &lpszProxyBypass,uint dwFlags);


int InternetOpenUrlW(int hInternetSession,string &lpszUrl,string &lpszHeaders,uint dwHeadersLength,uint dwFlags,uint dwContext);


int InternetConnectW(int hInternet,string &lpszServerName,int nServerPort,string &lpszUsername,string &lpszPassword,uint dwService,uint dwFlags,uint dwContext);


int HttpOpenRequestW(int hConnect,string &lpszVerb,string &lpszObjectName,string &lpszVersion,string &lpszReferer,string &lplpszAcceptTypes,uint dwFlags,uint dwContext);


int HttpSendRequestW(int hRequest,string &lpszHeaders,uint dwHeadersLength,uchar &lpOptional[],uint dwOptionalLength);


int InternetReadFile(int hFile,uchar &lpBuffer[],uint dwNumberOfBytesToRead,uint &lpdwNumberOfBytesRead);


int InternetCloseHandle(int hInternet);


#import




uchar uc_Buffer[1640000000], uc_DynBuf[]; // InternetReadFile() expects a static buffer.  The size of uc_Buffer is set to accommodate the specific number of bytes that will be downloaded.. will receive the html text downloaded from the web server


float VIX;





//------------------------------------------------------------------------------------------------------












//------------------------------------------first we check to see if an internet connection is available.


void OnInit()


  {


   bool bResult;  int i,iNet1,iNet2; //, iRequest;  


   


   string stURL="http://www.msn.com";


   bResult=InternetCheckConnectionW(stURL,1,0); // 1 == FLAG_ICC_FORCE_CONNECTION


   Print("InternetCheckConnectionW() returned ",bResult);


   if(!bResult) return;




   string stAgent="Mozilla/5.0",stNull="";


   iNet1 = InternetOpenW(stAgent, // _In_ LPCTSTR lpszAgent 


                         1,       // 1 == INTERNET_OPEN_TYPE_DIRECT


                         stNull,  // _In_ LPCTSTR lpszProxyName


                         stNull,  // _In_ LPCTSTR lpszProxyBypass


                         NULL);   // _In_ DWORD dwFlags


   Print("iNet1 == ",iNet1);


   if(iNet1==0) return;


//----------------------------------------------------------------------------------------------










//------------------------------------------a connection to the web server is established, initializing the handle iNet2 to download the html file.




   stURL="https://edition.cnn.com/markets/fear-and-greed";


   string stHdr="Accept: text/*";


   iNet2 = InternetOpenUrlW(iNet1,            // HINTERNET hInternet,


                            stURL,            // LPCWSTR   lpszUrl,


                            stHdr,            // LPCWSTR   lpszHeaders,


                            StringLen(stHdr), // DWORD     dwHeadersLength,


                            0x00080000,       // DWORD dwFlags, 0x00080000 == INTERNET_FLAG_NO_COOKIES


                            NULL);            // DWORD_PTR dwContext


   Print("iNet2 == ",iNet2);


   if(iNet2==0)


     {


      InternetCloseHandle(iNet1);


      return;


     }


//----------------------------------------------------------------------------------------------










//-------------------------------------------------Download data from the web server.




  uint uGet, uGot, uDst, uMax;


  uGet = 1638400000;    // number of bytes to download per call to InternetReadFile, must be at least 1 byte less than size of uc_Buffer


  uGot = uDst = 0; // uGot is number of bytes downloaded in call to InternetReadFile; uDst is total number of bytes downloaded.


  uMax = 18022400000;   // maximum number of bytes to download.




  do


  { bResult = InternetReadFile(iNet2,     // _In_  HINTERNET hFile


                               uc_Buffer, // _Out_ LPVOID lpBuffer


                               uGet,      // _In_  DWORD dwNumberOfBytesToRead


                               uGot);     // _Out_ LPDWORD lpdwNumberOfBytesRead




    uc_Buffer[uGot] = 0; // Terminate string in uc_Buffer by appending a null character.




    ArrayCopy(uc_DynBuf, // destination array 


              uc_Buffer, // source array 


              uDst,      // index at which writing to destination array begins 


              0,         // index at which copying from source array begins 


              uGot);     // number of elements to copy 


    uDst += uGot; // advance index of destination array for next pass in loop


  Print( " uc_Buffer:", CharArrayToString(uc_Buffer)); 


  }while(bResult && uGot > 0 && uDst < uMax);


 


  Print( "Size of uc_DynBuf == ", ArraySize(uc_DynBuf));


  Print("Bytes downloaded  == ", uDst);   


 //----------------------------------------------------------------------------------------------


 


 


 


 

 


 


 //-----------------------------Now we search for the meta tag of interest in the downloaded text




  i = StringFind(CharArrayToString(uc_DynBuf), "<span class=\"market-fng-gauge__dial-number-value\">", 0); // 0 == position from which search starts 


  Print("<span class=\"market-fng-gauge__dial-number-value\"> == ", i); 


  if(i == -1) {Print("<span class=\"market-fng-gauge__dial-number-value\"> not found.");  InternetCloseHandle(iNet1);  return;}




   i = i + 52; // Advance index to known location of text representing vix.

   Print("i= ", i);

  VIX = StringToDouble(StringSubstr(CharArrayToString(uc_Buffer), i, 2));

  Print("foundstring: ", StringSubstr(CharArrayToString(uc_Buffer), i, 2));

  Print("VIX: ", VIX);

  InternetCloseHandle(iNet1); // Done with wininet.

}//END void OnStart()




//----------------------------------------------------------------------------------------------




void OnTick()


{




    Comment("VIX:",VIX);


}

Fear and Greed Index - Investor Sentiment | CNN
  • edition.cnn.com
CNN’s Fear & Greed Index is a way to gauge stock market movements and whether stocks are fairly priced. The index uses seven market indicators to help answer the question: What emotion is driving the market now?
Carl Schreiber  

Please edit your posted code with the code bottom: Alt+s or the </> button from the header line.

To dld. from the internet you don't need the Windows functions (with example): https://www.mql5.com/de/docs/network/webrequest

If the code compiles without any error but doesn't do what it should you can use the debugger - a very powerful tool:

https://www.metatrader5.com/en/metaeditor/help/development/debug // Code debugging
https://www.mql5.com/en/articles/2041 // Error Handling and Logging in MQL5
https://www.mql5.com/en/articles/272 // Tracing, Debugging and Structural Analysis of Source Code
https://www.mql5.com/en/articles/35 // scroll down to: "Launching and Debugging"

(btw if you replace /en/ by /de/ in the MQ-Urls you'll get the German version).

Code debugging - Developing programs - MetaEditor Help
Code debugging - Developing programs - MetaEditor Help
  • www.metatrader5.com
MetaEditor has a built-in debugger allowing you to check a program execution step by step (by individual functions). Place breakpoints in the code...
Jan4654  

Scraping is easily detected with webrequests especially on big commercial sites like cnn and like.

Simple requests can not excute js code and or ip or asn belong to datacenter == detected your crap web crawler.

They will let web crawlers like google search engine and other friendly web crawlers poke around more freely.

Reason: