How to correctly read data from .cvs

[Manu Duarte]

//+------------------------------------------------------------------+
//|                                      Expert Advisor Template.mq5 |
//|                                                                  |
//+------------------------------------------------------------------+

// Include necessary files
#include <Trade/Trade.mqh>
#include <Files/File.mqh>
#include <Files/FileBin.mqh>
#include <Arrays/ArrayString.mqh>
#include <Arrays/ArrayObj.mqh>

input string FileName = "test.csv";      // CSV file name
input string SymbolName = "GBPUSD";      // Symbol name


// Declare global variables and objects
CTrade trade;
double value1;
double value2;
const int MAX_DATA_SIZE = 1000; // Maximum number of data points
// Declare your custom structure for the data
struct Data
{
    datetime sample_time;
    datetime ht_time;
    double ht_open;
    double ht_high;
    double ht_low;
    double ht_close;
    double ht_bollinger_mid;
    double ht_OHLC_max;
    double ht_OHLC_min;
    int    ht_liq_type;
    datetime lt_time;
    double lt_open;
    double lt_high;
    double lt_low;
    double lt_close;
    double lt_bollinger_mid;
    double lt_OHLC_max;
    double lt_OHLC_min;
    double lt_valid_range;
    double lt_tap_range;
    string dist_prox;
    datetime tapped_by;
    double sl;
    double entry;
    double tp;
    string dir;

    bool FromCSVLine(string line)
    {
        StringReplace(line, "\"", ""); // Remove double quotes from the line
        string parts[];
        StringSplit(line, ',', parts); // Split the line using comma as a delimiter

        if(ArraySize(parts) != 26) // Check that there are 26 fields in the line
            return false;

        // Parse and populate the data structure
        sample_time = StringToTime(parts[0]);
        ht_time = StringToTime(parts[1]);
        ht_open = StringToDouble(parts[2]);
        ht_high = StringToDouble(parts[3]);
        ht_low = StringToDouble(parts[4]);
        ht_close = StringToDouble(parts[5]);
        ht_bollinger_mid = StringToDouble(parts[6]);
        ht_OHLC_max = StringToDouble(parts[7]);
        ht_OHLC_min = StringToDouble(parts[8]);
        ht_liq_type = int(StringToInteger(parts[9]));
        lt_time = StringToTime(parts[10]);
        lt_open = StringToDouble(parts[11]);
        lt_high = StringToDouble(parts[12]);
        lt_low = StringToDouble(parts[13]);
        lt_close = StringToDouble(parts[14]);
        lt_bollinger_mid = StringToDouble(parts[15]);
        lt_OHLC_max = StringToDouble(parts[16]);
        lt_OHLC_min = StringToDouble(parts[17]);
        lt_valid_range = StringToDouble(parts[18]);
        lt_tap_range = StringToDouble(parts[19]);
        dist_prox = parts[20];
        tapped_by = StringToTime(parts[21]);
        sl = StringToDouble(parts[22]);
        entry = StringToDouble(parts[23]);
        tp = StringToDouble(parts[24]);
        dir = parts[25];

        return true;
    }
};

bool ReadCSVFile(string filename, Data &data[], uint &count)
{
    int line_count = 0;
    ResetLastError();

    // Open the file for reading
    int file_handle = FileOpen(filename, FILE_READ | FILE_CSV, ',');
    if(file_handle == INVALID_HANDLE)
    {
        Print("Failed to open file: ", filename, ". Error code: ", GetLastError());
        return false;
    }
  // Read the header row and print it
    string header = FileReadString(file_handle, CP_UTF8);
    Print("Header: ", header);
    // Read each line from the file and parse it into the data array
    string line;
    while(FileIsEnding(file_handle) == false)
    {
        line = FileReadString(file_handle);
        if (StringLen(line) == 0)
        {
            Print("Failed to read line ", line_count, " from file: ", filename, ". Error code: ", GetLastError());
            FileClose(file_handle);
            return false;
        }

        // Print the current line for debugging
        Print("Line ", line_count, ": ", line);

        // Skip the header row
        if(line_count == 0)
        {
            line_count++;
            continue;
        }

        // Parse the CSV line and populate the data array
        if(!data[line_count-1].FromCSVLine(line))
        {
            Print("Failed to parse line ", line_count, " from file: ", filename);
            FileClose(file_handle);
            return false;
        }

       
        
        line_count++;
    }

    // Set the count of parsed data
    count = line_count - 1;

    // Close the file
    FileClose(file_handle);

    return true;
}


// GetCandleInformation function definition
void GetCandleInformation(string symbol, ENUM_TIMEFRAMES sample_time_tf, ENUM_TIMEFRAMES ht_time_tf, ENUM_TIMEFRAMES lt_time_tf, int index, Data& data)
{
    data.sample_time = iTime(symbol, sample_time_tf, index);
    data.ht_time = iTime(symbol, ht_time_tf, index);
    data.ht_open = iOpen(symbol, ht_time_tf, index);
    data.ht_high = iHigh(symbol, ht_time_tf, index);
    data.ht_low = iLow(symbol, ht_time_tf, index);
    data.ht_close = iClose(symbol, ht_time_tf, index);

    double ht_bollinger_mid[];
    int ht_bands_handle = iBands(symbol, ht_time_tf, 20, 2, 0, PRICE_CLOSE);
    if(CopyBuffer(ht_bands_handle, 1, 0, 1, ht_bollinger_mid) != 1)
    {
        Print("Failed to get bollinger mid value for HT at index ", index);
        return;
    }
    data.ht_bollinger_mid = ht_bollinger_mid[0];

    data.ht_OHLC_max = MathMax(MathMax(data.ht_open, data.ht_high), MathMax(data.ht_low, data.ht_close));
    data.ht_OHLC_min = MathMin(MathMin(data.ht_open, data.ht_high), MathMin(data.ht_low, data.ht_close));
    data.ht_liq_type = (data.ht_close > data.ht_bollinger_mid) ? 1 : 2;

    data.lt_time = iTime(symbol, lt_time_tf, index);
    data.lt_open = iOpen(symbol, lt_time_tf, index);
    data.lt_high = iHigh(symbol, lt_time_tf, index);
    data.lt_low = iLow(symbol, lt_time_tf, index);
    data.lt_close = iClose(symbol, lt_time_tf, index);

    double lt_bollinger_mid[];
    int lt_bands_handle = iBands(symbol, lt_time_tf, 20, 2, 0, PRICE_CLOSE);
    if(CopyBuffer(lt_bands_handle, 1, 0, 1, lt_bollinger_mid) != 1)
    {
        Print("Failed to get bollinger mid value for LT at index ", index);
        return;
    }
    data.lt_bollinger_mid = lt_bollinger_mid[0];

    data.lt_OHLC_max = MathMax(MathMax(data.lt_open, data.lt_high), MathMax(data.lt_low, data.lt_close));
    data.lt_OHLC_min = MathMin(MathMin(data.lt_open, data.lt_high), MathMin(data.lt_low, data.lt_close));
    data.lt_valid_range = data.lt_close - data.lt_open;
    data.lt_tap_range = MathAbs(data.ht_close - data.lt_close) / (data.ht_OHLC_max - data.ht_OHLC_min);
    data.dist_prox = (data.ht_liq_type == 1) ? "far" : "close";
    data.tapped_by = (data.ht_close > data.lt_close) ? data.ht_time : data.lt_time;
    data.sl = (data.ht_liq_type == 1) ? data.lt_low : data.lt_high;
    data.entry = (data.ht_liq_type == 1) ? data.lt_close - data.lt_valid_range * 2 : data.ht_close + data.lt_valid_range * 2;
  data.tp = (data.ht_liq_type == 1) ? data.lt_close - data.lt_tap_range * data.lt_valid_range : data.ht_close + data.lt_tap_range * data.lt_valid_range;
  data.dir = (data.ht_close > data.lt_close) ? "long" : "short";
}


void DrawMarkersAndLines(Data &data)
{
  if (data.dir == "long")
  {
    // Remove any existing objects with the same names
    ObjectDelete(0, "BuyArrow");
    ObjectDelete(0, "StopLoss");
    ObjectDelete(0, "TakeProfit");

    // Draw the buy arrow
    ObjectCreate(0, "BuyArrow", OBJ_ARROW, 0, data.sample_time, data.entry);
    ObjectSetInteger(0, "BuyArrow", OBJPROP_COLOR, clrGreen);
    ObjectSetInteger(0, "BuyArrow", OBJPROP_ARROWCODE, 233);

    // Draw the stop loss line
    ObjectCreate(0, "StopLoss", OBJ_TREND, 0, data.tapped_by, data.sl, data.sample_time, data.sl);
    ObjectSetInteger(0, "StopLoss", OBJPROP_COLOR, clrRed);

    // Draw the take profit line
    ObjectCreate(0, "TakeProfit", OBJ_TREND, 0, data.tapped_by, data.tp, data.sample_time, data.tp);
    ObjectSetInteger(0, "TakeProfit", OBJPROP_COLOR, clrBlue);

    Print("DrawMarkersAndLines Checkpoint: Created objects for long trade at sample_time = ", TimeToString(data.sample_time));
  }
  else if (data.dir == "short")
  {
    // Remove any existing objects with the same names
    ObjectDelete(0, "SellArrow");
    ObjectDelete(0, "StopLoss");
    ObjectDelete(0, "TakeProfit");

    // Draw the sell arrow
    ObjectCreate(0, "SellArrow", OBJ_ARROW, 0, data.sample_time, data.entry);
    ObjectSetInteger(0, "SellArrow", OBJPROP_COLOR, clrRed);
    ObjectSetInteger(0, "SellArrow", OBJPROP_ARROWCODE, 234);

    // Draw the stop loss line
    ObjectCreate(0, "StopLoss", OBJ_TREND, 0, data.tapped_by, data.sl, data.sample_time, data.sl);
    ObjectSetInteger(0, "StopLoss", OBJPROP_COLOR, clrRed);

    // Draw the take profit line
    ObjectCreate(0, "TakeProfit", OBJ_TREND, 0, data.tapped_by, data.tp, data.sample_time, data.tp);
    ObjectSetInteger(0, "TakeProfit", OBJPROP_COLOR, clrBlue);

    Print("DrawMarkersAndLines Checkpoint: Created objects for short trade at sample_time = ", TimeToString(data.sample_time));
  }
}
// SaveScreenshot function definition
bool SaveScreenshot(string file_name)
{
  if(!ChartScreenShot(0, file_name, 800, 600, ALIGN_RIGHT))
  {
    Print("Failed to save screenshot: ", file_name);
    return false;
  }
  else
  {
    Print("Screenshot saved: ", file_name);
    return true;
  }
}

// OnStart function
int OnInit()
{
    Print("Using file name: ", FileName);

    Data data[];
    uint count = 0;
    if (ReadCSVFile("test.csv", data, count)) {
    Print("CSV file read successfully. Number of data rows: ", count);
} else {
    Print("Error reading CSV file.");
}
    for (uint i = 0; i < count; i++)
    {
        Print("Current index: ", i);
        Print("Sample time: ", TimeToString(data[i].sample_time));
        if (data[i].sample_time >= TimeCurrent() - (30 * 24 * 60 * 60) && data[i].sample_time <= TimeCurrent())
        {
            Print("Data within time range found.");

            int chartIndex = iBarShift(SymbolName, PERIOD_M15, data[i].sample_time);
            GetCandleInformation(SymbolName, PERIOD_M15, PERIOD_H4, PERIOD_M15, chartIndex, data[i]);

            DrawMarkersAndLines(data[i]);

            ChartRedraw();

            string screenshot_path = StringFormat("s_t_%s_%d.bmp", TimeToString(data[i].sample_time, TIME_DATE|TIME_SECONDS), i);
            if(!SaveScreenshot(screenshot_path))
            {
                Print("Failed to save screenshot for index: ", i);
            }
        }
        else
        {
            Print("Data not within time range.");
        }
    }

    return(INIT_SUCCEEDED);
}

This generates the following Output in the Terminal :  2023.04.13 09:51:50.356 mt5_chart_marker (EURUSD,M15) Header: 猢浡汰彥楴敭栬彴楴敭栬彴灯湥栬彴楨桧栬彴潬ⱷ瑨损潬敳栬彴潢汬湩敧彲業Ɽ瑨佟䱈彃慭ⱸ瑨佟䱈彃業Ɱ瑨江煩瑟灹ⱥ瑬瑟浩ⱥ瑬潟数Ɱ瑬桟杩ⱨ瑬江睯氬彴汣獯ⱥ瑬扟汯楬杮牥浟摩氬彴䡏䍌浟硡氬彴䡏䍌浟湩氬彴慶楬彤慲杮ⱥ瑬瑟灡牟湡敧∬搢獩㵴ⰰ牰硯ㄽ∢琬灡数彤祢猬ⱬ湥牴ⱹ灴搬物ഢㄊ㤹⸹㄰ㄮ‵〲㐺㨵〰ㄬ㤹⸹㄰ㄮ‵㈱〺㨰〰ㄬ㘮㠵ⰳ⸱㘶㘲ㄬ㘮㠴ⰵ⸱㔶ㄵㄬ㘮㘴㌴ㄬ㘮㈶ⰶ⸱㐶㔸⴬ⰱ㤱㤹〮⸱㔱ㄠ㨴〰〺ⰰ⸱㔶㔶ㄬ㘮㠵ⰶ⸱㐶㔸ㄬ㘮〵ⰲ⸱㔶㈸㔴ㄬ㘮㠵ⰶ⸱㐶㔸ㄬ㘮㠴ⰵ⸱㐶㘹㘰㘶ⰷⰰ㤱㤹〮⸱㔱㈠㨰㔱〺ⰰ⸱㐶㤸㈰ㄬ㘮㤴㜳ㄬ㘮㘵ㄲⰵ潬杮਍㤱㤹〮⸱㤱〠㨰〳〺ⰰ㤱㤹〮⸱㠱ㄠ㨶〰〺ⰰ⸱㔶㠳ㄬ㘮㐵ⰸ⸱㔶ㄱㄬ㘮㌵ⰴ⸱㔶㤲ⰳ⸱㔶㠴ㄬ㘮ㄵⰱㄭㄬ㤹⸹㄰ㄮ‸㜱ㄺ㨵〰ㄬ㘮㈵ⰴ⸱㔶㈳ㄬ㘮ㄵⰱ⸱㔶㤱ㄬ㘮㐵㈰ⰵ⸱㔶㈳ㄬ㘮ㄵⰱ⸱㔶ㄱㄬ㘮ㄵ㈴〬ㄬ㤹⸹㄰ㄮ‹〰ㄺ㨵〰ㄬ㘮ㄵ㌱ⰴ⸱㔶㈱ⰹ⸱㔶㘴㔵氬湯൧ㄊ㤹⸹㄰ㄮ‹㘰㌺㨰〰ㄬ㤹⸹㄰ㄮ‸㠰〺㨰〰ㄬ㘮㌵ⰲ⸱㔶㐵ㄬ㘮㤴ⰹ⸱㔶㘴ㄬ㘮ㄵⰸ⸱㔶㐵ㄬ㘮㤴ⰹㄭㄬ㤹⸹㄰ㄮ‸㤰㐺㨵〰ㄬ㘮㌵ⰳ⸱㔶㔳ㄬ㘮㤴ⰹ⸱㔶㈱ㄬ㘮㈵㐳ㄬ㘮㌵ⰵ⸱㐶㤹ㄬ㘮㤴ⰹ⸱㔶㐰ⰶⰱ㤱㤹〮⸱㤱〠㨵〳〺ⰰ⸱㔶㄰㐵ㄬ㘮㤴㤷ㄬ㘮㘴ㄹⰵ桳牯൴ㄊ㤹⸹㄰ㄮ‹㌱㌺㨰〰ㄬ㤹⸹㄰ㄮ‹㠰〺㨰〰ㄬ㘮

Which should be:  "sample_time,ht_time,ht_open,ht_high,ht_low,ht_close,ht_bollinger_mid,ht_OHLC_max,ht_OHLC_min,ht_liq_type,lt_time,lt_open,lt_high,lt_low,lt_close,lt_bollinger_mid,lt_OHLC_max,lt_OHLC_min,lt_valid_range,lt_tap_range,""dist=0,prox=1"",tapped_by,sl,entry,tp,dir"

What am I doing wrong?

[Carl Schreiber]

Manu Duarte:

This generates the following Output in the Terminal :  2023.04.13 09:51:50.356 mt5_chart_marker (EURUSD,M15) Header: 猢浡汰彥楴敭栬彴楴敭栬彴灯湥栬彴楨桧栬彴潬ⱷ瑨损潬敳栬彴潢汬湩敧彲業Ɽ瑨佟䱈彃慭ⱸ瑨佟䱈彃業Ɱ瑨江煩瑟灹ⱥ瑬瑟浩ⱥ瑬潟数Ɱ瑬桟杩ⱨ瑬江睯氬彴汣獯ⱥ瑬扟汯楬杮牥浟摩氬彴䡏䍌浟硡氬彴䡏䍌浟湩氬彴慶楬彤慲杮ⱥ瑬瑟灡牟湡敧∬搢獩㵴ⰰ牰硯ㄽ∢琬灡数彤祢猬ⱬ湥牴ⱹ灴搬物ഢㄊ㤹⸹㄰ㄮ‵〲㐺㨵〰ㄬ㤹⸹㄰ㄮ‵㈱〺㨰〰ㄬ㘮㠵ⰳ⸱㘶㘲ㄬ㘮㠴ⰵ⸱㔶ㄵㄬ㘮㘴㌴ㄬ㘮㈶ⰶ⸱㐶㔸⴬ⰱ㤱㤹〮⸱㔱ㄠ㨴〰〺ⰰ⸱㔶㔶ㄬ㘮㠵ⰶ⸱㐶㔸ㄬ㘮〵ⰲ⸱㔶㈸㔴ㄬ㘮㠵ⰶ⸱㐶㔸ㄬ㘮㠴ⰵ⸱㐶㘹㘰㘶ⰷⰰ㤱㤹〮⸱㔱㈠㨰㔱〺ⰰ⸱㐶㤸㈰ㄬ㘮㤴㜳ㄬ㘮㘵ㄲⰵ潬杮਍㤱㤹〮⸱㤱〠㨰〳〺ⰰ㤱㤹〮⸱㠱ㄠ㨶〰〺ⰰ⸱㔶㠳ㄬ㘮㐵ⰸ⸱㔶ㄱㄬ㘮㌵ⰴ⸱㔶㤲ⰳ⸱㔶㠴ㄬ㘮ㄵⰱㄭㄬ㤹⸹㄰ㄮ‸㜱ㄺ㨵〰ㄬ㘮㈵ⰴ⸱㔶㈳ㄬ㘮ㄵⰱ⸱㔶㤱ㄬ㘮㐵㈰ⰵ⸱㔶㈳ㄬ㘮ㄵⰱ⸱㔶ㄱㄬ㘮ㄵ㈴〬ㄬ㤹⸹㄰ㄮ‹〰ㄺ㨵〰ㄬ㘮ㄵ㌱ⰴ⸱㔶㈱ⰹ⸱㔶㘴㔵氬湯൧ㄊ㤹⸹㄰ㄮ‹㘰㌺㨰〰ㄬ㤹⸹㄰ㄮ‸㠰〺㨰〰ㄬ㘮㌵ⰲ⸱㔶㐵ㄬ㘮㤴ⰹ⸱㔶㘴ㄬ㘮ㄵⰸ⸱㔶㐵ㄬ㘮㤴ⰹㄭㄬ㤹⸹㄰ㄮ‸㤰㐺㨵〰ㄬ㘮㌵ⰳ⸱㔶㔳ㄬ㘮㤴ⰹ⸱㔶㈱ㄬ㘮㈵㐳ㄬ㘮㌵ⰵ⸱㐶㤹ㄬ㘮㤴ⰹ⸱㔶㐰ⰶⰱ㤱㤹〮⸱㤱〠㨵〳〺ⰰ⸱㔶㄰㐵ㄬ㘮㤴㤷ㄬ㘮㘴ㄹⰵ桳牯൴ㄊ㤹⸹㄰ㄮ‹㌱㌺㨰〰ㄬ㤹⸹㄰ㄮ‹㠰〺㨰〰ㄬ㘮

Which should be:  "sample_time,ht_time,ht_open,ht_high,ht_low,ht_close,ht_bollinger_mid,ht_OHLC_max,ht_OHLC_min,ht_liq_type,lt_time,lt_open,lt_high,lt_low,lt_close,lt_bollinger_mid,lt_OHLC_max,lt_OHLC_min,lt_valid_range,lt_tap_range,""dist=0,prox=1"",tapped_by,sl,entry,tp,dir"

What am I doing wrong?

I use this to read the whole csv file into an array of lines:

int getCsvLines( const string fName, string &l[], int flag=0, ushort sepLine='\n') { // alternative to FileOpen 
   uchar Bytes[]; 
  return( FileLoad(fName, Bytes, flag) ? StringSplit(CharArrayToString(Bytes), sepLine, l) : 0);
}

Hereafter you can split the lines into cells with StringSplit(..)

[William Roeder]

Manu Duarte: What am I doing wrong?

    int file_handle = FileOpen(filename, FILE_READ | FILE_CSV, ',');

1. Not specifying whether the file is ANSI or Unicode.
2. Your reads already split each field. You can't read the entire line and then split. Simplify your code or change the separator.