#!/usr/bin/env python
# coding: utf-8

# We will use the MT5 python package to control our MT5 terminal using python
# 
# We can fetch historical price data easily, and then leverage Python's Machine Learning Libraries to Quickly Build A Model And Evaluate it.

# In[97]:


#Import MetaTrader 5 Library
import MetaTrader5 as mt5

#Import datetime for selecting data
from datetime import datetime

#Plotting Data
import matplotlib.pyplot as plt

#Import pandas for handling data
import pandas as pd

#Import library for calculating technical indicators
import pandas_ta as ta

#Scoring metric to assess model accuracy
from sklearn.metrics import precision_score

#Import mutual information, a black box explanation technique
from sklearn.feature_selection import mutual_info_classif

#Import permutation importance, another black box explanation technique
from sklearn.inspection import permutation_importance

#Import our model
from xgboost import XGBClassifier

#Plotting model importance
from xgboost import plot_importance


# Creating login details

# In[6]:


#Enter your account number
login = enter_your_login_details
#Enter your password
password = 'enter_your_password'
#Enter your Broker's server
server = 'Deriv-Demo'


# In[35]:


#We can intialize the MT5 terminal and login to our account in the same step
if mt5.initialize(login=login,password=password,server=server):
    print('Logged in successfully')
else:
    print('Failed To Log in')


# In[14]:


dir(mt5)


# In[13]:


#To view all available symbols from your broker
symbols = mt5.symbols_get()

for index,value in enumerate(symbols):
    print(value.name)


# In[39]:


#We need to specify the dates we want to use in our dataset
date_from = datetime(2019,4,17)
date_to = datetime.now()


# In[46]:


#Fetching historical data
data = pd.DataFrame(mt5.copy_rates_range('Boom 1000 Index',mt5.TIMEFRAME_D1,date_from,date_to))


# In[47]:


#Let's convert the time from seconds to year-month-date
data['time'] = pd.to_datetime(data['time'],unit='s')

data


# In[48]:


#Let's create a function to preprocess our data
def preprocess(df):
    #All values of real_volume are 0 in this dataset
    df.drop(columns={'real_volume'},inplace=True) 
    #Calculating 14 period ATR
    df.ta.atr(length=14,append=True)
    #Calculating the growth in the value of the ATR, the second difference
    df['ATR Growth'] = df['ATRr_14'].diff().diff()
    #Calculating 14 period RSI
    df.ta.rsi(length=14,append=True)    
    #Calculating the rolling standard deviation of the RSI
    df['RSI Stdv'] = df['RSI_14'].rolling(window=14).std()
    #Calculating the mid point of the high and low price
    df['mid_point'] = ( ( df['high'] + df['low'] ) / 2 )  
    #We will keep track of the midpoint value of the previous day
    df['mid_point - 1'] = df['mid_point'].shift(1) 
    #How far is our price from the midpoint?
    df['height'] = df['close'] - df['mid_point']  
    #Drop any rows that have missing values
    df.dropna(axis=0,inplace=True)


# In[49]:


preprocess(data)

data


# In[50]:


#We want to predict whether tomorrow's close will be greater than today's close
#We can encode a dummy variable for that: 
#1 means tomorrow's close will be greater
#0 means today's close will be greater than tomorrow

data['target'] = (data['close'].shift(-1) > data['close']).astype(int)

data

#As you can see in the output below the first date is 2019-05-14, the close on 2019-05-15 was 8944.461
#So therefore on  2019-05-14 the correct forecast woudl've been 0 because the close price fell.


# In[73]:


#Seperating predictors and target
predictors = ['open','high','low','close','tick_volume','spread','ATRr_14','ATR Growth','RSI_14','RSI Stdv','mid_point','mid_point - 1','height']
target     = ['target']

train_start = 27
train_end = 1000

test_start = 1001


# In[74]:


#Train set
train_x = data.loc[train_start:train_end,predictors]
train_y = data.loc[train_start:train_end,target]

#Test set
test_x = data.loc[test_start:,predictors]
test_y = data.loc[test_start:,target]


# In[79]:


#Let us fit our model
black_box = XGBClassifier()
black_box.fit(train_x,train_y)


# In[80]:


#Let's see our model predictions
black_box_predictions = pd.DataFrame(black_box.predict(test_x),index=test_x.index)


# In[81]:


print(test_y.shape,black_box_predictions.shape)


# In[82]:


#Assesing model prediction accuracy
black_box_score = precision_score(test_y,black_box_predictions)

#Model precision score
black_box_score


# Fortunately XGBoost comes packaged with an inbuilt function to measure feature importance making our lives easier.
# However this is specific to this implementation of XGBoost, and not all black boxes contain useful functions to show feature importance in such an easy manner. 
# 
# For example neural networks and support vector machines don't have an equivalent function, you have to analyse and interpret the model weights to understand your model better.
# 
# Also another drawback is that the feature importance table doesn't show any interaction terms, does that mean none exist?

# In[90]:


#Our model has a 46% precision score, but which features are contributing the most and which ones aren't helpfull?
plot_importance(black_box)


# So now we know the actual feature importance, again remember this information may not always be easily accessible. 
# 
# But since this is a controlled environment, we're able to create our own data for demonstrational purposes.

# Permutation importance attempts to estimate the importance of each feature by randomly shuffling the values in each feature
# and then measuring the change in the model's loss function. The reasoning is that,the more your model relies on that feature
# the worse its performance will be if we randomly shuffle those values.

# In[93]:


#Now let us observe the disagreement problem
black_box_pi = permutation_importance(black_box,train_x,train_y)


# In[95]:


# Get feature importances and standard deviations
perm_importances = black_box_pi.importances_mean
perm_std = black_box_pi.importances_std
# Sort features based on importance
sorted_idx = perm_importances.argsort()


# Below we can see the importance of each column as calculated by Permutation Importance

# In[99]:


plt.barh(range(train_x.shape[1]), perm_importances[sorted_idx], xerr=perm_std[sorted_idx])
plt.yticks(range(train_x.shape[1]), train_x.columns[sorted_idx])
plt.xlabel('Permutation Importance')
plt.title('Permutation Importances')
plt.show()


# According to the calculations performed by permutation importance, the ATR reading is the most informative feature we have engineered. But we know from our ground truth that it's not, the ATR actually ranked sixth. The ATR growth is the most important feature! The second most important feature was the height, however permutation importance calculated that the ATR Growth was more important. The third most important feature was the RSI reading but our permutation importance calculated height as being more important.
# 
# This is the problem with black box explanation techniques, they are very good estimates of feature importance however they are prone to be wrong because at best they are only estimations. And not only that, but they can also disagree with each other.
# 

# Mutual information is our next black box explanation technique, it measures the reduction in uncertainty that is brought about by being aware of a feature's value.

# In[100]:


#Let's see if our black box explainers will disagree with each other by calculating mutual information
black_box_mi = mutual_info_classif(train_x,train_y)
black_box_mi = pd.Series(black_box_mi, name="MI Scores", index=train_x.columns)
black_box_mi = black_box_mi.sort_values(ascending=False)

black_box_mi


# As you can see, we have wildly different importance rankings. Permutation importance is ranking the features in almost reverse order compared to our ground truth and the mutual information calculation. If you didn't have the ground truth that we have in this example, which explainer were you going to rely on more?  
# 
# Furthermore, what if you used 5 different explanation techniques and they each gave you different importance rankings, then what? Do you pick the rankings that align with your beliefs about the workings of the real world, that's called confirmation bias. 
# 
# Also note permutation importance and mutual information are univariate metrics of feature importance. This means that they can't pick up on interaction terms between the features. Therefore, some features may appear uninformative when considered on their own but may be powerful when considered with their interaction pair.