Titanic

# import data
data_train = pd.read_csv("train.csv")
df=data_train.copy()

# display data head
df.head()

# import data
data_test = pd.read_csv("test.csv")

# display data head
data_test.head()

#get the shape of the data
df.shape

# Use info() to print a summary of the DataFrame
df.info()

# Check for columns that have missing values
df.isnull().sum().sort_values(ascending=False)

# Coverting the following to "category"
cat_columns = df.select_dtypes(include=["object", "category"]).columns.tolist()

for col in cat_columns:
    df[col] = df[col].astype("category")

# Use info() to print a summary of the DataFrame
df.info()

# Statistical summary of the data
df.describe(include="all").T

# Perform Univariate Analysis
Univariate_Analysis(data=df, features=df.columns, num=25)

print(round(len(df[df['Survived']==0])/len(df)*100,2),"% of passangers died in the training data.")

# heat map of numerical features.
fig, ax = plt.subplots(figsize=(20, 5))
sns.heatmap(
    df.corr(), ax=ax, annot=True, linewidths=0.05, fmt=".2f", cmap="magma"
)  # the color intensity is based on
# plt.savefig('HeatMap.jpg',bbox_inches ="tight",pad_inches = 0.5)
plt.show()

# performs a bivariate analysis
Bivariate_Analysis(data=df, feature="Survived", max_elements=35)

#make copy of dataframe
df_processed=df.copy()

#Drop features
df_processed.drop(['Name','Ticket'],axis=1,inplace=True)
df_processed.head()

#Covert Cabin to Deck
df_processed['Deck']=df_processed['Cabin'].apply(lambda x : x[:1])

#Remove Cabin
df_processed.drop(['Cabin'],axis=1,inplace=True)

df_processed.head()

df_processed['Deck'].value_counts()

#define function to replace values in a feature    
def replace_with(data,features=[],replace_struc=[],revert=False):
    # do the replacing
    for feature in features:
        replacement=replace_struc[features.index(feature)]
        if(revert==True): replacement = {v: k for k, v in replacement.items()}
        data[feature].replace(replacement, inplace=True)
    
    return data

# setup a dictionary to do the replacing
Sex = {"male": 0, "female": 1}
Embarked = {"C": 1,"Q": 2,"S": 3}
Deck = {"T": 0,"A": 1,"B": 2,"C": 3,"D": 4,"E": 5,"F": 6,"G": 7}

#use replace_with on the features that are not numerical
df_processed = replace_with(df_processed,['Sex','Embarked','Deck'],[Sex,Embarked,Deck])
    
# display the head
df_processed.head()

#imput missing values in train and test data for V1 and V2
imputer = KNNImputer(n_neighbors=5)

col_to_impute=['Embarked','Sex','Age', 'Deck']

df_processed[col_to_impute] = imputer.fit_transform(df_processed[col_to_impute])
df_processed[col_to_impute]=round(df_processed[col_to_impute])

df_processed.head()

#reverse the replace_with
df_processed = replace_with(df_processed,['Sex','Embarked','Deck'],[Sex,Embarked,Deck],revert=True)

# display the head
df_processed.head()

# performs a bivariate analysis
Bivariate_Analysis(data=df_processed, feature="Survived", max_elements=35)

#Bin decks into upper and lower decks
#Upper decks will be defined as decks A,B,C
#Lower decks will be defined as decks D,E,F,G

# setup a dictionary to do the replacing
DeckLevel = {"T": "upper","A": "upper","B": "upper","C": "upper","D": "lower","E": "lower","F": "lower","G": "lower"}

df_processed['DeckLevel']=df_processed['Deck'];

#use replace_with on the features that are not numerical
df_processed = replace_with(df_processed,['DeckLevel'],[DeckLevel])
    
# display the head
df_processed.head()

# performs a bivariate analysis
Bivariate_Analysis(data=df_processed, feature="Survived", max_elements=35)

# Run Detect_Outliers
Detect_Outliers(data=df_processed, features=df_processed.columns, limits=[0.25, 0.75])

# Separating features and the target column
X = df_modeling.drop("Survived", axis=1)
Y = df_modeling["Survived"]

# create dummies
X = pd.get_dummies(
    X,
    columns=X.select_dtypes(include=["object", "category"]).columns.tolist(),
    drop_first=True
)

# display head
X.head()

# Splitting the data into train and test sets in 75:25 ratio
x_train, x_val, y_train, y_val = train_test_split(
    X, Y, test_size=0.2, random_state=1, shuffle=True, stratify=Y
)

# get shape of training and test set
x_train.shape, x_val.shape

# Print percentage of training set
print("Percentage of classes in training set:")
print(y_train.value_counts(normalize=True))

# Print percentage of test set
print("Percentage of classes in test set:")
print(y_val.value_counts(normalize=True))

# setting the class weight
class_weighting = {
    0: round(y_train.value_counts(normalize=True)[1], 2),
    1: round(y_train.value_counts(normalize=True)[0], 2),
}
print("*" * 50)
print("Class Weighting:", class_weighting)

#build models using original data
models_original=BuildModels(models,x_train=x_train,y_train=y_train,score='accuracy')

#Display the validation performance across all models using the original data.
for model in models_original:
    name=type(model).__name__
    mod_pref=model_performance_classification_sklearn(model=model,predictors=x_val,target=y_val)
    print(name,"\n",mod_pref,"\n")

print("Before OverSampling, count of label '1': {}".format(sum(y_train == 1)))
print("Before OverSampling, count of label '0': {} \n".format(sum(y_train == 0)))

print("After OverSampling, count of label '1': {}".format(sum(y_train_over == 1)))
print("After OverSampling, count of label '0': {} \n".format(sum(y_train_over == 0)))

print("After OverSampling, the shape of train_X: {}".format(x_train_over.shape))
print("After OverSampling, the shape of train_y: {} \n".format(y_train_over.shape))

#build models using oversample data
models_over=BuildModels(models,x_train=x_train_over,y_train=y_train_over,score='accuracy')

#Display the validation performance across all models using the oversample data.
for model in models_over:
    name=type(model).__name__
    mod_pref=model_performance_classification_sklearn(model=model,predictors=x_val,target=y_val)
    print(name,"\n",mod_pref,"\n")

#Display the confusion matrix for the random forest model
rForest_over=RandomForestClassifier(random_state=1)
rForest_over.fit(x_train_over,y_train_over)
confusion_matrix_sklearn(model=rForest_over,predictors=x_val,target=y_val)

print("Before Under Sampling, count of label '1': {}".format(sum(y_train == 1)))
print("Before Under Sampling, count of label '0': {} \n".format(sum(y_train == 0)))

print("After Under Sampling, count of label '1': {}".format(sum(y_train_under == 1)))
print("After Under Sampling, count of label '0': {} \n".format(sum(y_train_under == 0)))

print("After Under Sampling, the shape of train_X: {}".format(x_train_under.shape))
print("After Under Sampling, the shape of train_y: {} \n".format(y_train_under.shape))

#build models using undersample data
models_under=BuildModels(models,x_train=x_train_under,y_train=y_train_under,score='accuracy')

#Display the validation performance across all models using the undersample data.
for model in models_under:
    name=type(model).__name__
    mod_pref=model_performance_classification_sklearn(model=model,predictors=x_val,target=y_val)
    print(name,"\n",mod_pref,"\n")

#Display the confusion matrix for the random forest model
rForest_under=RandomForestClassifier(random_state=1)
rForest_under.fit(x_train_under,y_train_under)
confusion_matrix_sklearn(model=rForest_under,predictors=x_val,target=y_val)

%%time
# defining model
Model = RandomForestClassifier(random_state=1,class_weight=class_weighting)

# Parameter grid to pass in RandomSearchCV
param_grid = { 
    "n_estimators": np.arange(10, 100,10), 
    "min_samples_leaf": np.arange(1, 8,1),
    "max_samples": np.arange(0.2, 1, 0.1),
    "max_features": np.arange(0.1, 1, 0.1),
    "max_depth": np.arange(10, 30, 1) 
}

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(
    estimator=Model, param_distributions=param_grid, n_iter=5, n_jobs = -1, scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(x_train,y_train)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

#Build Base Model
rForest_original= RandomForestClassifier(random_state=1,class_weight=class_weighting)

# Fit the model on training data
rForest_original.fit(x_train, y_train)

# Creating new model with best parameters
rForest_tuned_original = RandomForestClassifier(
    n_estimators=90,
    min_samples_leaf= 2,
    max_samples= 0.6,
    max_features= 0.2,
    max_depth= 26
)

# Fit the model on training data
rForest_tuned_original.fit(x_train, y_train)

#display performance of base model on validation set
print('Original')
pref_rForest_original = model_performance_classification_sklearn(
    model=rForest_original,predictors=x_val,target=y_val)
print(pref_rForest_original)

#display performance of tuned model on validation set
print('Tuned')
pref_rForest_original_tuned=model_performance_classification_sklearn(
    model=rForest_tuned_original,predictors=x_val,target=y_val)
print(pref_rForest_original_tuned)

#display performance of tuned model on training set
print('Tuned Performance Training')
pref_rForest_original_tuned_train=model_performance_classification_sklearn(
    model=rForest_tuned_original,predictors=x_train,target=y_train)
print(pref_rForest_original_tuned_train)

%%time
# defining model
Model = GradientBoostingClassifier(random_state=1)

param_grid = { 
                "n_estimators": np.arange(50,150,10), #100
                "max_features":np.arange(0.5,1.5,0.5), 
                "max_depth":np.arange(1,10,1), #3
                "max_leaf_nodes":np.arange(1,10,1),
                "learning_rate": np.arange(0.05,0.5,0.05), #0.1
                "subsample":np.arange(0.1,1.5,0.1),
             }

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(
    estimator=Model, param_distributions=param_grid, n_iter=5, n_jobs = -1, scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(x_train,y_train)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

#Build Base Model
GBM_original= GradientBoostingClassifier(random_state=1)

# Fit the model on training data
GBM_original.fit(x_train, y_train)

# Creating new model with best parameters
GBM_tuned_original = GradientBoostingClassifier(
    subsample= 0.7, 
    n_estimators= 75, 
    max_leaf_nodes= 5, 
    max_features= 0.5, 
    max_depth= 3,
    learning_rate= 0.1
)

# Fit the model on training data
GBM_tuned_original.fit(x_train, y_train)

#display performance of base model on validation set
print('Original')
pref_GBM_original = model_performance_classification_sklearn(
    model=GBM_original,predictors=x_val,target=y_val)
print(pref_GBM_original)

#display performance of tuned model on validation set
print('Tuned')
pref_GBM_original_tuned=model_performance_classification_sklearn(
    model=GBM_tuned_original,predictors=x_val,target=y_val)
print(pref_GBM_original_tuned)

#display performance of tuned model on training set
print('Tuned Performance Training')
pref_GBM_original_tuned_train=model_performance_classification_sklearn(
    model=GBM_tuned_original,predictors=x_train,target=y_train)
print(pref_GBM_original_tuned_train)

%%time
# defining model
Model = XGBClassifier(random_state=1)

param_grid = { 
                "n_estimators": np.arange(10,150,10), #100
                "max_depth":np.arange(1,10,1), #3
                "learning_rate": np.arange(0.05,0.5,0.05), #0.1
                "subsample":np.arange(0.1,1.5,0.1),
             }

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(
    estimator=Model, param_distributions=param_grid, n_iter=5, n_jobs = -1, scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(x_train,y_train)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

#Build Base Model
XGB_original= XGBClassifier(random_state=1)

# Fit the model on training data
XGB_original.fit(x_train, y_train)

# Creating new model with best parameters
XGB_tuned_original = XGBClassifier(
    subsample= 0.7, 
    n_estimators= 20, 
    max_depth= 9,
    learning_rate= 0.35
)

# Fit the model on training data
XGB_tuned_original.fit(x_train, y_train)

#display performance of base model on validation set
print('Original')
pref_XGB_original = model_performance_classification_sklearn(
    model=XGB_original,predictors=x_val,target=y_val)
print(pref_XGB_original)

#display performance of tuned model on validation set
print('Tuned')
pref_XGB_original_tuned=model_performance_classification_sklearn(
    model=XGB_tuned_original,predictors=x_val,target=y_val)
print(pref_XGB_original_tuned)

#display performance of tuned model on training set
print('Tuned Performance Training')
pref_XGB_original_tuned_train=model_performance_classification_sklearn(
    model=XGB_tuned_original,predictors=x_train,target=y_train)
print(pref_XGB_original_tuned_train)

%%time
# defining model
Model = XGBClassifier(random_state=1)

# Parameter grid to pass in RandomSearchCV
param_grid = { 
                "n_estimators": np.arange(10,150,10), #100
                "max_depth":np.arange(1,10,1), #3
                "learning_rate": np.arange(0.05,0.5,0.01), #0.1
                "subsample":np.arange(0.1,1.5,0.1),
             }

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=Model, param_distributions=param_grid, n_iter=5, n_jobs = -1,
                                   scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(x_train_over,y_train_over)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

#Build base model
XGB_over= XGBClassifier(random_state=1)

# Fit the model on training data
XGB_over.fit(x_train_over, y_train_over)

# Creating new model with best parameters
XGB_tuned_over = XGBClassifier(
    subsample= 0.7, 
    n_estimators= 20, 
    max_depth= 9,
    learning_rate= 0.35
)

# Fit the model on training data
XGB_tuned_over.fit(x_train_over, y_train_over)

#display performance of base model on validation set
print('Original')
pref_XGB_over = model_performance_classification_sklearn(
    model=XGB_over,predictors=x_val,target=y_val)
print(pref_XGB_over)

#display performance of tuned model on validation set
print('Tuned')
pref_XGB_over_tuned=model_performance_classification_sklearn(
    model=XGB_tuned_over,predictors=x_val,target=y_val)
print(pref_XGB_over_tuned)

#display performance of tuned model on training set
print('\nTuned Training Performance')
pref_XGB_over_tuned_train=model_performance_classification_sklearn(
    model=XGB_tuned_over,predictors=x_train_over,target=y_train_over)
print(pref_XGB_over_tuned_train)

#Display the confusion matrix for the random forest model
confusion_matrix_sklearn(model=XGB_tuned_over,predictors=x_val,target=y_val)

%%time
# defining model
Model = XGBClassifier(random_state=1)

# Parameter grid to pass in RandomSearchCV
param_grid = { 
                "subsample":np.arange(0.1,1,0.1), 
                "n_estimators": np.arange(10,150,10), 
                "max_depth":np.arange(1,10,1),
                "learning_rate": np.arange(0.05,0.5,0.05), 
             }

#Calling RandomizedSearchCV
randomized_cv = RandomizedSearchCV(estimator=Model, param_distributions=param_grid, n_iter=5, n_jobs = -1, scoring=scorer, cv=5, random_state=1)

#Fitting parameters in RandomizedSearchCV
randomized_cv.fit(x_train_under,y_train_under)

print("Best parameters are {} with CV score={}:" .format(randomized_cv.best_params_,randomized_cv.best_score_))

#Build base model
XGB_under= XGBClassifier(random_state=1)

# Fit the model on training data
XGB_under.fit(x_train_under, y_train_under)

# Creating new model with best parameters
XGB_tuned_under = XGBClassifier(
    subsample= 0.7, 
    n_estimators= 20, 
    max_depth= 9,
    learning_rate= 0.35
)

# Fit the model on training data
XGB_tuned_under.fit(x_train_under, y_train_under)

#display performance of base model on validation set
print('Original')
pref_XGB_under = model_performance_classification_sklearn(
    model=XGB_under,predictors=x_val,target=y_val)
print(pref_XGB_under)

#display performance of tuned model on validation set
print('Tuned')
pref_XGB_under_tuned=model_performance_classification_sklearn(
    model=XGB_tuned_under,predictors=x_val,target=y_val)
print(pref_XGB_under_tuned)

#display performance of tuned model on training set
print('\nTuned Training Performance')
pref_XGB_under_tuned_train=model_performance_classification_sklearn(
    model=XGB_tuned_under,predictors=x_train_under,target=y_train_under)
print(pref_XGB_under_tuned_train)

#Display the confusion matrix for the random forest model
confusion_matrix_sklearn(model=XGB_tuned_under,predictors=x_val,target=y_val)

# test performance comparison

models_comparison = pd.concat(
    [
        pref_rForest_original.T,
        pref_rForest_original_tuned.T,
        pref_GBM_original.T,
        pref_GBM_original_tuned.T,
        pref_XGB_original.T,
        pref_XGB_original_tuned.T,

        pref_XGB_over.T,
        pref_XGB_over_tuned.T,

        pref_XGB_under.T,
        pref_XGB_under_tuned.T,
    ],
    axis=1,
)

models_comparison.columns = [
    "rForest",
    "rForest tuned",
    "GBM",
    "GBM tuned",
    "XGB",
    "XGB tuned",
    "XGB OV",
    "XGB OV tuned",
    "XGB UN",
    "XGB UN tuned",
]

print("Validation Performance Comparison across the best tuned and untuned models:")
models_comparison

# Separating features and the target column
X = df_modeling.drop("Survived", axis=1)
Y = df_modeling["Survived"]

# create dummies
X = pd.get_dummies(
    X,
    columns=X.select_dtypes(include=["object", "category"]).columns.tolist(),
    drop_first=True
)

# display head
X.head()

model_final=XGBClassifier(
    subsample= 0.7, 
    n_estimators= 20, 
    max_depth= 9,
    learning_rate= 0.35,random_state=1
)
model_final.fit(x_train_over2,y_train_over2)

df_test=data_test.copy()
df_test.drop(['PassengerId','Name','Ticket','Cabin'],axis=1,inplace=True)

# create dummies
df_test = pd.get_dummies(
    df_test,
    columns=df_test.select_dtypes(include=["object", "category"]).columns.tolist(),
    drop_first=True
)

df_test.head()

#Using Final Model
#get predications
pred=model_final.predict(df_test)

#get predicated probability of survival
prob_of_survival=pd.DataFrame(model_final.predict_proba(df_test))[1]

#Make a copy of the test data
df_test2=df_test.copy()

#add a Survived column
df_test2['Survived']=pred

#Add a Probability of survival column
df_test2['prob_of_survival']=prob_of_survival*100

#Display the data head with the new columns
df_test2.head()

#get total number of 1's
print("Total number of surviving passangers in the test data:",len(df_test2[df_test2['Survived']==1]))

#Adjusting the model threshold

#Setting the threshold
threshold=0.74

#Get data frame of passengers likely to survive.
likely_to_survive=pd.DataFrame(df_test2[df_test2['prob_of_survival']>=threshold*100])

#Calculate and display the survivors and the percent of the total.
print("Out of", len(prob_of_survival), "passangers.", len(likely_to_survive), "are likely to survive.")
print(round(len(likely_to_survive)/len(prob_of_survival)*100,2),"% likely to survive.")

#Display list of passengers likely to survive from the test data set.
likely_to_survive.head(10)