RANDOM FOREST - PYSPARK¶
Configuraciones iniciales e importación de librerías¶
Configuración de sesión de Spark
In [10]:
import findspark
findspark.init()
In [11]:
from pyspark.sql import SparkSession
In [12]:
spark = SparkSession.builder.getOrCreate()
LibrerÃas a usar
In [13]:
#Pyspark libraries/modules
from pyspark.sql import *
from pyspark.sql import Window
from pyspark.sql import functions as F
from pyspark.sql.functions import row_number, monotonically_increasing_id
from pyspark.sql.types import StringType, DoubleType, IntegerType, ArrayType, DateType
#Python libraries/modules
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")
from scipy import stats
from scipy.stats import kstest
#Pyspark Machine Learning libraries/modules
from pyspark.mllib.stat import Statistics
from pyspark.ml import Pipeline
from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler, MinMaxScaler
from pyspark.ml.classification import RandomForestClassifier, GBTClassifier, LogisticRegression
from pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.stat import Statistics
import re
Caso a tratar¶
En esta oportunidad vamos a retomar el caso publicado anteriormente en el que realice un análisis exploratorio de una base de posibles clientes bancarios.
Si no lo viste te invito a pasar por el notebook anterior.
Ahora que la base se encuentra limpia procedemos a aplicar un modelo de clasificación Random
Pasos Iniciales¶
In [14]:
pd_df = pd.read_excel('bank_full_final.xlsx')
df = spark.createDataFrame(pd_df)
In [15]:
df = df.drop('Target_no','Target_yes')
In [16]:
print('filas',df.count(),len(df.columns),"columnas")
In [17]:
df.printSchema()
Codificación de variables Targets
In [18]:
df = df.withColumn('Target',F.when(F.col('Target') == 'no',0)
.otherwise(1))
Separación Train/Test (Hold-out)
In [19]:
train, test = df.randomSplit([0.80,0.20])
#test = test.drop('Target')
In [20]:
print('Filas training set {}'.format(train.count()))
print('Filas validation set {}'.format(test.count()))
In [21]:
print('Filas training set {}'.format(len(train.columns)))
print('Filas validation set {}'.format(len(test.columns)))
Selección de Features
In [22]:
feature = [c for c,t in df.dtypes if t != 'string']
In [23]:
feature.remove('Target')
feature
Out[23]:
In [24]:
from pyspark.ml.feature import VectorAssembler
assembler = VectorAssembler(inputCols=feature, outputCol='features')
train = assembler.transform(train)
test = assembler.transform(test)
In [25]:
train.select('features').take(1)
Out[25]:
In [26]:
test.show(1,vertical = True)
In [27]:
train.select('features').show(2)
In [28]:
from pyspark.ml.stat import Correlation
matrix = Correlation.corr(train.select('features'), 'features')
matrix_np = matrix.collect()[0]["pearson({})".format('features')].values
In [41]:
import seaborn as sns
matrix_np = matrix_np.reshape(len(feature),len(feature))
fig, ax = plt.subplots(figsize=(20,20))
ax = sns.heatmap(matrix_np, cmap="YlGnBu")
ax.xaxis.set_ticklabels(feature, rotation=270)
ax.yaxis.set_ticklabels(feature, rotation=0)
ax.set_title("Correlation Matrix")
plt.tight_layout()
plt.show()
RANDOM FOREST¶
In [30]:
from pyspark.ml.classification import RandomForestClassifier
rf = RandomForestClassifier(labelCol='Target', featuresCol='features', numTrees=50)
In [31]:
rf = rf.fit(train)
In [32]:
pred = rf.transform(test)
In [33]:
pred.select('Target', 'prediction', 'probability').show(5)
In [34]:
pred_pd = pred.select(['Target', 'prediction', 'probability']).toPandas()
In [35]:
pred_pd.head()
Out[35]:
In [36]:
import numpy as np
pred_pd['probability'] = pred_pd['probability'].map(lambda x: list(x))
pred_pd['encoded_Target'] = pred_pd['Target'].map(lambda x: np.eye(2)[int(x)])
In [37]:
y_pred = np.array(pred_pd['probability'].tolist())
y_true = np.array(pred_pd['encoded_Target'].tolist())
Performance Metric¶
In [38]:
from sklearn.metrics import auc, roc_curve
fpr, tpr, threshold = roc_curve(y_score=y_pred[:,0], y_true=y_true[:,0])
auc = auc(fpr, tpr)
print('AUC: {:.3f}'.format(auc))
In [45]:
plt.figure()
plt.subplots(figsize=(15,15))
plt.plot([0,1], [0,1], 'k--', color='orange')
plt.plot(fpr, tpr, label='auc = {:.3f}'.format(auc))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('Curva ROC')
plt.legend(loc='lower right')
plt.grid()
plt.show()
