9.1 Evaluate the accuracy of regression models.
a) Evaluation on training data.
train = pd.read_csv('/Users/boston_train.csv')
test = pd.read_csv('/Users/boston_test.csv')
# Random Forest Regression Model
from sklearn.ensemble import RandomForestRegressor
rfMod = RandomForestRegressor(random_state=29)
rfMod.fit(train.drop(["Target"], axis = 1), train["Target"])
# Evaluation on training data
predY = rfMod.predict(train.drop(["Target"], axis = 1))
# Determine coefficient of determination score
from sklearn.metrics import r2_score
r2_rf = r2_score(train["Target"], predY)
print("Random forest regression model r^2 score (coefficient of determination): %f" % r2_rf)
## Random forest regression model r^2 score (coefficient of determination): 0.975233b) Evaluation on testing data.
# Random Forest Regression Model (rfMod)
# Evaluation on testing data
predY = rfMod.predict(test.drop(["Target"], axis = 1))
# Determine coefficient of determination score
r2_rf = r2_score(test["Target"], predY)
print("Random forest regression model r^2 score (coefficient of determination): %f" % r2_rf)
## Random forest regression model r^2 score (coefficient of determination): 0.833687The sklearn metric r2_score is only one option for assessing a regression model. Please go here for more information about other sklearn regression metrics.
9.2 Evaluate the accuracy of classification models.
a) Evaluation on training data.
train = pd.read_csv('/Users/digits_train.csv')
test = pd.read_csv('/Users/digits_test.csv')
# Random Forest Classification Model
from sklearn.ensemble import RandomForestClassifier
rfMod = RandomForestClassifier(random_state=29)
rfMod.fit(train.drop(["Target"], axis = 1), train["Target"])
# Evaluation on training data
predY = rfMod.predict(train.drop(["Target"], axis = 1))
# Determine accuracy score
from sklearn.metrics import accuracy_score
accuracy_rf = accuracy_score(train["Target"], predY)
print("Random forest model accuracy: %f" % accuracy_rf)
## Random forest model accuracy: 1.000000
b) Evaluation on testing data.
# Random Forest Classification Model (rfMod)
# Evaluation on testing data
predY = rfMod.predict(test.drop(["Target"], axis = 1))
# Determine accuracy score
accuracy_rf = accuracy_score(test["Target"], predY)
print("Random forest model accuracy: %f" % accuracy_rf)
Note: The sklearn metric accuracy_score is only one option for assessing a classification model. Please go here for more information about other sklearn classification metrics.
9.3 Evaluation with cross validation.
a) KFold
# Notice we are using a new data set that need to be read into the
# environment
breastcancer = pd.read_csv('/Users/breastcancer.csv')
from sklearn import model_selection
from sklearn.ensemble import RandomForestClassifier
X = breastcancer.drop(["Target"], axis = 1)
Y = breastcancer["Target"]
kfold = model_selection.KFold(n_splits = 5, random_state = 29)
rfMod = RandomForestClassifier(random_state = 29)
results = model_selection.cross_val_score(rfMod, X, Y, cv = kfold)
print("Accuracy: %.2f%% +/- %.2f%%" % (results.mean()*100,
results.std()*100))
b) ShuffleSplit
shuffle = model_selection.ShuffleSplit(n_splits = 5, random_state = 29)
rfMod = RandomForestClassifier(random_state = 29)
results = model_selection.cross_val_score(rfMod, X, Y, cv = shuffle)
print("Accuracy: %.2f%% +/- %.2f%%" % (results.mean()*100,
results.std()*100))