Scikit-Learn 机器学习笔记 – 线性回归、逻辑回归、softmax回归
import numpy as np
from matplotlib import pyplot as plt
def create_dataset():
X = 2 * np.random.rand(100, 1)
y = 4 + 3*X + np.random.randn(100, 1)
return X, y
def linear_regression_analysis(X, y):
X_b = np.c_[np.ones((100, 1)), X]
theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y)
print("线性回归解析解为:", theta_best)
sample = np.array([[0], [2]])
sample_b = np.c_[np.ones((2, 1)), sample]
predict = sample_b.dot(theta_best)
plt.plot(sample, predict, 'r-')
plt.plot(X, y, 'b.')
plt.axis([0, 2, 0, 15])
plt.show()
return X_b
def linear_regression_sk(X, y):
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(X, y)
print('sk-learn线性回归解析解:', 'b:', lin_reg.intercept_, 'w:', lin_reg.coef_)
def linear_regression_batch_gd(X_b, y):
learning_rate = 0.1
max_iterations = 1000
m = 100
theta = np.random.randn(2, 1)
for n in range(max_iterations):
gradients = 2/m * X_b.T.dot(X_b.dot(theta)-y)
theta = theta - learning_rate*gradients
print('线性回归批量梯度下降法解:', theta)
def linear_regression_stochastic_gd(X_b, y):
n_epochs = 50
m = 100
theta = np.random.randn(2, 1)
for epoch in range(n_epochs):
for i in range(m):
random_index = np.random.randint(m)
xi = X_b[random_index:random_index+1]
yi = y[random_index:random_index+1]
gradients = 2 * xi.T.dot(xi.dot(theta) - yi)
learning_rate = 1.0/(epoch*m + i + 10)
theta = theta - learning_rate*gradients
print('线性回归随机梯度下降法解:', theta)
def linear_regression_stochastic_gd_sk(X, y):
from sklearn.linear_model import SGDRegressor
sgd_reg = SGDRegressor(n_iter=50, penalty=None, eta0=0.1)
sgd_reg.fit(X, y.ravel())
print('sk-learn线性回归随机梯度下降法解:', 'b:', sgd_reg.intercept_, 'w:', sgd_reg.coef_)
def create_dataset_poly():
m = 100
X1 = 6 * np.random.rand(m, 1) - 3
y1 = 0.5 * X1 ** 2 + X1 + 2 + np.random.randn(m, 1)
return X1, y1
def polynomial_regression(X, y):
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
poly_features = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly_features.fit_transform(X)
lin_reg_poly = LinearRegression()
lin_reg_poly.fit(X_poly, y)
print('多项式回归解:', 'b:', lin_reg_poly.intercept_, 'w:', lin_reg_poly.coef_)
return lin_reg_poly
def plot_learning_curves(model, X, y):
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)
train_errors, val_errors = [], []
for m in range(1, len(X_train)):
model.fit(X_train[:m], y_train[:m])
y_train_predict = model.predict(X_train[:m])
y_val_predict = model.predict(X_val)
train_errors.append(mean_squared_error(y_train_predict, y_train[:m]))
val_errors.append(mean_squared_error(y_val_predict, y_val))
plt.plot(np.sqrt(train_errors), "r-+", linewidth=2, label="train")
plt.plot(np.sqrt(val_errors), "b-", linewidth=3, label="val")
plt.show()
def ridge_regression_analysis(X, y):
from sklearn.linear_model import Ridge
ridge_reg = Ridge(alpha=1, solver="cholesky")
ridge_reg.fit(X, y)
print('岭回归解:', 'b:', ridge_reg.intercept_, 'w:', ridge_reg.coef_)
def lasso_regression_analysis(X, y):
from sklearn.linear_model import Lasso
lasso_reg = Lasso(alpha=0.1)
lasso_reg.fit(X, y)
print('Lasso 回归解:', 'b:', lasso_reg.intercept_, 'w:', lasso_reg.coef_)
def regularization_regression_gd(X, y):
from sklearn.linear_model import SGDRegressor
sgd_reg = SGDRegressor(penalty="l2")
sgd_reg.fit(X, y.ravel())
print('l2梯度下降法解:', 'b:', sgd_reg.intercept_, 'w:', sgd_reg.coef_)
def elasticnet_regression_gd(X, y):
from sklearn.linear_model import ElasticNet
elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5)
elastic_net.fit(X, y)
print('弹性网络解:', 'b:', elastic_net.intercept_, 'w:', elastic_net.coef_)
def early_stoping(X, y):
from sklearn.base import clone
from sklearn.linear_model import SGDRegressor
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
sgd_reg = SGDRegressor(max_iter=1, warm_start=True, penalty=None, learning_rate="constant", eta0=0.0005)
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)
minimum_val_error = float("inf")
best_epoch = None
best_model = None
for epoch in range(1000):
sgd_reg.fit(X_train, y_train.ravel())
y_val_predict = sgd_reg.predict(X_val)
val_error = mean_squared_error(y_val_predict, y_val)
if val_error < minimum_val_error:
minimum_val_error = val_error
best_epoch = epoch
best_model = clone(sgd_reg)
print('stopping in:', best_epoch)
def load_dataset_flower():
from sklearn import datasets
iris = datasets.load_iris()
return iris
def logistic_classify(iris):
from sklearn.linear_model import LogisticRegression
X = iris["data"][:, 3:]
y = (iris["target"] == 2).astype(np.int)
log_reg = LogisticRegression()
log_reg.fit(X, y)
X_new = np.linspace(0, 3, 1000).reshape(-1, 1)
y_proba = log_reg.predict_proba(X_new)
plt.plot(X_new, y_proba[:, 1], "g-", label="Iris-Virginica")
plt.plot(X_new, y_proba[:, 0], "b--", label="Not Iris-Virginica")
plt.show()
def softmax_classify(iris):
from sklearn.linear_model import LogisticRegression
X = iris["data"][:, (2, 3)]
y = iris["target"]
softmax_reg = LogisticRegression(multi_class="multinomial", solver="lbfgs", C=10)
softmax_reg.fit(X, y)
predict = softmax_reg.predict([[5, 2]])
predict_pro = softmax_reg.predict_proba([[5, 2]])
print('softmax回归预测为:', predict, '各类概率为', predict_pro)
if __name__ == '__main__':
X, y = create_dataset()
iris = load_dataset_flower()
logistic_classify(iris)
softmax_classify(iris)