吴达恩机器学习python代码

“#吴达恩机器学习python代码1”
作业代码原文章链接

Coursera-ML-AndrewNg-Notes/code/ex1-linear regression/1.linear_regreesion_v1.ipynb
linear regreesion（线性回归）
¶注意：python版本为3.6， 安装TensorFlow的方法：pip install

tensorflowIn [1]:

    import pandas as pd
    import seaborn as sns
    sns.set(context="notebook", style="whitegrid", palette="dark")
    import matplotlib.pyplot as plt
    import tensorflow as tf
    import numpy as np

In [2]:df = pd.read_csv('ex1data1.txt', names=['population', 'profit'])#读取数据并赋予列名

In [3]:`df.head()#看前五行

In [5]:`sns.lmplot('population', 'profit', df, size=6, fit_reg=False)`

plt.show()

 In [6]:def get_X(df):#读取特征

“”"

use concat to add intersect feature to avoid side effect

not efficient for big dataset though

“”"

ones = pd.DataFrame({'ones': np.ones(len(df))})#ones是m行1列的dataframe
data = pd.concat([ones, df], axis=1)  # 合并数据，根据列合并
return data.iloc[:, :-1].as_matrix()  # 这个操作返回 ndarray,不是矩阵


def get_y(df):#读取标签

‘’‘assume the last column is the target’’’

return np.array(df.iloc[:, -1])#df.iloc[:, -1]是指df的最后一列


def normalize_feature(df):

“”“Applies function along input axis(default 0) of DataFrame.”""

return df.apply(lambda column: (column - column.mean()) / column.std())#特征缩放

多变量的假设 h 表示为：[{{h}{\theta }}\left( x \right)={{\theta }{0}}+{{\theta }{1}}{{x}{1}}+{{\theta }{2}}{{x}{2}}+…+{{\theta }{n}}{{x}{n}}] 这个公式中有n+1个参数和n个变量，为了使得公式能够简化一些，引入，则公式转化为：
此时模型中的参数是一个n+1维的向量，任何一个训练实例也都是n+1维的向量，特征矩阵X的维度是 m*(n+1)。 因此公式可以简化为：，其中上标T代表矩阵转置。In [7]:def linear_regression(X_data, y_data, alpha, epoch, optimizer=tf.train.GradientDescentOptimizer):# 这个函数是旧金山的一个大神Lucas Shen写的

# placeholder for graph input
X = tf.placeholder(tf.float32, shape=X_data.shape)
y = tf.placeholder(tf.float32, shape=y_data.shape)

# construct the graph
with tf.variable_scope('linear-regression'):
    W = tf.get_variable("weights",
                        (X_data.shape[1], 1),
                        initializer=tf.constant_initializer())  # n*1

    y_pred = tf.matmul(X, W)  # m*n @ n*1 -> m*1

    loss = 1 / (2 * len(X_data)) * tf.matmul((y_pred - y), (y_pred - y), transpose_a=True)  # (m*1).T @ m*1 = 1*1

opt = optimizer(learning_rate=alpha)
opt_operation = opt.minimize(loss)

# run the session
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    loss_data = []

    for i in range(epoch):
        _, loss_val, W_val = sess.run([opt_operation, loss, W], feed_dict={X: X_data, y: y_data})
        loss_data.append(loss_val[0, 0])  # because every loss_val is 1*1 ndarray

        if len(loss_data) > 1 and np.abs(loss_data[-1] - loss_data[-2]) < 10 ** -9:  # early break when it's converged
            # print('Converged at epoch {}'.format(i))
            break

# clear the graph
tf.reset_default_graph()
return {'loss': loss_data, 'parameters': W_val}  # just want to return in row vector format

In [8]:data = pd.read_csv(‘ex1data1.txt’, names=[‘population’, ‘profit’])#读取数据，并赋予列名

data.head()#看下数据前5行
Out[8]:populationprofit06.110117.592015.52779.130228.518613.662037.003211.854045.85986.8233计算代价函数¶其中：[{{h}{\theta }}\left( x \right)={{\theta }^{T}}X={{\theta }{0}}{{x}{0}}+{{\theta }{1}}{{x}{1}}+{{\theta }{2}}{{x}{2}}+…+{{\theta }{n}}{{x}_{n}}]In [9]:X = get_X(data)
print(X.shape, type(X))

y = get_y(data)
print(y.shape, type(y))
#看下数据维度
(97, 2) <class ‘numpy.ndarray’>
(97,) <class ‘numpy.ndarray’>
In [10]:theta = np.zeros(X.shape[1])#X.shape[1]=2,代表特征数n
In [11]:def lr_cost(theta, X, y):

“”"

X: R(m*n), m 样本数, n 特征数

y: R(m)

theta : R(n), 线性回归的参数

“”"

m = X.shape[0]#m为样本数

inner = X @ theta - y  # R(m*1)，X @ theta等价于X.dot(theta)

# 1*m @ m*1 = 1*1 in matrix multiplication
# but you know numpy didn't do transpose in 1d array, so here is just a
# vector inner product to itselves
square_sum = inner.T @ inner
cost = square_sum / (2 * m)

return cost

In [12]:lr_cost(theta, X, y)#返回theta的值
Out[12]:32.072733877455676batch gradient decent（批量梯度下降）¶ In [13]:def gradient(theta, X, y):
m = X.shape[0]

inner = X.T @ (X @ theta - y)  # (m,n).T @ (m, 1) -> (n, 1)，X @ theta等价于X.dot(theta)

return inner / m

In [14]:def batch_gradient_decent(theta, X, y, epoch, alpha=0.01):

拟合线性回归，返回参数和代价

epoch: 批处理的轮数

“”"

cost_data = [lr_cost(theta, X, y)]
_theta = theta.copy()  # 拷贝一份，不和原来的theta混淆

for _ in range(epoch):
    _theta = _theta - alpha * gradient(_theta, X, y)
    cost_data.append(lr_cost(_theta, X, y))

return _theta, cost_data

#批量梯度下降函数
In [15]:epoch = 500
final_theta, cost_data = batch_gradient_decent(theta, X, y, epoch)
In [16]:final_theta
#最终的theta
Out[16]:array([-2.28286727, 1.03099898])In [19]:cost_data

看下代价数据

Out[19]:[32.072733877455676,
6.7371904648700109,
5.9315935686049563,
5.9011547070813881,
5.8952285864442207,
5.8900949431173304,
5.8850041584436461,
5.8799324804914175,
5.8748790947625729,
5.8698439118063854,
5.8648268653129305,
5.8598278899321805,
5.8548469205722897,
5.849883892376587,
5.8449387407220339,
5.8400114012183613,
5.8351018097072282,
5.8302099022613882,
5.825335615183862,
5.8204788850070992,
5.8156396484921542,
5.8108178426278689,
5.8060134046300451,
5.8012262719406298,
5.7964563822268991,
5.7917036733806526,
5.7869680835173956,
5.7822495509755392,
5.7775480143155962,
5.7728634123193823,
5.7681956839892123,
5.7635447685471197,
5.7589106054340471,
5.7542931343090782,
5.7496922950486287,
5.7451080277456841,
5.7405402727090138,
5.7359889704623814,
5.7314540617437917,
5.7269354875047016,
5.7224331889092568,
5.7179471073335293,
5.7134771843647494,
5.7090233618005488,
5.7045855816481987,
5.7001637861238557,
5.6957579176518154,
5.6913679188637518,
5.6869937325979798,
5.6826353018987072,
5.6782925700152918,
5.6739654804015052,
5.6696539767147964,
5.6653580028155526,
5.6610775027663793,
5.6568124208313595,
5.6525627014753335,
5.6483282893631808,
5.6441091293590917,
5.6399051665258559,
5.6357163461241351,
5.6315426136117717,
5.6273839146430547,
5.6232401950680257,
5.6191114009317777,
5.6149974784737404,
5.6108983741269842,
5.6068140345175319,
5.602744406463648,
5.5986894369751594,
5.5946490732527598,
5.5906232626873233,
5.5866119528592151,
5.5826150915376234,
5.5786326266798527,
5.5746645064306781,
5.5707106791216434,
5.5667710932704031,
5.5628456975800509,
5.558934440938442,
5.5550372724175441,
5.5511541412727547,
5.5472849969422562,
5.5434297890463515,
5.5395884673868077,
5.5357609819462041,
5.5319472828872751,
5.5281473205522715,
5.5243610454623058,
5.5205884083167129,
5.5168293599924025,
5.5130838515432252,
5.5093518341993333,
5.5056332593665385,
5.5019280786256992,
5.4982362437320651,
5.4945577066146658,
5.490892419375677,
5.4872403342898046,
5.483601403803652,
5.479975580535112,
5.476362817272741,
5.472763066975153,
5.4691762827703982,
5.465602417955358,
5.462041425995138,
5.4584932605224585,
5.4549578753370467,
5.451435224405051,
5.4479252618584244,
5.4444279419943333,
5.4409432192745655,
5.4374710483249329,
5.4340113839346866,
5.4305641810559191,
5.4271293948029848,
5.4237069804519171,
5.4202968934398381,
5.4168990893643816,
5.4135135239831227,
5.4101401532129882,
5.4067789331296936,
5.4034298199671653,
5.4000927701169754,
5.3967677401277676,
5.3934546867046969,
5.3901535667088618,
5.3868643371567471,
5.3835869552196609,
5.3803213782231776,
5.3770675636465821,
5.3738254691223171,
5.37059505243543,
5.3673762715230238,
5.364169084473712,
5.3609734495270676,
5.3577893250730844,
5.3546166696516337,
5.3514554419519165,
5.3483056008119441,
5.345167105217981,
5.3420399143040287,
5.3389239873512837,
5.3358192837876031,
5.3327257631869891,
5.329643385269053,
5.3265721098984988,
5.3235118970845869,
5.3204627069806296,
5.3174244998834608,
5.3143972362329235,
5.3113808766113557,
5.3083753817430726,
5.3053807124938608,
5.3023968298704647,
5.2994236950200815,
5.2964612692298516,
5.2935095139263604,
5.2905683906751291,
5.2876378611801176,
5.2847178872832314,
5.2818084309638111,
5.2789094543381534,
5.2760209196590013,
5.2731427893150702,
5.2702750258305437,
5.267417591864592,
5.2645704502108854,
5.2617335637971108,
5.2589068956844836,
5.2560904090672738,
5.2532840672723218,
5.250487833758565,
5.2477016721165555,
5.2449255460679938,
5.2421594194652528,
5.2394032562909016,
5.2366570206572511,
5.2339206768058695,
5.2311941891071285,
5.2284775220597366,
5.2257706402902713,
5.2230735085527291,
5.2203860917280558,
5.2177083548236967,
5.2150402629731367,
5.2123817814354494,
5.2097328755948435,
5.2070935109602079,
5.2044636531646722,
5.2018432679651498,
5.1992323212418956,
5.1966307789980615,
5.1940386073592597,
5.1914557725731072,
5.1888822410088018,
5.1863179791566756,
5.1837629536277587,
5.1812171311533488,
5.1786804785845755,
5.1761529628919671,
5.1736345511650219,
5.1711252106117822,
5.168624908558404,
5.1661336124487311,
5.1636512898438749,
5.1611779084217888,
5.1587134359768489,
5.1562578404194337,
5.1538110897755045,
5.1513731521861974,
5.1489439959073957,
5.1465235893093233,
5.1441119008761387,
5.1417088992055149,
5.1393145530082336,
5.1369288311077792,
5.1345517024399312,
5.1321831360523653,
5.1298231011042379,
5.1274715668657986,
5.1251285027179803,
5.1227938781520068,
5.1204676627689922,
5.1181498262795433,
5.1158403385033679,
5.1135391693688836,
5.1112462889128247,
5.1089616672798481,
5.1066852747221496,
5.1044170815990766,
5.1021570583767355,
5.0999051756276197,
5.0976614040302106,
5.0954257143686092,
5.0931980775321497,
5.09097846451502,
5.0887668464158873,
5.0865631944375176,
5.0843674798864011,
5.0821796741723864,
5.079999748808298,
5.0778276754095675,
5.0756634256938717,
5.0735069714807555,
5.0713582846912679,
5.0692173373475962,
5.0670841015727053,
5.0649585495899681,
5.0628406537228097,
5.0607303863943418,
5.0586277201270109,
5.0565326275422313,
5.0544450813600363,
5.0523650543987193,
5.0502925195744792,
5.0482274499010735,
5.0461698184894566,
5.0441195985474412,
5.0420767633793417,
5.0400412863856268,
5.038013141062577,
5.0359923010019392,
5.0339787398905793,
5.0319724315101411,
5.0299733497367072,
5.0279814685404549,
5.025996761985323,
5.0240192042286669,
5.0220487695209268,
5.0200854322052928,
5.018129166717368,
5.0161799475848339,
5.0142377494271289,
5.0123025469551052,
5.0103743149707087,
5.0084530283666435,
5.0065386621260481,
5.0046311913221766,
5.00273059111806,
5.0008368367661991,
4.9989499036082261,
4.9970697670745947,
4.9951964026842566,
4.9933297860443435,
4.9914698928498451,
4.9896166988833013,
4.9877701800144782,
4.9859303122000602,
4.9840970714833324,
4.9822704339938717,
4.9804503759472336,
4.978636873644648,
4.9768299034726979,
4.9750294419030316,
4.9732354654920341,
4.9714479508805409,
4.9696668747935222,
4.967892214039785,
4.9661239455116677,
4.9643620461847444,
4.9626064931175193,
4.9608572634511301,
4.9591143344090529,
4.9573776832968033,
4.9556472875016393,
4.9539231244922703,
4.9522051718185605,
4.9504934071112396,
4.9487878080816099,
4.9470883525212574,
4.9453950183017596,
4.9437077833743981,
4.9420266257698762,
4.9403515235980295,
4.9386824550475357,
4.9370193983856412,
4.9353623319578688,
4.9337112341877427,
4.9320660835764993,
4.9304268587028188,
4.9287935382225347,
4.9271661008683632,
4.925544525449622,
4.9239287908519627,
4.9223188760370791,
4.9207147600424523,
4.9191164219810677,
4.9175238410411399,
4.9159369964858524,
4.9143558676530761,
4.9127804339551107,
4.9112106748784097,
4.9096465699833125,
4.9080880989037841,
4.9065352413471484,
4.904987977093815,
4.903446285997032,
4.9019101479826102,
4.9003795430486665,
4.8988544512653656,
4.897334852774657,
4.8958207277900163,
4.894312056596192,
4.8928088195489439,
4.8913109970747897,
4.8898185696707479,
4.8883315179040903,
4.8868498224120795,
4.8853734639017246,
4.8839024231495243,
4.8824366810012201,
4.8809762183715479,
4.8795210162439853,
4.8780710556705085,
4.8766263177713407,
4.8751867837347129,
4.8737524348166099,
4.8723232523405349,
4.8708992176972616,
4.869480312344594,
4.8680665178071223,
4.8666578156759863,
4.8652541876086328,
4.863855615328573,
4.8624620806251588,
4.8610735653533252,
4.8596900514333718,
4.858311520850715,
4.8569379556556642,
4.8555693379631784,
4.8542056499526396,
4.8528468738676169,
4.8514929920156389,
4.8501439867679608,
4.8487998405593355,
4.8474605358877847,
4.8461260553143699,
4.8447963814629684,
4.8434714970200456,
4.8421513847344286,
4.8408360274170814,
4.8395254079408829,
4.8382195092404041,
4.8369183143116823,
4.8356218062120035,
4.8343299680596772,
4.8330427830338261,
4.8317602343741575,
4.8304823053807446,
4.8292089794138171,
4.8279402398935405,
4.8266760702997979,
4.8254164541719788,
4.8241613751087611,
4.8229108167678989,
4.8216647628660114,
4.8204231971783704,
4.819186103538688,
4.8179534658389018,
4.8167252680289776,
4.8155014941166856,
4.8142821281674033,
4.8130671543039014,
4.8118565567061413,
4.8106503196110664,
4.809448427312395,
4.8082508641604242,
4.8070576145618169,
4.8058686629794032,
4.8046839939319756,
4.8035035919940903,
4.8023274417958657,
4.8011555280227798,
4.7999878354154761,
4.7988243487695552,
4.7976650529353906,
4.7965099328179184,
4.795358973376449,
4.7942121596244665,
4.7930694766294364,
4.7919309095126117,
4.790796443448837,
4.7896660636663553,
4.7885397554466147,
4.7874175041240843,
4.7862992950860539,
4.7851851137724477,
4.7840749456756351,
4.7829687763402395,
4.7818665913629541,
4.7807683763923503,
4.7796741171286952,
4.7785837993237585,
4.7774974087806346,
4.7764149313535507,
4.7753363529476918,
4.7742616595190066,
4.773190837074031,
4.7721238716697076,
4.7710607494131958,
4.770001456461701,
4.7689459790222868,
4.7678943033516976,
4.7668464157561825,
4.7658023025913154,
4.7647619502618133,
4.7637253452213626,
4.7626924739724474,
4.7616633230661636,
4.7606378791020525,
4.7596161287279255,
4.7585980586396799,
4.7575836555811408,
4.7565729063438775,
4.755565797767038,
4.7545623167371724,
4.7535624501880669,
4.7525661851005694,
4.7515735085024247,
4.7505844074680983,
4.7495988691186195,
4.7486168806214,
4.7476384291900784,
4.7466635020843446,
4.745692086609786,
4.7447241701177063,
4.7437597400049727,
4.7427987837138508,
4.7418412887318322,
4.7408872425914828,
4.7399366328702737,
4.7389894471904226,
4.7380456732187284,
4.7371052986664157,
4.7361683112889716,
4.7352346988859892,
4.7343044493010051,
4.7333775504213413,
4.7324539901779517,
4.7315337565452618,
4.7306168375410103,
4.7297032212260985,
4.7287928957044265,
4.7278858491227513,
4.7269820696705187,
4.7260815455797172,
4.7251842651247218,
4.7242902166221432,
4.7233993884306775,
4.7225117689509464,
4.7216273466253593,
4.7207461099379495,
4.7198680474142325,
4.7189931476210534,
4.7181213991664395,
4.7172527906994519,
4.7163873109100365,
4.7155249485288744,
4.7146656923272445,
4.7138095311168664]In [17]:# 计算最终的代价
lr_cost(final_theta, X, y)
Out[17]:4.7138095311168664visualize cost data（代价数据可视化）¶In [18]:ax = sns.tsplot(cost_data, time=np.arange(epoch+1))
ax.set_xlabel(‘epoch’)
ax.set_ylabel(‘cost’)
plt.show()
#可以看到从第二轮代价数据变换很大，接下来平稳了
In [20]:b = final_theta[0] # intercept，Y轴上的截距
m = final_theta[1] # slope，斜率

plt.scatter(data.population, data.profit, label=“Training data”)
plt.plot(data.population, data.population*m + b, label=“Prediction”)
plt.legend(loc=2)
plt.show()
3- 选修章节¶In [22]:raw_data = pd.read_csv(‘ex1data2.txt’, names=[‘square’, ‘bedrooms’, ‘price’])
raw_data.head()
Out[22]:squarebedroomsprice021043399900116003329900224003369000314162232000430004539900标准化数据¶最简单的方法是令：其中 是平均值，sn 是标准差。In [23]:def normalize_feature(df):

“”“Applies function along input axis(default 0) of DataFrame.”""

return df.apply(lambda column: (column - column.mean()) / column.std())

In [24]:data = normalize_feature(raw_data)
data.head()
Out[24]:squarebedroomsprice00.130010-0.2236750.4757471-0.504190-0.223675-0.08407420.502476-0.2236750.2286263-0.735723-1.537767-0.86702541.2574761.0904171.5953892. multi-var batch gradient decent（多变量批量梯度下降）¶In [25]:X = get_X(data)
print(X.shape, type(X))

y = get_y(data)
print(y.shape, type(y))#看下数据的维度和类型
(47, 3) <class ‘numpy.ndarray’>
(47,) <class ‘numpy.ndarray’>
In [26]:alpha = 0.01#学习率
theta = np.zeros(X.shape[1])#X.shape[1]：特征数n
epoch = 500#轮数
In [27]:final_theta, cost_data = batch_gradient_decent(theta, X, y, epoch, alpha=alpha)
In [31]:sns.tsplot(time=np.arange(len(cost_data)), data = cost_data)
plt.xlabel(‘epoch’, fontsize=18)
plt.ylabel(‘cost’, fontsize=18)
plt.show()
In [29]:final_theta
Out[29]:array([ -1.20848957e-16, 8.30383883e-01, 8.23982853e-04])3. learning rate（学习率）¶In [30]:base = np.logspace(-1, -5, num=4)
candidate = np.sort(np.concatenate((base, base*3)))
print(candidate)
[ 1.00000000e-05 3.00000000e-05 2.15443469e-04 6.46330407e-04
4.64158883e-03 1.39247665e-02 1.00000000e-01 3.00000000e-01]
In [32]:epoch=50

fig, ax = plt.subplots(figsize=(16, 9))

for alpha in candidate:
_, cost_data = batch_gradient_decent(theta, X, y, epoch, alpha=alpha)
ax.plot(np.arange(epoch+1), cost_data, label=alpha)

ax.set_xlabel(‘epoch’, fontsize=18)
ax.set_ylabel(‘cost’, fontsize=18)
ax.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
ax.set_title(‘learning rate’, fontsize=18)
plt.show()
4. normal equation（正规方程）¶正规方程是通过求解下面的方程来找出使得代价函数最小的参数的： 。 假设我们的训练集特征矩阵为 X（包含了）并且我们的训练集结果为向量 y，则利用正规方程解出向量 。 上标T代表矩阵转置，上标-1 代表矩阵的逆。设矩阵，则：梯度下降与正规方程的比较：梯度下降：需要选择学习率α，需要多次迭代，当特征数量n大时也能较好适用，适用于各种类型的模型正规方程：不需要选择学习率α，一次计算得出，需要计算，如果特征数量n较大则运算代价大，因为矩阵逆的计算时间复杂度为O(n3)，通常来说当n小于10000 时还是可以接受的，只适用于线性模型，不适合逻辑回归模型等其他模型In [42]:# 正规方程
def normalEqn(X, y):
theta = np.linalg.inv(X.T@X)@X.T@y#X.T@X等价于X.T.dot(X)
return theta
In [37]:final_theta2=normalEqn(X, y)#感觉和批量梯度下降的theta的值有点差距
final_theta2
Out[37]:array([ -1.14491749e-16, 8.84765988e-01, -5.31788197e-02])run the tensorflow graph over several optimizer¶In [38]:X_data = get_X(data)
print(X_data.shape, type(X_data))

y_data = get_y(data).reshape(len(X_data), 1) # special treatment for tensorflow input data
print(y_data.shape, type(y_data))
(47, 3) <class ‘numpy.ndarray’>
(47, 1) <class ‘numpy.ndarray’>
In [39]:epoch = 2000
alpha = 0.01
In [40]:optimizer_dict={‘GD’: tf.train.GradientDescentOptimizer,
‘Adagrad’: tf.train.AdagradOptimizer,
‘Adam’: tf.train.AdamOptimizer,
‘Ftrl’: tf.train.FtrlOptimizer,
‘RMS’: tf.train.RMSPropOptimizer
}
results = []
for name in optimizer_dict:
res = linear_regression(X_data, y_data, alpha, epoch, optimizer=optimizer_dict[name])
res[‘name’] = name
results.append(res)
画图¶In [41]:fig, ax = plt.subplots(figsize=(16, 9))

for res in results:
loss_data = res[‘loss’]

print(‘for optimizer {}’.format(res[‘name’]))

print(‘final parameters\n’, res[‘parameters’])

print(‘final loss={}\n’.format(loss_data[-1]))

ax.plot(np.arange(len(loss_data)), loss_data, label=res['name'])

ax.set_xlabel(‘epoch’, fontsize=18)
ax.set_ylabel(‘cost’, fontsize=18)
ax.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
ax.set_title(‘different optimizer’, fontsize=18)
plt.show()