【日常】手写三层反向传播神经网络（损失函数交叉熵+正则项+反向求导）

课程的一次作业，虽然没什么用，但是手写一遍dense确实能加深对神经网络的理解，尤其是反向传播求导这一块。

资源已经上传，不过最近CSDN犯病不能改资源积分了。留个BDY链接了?

#-*- coding:UTF-8 -*-
import numpy as np
import pandas as pd
from scipy.io import loadmat
from scipy.linalg import norm
import matplotlib.pyplot as plt

""" 百度云链接：https://pan.baidu.com/s/1kdSoUcK9PFjUEfRiUI9pdw """
""" 密码：iycw """

"""
	Nesterov's方法是先根据历史信息走到一个点,
	再根据那个点的gradient来走一段更新;
	这恰好与动量方法的次序是相反的,
	动量方法是先根据当前点的gradient来走一段更新,
	然后再根据历史信息往前面搞一段距离;
"""
def train(wd,n_hidden,n_iters,learning_rate,momentum_mul,do_early_stopping=False,minibatch_size=10,isNestrov=False):
	"""
		· wd: 权重衰减
		· n_hidden: 隐层结点数量
		· n_iters: 随机梯度下降迭代次数
		· learning_rate: 学习速率
		· momentum_mul: 速率衰减系数(这个系数将附加在前一次的动量上,然后瞎搞)
		· do_early_stopping: 是否提早结束（如果是则简单的输出过去epoch中最优的那个）
		· minibatch_size: 随机梯度下降的小批尺寸
		· inNestrov: 是否使用Nestrov方法
		· return: 数据集上的分类损失
	"""
	data_file = loadmat("data.mat",squeeze_me=True,struct_as_record=False)
	data = data_file["data"]											 # 读取数据
	"""
		· data.training.inputs —— 256×1000
		· data.training.targets —— 10×1000
		· data.validation.inputs —— 256×1000
		· data.validation.targets —— 10×1000
		· data.test.inputs —— 256×9000
		· data.test.targets —— 10×9000
	"""
	data_train = {"X":data.training.inputs,"y":data.training.targets}
	data_valid = {"X":data.validation.inputs,"y":data.validation.targets}
	data_test = {"X":data.test.inputs,"y":data.test.targets}
	n_train = data_train["X"].shape[1]									 # 训练集样本数量							 
	params = initial_model(n_hidden)									 # 初始化两个权重矩阵的参数
	theta = model2theta(params)											 # 将两个矩阵压扁拼接成向量
	test_gradient(params,data_train,wd,n_hidden)						 # 检测梯度是否有问题
	v = 0																 # 初始化速率
	loss_train = []														 # 储存训练时的损失函数值
	loss_valid = []														 # 储存验证时的损失函数值
	best = {}															 # 储存最优参数
	if do_early_stopping:												 # 提前结束
		best["theta"] = 0
		best["loss_valid"] = np.inf
		best["iter"] = -1

	for t in range(n_iters+1):											 # 随机梯度下降迭代算法
		batch_start = (t*minibatch_size) % n_train						 # 每次取一定规模的样本来搞搞
		data_batch = {
			"X": data_train["X"][:,batch_start:batch_start+minibatch_size],
			"y": data_train["y"][:,batch_start:batch_start+minibatch_size],
		}
		if isNestrov:													 # Nestrov's方法
			temp = theta + momentum_mul*v								 # 先照着前一次的情况再苟一段距离
			loss,grad = eval_obj_grad(theta2model(temp),data_batch,wd,n_hidden)
			grad_vec = model2theta(grad)								 # 梯度转向量
			v = momentum_mul*v - grad_vec								 # 得到本次的实际下降方向
			theta -= learning_rate*v									 # 找到更新后的参数值

		else:															 # 传统动量
			loss,grad = eval_obj_grad(theta2model(theta),data_batch,wd,n_hidden)
			grad_vec = model2theta(grad)								 # 梯度转向量
			v = momentum_mul*v - grad_vec								 # 计算经过动量调整后的下降方向
			theta += learning_rate*v									 # 找到更新后的参数值

		params = theta2model(theta)										 # 将调整好的theta转化为params的格式(即两个权重矩阵)
		loss = eval_obj(params,data_train,wd)							 # 计算训练损失
		loss_train.append(loss)											 # 储存训练损失
		loss = eval_obj(params,data_valid,wd)							 # 计算验证损失
		loss_valid.append(loss)											 # 存储验证损失
		if do_early_stopping and loss_valid[-1]<best["loss_valid"]:		 # 如果验证损失比当前最好要小, 则更新best
			best["theta"] = theta.copy()
			best["loss_valid"] = loss_valid[-1]
			best["iter"] = t
		if t%(max(1,n_iters//10))==0:									 # 每过10次迭代将输出信息
			print("After %d iterations - ||theta|| is %.4e - training loss is %.4e - and validation loss is %.4e\n"%(t,norm(theta),loss_train[-1],loss_valid[-1]))
	test_gradient(params,data_train,wd,n_hidden)
	""" 实验绘图 """
	plt.close()
	plt.figure()
	plt.plot(loss_train,label="training loss")
	plt.plot(loss_valid,label="validation loss")
	plt.legend(loc="best")
	plt.show()
	if do_early_stopping:
		print("Early stopping: validation loss: %.3e, was lowest after %d iterations" % (best["loss_valid"],best["iter"]))
		theta = best["theta"]
	params = theta2model(theta)
	""" 检查效果 """
	datasets = [data_train,data_valid,data_test]
	acc = [accuracy(params,x) for x in datasets]
	classification_loss = [eval_obj(params,x,0) for x in datasets]
	print("Accuracy: training %.4f,validation %.4f,testing %.4f" % (acc[0],acc[1],acc[2]))
	info = {
		"loss_train": classification_loss[0],
		"loss_valid": classification_loss[1],
		"loss_test": classification_loss[2],
	}
	print(info)

def eval_obj(params,data,wd):											 # 计算损失函数
	W_hid,W_out = params["W_hid"],params["W_out"]						 # 100×256 10×100
	X,y = data["X"],data["y"]											 # 256×1000 10×1000
	""" 前向传播 """
	hidden_in = np.dot(W_hid,X)
	hidden_out = sigmoid(hidden_in)
	y_hat = np.dot(W_out,hidden_out)
	exps = np.exp(y_hat)
	y_hat = exps/(np.sum(exps,axis=0))									 # softmax输出
	assert y_hat.shape==y.shape											 # 断言
	loss = 0
	for i in range(y_hat.shape[1]):
		loss -= np.dot(y[:,i].T,np.log(y_hat[:,i]))
	loss /= y_hat.shape[1]
	loss += wd/2*(np.linalg.norm(W_hid,ord="fro")**2+np.linalg.norm(W_out,ord="fro")**2)
	return loss

def eval_obj_grad(params,data,wd,n_hidden):								 # 计算模型的损失函数与梯度
	W_hid,W_out = params["W_hid"],params["W_out"]						 # 100×256 10×100
	X,y = data["X"],data["y"]											 # 256×1000 10×1000
	m = y.shape[1]														 # 获取样本数量
	""" 前向传播 """
	hidden_in = np.dot(W_hid,X)											 # 计算隐层输入 100×1000
	hidden_out = sigmoid(hidden_in)										 # 计算隐层输出 100×1000
	y_hat = np.dot(W_out,hidden_out)									 # 计算输出层输入 10×1000
	exps = np.exp(y_hat)
	y_hat = exps/(np.sum(exps,axis=0))									 # softmax输出 10×1000
	assert y_hat.shape==y.shape											 # 断言
	loss = 0
	for i in range(y_hat.shape[1]):
		loss -= np.dot(y[:,i].T,np.log(y_hat[:,i]))
	loss /= y_hat.shape[1]
	loss += wd/2*(np.linalg.norm(W_hid,ord="fro")**2+np.linalg.norm(W_out,ord="fro")**2)
	""" 反向传播 """
	dz_2 = y_hat - y													 # 计算预测与实际的误差 10×1000
	grad_W_out = 1/m * np.dot(dz_2,hidden_out.T)						 # 转置很关键 10×100
	dz_1 = np.dot(W_out.T,dz_2) * hidden_out*(1-hidden_out)				 # 对sigmoid激活函数求导 100×1000
	grad_W_hid = 1/m * np.dot(dz_1,X.T)									 # 100×256
	grad = {															 # 返回梯度
		"W_out": grad_W_out + wd*W_out,									 # SGD飞了我才想起来还有正则项的梯度
		"W_hid": grad_W_hid + wd*W_hid,									 # SGD飞了我才想起来还有正则项的梯度
	}
	return loss,grad

def initial_model(n_hid):
	n_params = (256+10)*n_hid											 # 表明 输入->隐层 & 隐层->输出 的两个权重矩阵总的元素数量
	as_row_vector = np.cos(np.arange(n_params))							 # 返回darray([0,1,2,...,n_params-1])
	params = {}
	params["W_hid"] = as_row_vector[:256*n_hid].reshape((n_hid,256))*0.1
	params["W_out"] = as_row_vector[256*n_hid:].reshape((10,n_hid))*0.1
	return params

def test_gradient(params,data,wd,n_hidden):								 # 测试梯度
	loss,analytic_grad = eval_obj_grad(params,data,wd,n_hidden)			 # 获取当前参数的损失函数值及梯度
	num_checks = 100
	theta = model2theta(params)
	grad_ana = model2theta(analytic_grad)
	delta = 1e-4
	threshold = 1e-5
	for i in range(num_checks):
		ind = (i*1299283) % theta.size
		grad_ind_ana = grad_ana[ind]
		theta1 = theta.copy()
		theta1[ind] += delta
		l1 = eval_obj(theta2model(theta1),data,wd)
		theta2 = theta.copy()
		theta2[ind] -= delta
		l2 = eval_obj(theta2model(theta2),data,wd)
		grad_ind_fin = (l1-l2)/(2*delta)
		diff = abs(grad_ind_ana - grad_ind_fin)
		if diff<threshold: continue
		if diff/(abs(grad_ind_ana)+abs(grad_ind_fin))<threshold: continue
		raise AssertionError("%d-th: l %.3e\nl1 %.3e\nl2 %.3e\nanalytic %.3e\nfd %.3e\ndiff %.3e\n" % (i,loss,l1,l2,grad_ind_ana,grad_ind_fin,diff))
	print("Gradient test passed")

def model2theta(params):												 # 将两个权重矩阵参数转化为向量再拼起来
	theta = np.concatenate((params["W_out"].flatten(),params["W_hid"].flatten()))
	return theta

def theta2model(theta):													 # 将向量形式转化为两个权重矩阵输出
	n_hid = theta.size // (256+10)										 # theta.size是theta的向量维度, a//b运算符相当于int(a/b6)
	params = {}
	params["W_out"] = np.reshape(theta[:n_hid*10],(10,n_hid))
	params["W_hid"] = np.reshape(theta[n_hid*10:],(n_hid,256))
	return params

def accuracy(params,data):												 # 计算准确度
	W_hid,W_out = params["W_hid"],params["W_out"]
	index_transpose = np.nonzero(data["y"].T)
	true_label = index_transpose[1]
	a_hidden = W_hid.dot(data["X"])
	h_hidden = sigmoid(a_hidden)
	a_out = W_out.dot(h_hidden)
	pred = a_out.argmax(axis=0)
	return np.mean(pred==true_label)

def log_sum_exp(X):														 # 对矩阵X每列的求以自然底数的指数累和, 对得到的行向量去自然对数
	return np.log(np.sum(np.exp(X),axis=0))
	
def sigmoid(X):															 # 激活函数
	return 1/(1+np.exp(-X))

def grad_sigmoid(X):													 # 激活函数求导
	return sigmoid(X)*(1-sigmoid(X))

def P2Q1():
	print("Part2 Question1...")
	params = [															 # 存一些需要测试的参数
		[0,10,70,0.005,0,False,4],								
		[0,10,70,0.01,0,False,4],									
		[0,10,70,0.05,0,False,4],										
		[0,10,70,0.2,0,False,4],										
		[0,10,70,1.0,0,False,4],
		[0,10,70,5.0,0,False,4],
	]
	for param in params:
		print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(param[0],param[1],param[2],param[3],param[4],param[5],param[6]))
		train(param[0],param[1],param[2],param[3],param[4],param[5],param[6])

def P2Q2():
	print("Part2 Question2...")
	param = [0,10,100,None,None,False,4]
	learning_rates = [0.01,0.05,0.2,1.0,5.0]
	momentums_muls = [0,0.5,0.9]	
	for learning_rate in learning_rates:
		for momentum_mul in momentums_muls:
			tempParam = param[:]
			tempParam[3] = learning_rate
			tempParam[4] = momentum_mul
			print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6]))
			train(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6])

def P2Q3():
	""" 与P2Q1比较 """
	print("Part2 Question3...")
	params = [															 # 存一些需要测试的参数
		[0,10,70,0.005,0,False,4],								
		[0,10,70,0.01,0,False,4],									
		[0,10,70,0.05,0,False,4],										
		[0,10,70,0.2,0,False,4],										
		[0,10,70,1.0,0,False,4],
		[0,10,70,5.0,0,False,4],
	]
	for param in params:
		print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(param[0],param[1],param[2],param[3],param[4],param[5],param[6]))
		train(param[0],param[1],param[2],param[3],param[4],param[5],param[6],True)
	print("###########################################################")
	print("########################  分界线  ##########################")
	print("###########################################################")
	""" 与P2Q2比较 """
	param = [0,10,100,None,None,False,4]
	learning_rates = [0.01,0.05,0.2,1.0,5.0]
	momentums_muls = [0,0.5,0.9]	
	for learning_rate in learning_rates:
		for momentum_mul in momentums_muls:
			tempParam = param[:]
			tempParam[3] = learning_rate
			tempParam[4] = momentum_mul
			print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6]))
			train(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6],True)

def P3Q1():
	print("Part3 Question1...")
	params = [															 # 存一些需要测试的参数
		[0,200,1000,0.2,0.9,False,4],								
		[1e-4,200,1000,0.2,0.9,False,4],								
		[1e-3,200,1000,0.2,0.9,False,4],								
		[1e-2,200,1000,0.2,0.9,False,4],								
		[1e-1,200,1000,0.2,0.9,False,4],								
		[1,200,1000,0.2,0.9,False,4],								
		[10,200,1000,0.2,0.9,False,4],								
	]
	for param in params:
		print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(param[0],param[1],param[2],param[3],param[4],param[5],param[6]))
		train(param[0],param[1],param[2],param[3],param[4],param[5],param[6])

def P3Q2():
	print("Part3 Question2...")
	param = [0,None,1000,0.2,0.9,None,4]
	n_hiddens = [10,50,100,200,300]
	for n_hidden in n_hiddens:
		tempParam = param[:]
		tempParam[1] = n_hidden
		tempParam[5] = True
		print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6]))
		train(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6],True)

		tempParam = param[:]
		tempParam[1] = n_hidden
		tempParam[5] = False
		print("正在测试参数：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {}...".format(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6]))
		train(tempParam[0],tempParam[1],tempParam[2],tempParam[3],tempParam[4],tempParam[5],tempParam[6],True)

def train_1(wd,n_hidden,n_iters,learning_rate,momentum_mul,do_early_stopping=False,minibatch_size=10,isNestrov=False):
	"""
		· wd: 权重衰减
		· n_hidden: 隐层结点数量
		· n_iters: 随机梯度下降迭代次数
		· learning_rate: 学习速率
		· momentum_mul: 速率衰减系数(这个系数将附加在前一次的动量上,然后瞎搞)
		· do_early_stopping: 是否提早结束（如果是则简单的输出过去epoch中最优的那个）
		· minibatch_size: 随机梯度下降的小批尺寸
		· inNestrov: 是否使用Nestrov方法
		· return: 数据集上的分类损失
	"""
	data_file = loadmat("data.mat",squeeze_me=True,struct_as_record=False)
	data = data_file["data"]											 # 读取数据
	"""
		· data.training.inputs —— 256×1000
		· data.training.targets —— 10×1000
		· data.validation.inputs —— 256×1000
		· data.validation.targets —— 10×1000
		· data.test.inputs —— 256×9000
		· data.test.targets —— 10×9000
	"""
	data_train = {"X":data.training.inputs,"y":data.training.targets}
	data_valid = {"X":data.validation.inputs,"y":data.validation.targets}
	data_test = {"X":data.test.inputs,"y":data.test.targets}
	n_train = data_train["X"].shape[1]									 # 训练集样本数量							 
	params = initial_model(n_hidden)									 # 初始化两个权重矩阵的参数
	theta = model2theta(params)											 # 将两个矩阵压扁拼接成向量
	test_gradient(params,data_train,wd,n_hidden)						 # 检测梯度是否有问题
	v = 0																 # 初始化速率
	loss_train = []														 # 储存训练时的损失函数值
	loss_valid = []														 # 储存验证时的损失函数值
	best = {}															 # 储存最优参数
	if do_early_stopping:												 # 提前结束
		best["theta"] = 0
		best["loss_valid"] = np.inf
		best["iter"] = -1

	for t in range(n_iters+1):											 # 随机梯度下降迭代算法
		batch_start = (t*minibatch_size) % n_train						 # 每次取一定规模的样本来搞搞
		data_batch = {
			"X": data_train["X"][:,batch_start:batch_start+minibatch_size],
			"y": data_train["y"][:,batch_start:batch_start+minibatch_size],
		}
		if isNestrov:													 # Nestrov's方法
			temp = theta + momentum_mul*v								 # 先照着前一次的情况再苟一段距离
			loss,grad = eval_obj_grad(theta2model(temp),data_batch,wd,n_hidden)
			grad_vec = model2theta(grad)								 # 梯度转向量
			v = momentum_mul*v - grad_vec								 # 得到本次的实际下降方向
			theta -= learning_rate*v									 # 找到更新后的参数值

		else:															 # 传统动量
			loss,grad = eval_obj_grad(theta2model(theta),data_batch,wd,n_hidden)
			grad_vec = model2theta(grad)								 # 梯度转向量
			v = momentum_mul*v - grad_vec								 # 计算经过动量调整后的下降方向
			theta += learning_rate*v									 # 找到更新后的参数值

		params = theta2model(theta)										 # 将调整好的theta转化为params的格式(即两个权重矩阵)
		loss = eval_obj(params,data_train,wd)							 # 计算训练损失
		loss_train.append(loss)											 # 储存训练损失
		loss = eval_obj(params,data_valid,wd)							 # 计算验证损失
		loss_valid.append(loss)											 # 存储验证损失
		if do_early_stopping and loss_valid[-1]<best["loss_valid"]:		 # 如果验证损失比当前最好要小, 则更新best
			best["theta"] = theta.copy()
			best["loss_valid"] = loss_valid[-1]
			best["iter"] = t
		if t%(max(1,n_iters//10))==0:									 # 每过10次迭代将输出信息
			print("After %d iterations - ||theta|| is %.4e - training loss is %.4e - and validation loss is %.4e\n"%(t,norm(theta),loss_train[-1],loss_valid[-1]))
	test_gradient(params,data_train,wd,n_hidden)
	""" 实验绘图 """
	plt.close()
	plt.figure()
	plt.plot(loss_train,label="training loss")
	plt.plot(loss_valid,label="validation loss")
	plt.legend(loc="best")
	plt.show()
	if do_early_stopping:
		print("Early stopping: validation loss: %.3e, was lowest after %d iterations" % (best["loss_valid"],best["iter"]))
		theta = best["theta"]
	params = theta2model(theta)
	""" 检查效果 """
	datasets = [data_train,data_valid,data_test]
	acc = [accuracy(params,x) for x in datasets]
	classification_loss = [eval_obj(params,x,0) for x in datasets]
	print("Accuracy: training %.4f,validation %.4f,testing %.4f" % (acc[0],acc[1],acc[2]))
	return acc[1]

def train_2(wd,n_hidden,n_iters,learning_rate,momentum_mul,do_early_stopping=False,minibatch_size=10,isNestrov=False):
	"""
		· wd: 权重衰减
		· n_hidden: 隐层结点数量
		· n_iters: 随机梯度下降迭代次数
		· learning_rate: 学习速率
		· momentum_mul: 速率衰减系数(这个系数将附加在前一次的动量上,然后瞎搞)
		· do_early_stopping: 是否提早结束（如果是则简单的输出过去epoch中最优的那个）
		· minibatch_size: 随机梯度下降的小批尺寸
		· inNestrov: 是否使用Nestrov方法
		· return: 数据集上的分类损失
	"""
	data_file = loadmat("data.mat",squeeze_me=True,struct_as_record=False)
	data = data_file["data"]											 # 读取数据
	"""
		· data.training.inputs —— 256×1000
		· data.training.targets —— 10×1000
		· data.validation.inputs —— 256×1000
		· data.validation.targets —— 10×1000
		· data.test.inputs —— 256×9000
		· data.test.targets —— 10×9000
	"""
	data_valid = {"X":data.training.inputs,"y":data.training.targets}
	data_train = {"X":data.validation.inputs,"y":data.validation.targets}
	data_test = {"X":data.test.inputs,"y":data.test.targets}
	n_train = data_train["X"].shape[1]									 # 训练集样本数量							 
	params = initial_model(n_hidden)									 # 初始化两个权重矩阵的参数
	theta = model2theta(params)											 # 将两个矩阵压扁拼接成向量
	test_gradient(params,data_train,wd,n_hidden)						 # 检测梯度是否有问题
	v = 0																 # 初始化速率
	loss_train = []														 # 储存训练时的损失函数值
	loss_valid = []														 # 储存验证时的损失函数值
	best = {}															 # 储存最优参数
	if do_early_stopping:												 # 提前结束
		best["theta"] = 0
		best["loss_valid"] = np.inf
		best["iter"] = -1

	for t in range(n_iters+1):											 # 随机梯度下降迭代算法
		batch_start = (t*minibatch_size) % n_train						 # 每次取一定规模的样本来搞搞
		data_batch = {
			"X": data_train["X"][:,batch_start:batch_start+minibatch_size],
			"y": data_train["y"][:,batch_start:batch_start+minibatch_size],
		}
		if isNestrov:													 # Nestrov's方法
			temp = theta + momentum_mul*v								 # 先照着前一次的情况再苟一段距离
			loss,grad = eval_obj_grad(theta2model(temp),data_batch,wd,n_hidden)
			grad_vec = model2theta(grad)								 # 梯度转向量
			v = momentum_mul*v - grad_vec								 # 得到本次的实际下降方向
			theta -= learning_rate*v									 # 找到更新后的参数值

		else:															 # 传统动量
			loss,grad = eval_obj_grad(theta2model(theta),data_batch,wd,n_hidden)
			grad_vec = model2theta(grad)								 # 梯度转向量
			v = momentum_mul*v - grad_vec								 # 计算经过动量调整后的下降方向
			theta += learning_rate*v									 # 找到更新后的参数值

		params = theta2model(theta)										 # 将调整好的theta转化为params的格式(即两个权重矩阵)
		loss = eval_obj(params,data_train,wd)							 # 计算训练损失
		loss_train.append(loss)											 # 储存训练损失
		loss = eval_obj(params,data_valid,wd)							 # 计算验证损失
		loss_valid.append(loss)											 # 存储验证损失
		if do_early_stopping and loss_valid[-1]<best["loss_valid"]:		 # 如果验证损失比当前最好要小, 则更新best
			best["theta"] = theta.copy()
			best["loss_valid"] = loss_valid[-1]
			best["iter"] = t
		if t%(max(1,n_iters//10))==0:									 # 每过10次迭代将输出信息
			print("After %d iterations - ||theta|| is %.4e - training loss is %.4e - and validation loss is %.4e\n"%(t,norm(theta),loss_train[-1],loss_valid[-1]))
	test_gradient(params,data_train,wd,n_hidden)
	""" 实验绘图 """
	plt.close()
	plt.figure()
	plt.plot(loss_train,label="training loss")
	plt.plot(loss_valid,label="validation loss")
	plt.legend(loc="best")
	plt.show()
	if do_early_stopping:
		print("Early stopping: validation loss: %.3e, was lowest after %d iterations" % (best["loss_valid"],best["iter"]))
		theta = best["theta"]
	params = theta2model(theta)
	""" 检查效果 """
	datasets = [data_train,data_valid,data_test]
	acc = [accuracy(params,x) for x in datasets]
	classification_loss = [eval_obj(params,x,0) for x in datasets]
	print("Accuracy: training %.4f,validation %.4f,testing %.4f" % (acc[0],acc[1],acc[2]))
	return acc[1]

def P3Q3():
	wds = [0,1e-4,1e-3,1e-2,1e-1,1,10]
	n_hiddens = [10,50,100,200,300]
	n_iters = [1000]
	learning_rates = [0.2]
	momentum_muls = [0.9]
	do_early_stoppings = [True,False]
	minibatch_sizes = [4]
	isNestrovs = [False]
	results = {}														 # 存储所有结果
	count = 0															 # 计数
	for wd in wds:
		for n_hidden in n_hiddens:
			for n_iter in n_iters:
				for learning_rate in learning_rates:
					for momentum_mul in momentum_muls:
						for do_early_stopping in do_early_stoppings:
							for minibatch_size in minibatch_sizes:
								for isNestrov in isNestrovs:
									count += 1
									print("###################################################")
									print("################### 第{}个组合 #####################".format(count))
									print("###################################################")
									print("正在测试参数 - 1折：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {} - isNestrov is {}...".format(wd,n_hidden,n_iter,learning_rate,momentum_mul,do_early_stopping,minibatch_size,isNestrov))
									acc1 = train_1(wd,n_hidden,n_iter,learning_rate,momentum_mul,do_early_stopping,minibatch_size,isNestrov)
									print("正在测试参数 - 2折：wd is {} - n_hidden is {} - n_iters is {} - learning_rate is {} - momentum_mul is {} - do_early_stopping is {} - minibatch_size is {} - isNestrov is {}...".format(wd,n_hidden,n_iter,learning_rate,momentum_mul,do_early_stopping,minibatch_size,isNestrov))									
									acc2 = train_2(wd,n_hidden,n_iter,learning_rate,momentum_mul,do_early_stopping,minibatch_size,isNestrov)
									results["{}-{}-{}".format(wd,n_hidden,do_early_stopping)] = (acc1+acc2)/2
	for key,value in results.items():
		print("{}\t平均精度 {}".format(key,value))

if __name__ == "__main__":
	P3Q3()