1. Clase de perceptrón multicapa
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace MulLinearRegression
{
public class Net
{
/// <summary>
/// 激活类型
/// </summary>
public enum ACTIVE_TYPE
{
X,
SIGMOID,
TANH,
ReLU,
Sign
}
/// <summary>
/// 损失函数类型
/// </summary>
public enum COST_TYPE
{
MIN_LEAST_SQUARE,
MAX_LIKELIHOOD,
CORSS_ENTROY,
}
/// <summary>
/// 感知机层类型
/// </summary>
public enum LAYER_TYPE
{
I_LAY,
H_LAY,
O_LAY
}
/// <summary>
/// 感知机层
/// </summary>
internal class Layer
{
public int NeuronNum;
public int WeightNum;
public double[,] Weights;
public double[,] Biases;
public double[,] PreActSet;
public double[,] ActedSet;
public List<double[,]> DeltaWeights=new List<double[,]>();
public List<double[,]> DeltaBiases=new List<double[,]>();
public double[,] FakeWeights;
public double[,] FakeBiases;
public LAYER_TYPE LayType;
public int AccumTimes = 0;
public Layer()
{
}
public void Create(int neuNum,int weiNum,LAYER_TYPE lType)
{
this.NeuronNum= neuNum;
this.WeightNum= weiNum;
this.LayType= lType;
this.Weights = new double[this.NeuronNum, this.WeightNum];
this.Biases = new double[1, this.NeuronNum];
ParamsRandom();
}
private void ParamsRandom()
{
for (int i = 0; i < this.NeuronNum; i++)
{
this.Biases[0, i] = new Random().NextDouble();
for (int j = 0; j < this.WeightNum; j++)
{
this.Weights[i, j] = new Random().NextDouble();
}
}
}
public void WeightedSum(double[,] iDatas)
{
this.PreActSet = Matrix.MultiplyMatrix(iDatas, Matrix.Transpose(this.Weights));
double[,] strBias = Matrix.VectorStretch(this.Biases, this.PreActSet.GetLength(0));
this.PreActSet = Matrix.AddMatrix(this.PreActSet, strBias);
}
public void Active(ACTIVE_TYPE aType)
{
switch (aType)
{
case ACTIVE_TYPE.SIGMOID:
{
this.ActedSet = Matrix.MatrixSigmoid(this.PreActSet,Grad:false);
break;
}
case ACTIVE_TYPE.TANH:
{
this.ActedSet = Matrix.MatrixTanh(this.PreActSet, Grad: false);
break;
}
case ACTIVE_TYPE.ReLU:
{
this.ActedSet = Matrix.MatrixReLU(this.PreActSet, Grad: false);
break;
}
}
}
public void DeltaAccumulate(double[,] dWeig, double[,] dBias)
{
this.DeltaWeights.Add((double[,])dWeig.Clone());
this.DeltaBiases.Add((double[,])dBias.Clone());
}
public void Update(double eta)
{
double[,] avgWeights = new double[0, 0], avgBiases = new double[0, 0];
for (int i = 0; i < this.DeltaWeights.Count; i++)
{
if (i == 0)
{
avgWeights = (double[,])this.DeltaWeights[i].Clone();
avgBiases = (double[,])this.DeltaBiases[i].Clone();
}
else
{
avgWeights = Matrix.AddMatrix(avgWeights, this.DeltaWeights[i]);
avgBiases = Matrix.AddMatrix(avgBiases, this.DeltaBiases[i]);
}
if (i == this.DeltaWeights.Count)
{
avgWeights = Matrix.MultiplyConst(avgWeights, 1.0 / (this.DeltaWeights.Count));
avgBiases = Matrix.MultiplyConst(avgBiases, 1.0 / (this.DeltaWeights.Count));
}
}
this.DeltaWeights.Clear();
this.DeltaBiases.Clear();
this.Weights = Matrix.AddMatrix(this.Weights,Matrix.MultiplyConst(avgWeights,-1.0*eta));
this.Biases = Matrix.AddMatrix(this.Biases, Matrix.MultiplyConst(avgBiases, -1.0*eta));
}
}
public double[,] DataSet;//样本数据
public double[,] Labels;//标签
public int HiddenLayerNum;//隐藏层数量
public int[] HiddNeuronsNumPerLay;//隐藏层神经元数量
private Layer[] net;//多层感知机
public int Iteration;//迭代次数
public double eta;//学习率
private ACTIVE_TYPE HiddenActType;
private ACTIVE_TYPE OutActType;
private int LayerNumber;//感知机层数量
public double[,] Predicts;//预测结果
private COST_TYPE CostType;
private int SampleNum;
private double epsilon=1e-5;
private List<double> TotalCost=new List<double>();
/// <summary>
/// 创建神经网络
/// </summary>
/// <param name="datas">样本数据集</param>
/// <param name="labels">标签</param>
/// <param name="hiddNum">隐藏层数量</param>
/// <param name="hiddPerLay">每层隐藏层神经元数量</param>
/// <param name="iter">迭代次数</param>
/// <param name="lr">学习率</param>
/// <param name="hiddType">隐藏层激活函数类型</param>
/// <param name="outType">输出层激活函数类型</param>
/// <param name="costType">损失函数类型</param>
public void Create(double[,] datas, double[,] labels, int hiddNum, int[] hiddPerLay,int iter,double lr,
ACTIVE_TYPE hiddType,ACTIVE_TYPE outType,COST_TYPE costType)
{
this.DataSet= datas;
this.Labels= labels;
this.SampleNum = this.Labels.GetLength(0);
this.HiddenLayerNum= hiddNum;
this.LayerNumber = this.HiddenLayerNum + 2;
this.HiddNeuronsNumPerLay= hiddPerLay;
this.Iteration= iter;
this.eta= lr;
this.OutActType= outType;
this.HiddenActType = hiddType;
this.net = new Layer[this.LayerNumber];
this.CostType = costType;
this.Initial();
}
/// <summary>
/// 神经网络初始化，搭建框架，权重系数、偏置系数随机生成0-1
/// </summary>
private void Initial()
{
int l = 0;
while (l < this.LayerNumber)
{
this.net[l] = new Layer();
if (l == 0)
{
this.net[l].ActedSet = (double[,])this.DataSet.Clone();
this.net[l].NeuronNum = this.net[l].ActedSet.GetLength(1);
}
else if(l!=this.LayerNumber-1)
{
this.net[l].Create(this.HiddNeuronsNumPerLay[l - 1], this.net[l - 1].NeuronNum, LAYER_TYPE.H_LAY);
}
else
{
this.net[l].Create(this.Labels.GetLength(1), this.net[l - 1].NeuronNum, LAYER_TYPE.O_LAY);
}
l++;
}
}
/// <summary>
/// 正向传播
/// </summary>
private void FowardPropgation()
{
int l = 1;
while (l < this.LayerNumber )
{
this.net[l].WeightedSum(this.net[l - 1].ActedSet);
this.net[l].Active(l != this.LayerNumber - 1 ? this.HiddenActType : this.OutActType);
l++;
}
this.Predicts = (double[,])this.net[this.LayerNumber - 1].ActedSet.Clone();
}
/// <summary>
/// 损失函数梯度
/// </summary>
/// <param name="activations"></param>
/// <param name="labels"></param>
/// <param name="costType"></param>
/// <returns></returns>
private double[,] CostGrad(double[,] activations,double[,] labels,COST_TYPE costType)
{
int r = activations.GetLength(0);
int c = activations.GetLength(1);
double[,] ret = new double[r, c];
switch (costType)
{
case COST_TYPE.MIN_LEAST_SQUARE:
{
for (int i = 0; i < r; i++)
{
for (int j = 0; j < c; j++)
{
ret[i, j] = 2.0 * (activations[i, j] - labels[i, j]);
}
}
break;
}
case COST_TYPE.MAX_LIKELIHOOD:
{
break;
}
case COST_TYPE.CORSS_ENTROY:
{
break;
}
}
return ret;
}
/// <summary>
/// 激活函数梯度
/// </summary>
/// <param name="activations"></param>
/// <param name="actType"></param>
/// <returns></returns>
private double[,] ActiveGrad(double[,] activations,ACTIVE_TYPE actType)
{
int r = activations.GetLength(0);
int c = activations.GetLength(1);
double[,] ret = new double[r, c];
switch (this.OutActType)
{
case ACTIVE_TYPE.SIGMOID:
{
ret = Matrix.MatrixSigmoid(activations, Grad: true);
break;
}
case ACTIVE_TYPE.TANH:
{
ret = Matrix.MatrixTanh(activations, Grad: true);
break;
}
case ACTIVE_TYPE.ReLU:
{
ret = Matrix.MatrixReLU(activations, Grad: true);
break;
}
}
return ret;
}
/// <summary>
/// 反向传播
/// </summary>
private void BackPropgation()
{
double[,] dc_dz;
double[,] dz_dw;
double[,] dc_da_1=new double[0,0];
int l = this.LayerNumber - 1;
while (l >= 1)
{
if (l == this.LayerNumber - 1)
{
dc_dz = CostGrad(this.net[l].ActedSet, this.Labels, costType: this.CostType);
dc_dz = Matrix.HadamardProduct(dc_dz, ActiveGrad(this.net[l].ActedSet, this.OutActType));
}
else
{
double[,] da_dz = ActiveGrad(this.net[l].ActedSet, this.HiddenActType);
dc_dz = Matrix.HadamardProduct(dc_da_1, da_dz);
}
dc_da_1 = new double[0, 0];
dz_dw = this.net[l - 1].ActedSet;
for (int r = 0; r < this.SampleNum; r++)
{
//更新当前图层每个样本权重及偏置
double[,] dc_dz_col = Matrix.Transpose(Matrix.SubMatrix(dc_dz, r, RowGet: true));
double[,] dz_dw_row = Matrix.SubMatrix(dz_dw, r, RowGet: true);
double[,] delta_w = Matrix.MultiplyMatrix(dc_dz_col, dz_dw_row);
double[,] delta_b = Matrix.SubMatrix(dc_dz, r, RowGet: true);
this.net[l].DeltaAccumulate(delta_w, delta_b);
//计算每个样本的dC/da(L-1)
this.net[l].FakeWeights = Matrix.AddMatrix(this.net[l].Weights, Matrix.MultiplyConst(this.net[l].DeltaWeights[r], -1.0 * eta));
double[,] dc_da_1_row = Matrix.MultiplyMatrix(Matrix.SubMatrix(dc_dz, r, RowGet: true), this.net[l].FakeWeights);
dc_da_1 = Matrix.MatrixCombine(dc_da_1, dc_da_1_row, RowCombine:true);
}
//更新当前层权重及偏置
this.net[l].Update(this.eta);
l--;
}
}
/// <summary>
/// 损失/代价监控
/// </summary>
private void CostMonitor()
{
switch (this.CostType)
{
case COST_TYPE.MIN_LEAST_SQUARE:
{
double cost = Matrix.Norm(
Matrix.AddMatrix(this.net[this.LayerNumber - 1].ActedSet,
Matrix.MultiplyConst(this.Labels, -1)), NormType.F);
this.TotalCost.Add(cost * cost * (1.0 / this.SampleNum));
break;
}
case COST_TYPE.MAX_LIKELIHOOD:
{
break;
}
case COST_TYPE.CORSS_ENTROY:
{
double cost = 0;
double[,] logActivation = Matrix.MatrixLog(this.net[this.LayerNumber - 1].ActedSet, 2);
int r=logActivation.GetLength(0),c=logActivation.GetLength(1);
logActivation = Matrix.HadamardProduct(logActivation, this.Labels);
for(int i = 0; i < r; i++)
{
for(int j = 0; j < c; j++)
{
cost += logActivation[i, j];
}
}
cost /= ((double)this.SampleNum);
break;
}
}
}
/// <summary>
/// 训练
/// </summary>
public void Training()
{
int it = this.Iteration;
while (it > 0)
{
if(it!=this.Iteration && this.TotalCost.Last() < this.epsilon)
{
break;
}
this.FowardPropgation();
this.CostMonitor();
this.BackPropgation();
it--;
}
this.FowardPropgation();
}
}
}
2. Muestra de prueba de perceptrón multicapa
static void Main(string[] args)
{
double[,] testx = {
{
1, 1, 1 }, {
2, 2, 2 }, {
2, 2, 2 }, {
2, 2, 2 }, {
1, 1, 1 }, {
2, 2, 2 }, {
2, 2, 2 }, {
2, 2, 2 }, {
1, 1, 3 }, {
1,1,3}, {
2, 2, 3 } };
double[,] testLab = {
{
1, 0 }, {
1, 0 }, {
1, 0 }, {
1, 0 }, {
1, 0 }, {
1, 0 }, {
1, 0 }, {
1, 0 }, {
0, 1 }, {
0, 1 }, {
0, 1 } };
//double[,] testLab = { { 1}, { 1 }, { 1 }, { 1 }, { 1 }, { 1 }, { 1}, { 1 }, { 0}, { 0 }, { 0 } };
Net tNet = new Net();
tNet.Create(testx, testLab, 2, new int[] {
8, 8 }, iter: 10000, lr: 0.1, Net.ACTIVE_TYPE.SIGMOID, Net.ACTIVE_TYPE.SIGMOID, Net.COST_TYPE.MIN_LEAST_SQUARE);
tNet.Training();
}