CS229 Machine Learning作业代码:Problem Set 1

牛顿法求解二分类逻辑回归参数

Repeat{

\(\theta:=\theta-H^{-1}\nabla_\theta l(\theta)\)

}

其中，Hessian矩阵\(H\in \mathbb R^{(n+1)\times (n+1)}\)

\[(H)_{i,j}=\frac {\partial^2J}{\partial \theta_i\partial \theta_j}\]

具体公式推导过程如下

logistic_reg.m

clear;
close all;

%Plot the data
X=load("logistic_x.txt");
y=load("logistic_y.txt");

pos=find(y==1);
neg=find(y==-1);
figure();
plot(X(pos,1),X(pos,2),'bx');
hold on;
plot(X(neg,1),X(neg,2),'yo');
m=size(X,1);
n=size(X,2);
y(neg)=0;
X=[ones(m,1),X];

%logistic regression using Newton method
theta=zeros(n+1,1);

for it=1:10
    nabla=zeros(n+1,1);
    H=zeros(n+1,n+1); %hessian matrix
    J=0;
    for t=1:m
        tmp=sigmoid(X(t,:)*theta);
        J=J-(y(t)*log(tmp)+(1-y(t))*log(1-tmp));
    end
    J
    for t=1:m
        for i=1:n+1
            tmp=sigmoid(X(t,:)*theta);
            nabla(i)=nabla(i)+(tmp-y(t))*X(t,i);
        end
    end
    for t=1:m
        tmp=sigmoid(X(t,:)*theta);
        for i=1:n+1
            for j=1:n+1
                if(i>=j)
                    H(i,j)=H(i,j)+tmp*(1-tmp)*X(t,i)*X(t,j);
                else
                    H(i,j)=H(j,i);
                end
            end
        end
    end
    theta=theta-inv(H)*nabla;

    plotBoundary(X,theta);
end

plotBoundary.m

function plotBoundary(X,theta)
    plotx=[min(X(:,2))-2,max(X(:,2))+2];
    ploty=-(theta(1)+theta(2).*plotx)./theta(3);
    plot(plotx,ploty,'-');
end

sigmoid.m

function ans=sigmoid(x)
    ans=1.0/(1.0+exp(-x));
end

运行结果