Supervised Local Linear Embedding Algorithm (SLLE Algorithm)

% SLLE ALGORITHM (using K nearest neighbors)
%
% [Y] = lle(X,K,dmax,a)
%
% X = data as D x N matrix (D = dimensionality, N = #points)
% K = number of neighbors
% dmax = max embedding dimensionality
% Y = embedding as dmax x N matrix
% a=increment factor

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

function [Y] = lle(X,K,d,a)

[D,N] = size(X);
fprintf(1,'SLLE running on %d points in %d dimensions\n',N,D);


% STEP1: COMPUTE PAIRWISE DISTANCES & FIND NEIGHBORS
fprintf(1,'-->Finding %d nearest neighbours.\n',K);

X2 = sum(X.^2,1);
distance = repmat(X2,N,1)+repmat(X2',1,N)-2*X'*X;
B=ones(N);
R=N/(d+1);
for i=1:d+1;
    B(1+R*(i-1):R*i,1+R*(i-1):R*i)=zeros(R);
end;
distance1=distance+a*max(max(distance))*B;

[sorted,index] = sort(distance1);
neighborhood = index(2:(1+K),:);

 

% STEP2: SOLVE FOR RECONSTRUCTION WEIGHTS
fprintf(1,'-->Solving for reconstruction weights.\n');

if(K>D)
  fprintf(1,'   [note: K>D; regularization will be used]\n');
  tol=1e-3; % regularlizer in case constrained fits are ill conditioned
else
  toll=0;
end;
toll=1e-3;
W = zeros(K,N);
for ii=1:N
   z = X(:,neighborhood(:,ii))-repmat(X(:,ii),1,K); % shift ith pt to origin
   C = z'*z;                                        % local covariance
   C = C + eye(K,K)*tol*trace(C);                   % regularlization (K>D)
   W(:,ii) = C\ones(K,1);                           % solve Cw=1
   W (:, ii) = W (:, ii) / sum (W (:, ii)); % enforce sum (w) = 1
end;


% STEP 3: COMPUTE EMBEDDING FROM EIGENVECTS OF COST MATRIX M=(I-W)'(I-W)
fprintf(1,'-->Computing embedding.\n');

% M=eye(N,N); % use a sparse matrix with storage for 4KN nonzero elements
M = sparse(1:N,1:N,ones(1,N),N,N,4*K*N);
for ii=1:N
   w = W (:, ii);
   jj = neighborhood (:, ii);
   M (ii, jj) = M (ii, jj) - w ';
   M (jj, ii) = M (jj, ii) - w;
   M (jj, jj) = M (jj, jj) + w * w ';
end;

% CALCULATION OF EMBEDDING
options.disp = 0; options.isreal = 1; options.issym = 1;
[Y,eigenvals] = eigs(M,d+1,0,options);
Y = Y(:,1:d)'*sqrt(N); % bottom evect is [1,1,1,1...] with eval 0


fprintf(1,'Done.\n');

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


% other possible regularizers for K>D
%   C = C + tol*diag(diag(C));                       % regularlization
%   C = C + eye(K,K)*tol*trace(C)*K;                 % regularlization

　　

Test case (Swiss roll, looks delicious):

clear all,clc

N = 2000;

K = 12;
d = 3;
a=0;

% Plot true manfold

tt0 = (3 * pi / 2) * (1 + 2 * [0: 0.02: 1]); hh = [0: 0.125: 1] * 30;

xx  = (tt0.*cos(tt0))'*ones(size(hh));

yy  = ones(size(tt0))'*hh;

zz  = (tt0.*sin(tt0))'*ones(size(hh));

cc  = tt0'*ones(size(hh));

subplot(1,3,1); cla;

surf(xx,yy,zz,cc);

view([12 20]); grid off; axis off; hold on;

lnx = -5 * [3.3.3; 3, -4.3]; lny = [0.0.0; 32.0.0]; lnz = -5 * [3.3.3; 3.3, -3];

lnh=line(lnx,lny,lnz);

set(lnh,'Color',[1,1,1],'LineWidth',2,'LineStyle','-','Clipping','off');

axis([-15,20,0,32,-15,15]);

 

%generate sample data

tt = (3*pi/2)*(1+2*rand(1,N));

height = 21*rand(1,N);

X  = [tt.*cos(tt); height; tt.*sin(tt)];

%scatter plot of sampled data

subplot(1,3,2); cla;

scatter3(X(1,:),X(2,:),X(3,:),12,tt,'+');

view([12 20]); grid off; axis off; hold on;

lnh=line(lnx,lny,lnz);

set(lnh,'Color',[1,1,1],'LineWidth',2,'LineStyle','-','Clipping','off');

axis([-15,20,0,32,-15,15]); drawnow;

 

%run LLE algorithm

Y=lle(X,K,d);

 

%scatterplot of embedding

subplot(1,3,3); cla;

scatter(Y(1,:),Y(2,:),12,tt,'+');

grid off;

set(gca,'XTick',[]); set(gca,'YTick',[]);