Python example to implement Levenberg-Marquardt algorithm

This is the first time to write a technical blog post. If there are mistakes, please give pointers.
This blog post summarizes the learning of the LM algorithm through an example, and the programming language is python.
I won't talk about the theory, there are a lot of them on the Internet.
Fits the parameters a, b, c in the function y(x) = exp(a*x^2 + b * x + c) . Without further ado, let’s go directly to the code:

# -*- coding:utf-8 -*-
# autor: HuangYuliang
import numpy as np
from numpy import matrix as mat
from matplotlib import pyplot as plt
import random

n = 100
a1,b1,c1 = 1,3,2 # This is the real parameter of the function y(x) that needs to be fitted
h = np.linspace(0,1,n) # generate data with noise
y = [np.exp(a1*i**2+b1*i+c1)+random.gauss(0,4) for i in h]
y = mat(y) # convert to matrix form
 
def Func(abc,iput): # The function to be fitted, abc is a matrix containing three parameters [[a],[b],[c]]
    a = abc[0,0]
    b = abc[1,0]
    c = abc[2,0]
    return np.exp(a*iput**2+b*iput+c)

def Deriv(abc,iput,n): # Find partial derivatives of functions
    x1 = abc.copy()
    x2 = abc.copy()
    x1[n,0] -= 0.000001
    x2[n,0] += 0.000001
    p1 = Func (x1, iput)
    p2 = Func (x2, iput)
    d = (p2-p1)*1.0/(0.000002)
    return d
         
J = mat(np.zeros((n,3))) #Jacobian matrix
fx = mat(np.zeros((n,1))) # f(x) 100*1 error
fx_tmp = mat(np.zeros((n,1)))
xk = mat([[0.8],[2.7],[1.5]]) # parameter initialization
lase_mse = 0
step = 0
u,v= 1.2
conve = 100

while (conve):
       
    mse,mse_tmp = 0,0
    step += 1  
    for i in range(n):
        fx[i] = Func(xk,h[i]) - y[0,i] # Note that it cannot be written as y - Func , otherwise it will diverge
        mse += fx[i,0]**2
        
        for j in range(3):
            J[i,j] = Deriv(xk,h[i],j) # Numerical derivation                                                    
    mse /= n # range constraints

    H = J.T*J + u*np.eye(3)   # 3*3
    dx = -HI * JT*fx # Note that there is a negative sign here, which corresponds to the sign of fx = Func - y
    xk_tmp = xk.copy()
    xk_tmp += dx
    
    for j in range(n):
        fx_tmp[i] =  Func(xk_tmp,h[i]) - y[0,i]  
        mse_tmp += fx_tmp[i,0]**2
    mse_tmp /= n
     
    q = (mse - mse_tmp)/((0.5*dx.T*(u*dx - J.T*fx))[0,0])
    
    if q > 0:
        s = 1.0/3.0
        v = 2
        mse = mse_tmp
        xk = xk_tmp
        temp = 1 - pow(2*q-1,3)

        if s > temp:
            u = u*s
        else:
            u = u*temp
    else:
        u = u*v
        v = 2*v
        xk = xk_tmp

    print "step = %d,abs(mse-lase_mse) = %.8f" %(step,abs(mse-lase_mse))  
    if abs(mse-lase_mse)<0.000001:
        break
       
    last_mse = mse # record the position of the last mse
    conve -= 1
print xk
 
z = [Func(xk,i) for i in h] #Draw a graph with the fitted parameters

plt.figure(0)
plt.scatter(h,y,s = 4)
plt.plot(h,z,'r')
plt.show()

The fitting effect is shown in the following figure:

The fitting results are as follows:

step = 1,abs(mse-lase_mse) = 6427.823881782
step = 2,abs(mse-lase_mse) = 1348.739146779
step = 3,abs(mse-lase_mse) = 7725.330011166
step = 4,abs(mse-lase_mse) = 51.226834643
step = 5,abs(mse-lase_mse) = 0.013664543
step = 6,abs(mse-lase_mse) = 0.000026830
step = 7,abs(mse-lase_mse) = 0.000000194
参数：[[ 0.97256842]
[ 3.03242402]
[ 1.99001528]]

It can be seen that the algorithm converges after seven iterations. And it is very close to the true value [1, 3, 2].

This blog post references the next blog post:

http://blog.csdn.net/jinshengtao/article/details/53310804

Python example to implement Levenberg-Marquardt algorithm

Guess you like