Express implementation principle learning - simple simulation

This notebook learns the implementation principle of Express by simulating some functions of Express.

Initialize the application

mkdir my-express
cd ./my-express
npm init -y
npm i express

add app.js:

// app.js
const express = require('express')

const app = express()

app.get('/', (req, res) => {
    
    
  res.end('get /')
})

app.get('/about', (req, res) => {
    
    
  res.end('get /about')
})

app.listen(3000, () => {
    
    
  console.log('server is running on http://localhost:3000')
})

source code structure

From the code point of view, the npm package of Express app.jswill be loaded first , and it will look for the field in .require('express')node_modules/express/package.jsonmain

mainThe field specifies the entry file of the module. If there is no mainfield, the file in the root directory of the npm package will be loaded by default index.js.

node_modules/express/index.jsjust imported another module:

'use strict';
module.exports = require('./lib/express');

node_modules/express/lib/express.jsMainly exported a createApplicationmethod

// 只截取了片段
// 主要导出了一个 `createApplication` 方法
exports = module.exports = createApplication;

// 创建并返回 app
function createApplication() {
    
    
  // app 本身是一个函数
  var app = function(req, res, next) {
    
    
    app.handle(req, res, next);
  };

  // 通过 mixin 方法扩展其他成员
  mixin(app, EventEmitter.prototype, false);
  // proto --> require('./application')
  mixin(app, proto, false);

  // 扩展 request
  app.request = Object.create(req, {
    
    
    app: {
    
     configurable: true, enumerable: true, writable: true, value: app }
  })

  // 扩展 response
  app.response = Object.create(res, {
    
    
    app: {
    
     configurable: true, enumerable: true, writable: true, value: app }
  })

  app.init();
  return app;
}

createApplicationThe method creates and returns an app function, which internally mixinextends many other members through the method.

The core thing in the members is that node_modules/express/lib/application.jsan app object is created and returned internally, and then many members are added to this object, such as use, router, listenand so on:

// 只截取了片段
var app = exports = module.exports = {
    
    };

app.use = function use(fn) {
    
    ...}
app.route = function route(path) {
    
    ...}
app.listen = function listen() {
    
    ...}

mixinThe method is to mix the members of application.jsthe returned object into the object created by the appcurrent method .createApplicationapp

Others include:

• Extends requestand responseobjects on top of node's http module: lib/request.jsandlib/response.js
• Some utility functions are built in:lib/utils.js
• Related setting processing of the template engine:lib/view.js
• Related processing of routing:lib/router
• Internal middleware:lib/middleware

Next, imitate the directory structure of Express to implement some content.

Quick experience

Create a directory under the project express, implement the simulated express, and add files under the directory.

// express\index.js
module.exports = require('./lib/express')

// express\lib\express.js
const http = require('http')
const url = require('url')

// 路由表
const routes = [
  // { path: '', method: '', handler: () => { } }
]

function createApplication() {
    
    
  return {
    
    
    get(path, handler) {
    
    
      // 收集路由
      routes.push({
    
    
        path,
        method: 'get',
        handler,
      })
    },
    listen(...args) {
    
    
      const server = http.createServer((req, res) => {
    
    
        /* 接收到请求的时候处理路由 */
        const {
    
     pathname } = url.parse(req.url)
        const method = req.method.toLowerCase()
        const route = routes.find(route => route.path === pathname && route.method === method)
        if (route) {
    
    
          return route.handler(req, res)
        }

        res.end('404 Not Found.')
      })

      server.listen(...args)
    }
  }
}

module.exports = createApplication

Extract the modules that create the App

In order to extend the app conveniently, the internal methods are now extracted into a constructor, and newinstances are created by means of , and these methods are defined as instance methods of App.

// express\lib\application.js
const http = require('http')
const url = require('url')

function App() {
    
    
  // 路由表
  this.routes = []
}

App.prototype.get = function (path, handler) {
    
    
  // 收集路由
  this.routes.push({
    
    
    path,
    method: 'get',
    handler,
  })
}

App.prototype.listen = function (...args) {
    
    
  const server = http.createServer((req, res) => {
    
    
    /* 接收到请求的时候处理路由 */
    const {
    
     pathname } = url.parse(req.url)
    const method = req.method.toLowerCase()
    const route = this.routes.find(route => route.path === pathname && route.method === method)
    if (route) {
    
    
      return route.handler(req, res)
    }

    res.end('404 Not Found.')
  })

  server.listen(...args)
}

module.exports = App

// express\lib\express.js
const App = require('./application')

function createApplication() {
    
    
  return new App()
}

module.exports = createApplication

Extract routing module

source code:node_modules\express\lib\router\index.js

In order to more conveniently develop and manage routing in the app, it is recommended to encapsulate routing-related processing into a separate module.

Similarly, create a Router constructor, and all routing-related operations are performed through Router's routing instance.

// router\index.js
const url = require('url')

function Router() {
    
    
  // 路由记录栈
  this.stack = []
}

Router.prototype.get = function (path, handler) {
    
    
  this.stack.push({
    
    
    path,
    method: 'get',
    handler,
  })
}

Router.prototype.handle = function (req, res) {
    
    
  const {
    
     pathname } = url.parse(req.url)
  const method = req.method.toLowerCase()
  const route = this.stack.find(route => route.path === pathname && route.method === method)
  if (route) {
    
    
    return route.handler(req, res)
  }

  res.end('404 Not Found.')
}

module.exports = Router

The routing table routesis changed to the name in express stack.

// express\lib\application.js
const http = require('http')
const Router = require('./router')

function App() {
    
    
  // 路由表
  this._router = new Router()
}

App.prototype.get = function (path, handler) {
    
    
  this._router.get(path, handler)
}

App.prototype.listen = function (...args) {
    
    
  const server = http.createServer((req, res) => {
    
    
    this._router.handle(req, res)
  })

  server.listen(...args)
}

module.exports = App

Handle different request methods

routing example

// app.js
app.get('/', (req, res) => {
    
    
  res.end('get /')
})

app.get('/about', (req, res) => {
    
    
  res.end('get /about')
})

app.post('/about', (req, res) => {
    
    
  res.end('post /about')
})

app.patch('/about', (req, res) => {
    
    
  res.end('patch /about')
})

app.delete('/about', (req, res) => {
    
    
  res.end('delete /about')
})

Express source code

The request processing logic of each method is the same, but the method name is different, and common request methods can be traversed to add processing logic in batches.

Express internally uses the methods package, which exports common HTTP request methods.

It has only one js file ( node_modules\methods\index.js):

'use strict';
// 导入 nodejs 原生模块 http
var http = require('http');

// 优先 http 中支持的 methods，如果没有则导出手动整理的 methods
module.exports = getCurrentNodeMethods() || getBasicNodeMethods();

// 获取 nodejs 原生模块 http 的 METHODS 属性
// 它标识 node 环境中支持的 HTTP 请求方法
// 将它们遍历小写处理并返回
function getCurrentNodeMethods() {
    
    
  return http.METHODS && http.METHODS.map(function lowerCaseMethod(method) {
    
    
    return method.toLowerCase();
  });
}

// 手动整理 methods
function getBasicNodeMethods() {
    
    
  return [
    'get',
    'post',
    'put',
    'head',
    'delete',
    'options',
    'trace',
    'copy',
    'lock',
    'mkcol',
    'move',
    'purge',
    'propfind',
    'proppatch',
    'unlock',
    'report',
    'mkactivity',
    'checkout',
    'merge',
    'm-search',
    'notify',
    'subscribe',
    'unsubscribe',
    'patch',
    'search',
    'connect'
  ];
}

Simulation implementation

You can use the methods module directly:

// express\lib\application.js
const http = require('http')
const Router = require('./router')
const methods = require('methods')

function App() {
    
    
  // 路由表
  this._router = new Router()
}

// 为每个请求方法添加处理函数
methods.forEach(method => {
    
    
  App.prototype[method] = function (path, handler) {
    
    
    this._router[method](path, handler)
  }
})

App.prototype.listen = function (...args) {
    
    
  const server = http.createServer((req, res) => {
    
    
    this._router.handle(req, res)
  })

  server.listen(...args)
}

module.exports = App

// router\index.js
const url = require('url')
const methods = require('methods')

function Router() {
    
    
  // 路由记录栈
  this.stack = []
}

// 为每个请求方法添加处理函数
methods.forEach(method => {
    
    
  Router.prototype[method] = function (path, handler) {
    
    
    this.stack.push({
    
    
      path,
      method,
      handler,
    })
  }
})

Router.prototype.handle = function (req, res) {
    
    
  const {
    
     pathname } = url.parse(req.url)
  const method = req.method.toLowerCase()
  const route = this.stack.find(route => route.path === pathname && route.method === method)
  if (route) {
    
    
    return route.handler(req, res)
  }

  res.end('404 Not Found.')
}

module.exports = Router

Note: Currently, these third-party packages can be used directly because express is installed in this project, so its dependencies are installed. If you want to release your own implementation of express separately, you should install dependencies in the directory of this package:

# 进入到模拟 express 的模块目录下
cd express
npm init -y
# 安装依赖
npm i methods
# 建议安装和 Express 一样的版本
# npm i [email protected]

Now, when loading methods in express (simulated), the ones /expressunder the current directory ( ) will node_modules/methodsbe loaded first.

Implement more powerful routing path matching mode

source code:node_modules\express\lib\router\layer.js

Express internally supports multiple matching modes for routing paths: Route paths

routing example

// app.js
app.get('/', function (req, res) {
    
    
  res.end('root')
})

app.get('/about', function (req, res) {
    
    
  res.end('about')
})

app.get('/random.text', function (req, res) {
    
    
  res.end('random.text')
})

app.get('/ab?cd', function (req, res) {
    
    
  res.end('ab?cd')
})

app.get('/ab+cd', function (req, res) {
    
    
  res.end('ab+cd')
})

app.get('/ab*cd', function (req, res) {
    
    
  res.end('ab*cd')
})

app.get('/ab(cd)?e', function (req, res) {
    
    
  res.end('ab(cd)?e')
})

app.get(/a/, function (req, res) {
    
    
  res.end('/a/')
})

app.get(/.*fly$/, function (req, res) {
    
    
  res.end('/.*fly$/')
})

app.get('/users/:userId/books/:bookId', function (req, res) {
    
    
  res.end(req.params)
})

PS: The method is a method extended by response.send()Express on the basis of the nodejs object. It has not been defined yet, and there is no method itself , so the method used herehttp.ServerResponsehttp.ServerResponse.send()res.end()

Express source code

The module used internally by Express path-to-regexpimplements route matching and can resolve dynamic parameters.

// node_modules\express\lib\router\layer.js
var pathRegexp = require('path-to-regexp');

this.regexp = pathRegexp(path, this.keys = [], opts);

Simulation implementation

install dependencies

# 在 /express 目录下执行
# 注意：path-to-regexp 最新版已经不支持 Express 4 了，要安装老版本，版本号参考 Express 包下的 package.json
npm i [email protected]

processing path

// router\index.js
const url = require('url')
const methods = require('methods')
const pathRegexp = require('path-to-regexp')

function Router() {
    
    
  // 路由记录栈
  this.stack = []
}

// 为每个请求方法添加处理函数
methods.forEach(method => {
    
    
  Router.prototype[method] = function (path, handler) {
    
    
    this.stack.push({
    
    
      path,
      method,
      handler,
    })
  }
})

Router.prototype.handle = function (req, res) {
    
    
  const {
    
     pathname } = url.parse(req.url)
  const method = req.method.toLowerCase()
  const route = this.stack.find(route => {
    
    
    const keys = []
    // 基于配置的路由匹配规则，生成一个正则表达式
    const regexp = pathRegexp(route.path, keys, {
    
    })
    // 匹配实际请求路径的结果
    const match = regexp.exec(pathname)
    return match && route.method === method
  })

  if (route) {
    
    
    return route.handler(req, res)
  }

  res.end('404 Not Found.')
}

module.exports = Router

Handling dynamic path parameters

Assuming access to '/users/:userId/books/:bookId'routing address: http://localhost:3000/users/tom/books/123.

path-to-regexpDynamic parameters will be parsed, and the parsed parameter names will be filled into the incoming keysarray. The parameter values ​​can be viewed through the regular matching results:

const keys = []
const regexp = pathRegexp(route.path, keys, {
    
    })
const match = regexp.exec(pathname)
console.log('keys =>',keys)
console.log('match =>',match)

Print result:

keys => [
  {
    
     name: 'userId', optional: false, offset: 8 },
  {
    
     name: 'bookId', optional: false, offset: 30 }
]
match => [
  '/users/tom/books/123',
  'tom',
  '123',
  index: 0,
  input: '/users/tom/books/123',
  groups: undefined
]

So you can map the parameter name and parameter value together and mount it req.paramson :

// router\index.js
const route = this.stack.find(route => {
    
    
  const keys = []
  // 基于配置的路由匹配规则，生成一个正则表达式
  const regexp = pathRegexp(route.path, keys, {
    
    })
  // 匹配实际请求路径的结果
  const match = regexp.exec(pathname)

  // 解析动态参数
  if (match) {
    
    
    req.params = req.params || {
    
    }
    keys.forEach((key, index) => {
    
    
      req.params[key.name] = match[index + 1]
    })
  }

  return match && route.method === method
})

Extract Layer processing module

Express source code

source code:node_modules\express\lib\router\layer.js

In Express, the routing path matching is separately encapsulated into a module for management.

// node_modules\express\lib\router\index.js
var layer = new Layer(path, {
    
    
    sensitive: this.caseSensitive,
    strict: this.strict,
    end: true
}, route.dispatch.bind(route));

layer.route = route;

// 路由栈里存储的 Layer 实例
this.stack.push(layer);

// 调用的 layer 的 match 方法进行匹配
layer = stack[idx++];
match = matchLayer(layer, path);

function matchLayer(layer, path) {
    
    
  try {
    
    
    return layer.match(path);
  } catch (err) {
    
    
    return err;
  }
}

// node_modules\express\lib\router\layer.js
module.exports = Layer;

function Layer(path, options, fn) {
    
    
  if (!(this instanceof Layer)) {
    
    
    return new Layer(path, options, fn);
  }

  debug('new %o', path)
  var opts = options || {
    
    };

  this.handle = fn;
  this.name = fn.name || '<anonymous>';
  this.params = undefined;
  this.path = undefined;
  // 实例化时就生成了路由的正则表达式
  this.regexp = pathRegexp(path, this.keys = [], opts);

  // set fast path flags
  this.regexp.fast_star = path === '*'
  this.regexp.fast_slash = path === '/' && opts.end === false
}

Layer.prototype.match = function match(path) {
    
    ...}

Simulation implementation

// express\lib\router\layer.js
const pathRegexp = require('path-to-regexp')

function Layer(path, handler) {
    
    
  this.path = path
  this.handler = handler
  this.keys = []
  this.regexp = pathRegexp(path, this.keys, {
    
    })
  this.params = {
    
    }
}

Layer.prototype.match = function (pathname) {
    
    
  const match = this.regexp.exec(pathname)

  if (match) {
    
    
    this.keys.forEach((key, index) => {
    
    
      this.params[key.name] = match[index + 1]
    })
    return true
  }

  return false
}

module.exports = Layer

// router\index.js
const url = require('url')
const methods = require('methods')
const Layer = require('./layer')

function Router() {
    
    
  // 路由记录栈
  this.stack = []
}

// 为每个请求方法添加处理函数
methods.forEach(method => {
    
    
  Router.prototype[method] = function (path, handler) {
    
    
    const layer = new Layer(path, handler)
    layer.method = method
    this.stack.push(layer)
  }
})

Router.prototype.handle = function (req, res) {
    
    
  const {
    
     pathname } = url.parse(req.url)
  const method = req.method.toLowerCase()
  
  const layer = this.stack.find(layer => {
    
    
    const match = layer.match(pathname)

    if (match) {
    
    
      req.params = req.params || {
    
    }
      Object.assign(req.params, layer.params)
    }

    return match && layer.method === method
  })

  if (layer) {
    
    
    return layer.handler(req, res)
  }

  res.end('404 Not Found.')
}

module.exports = Router

A middleware function that implements a single handler function

routing example

app.get('/', (req, res, next) => {
    
    
  console.log(1)
  next()
}, (req, res, next) => {
    
    
  console.log(2)
  next()
}, (req, res, next) => {
    
    
  console.log(3)
  next()
})

app.get('/', (req, res) => {
    
    
  // 响应
  res.end('get /')
})

app.get('/foo', (req, res, next) => {
    
    
  console.log('foo 1')
  next()
})

app.get('/foo', (req, res, next) => {
    
    
  console.log('foo 2')
  next()
})

app.get('/foo', (req, res) => {
    
    
  // 响应
  res.end('get /foo')
})

Simulation implementation

It is not possible to directly traverse each matched route handler. Considering that there may be asynchronous operations in the processing function, you can customize a nextmethod (internal processing of the next route), pass it as a parameter , and decide to process the next route handlerby yourself (call ) timing.handlernext

// router\index.js
const url = require('url')
const methods = require('methods')
const Layer = require('./layer')

function Router() {
    
    
  // 路由记录栈
  this.stack = []
}

// 为每个请求方法添加处理函数
methods.forEach(method => {
    
    
  Router.prototype[method] = function (path, handler) {
    
    
    const layer = new Layer(path, handler)
    layer.method = method
    this.stack.push(layer)
  }
})

Router.prototype.handle = function (req, res) {
    
    
  const {
    
     pathname } = url.parse(req.url)
  const method = req.method.toLowerCase()

  let index = 0
  const next = () => {
    
    
    if (index >= this.stack.length) {
    
    
      return res.end(`Cannot GET ${
      
      pathname}`)
    }

    const layer = this.stack[index++]
    const match = layer.match(pathname)

    if (match) {
    
    
      req.params = req.params || {
    
    }
      Object.assign(req.params, layer.params)
    }

    if (match && layer.method === method) {
    
    
      return layer.handler(req, res, next)
    }

    // 如果当前请求没有匹配到路由，继续遍历下一个路由
    next()
  }

  // 调用执行
  next()
}

module.exports = Router

// 访问 `/foo` 打印结果：
foo 1
foo 2

// 访问 `/ 打印结果（只执行了每个路由的第一个处理函数）：
1

Middleware that implements multiple handler functions

Idea analysis

Each item stored in the current routing table stack is a separate layer, each containing a processing function handler.

In fact, Express also stores a routing table (Route) in the routing table (Router), which stores the layer created for each processing function, so that multiple processing functions of each route can be traversed.

The data structure is similar to:

Router
  stack [
    Layer 1 {
    
    
      path: 'xx', // 请求路径
      dispatch, // 处理方法，内部调用 Route 下的 dispatch
      Route {
    
    
        stack [
          Layer {
    
     path: 'xx', method: 'xxx', handler: 'xxx' },
          Layer {
    
     path: 'xx', method: 'xxx', handler: 'xxx' },
          Layer {
    
     path: 'xx', method: 'xxx', handler: 'xxx' }
        ],
        dispatch // 遍历调用 stack 中的 handler
      }
    },
    Layer 2 {
    
    
      path: 'xx', // 请求路径
      dispatch, // 处理方法
      Route {
    
    
        stack [
          Layer {
    
     path: 'xx', method: 'xxx', handler: 'xxx' },
          Layer {
    
     path: 'xx', method: 'xxx', handler: 'xxx' },
          Layer {
    
     path: 'xx', method: 'xxx', handler: 'xxx' }
        ],
        dispatch // 遍历调用 stack 中的 handler
      }
    },
  ]

Organization Data Structure

// express\lib\router\route.js
const methods = require('methods')
const Layer = require('./layer')

function Route() {
    
    
  this.stack = [
    // { path, method, handler }
  ]
}

// 遍历执行当前路由对象中所有的处理函数
Route.prototype.dispatch = function () {
    
     }

methods.forEach(method => {
    
    
  Route.prototype[method] = function (path, handlers) {
    
    
    handlers.forEach(handler => {
    
    
      const layer = new Layer(path, handler)
      layer.method = method
      this.stack.push(layer)
    })
  }
})

module.exports = Route

// router\index.js
const url = require('url')
const methods = require('methods')
const Layer = require('./layer')
const Route = require('./route')

function Router() {
    
    
  // 路由记录栈
  this.stack = []
}

// 为每个请求方法添加处理函数
methods.forEach(method => {
    
    
  Router.prototype[method] = function (path, handlers) {
    
    
    const route = new Route()
    const layer = new Layer(path, route.dispatch.bind(route))
    layer.route = route
    this.stack.push(layer)
    route[method](path, handlers)
  }
})

Router.prototype.handle = function (req, res) {
    
    ...}

module.exports = Router

// express\lib\application.js
...

methods.forEach(method => {
    
    
  // 接受全部处理函数
  App.prototype[method] = function (path, ...handlers) {
    
    
    this._router[method](path, handlers)
  }
})

...

You can print the route to see the result:

// app.js
app.get('/foo', (req, res) => {
    
    
  // 响应
  res.end('get /foo')
})

console.log(app._router)

Execute processing function

Express can configure routes through the following API:

app.route('/foo')
  .get((req, res) => {
    
    })
  .post((req, res) => {
    
    })

When executing the processing function, the outer route matches the path, and the inner route matches the request method, so the matching method is removed here:

// router\index.js
Router.prototype.handle = function (req, res) {
    
    
  ...
  const next = () => {
    
    
	...
    // 顶层只匹配请求路径，内层匹配请求方法
    // if (match && layer.method === method) {
    
    
    if (match) {
    
    
      // 顶层调用的 handler 就是 route.dispatch 函数
      return layer.handler(req, res, next)
    }
    next()
  }
  next()
}

// express\lib\router\route.js
const methods = require('methods')
const Layer = require('./layer')

...

// 遍历执行当前路由对象中所有的处理函数
// 遍历内层的 stack，内层遍历结束就要返回到外层，所以第三个参数名取名 out 而不是 next
Route.prototype.dispatch = function (req, res, out) {
    
    
  const method = req.method.toLowerCase()

  let index = 0
  const next = () => {
    
    
    if (index >= this.stack.length) {
    
    
      return out()
    }

    const layer = this.stack[index++]

    if (layer.method === method) {
    
    
      return layer.handler(req, res, next)
    }

    next()
  }

  next()
}

...

module.exports = Route

Visit again /to print the result:

1
2
3

Implement the use method

Review the rules of use

// 不验证请求方法和请求路径，任何请求都会处理
// app.use((req, res, next) => {
    
    
//   res.end('hello')
// })

// 匹配的不是 `/foo`，而是以 `/foo` 作为前置路径的路由
// app.use('/foo', (req, res, next) => {
    
    
//   res.end('foo')
// })

// 与不传第一个路径参数等效
// app.use('/', (req, res, next) => {
    
    
//   res.end('hello')
// })

// 可以挂载多个请求处理函数
app.use('/', (req, res, next) => {
    
    
  console.log(1)
  next()
}, (req, res, next) => {
    
    
  console.log(2)
  next()
}, (req, res, next) => {
    
    
  res.end('hello')
})

Simulation implementation

// express\lib\application.js
App.prototype.use = function (path, ...handlers) {
    
    
  this._router.use(path, handlers)
}

// router\index.js
Router.prototype.use = function (path, handlers) {
    
    
  // 没有传路径的时候
  if (typeof path === 'function') {
    
    
    handlers.unshift(path)
    path = '/' // 任何路径都以它开头
  }

  handlers.forEach(handler => {
    
    
    const layer = new Layer(path, handler)
    // 标记是否是 use 类型的中间件
    layer.isUseMiddleware = true
    this.stack.push(layer)
  })
}

// express\lib\router\layer.js
const pathRegexp = require('path-to-regexp')

function Layer(path, handler) {
    
    
  this.path = path
  this.handler = handler
  this.keys = []
  this.regexp = pathRegexp(path, this.keys, {
    
    })
  this.params = {
    
    }
}

Layer.prototype.match = function (pathname) {
    
    
  const match = this.regexp.exec(pathname)

  if (match) {
    
    
    this.keys.forEach((key, index) => {
    
    
      this.params[key.name] = match[index + 1]
    })
    return true
  }

  // 匹配 use 中间件的路径
  if (this.isUseMiddleware) {
    
    
    if (this.path === '/') {
    
    
      return true
    }

    if (pathname.startsWith(`${
      
      this.path}/`)) {
    
    
      return true
    }
  }

  return false
}

module.exports = Layer