1. stream module
stream is another module provided by Node.js that is only available on the server side, and its purpose is to support the data structure of "stream".
What is a stream? A stream is an abstract data structure. Imagine the flow of water, flowing from one place (such as a waterworks) to another (such as your sink) as it flows through a pipe. We can also think of data as a data stream. For example, when you type on the keyboard, you can connect each character in turn and see it as a character stream. This stream is input from the keyboard to the application. In fact, it also corresponds to a name: the standard input stream (stdin).
If the application outputs characters to the display one by one, this can also be regarded as a stream, and this stream also has a name: the standard output stream (stdout). The characteristic of the stream is that the data is ordered, and must be read or written sequentially, and cannot be randomly positioned like Array.
Some streams are used to read data. For example, when reading data from a file, you can open a file stream and then continuously read data from the file stream. Some streams are used to write data. For example, when writing data to a file, you only need to continuously write data into the file stream.
In Node.js, a stream is also an object. We only need to respond to stream events: the data event indicates that the data of the stream can be read, and the end event indicates that the stream has reached the end and there is no data to read. , the error event indicates an error.
1.1 Read stream
const fs = require('fs');
//创建读取流
let rs = fs.createReadStream('hello.txt', 'utf-8');
rs.on('open', function () {
console.log('读取的文件已打开');
}).on('close', function () {
console.log('读取流结束');
}).on('error', err => {
console.log(err);
}).on('data', function (chunk) {
//每一批数据流入完成
console.log('单批数据流入:' + chunk.length);
console.log(chunk);
});
It should be noted that the data event may occur multiple times, and the chunk passed each time is part of the stream data.
read video
const fs = require('fs');
//创建读取流
let rs = fs.createReadStream('video.mp4');
//每一批数据流入完成
rs.on('data', function (chunk) {
console.log('单批数据流入:' + chunk.length);
console.log(chunk);
});
1.2 Write stream
To write to a file as a stream, just keep calling the write() method and end with end():
const fs = require('fs');
//创建写入流
let ws = fs.createWriteStream('hello.txt', 'utf-8');
//监听文件打开事件
ws.on('open', function () {
console.log('文件打开');
});
//监听文件关闭事件
ws.on('close', function () {
console.log('文件写入完成，关闭');
});
//文件流式写入
ws.write('helloworld1!', function (err) {
if (err) {
console.log(err);
} else {
console.log('内容1流入完成');
}
});
ws.write('helloworld2!', function (err) {
if (err) {
console.log(err);
} else {
console.log('内容2流入完成');
}
});
//文件写入完成
ws.end(function () {
console.log('文件写入关闭');
});
pipeJust as two water pipes can be chained to make a longer pipe, two streams can be chained. After a Readablestream Writableis connected to a stream, all data automatically Readableenters Writablethe stream from the stream. This operation is called pipe.
In Node.js, Readablestreams have a pipe()method that does just that.
Let's pipe()chain a file stream to another file stream so that all the data from the source file is automatically written to the destination file, so this is actually a program that copies files:
const fs = require('fs');
//创建读取流
let rs = fs.createReadStream('video.mp4');
let ws = fs.createWriteStream('b.mp4');
rs.on('close', function () {
console.log('读取流结束');
});
rs.pipe(ws);
pipe principle
const fs = require('fs');
//创建读取流
let rs = fs.createReadStream('video.mp4');
let ws = fs.createWriteStream('b.mp4');
rs.on('close', function () {
ws.end();
console.log('读取流结束');
});
//每一批数据流入完成
rs.on('data', function (chunk) {
console.log('单批数据流入:' + chunk.length);
ws.write(chunk, () => {
console.log('单批输入流入完成');
});
});
2. Resource compression module zib
2.1 Overview
Students who have done web performance optimization should be familiar with the performance optimization killer gzip. The browser initiates a resource request to the server, such as downloading a js file, the server first compresses the resource, and then returns it to the browser, so as to save traffic and speed up access.
The browser adds Accept-Encoding to the HTTP request header to tell the server, "You can use gzip or defalte algorithm to compress resources".
Accept-Encoding:gzip, deflate
So, in nodejs, how to compress resources? The answer is the Zlib module. =
2.2 Compression example
With a very simple few lines of code, the gzip compression of local files is completed.
var fs = require('fs');
var zlib = require('zlib');
var gzip = zlib.createGzip();
var readstream = fs.createReadStream('./extra/fileForCompress.txt');
var writestream = fs.createWriteStream('./extra/fileForCompress.txt.gz');
readstream.pipe(gzip).pipe(writestream);
2.3 Example of decompression
It is also very simple, it is a reverse operation.
var fs = require('fs');
var zlib = require('zlib');
var gunzip = zlib.createGunzip();
var readstream = fs.createReadStream('./extra/fileForCompress.txt.gz');
var writestream = fs.createWriteStream('./extra/fileForCompress1.txt');
readstream.pipe(gunzip).pipe(writestream);
2.4 Server-side gzip compression
First determine whether the accept-encoding header is included, and the value is gzip.
- No: return the uncompressed file.
- Yes: return the gzip compressed file.
var http = require('http');
var zlib = require('zlib');
var fs = require('fs');
var filepath = './extra/fileForGzip.html';
var server = http.createServer(function(req, res){
var acceptEncoding = req.headers['accept-encoding'];
var gzip;
if(acceptEncoding.indexOf('gzip')!=-1){
// 判断是否需要gzip压缩
gzip = zlib.createGzip();
// 记得响应 Content-Encoding，告诉浏览器：文件被 gzip 压缩过
res.writeHead(200, {
'Content-Encoding': 'gzip'
});
fs.createReadStream(filepath).pipe(gzip).pipe(res);
}else{
fs.createReadStream(filepath).pipe(res);
}
});
server.listen('3000');
return js large file
const fs = require('fs');
const zlib = require('zlib');//这两个要写在fs模块后面
const gzip = zlib.createGzip();
const http = require('http');
http
.createServer((req, res) => {
let rs = fs.createReadStream('hello.js');
res.writeHead(200, {
'Content-Type': 'application/x-javascript;charset=utf-8',
'Content-Encoding': 'gzip',
});
rs.pipe(gzip).pipe(res);
})
.listen(3000, () => {
console.log('server start');
});
2.5 Server string gzip compression
The code is similar to the previous example. Here zlib.gzipSync(str) is used to compress the string with gzip.
var http = require('http');
var zlib = require('zlib');
var responseText = 'hello world';
var server = http.createServer(function(req, res){
var acceptEncoding = req.headers['accept-encoding'];
if(acceptEncoding.indexOf('gzip')!=-1){
res.writeHead(200, {
'content-encoding': 'gzip'
});
res.end(zlib.gzipSync(responseText) );
}else{
res.end(responseText);
}
});
server.listen('3000');
3. Data encryption module crypto
cryptoThe purpose of the module is to provide common encryption and hashing algorithms. It is not impossible to implement these functions in pure JavaScript code, but it will be very slow. cyptoAfter Nodejs uses C/C++ to implement these algorithms, it is exposed as a JavaScript interface through this module, which is convenient to use and fast to run.
3.1 hash example
hash.digest([encoding]): Compute summary. encodingCan be hex, latin1or base64. If specified encoding, returns the string. Otherwise, return Bufferthe instance. Note that hash.digest()after the call, hashthe object is invalidated, and an error will occur if it is called again.
hash.update(data[, input_encoding]): input_encodingCan be utf8, asciior latin1. If data is a string and is not specified input_encoding, it defaults to utf8. Note that hash.update()methods can be called multiple times.
var crypto = require('crypto');
var fs = require('fs');
var content = fs.readFileSync('./test.txt', {
encoding: 'utf8'});
var hash = crypto.createHash('sha256');
var output;
hash.update(content);
output = hash.digest('hex');
console.log(output);
// 输出内容为：
// b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
It can also be like this:
var crypto = require('crypto');
var fs = require('fs');
var input = fs.createReadStream('./test.txt', {
encoding: 'utf8'});
var hash = crypto.createHash('sha256');
hash.setEncoding('hex');
input.pipe(hash).pipe(process.stdout)
// 输出内容为：
// b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
hash.digest()后，再次调用digest()或者update()
var crypto = require('crypto');
var fs = require('fs');
var content = fs.readFileSync('./test.txt', {
encoding: 'utf8'});
var hash = crypto.createHash('sha256');
var output;
hash.update(content);
hash.digest('hex');
// 报错：Error: Digest already called
hash.update(content);
// 报错：Error: Digest already called
hash.digest('hex');
3.2 HMAC example
The full name of HMAC is Hash-based Message Authentication Code, which is the salt operation in hash.
When it comes to use, it is similar to the hash module, just select the hash algorithm and specify "salt".
Example 1:
var crypto = require('crypto');
var fs = require('fs');
var secret = 'secret';
var hmac = crypto.createHmac('sha256', secret);
var input = fs.readFileSync('./test.txt', {
encoding: 'utf8'});
hmac.update(input);
console.log( hmac.digest('hex') );
// 输出：
// 734cc62f32841568f45715aeb9f4d7891324e6d948e4c6c60c0621cdac48623a
Example 2:
var crypto = require('crypto');
var fs = require('fs');
var secret = 'secret';
var hmac = crypto.createHmac('sha256', secret);
var input = fs.createReadStream('./test.txt', {
encoding: 'utf8'});
hmac.setEncoding('hex');
input.pipe(hmac).pipe(process.stdout)
// 输出：
// 734cc62f32841568f45715aeb9f4d7891324e6d948e4c6c60c0621cdac48623a
3.3 MD5 example
MD5 (Message-Digest Algorithm) is a hash function (also known as hash algorithm, digest algorithm) widely used in the field of computer security, mainly used to ensure the integrity and consistency of messages. Common application scenarios include password protection, download file verification, etc.
features
- Fast operation speed: for jquery.js to find md5 value, 57254 characters, time-consuming 1.907ms
- Fixed output length: The input length is not fixed, and the output length is fixed (128 bits).
- The operation is irreversible: When the operation result is known, the original string cannot be obtained through the inverse operation.
- Highly discrete: small changes in the input can lead to huge differences in the results of the operation.
- Weak collision: Different inputs may have the same hash value.
Application Scenario
- File integrity verification: For example, when downloading a software from the Internet, the general website will attach the md5 value of the software to the web page. After the user downloads the software, he can perform md5 calculation on the downloaded software, and then compare it with the md5 value on the website. Make a comparison to ensure that the downloaded software is complete (or correct);
- Password protection: save the password after md5 to the database instead of saving the plaintext password to avoid leakage of the plaintext password after events such as dragging the database;
- Anti-tampering: For example, the anti-tampering of digital certificates uses the digest algorithm. (Of course, in combination with digital signatures and other means);
var crypto = require('crypto');
var md5 = crypto.createHash('md5');
var result = md5.update('a').digest('hex');
// 输出：0cc175b9c0f1b6a831c399e269772661
console.log(result);
3.4 Example: password protection
As mentioned earlier, it is very unsafe to save plaintext passwords to the database, and at worst, md5 must be performed to save them. For example, the user password is 123456, after running md5, the output is: e10adc3949ba59abbe56e057f20f883e.
This has at least two advantages:
- Anti-internal attack: The website owner does not know the user's plaintext password, preventing the website owner from doing bad things with the user's plaintext password.
- Anti-external attack: If the website is hacked, the hacker can only get the password after md5, not the user's plaintext password.
The sample code is as follows:
var crypto = require('crypto');
function cryptPwd(password) {
var md5 = crypto.createHash('md5');
return md5.update(password).digest('hex');
}
var password = '123456';
var cryptedPassword = cryptPwd(password);
console.log(cryptedPassword);
// 输出：e10adc3949ba59abbe56e057f20f883e
It is not safe to simply perform md5 on passwords
. As mentioned earlier, the security is improved by performing md5 calculations on user passwords. But in fact, such security is very poor, why?
Modify the above example a little bit, maybe you will understand. The same plaintext password has the same md5 value.
var crypto = require('crypto');
function cryptPwd(password) {
var md5 = crypto.createHash('md5');
return md5.update(password).digest('hex');
}
var password = '123456';
console.log( cryptPwd(password) );
// 输出：e10adc3949ba59abbe56e057f20f883e
console.log( cryptPwd(password) );
// 输出：e10adc3949ba59abbe56e057f20f883e
That is to say, when the attacker knows that the algorithm is md5, and the password value stored in the database is, e10adc3949ba59abbe56e057f20f883etheoretically, it can be guessed that the user's plaintext password is 123456.
In fact, this is how the rainbow table is brute force cracked: the md5 value of the common plaintext password is calculated and stored in advance, and then matched with the password stored in the website database, the user's plaintext password can be quickly found. (The specific details are not explored here)
So, is there any way to further improve security? The answer is: salt the password.
Adding salt to the password
The word "salting" seems mysterious, but the principle is actually very simple, that is, after inserting a specific string in a specific position of the password, and then performing the md5 operation on the modified string.
Examples are as follows. For the same password, when the "salt" value is different, the difference in md5 value is very large. By adding salt to the password, the most basic brute force cracking can be prevented. If the attacker does not know the "salt" value in advance, it will be very difficult to crack.
var crypto = require('crypto');
function cryptPwd(password, salt) {
// 密码“加盐”
var saltPassword = password + ':' + salt;
console.log('原始密码：%s', password);
console.log('加盐后的密码：%s', saltPassword);
// 加盐密码的md5值
var md5 = crypto.createHash('md5');
var result = md5.update(saltPassword).digest('hex');
console.log('加盐密码的md5值：%s', result);
}
cryptPwd('123456', 'abc');
// 输出：
// 原始密码：123456
// 加盐后的密码：123456:abc
// 加盐密码的md5值：51011af1892f59e74baf61f3d4389092
cryptPwd('123456', 'bcd');
// 输出：
// 原始密码：123456
// 加盐后的密码：123456:bcd
// 加盐密码的md5值：55a95bcb6bfbaef6906dbbd264ab4531