Swoole large data volume transmission solution

Swoole large data volume transmission solution

Question: To transmit a large amount of data length>1M in Swoole TCP without compression,
first use node's mockjs to generate a file larger than 1M

	var jsonfile = require('jsonfile')
	var file = './data.json'
	let fs = require("fs");
	var Mock = require('mockjs')
	
	function injectContentTofileUntil(fileSizeInMega) {
	    let obj = jsonfile.readFileSync(file)
	    console.log(obj)
	    if(!obj){
	        obj=[]
	    }
	
	    for (let index = 0; index < 1000; index++) {
	        obj.push({
	            name: Mock.Random.cname(),
	            country: Mock.Random.county(true),
	        })
	    }
	
	    jsonfile.writeFileSync(file, obj)
	    let fileSizeInMegabytes = getFileSizeInMega(file)
	    if (fileSizeInMegabytes >= fileSizeInMega) {
	        return true
	    } else {
	        return false
	    }
	}
	
	function getFileSizeInMega(filePath) {
	    let stats = fs.statSync(file)
	    let fileSizeInBytes = stats.size
	    let fileSizeInMegabytes = fileSizeInBytes / 1000000.0
	    return fileSizeInMegabytes
	}
	
	while (!injectContentTofileUntil(1)) {
	    console.log(getFileSizeInMega(file));
	}

The file is an unordered file larger than 1M, and the file is transferred to the server without compression.

1. Start the server and client transmission normally

Server.php

	error_reporting(0);
	$client = new swoole_client(SWOOLE_SOCK_TCP, SWOOLE_SOCK_ASYNC);
	$filePath = realpath(__DIR__ . '/./jstools/data.json');
	$fileContent = file_get_contents($filePath);
	
	$client->on("connect", function ($cli) {
	    global $fileContent;
	    $cli->send($fileContent);
	});
	$client->on("receive", function ($cli, $data) {
	    echo "received: {$data}\n";
	});
	$client->on("error", function ($cli) {
	    echo "connect failed\n";
	});
	$client->on("close", function ($cli) {
	    echo "connection close\n";
	});
	$client->connect("127.0.0.1", 8091, 0.5);

Client.php

	$server = new swoole_server("0.0.0.0", 8081);
	$server->on('connect', function ($server, $fd){
	    echo "connection open: {$fd}\n";
	});
	$server->on('receive', function ($server, $fd, $reactor_id, $data) {
	    echo "received:" . $data;
	    $server->close($fd);
	    file_put_contents('./response.json', $data);
	});
	$server->on('close', function ($server, $fd) {
	    echo "connection close: {$fd}\n";
	});
	$server->start();

Executing the command
php server.php
php client.php reports an
error:
WARNING swReactor_write (ERROR 1201): data is too large, cannot exceed buffer size.
Reason:
insufficient buffer overflow

2. Modify the problem of insufficient buffer

Set the client maximum buffer:
"socketbuffersize"=>210241024,

报错:
2018-05-18 17:46:09 *6146.0 ERROR swWorker_discard_data (ERROR 1007): 1received the wrong data8180 bytes from socket#1

Reason:
When the swoole receives the content that meets the buffer size, it will automatically end the transmission, and the data has been sent to the consumer, but the data has not been transmitted. There are several solutions to this problem:

• 1. Compression
• 2. Tell swoole that the transmission has not ended when it reaches the buffer size. This article only discusses this scheme

3. Realize data transmission segmentation

server settings:

	'package_eof' => "\r\n\r\n",
	'open_eof_split' => true,

client settings:

	'package_eof' => "\r\n\r\n",

can be transmitted normally.

4. Advantages and disadvantages of this program:

Advantages: Data of any size can be transmitted normally
Disadvantages: You need to manually add the
meaning of the eof field to the stream, please refer to the swoole documentation yourself.

Code address: https://gitee.com/sharpcx/swoole_big_str

{{o.name}}

{{m.name}}