初期化.py
import logging
import socketio
from flask import Flask, request
from flask_appbuilder import AppBuilder, SQLA
from flask_cors import CORS
from flask_redis import FlaskRedis
from flask_ckeditor import CKEditor
from flask_socketio import SocketIO, emit
from .models import RDMessage
from .myKafka.myKafkaManager import myKafkaManager
"""
Logging configuration
"""
logging.basicConfig(format="%(asctime)s:%(levelname)s:%(name)s:%(message)s")
logging.getLogger().setLevel(logging.DEBUG)
app = Flask(__name__)
app.config.from_object("config")
# CORS(app, supports_credentials=True)
CORS(app, resources={
r"/*": {
"origins": "*"}})
db = SQLA(app)
async_mode = 'threading'
kafkaList = app.config.get('KAFKALIST')
# print('kafkaList', kafkaList)
#url = "kafka://{}".format(kafkaList[0])
url = ["kafka://{}".format(i) for i in kafkaList]
# socketioApp = socketio.Server(async_mode=async_mode, cors_allowed_origins='*', client_manager=socketio.KafkaManager(url))
# app.wsgi_app = socketio.WSGIApp(socketioApp, app.wsgi_app)
socketioApp =SocketIO(app, async_mode=async_mode, cors_allowed_origins='*')
mgr = None
sio = None
if len(kafkaList)>0:
# mgr = socketio.KafkaManager(url)
mgr = myKafkaManager(url)
# socketioApp = SocketIO(app, async_mode=async_mode, cors_allowed_origins='*', client_manager=mgr)
socketioApp = socketio.Server(async_mode=async_mode, cors_allowed_origins='*', client_manager=mgr)
app.wsgi_app = socketio.WSGIApp(socketioApp, app.wsgi_app)
# mgr = socketio.KafkaManager(url)
# sio = socketio.Server(client_manager=mgr)
# app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
# send CORS headers 2022-4-18 10:48:41
@app.after_request
def after_request(response):
print('request.url====', request.url)
response.headers.add('Access-Control-Allow-Origin', '*')
if request.method == 'OPTIONS':
response.headers['Access-Control-Allow-Methods'] = '*'
headers = request.headers.get('Access-Control-Request-Headers')
if headers:
response.headers['Access-Control-Allow-Headers'] = headers
##response.headers['Authorization'] = 'xiaominggessdfs3432ds34ds32432cedsad332e23'
return response
redis_store = FlaskRedis()
redis_store.init_app(app)
ckeditor = CKEditor()
ckeditor.init_app(app)
base_template = "app/base.html"
appbuilder = AppBuilder(app, db.session, base_template=base_template)
"""
from sqlalchemy.engine import Engine
from sqlalchemy import event
#Only include this for SQLLite constraints
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
# Will force sqllite contraint foreign keys
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
"""
from . import views
# from app.view import myEvent
from app.view import mydemo
if len(kafkaList)>0:
from app.view import myKafkaEvent
else:
from app.view import myEvent
mydemo.py
from flask import request, current_app, send_from_directory, render_template
from flask_appbuilder import BaseView, expose
from app import appbuilder
class MyDemoView(BaseView):
route_base = "/mydemo"
@expose( '/index', methods = ['GET'] )
def index(self):
return render_template('myTemp/myDemo/index.html')
appbuilder.add_view_no_menu(MyDemoView())
myKafkaEvent.py
import datetime
import json
import uuid
from threading import Lock
import socketio
from app import socketioApp, mgr, appbuilder
from app.common.function import sendShortMessageFun
from app.models import RDMessage
from app.myKafka.myKafkaConsumer import background_thread
from app.view.common import addRDMessage
@socketioApp.event
def connect(sid, environ):
print('socketioApp connect ', sid, environ)
@socketioApp.event
def my_message(sid, data):
print('socketioApp message ', sid, data)
@socketioApp.event
def disconnect(sid):
print('socketioApp disconnect ', sid)
@socketioApp.on('*')
def catch_all(event, sid, data):
print("服务器已经接收到消息 catch_all ：", event, sid, data)
if isinstance(data, dict):
mqtt_msg = data
else:
mqtt_msg = json.loads(data)
topic = event
print("服务器已经接收到消息：", topic, mqtt_msg)
if topic in ["message"]:
mqtt_msg['msg'] = '服务器已经接收到消息'
socketioApp.emit("response", mqtt_msg)
if topic in ["demo1"]:
socketioApp.emit("response", mqtt_msg)
myEvent.py
import datetime
import json
import uuid
from threading import Lock
import socketio
from flask import current_app
from flask_socketio import emit
from kafka import KafkaConsumer
from app import socketioApp, appbuilder
from app.common.function import sendShortMessageFun
from app.models import RDMessage
from app.view.common import addRDMessage
thread = None
thread_lock = Lock()
consumer = None
@socketioApp.event
def connect():
# print('socketioApp connect ', sid, environ)
kafkaList = current_app.config.get('KAFKALIST')
print('kafkaList', kafkaList)
# 这个方式失败，看到了kafka 消息，就看不到socket 消息
# if len(kafkaList) > 0:
# consumer = KafkaConsumer(bootstrap_servers=kafkaList)
# topics = get_topics(consumer)
# consumer.subscribe([i for i in topics]) # 消费者订阅多个topic
# global thread
# with thread_lock:
# if thread is None:
# socketioApp.sleep(1)
# thread = socketioApp.start_background_task(target=background_thread1(consumer))
print("服务器已经接收到消息 connect：", kafkaList, thread)
@socketioApp.on("message")
def handle_event_message(message):
print("服务器已经接收到消息：", message)
message['msg'] = '服务器已经接收到消息'
emit("response", message)
@socketioApp.event
def connect_error(sid):
print("The connection failed!", sid)
@socketioApp.event
def disconnect(sid):
print("I'm disconnected!", sid)
@socketioApp.on("demo1", namespace='/')
def handle_event_send(message):
print("服务器已经接收到消息 demo1 ：", message)
if isinstance(message, dict):
mqtt_msg = message
else:
mqtt_msg = json.loads(message)
emit("response", mqtt_msg)
myKafkaManager.pyをハイライトします。
import datetime
import json
import pickle
import uuid
from app.view.common import addRDMessage
try:
import kafka
except ImportError:
kafka = None
from kafka import KafkaConsumer
from socketio import KafkaManager
class myKafkaManager(KafkaManager):
name = 'kafka'
def __init__(self, url='kafka://localhost:9092', channel='socketio',
write_only=False):
if kafka is None:
raise RuntimeError('kafka-python package is not installed '
'(Run "pip install kafka-python" in your '
'virtualenv).')
super(KafkaManager, self).__init__(channel=channel,
write_only=write_only)
urls = [url] if isinstance(url, str) else url
self.kafka_urls = [url[8:] if url != 'kafka://' else 'localhost:9092'
for url in urls]
self.producer = kafka.KafkaProducer(bootstrap_servers=self.kafka_urls)
self.consumer = kafka.KafkaConsumer(self.channel,
bootstrap_servers=self.kafka_urls)
self._get_topics()
print('__init__', self.topics, self.channel)
topics = [i for i in self.topics]
topics.append(self.channel)
self.consumerAll = kafka.KafkaConsumer(bootstrap_servers=self.kafka_urls)
self.consumerAll.subscribe(topics) # 消费者订阅多个topic
def _publish(self, data):
self.producer.send(self.channel, value=pickle.dumps(data))
self.producer.flush()
def _kafka_listen(self):
for message in self.consumer:
yield message
def _listen(self):
for message in self._kafka_listen():
if message.topic == self.channel:
yield pickle.loads(message.value)
def _get_topics(self):
self.topics = self.consumer.topics()
def _kafka_listen_all(self):
print('_kafka_listen_all', self.topics, self.channel)
for message in self.consumerAll:
yield message
def _listen_all(self):
for message in self._kafka_listen_all():
if message.topic == self.channel:
yield pickle.loads(message.value)
if message.topic in ["demo1", "demo2"]:
if isinstance(message.value, bytes):
try:
res = {
}
res['method'] = 'emit'
res['namespace'] = list(self.get_namespaces())[0]
res['room'] = None
res['skip_sid'] = None
res['callback'] = None
res['host_id'] = self.host_id
res['event'] = message.topic
msg = json.loads(message.value.decode("utf-8"))
res['data'] = msg
except:
pass
yield res
def _thread(self):
for message in self._listen_all():
data = None
if isinstance(message, dict):
data = message
else:
if isinstance(message, bytes): # pragma: no cover
try:
data = pickle.loads(message)
except:
pass
if data is None:
try:
data = json.loads(message)
except:
pass
if data and 'method' in data:
self._get_logger().info('pubsub message: {}'.format(
data['method']))
try:
if data['method'] == 'emit':
self._handle_emit(data)
elif data['method'] == 'callback':
self._handle_callback(data)
elif data['method'] == 'disconnect':
self._handle_disconnect(data)
elif data['method'] == 'close_room':
self._handle_close_room(data)
except:
self.server.logger.exception(
'Unknown error in pubsub listening thread')
Kafka_Producer.py
が Kafka メッセージ テストを送信します
# kafka_producer.py
import json
import time
from kafka import KafkaProducer, KafkaConsumer
from time import sleep
kafkaList = ['192.168.6.15:9092', '192.168.6.21:9092', '192.168.6.26:9092']
def start_producer1():
producer = KafkaProducer(bootstrap_servers=kafkaList[0], value_serializer=lambda m:json.dumps(m).encode("utf-8"))
for i in range(0, 1000):
msg = 'msg is ' + str(i)
future = producer.send('BDLY_TERMINAL_MSG_2.1', msg.encode('utf-8'))
try:
future.get(timeout=10)
# 监控是否发送成功
except Exception as e:
print(str(e))
sleep(3)
def start_producer(topic):
print(topic)
producer = KafkaProducer(bootstrap_servers=kafkaList, key_serializer=lambda k: json.dumps(k).encode(), value_serializer=lambda v: json.dumps(v).encode(), api_version=(0, 1, 0))
for i in range(0, 3):
# 获得当前时间时间戳
now = int(time.time())
# 转换为其他日期格式,如:"%Y-%m-%d %H:%M:%S"
timeStruct = time.localtime(now)
strTime = time.strftime("%Y-%m-%d %H:%M:%S", timeStruct)
msg = {
}
msg['code'] = 0
msg['data'] = {
}
msg['data']['user_form'] = 888888
msg['data']['user_to'] = 888899
msg['data']['content'] = strTime
msg['data']['msgType'] = "1"
msg['data']['sendTime'] = int(time.time()*1000)
print(msg)
# 同一个key值，会被送至同一个分区
future = producer.send(topic, msg, key=msg['userId'])
try:
future.get(timeout=10)
# 监控是否发送成功
except Exception as e:
print(str(e))
sleep(3)
def get_topics():
consumer = KafkaConsumer(group_id='test', bootstrap_servers=kafkaList)
topics = consumer.topics()
print(topics)
return topics
if __name__ == '__main__':
topics = get_topics()
# topic = 'demo2'
topic = 'demo1'
start_producer(topic)
print('success')
インデックス.html
<html>
<head>
<script type="text/javascript"
src="https://code.jquery.com/jquery-3.4.0.min.js"></script>
<script crossorigin="anonymous" integrity="sha512-74AKPNm8Tfd5E9c4otg7XNkIVfIe5ynON7wehpX/9Tv5VYcZvXZBAlcgOAjLHg6HeWyLujisAnle6+iKnyWd9Q==" src="https://lib.baomitu.com/socket.io/4.0.0/socket.io.js"></script>
</head>
<body>
<h1>socket</h1>
<input type="text" name="topic" id="topic" value="send_BDLY_TERMINAL_MSG_4.0">
<input type="text" name="sendUserId" id="sendUserId" value="378191">
<input type="text" name="message" id="message">
<input type="button" name="" id="push" value="发送">
<div id="log"></div>
<script type="text/javascript" charset="utf-8">
$(document).ready(function () {
var namespace = ""
//'http://' + document.domain + ':' + location.port + namespace
var url = 'http://' + document.domain + ':' + location.port + namespace
var socket = io.connect(url);
// 测试消息
socket.emit("message", {
"data": url });
socket.on('connect', function (data) {
console.log('connect',url)
socket.emit('message', {
'data': 'I\'m connected!' });
});
socket.on('disconnect', function(data){
socket.emit('message', {
'data': 'I\'m disconnected!' });
});
socket.on('response', function (data) {
console.log('response',data)
});
$("#push").click(function(){
var topic=$("#topic").val();
var msg = $("#message").val();
var sendUserId = $("#sendUserId").val();
//message='发送内容:'+$("#message").val()+'=====发送时间:'+gettime();
var message={
};
message['sendUserId']=sendUserId;
message['content']=msg;
socket.emit(topic, message);
})
});
</script>
</body>
</html>
参考ドキュメント
https://python-socketio.readthedocs.io/en/latest/intro.html