一.安装redis
# redis6.2.6安装
yum install gcc
# 安装依赖
yum install -y gcc
# 新建安装目录,进到相应目录
mkdir /data/Downloads
cd /data/Downloads
# 下载
wget https://download.redis.io/releases/redis-6.2.6.tar.gz
# 下载完毕后开始解压
tar -zxvf redis-6.2.6.tar.gz
# 编译并安装
cd redis-6.2.6
make && make install PREFIX=/usr/local/redis
cd /usr/local/redis
# 新建日志目录
mkdir logs
# 将配置文件复制相应的目录
cp /data/Downloads/redis-6.2.6/redis.conf /usr/local/redis/bin
cd ./bin
# 修改redis配置
cat redis.conf | grep -v "#" | grep -v "^$" > redis-12389.conf
port 12389
pidfile /var/run/redis_12389.pid
dbfilename dump_12389.rdb
requirepass requirepass
logfile "/usr/local/redis/logs/redis.log"
maxclients 100000
stop-writes-on-bgsave-error no
# 由于改为supervisor监控
# daemonize no
# 启动redis
./redis-server /usr/local/redis/bin/redis-12389.conf
ln -s /usr/local/redis/bin/redis-cli /usr/bin/redis-cli
二.安装mysql
略
# 创建superset库
CREATE DATABASE `superset` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
三.superset安装
3.1安装superset依赖
sudo yum install -y gcc gcc-c++ libffi-devel python3-devel python3-pip python3-wheel openssl-devel cyrus-sasl-devel openldap-devel
pip3 install --upgrade setuptools pip
pip3 install flower
pip3 install flask
pip3 install redis
pip3 install pymysql
pip3 install apache-superset
yum install -y nginx
pip3 install uwsgi
pip3 install gunicorn
# mysql依赖
yum install -y mysql-devel
pip3 install mysqlclient
# hive依赖
pip3 install 'pyhive[hive]'
3.2superset配置路径
mkdir -p /opt/superset/conf
# 将配置文件拷贝到配置路径
cp /usr/local/lib/python3.7/site-packages/superset/config.py /opt/superset/conf/superset_config.py
cd /opt/superset/conf/
# 设置superset路径 修改环境变量
vim /etc/profile
export SUPERSET_CONFIG_PATH=/opt/superset/conf/superset_config.py
source /etc/profile
# 初始化数据库
superset db upgrade
# 通过运行以下命令完成安装
# 创建管理员用户 (在设置密码之前，系统会提示您设置用户名、姓和名)
# 设置FLASK_APP环境变量
export FLASK_APP=superset
superset fab create-admin
# 加载一些数据来展示
# superset load_examples
# 创建默认角色和权限
superset init
# 在端口8088上启动一个开发web服务器, 使用-p绑定到其他端口(调试模式启动)
# superset run -p 8088 --with-threads --reload --debugger
superset_config.py 相关配置
# Superset specific config
from datetime import timedelta
from flask import session
from flask import Flask
from celery.schedules import crontab
ROW_LIMIT = 5000
SUPERSET_WEBSERVER_PORT = 8088
# Flask App Builder configuration
# Your App secret key
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'
# The SQLAlchemy connection string to your database backend
# This connection defines the path to the database that stores your
# superset metadata (slices, connections, tables, dashboards, ...).
# Note that the connection information to connect to the datasources
# you want to explore are managed directly in the web UI
# SQLALCHEMY_DATABASE_URI = 'sqlite:path/to/superset.db'
SQLALCHEMY_DATABASE_URI = 'mysql://user:[email protected]/superset'
# Flask-WTF flag for CSRF
WTF_CSRF_ENABLED = True
# Add endpoints that need to be exempt from CSRF protection
WTF_CSRF_EXEMPT_LIST = []
# A CSRF token that expires in 1 year
WTF_CSRF_TIME_LIMIT = 60 * 60 * 24 * 365
# Set this API key to enable Mapbox visualizations
MAPBOX_API_KEY = ''
def make_session_permanent():
'''
Enable maxAge for the cookie 'session'
'''
session.permanent = True
# Set up max age of session to 24 hours
PERMANENT_SESSION_LIFETIME = timedelta(hours=24)
def FLASK_APP_MUTATOR(app: Flask) -> None:
app.before_request_funcs.setdefault(None, []).append(make_session_permanent)
# 标志功能
FEATURE_FLAGS = {
'CLIENT_CACHE': False,
'ENABLE_EXPLORE_JSON_CSRF_PROTECTION': False,
'PRESTO_EXPAND_DATA': False,
# 缓存缩略图
"THUMBNAILS": True,
"THUMBNAILS_SQLA_LISTENERS": True,
}
THUMBNAIL_CACHE_CONFIG = {
"CACHE_TYPE": "redis",
"CACHE_NO_NULL_WARNING": False,
"CACHE_NO_NULL_WARNING": False,
'CACHE_REDIS_PORT': 12389, # 配置端口号
'CACHE_REDIS_URL': 'redis://:[email protected]:12389' # 配置 URL
}
CACHE_CONFIG = {
'CACHE_TYPE': 'redis', # 使用 Redis
'CACHE_REDIS_HOST': '127.0.0.1', # 配置域名
'CACHE_REDIS_PORT': 12389, # 配置端口号
'CACHE_REDIS_URL': 'redis://:[email protected]:12389' # 配置 URL
}
# 缓存
# 对于图表数据Superset 上升了一个“超时搜索路径”，从切片的配置到数据源、数据库的配置，然后最终回退到 中定义的全局默认值DATA_CACHE_CONFIG
DATA_CACHE_CONFIG = {
'CACHE_TYPE': 'redis',
'CACHE_DEFAULT_TIMEOUT': 60 * 60 * 24, # 1 day default (in secs)
'CACHE_KEY_PREFIX': 'superset_results',
'CACHE_REDIS_URL': 'redis://:[email protected]:12389/0',
}
# 设置CELERY
class CeleryConfig: # pylint: disable=too-few-public-methods
# BROKER_URL = "sqla+sqlite:///celerydb.sqlite"
# 设置broker的信息
# 其中password是密码，密码前必须加冒号
BROKER_URL = 'redis://:[email protected]:12389/0'
CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks")
# 设置结果存储信息
CELERY_RESULT_BACKEND = 'redis://:[email protected]:12389/1'
CELERYD_LOG_LEVEL = "DEBUG"
CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_ACKS_LATE = False
CELERY_ANNOTATIONS = {
"sql_lab.get_sql_results": {
"rate_limit": "100/s"},
"email_reports.send": {
"rate_limit": "1/s",
"time_limit": 120,
"soft_time_limit": 150,
"ignore_result": True,
},
}
# CELERYBEAT_SCHEDULE = {
# "email_reports.schedule_hourly": {
# "task": "email_reports.schedule_hourly",
# "schedule": crontab(minute=1, hour="*"),
# }
# }
# Superset 有一个 Celery 任务，它会根据不同的策略定期预热缓存。
# #这将每小时缓存前 5 个最受欢迎的仪表板中的所有图表。对于其他策略，请检查superset/tasks/cache.py文件
CELERYBEAT_SCHEDULE = {
'cache-warmup-hourly': {
'task': 'cache-warmup',
'schedule': crontab(minute=0, hour='*'), # hourly
'kwargs': {
'strategy_name': 'top_n_dashboards',
'top_n': 5,
'since': '7 days ago',
},
},
}
CELERY_CONFIG = CeleryConfig # pylint: disable=invalid-name
# 设置默认语种
BABEL_DEFAULT_LOCALE = "zh"
# Your application default translation path
BABEL_DEFAULT_FOLDER = "superset/translations"
# The allowed translation for you app
LANGUAGES = {
# "en": {"flag": "us", "name": "English"},
#
# "es": {"flag": "es", "name": "Spanish"},
# "it": {"flag": "it", "name": "Italian"},
# "fr": {"flag": "fr", "name": "French"},
"zh": {
"flag": "cn", "name": "Chinese"},
# "ja": {"flag": "jp", "name": "Japanese"},
# "de": {"flag": "de", "name": "German"},
# "pt": {"flag": "pt", "name": "Portuguese"},
# "pt_BR": {"flag": "br", "name": "Brazilian Portuguese"},
# "ru": {"flag": "ru", "name": "Russian"},
# "ko": {"flag": "kr", "name": "Korean"},
}
GLOBAL_ASYNC_QUERIES_REDIS_CONFIG = {
"port": 12389,
"host": "127.0.0.1",
"password": "password",
"db": 0,
}
3.3superset启动
# 使用gunicorn启动
# gunicorn相关配置参考:https://docs.gunicorn.org/en/stable/design.html
# 如果不使用Gunicorn,通过在superset_config.py中设置COMPRESS_REGISTER = False 来禁用flask-compress
# 16666端口需要开放tcp以及udp
gunicorn -w 10 -k gevent --timeout 120 -b 0.0.0.0:16666 --limit-request-line 0 --limit-request-field_size 0 "superset.app:create_app()"
# 要启动 Celery Worker 以利用配置，请运行以下命令
celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4
# 要启动一个计划定期后台作业的作业，请运行以下命令
celery --app=superset.tasks.celery_app:app beat
四.mysql,hive可视化展示
4.1mysql可视化展示
4.1.1添加mysql连接,添加mysql数据集(Datasets)
五.superset的supervisor配置文件
[program:superset_celery_worker]
environment=FLASK_APP="superset",SUPERSET_CONFIG_PATH="/opt/superset/conf/superset_config.py"
command = /usr/local/bin/celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4
user = root
startsecs = 3
redirect_stderr = true
stdout_logfile_maxbytes = 550MB
stdout_logfile_backups = 10
stdout_logfile = /var/lib/docker/containers/facebook_google/superset_celery_worker.log
参考文档 https://superset.apache.org/
参考配置文档 https://github.com/apache/superset/blob/master/superset/config.py
数据库驱动参考https://superset.apache.org/docs/databases/installing-database-drivers