feat: 优化访问速度

Dockerfile

@@ -39,6 +39,7 @@ ENV S6_SERVICES_GRACETIME=30000 \
     PUID=0 \
     PGID=0 \
     UMASK=000 \
+    NT_PORT=3000 \
     WORKDIR="/nas-tools"
 RUN mkdir ${WORKDIR}
 ADD ./ ${WORKDIR}/

README.md

@@ -6,26 +6,38 @@
 - 支持新版馒头刷流和下载
 - 3.3.0 以上版本通过 gunicorn 部署，并且依赖redis，其他版本安装请自行安装redis，docker 3.2.6及以下版本需要重新拉取新镜像
 
-### 馒头站点维护
+### 站点维护
 
-- 添加 User-Agent
+1. 馒头站点维护
+   - 添加 User-Agent
 
-- 不需要模拟登陆只用添加令牌
+   - 不需要模拟登陆只用添加令牌
 
-  1. 令牌获取
+   1. 令牌获取
 
-     打开馒头 控制台 > 实验室 > 存取令牌 > 建立存取令牌
+      打开馒头 控制台 > 实验室 > 存取令牌 > 建立存取令牌
 
-     1. 复制令牌到 nas-tools 站点维护
-     2. 添加请求头参数 格式：{"x-api-key": "令牌"}
+      1. 复制令牌到 nas-tools 站点维护
+      2. 添加请求头参数 格式：{"x-api-key": "令牌"}
+
+   2. 馒头签到
+
+      馒头模拟登录需要添加 auth 参数，通过 auth 访问接口，不排除禁用账户的可能
+      添加请求头参数 格式：{"x-api-key": "令牌", "authorization": "auth参数"}
+
+      auth参数获取方式
+      ![mt-auth.png](https://raw.github.com/linyuan0213/nas-tools/master/img/mt-auth.png)
+
+2. FSM站点维护
+   - 添加 User-Agent
+
+   - API TOKEN 获取方式
+      1. 进入站点下拉到最后，找到API链接，点击进入，如图
+         ![fsm-api-1.png](https://raw.github.com/linyuan0213/nas-tools/master/img/fsm-api-1.png)
+      2. 点击一键生成生成 API TOKEN，如图
+         ![fsm-api-2.png](https://raw.github.com/linyuan0213/nas-tools/master/img/fsm-api-2.png)
+      3. 添加请求头参数，格式： {"APITOKEN": "这里填 api token"}， 将复制的API TOKEN 填入
 
-  2. 馒头签到
-
-     馒头模拟登录需要添加 auth 参数，通过 auth 访问接口，不排除禁用账户的可能
-     添加请求头参数 格式：{"x-api-key": "令牌", "authorization": "auth参数"}
-
-     auth参数获取方式
-     ![mt-auth.png](https://raw.github.com/linyuan0213/nas-tools/master/img/mt-auth.png)
 
 
 ### 验证码识别

app/conf/systemconfig.py

@@ -5,7 +5,6 @@
 from app.utils.types import SystemConfigKey
 
 
-@singleton
 class SystemConfig:
     # 系统设置
     systemconfig = {}

app/plugins/plugin_manager.py

@@ -10,14 +10,14 @@
 from app.utils.commons import singleton
 from app.utils.types import SystemConfigKey
 from config import Config
+from app.utils import RedisStore
 
 
 @singleton
 class PluginManager:
     """
     插件管理器
     """
-    systemconfig = None
     eventmanager = None
 
     # 用户插件目录
@@ -28,7 +28,7 @@ class PluginManager:
     # 插件列表
     _plugins = {}
     # 运行态插件列表
-    _running_plugins = {}
+    _running_plugins = None
     # 配置Key
     _config_key = "plugin.%s"
     # 事件处理线程
@@ -39,6 +39,7 @@ class PluginManager:
     def __init__(self):
         # config/plugins 是插件py文件目录，config/plugins/xxx是插件数据目录
         self.user_plugin_path = Config().get_user_plugin_path()
+        self._running_plugins = RedisStore()
         if not os.path.exists(self.user_plugin_path):
             os.makedirs(self.user_plugin_path)
         self.system_plugin_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "modules")
@@ -48,7 +49,6 @@ def __init__(self):
         self.init_config()
 
     def init_config(self):
-        self.systemconfig = SystemConfig()
         self.eventmanager = EventManager()
         # 停止已有插件
         self.stop_service()
@@ -106,17 +106,19 @@ def __load_plugins(self):
         # 排序
         plugins.sort(key=lambda x: x.module_order if hasattr(x, "module_order") else 0)
         # 用户已安装插件列表
-        user_plugins = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins) or []
-        self._running_plugins = {}
+        user_plugins = SystemConfig().get(SystemConfigKey.UserInstalledPlugins) or []
+        log.debug(f"user_plugins: {user_plugins}")
         self._plugins = {}
         for plugin in plugins:
             module_id = plugin.__name__
             self._plugins[module_id] = plugin
             # 未安装的跳过加载
             if module_id not in user_plugins:
+                self._running_plugins.delete_obj('plugin', module_id)
                 continue
             # 生成实例
-            self._running_plugins[module_id] = plugin()
+            self._running_plugins.add_object(key='plugin', obj_id=module_id, obj=plugin())
+            # self._running_plugins[module_id] = plugin()
             # 初始化配置
             self.reload_plugin(module_id)
             log.info(f"加载插件：{plugin}")
@@ -125,32 +127,32 @@ def run_plugin(self, pid, method, *args, **kwargs):
         """
         运行插件
         """
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return None
-        if not hasattr(self._running_plugins[pid], method):
+        if not hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), method):
             return
         try:
-            return getattr(self._running_plugins[pid], method)(*args, **kwargs)
+            return getattr(self._running_plugins.load_object(key='plugin', obj_id=pid), method)(*args, **kwargs)
         except Exception as err:
             print(str(err), traceback.format_exc())
 
     def get_running_plugin_ids(self):
         """
         获取所有运行态插件ID
         """
-        return list(self._running_plugins.keys())
+        return list(self._running_plugins.get_all_object_keys('plugin'))
 
     def reload_plugin(self, pid):
         """
         生效插件配置
         """
         if not pid:
             return
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return
-        if hasattr(self._running_plugins[pid], "init_config"):
+        if hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), "init_config"):
             try:
-                self._running_plugins[pid].init_config(self.get_plugin_config(pid))
+                self._running_plugins.load_object(key='plugin', obj_id=pid).init_config(self.get_plugin_config(pid))
                 log.debug(f"生效插件配置：{pid}")
             except Exception as err:
                 print(str(err))
@@ -159,7 +161,7 @@ def __stop_plugins(self):
         """
         停止所有插件
         """
-        for plugin in self._running_plugins.values():
+        for plugin in self._running_plugins.get_all_object_vals('plugin'):
             if hasattr(plugin, "stop_service"):
                 plugin.stop_service()
 
@@ -169,46 +171,46 @@ def get_plugin_config(self, pid):
         """
         if not self._plugins.get(pid):
             return {}
-        return self.systemconfig.get(self._config_key % pid) or {}
+        return SystemConfig().get(self._config_key % pid) or {}
 
     def get_plugin_page(self, pid):
         """
         获取插件额外页面数据
         :return: 标题，页面内容，确定按钮响应函数
         """
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return None, None, None
-        if not hasattr(self._running_plugins[pid], "get_page"):
+        if not hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), "get_page"):
             return None, None, None
-        return self._running_plugins[pid].get_page()
+        return self._running_plugins.load_object(key='plugin', obj_id=pid).get_page()
 
     def get_plugin_script(self, pid):
         """
         获取插件额外脚本
         """
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return None
-        if not hasattr(self._running_plugins[pid], "get_script"):
+        if not hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), "get_script"):
             return None
-        return self._running_plugins[pid].get_script()
+        return self._running_plugins.load_object(key='plugin', obj_id=pid).get_script()
 
     def get_plugin_state(self, pid):
         """
         获取插件状态
         """
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return None
-        if not hasattr(self._running_plugins[pid], "get_state"):
+        if not hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), "get_state"):
             return None
-        return self._running_plugins[pid].get_state()
+        return self._running_plugins.load_object(key='plugin', obj_id=pid).get_state()
 
     def save_plugin_config(self, pid, conf):
         """
         保存插件配置
         """
         if not self._plugins.get(pid):
             return False
-        return self.systemconfig.set(self._config_key % pid, conf)
+        return SystemConfig().set(self._config_key % pid, conf)
 
     @staticmethod
     def __get_plugin_color(plugin):
@@ -229,7 +231,7 @@ def get_plugins_conf(self, auth_level):
         获取所有插件配置
         """
         all_confs = {}
-        for pid, plugin in self._running_plugins.items():
+        for pid, plugin in self._running_plugins.get_all_key_values('plugin').items():
             # 基本属性
             conf = {}
             # 权限
@@ -275,7 +277,7 @@ def get_plugin_apps(self, auth_level):
         获取所有插件
         """
         all_confs = {}
-        installed_apps = self.systemconfig.get(SystemConfigKey.UserInstalledPlugins) or []
+        installed_apps = SystemConfig().get(SystemConfigKey.UserInstalledPlugins) or []
         for pid, plugin in self._plugins.items():
             # 基本属性
             conf = {}
@@ -324,7 +326,7 @@ def get_plugin_commands(self):
         }]
         """
         ret_commands = []
-        for _, plugin in self._running_plugins.items():
+        for _, plugin in self._running_plugins.get_all_key_values('plugin').items():
             if hasattr(plugin, "get_command"):
                 ret_commands.append(plugin.get_command())
         return ret_commands
@@ -333,18 +335,18 @@ def run_plugin_method(self, pid, method, *args, **kwargs):
         """
         运行插件方法
         """
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return None
-        if not hasattr(self._running_plugins[pid], method):
+        if not hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), method):
             return None
-        return getattr(self._running_plugins[pid], method)(*args, **kwargs)
+        return getattr(self._running_plugins.load_object(key='plugin', obj_id=pid), method)(*args, **kwargs)
 
     def get_plugin_method(self, pid, method):
         """
         获取插件方法
         """
-        if not self._running_plugins.get(pid):
+        if not self._running_plugins.load_object(key='plugin', obj_id=pid):
             return None
-        if not hasattr(self._running_plugins[pid], method):
+        if not hasattr(self._running_plugins.load_object(key='plugin', obj_id=pid), method):
             return None
-        return getattr(self._running_plugins[pid], method)
+        return getattr(self._running_plugins.load_object(key='plugin', obj_id=pid), method)

app/sites/sites.py

@@ -6,11 +6,9 @@
 from app.message import Message
 from app.sites.site_limiter import SiteRateLimiter
 from app.utils import RequestUtils, StringUtils, JsonUtils
-from app.utils.commons import singleton
 from config import Config
 
 
-@singleton
 class Sites:
     message = None
     dbhelper = None

app/utils/redis_store.py

@@ -1,4 +1,5 @@
 import redis
+import pickle
 
 from config import REDIS_HOST, REDIS_PORT
 
@@ -36,3 +37,31 @@ def llen(self, name):
 
     def delete(self, *keys):
         self.client.delete(*keys)
+
+    def add_object(self, key, obj_id, obj):
+        serialize_obj = pickle.dumps(obj)
+        self.client.hset(key, obj_id, serialize_obj)
+
+    def load_object(self, key, obj_id):
+        serialize_obj = self.client.hget(key, obj_id)
+        if serialize_obj:
+            return pickle.loads(serialize_obj)
+        else:
+            return None
+
+    def delete_obj(self, key, obj_id):
+        self.client.hdel(key, obj_id)
+
+    def get_all_object_keys(self, key):
+        return [key.decode('utf-8') for key in self.client.hkeys(key)]
+
+    def get_all_object_vals(self, key):
+        return [pickle.loads(val) for val in self.client.hvals(key)]
+
+    def get_all_key_values(self, key):
+        all_key_values = {}
+        hash_data = self.client.hgetall(key)
+        for hash_key, serialized_value in hash_data.items():
+            value = pickle.loads(serialized_value)
+            all_key_values[hash_key.decode('utf-8')] = value
+        return all_key_values

docker-compose.yml

@@ -11,7 +11,8 @@ services:
       - PUID=0    # 想切换为哪个用户来运行程序，该用户的uid
       - PGID=0    # 想切换为哪个用户来运行程序，该用户的gid
       - UMASK=000 # 掩码权限，默认000，可以考虑设置为022
-      - NASTOOL_AUTO_UPDATE=false  # 如需在启动容器时自动升级程程序请设置为true
+      - WORKERS=3  # 设置进程数，不填写该参数默认为 cpu核心数 * 2 + 1
+      - NT_PORT=3000 # 不填写 默认3000
     restart: always
     network_mode: bridge
     hostname: nas-tools

docker/rootfs/etc/services.d/NAStool/run

@@ -3,11 +3,6 @@
 
 umask ${UMASK}
 
-if [ -f ${NASTOOL_CONFIG} ]; then
-    NT_PORT=$(awk -F"[' ]+" '/web_port/{print $3}' /config/config.yaml)
-else
-    NT_PORT=3000
-fi
 exec \
     s6-notifyoncheck -d -n 300 -w 1000 -c "nc -z localhost ${NT_PORT}" \
     cd ${WORKDIR} s6-setuidgid nt sh ./start-server.sh

gunicorn.conf.py

@@ -2,6 +2,7 @@
 import os
 import hashlib
 import random
+import multiprocessing
 
 
 os.environ['SERVER_INSTANCE'] = hashlib.md5(str(random.random()).encode()).hexdigest()
@@ -22,9 +23,9 @@
 timeout = 60  # 超时
 daemon = False  # 是否后台运行
 debug = False
-workers = 1  # 进程数
+workers = os.environ.get('WORKERS') if os.environ.get('WORKERS') else multiprocessing.cpu_count() * 2 + 1  # 进程数
 worker_class = "gthread"
-threads = 10  # 指定每个进程开启的线程数
+threads = 5  # 指定每个进程开启的线程数
 loglevel = 'info'  # 日志级别，这个日志级别指的是错误日志的级别，而访问日志的级别无法设置
 pidfile = os.path.join(ROOT_PATH, "gunicorn.pid")  # 存放Gunicorn进程pid的位置，便于跟踪