From 38787b1e4ed861b6c19ae001c72e53e94c1f4db0 Mon Sep 17 00:00:00 2001 From: "mark.tian" Date: Tue, 9 Dec 2025 19:09:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96flower=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LOGGING_README.md | 105 ++++++++++++++++++++++++++++++++++++++++++++++ README.md | 1 - call_api_task.py | 4 +- celery_app.py | 8 +++- flower_config.py | 2 +- logging_config.py | 69 ++++++++++++++++++++++++++++++ run_flower.py | 55 +++++++++++++++++------- 7 files changed, 224 insertions(+), 20 deletions(-) create mode 100644 LOGGING_README.md create mode 100644 logging_config.py diff --git a/LOGGING_README.md b/LOGGING_README.md new file mode 100644 index 0000000..7e4c6da --- /dev/null +++ b/LOGGING_README.md @@ -0,0 +1,105 @@ +# 日志系统说明 + +## 日志配置 + +项目已配置统一的文件日志系统，所有日志文件存储在 `logs/` 目录下。 + +## 日志文件 + +### 1. 主要日志文件 +- `logs/call_api_task.log` - 呼叫任务API执行日志 +- `logs/celery_app.log` - Celery应用日志 +- `logs/app.log` - 应用主日志 + +### 2. 日志格式 +``` +2025-12-09 17:34:49 - call_api_task - INFO - 开始遍历API配置文件执行调用任务 - 当前时间: 2025-12-09 17:34:49 +``` + +格式说明： +- 时间戳 +- 模块名 +- 日志级别 (INFO/WARNING/ERROR) +- 日志消息 + +## 使用方法 + +### 1. 在代码中使用日志 + +```python +from logging_config import get_logger + +# 获取logger +logger = get_logger('your_module_name') + +# 记录日志 +logger.info("信息日志") +logger.warning("警告日志") +logger.error("错误日志") +``` + +### 2. 查看日志 + +```bash +# 查看最新日志 +tail -f logs/call_api_task.log + +# 查看日志文件 +ls -la logs/ +``` + +## 日志级别 + +- **INFO**: 正常执行信息 +- **WARNING**: 警告信息，但不影响执行 +- **ERROR**: 错误信息，可能导致任务失败 + +## 日志管理 + +### 自动创建 +- 日志目录 `logs/` 会自动创建 +- 日志文件会自动创建和追加 + +### 日志轮转 +- 目前使用简单的追加模式 +- 建议定期清理旧的日志文件 + +## 示例 + +### 任务执行日志示例 +``` +2025-12-09 17:34:49 - call_api_task - INFO - 开始遍历API配置文件执行调用任务 +2025-12-09 17:34:49 - call_api_task - INFO - 开始处理API配置: main_api +2025-12-09 17:34:49 - call_api_task - INFO - 正在调用任务API: POST https://api.example.com +2025-12-09 17:34:52 - call_api_task - INFO - 任务API调用成功，响应数据: {...} +2025-12-09 17:34:52 - call_api_task - INFO - 遍历API配置文件完成 - 成功: 1/1 +``` + +### 错误日志示例 +``` +2025-12-09 17:35:15 - call_api_task - WARNING - 任务API需要重试，当前重试次数: 1/3 +2025-12-09 17:35:15 - call_api_task - ERROR - 任务API失败，状态码: 500 +``` + +## 故障排查 + +1. **检查日志目录权限** + ```bash + ls -la logs/ + ``` + +2. **查看最新错误** + ```bash + grep "ERROR" logs/*.log + ``` + +3. **实时监控日志** + ```bash + tail -f logs/call_api_task.log + ``` + +## 注意事项 + +- 日志文件会持续增长，建议定期备份和清理 +- 敏感信息不应写入日志文件 +- 日志文件使用UTF-8编码 \ No newline at end of file diff --git a/README.md b/README.md index 3724417..f7ce574 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ - `run_flower.py`: 启动Flower监控的脚本 - `flower_config.py`: Flower监控配置 - `start_all.py`: 一键启动所有服务 -- `test_task.py`: 测试脚本 ## 安装依赖 ```bash diff --git a/call_api_task.py b/call_api_task.py index e1d6d1b..e104b86 100644 --- a/call_api_task.py +++ b/call_api_task.py @@ -4,8 +4,10 @@ from datetime import datetime import logging import requests from api_config import API_CONFIG, RETRY_CONFIG +from logging_config import get_logger -logger = logging.getLogger(__name__) +# 获取配置好的logger +logger = get_logger('call_api_task') @app.task(bind=True, name='call_api', max_retries=None) def execute_call_api_task(self): diff --git a/celery_app.py b/celery_app.py index 3eb8295..f064d08 100644 --- a/celery_app.py +++ b/celery_app.py @@ -1,10 +1,14 @@ from celery import Celery from datetime import datetime import logging +from logging_config import get_logger -# 配置日志 +# 配置基本日志 logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) + +# 获取配置好的logger +logger = get_logger('celery') +celery_logger = get_logger('celery_app') # 创建Celery应用 app = Celery( diff --git a/flower_config.py b/flower_config.py index a5d5a44..cd8e610 100644 --- a/flower_config.py +++ b/flower_config.py @@ -6,7 +6,7 @@ FLOWER_CONFIG = { 'result_backend': 'redis://localhost:6379/0', 'port': 5555, 'address': '0.0.0.0', # 允许外部访问 - 'basic_auth': None, # 基础认证，格式: 'username:password' + 'basic_auth': 'admin:admin123', # 基础认证，格式: 'username:password' 'oauth_redirect_url': None, 'db': None, # 使用SQLite存储监控数据 'inspect_timeout': 1000, diff --git a/logging_config.py b/logging_config.py new file mode 100644 index 0000000..3881371 --- /dev/null +++ b/logging_config.py @@ -0,0 +1,69 @@ +"""统一日志配置模块""" +import logging +import os +from datetime import datetime + +def setup_logging(logger_name, log_filename=None): + """ + 设置统一的文件日志配置 + + Args: + logger_name: logger名称 + log_filename: 日志文件名，如果不提供则使用logger_name + + Returns: + configured logger: 配置好的logger + """ + # 创建日志目录 + log_dir = "logs" + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + # 设置日志文件名 + if log_filename is None: + log_filename = f"{logger_name}.log" + + log_file = os.path.join(log_dir, log_filename) + + # 配置文件处理器 + file_handler = logging.FileHandler(log_file, encoding='utf-8') + file_handler.setLevel(logging.INFO) + + # 设置日志格式 + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + file_handler.setFormatter(formatter) + + # 获取或创建logger + logger = logging.getLogger(logger_name) + + # 避免重复添加handler + if not logger.handlers: + logger.addHandler(file_handler) + logger.setLevel(logging.INFO) + + return logger + +def get_logger(logger_name): + """获取配置好的logger""" + return setup_logging(logger_name) + +# 全局日志配置 +def setup_global_logging(): + """设置全局应用日志""" + # 应用主日志 + app_logger = setup_logging('app', 'app_main.log') + + # Celery日志 + celery_logger = setup_logging('celery', 'celery_app.log') + + # 任务日志 + task_logger = setup_logging('call_api_task', 'call_api_tasks.log') + + return { + 'app': app_logger, + 'celery': celery_logger, + 'call_api_task': task_logger + } \ No newline at end of file diff --git a/run_flower.py b/run_flower.py index 4830805..81796b1 100644 --- a/run_flower.py +++ b/run_flower.py @@ -1,25 +1,50 @@ #!/usr/bin/env python3 """启动Flower监控服务""" -from celery import Celery -from flower import command as flower_command +import os +import sys +import subprocess +from celery_app import app +from flower_config import FLOWER_CONFIG def start_flower(): """启动Flower监控""" print("启动Flower监控服务...") - print("监控地址: http://localhost:5555") + print(f"监控地址: http://localhost:{FLOWER_CONFIG['port']}") print("按 Ctrl+C 停止服务") - # Flower配置 - flower_options = { - 'broker': 'redis://localhost:6379/0', - 'port': 5555, - 'basic_auth': None, # 可以设置用户名:密码，如 'admin:password' - 'inspect_timeout': 1000, - 'purge_offline_workers': 60, - } - - # 启动Flower - flower_command.flower_command(flower_options) + try: + # 使用celery_app中的app对象启动flower + app.worker_main(['flower', + '--broker', app.conf.broker_url, + '--port', str(FLOWER_CONFIG['port']), + '--address', FLOWER_CONFIG['address'], + '--inspect_timeout', str(FLOWER_CONFIG['inspect_timeout']), + '--purge_offline_workers', str(FLOWER_CONFIG['purge_offline_workers']), + '--enable_events', + 'true' if FLOWER_CONFIG['enable_events'] else 'false']) + except KeyboardInterrupt: + print("\nFlower监控服务已停止") + except Exception as e: + print(f"启动Flower失败: {e}") + print("请检查:") + print("1. Redis是否正在运行 (redis-server)") + print("2. 端口5555是否被占用") + print("3. Celery worker是否正在运行") + +def start_flower_simple(): + """简单启动方式""" + print("尝试简单启动Flower...") + try: + # 使用celery_app中的app对象简单启动 + app.worker_main(['flower', '--broker=redis://localhost:6379/0', '--port=5555', '--address=127.0.0.1']) + except Exception as e: + print(f"简单启动失败: {e}") if __name__ == '__main__': - start_flower() \ No newline at end of file + # 先尝试正常启动 + try: + start_flower() + except Exception as e: + print(f"正常启动失败: {e}") + print("尝试简单启动...") + start_flower_simple() \ No newline at end of file