增加组装推送消息的逻辑
This commit is contained in:
mark.tian
@@ -2,20 +2,17 @@
|
||||
Celery任务定义
|
||||
"""
|
||||
from celery import current_task
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy import create_engine
|
||||
import json
|
||||
from app.celery_app import celery_app
|
||||
from app.config import settings
|
||||
from app.logger import get_logger
|
||||
from app.database import CallbackFailureLog, ExternalApiLog, CallbackFailureData
|
||||
from app.callback_service import (
|
||||
save_callback_data_items,
|
||||
get_uncompleted_callback_log,
|
||||
get_callback_log_data,
|
||||
check_phone_number_threshold,
|
||||
call_external_api_with_retry,
|
||||
mark_callback_log_completed,
|
||||
_log_dtc_push_call
|
||||
get_related_records_by_unique_data_list
|
||||
)
|
||||
from app.redis_lock import redis_manager
|
||||
|
||||
@@ -29,7 +26,7 @@ engine = create_engine(
|
||||
)
|
||||
|
||||
@celery_app.task(bind=True, name='push_data_to_dtc')
|
||||
def push_data_to_dtc_task(self):
|
||||
def push_data_to_dtc_task():
|
||||
"""
|
||||
推送数据给DTC的Celery任务
|
||||
自动获取一条未完成的回调请求进行处理
|
||||
@@ -50,6 +47,12 @@ def push_data_to_dtc_task(self):
|
||||
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': 0, 'total': 100, 'status': f'正在获取下一条回调请求日志...'}
|
||||
)
|
||||
|
||||
# 获取一条未完成的回调请求（按创建时间取最小值）
|
||||
query_success, callback_log_data = get_uncompleted_callback_log(conn)
|
||||
if not query_success:
|
||||
@@ -65,10 +68,11 @@ def push_data_to_dtc_task(self):
|
||||
|
||||
callback_log_id, site_id, request_headers_json, request_body_json = callback_log_data
|
||||
logger.info(f"📋 获取到未完成的回调请求: ID={callback_log_id}, site_id={site_id}")
|
||||
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': len(valid_phone_numbers), 'total': len(phone_numbers), 'status': f'检查手机号: {phone_number}'}
|
||||
meta={'current': 25, 'total': 100, 'status': f'获取回调请求成功(ID={callback_log_id}, site_id={site_id})，开始分析处理...'}
|
||||
)
|
||||
|
||||
# 解析请求头和请求体
|
||||
@@ -83,78 +87,104 @@ def push_data_to_dtc_task(self):
|
||||
logger.error(f"❌ 保存callback_data:{callback_log_id}失败，停止任务执行")
|
||||
raise Exception(f"保存callback_data:{callback_log_id}失败，任务停止执行")
|
||||
|
||||
# 提取所有手机号并去重
|
||||
phone_numbers_set = set()
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': 50, 'total': 100, 'status': f'请求体中通话明细处理成功(ID={callback_log_id}, site_id={site_id})，开始过滤需要转发的通话记录...'}
|
||||
)
|
||||
|
||||
# 根据手机号、task_id、user_id给data_list去重
|
||||
data_list = request_body.get('data', [])
|
||||
unique_data_list = []
|
||||
seen_records = set()
|
||||
original_count = len(data_list)
|
||||
|
||||
for item in data_list:
|
||||
# 提取手机号
|
||||
number_data = item.get('number_data', {})
|
||||
phone_number = number_data.get('number')
|
||||
if phone_number:
|
||||
phone_numbers_set.add(phone_number)
|
||||
phone_number = number_data.get('number', '')
|
||||
|
||||
# 提取task_id
|
||||
task = item.get('task', {})
|
||||
task_id = task.get('id', '')
|
||||
|
||||
# 提取user_id
|
||||
user_id = item.get('user_id', '')
|
||||
|
||||
# 创建唯一标识
|
||||
unique_key = (phone_number, task_id, user_id)
|
||||
|
||||
# 如果这个组合没见过，则添加到去重列表中
|
||||
if unique_key not in seen_records:
|
||||
seen_records.add(unique_key)
|
||||
unique_data_list.append(item)
|
||||
|
||||
phone_numbers = list(phone_numbers_set)
|
||||
logger.info(f"📱 提取到的手机号列表（去重后）: {phone_numbers}")
|
||||
logger.info(f"🔄 数据去重完成: 原始数据 {original_count} 条，去重后 {len(unique_data_list)} 条 - callback_log_id: {callback_log_id}, site_id: {site_id}")
|
||||
|
||||
# 过滤出需要处理的手机号（不超过阈值的）
|
||||
valid_phone_numbers = []
|
||||
for phone_number in phone_numbers:
|
||||
if len(unique_data_list) < original_count:
|
||||
logger.info(f"🗑️ 移除了 {original_count - len(unique_data_list)} 条重复数据 - callback_log_id: {callback_log_id}, site_id: {site_id}")
|
||||
|
||||
|
||||
# 查询相关数据：根据手机号、task_id、user_id作为条件，按创建时间排序获取前三条记录
|
||||
query_success, related_records = get_related_records_by_unique_data_list(
|
||||
conn, unique_data_list, callback_log_id, settings.count_threshold
|
||||
)
|
||||
|
||||
if not query_success:
|
||||
logger.error("❌ 查询相关记录失败，停止任务执行")
|
||||
raise Exception("查询相关记录失败，任务停止执行")
|
||||
|
||||
# 判断是否有相关记录需要处理
|
||||
if not related_records:
|
||||
logger.info(f"📋 没有有效的通过记录需要处理，直接返回 - callback_log_id: {callback_log_id}, site_id={site_id}")
|
||||
return {"status": "completed", "message": "没有有效的通过记录需要处理"}
|
||||
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': 70, 'total': 100, 'status': f'需要推送的通过记录已获取成功(ID={callback_log_id}, site_id={site_id})，准备转发...'}
|
||||
)
|
||||
|
||||
if related_records:
|
||||
# 创建只包含有效数据项的请求体
|
||||
filtered_request_body = request_body.copy()
|
||||
filtered_request_body['data'] = related_records
|
||||
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': len(valid_phone_numbers), 'total': len(phone_numbers), 'status': f'检查手机号: {phone_number}'}
|
||||
)
|
||||
|
||||
# 检查手机号是否超过阈值
|
||||
exceeds_threshold, query_success = check_phone_number_threshold(conn, phone_number)
|
||||
|
||||
if not query_success:
|
||||
logger.error(f"❌ 查询手机号 {phone_number} 失败，跳过处理")
|
||||
continue
|
||||
|
||||
if exceeds_threshold:
|
||||
logger.info(f"✅ 手机号 {phone_number} 出现次数超过阈值，跳过处理")
|
||||
continue
|
||||
|
||||
valid_phone_numbers.append(phone_number)
|
||||
|
||||
logger.info(f"📱 需要处理的手机号数量: {len(valid_phone_numbers)}")
|
||||
|
||||
# 推送数据给DTC（在循环外执行一次）
|
||||
processed_count = 0
|
||||
skipped_count = len(phone_numbers) - len(valid_phone_numbers)
|
||||
|
||||
if valid_phone_numbers:
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': 0, 'total': len(valid_phone_numbers), 'status': '开始推送数据给DTC'}
|
||||
meta={'current': 85, 'total': 100, 'status': '开始推送数据给DTC(ID={callback_log_id}, site_id={site_id})...'}
|
||||
)
|
||||
|
||||
success, retry_count = call_external_api_with_retry(
|
||||
conn=conn,
|
||||
request_body=request_body,
|
||||
request_body=filtered_request_body,
|
||||
request_headers=request_headers,
|
||||
max_retries=settings.external_api_retry_max,
|
||||
callback_failure_log_id=callback_log_id
|
||||
)
|
||||
|
||||
if success:
|
||||
logger.info(f"✅ 推送数据给DTC成功，处理的手机号数量: {len(valid_phone_numbers)}, 重试次数: {retry_count}")
|
||||
processed_count = len(valid_phone_numbers)
|
||||
logger.info(f"✅ 推送数据给DTC成功，处理的数据项数量: {len(related_records)}, 重试次数: {retry_count} - callback_log_id: {callback_log_id}, site_id: {site_id}")
|
||||
else:
|
||||
logger.error(f"❌ 推送数据给DTC失败，重试次数: {retry_count}")
|
||||
logger.error(f"❌ 推送数据给DTC失败，重试次数: {retry_count} - callback_log_id: {callback_log_id}, site_id: {site_id}")
|
||||
else:
|
||||
logger.info("📋 没有有效的手机号需要处理")
|
||||
logger.info(f"📋 没有有效的通过记录需要处理 - callback_log_id: {callback_log_id}, site_id: {site_id}")
|
||||
|
||||
logger.info(f"🎉 推送数据给DTC处理完成，成功: {processed_count}, 跳过: {skipped_count}")
|
||||
logger.info(f"🎉 推送数据给DTC处理完成 - callback_log_id: {callback_log_id}, site_id: {site_id}")
|
||||
|
||||
# 标记CallbackFailureLog为已完成
|
||||
mark_callback_log_completed(conn, callback_log_id)
|
||||
|
||||
# 更新任务状态
|
||||
current_task.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': 100, 'total': 100, 'status': '标记回调请求日志为已完成(ID={callback_log_id}, site_id={site_id})'}
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"processed": processed_count,
|
||||
"skipped": skipped_count,
|
||||
"total": len(phone_numbers)
|
||||
"message": "任务完成"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user