chore: 基线提交（重构前快照）
重构前保存现有 apps/ 结构基线，供后续各阶段回滚。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,368 @@
|
||||
"""
|
||||
定时任务调度核心模块
|
||||
|
||||
使用 APScheduler 管理和执行定时任务。
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
|
||||
import requests
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.jobstores.base import JobLookupError
|
||||
|
||||
from config import FASTAPI_HOST, FASTAPI_PORT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 调度器实例
|
||||
_scheduler: Optional[BackgroundScheduler] = None
|
||||
|
||||
|
||||
def get_scheduler() -> BackgroundScheduler:
|
||||
"""获取调度器单例"""
|
||||
global _scheduler
|
||||
if _scheduler is None:
|
||||
_scheduler = BackgroundScheduler(timezone='Asia/Shanghai')
|
||||
return _scheduler
|
||||
|
||||
|
||||
def build_full_url(url: str) -> str:
|
||||
"""构建完整URL，将相对路径拼接为带域名的URL"""
|
||||
if not url:
|
||||
return url
|
||||
if url.startswith(('http://', 'https://')):
|
||||
return url
|
||||
if url.startswith('/'):
|
||||
url = f"http://{FASTAPI_HOST}:{FASTAPI_PORT}{url}"
|
||||
return url
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 任务执行核心函数
|
||||
# ============================================================
|
||||
|
||||
def execute_job(
|
||||
job_id: int,
|
||||
url: Optional[str] = None,
|
||||
is_sync: int = 0,
|
||||
job_trigger: Optional[str] = None,
|
||||
params: Optional[Dict] = None
|
||||
) -> Dict:
|
||||
"""执行任务（调度器调用入口），执行完成后写入任务日志"""
|
||||
start_time = datetime.now()
|
||||
result = {'success': False, 'error': '未知错误'}
|
||||
try:
|
||||
logger.info(f"执行定时任务 [ID: {job_id}, URL: {url}, Trigger: {job_trigger}]")
|
||||
|
||||
if url:
|
||||
result = _execute_by_url(job_id, url, is_sync, params)
|
||||
elif job_trigger:
|
||||
result = _execute_by_trigger(job_trigger, job_id, params)
|
||||
else:
|
||||
result = {'success': False, 'error': '任务既没有URL也没有触发器'}
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"执行任务异常 [ID: {job_id}]: {str(e)}")
|
||||
result = {'success': False, 'error': str(e)}
|
||||
return result
|
||||
|
||||
finally:
|
||||
_record_execution_log(job_id, result, start_time)
|
||||
|
||||
|
||||
def _record_execution_log(job_id: int, result: Dict, start_time: datetime):
|
||||
"""记录一次任务执行日志"""
|
||||
try:
|
||||
from apps.services.job_log import record_job_log
|
||||
|
||||
end_time = datetime.now()
|
||||
success = bool(result.get('success'))
|
||||
status = 0 if success else 1
|
||||
if not success:
|
||||
job_message = '任务执行失败'
|
||||
exception_info = str(result.get('error'))
|
||||
elif result.get('status') == 'triggered':
|
||||
job_message = '异步任务已触发'
|
||||
exception_info = None
|
||||
else:
|
||||
job_message = '任务执行成功'
|
||||
exception_info = None
|
||||
|
||||
record_job_log(job_id, start_time, end_time, status,
|
||||
exception_info=exception_info, job_message=job_message)
|
||||
except Exception as e:
|
||||
logger.error(f"记录任务日志失败 [ID: {job_id}]: {str(e)}")
|
||||
|
||||
|
||||
def _execute_by_url(job_id: int, url: str, is_sync: int, params: Optional[Dict] = None) -> Dict:
|
||||
"""通过URL方式执行任务"""
|
||||
full_url = build_full_url(url)
|
||||
request_data = {
|
||||
'job_id': job_id,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'params': params or {}
|
||||
}
|
||||
|
||||
if is_sync == 1:
|
||||
return _send_sync_request(full_url, request_data, job_id)
|
||||
else:
|
||||
return _send_async_request(full_url, request_data, job_id)
|
||||
|
||||
|
||||
def _send_sync_request(url: str, data: Dict, job_id: int) -> Dict:
|
||||
"""发送同步HTTP请求"""
|
||||
try:
|
||||
logger.info(f"发送同步请求 [ID: {job_id}, URL: {url}]")
|
||||
headers = {'Content-Type': 'application/json', 'User-Agent': 'Scheduler/1.0'}
|
||||
response = requests.post(url, json=data, timeout=30, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
try:
|
||||
result_data = response.json()
|
||||
except Exception:
|
||||
result_data = {'text': response.text[:200]}
|
||||
|
||||
return {'success': True, 'status_code': response.status_code, 'data': result_data}
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def _send_async_request(url: str, data: Dict, job_id: int) -> Dict:
|
||||
"""发送异步HTTP请求（独立线程）"""
|
||||
|
||||
def async_request():
|
||||
try:
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
resp = requests.post(url, json=data, timeout=30, headers=headers)
|
||||
logger.info(f"异步任务完成 [ID: {job_id}, 状态码: {resp.status_code}]")
|
||||
except Exception as e:
|
||||
logger.error(f"异步任务异常 [ID: {job_id}]: {str(e)}")
|
||||
|
||||
thread = threading.Thread(target=async_request)
|
||||
thread.start()
|
||||
return {'success': True, 'status': 'triggered'}
|
||||
|
||||
|
||||
def _execute_by_trigger(trigger: str, job_id: int, params: Optional[Dict] = None) -> Dict:
|
||||
"""通过触发器方式执行任务"""
|
||||
try:
|
||||
from apps.services.job_executor import job_executor
|
||||
# 剔除 job_id，避免与 execute(job_id=...) 关键字参数冲突
|
||||
trigger_params = dict(params or {})
|
||||
trigger_params.pop('job_id', None)
|
||||
result = job_executor.execute(trigger, job_id=job_id, **trigger_params)
|
||||
# 处理器返回 success=False 时透传失败，而非一律视为成功
|
||||
if isinstance(result, dict) and result.get('success') is False:
|
||||
error = result.get('message') or result.get('error') or '任务执行失败'
|
||||
return {'success': False, 'trigger': trigger, 'error': error}
|
||||
return {'success': True, 'trigger': trigger, 'data': result}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 调度器任务管理函数
|
||||
# ============================================================
|
||||
|
||||
def _parse_job_params(params: Optional[str]) -> Dict:
|
||||
"""解析任务参数（JSON字符串 -> dict），容错非法输入"""
|
||||
if not params:
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(params)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except Exception:
|
||||
logger.warning(f"任务参数不是合法JSON，已忽略: {str(params)[:200]}")
|
||||
return {}
|
||||
|
||||
|
||||
def _build_misfire_config(execute_policy) -> Dict:
|
||||
"""根据执行策略映射 APScheduler 的 misfire/coalesce 参数"""
|
||||
if execute_policy == 3:
|
||||
# 放弃执行：错过的运行不补
|
||||
return {'misfire_grace_time': 0, 'coalesce': True}
|
||||
if execute_policy == 2:
|
||||
# 执行一次：短时间内错过的补跑一次
|
||||
return {'misfire_grace_time': 600, 'coalesce': True}
|
||||
# 立即执行（默认）：错过立即补跑一次
|
||||
return {'misfire_grace_time': 3600, 'coalesce': True}
|
||||
|
||||
|
||||
def add_job_to_scheduler(job_dict: dict) -> bool:
|
||||
"""添加任务到调度器"""
|
||||
try:
|
||||
job_id = job_dict.get('id')
|
||||
job_status = job_dict.get('status')
|
||||
job_cron = job_dict.get('cron_expression')
|
||||
job_url = job_dict.get('url')
|
||||
job_sync = job_dict.get('is_sync', 0)
|
||||
job_trigger = job_dict.get('job_trigger')
|
||||
job_name = job_dict.get('job_name', '')
|
||||
job_params = _parse_job_params(job_dict.get('params'))
|
||||
|
||||
if job_status != 1:
|
||||
return False
|
||||
|
||||
if not job_cron:
|
||||
logger.warning(f"跳过添加任务: Cron表达式为空 (id={job_id})")
|
||||
return False
|
||||
|
||||
# 验证Cron表达式
|
||||
try:
|
||||
trigger = CronTrigger.from_crontab(job_cron)
|
||||
next_run = trigger.get_next_fire_time(None, datetime.now())
|
||||
if next_run is None:
|
||||
logger.warning(f"Cron表达式不会触发 (id={job_id})")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Cron表达式无效 (id={job_id}): {str(e)}")
|
||||
return False
|
||||
|
||||
scheduler = get_scheduler()
|
||||
scheduler_job_id = f"job_{job_id}"
|
||||
|
||||
# 移除已存在的同名任务
|
||||
try:
|
||||
scheduler.remove_job(scheduler_job_id)
|
||||
except JobLookupError:
|
||||
pass
|
||||
|
||||
misfire_config = _build_misfire_config(job_dict.get('execute_policy', 1))
|
||||
|
||||
scheduler.add_job(
|
||||
func=execute_job,
|
||||
trigger=trigger,
|
||||
args=[job_id, job_url, job_sync, job_trigger, job_params],
|
||||
id=scheduler_job_id,
|
||||
name=job_name,
|
||||
replace_existing=True,
|
||||
**misfire_config
|
||||
)
|
||||
|
||||
logger.info(f"任务已添加到调度器 [ID: {job_id}, cron: {job_cron}]")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"添加任务到调度器异常: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def remove_job_from_scheduler(job_id: int) -> bool:
|
||||
"""从调度器移除任务"""
|
||||
try:
|
||||
get_scheduler().remove_job(f"job_{job_id}")
|
||||
return True
|
||||
except JobLookupError:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"移除任务异常: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def pause_job_in_scheduler(job_id: int) -> bool:
|
||||
"""暂停调度器中的任务"""
|
||||
try:
|
||||
get_scheduler().pause_job(f"job_{job_id}")
|
||||
return True
|
||||
except JobLookupError:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"暂停任务异常: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def resume_job_in_scheduler(job_id: int) -> bool:
|
||||
"""恢复调度器中的任务"""
|
||||
try:
|
||||
get_scheduler().resume_job(f"job_{job_id}")
|
||||
return True
|
||||
except JobLookupError:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"恢复任务异常: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def run_job_once(job_id: int, params: Optional[Dict] = None) -> Dict:
|
||||
"""立即执行一次任务（不依赖调度器）"""
|
||||
try:
|
||||
from apps.models.job import Job
|
||||
from extensions import get_db_session
|
||||
|
||||
db = get_db_session()
|
||||
job = db.query(Job).filter(Job.id == job_id, Job.is_delete == 0).first()
|
||||
if not job:
|
||||
return {'success': False, 'error': '任务不存在'}
|
||||
|
||||
return execute_job(job.id, job.url, job.is_sync, job.job_trigger, params or {})
|
||||
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 调度器初始化
|
||||
# ============================================================
|
||||
|
||||
def init_scheduler() -> int:
|
||||
"""初始化调度器，启动并从数据库加载所有启用的任务"""
|
||||
try:
|
||||
from apps.models.job import Job
|
||||
from extensions.sqlalchemy import SessionLocal
|
||||
|
||||
scheduler = get_scheduler()
|
||||
if not scheduler.running:
|
||||
scheduler.start()
|
||||
logger.info("调度器已启动")
|
||||
|
||||
# lifespan 启动阶段无请求上下文，需用独立会话而非请求级 get_db_session()
|
||||
session = SessionLocal()
|
||||
try:
|
||||
jobs = session.query(Job).filter(Job.status == 1, Job.is_delete == 0).all()
|
||||
# 转为字典避免 Session 分离问题
|
||||
job_dicts = [job.to_dict() for job in jobs]
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
success_count = 0
|
||||
for job_dict in job_dicts:
|
||||
if add_job_to_scheduler(job_dict):
|
||||
success_count += 1
|
||||
|
||||
logger.info(f"调度器初始化完成，加载 {success_count}/{len(job_dicts)} 个任务")
|
||||
return success_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"初始化调度器异常: {str(e)}")
|
||||
return 0
|
||||
|
||||
|
||||
def shutdown_scheduler():
|
||||
"""关闭调度器"""
|
||||
try:
|
||||
scheduler = get_scheduler()
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=False)
|
||||
logger.info("调度器已关闭")
|
||||
except Exception as e:
|
||||
logger.error(f"关闭调度器异常: {str(e)}")
|
||||
|
||||
|
||||
def reload_scheduler() -> int:
|
||||
"""重新加载调度器：清空所有任务，重新从数据库加载"""
|
||||
try:
|
||||
scheduler = get_scheduler()
|
||||
scheduler.remove_all_jobs()
|
||||
return init_scheduler()
|
||||
except Exception as e:
|
||||
logger.error(f"重新加载调度器异常: {str(e)}")
|
||||
return 0
|
||||
Reference in New Issue
Block a user