ClaudeandClaude
dbcce79536
重构前保存现有 apps/ 结构基线，供后续各阶段回滚。 Co-Authored-By: Claude <noreply@anthropic.com>
369 lines
12 KiB
Python
369 lines
12 KiB
Python
"""
|
|
定时任务调度核心模块
|
|
|
|
使用 APScheduler 管理和执行定时任务。
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import threading
|
|
from datetime import datetime
|
|
from typing import Dict, Optional
|
|
|
|
import requests
|
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
from apscheduler.jobstores.base import JobLookupError
|
|
|
|
from config import FASTAPI_HOST, FASTAPI_PORT
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# 调度器实例
|
|
_scheduler: Optional[BackgroundScheduler] = None
|
|
|
|
|
|
def get_scheduler() -> BackgroundScheduler:
|
|
"""获取调度器单例"""
|
|
global _scheduler
|
|
if _scheduler is None:
|
|
_scheduler = BackgroundScheduler(timezone='Asia/Shanghai')
|
|
return _scheduler
|
|
|
|
|
|
def build_full_url(url: str) -> str:
|
|
"""构建完整URL，将相对路径拼接为带域名的URL"""
|
|
if not url:
|
|
return url
|
|
if url.startswith(('http://', 'https://')):
|
|
return url
|
|
if url.startswith('/'):
|
|
url = f"http://{FASTAPI_HOST}:{FASTAPI_PORT}{url}"
|
|
return url
|
|
|
|
|
|
# ============================================================
|
|
# 任务执行核心函数
|
|
# ============================================================
|
|
|
|
def execute_job(
|
|
job_id: int,
|
|
url: Optional[str] = None,
|
|
is_sync: int = 0,
|
|
job_trigger: Optional[str] = None,
|
|
params: Optional[Dict] = None
|
|
) -> Dict:
|
|
"""执行任务（调度器调用入口），执行完成后写入任务日志"""
|
|
start_time = datetime.now()
|
|
result = {'success': False, 'error': '未知错误'}
|
|
try:
|
|
logger.info(f"执行定时任务 [ID: {job_id}, URL: {url}, Trigger: {job_trigger}]")
|
|
|
|
if url:
|
|
result = _execute_by_url(job_id, url, is_sync, params)
|
|
elif job_trigger:
|
|
result = _execute_by_trigger(job_trigger, job_id, params)
|
|
else:
|
|
result = {'success': False, 'error': '任务既没有URL也没有触发器'}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
logger.error(f"执行任务异常 [ID: {job_id}]: {str(e)}")
|
|
result = {'success': False, 'error': str(e)}
|
|
return result
|
|
|
|
finally:
|
|
_record_execution_log(job_id, result, start_time)
|
|
|
|
|
|
def _record_execution_log(job_id: int, result: Dict, start_time: datetime):
|
|
"""记录一次任务执行日志"""
|
|
try:
|
|
from apps.services.job_log import record_job_log
|
|
|
|
end_time = datetime.now()
|
|
success = bool(result.get('success'))
|
|
status = 0 if success else 1
|
|
if not success:
|
|
job_message = '任务执行失败'
|
|
exception_info = str(result.get('error'))
|
|
elif result.get('status') == 'triggered':
|
|
job_message = '异步任务已触发'
|
|
exception_info = None
|
|
else:
|
|
job_message = '任务执行成功'
|
|
exception_info = None
|
|
|
|
record_job_log(job_id, start_time, end_time, status,
|
|
exception_info=exception_info, job_message=job_message)
|
|
except Exception as e:
|
|
logger.error(f"记录任务日志失败 [ID: {job_id}]: {str(e)}")
|
|
|
|
|
|
def _execute_by_url(job_id: int, url: str, is_sync: int, params: Optional[Dict] = None) -> Dict:
|
|
"""通过URL方式执行任务"""
|
|
full_url = build_full_url(url)
|
|
request_data = {
|
|
'job_id': job_id,
|
|
'timestamp': datetime.now().isoformat(),
|
|
'params': params or {}
|
|
}
|
|
|
|
if is_sync == 1:
|
|
return _send_sync_request(full_url, request_data, job_id)
|
|
else:
|
|
return _send_async_request(full_url, request_data, job_id)
|
|
|
|
|
|
def _send_sync_request(url: str, data: Dict, job_id: int) -> Dict:
|
|
"""发送同步HTTP请求"""
|
|
try:
|
|
logger.info(f"发送同步请求 [ID: {job_id}, URL: {url}]")
|
|
headers = {'Content-Type': 'application/json', 'User-Agent': 'Scheduler/1.0'}
|
|
response = requests.post(url, json=data, timeout=30, headers=headers)
|
|
response.raise_for_status()
|
|
|
|
try:
|
|
result_data = response.json()
|
|
except Exception:
|
|
result_data = {'text': response.text[:200]}
|
|
|
|
return {'success': True, 'status_code': response.status_code, 'data': result_data}
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
return {'success': False, 'error': str(e)}
|
|
|
|
|
|
def _send_async_request(url: str, data: Dict, job_id: int) -> Dict:
|
|
"""发送异步HTTP请求（独立线程）"""
|
|
|
|
def async_request():
|
|
try:
|
|
headers = {'Content-Type': 'application/json'}
|
|
resp = requests.post(url, json=data, timeout=30, headers=headers)
|
|
logger.info(f"异步任务完成 [ID: {job_id}, 状态码: {resp.status_code}]")
|
|
except Exception as e:
|
|
logger.error(f"异步任务异常 [ID: {job_id}]: {str(e)}")
|
|
|
|
thread = threading.Thread(target=async_request)
|
|
thread.start()
|
|
return {'success': True, 'status': 'triggered'}
|
|
|
|
|
|
def _execute_by_trigger(trigger: str, job_id: int, params: Optional[Dict] = None) -> Dict:
|
|
"""通过触发器方式执行任务"""
|
|
try:
|
|
from apps.services.job_executor import job_executor
|
|
# 剔除 job_id，避免与 execute(job_id=...) 关键字参数冲突
|
|
trigger_params = dict(params or {})
|
|
trigger_params.pop('job_id', None)
|
|
result = job_executor.execute(trigger, job_id=job_id, **trigger_params)
|
|
# 处理器返回 success=False 时透传失败，而非一律视为成功
|
|
if isinstance(result, dict) and result.get('success') is False:
|
|
error = result.get('message') or result.get('error') or '任务执行失败'
|
|
return {'success': False, 'trigger': trigger, 'error': error}
|
|
return {'success': True, 'trigger': trigger, 'data': result}
|
|
except Exception as e:
|
|
return {'success': False, 'error': str(e)}
|
|
|
|
|
|
# ============================================================
|
|
# 调度器任务管理函数
|
|
# ============================================================
|
|
|
|
def _parse_job_params(params: Optional[str]) -> Dict:
|
|
"""解析任务参数（JSON字符串 -> dict），容错非法输入"""
|
|
if not params:
|
|
return {}
|
|
try:
|
|
parsed = json.loads(params)
|
|
if isinstance(parsed, dict):
|
|
return parsed
|
|
except Exception:
|
|
logger.warning(f"任务参数不是合法JSON，已忽略: {str(params)[:200]}")
|
|
return {}
|
|
|
|
|
|
def _build_misfire_config(execute_policy) -> Dict:
|
|
"""根据执行策略映射 APScheduler 的 misfire/coalesce 参数"""
|
|
if execute_policy == 3:
|
|
# 放弃执行：错过的运行不补
|
|
return {'misfire_grace_time': 0, 'coalesce': True}
|
|
if execute_policy == 2:
|
|
# 执行一次：短时间内错过的补跑一次
|
|
return {'misfire_grace_time': 600, 'coalesce': True}
|
|
# 立即执行（默认）：错过立即补跑一次
|
|
return {'misfire_grace_time': 3600, 'coalesce': True}
|
|
|
|
|
|
def add_job_to_scheduler(job_dict: dict) -> bool:
|
|
"""添加任务到调度器"""
|
|
try:
|
|
job_id = job_dict.get('id')
|
|
job_status = job_dict.get('status')
|
|
job_cron = job_dict.get('cron_expression')
|
|
job_url = job_dict.get('url')
|
|
job_sync = job_dict.get('is_sync', 0)
|
|
job_trigger = job_dict.get('job_trigger')
|
|
job_name = job_dict.get('job_name', '')
|
|
job_params = _parse_job_params(job_dict.get('params'))
|
|
|
|
if job_status != 1:
|
|
return False
|
|
|
|
if not job_cron:
|
|
logger.warning(f"跳过添加任务: Cron表达式为空 (id={job_id})")
|
|
return False
|
|
|
|
# 验证Cron表达式
|
|
try:
|
|
trigger = CronTrigger.from_crontab(job_cron)
|
|
next_run = trigger.get_next_fire_time(None, datetime.now())
|
|
if next_run is None:
|
|
logger.warning(f"Cron表达式不会触发 (id={job_id})")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"Cron表达式无效 (id={job_id}): {str(e)}")
|
|
return False
|
|
|
|
scheduler = get_scheduler()
|
|
scheduler_job_id = f"job_{job_id}"
|
|
|
|
# 移除已存在的同名任务
|
|
try:
|
|
scheduler.remove_job(scheduler_job_id)
|
|
except JobLookupError:
|
|
pass
|
|
|
|
misfire_config = _build_misfire_config(job_dict.get('execute_policy', 1))
|
|
|
|
scheduler.add_job(
|
|
func=execute_job,
|
|
trigger=trigger,
|
|
args=[job_id, job_url, job_sync, job_trigger, job_params],
|
|
id=scheduler_job_id,
|
|
name=job_name,
|
|
replace_existing=True,
|
|
**misfire_config
|
|
)
|
|
|
|
logger.info(f"任务已添加到调度器 [ID: {job_id}, cron: {job_cron}]")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"添加任务到调度器异常: {str(e)}")
|
|
return False
|
|
|
|
|
|
def remove_job_from_scheduler(job_id: int) -> bool:
|
|
"""从调度器移除任务"""
|
|
try:
|
|
get_scheduler().remove_job(f"job_{job_id}")
|
|
return True
|
|
except JobLookupError:
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"移除任务异常: {str(e)}")
|
|
return False
|
|
|
|
|
|
def pause_job_in_scheduler(job_id: int) -> bool:
|
|
"""暂停调度器中的任务"""
|
|
try:
|
|
get_scheduler().pause_job(f"job_{job_id}")
|
|
return True
|
|
except JobLookupError:
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"暂停任务异常: {str(e)}")
|
|
return False
|
|
|
|
|
|
def resume_job_in_scheduler(job_id: int) -> bool:
|
|
"""恢复调度器中的任务"""
|
|
try:
|
|
get_scheduler().resume_job(f"job_{job_id}")
|
|
return True
|
|
except JobLookupError:
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"恢复任务异常: {str(e)}")
|
|
return False
|
|
|
|
|
|
def run_job_once(job_id: int, params: Optional[Dict] = None) -> Dict:
|
|
"""立即执行一次任务（不依赖调度器）"""
|
|
try:
|
|
from apps.models.job import Job
|
|
from extensions import get_db_session
|
|
|
|
db = get_db_session()
|
|
job = db.query(Job).filter(Job.id == job_id, Job.is_delete == 0).first()
|
|
if not job:
|
|
return {'success': False, 'error': '任务不存在'}
|
|
|
|
return execute_job(job.id, job.url, job.is_sync, job.job_trigger, params or {})
|
|
|
|
except Exception as e:
|
|
return {'success': False, 'error': str(e)}
|
|
|
|
|
|
# ============================================================
|
|
# 调度器初始化
|
|
# ============================================================
|
|
|
|
def init_scheduler() -> int:
|
|
"""初始化调度器，启动并从数据库加载所有启用的任务"""
|
|
try:
|
|
from apps.models.job import Job
|
|
from extensions.sqlalchemy import SessionLocal
|
|
|
|
scheduler = get_scheduler()
|
|
if not scheduler.running:
|
|
scheduler.start()
|
|
logger.info("调度器已启动")
|
|
|
|
# lifespan 启动阶段无请求上下文，需用独立会话而非请求级 get_db_session()
|
|
session = SessionLocal()
|
|
try:
|
|
jobs = session.query(Job).filter(Job.status == 1, Job.is_delete == 0).all()
|
|
# 转为字典避免 Session 分离问题
|
|
job_dicts = [job.to_dict() for job in jobs]
|
|
finally:
|
|
session.close()
|
|
|
|
success_count = 0
|
|
for job_dict in job_dicts:
|
|
if add_job_to_scheduler(job_dict):
|
|
success_count += 1
|
|
|
|
logger.info(f"调度器初始化完成，加载 {success_count}/{len(job_dicts)} 个任务")
|
|
return success_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"初始化调度器异常: {str(e)}")
|
|
return 0
|
|
|
|
|
|
def shutdown_scheduler():
|
|
"""关闭调度器"""
|
|
try:
|
|
scheduler = get_scheduler()
|
|
if scheduler.running:
|
|
scheduler.shutdown(wait=False)
|
|
logger.info("调度器已关闭")
|
|
except Exception as e:
|
|
logger.error(f"关闭调度器异常: {str(e)}")
|
|
|
|
|
|
def reload_scheduler() -> int:
|
|
"""重新加载调度器：清空所有任务，重新从数据库加载"""
|
|
try:
|
|
scheduler = get_scheduler()
|
|
scheduler.remove_all_jobs()
|
|
return init_scheduler()
|
|
except Exception as e:
|
|
logger.error(f"重新加载调度器异常: {str(e)}")
|
|
return 0
|