27 lines
1.0 KiB
Python
27 lines
1.0 KiB
Python
import importlib
|
|
|
|
from database.database import get_session
|
|
from database.tinformationsource.curd import get_active_information_sources
|
|
from log.log_manager import logger
|
|
from task.manager_task import execute_task
|
|
|
|
|
|
def main_spider_task():
|
|
with get_session() as db:
|
|
information_sources = get_active_information_sources(db)
|
|
for information_source in information_sources:
|
|
if information_source.module is None or information_source.method is None:
|
|
logger.error(f"{information_source.title} module or method is None")
|
|
continue
|
|
# 动态导入模块和函数
|
|
module = importlib.import_module(information_source.module)
|
|
task_function = getattr(module, information_source.method)
|
|
try:
|
|
task_function(information_source)
|
|
except Exception as e:
|
|
logger.error(f"{information_source.title} task error: {e}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
execute_task(main_spider_task)
|