45 lines
1.7 KiB
Python
45 lines
1.7 KiB
Python
# -*- coding: utf-8 -*-
|
||
import time
|
||
from typing import Dict, Any
|
||
|
||
from public_function.asyn_mysql import AsyncMySQL
|
||
from public_function.redis_task_manager import RedisTaskManager
|
||
|
||
|
||
class AllTask:
|
||
def __init__(self, config_data: Dict[str, Any]):
|
||
self.config_data = config_data
|
||
self.redis_conn = RedisTaskManager(self.config_data["redis_config"])
|
||
self.db_pool: Optional[AsyncMySQL] = AsyncMySQL(self.config_data["advert_policy"])
|
||
|
||
async def deal_shopee_task(self, param):
|
||
# 查询redis数据库，redis 数据库存在该数据直接返回
|
||
key = f"{param['app_name']}:{param['store_id']}:{param['goods_id']}"
|
||
result = self.redis_conn.read_data(key)
|
||
if result:
|
||
return result
|
||
# 调用对应爬虫任务
|
||
# 任务结束后开始等待
|
||
endtime = time.time() + 55
|
||
while time.time() < endtime:
|
||
await time.sleep(1)
|
||
result = self.redis_conn.read_data(key)
|
||
if result:
|
||
return result
|
||
return []
|
||
|
||
async def task_distribution(self, data: Dict[str, Any]):
|
||
# 需要对任务进行记录
|
||
try:
|
||
await self.db_pool.initialize()
|
||
# 将任务记录到mysql
|
||
param = {"app_name": data["app_name"], "task_id": data["task_id"], "country": data["country"]}
|
||
await self.db_pool.insert_many(table="crawler_task_record_info", data=param)
|
||
except Exception as e:
|
||
print("将任务记录到数据库失败，失败原因为：{}".format(e))
|
||
if param["app_name"] == "Shopee":
|
||
result = await self.deal_shopee_task(param)
|
||
if result:
|
||
return result
|
||
return None
|