ljj
/
crawler_task_management
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

代码优化

This commit is contained in:
liujianjiang 2025-11-28 17:38:20 +08:00
parent 45edcdea6b
commit 924537fa4c
3 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
main.py
View File

@ -5,7 +5,7 @@ import asyncio
import uvicorn import uvicorn
from pathlib import Path from pathlib import Path
from typing import Dict, Any, Optional, List from typing import Dict, Any, Optional, List
from fastapi import FastAPI, HTTPException, Depends from fastapi import FastAPI, HTTPException, Depends, Header
from public_function.auth import verify_tk_token from public_function.auth import verify_tk_token
from public_function.public_func import read_config, create_logger from public_function.public_func import read_config, create_logger
from model.model import GoodsInfo, DataReceive, AccountCreate, AccountUpdate, DeviceResetData, CrawlerItem from model.model import GoodsInfo, DataReceive, AccountCreate, AccountUpdate, DeviceResetData, CrawlerItem
@ -169,7 +169,7 @@ async def receive_data(task_data: DataReceive, task_manager: Any = Depends(get_t
@app.post("/get_goods_info", summary="客户端获取商品数据") @app.post("/get_goods_info", summary="客户端获取商品数据")
async def get_goods_info(task_data: GoodsInfo, task_manager: Any = Depends(get_task_manager)): async def get_goods_info(task_data: GoodsInfo, task_manager: Any = Depends(get_task_manager), token: Optional[str] = Header(None)):
"""客户端获取商品数据接口""" """客户端获取商品数据接口"""
task_id = uuid.uuid4().hex task_id = uuid.uuid4().hex
try: try:
@ -177,6 +177,7 @@ async def get_goods_info(task_data: GoodsInfo, task_manager: Any = Depends(get_t
params['task_id'] = task_id params['task_id'] = task_id
params["app_name"] = "Shopee" params["app_name"] = "Shopee"
params['region'] = params["host"] params['region'] = params["host"]
params["token"] = token
await task_manager.insert_task_record(params) await task_manager.insert_task_record(params)
print(f"开始处理任务: {task_id}") print(f"开始处理任务: {task_id}")
result = await task_manager.deal_shopee_task(params) result = await task_manager.deal_shopee_task(params)

2
public_function/table_log.sql
View File

@ -6,11 +6,13 @@ create table crawler_task_record_info
shop_id VARCHAR(50) NOT NULL COMMENT '店铺ID', shop_id VARCHAR(50) NOT NULL COMMENT '店铺ID',
item_id VARCHAR(50) NOT NULL COMMENT '商品ID', item_id VARCHAR(50) NOT NULL COMMENT '商品ID',
region VARCHAR(50) NOT NULL COMMENT '客户所在地区', region VARCHAR(50) NOT NULL COMMENT '客户所在地区',
token VARCHAR(128) NOT NULL COMMENT '用户标识',
status int NOT NULL DEFAULT 1 COMMENT '任务状态：1、开始执行；2、执行成功；3、执行失败', status int NOT NULL DEFAULT 1 COMMENT '任务状态：1、开始执行；2、执行成功；3、执行失败',
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (id), PRIMARY KEY (id),
UNIQUE KEY uk_task_id (task_id), UNIQUE KEY uk_task_id (task_id),
KEY idx_status (status),
KEY idx_status (status) KEY idx_status (status)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='爬虫任务执行记录表'; )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='爬虫任务执行记录表';

3
task_management/all_task_management.py
View File

@ -53,10 +53,11 @@ class AllTask:
await self.db_pool.initialize() await self.db_pool.initialize()
try: try:
insert_param = {"task_id": params["task_id"], "app_name": params["app_name"], "region": params["host"], insert_param = {"task_id": params["task_id"], "app_name": params["app_name"], "region": params["host"],
"shop_id": params["shop_id"], "item_id": params["item_id"], "status": 1} "shop_id": params["shop_id"], "item_id": params["item_id"], "status": 1, "token": params["token"]}
result = await self.db_pool.insert_many("crawler_task_record_info", [insert_param]) result = await self.db_pool.insert_many("crawler_task_record_info", [insert_param])
if result: if result:
print(f"{params['task_id']} 数据存入mysql{result} 行成功") print(f"{params['task_id']} 数据存入mysql{result} 行成功")
else:
print(f"{params['task_id']} 数据存入mysql失败") print(f"{params['task_id']} 数据存入mysql失败")
except Exception as e: except Exception as e:
print(f"{params['task_id']} 数据存入mysql失败，失败原因为{e}") print(f"{params['task_id']} 数据存入mysql失败，失败原因为{e}")