ljj
/
crawler_task_management
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

代码优化

This commit is contained in:
liujianjiang 2025-11-28 17:38:20 +08:00
parent 45edcdea6b
commit 924537fa4c
3 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
main.py
View File

@ -5,7 +5,7 @@ import asyncio
import uvicorn
from pathlib import Path
from typing import Dict, Any, Optional, List
from fastapi import FastAPI, HTTPException, Depends
from fastapi import FastAPI, HTTPException, Depends, Header
from public_function.auth import verify_tk_token
from public_function.public_func import read_config, create_logger
from model.model import GoodsInfo, DataReceive, AccountCreate, AccountUpdate, DeviceResetData, CrawlerItem
@ -169,7 +169,7 @@ async def receive_data(task_data: DataReceive, task_manager: Any = Depends(get_t
@app.post("/get_goods_info", summary="客户端获取商品数据")
async def get_goods_info(task_data: GoodsInfo, task_manager: Any = Depends(get_task_manager)):
async def get_goods_info(task_data: GoodsInfo, task_manager: Any = Depends(get_task_manager), token: Optional[str] = Header(None)):
"""客户端获取商品数据接口"""
task_id = uuid.uuid4().hex
try:
@ -177,6 +177,7 @@ async def get_goods_info(task_data: GoodsInfo, task_manager: Any = Depends(get_t
params['task_id'] = task_id
params["app_name"] = "Shopee"
params['region'] = params["host"]
params["token"] = token
await task_manager.insert_task_record(params)
print(f"开始处理任务: {task_id}")
result = await task_manager.deal_shopee_task(params)

18
public_function/table_log.sql
View File

@ -1,16 +1,18 @@
create table crawler_task_record_info
(
id INT(11) NOT NULL AUTO_INCREMENT COMMENT '表自增ID',
task_id varchar(50) NOT NULL COMMENT '任务ID',
app_name VARCHAR(50) NOT NULL COMMENT 'app名称',
shop_id VARCHAR(50) NOT NULL COMMENT '店铺ID',
item_id VARCHAR(50) NOT NULL COMMENT '商品ID',
region VARCHAR(50) NOT NULL COMMENT '客户所在地区',
status int NOT NULL DEFAULT 1 COMMENT '任务状态：1、开始执行；2、执行成功；3、执行失败',
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
task_id varchar(50) NOT NULL COMMENT '任务ID',
app_name VARCHAR(50) NOT NULL COMMENT 'app名称',
shop_id VARCHAR(50) NOT NULL COMMENT '店铺ID',
item_id VARCHAR(50) NOT NULL COMMENT '商品ID',
region VARCHAR(50) NOT NULL COMMENT '客户所在地区',
token VARCHAR(128) NOT NULL COMMENT '用户标识',
status int NOT NULL DEFAULT 1 COMMENT '任务状态：1、开始执行；2、执行成功；3、执行失败',
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (id),
UNIQUE KEY uk_task_id (task_id),
KEY idx_status (status),
KEY idx_status (status)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='爬虫任务执行记录表';

5
task_management/all_task_management.py
View File

@ -53,11 +53,12 @@ class AllTask:
await self.db_pool.initialize()
try:
insert_param = {"task_id": params["task_id"], "app_name": params["app_name"], "region": params["host"],
"shop_id": params["shop_id"], "item_id": params["item_id"], "status": 1}
"shop_id": params["shop_id"], "item_id": params["item_id"], "status": 1, "token": params["token"]}
result = await self.db_pool.insert_many("crawler_task_record_info", [insert_param])
if result:
print(f"{params['task_id']} 数据存入mysql{result} 行成功")
print(f"{params['task_id']} 数据存入mysql失败")
else:
print(f"{params['task_id']} 数据存入mysql失败")
except Exception as e:
print(f"{params['task_id']} 数据存入mysql失败，失败原因为{e}")