获取tr下td的最佳方式

tr = bs.select(".list_tab tr")
# 一般tr[0]是表头数据，所以从1开始循环处理
for item in tr[1:]:
    tds = list(item.stripped_strings)
    # tds 按数组下标获取对应的值就可以了

登录后保持cookie

# 保持session
s = requests.Session()
# 请求登录
r = s.post(loginUrl, headers=self.headers, data=formData)
# 请求数据
r = s.get(dataUrl)

存储及使用cookie

存储cookie

s = requests.Session()
r = s.post(loginUrl, headers=self.headers, data=formData)
cookies = r.cookies.get_dict()
cookie_dict = requests.utils.dict_from_cookiejar(r.cookies)
cookie_str = json.dumps(cookie_dict)
# 将cookie以字符串形式存入redis
self.red.set("sale:spider:cookie", cookie_str)

使用cookie

cur_cookie = self.red.get("sale:spider:cookie")
session = requests.Session()
cookie_dict = re.sub("'", "\"", cur_cookie)
cookie_dict = json.loads(cookie_dict)
session.cookies = requests.utils.cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True)
 r = session.get(dataUrl, headers=self.headers)

获取字符串里的数值,用户提取价格等场景

newPrice = re.findall(r"\d+\.?\d*", oldPrice)
newPrice = float(newPrice)

获取最外层文本数据，不包含下级

数据如：
<p>价格：<span>12.5元/只</span></p>
如何只提取“价格”这两个字呢?

ul = bs.select(".msg_detail ul li")

arcitle_title = bs.select_one("body > div.article_box > p")

release_time = bs.select_one("div.article_time.cz_center span")

company_name = bs.select_one("body > div.msg_detail > ul > li > span")

读取配置文件

# 配置文件config.ini
[env]
# dev, test, prod
select = prod

# 读取配置文件
self.config = configparser.ConfigParser()
self.config.read("pig-config.ini", encoding="utf-8")
self.env = self.config["env"]["select"]

日志初始化

self.log = logging.getLogger()
self.log.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")

log_file_handler = TimedRotatingFileHandler(filename="./logs/log.txt", when="midnight", interval=1,
                                  backupCount=90)
        log_file_handler.setFormatter(formatter)
        log_file_handler.setLevel(logging.INFO)
        self.log.addHandler(log_file_handler)

stream_handler = logging.StreamHandler(sys.stdout)
        stream_handler.setFormatter(formatter)
        self.log.addHandler(stream_handler)