简单的看一二例子即可

import requests
#代理ip 高频的ip容易被封,所以使用ip代理
#免费代理 ip:www.goubanjia.com    快代理  西祠代理
headers={
    \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36\'
}
url=\'https://www.baidu.com/s?wd=ip\'
#ip代理的使用proxies,先指定协议的类型,http,https,再指定ip的端口
page_text=requests.get(url=url,headers=headers,proxies={\'https\':\'1.10.186.79:8080\'}).text

with open(\'ip.html\',\'w\',encoding=\'utf-8\') as f:
    f.write(page_text)

import requests
import re
import random
from multiprocessing.dummy import Pool
from lxml import etree

#多线程爬取一定是在耗时比较多的情况下
headers={
    \'User-Agent\':\'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER\'
}
#爬取第一页数据
url=\'https://www.pearvideo.com/category_6\'
page_text=requests.get(url=url,headers=headers).text
tree=etree.HTML(page_text)
#xpath解析返回一个列表
li_list=tree.xpath(\'//ul[@id="listvideoListUl"]/li\')
# print(li_list)
#循环列表获取视频的url

#解析出视频详情页的url
video_url_list=[]
for li in li_list:
    detail_url=\'https://www.pearvideo.com/\'+li.xpath(\'./div/a/@href\')[0]
    detail_page_text=requests.get(url=detail_url,headers=headers).text
    ex=\'srcUrl="(.*?)",vdoUrl\'
    video_url=re.findall(ex,detail_page_text,re.S)[0]#re.单行匹配,一定要记得写,返回的额还是列表
    # 存放的是所有视频的url
    video_url_list.append(video_url.rstrip(\'"\'))

#从网页直接获取视频
def video_download(url):
    return requests.get(url=url,headers=headers).content
#保存下来的视频
def save_video(data):
    name=str(random.randint(1,5000))+\'.mp4\'
    with open(name,\'wb\') as f:
        f.write(data)
        print(name+\'保存成功\')

if __name__ == \'__main__\':
    # #获取视频数据和写入都是比较耗时间的,我我们就用多进程进行会比较块
    # #实例化一个进程池
    pool=Pool(5)
    #map函数接收两个参数,第一是函数,第二是可迭代对象,函数会依次作用到可迭代对象的每一个元素
    video_data=pool.map(video_download,video_url_list)
    print("-->",video_data)
    #持久化存储视频的过程也是比价耗时间的,所以需要多进程会比较块一点
    pool.map(save_video,video_data)
    pool.close()
    pool.join()