Python图片爬虫
1.今天给大家介绍自己写的一个图片爬虫，说白了就是从网页自动上下载需要的图片
2.首先选取目标为：http://www.zhangzishi.cc/涨姿势这个网站如下图，我们的目标就是爬取该网站福利社的所有美图
3.福利社地址为http://www.zhangzishi.cc/category/welfare，获取图片就是获取所有网站图片的url地址,首先
A.打开URL,获取html代码
def url_open(url): req = urllib.request.Request(url) req.add_header(\'User-Agent\',\'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36\') response = urllib.request.urlopen(req) html = response.read() print(\'url_open\') return html
B.从html代码中摘取网页链接，返回的是一个列表
def page_htmls(url,count): html = url_open(url).decode(\'utf-8\') pages = [] a = html.find(\'a target="_blank" href=\') i = 0 while a != -1: i += 1 b = html.find(\'.html\',a,a+200) if b != -1: pages.append(html[a+24:b+5]) else: b = a + 24 a = html.find(\'a target="_blank" href=\',b) if i == count: break for each in pages: print(each) return pages
C.从每一个链接页中获取图片地址，我这用了两种方法
def find_imgs(url): html = url_open(url).decode(\'utf-8\') imgs = [] a = html.find(\'img src=\') while a != -1: b = html.find(\'.jpg\',a,a+100) if b != -1: if html[a+9:b+4].find(\'http\') == -1: imgs.append(\'http:\'+html[a+9:b+4]) else: imgs.append(html[a+9:b+4]) else: b = a + 9 a = html.find(\'img src=\',b) \'\'\' for each in imgs: print(each) \'\'\' return imgs def imgurl_get(url): html = url_open(url).decode(\'utf-8\') imgurls = [] a = html.find(\'color: #555555;" src=\') while a != -1: b = html.find(\'.jpg\',a,a+100) if b != -1: imgurls.append(\'http:\'+html[a+22:b+4]) else: b = a + 22 a = html.find(\'color: #555555;" src=\',b) return imgurls
D.根据图片url下载图片到文件
def save_imgs(folder,imgs): for ea in imgs: filename = ea.split(\'/\')[-1] with open(filename,\'wb\') as f: img = url_open(ea) f.write(img) def download_mm(folder=\'H:\\xxoo2\',page_count = 100,count = 100): main_url = \'http://www.zhangzishi.cc/category/welfare\' main_urls = [] for i in range(count): main_urls.append(main_url+\'/page/\'+str(i+1)) os.mkdir(folder) os.chdir(folder) for url in main_urls: htmls = page_htmls(url,page_count) for page in htmls: imgurls = imgurl_get(page) save_imgs(folder,imgurls)
E.开始下载
def download__img(folder=\'H:\\xxoo\',page_count=100): main_url = \'http://www.zhangzishi.cc/category/welfare\' os.mkdir(folder) os.chdir(folder) htmls = page_htmls(main_url,page_count) for page in htmls: imgs_url = find_imgs(page) save_imgs(folder,imgs_url) if __name__ == \'__main__\': download_mm() #download__img()
F:下载结果
顺便附上全部代码：
import urllib.request import os def url_open(url): req = urllib.request.Request(url) req.add_header(\'User-Agent\',\'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36\') response = urllib.request.urlopen(req) html = response.read() print(\'url_open\') return html def page_htmls(url,count): html = url_open(url).decode(\'utf-8\') pages = [] a = html.find(\'a target="_blank" href=\') i = 0 while a != -1: i += 1 b = html.find(\'.html\',a,a+200) if b != -1: pages.append(html[a+24:b+5]) else: b = a + 24 a = html.find(\'a target="_blank" href=\',b) if i == count: break for each in pages: print(each) return pages \'\'\' \'\'\' def find_imgs(url): html = url_open(url).decode(\'utf-8\') imgs = [] a = html.find(\'img src=\') while a != -1: b = html.find(\'.jpg\',a,a+100) if b != -1: if html[a+9:b+4].find(\'http\') == -1: imgs.append(\'http:\'+html[a+9:b+4]) else: imgs.append(html[a+9:b+4]) else: b = a + 9 a = html.find(\'img src=\',b) \'\'\' for each in imgs: print(each) \'\'\' return imgs def imgurl_get(url): html = url_open(url).decode(\'utf-8\') imgurls = [] a = html.find(\'color: #555555;" src=\') while a != -1: b = html.find(\'.jpg\',a,a+100) if b != -1: imgurls.append(\'http:\'+html[a+22:b+4]) else: b = a + 22 a = html.find(\'color: #555555;" src=\',b) return imgurls \'\'\' for each in imgurls: print(each) \'\'\' def save_imgs(folder,imgs): for ea in imgs: filename = ea.split(\'/\')[-1] with open(filename,\'wb\') as f: img = url_open(ea) f.write(img) def download_mm(folder=\'H:\\xxoo2\',page_count = 100,count = 100): main_url = \'http://www.zhangzishi.cc/category/welfare\' main_urls = [] for i in range(count): main_urls.append(main_url+\'/page/\'+str(i+1)) os.mkdir(folder) os.chdir(folder) for url in main_urls: htmls = page_htmls(url,page_count) for page in htmls: imgurls = imgurl_get(page) save_imgs(folder,imgurls) def download__img(folder=\'H:\\xxoo\',page_count=100): main_url = \'http://www.zhangzishi.cc/category/welfare\' os.mkdir(folder) os.chdir(folder) htmls = page_htmls(main_url,page_count) for page in htmls: imgs_url = find_imgs(page) save_imgs(folder,imgs_url) if __name__ == \'__main__\': download_mm() #download__img()
版权声明：本文为huipengbo原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。