python爬虫系列(5.2-使用多线程下载图片)

一、基本步骤讲解

1、导包

import os

import time

import random

import shutil

import re

import queue

import threading

import requests

from lxml import etree

2、定义全局变量

path = os.path.join(os.path.dirname(__file__), 'doutula')

headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',

}

3、定义一个生产者获取图片地址

class Procuder(threading.Thread):

"""

定义一个生产者(抓取网页上的图片url)

"""

def __init__(self, page_queue, img_queue, *args, **kwargs):

super(Procuder, self).__init__(*args, **kwargs)

self.page_queue = page_queue

self.img_queue = img_queue

def run(self):

while True:

if self.page_queue.empty():

break

# 从队列中获取url地址然后添加到方法中

url = self.page_queue.get()

self.parse_page(url)

def parse_page(self, url):

"""

定义获取全部的图片url地址,追加到图片队列中

:param url:

:return:

"""

response = requests.get(url=url, headers=headers)

time.sleep(random.randrange(2))

if response.status_code == 200:

html = etree.HTML(response.text)

imgs = html.xpath('//div[@class="random_article"]/div[@class="col-xs-6 col-sm-3"]/img')

for img in imgs:

if img.get('class') == 'gif':

continue

img_url = img.xpath("./@data-original")[0]

alt = img.xpath('./@alt')[0]

# 正则替换中文中特殊字符

name = re.sub(re.compile('[，。\.\+]'), '', alt)

# 使用os模块中splitext截取文件的后缀名

filename = name + os.path.splitext(img_url)[1]

# 获取到了添加到图片队列中

self.img_queue.put((img_url, filename))

4、定义一个消费者下载图片

class Consumer(threading.Thread):

"""

定义一个消费者类(从生产者中获取图片url地址下载)

"""

def __init__(self, page_queue, img_queue, *args, **kwargs):

super(Consumer, self).__init__(*args, **kwargs)

self.page_queue = page_queue

self.img_queue = img_queue

def run(self):

while True:

if self.img_queue.empty() and self.page_queue.empty():

break

# 从队列里面获取一个值

img_url, filename = self.img_queue.get()

time.sleep(random.randrange(2))

response = requests.get(url=img_url, headers=headers)

print('正在下载==>', img_url, '===', filename)

with open(os.path.join(path, filename), 'wb') as fp:

fp.write(response.content)

5、定义一个main()主运行方法

def create_dir():

"""

定义创建一个文件夹的方法(有就删除再创建没用就删除)

:return:

"""

if os.path.exists(path):

shutil.rmtree(path)

os.makedirs(path)

else:

os.makedirs(path)

def main():

"""

创建一个运行的函数

:return:

"""

# 先创建文件夹

create_dir()

page_queue = queue.Queue(100)

img_queue = queue.Queue(1000)

# 下载图片为1-5页

for x in range(1, 5):

url = 'http://www.doutula.com/article/list/?page={0}'.format(x)

page_queue.put(url)

# 开启5个生产者与5个消费者线程

for x in range(5):

c = Consumer(page_queue, img_queue)

p = Procuder(page_queue, img_queue)

c.start()

p.start()

if __name__ == '__main__':

main()

二、全部的代码

import os

import time

import random

import shutil

import re

import queue

import threading

import requests

from lxml import etree

path = os.path.join(os.path.dirname(__file__), 'doutula')

headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',

}

class Procuder(threading.Thread):

"""

定义一个生产者(抓取网页上的图片url)

"""

def __init__(self, page_queue, img_queue, *args, **kwargs):

super(Procuder, self).__init__(*args, **kwargs)

self.page_queue = page_queue

self.img_queue = img_queue

def run(self):

while True:

if self.page_queue.empty():

break

# 从队列中获取url地址然后添加到方法中

url = self.page_queue.get()

self.parse_page(url)

def parse_page(self, url):

"""

定义获取全部的图片url地址,追加到图片队列中

:param url:

:return:

"""

response = requests.get(url=url, headers=headers)

time.sleep(random.randrange(2))

if response.status_code == 200:

html = etree.HTML(response.text)

imgs = html.xpath('//div[@class="random_article"]/div[@class="col-xs-6 col-sm-3"]/img')

for img in imgs:

if img.get('class') == 'gif':

continue

img_url = img.xpath("./@data-original")[0]

alt = img.xpath('./@alt')[0]

# 正则替换中文中特殊字符

name = re.sub(re.compile('[，。\.\+]'), '', alt)

# 使用os模块中splitext截取文件的后缀名

filename = name + os.path.splitext(img_url)[1]

# 获取到了添加到图片队列中

self.img_queue.put((img_url, filename))

class Consumer(threading.Thread):

"""

定义一个消费者类(从生产者中获取图片url地址下载)

"""

def __init__(self, page_queue, img_queue, *args, **kwargs):

super(Consumer, self).__init__(*args, **kwargs)

self.page_queue = page_queue

self.img_queue = img_queue

def run(self):

while True:

if self.img_queue.empty() and self.page_queue.empty():

break

# 从队列里面获取一个值

img_url, filename = self.img_queue.get()

time.sleep(random.randrange(2))

response = requests.get(url=img_url, headers=headers)

print('正在下载==>', img_url, '===', filename)

with open(os.path.join(path, filename), 'wb') as fp:

fp.write(response.content)

def create_dir():

"""

定义创建一个文件夹的方法(有就删除再创建没用就删除)

:return:

"""

if os.path.exists(path):

shutil.rmtree(path)

os.makedirs(path)

else:

os.makedirs(path)

def main():

"""

创建一个运行的函数

:return:

"""

# 先创建文件夹

create_dir()

page_queue = queue.Queue(100)

img_queue = queue.Queue(1000)

# 下载图片为1-5页

for x in range(1, 5):

url = 'http://www.doutula.com/article/list/?page={0}'.format(x)

page_queue.put(url)

# 开启5个生产者与5个消费者线程

for x in range(5):

c = Consumer(page_queue, img_queue)

p = Procuder(page_queue, img_queue)

c.start()

p.start()

if __name__ == '__main__':

main()