1 基本介绍

taskpool.py代码如下，

import Queue
import threading
import time


class Reportthread(threading.Thread):
    """
    reportthread
    """
    def __init__(self, iqueue, max):
        """
        init
        :param iqueue:
        :param max:
        """
        threading.Thread.__init__(self)
        self.iqueue = iqueue
        self.max = max

    def run(self):
        """
        run
        :return:
        """
        while True:
            s = self.iqueue.qsize()
            if s == 0:
                break
            print 'reportthread, %d' % s
            time.sleep(1)


class Taskthread(threading.Thread):
    """
    Taskthread
    """

    def __init__(self, iqueue, oqueue, equeue):
        """
        init
        :param iqueue:
        :param oqueue:
        :param equeue:
        """
        threading.Thread.__init__(self)
        self.iqueue = iqueue
        self.oqueue = oqueue
        self.equeue = equeue

    def run(self):
        """
        run
        :return:
        """
        while True:
            if self.iqueue.empty():
                return
            try:
                func, param = self.iqueue.get(timeout=1)
            except Exception as e:
                self.equeue.put((None, e))
                return

            try:
                result = func(param)
            except Exception as e:
                self.equeue.put((param, e))
            else:
                self.oqueue.put((param, result))
            finally:
                self.iqueue.task_done()


def process(func, params, size=2, report=False):
    """
    process
    :param func:
    :param params:
    :param size:
    :param report:
    :return:
    """
    iqueue = Queue.Queue(maxsize=1000000)
    oqueue = Queue.Queue()
    equeue = Queue.Queue()
    for param in params:
        iqueue.put((func, param))

    for i in range(size):
        t = Taskthread(iqueue, oqueue, equeue)
        t.setDaemon(True)
        t.start()

    rpt = None
    if report:
        rpt = Reportthread(iqueue, len(params))
        rpt.setDaemon(True)
        rpt.start()
    iqueue.join()
    if report:
        rpt.join()

    output = []
    while not oqueue.empty():
        output.append(oqueue.get())
    exception = []
    while not equeue.empty():
        exception.append(equeue.get())

    return (output, exception)

2 使用

2.1 case1

main.py代码如下，

import os 
import time 
import sys 
import taskpool 

def worker(input):
    x, y = input 
    return x * y 

def main_v1():
    input_list = [(1,2),(3,4),(5,6),(7,8),(9,10),(10,11)]
    results, exception = taskpool.process(worker, input_list, 2, True) #2表示线程数
    for result in results:
        print("input = {}, output = {}.".format(result[0], result[1]))

    return 

if __name__ == "__main__":
    start_t = time.time()

    main_v1()

    end_t = time.time()
    print("used time: {}!".format((end_t - start_t) / 60.0)) 

执行代码，结果为，

input = (1, 2), output = 2.
input = (3, 4), output = 12.
input = (5, 6), output = 30.
input = (7, 8), output = 56.
input = (9, 10), output = 90.
input = (10, 11), output = 110.
used time: 1.52150789897e-05!