concurrent.futures.ProcessPoolExecutor.map may be slow in some cases


A simple test using Python 3.3

Sample Code

from __future__ import print_function

from concurrent import futures
import math
import multiprocessing


def is_prime(num):
    if num % 2 == 0:
        return False

    sqrt_num = int(math.floor(math.sqrt(num)))
    for i in range(3, sqrt_num + 1, 2):
        if num % i == 0:
            return False

    return True


def prime_worker(count):
    return sorted(num for num in range(count) if is_prime(num))


def future_prime_worker(count):
    with futures.ProcessPoolExecutor(4) as executor:
        numbers = range(count)
        return sorted(num for num, prime in
                      zip(numbers, executor.map(is_prime, numbers)) if prime)


def multiprocess_prime_worker(count):
    pool = multiprocessing.Pool(4)
    numbers = range(count)
    return sorted(num for num, prime in
                  zip(numbers, pool.map(is_prime, numbers)) if prime)


if __name__ == '__main__':
    import timeit
    t = timeit.timeit("prime_worker(200000)",
                      number=1,
                      setup="from __main__ import prime_worker")
    print (t)

    t = timeit.timeit("multiprocess_prime_worker(200000)",
                      number=1,
                      setup="from __main__ import multiprocess_prime_worker")
    print (t)

    t = timeit.timeit("future_prime_worker(200000)",
                      number=1,
                      setup="from __main__ import future_prime_worker")
    print (t)

Result

1.1414704178459942
0.7401300449855626
88.23592492006719

References:
concurrent.futures.ProcessPoolExecutor.map() doesn’t batch function arguments by chunks


Leave a Reply

Your email address will not be published. Required fields are marked *