PythonParallel - mwicat/personal GitHub Wiki

python parallel map

from multiprocessing import Pool


pool_size = 100
pool = Pool(processes=pool_size)

lst = range(10)

def f(x):
    return x*2

for i, result in enumerate(pool.imap_unordered(count, lst)):
    print('%d/%d' % (i, len(lst)))

parallel apply

from multiprocessing import Pool


pool_size = 100
pool = Pool(processes=pool_size)

lst = range(10)

def f(x):
    return x*2

def callback(result):
    print('finished')


x = 10
pool.apply_async(f, x, {}, callback)

parallel requests

sudo pip install requests-futures

from requests_futures.sessions import FuturesSession

session = FuturesSession(max_workers=1000)

def on_done():
    res = future.result()
    if res.status_code != 200:
        print(res.status_code)
    print(res.content)


fs = []

future = session.get(url, headers=headers)
future.add_done_callback(on_done)
fs.append(future)

Threading

from threading import Thread
thread = Thread(target=func1, args=(arg1, arg2))
thread.start()
thread.join()