import threading
class SummingThread(threading.Thread):def __init__(self,low,high):super(SummingThread, self).__init__()self.low=lowself.high=highself.total=0
def run(self):for i in range(self.low,self.high):self.total+=i
thread1 = SummingThread(0,500000)thread2 = SummingThread(500000,1000000)thread1.start() # This actually causes the thread to runthread2.start()thread1.join() # This waits until the thread has completedthread2.join()# At this point, both threads have completedresult = thread1.total + thread2.totalprint result
import Queueimport threadingimport urllib2
# Called by each threaddef get_url(q, url):q.put(urllib2.urlopen(url).read())
theurls = ["http://google.com", "http://yahoo.com"]
q = Queue.Queue()
for u in theurls:t = threading.Thread(target=get_url, args = (q,u))t.daemon = Truet.start()
s = q.get()print s
import threadingfrom random import randintfrom time import sleep
def print_number(number):
# Sleeps a random 1 to 10 secondsrand_int_var = randint(1, 10)sleep(rand_int_var)print "Thread " + str(number) + " slept for " + str(rand_int_var) + " seconds"
thread_list = []
for i in range(1, 10):
# Instantiates the thread# (i) does not make a sequence, so (i,)t = threading.Thread(target=print_number, args=(i,))# Sticks the thread in a list so that it remains accessiblethread_list.append(t)
# Starts threadsfor thread in thread_list:thread.start()
# This blocks the calling thread until the thread whose join() method is called is terminated.# From http://docs.python.org/2/library/threading.html#thread-objectsfor thread in thread_list:thread.join()
# Demonstrates that the main process waited for threads to completeprint "Done"
try:# For Python 3import queuefrom urllib.request import urlopenexcept:# For Python 2import Queue as queuefrom urllib2 import urlopen
import threading
worker_data = ['http://google.com', 'http://yahoo.com', 'http://bing.com']
# Load up a queue with your data. This will handle lockingq = queue.Queue()for url in worker_data:q.put(url)
# Define a worker functiondef worker(url_queue):queue_full = Truewhile queue_full:try:# Get your data off the queue, and do some workurl = url_queue.get(False)data = urlopen(url).read()print(len(data))
except queue.Empty:queue_full = False
# Create as many threads as you wantthread_count = 5for i in range(thread_count):t = threading.Thread(target=worker, args = (q,))t.start()
import Queueimport threadingimport multiprocessingimport subprocess
q = Queue.Queue()for i in range(30): # Put 30 tasks in the queueq.put(i)
def worker():while True:item = q.get()# Execute a task: call a shell program and wait until it completessubprocess.call("echo " + str(item), shell=True)q.task_done()
cpus = multiprocessing.cpu_count() # Detect number of coresprint("Creating %d threads" % cpus)for i in range(cpus):t = threading.Thread(target=worker)t.daemon = Truet.start()
q.join() # Block until all tasks are done
import urllib2from multiprocessing.dummy import Pool as ThreadPool
urls = ['http://www.python.org','http://www.python.org/about/','http://www.onlamp.com/pub/a/python/2003/04/17/metaclasses.html','http://www.python.org/doc/','http://www.python.org/download/','http://www.python.org/getit/','http://www.python.org/community/','https://wiki.python.org/moin/',]
# Make the Pool of workerspool = ThreadPool(4)
# Open the URLs in their own threads# and return the resultsresults = pool.map(urllib2.urlopen, urls)
# Close the pool and wait for the work to finishpool.close()pool.join()
def sqr(val):import timetime.sleep(0.1)return val * val
def process_result(result):print(result)
def process_these_asap(tasks):import concurrent.futures
with concurrent.futures.ProcessPoolExecutor() as executor:futures = []for task in tasks:futures.append(executor.submit(sqr, task))
for future in concurrent.futures.as_completed(futures):process_result(future.result())# Or instead of all this just do:# results = executor.map(sqr, tasks)# list(map(process_result, results))
def main():tasks = list(range(10))print('Processing {} tasks'.format(len(tasks)))process_these_asap(tasks)print('Done')return 0
if __name__ == '__main__':import syssys.exit(main())
import concurrent.futuresimport urllib.request
URLS = ['http://www.foxnews.com/','http://www.cnn.com/','http://europe.wsj.com/','http://www.bbc.co.uk/','http://some-made-up-domain.com/']
# Retrieve a single page and report the URL and contentsdef load_url(url, timeout):with urllib.request.urlopen(url, timeout=timeout) as conn:return conn.read()
# We can use a with statement to ensure threads are cleaned up promptlywith concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:# Start the load operations and mark each future with its URLfuture_to_url = {executor.submit(load_url, url, 60): url for url in URLS}for future in concurrent.futures.as_completed(future_to_url):url = future_to_url[future]try:data = future.result()except Exception as exc:print('%r generated an exception: %s' % (url, exc))else:print('%r page is %d bytes' % (url, len(data)))
import concurrent.futuresimport math
PRIMES = [112272535095293,112582705942171,112272535095293,115280095190773,115797848077099,1099726899285419]
def is_prime(n):if n % 2 == 0:return False
sqrt_n = int(math.floor(math.sqrt(n)))for i in range(3, sqrt_n + 1, 2):if n % i == 0:return Falsereturn True
def main():with concurrent.futures.ProcessPoolExecutor() as executor:for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)):print('%d is prime: %s' % (number, prime))
if __name__ == '__main__':main()
from concurrent.futures import ThreadPoolExecutor, as_completed
def get_url(url):# Your actual program here. Using threading.Lock() if necessaryreturn ""
# List of URLs to fetchurls = ["url1", "url2"]
with ThreadPoolExecutor(max_workers = 5) as executor:
# Create threadsfutures = {executor.submit(get_url, url) for url in urls}
# as_completed() gives you the threads once finishedfor f in as_completed(futures):# Get the resultsrs = f.result()
import mathimport timeitimport threadingimport multiprocessingfrom concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
def time_stuff(fn):"""Measure time of execution of a function"""def wrapper(*args, **kwargs):t0 = timeit.default_timer()fn(*args, **kwargs)t1 = timeit.default_timer()print("{} seconds".format(t1 - t0))return wrapper
def find_primes_in(nmin, nmax):"""Compute a list of prime numbers between the given minimum and maximum arguments"""primes = []
# Loop from minimum to maximumfor current in range(nmin, nmax + 1):
# Take the square root of the current numbersqrt_n = int(math.sqrt(current))found = False
# Check if the any number from 2 to the square root + 1 divides the current numnber under considerationfor number in range(2, sqrt_n + 1):
# If divisible we have found a factor, hence this is not a prime number, lets move to the next oneif current % number == 0:found = Truebreak
# If not divisible, add this number to the list of primes that we have found so farif not found:primes.append(current)
# I am merely printing the length of the array containing all the primes, but feel free to do what you wantprint(len(primes))
@time_stuffdef sequential_prime_finder(nmin, nmax):"""Use the main process and main thread to compute everything in this case"""find_primes_in(nmin, nmax)
@time_stuffdef threading_prime_finder(nmin, nmax):"""If the minimum is 1000 and the maximum is 2000 and we have four workers,1000 - 1250 to worker 11250 - 1500 to worker 21500 - 1750 to worker 31750 - 2000 to worker 4so let’s split the minimum and maximum values according to the number of workers"""nrange = nmax - nminthreads = []for i in range(8):start = int(nmin + i * nrange/8)end = int(nmin + (i + 1) * nrange/8)
# Start the thread with the minimum and maximum split up to compute# Parallel computation will not work here due to the GIL since this is a CPU-bound taskt = threading.Thread(target = find_primes_in, args = (start, end))threads.append(t)t.start()
# Don’t forget to wait for the threads to finishfor t in threads:t.join()
@time_stuffdef processing_prime_finder(nmin, nmax):"""Split the minimum, maximum interval similar to the threading method above, but use processes this time"""nrange = nmax - nminprocesses = []for i in range(8):start = int(nmin + i * nrange/8)end = int(nmin + (i + 1) * nrange/8)p = multiprocessing.Process(target = find_primes_in, args = (start, end))processes.append(p)p.start()
for p in processes:p.join()
@time_stuffdef thread_executor_prime_finder(nmin, nmax):"""Split the min max interval similar to the threading method, but use a thread pool executor this time.This method is slightly faster than using pure threading as the pools manage threads more efficiently.This method is still slow due to the GIL limitations since we are doing a CPU-bound task."""nrange = nmax - nminwith ThreadPoolExecutor(max_workers = 8) as e:for i in range(8):start = int(nmin + i * nrange/8)end = int(nmin + (i + 1) * nrange/8)e.submit(find_primes_in, start, end)
@time_stuffdef process_executor_prime_finder(nmin, nmax):"""Split the min max interval similar to the threading method, but use the process pool executor.This is the fastest method recorded so far as it manages process efficiently + overcomes GIL limitations.RECOMMENDED METHOD FOR CPU-BOUND TASKS"""nrange = nmax - nminwith ProcessPoolExecutor(max_workers = 8) as e:for i in range(8):start = int(nmin + i * nrange/8)end = int(nmin + (i + 1) * nrange/8)e.submit(find_primes_in, start, end)
def main():nmin = int(1e7)nmax = int(1.05e7)print("Sequential Prime Finder Starting")sequential_prime_finder(nmin, nmax)print("Threading Prime Finder Starting")threading_prime_finder(nmin, nmax)print("Processing Prime Finder Starting")processing_prime_finder(nmin, nmax)print("Thread Executor Prime Finder Starting")thread_executor_prime_finder(nmin, nmax)print("Process Executor Finder Starting")process_executor_prime_finder(nmin, nmax)if __name__ == "__main__":main()
这是我的Mac OS X四核机器上的结果
Sequential Prime Finder Starting9.708213827005238 secondsThreading Prime Finder Starting9.81836523200036 secondsProcessing Prime Finder Starting3.2467174359990167 secondsThread Executor Prime Finder Starting10.228896902000997 secondsProcess Executor Finder Starting2.656402041000547 seconds
from concurrent.futures import ThreadPoolExecutor, as_completedfrom time import sleep, time
def concurrent(max_worker):futures = []tic = time()with ThreadPoolExecutor(max_workers=max_worker) as executor:futures.append(executor.submit(sleep, 2)) # Two seconds sleepfutures.append(executor.submit(sleep, 1))futures.append(executor.submit(sleep, 7))futures.append(executor.submit(sleep, 3))for future in as_completed(futures):if future.result() is not None:print(future.result())print(f'Total elapsed time by {max_worker} workers:', time()-tic)
concurrent(5)concurrent(4)concurrent(3)concurrent(2)concurrent(1)
输出:
Total elapsed time by 5 workers: 7.007831811904907Total elapsed time by 4 workers: 7.007944107055664Total elapsed time by 3 workers: 7.003149509429932Total elapsed time by 2 workers: 8.004627466201782Total elapsed time by 1 workers: 13.013478994369507
#!/bin/pythonfrom multiprocessing.dummy import Poolfrom subprocess import PIPE,Popenimport timeimport os
# In the variable pool_size we define the "parallelness".# For CPU-bound tasks, it doesn't make sense to create more Pool processes# than you have cores to run them on.## On the other hand, if you are using I/O-bound tasks, it may make sense# to create a quite a few more Pool processes than cores, since the processes# will probably spend most their time blocked (waiting for I/O to complete).pool_size = 8
def do_ping(ip):if os.name == 'nt':print ("Using Windows Ping to " + ip)proc = Popen(['ping', ip], stdout=PIPE)return proc.communicate()[0]else:print ("Using Linux / Unix Ping to " + ip)proc = Popen(['ping', ip, '-c', '4'], stdout=PIPE)return proc.communicate()[0]
os.system('cls' if os.name=='nt' else 'clear')print ("Running using threads\n")start_time = time.time()pool = Pool(pool_size)website_names = ["www.google.com","www.facebook.com","www.pinterest.com","www.microsoft.com"]result = {}for website_name in website_names:result[website_name] = pool.apply_async(do_ping, args=(website_name,))pool.close()pool.join()print ("\n--- Execution took {} seconds ---".format((time.time() - start_time)))
# Now we do the same without threading, just to compare timeprint ("\nRunning NOT using threads\n")start_time = time.time()for website_name in website_names:do_ping(website_name)print ("\n--- Execution took {} seconds ---".format((time.time() - start_time)))
# Here's one way to print the final output from the threadsoutput = {}for key, value in result.items():output[key] = value.get()print ("\nOutput aggregated in a Dictionary:")print (output)print ("\n")
print ("\nPretty printed output: ")for key, value in output.items():print (key + "\n")print (value)
import queue as Queueimport threadingimport urllib.request
# Called by each threaddef get_url(q, url):q.put(urllib.request.urlopen(url).read())
theurls = ["http://google.com", "http://yahoo.com", "http://www.python.org","https://wiki.python.org/moin/"]
q = Queue.Queue()def thread_func():for u in theurls:t = threading.Thread(target=get_url, args = (q,u))t.daemon = Truet.start()
s = q.get()
def non_thread_func():for u in theurls:get_url(q,u)
s = q.get()
import timefrom concurrent.futures import ThreadPoolExecutor, as_completedimport threading
def a(a=1, b=2):print(a)time.sleep(5)print(b)return a+b
def b(**kwargs):if "a" in kwargs:print("am b")else:print("nothing")
to_do=[]executor = ThreadPoolExecutor(max_workers=4)ex1=executor.submit(a)to_do.append(ex1)ex2=executor.submit(b, **{"a":1})to_do.append(ex2)
for future in as_completed(to_do):print("Future {} and Future Return is {}\n".format(future, future.result()))
print("threading")
to_do=[]to_do.append(threading.Thread(target=a))to_do.append(threading.Thread(target=b, kwargs={"a":1}))
for threads in to_do:threads.start()
for threads in to_do:threads.join()
from threading import Thread
def test():for i in range(0, 100):print(i)
thread_list = []
for _ in range(0, 10):thread = Thread(target=test)thread_list.append(thread)
for thread in thread_list:thread.start()
for thread in thread_list:thread.join()
并且,下面的代码是上面运行10线程并发的代码中的速记#0循环版本,打印从0到99的数字:
from threading import Thread
def test():[print(i) for i in range(0, 100)]
thread_list = [Thread(target=test) for _ in range(0, 10)]
[thread.start() for thread in thread_list]
[thread.join() for thread in thread_list]