1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
| import multiprocessing
import os
import requests
class MultiProcDownloader(object):
def __init__(self, urls):
self.urls = urls
def run(self):
jobs = []
for url in self.urls:
process = multiprocessing.Process(target=self.worker, args=(url,))
jobs.append(process)
process.start()
for job in jobs:
job.join()
def worker(self, url):
fname = os.path.basename(url)
msg = 'Starting download of %s'%fname
print(msg, multiprocessing.current_process().name)
r = requests.get(url)
with open(fname, 'wb') as f:
f.write(r.content)
if __name__=='__main__':
urls = ["http://www.irs.gov/pub/irs-pdf/f1040.pdf",
"http://www.irs.gov/pub/irs-pdf/f1040a.pdf",
"http://www.irs.gov/pub/irs-pdf/f1040ez.pdf",
"http://www.irs.gov/pub/irs-pdf/f1040es.pdf",
"http://www.irs.gov/pub/irs-pdf/f1040sb.pdf"]
downloader = MultiProcDownloader(urls)
downloader.run()
|