How can i limit selenium browser count? [duplicate]
The following sample program creates a thread pool with only 2 threads for demo purposes and then scrapes 4 URLs to get their titles:
from multiprocessing.pool import ThreadPool
from bs4 import BeautifulSoup
from selenium import webdriver
import threading
import gc
class Driver:
def __init__(self):
options = webdriver.ChromeOptions()
options.add_argument("--headless")
# suppress logging:
options.add_experimental_option('excludeSwitches', ['enable-logging'])
self.driver = webdriver.Chrome(options=options)
print('The driver was just created.')
def __del__(self):
self.driver.quit() # clean up driver when we are cleaned up
print('The driver has terminated.')
threadLocal = threading.local()
def create_driver():
the_driver = getattr(threadLocal, 'the_driver', None)
if the_driver is None:
the_driver = Driver()
setattr(threadLocal, 'the_driver', the_driver)
return the_driver.driver
def get_title(url):
driver = create_driver()
driver.get(url)
source = BeautifulSoup(driver.page_source, "lxml")
title = source.select_one("title").text
print(f"{url}: '{title}'")
# just 2 threads in our pool for demo purposes:
with ThreadPool(2) as pool:
urls = [
'https://www.google.com',
'https://www.microsoft.com',
'https://www.ibm.com',
'https://www.yahoo.com'
]
pool.map(get_title, urls)
# must be done before terminate is explicitly or implicitly called on the pool:
del threadLocal
gc.collect()
# pool.terminate() is called at exit of with block
Prints:
The driver was just created.
The driver was just created.
https://www.google.com: 'Google'
https://www.microsoft.com: 'Microsoft - Official Home Page'
https://www.ibm.com: 'IBM - United States'
https://www.yahoo.com: 'Yahoo'
The driver has terminated.
The driver has terminated.