@Kaizen , @Evan_MCK : I refactored here a notebook with the elements collected from your posts. I works.
# imports needed for notebook
from datetime import datetime
import dateutil.relativedelta
import os
import time
import urllib.request, json
def get_latest_driver_url():
with urllib.request.urlopen("https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json") as url:
data = json.load(url)
print(data['channels']['Stable']['version'])
url = data['channels']['Stable']['downloads']['chromedriver'][0]['url']
# print(url)
# set the url as environment variable to use in scripting
# os.environ['latest_chromedriver_url']= url
return url
latest_chromedriver_url = get_latest_driver_url()
print(latest_chromedriver_url)
Make an init script
dbutils.fs.mkdirs("dbfs:/databricks/scripts/")
dbutils.fs.put("/databricks/scripts/selenium-install.sh",f"""
#!/bin/bash
latest_chromedriver_url="{latest_chromedriver_url}"
wget -N $latest_chromedriver_url -O /tmp/chromedriver_linux64.zip
rm -rf /tmp/chromedriver/
unzip /tmp/chromedriver_linux64.zip -d /tmp/chromedriver/
sudo apt-get clean && sudo apt-get update --fix-missing -y
sudo curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add
sudo echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
sudo apt-get -y update
sudo apt-get -y install google-chrome-stable
""", True)
display(dbutils.fs.ls("dbfs:/databricks/scripts/"))
Run it (or put in cluster init script config to automatically run it at cluster start)
%sh
/dbfs/databricks/scripts/selenium-install.sh
Install Selenium and restart Python kernel (or put it in PiPy package to install at start of cluster)
%pip install selenium
dbutils.library.restartPython()
Init the driver
# imports needed for notebook
from datetime import datetime
import dateutil.relativedelta
import os
import time
import urllib.request, json
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
def init_chrome_browser(download_path, chrome_driver_path, url):
options = Options()
prefs = {'download.default_directory' : download_path, 'profile.default_content_setting_values.automatic_downloads': 1, "download.prompt_for_download": False,
"download.directory_upgrade": True, "safebrowsing.enabled": True ,
"translate_whitelists": {"vi":"en"},
"translate":{"enabled":"true"}}
options.add_experimental_option('prefs', prefs)
options.add_argument('--no-sandbox')
options.add_argument('--headless') # wont work without this feature in databricks can't display browser
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--start-maximized')
options.add_argument('window-size=2560,1440')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--ignore-ssl-errors')
options.add_argument('--lang=en')
options.add_experimental_option('excludeSwitches', ['enable-logging'])
print(f"{datetime.now()} Launching Chrome...")
browser = webdriver.Chrome(service=Service(chrome_driver_path), options=options)
print(f"{datetime.now()} Chrome launched.")
browser.get(url)
print(f"{datetime.now()} Browser ready to use.")
return browser
driver = init_chrome_browser(
download_path="/tmp/downloads",
chrome_driver_path="/tmp/chromedriver/chromedriver-linux64/chromedriver",
url= "https://www.google.com"
)
Test it
from selenium.webdriver.common.by import By
driver.find_element(By.CSS_SELECTOR, "img").get_attribute("alt")
Close the driver
driver.quit()
