2martens/allris-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed URL and headless mode for Firefox on GUI-free host

Browse Source
This commit is contained in:
Jim Martens
2020-01-12 00:16:22 +01:00
parent 7ade635756
commit 8c271853e9
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/twomartens/allrisscraper/main.py
View File

@ -32,7 +32,6 @@ from twomartens.allrisscraper import definitions
ALLRIS_LOGIN: str = "https://2martens.de/allris-eimsbüttel" ALLRIS_LOGIN: str = "https://2martens.de/allris-eimsbüttel"
ALLRIS_OPEN: str = "https://2martens.de/bezirk-eimsbüttel" ALLRIS_OPEN: str = "https://2martens.de/bezirk-eimsbüttel"
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3112.50 Safari/537.36'
_CONFIG_PROPS = { _CONFIG_PROPS = {
"Default": { "Default": {
"district": "Eimsbüttel", "district": "Eimsbüttel",
@ -71,13 +70,12 @@ def main() -> None:
options = Options() options = Options()
options.headless = True options.headless = True
options.add_argument(f"user-agent={user_agent}")
driver = webdriver.WebDriver(options=options) driver = webdriver.WebDriver(options=options)
driver.implicitly_wait(2) driver.implicitly_wait(2)
driver.get(ALLRIS_LOGIN) driver.get(ALLRIS_LOGIN)
login(driver, username=username, password=password) login(driver, username=username, password=password)
driver.get("https://gateway.hamburg.de/HamburgGateway/Service/StartService/113") driver.get("https://gateway.hamburg.de/HamburgGateway/Service/StartService/113")
driver.get(f"{base_url}/ri/si012.asp") driver.get(f"{base_url}/si012.asp")
meetings = get_meetings(driver) meetings = get_meetings(driver)
download_documents(driver, meetings, pdf_location, base_url, district) download_documents(driver, meetings, pdf_location, base_url, district)
driver.close() driver.close()