2martens
/
allris-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed login procedure and updated changelog/README

This commit is contained in:
Jim Martens 2020-12-06 11:46:48 +01:00
parent 6249e60276
commit a5f49b27fd
4 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.rst
View File

@ -3,6 +3,10 @@ Changelog
This changelog contains a list of versions with their respective high-level changes. This changelog contains a list of versions with their respective high-level changes.
v0.5.1:
- fixed login procedure
v0.5.0:
- added support for not public video conferences
v0.4.1: v0.4.1:
- specify window size for internal scraping - specify window size for internal scraping
v0.4.0: v0.4.0:

2
README.rst
View File

@ -6,7 +6,7 @@ ALLRIS Scraper
.. image:: https://img.shields.io/pypi/pyversions/twomartens.allrisscraper.svg .. image:: https://img.shields.io/pypi/pyversions/twomartens.allrisscraper.svg
:alt: Python 3.7 and 3.8 :alt: Python 3.7 and 3.8
.. image:: https://img.shields.io/pypi/v/twomartens.allrisscraper.svg .. image:: https://img.shields.io/pypi/v/twomartens.allrisscraper.svg
:alt: version 0.4.1 :alt: version 0.5.1
This scraper offers both public and private scraping. The latter requires your username and password and performs the This scraper offers both public and private scraping. The latter requires your username and password and performs the
following tasks for you: following tasks for you:

2
setup.py
View File

@ -30,7 +30,7 @@ setup(
author="Jim Martens", author="Jim Martens",
author_email="github@2martens.de", author_email="github@2martens.de",
url="https://git.2martens.de/2martens/allris-scraper", url="https://git.2martens.de/2martens/allris-scraper",
version="0.4.1", version="0.5.1",
namespace_packages=["twomartens"], namespace_packages=["twomartens"],
packages=find_packages('src', exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), packages=find_packages('src', exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
package_dir={'': 'src'}, package_dir={'': 'src'},

9
src/twomartens/allrisscraper/internal.py
View File

@ -53,7 +53,7 @@ def main(_) -> None:
driver = webdriver.Firefox(firefox_binary=binary, options=options, executable_path=geckodriver) driver = webdriver.Firefox(firefox_binary=binary, options=options, executable_path=geckodriver)
driver.set_window_size(1920, 1080) driver.set_window_size(1920, 1080)
driver.delete_all_cookies() driver.delete_all_cookies()
driver.implicitly_wait(5) driver.implicitly_wait(10)
driver.get(ALLRIS_LOGIN) driver.get(ALLRIS_LOGIN)
login(driver, username=username, password=password) login(driver, username=username, password=password)
driver.get("https://serviceportal.hamburg.de/HamburgGateway/Service/StartService/ALLMAnd") driver.get("https://serviceportal.hamburg.de/HamburgGateway/Service/StartService/ALLMAnd")
@ -64,11 +64,14 @@ def main(_) -> None:
def login(driver: webdriver.Firefox, username: str, password: str) -> None: def login(driver: webdriver.Firefox, username: str, password: str) -> None:
collapsible = driver.find_element_by_id("headingLoginWithUsernamePassword")
if not collapsible.get_attribute("aria-expanded"):
collapsible.click()
login_field = driver.find_element_by_id("Username") login_field = driver.find_element_by_id("Username")
login_field.send_keys(username) login_field.send_keys(username)
password_field = driver.find_element_by_id("Password") password_field = driver.find_element_by_id("Password")
password_field.send_keys(password) password_field.send_keys(password)
button = driver.find_element_by_class_name("btn-primary") button = driver.find_element_by_id("LoginUsingUsernamePassword-Submit")
button.click() button.click()
@ -175,7 +178,7 @@ def get_abbreviated_committee_name(name: str, district: str) -> str:
abbreviated_name = "" abbreviated_name = ""
if name.startswith(start_video_conf): if name.startswith(start_video_conf):
name = name.removeprefix(start_video_conf) name = name[len(start_video_conf):]
if name.startswith(start_plenary): if name.startswith(start_plenary):
abbreviated_name = "BV" abbreviated_name = "BV"
elif name.startswith(start_committee): elif name.startswith(start_committee):