You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

210 lines
8.6 KiB

  1. # -*- coding: utf-8 -*-
  2. # Copyright 2020 Jim Martens
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import configparser
  16. import os
  17. from datetime import date
  18. from datetime import time
  19. from typing import List
  20. from typing import Tuple
  21. from urllib import request
  22. from selenium import webdriver
  23. from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
  24. from selenium.webdriver.firefox.options import Options
  25. from selenium.webdriver.remote.webelement import WebElement
  26. from twomartens.allrisscraper import config as config_module
  27. from twomartens.allrisscraper import definitions
  28. from twomartens.allrisscraper import meeting
  29. from twomartens.allrisscraper.definitions import ALLRIS_LOGIN
  30. def main(_) -> None:
  31. config_file = f"{os.getcwd()}/tm-allris-scraper-config.ini"
  32. if not config_module.initialize_config(config_file):
  33. return
  34. config = configparser.ConfigParser()
  35. config.read(config_file)
  36. district = config["Default"]["district"]
  37. username = config["Default"]["username"]
  38. password = config["Default"]["password"]
  39. pdf_location = config["Default"]["pdflocation"]
  40. firefox_binary = config["Default"]["firefoxBinary"]
  41. geckodriver = config["Default"]["geckodriver"]
  42. base_url = definitions.BASE_LINKS[district]
  43. options = Options()
  44. options.headless = True
  45. binary = FirefoxBinary(firefox_binary)
  46. driver = webdriver.Firefox(firefox_binary=binary, options=options, executable_path=geckodriver)
  47. driver.delete_all_cookies()
  48. driver.implicitly_wait(5)
  49. driver.get(ALLRIS_LOGIN)
  50. login(driver, username=username, password=password)
  51. driver.get("https://serviceportal.hamburg.de/HamburgGateway/Service/StartService/ALLMAnd")
  52. driver.get(f"{base_url}/si012.asp")
  53. meetings = get_meetings(driver)
  54. download_documents(driver, meetings, pdf_location, base_url, district)
  55. driver.close()
  56. def login(driver: webdriver.Firefox, username: str, password: str) -> None:
  57. login_field = driver.find_element_by_id("Username")
  58. login_field.send_keys(username)
  59. password_field = driver.find_element_by_id("Password")
  60. password_field.send_keys(password)
  61. button = driver.find_element_by_class_name("btn-primary")
  62. button.click()
  63. def get_meetings(driver: webdriver.Firefox) -> List[meeting.Meeting]:
  64. elements = driver.find_elements_by_class_name("zl12")
  65. elements.extend(driver.find_elements_by_class_name("zl11"))
  66. meetings = list()
  67. for element in elements:
  68. tds = element.find_elements_by_tag_name("td")
  69. date_obj = get_day(tds[0].text)
  70. time_obj = time.fromisoformat(str(tds[1].text).rstrip())
  71. agenda_link = tds[4].find_element_by_tag_name("a").get_property("href")
  72. name = tds[4].find_element_by_tag_name("a").text
  73. location = tds[5].text
  74. meetings.append(meeting.Meeting(name=name, date=date_obj,
  75. time=time_obj, end_time=None,
  76. link=agenda_link, location=location,
  77. agenda=None, address=None))
  78. return meetings
  79. def download_documents(driver: webdriver.Firefox, meetings: List[meeting.Meeting],
  80. pdf_location: str, base_url: str, district: str) -> None:
  81. base_link = f"{base_url}/do027.asp"
  82. for _meeting in meetings:
  83. driver.get(_meeting.link)
  84. td = driver.find_element_by_xpath("//table[@class='tk1']//td[@class='me1']")
  85. form_elements = td.find_elements_by_tag_name("form")
  86. agenda_link, total_link, invitation_link = get_links(form_elements, base_link)
  87. if len(agenda_link) > 0:
  88. driver.get(agenda_link)
  89. save_pdf(driver.current_url, f"{get_formatted_filename(pdf_location, _meeting, district)}/Tagesordnung.pdf")
  90. if len(total_link) > 0:
  91. driver.get(total_link)
  92. save_pdf(driver.current_url, f"{get_formatted_filename(pdf_location, _meeting, district)}/Mappe.pdf")
  93. if len(invitation_link) > 0:
  94. driver.get(invitation_link)
  95. save_pdf(driver.current_url, f"{get_formatted_filename(pdf_location, _meeting, district)}/Einladung.pdf")
  96. save_file(_meeting.location, f"{get_formatted_filename(pdf_location, _meeting, district)}/ort.txt")
  97. def get_links(form_elements: List[WebElement], base_link: str) -> Tuple[str, str, str]:
  98. agenda_name = "Tagesordnung"
  99. updated_agenda_name = "Aktuelle TO"
  100. total_name = "Alle Dokumente zur Sitzung im Paket"
  101. total_short_name = "Mappe"
  102. invitation_name = "Einladung"
  103. links = {}
  104. for element in form_elements:
  105. name = element.find_element_by_class_name("il2_p").get_property("value")
  106. link = f"{base_link}?DOLFDNR={element.find_element_by_name('DOLFDNR').get_property('value')}&options=64"
  107. if name == agenda_name:
  108. links[agenda_name] = link
  109. if name == updated_agenda_name:
  110. links[agenda_name] = link
  111. if name == total_name:
  112. links[total_short_name] = link
  113. if name == invitation_name:
  114. links[invitation_name] = link
  115. if agenda_name not in links:
  116. links[agenda_name] = ""
  117. if invitation_name not in links:
  118. links[invitation_name] = ""
  119. if total_short_name not in links:
  120. links[total_short_name] = ""
  121. return links[agenda_name], links[total_short_name], links[invitation_name]
  122. def get_formatted_filename(pdf_location: str, meeting_obj: meeting.Meeting, district: str) -> str:
  123. return f"{pdf_location}{meeting_obj.date.isoformat()}_{get_abbreviated_committee_name(meeting_obj.name, district)}"
  124. def save_pdf(url: str, dest: str) -> None:
  125. file_data: request = request.urlopen(url)
  126. data_to_write = file_data.read()
  127. os.makedirs(os.path.dirname(dest), exist_ok=True)
  128. with open(dest, "wb") as file:
  129. file.write(data_to_write)
  130. def save_file(content: str, dest: str) -> None:
  131. os.makedirs(os.path.dirname(dest), exist_ok=True)
  132. with open(dest, "w") as file:
  133. file.write(content)
  134. def get_day(date_str: str) -> date:
  135. date_elements = date_str[date_str.find(",") + 1:].split(".")
  136. return date(int(date_elements[-1]), int(date_elements[-2]), int(date_elements[-3]))
  137. def get_abbreviated_committee_name(name: str, district: str) -> str:
  138. start_committee = "Sitzung des Ausschusses"
  139. start_regional_committee = "Sitzung des Regionalausschusses"
  140. start_plenary = "Sitzung der Bezirksversammlung"
  141. start_youth_help_committee = "Sitzung des Jugendhilfeausschusses"
  142. start_other_committee = "Sitzung des"
  143. end_other_committee = "ausschusses"
  144. abbreviated_name = ""
  145. if name.startswith(start_plenary):
  146. abbreviated_name = "BV"
  147. elif name.startswith(start_committee):
  148. second_part = name[len(start_committee):]
  149. second_split = second_part.split(sep=",")
  150. abbreviated_name = get_abbreviation(second_split)
  151. if len(abbreviated_name) == 1:
  152. abbreviated_name = f"A{abbreviated_name}"
  153. elif name.startswith(start_regional_committee):
  154. second_part = name[len(start_regional_committee):]
  155. second_split = second_part.split(sep="/")
  156. abbreviated_name = f"Ra{get_abbreviation(second_split)}"
  157. elif name.startswith(start_youth_help_committee):
  158. abbreviated_name = "JHA"
  159. elif name.startswith(start_other_committee) and name.endswith(end_other_committee):
  160. core_name = name[len(start_other_committee):-len(end_other_committee)]
  161. abbreviated_name = core_name
  162. if abbreviated_name in definitions.ABBREVIATIONS[district]:
  163. abbreviated_name = definitions.ABBREVIATIONS[district][abbreviated_name]
  164. return abbreviated_name
  165. def get_abbreviation(name):
  166. abbreviated_name = ""
  167. for part in name:
  168. part = part.lstrip()
  169. if "und" in part:
  170. part_split = part.split("und")
  171. first_part = part_split[0].lstrip()
  172. second_part = part_split[1].lstrip()
  173. abbreviated_name = f"{abbreviated_name}{first_part[:1].capitalize()}{second_part[:1].capitalize()}"
  174. else:
  175. abbreviated_name = f"{abbreviated_name}{part[:1].capitalize()}"
  176. return abbreviated_name