diff --git a/Pipfile b/Pipfile index ad2fc0f..950b6f3 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,7 @@ name = "pypi" selenium = "*" requests = "*" rich = "*" +"pdfminer.six" = "*" [dev-packages] pylint = "*" diff --git a/main.py b/main.py index dd014b1..b4b5a95 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,13 @@ import argparse import csv import logging +import io import os import re import urllib.parse as urlparse from urllib.parse import parse_qs +from pdfminer.high_level import extract_text import requests from rich.logging import RichHandler from rich.console import Console @@ -28,6 +30,8 @@ class BaseTester: """ def __init__(self, user: str, passwd: str, dev: bool, show_browser: bool): self.console = Console() + # Disable pdfminers logger cause its annoying as fuck and provides no value + logging.getLogger("pdfminer").setLevel(100) logging.basicConfig( level="INFO", format="%(message)s", @@ -147,23 +151,21 @@ class BaseTester: return self.wait.until(EC.element_to_be_clickable(locator)) -class ContactExport(BaseTester): - """Tests if exporting contacts from a search returns a CSV file with all contacts.""" +class SearchExportTester(BaseTester): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.title("Contact Export") - self.desc( - "Testing if exporting contacts from a search returns a CSV file with all contacts." - ) - self.search_url = self.base_url + "/civicrm/contact/search" - self.contact_selectall_id = "CIVICRM_QFID_ts_all_4" - self.contact_dropdown_id = "select2-chosen-4" - self.contact_exportoption_id = "select2-result-label-15" + def _get_export_id(self) -> str: + """Parses url to get the param used to ID what search we are currently doing""" + export_page_url = self.browser.current_url + parsed = urlparse.urlparse(export_page_url) + qf_key = parse_qs(parsed.query)['qfKey'] + self.debug("got qf_key '%s' from url" % qf_key) + return qf_key - def download_csv(self): + def download(self, data: dict) -> requests.Response: """ - Download export CSV manually using requests + Download exports from searches manually using requests This is because managing downloads within Selenium is a nightmare Function tries to replicate the download within the browser as much as possible """ @@ -180,51 +182,60 @@ class ContactExport(BaseTester): session.cookies.update( {session_cookie["name"]: session_cookie["value"]} ) - qf_key = self.get_export_id() # Most of this data is replicating the export settings so that the response is the csv data data = { - "qfKey": qf_key, + "qfKey": self._get_export_id(), "entryURL": self.base_url + "/civicrm/contact/search", - "_qf_Select_next": "Continue", - "exportOption": 1, - "mergeOption": 0, - "postal_greeting": 1, - "addressee": 1, + **data # Add provided export data required for specific export requests } res = session.request( "POST", self.base_url + "/civicrm/contact/search", data=data ) return res - def calculate_exported_contacts_number(self): + +class ContactExport(SearchExportTester): + """Tests if exporting contacts from a search returns a CSV file with all contacts.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.title("Contact Export") + self.desc( + "Testing if exporting contacts from a search returns a CSV file with all expected contacts." + ) + self.search_url = self.base_url + "/civicrm/contact/search" + + self.contact_selectall_id = "CIVICRM_QFID_ts_all_4" + self.contact_dropdown_id = "select2-chosen-4" + self.contact_exportoption_id = "select2-result-label-15" + + def download_csv(self): + # Data of the request that is specific for this export and not genertic like qr_key + data = { + "_qf_Select_next": "Continue", + "exportOption": 1, + "mergeOption": 0, + "postal_greeting": 1, + "addressee": 1, + } + return self.download(data) + + def calculate_exported_contacts_number(self) -> int: """ - Downloads csv, saves it to tmp, opens it and checks if the number of contacts is equal to the reported number of contacts within the web ui - Has to be saved to disk due to the csv lib only wanting a file object, not str + Downloads csv, opens it in a string buffer using StringIO and passes it to the csv lib + Counts number of rows (excl. header) and returns that value """ res = self.download_csv() - file_name = "/tmp/exportedRecords.csv" - with open(file_name, "w") as csv_file: - self.debug("writing csv to '%s'" % file_name) - csv_file.write(res.text) - with open(file_name, "r") as csv_file: - # Dict reader should remove headers - exported_csv = csv.DictReader(csv_file) - exported_number_exports = sum(1 for row in exported_csv) - self.debug( - "found %d rows in exported csv excluding header row" % - exported_number_exports - ) - self.debug("deleting '%s'" % file_name) - os.remove(file_name) - return exported_number_exports + csv_file = io.StringIO(res.text) - def get_export_id(self) -> str: - """Parses url to get the param used to ID what search we are currently doing""" - export_page_url = self.browser.current_url - parsed = urlparse.urlparse(export_page_url) - qf_key = parse_qs(parsed.query)['qfKey'] - self.debug("got qf_key '%s' from url" % qf_key) - return qf_key + # Dict reader should remove headers + exported_csv = csv.DictReader(csv_file) + exported_number_exports = sum(1 for row in exported_csv) + self.debug( + "found %d rows in exported csv excluding header row" % + exported_number_exports + ) + + return exported_number_exports def _test(self, search_term: str): """ @@ -233,7 +244,7 @@ class ContactExport(BaseTester): Search for the search term Select all contacts and set them to export Get the login cookie and search ID and download the export manually with requests - Save the file in tmp + Save the file in tmp Read it to check the number of exported contacts is the same as reported in the UI """ self.browser.get(self.search_url) @@ -341,6 +352,79 @@ class ActivitiesTab(BaseTester): self._test_all(cid_tuple) +class SteeringCommitteePrintLabels(SearchExportTester): + """Tests the pdf labels for the SteeringCommitee show all contacts names""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.title("Steering Committee Print Labels") + self.desc( + "Testing the pdf labels for the SteeringCommittee show all contacts names" + ) + self.search_url = self.base_url + "/civicrm/contact/search" + self.group_dropdown = "s2id_autogen2" + self.search_button = "_qf_Basic_refresh" + self.mail_label_option = "select2-result-label-19" + self.contact_selectall_id = "CIVICRM_QFID_ts_all_4" + self.contact_dropdown_id = "select2-chosen-4" + # Using this to count amount of people in the exported pdf + # This will fail if someone is added that doesn't have a UK adddress + self.pdf_search_string = "UNITED KINGDOM" + + def _test(self): + self.browser.get(self.search_url) + group_dropdown = self.find_element_by_id(self.group_dropdown) + group_dropdown.click() + group_dropdown.send_keys("Steering Committee" + Keys.ENTER) + self.find_element_by_id(self.search_button).click() + self.wait_until_visible( + (By.ID, "alpha-filter") + ) #wait for table to load + self.debug("table of results has loaded") + # TODO: Refactor this in base class as a helper function because we do this in another test + results_text = self.find_element_by_css_selector( + ".form-layout-compressed > tbody:nth-child(1) > tr:nth-child(2) > td:nth-child(2) > label:nth-child(2)" + ).text + matches = re.findall( + r"(\d+)", results_text + ) # Should just be one match in normal cases + result_no = int(matches[0]) + self.debug("exporting results using the magic dropdown") + self.find_element_by_id(self.contact_selectall_id).click() + self.find_element_by_id(self.contact_dropdown_id).click() + self.find_element_by_id(self.mail_label_option).click() + + self.wait_until_visible((By.CSS_SELECTOR, ".crm-block")) + # By omitting the field, we are effectively disabling the do not mail filter + data = { + "_qf_default": "Label:submit", + # Smallest labels, will show 24 contacts on one page + "label_name": "3475", + "location_type_id": "", + "_qf_Label_submit": "Make+Mailing+Labels" + } + res = self.download(data) + pdf_text = extract_text(io.BytesIO(res.content)) + label_count = pdf_text.count(self.pdf_search_string) + if result_no == label_count: + self.passed( + "no missing mailing labels from pdf export. Expected: %d, Actual %d" + % (result_no, label_count) + ) + else: + self.failed( + "missing mailing labels from the pdf export. Expected: %d, Actual %d" + % (result_no, label_count) + ) + + def test(self): + try: + self.login() + self._test() + finally: + self.logout() + self.browser.close() + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="") parser.add_argument( @@ -365,19 +449,14 @@ if __name__ == "__main__": ) parser.set_defaults(dev=False, show_browser=False) arguments = parser.parse_args() - ActivitiesTab( + cl_arg = ( arguments.user, arguments.passwd, arguments.dev, arguments.show_browser - ).test_all_hardcoded_contacts() - ContactExport( - arguments.user, arguments.passwd, arguments.dev, arguments.show_browser - ).test_hardcoded_search_terms() + ) + SteeringCommitteePrintLabels(*cl_arg).test() + ActivitiesTab(*cl_arg).test_all_hardcoded_contacts() + ContactExport(*cl_arg).test_hardcoded_search_terms() # Mailing list # Load mailing list test and enter test data # send test email and check for soft crashes # manual check the email sent - -# Print labels -# Search contacts > by group "Steering Committee" > search -# Print labels for group, untick "do not mail" check box -# Resulting pdf might have missing names