From 1d4ac97fa6a94c09c0354973be2afc6b2992016c Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 18 Nov 2024 12:15:19 +0100 Subject: [PATCH] event check is done, I think --- caldav/davclient.py | 19 ++- caldav/lib/debug.py | 2 + caldav/lib/error.py | 11 +- caldav/objects.py | 2 +- check_server_compatibility.py | 284 +++++++++++++++++++++++----------- tests/compatibility_issues.py | 11 +- tests/conf.py | 7 +- tests/test_caldav.py | 21 +-- 8 files changed, 241 insertions(+), 116 deletions(-) diff --git a/caldav/davclient.py b/caldav/davclient.py index 7d6b475e..3a1a2dbe 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -211,6 +211,7 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] status = None href: Optional[str] = None propstats: List[_Element] = [] + check_404 = False ## special for purelymail error.assert_(response.tag == dav.Response.tag) for elem in response: if elem.tag == dav.Status.tag: @@ -223,13 +224,27 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] href = unquote(elem.text) elif elem.tag == dav.PropStat.tag: propstats.append(elem) + elif elem.tag == '{DAV:}error': + ## This happens with purelymail on a 404. + ## This code is mostly moot, but in debug + ## mode I want to be sure we do not toss away any data + children = elem.getchildren() + error.assert_(len(children)==1) + error.assert_(children[0].tag=='{https://purelymail.com}does-not-exist') + check_404 = True else: - error.assert_(False) + ## i.e. purelymail may contain one more tag, ... + ## This is probably not a breach of the standard. It may + ## probably be ignored. But it's something we may want to + ## know. + error.weirdness("unexpected element found in response", elem) error.assert_(href) + if check_404: + error.assert_('404' in status) ## TODO: is this safe/sane? ## Ref https://github.com/python-caldav/caldav/issues/435 the paths returned may be absolute URLs, ## but the caller expects them to be paths. Could we have issues when a server has same path - ## but different URLs for different elements? + ## but different URLs for different elements? Perhaps href should always be made into an URL-object? if ":" in href: href = unquote(URL(href).path) return (cast(str, href), propstats, status) diff --git a/caldav/lib/debug.py b/caldav/lib/debug.py index 9c7bcd2d..cdb4951e 100644 --- a/caldav/lib/debug.py +++ b/caldav/lib/debug.py @@ -2,6 +2,8 @@ def xmlstring(root): + if isinstance(root, str): + return root if hasattr(root, "xmlelement"): root = root.xmlelement() return etree.tostring(root, pretty_print=True).decode("utf-8") diff --git a/caldav/lib/error.py b/caldav/lib/error.py index 93e5e004..3d795f2c 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -25,6 +25,13 @@ else: log.setLevel(logging.WARNING) +def weirdness(*reasons): + from caldav.lib.debug import xmlstring + reason = " : ".join([xmlstring(x) for x in reasons]) + log.warning(f"Deviation from expectations found: {reason}") + if debugmode == "DEBUG_PDB": + log.error(f"Dropping into debugger due to {reason}") + import pdb; pdb.set_trace() def assert_(condition: object) -> None: try: @@ -36,9 +43,7 @@ def assert_(condition: object) -> None: ) elif debugmode == "DEBUG_PDB": log.error("Deviation from expectations found. Dropping into debugger") - import pdb - - pdb.set_trace() + import pdb; pdb.set_trace() else: raise diff --git a/caldav/objects.py b/caldav/objects.py index 13cc7408..fcdfed7b 100644 --- a/caldav/objects.py +++ b/caldav/objects.py @@ -791,7 +791,7 @@ def _create( if name: try: self.set_properties([display_name]) - except: + except Exception as e: ## TODO: investigate. Those asserts break. error.assert_(False) try: diff --git a/check_server_compatibility.py b/check_server_compatibility.py index b5b6964f..d6268169 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -6,30 +6,37 @@ from caldav.lib.error import AuthorizationError, DAVError, NotFoundError from caldav.lib.python_utilities import to_local from caldav.elements import dav +from json import dumps +import caldav import datetime import click import time -import json import uuid +def _delay_decorator(f, delay=10): + def foo(*a, **kwa): + time.sleep(delay) + return f(*a, **kwa) + return foo + class ServerQuirkChecker(): def __init__(self, client_obj): self.client_obj = client_obj self.flags_checked = {} self.other_info = {} + def set_flag(self, flag, value=True): + if flag == 'rate_limited': + self.client_obj.request = _delay_decorator(self.client_obj.request) + elif flag == 'search_delay': + caldav.Calendar.search = _delay_decorator(caldav.Calendar.search, 60) + if hasattr(self, '_default_calendar'): + self._default_calendar.search = _delay_decorator(self._default_calendar.search, 60) + self.flags_checked[flag] = value + def _try_make_calendar(self, cal_id, **kwargs): calmade = False - ## Check on "no_default_calendar" flag - try: - cals = self.principal.calendars() - cals[0].events() - self.flags_checked["no_default_calendar"] = False - except: - import pdb; pdb.set_trace() - self.flags_checked["no_default_calendar"] = True - ## In case calendar already exists ... wipe it first try: self.principal.calendar(cal_id=cal_id).delete() @@ -39,13 +46,19 @@ def _try_make_calendar(self, cal_id, **kwargs): ## create the calendar try: cal = self.principal.make_calendar(cal_id=cal_id, **kwargs) - ## calendar creation probably went OK + ## calendar creation probably went OK, but we need to be sure... + cal.events() + ## calendar creation must have gone OK. calmade = True - self.flags_checked['no_mkcalendar'] = False - self.flags_checked['read_only'] = False + self.set_flag('no_mkcalendar', False) + self.set_flag('read_only', False) except Exception as e: ## calendar creation created an exception - return exception cal = self.principal.calendar(cal_id=cal_id) + try: + cal.events() + except: + cal = None if not cal: ## cal not made and does not exist, exception thrown. ## Caller to decide why the calendar was not made @@ -57,12 +70,12 @@ def _try_make_calendar(self, cal_id, **kwargs): cal.delete() try: cal = self.principal.calendar(cal_id=cal_id) - cal.events() + events = cal.events() except NotFoundError: cal = None ## Delete throw no exceptions, but was the calendar deleted? - if calmade and (not cal or self.flags_checked.get('non_existing_calendar_found')): - self.flags_checked['no_delete_calendar'] = False + if (not cal or (self.flags_checked.get('non_existing_calendar_found' and len(events)==0))): + self.set_flag('no_delete_calendar', False) ## Calendar probably deleted OK. ## (in the case of non_existing_calendar_found, we should add ## some events t o the calendar, delete the calendar and make @@ -77,23 +90,22 @@ def _try_make_calendar(self, cal_id, **kwargs): cal.events() ## Calendar not deleted, but no exception thrown. ## Perhaps it's a "move to thrashbin"-regime on the server - self.flags_checked['no_delete_calendar'] = 'maybe' + self.set_flag('no_delete_calendar', 'maybe') except NotFoundError as e: ## Calendar was deleted, it just took some time. - self.flags_checked['no_delete_calendar'] = False - self.flags_checked['rate_limited'] = True + self.set_flag('no_delete_calendar', False) + self.set_flag('rate_limited', True) return (calmade, e) return (calmade, None) except Exception as e: - self.flags_checked['no_delete_calendar'] = True + self.set_flag('no_delete_calendar', True) time.sleep(10) try: cal.delete() - self.flags_Checked['no_delete_calendar'] = False - self.flags_Checked['rate_limited'] = True + self.set_flag('no_delete_calendar', False) + self.set_flag('rate_limited', True) except Exception as e2: pass - return (calmade, e) def check_principal(self): ## TODO @@ -103,21 +115,31 @@ def check_principal(self): ## In any case, this test will stop the script on authorization problems try: self.principal = self.client_obj.principal() - self.flags_checked['no-current-user-principal'] = False + self.set_flag('no-current-user-principal', False) except AuthorizationError: raise except DAVError: - self.flags_checked['no-current-user-principal'] = True + self.set_flag('no-current-user-principal', True) def check_mkcalendar(self): try: cal = self.principal.calendar(cal_id="this_should_not_exist") cal.events() - self.flags_checked["non_existing_calendar_found"] = True + self.set_flag("non_existing_calendar_found", True) except NotFoundError: - self.flags_checked["non_existing_calendar_found"] = False - except: + self.set_flag("non_existing_calendar_found", False) + except Exception as e: import pdb; pdb.set_trace() + pass + ## Check on "no_default_calendar" flag + try: + cals = self.principal.calendars() + cals[0].events() + self.set_flag("no_default_calendar", False) + self._default_calendar = cals[0] + except: + self.set_flag("no_default_calendar", True) + makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: self._default_calendar = self.principal.make_calendar(name="Yep", cal_id="pythoncaldav-test") @@ -125,34 +147,36 @@ def check_mkcalendar(self): makeret = self._try_make_calendar(cal_id="pythoncaldav-test") if makeret[0]: self._default_calendar = self.principal.make_calendar(cal_id="pythoncaldav-test") - self.flags_checked['no_displayname'] = True + self.set_flag('no_displayname', True) return unique_id1 = "testcalendar-" + str(uuid.uuid4()) unique_id2 = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id1) if makeret[0]: self._default_calendar = self.principal.make_calendar(cal_id=unique_id2) - self.flags_checked['unique_calendar_ids'] = True + self.set_flag('unique_calendar_ids', True) unique_id = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id, name='Yep') - if not makeret[0]: + if not makeret[0] and not self.flags_checked.get('no_mkcalendar', True): self.flags_checked['no_displayname'] = True + if not 'no_mkcalendar' in self.flags_checked: + self.set_flag('no_mkcalendar', True) def check_support(self): - self.flags_checked['dav_not_supported'] = True - self.flags_checked['no_scheduling'] = True + self.set_flag('dav_not_supported', True) + self.set_flag('no_scheduling', True) try: - self.flags_checked['dav_not_supported'] = not self.client_obj.check_dav_support() - self.flags_checked['no_scheduling'] = not self.client_obj.check_scheduling_support() + self.set_flag('dav_not_supported', not self.client_obj.check_dav_support()) + self.set_flag('no_scheduling', not self.client_obj.check_scheduling_support()) except: pass if not self.flags_checked['no_scheduling']: try: inbox = self.principal.schedule_inbox() outbox = self.principal.schedule_outbox() - self.flags_checked['no_scheduling_mailbox'] = False + self.set_flag('no_scheduling_mailbox', False) except: - self.flags_checked['no_scheduling_mailbox'] = True + self.set_flag('no_scheduling_mailbox', True) def check_propfind(self): try: @@ -172,68 +196,144 @@ def check_propfind(self): ] ) assert "resourcetype" in to_local(foo.raw) - self.flags_checked['propfind_allprop_failure'] = False + self.set_flag('propfind_allprop_failure', False) except: - self.flags_checked['propfind_allprop_failure'] = True + self.set_flag('propfind_allprop_failure', True) def check_event(self): cal = self._default_calendar - cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], uid='check_event_1') - cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') - try: - objcnt = len(cal.objects()) - except: - objcnt = 0 - if objcnt != 2: - if len(cal.events()) == 2: - self.flags_checked['search_always_needs_comptype'] = True + + ## Two simple events with text fields, dtstart=now and no dtend + obj1 = cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], class_='CONFIDENTIAL', uid='check_event_1') + obj2 = cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') + try: ## try-finally-block covering testing of obj1 and obj2 + try: + foo = cal.event_by_uid('check_event_2') + assert(foo) + self.set_flag('object_by_uid_is_broken', False) + except: + time.sleep(60) + try: + foo = cal.event_by_uid('check_event_2') + assert(foo) + self.set_flag('search_delay') + self.set_flag('object_by_uid_is_broken', False) + except: + self.set_flag('object_by_uid_is_broken', True) + + try: + objcnt = len(cal.objects()) + except: + objcnt = 0 + if objcnt != 2: + if len(cal.events()) == 2: + self.set_flag('search_always_needs_comptype', True) + else: + import pdb; pdb.set_trace() + pass + ## we should not be here else: - import pdb; pdb.set_trace() - ## we should not be here - else: - self.flags_checked['search_always_needs_comptype'] = False + self.set_flag('search_always_needs_comptype', False) - events = cal.search(summary='Test event 1') - if len(events) == 0: + ## purelymail writes things to an index as a background thread + ## and have delayed search. Let's test for that first. events = cal.search(summary='Test event 1', event=True) - if len(events)==1: - self.flags_checked['search_needs_comptype'] = True - self.flags_checked['no_search'] = False - self.flags_checked['text_search_not_working'] = False + if len(events) == 0: + events = cal.search(summary='Test event 1', event=True) + if len(events) == 1: + self.set_flag('rate_limited', True) + if len(events) == 1: + self.set_flag('no_search', False) + self.set_flag('text_search_not_working', False) else: + self.set_flag('text_search_not_working', True) + + objs = cal.search(summary='Test event 1') + if len(objs) == 0 and len(events)==1: + self.set_flag('search_needs_comptype', True) + elif len(objs) == 1: + self.set_flag('search_always_needs_comptype', False) + + if not self.flags_checked['text_search_not_working']: + events = cal.search(summary='test event 1', event=True) + if len(events) == 1: + self.set_flag('text_search_is_case_insensitive', True) + elif len(events)==0: + self.set_flag('text_search_is_case_insensitive', False) + else: + ## we should not be here + import pdb; pdb.set_trace() + pass + events = cal.search(summary='test event', event=True) + if len(events)==2: + self.set_flag('text_search_is_exact_match_only', False) + elif len(events)==0: + self.set_flag('text_search_is_exact_match_only', 'maybe') + ## may also be text_search_is_exact_match_sometimes + events = cal.search(summary='Test event 1', class_='CONFIDENTIAL', event=True) + if len(events)==1: + self.set_flag('combined_search_not_working', False) + elif len(events)==0: + self.set_flag('combined_search_not_working', True) + else: + import pdb; pdb.set_trace() + ## We should not be here + pass + try: + events = cal.search(category='foo', event=True) + except: + events = [] + if len(events) == 1: + self.set_flag("category_search_yields_nothing", False) + elif len(events) == 0: + self.set_flag("category_search_yields_nothing", True) + else: + ## we should not be here import pdb; pdb.set_trace() - ## we should not be here ... unless search is not working? - elif len(events) == 1: - self.flags_checked['no_search'] = False - self.flags_checked['text_search_not_working'] = False - self.flags_checked['search_always_needs_comptype'] = False - events = cal.search(summary='test event 1', event=True) - if len(events) == 1: - self.flags_checked['text_search_is_case_insensitive'] = True - elif len(events)==0: - self.flags_checked['text_search_is_case_insensitive'] = False - else: - ## we should not be here - import pdb; pdb.set_trace() - pass - events = cal.search(summary='test event', event=True) - if len(events)==2: - self.flags_checked['text_search_is_exact_match_only'] = False - elif len(events)==0: - self.flags_checked['text_search_is_exact_match_only'] = 'maybe' - ## may also be text_search_is_exact_match_sometimes + pass + + + events = cal.search(summary='test event', class_='CONFIDENTIAL', event=True) + finally: + obj1.delete() + obj2.delete() + + ## Recurring events try: - events = cal.search(category='foo', event=True) + yearly_time = cal.add_event(dtstart=datetime.datetime(2000, 1, 1, 0, 0), summary="Yearly timed event", uid='firework_event', rrule={'FREQ': 'YEARLY'}) + yearly_day = cal.add_event(dtstart=datetime.date(2000, 5, 1), summary="Yearly day event", uid='full_day_event', rrule={'FREQ': 'YEARLY'}) except: - events = [] - if len(events) == 1: - self.flags_checked["category_search_yields_nothing"] = False - elif len(events) == 0: - self.flags_checked["category_search_yields_nothing"] = True - else: - ## we should not be here + ## should not be here import pdb; pdb.set_trace() - pass + raise + try: + try: + events = cal.search(start=datetime.datetime(2001, 4, 1), end=datetime.datetime(2002,2,2), event=True) + assert len(events)==2 + self.set_flag('no_recurring', False) + except: + self.set_flag('no_recurring', True) + + if not (self.flags_checked['no_recurring']): + events = cal.search(start=datetime.datetime(2001, 4, 1), end=datetime.datetime(2002,2,2), event=True, expand='server') + assert len(events)==2 + if 'RRULE' in events[0].data: + assert 'RRULE' in events[1].data + assert not 'RECURRENCE-ID' in events[0].data + assert not 'RECURRENCE-ID' in events[1].data + self.set_flag('no_expand', True) + else: + assert not 'RRULE' in events[1].data + self.set_flag('no_expand', False) + if 'RECURRENCE-ID' in events[0].data and 'DTSTART:2001' in events[0].data: + assert 'RECURRENCE-ID' in events[1].data + self.set_flag('broken_expand', False) + else: + self.set_flag('broken_expand', True) + + finally: + yearly_time.delete() + yearly_day.delete() def check_all(self): try: @@ -246,8 +346,11 @@ def check_all(self): import pdb; pdb.set_trace() raise finally: - if self._default_calendar: - self._default_calendar.delete() + if self._default_calendar and not self.flags_checked['no_mkcalendar']: + try: + self._default_calendar.delete() + except: + pass def report(self, verbose, json): if verbose: @@ -261,7 +364,7 @@ def report(self, verbose, json): self.diff1 = set(self.client_obj.incompatibilities) - flags_found self.diff2 = flags_found - set(self.client_obj.incompatibilities) if json: - click.echo(json.dumps({'flags_checked': self.flags_checked, 'diff1': self.diff1, 'diff2': self.diff2})) + click.echo(dumps({'name': self.client_obj.server_name, 'url': str(self.client_obj.url), 'flags_checked': self.flags_checked, 'diff1': list(self.diff1), 'diff2': list(self.diff2)}, indent=4)) click.echo() return if verbose is False: @@ -278,6 +381,7 @@ def report(self, verbose, json): click.echo(f"-{x}") for x in self.diff2: click.echo(f"+{x}") + click.echo() for x in self.flags_checked: if self.flags_checked[x]: click.echo(f"## {x}") diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 6ed3d84e..8c2015d5 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -15,6 +15,9 @@ 'rate_limited': """It may be needed to pause a bit between each request when doing tests""", + 'search_delay': + """Server populates indexes through some background job, so it takes some time from an event is added/edited until it's possible to search for it""", + 'cleanup_calendar': """Remove everything on the calendar for every test""", @@ -229,6 +232,7 @@ ## TODO: remove this when shredding support for python 3.7 ## https://github.com/jelmer/xandikos/pull/194 'category_search_yields_nothing', + "search_needs_comptype", ## scheduling is not supported "no_scheduling", @@ -249,6 +253,7 @@ 'text_search_is_case_insensitive', 'text_search_is_exact_match_sometimes', 'combined_search_not_working', + "search_needs_comptype", ## extra features not specified in RFC5545 "calendar_order", @@ -434,11 +439,7 @@ ## Purelymail claims that the search indexes are "lazily" populated, ## so search works some minutes after the event was created/edited. - ## I tried adding arbitrary delays in commit 5d052b1 but still didn't - ## manage to get search to work. Should eventually do more research - ## into this. (personal email communication with contact@purelymail.com) - 'no_search', - 'object_by_uid_is_broken', + 'search_delay' ] # fmt: on diff --git a/tests/conf.py b/tests/conf.py index 2aa38a8f..2747cefa 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -258,12 +258,7 @@ def client( no_args = not any (x for x in kwargs if kwargs[x] is not None) if idx is None and no_args and caldav_servers: ## No parameters given - find the first server in caldav_servers list - idx = 0 - while idx < len(caldav_servers) and not caldav_servers[idx].get("enable", True): - idx += 1 - if idx == len(caldav_servers): - return None - return client(idx=idx) + return client(idx=0) elif idx is not None and no_args and caldav_servers: return client(**caldav_servers[idx]) elif name is not None and no_args and caldav_servers: diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 237dff49..da10b3cb 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -556,6 +556,12 @@ def testInviteAndRespond(self): ## inbox/outbox? +def _delay_decorator(f, t=60): + def foo(*a, **kwa): + time.sleep(t) + return f(*a, **kwa) + return foo + class RepeatedFunctionalTestsBaseClass(object): """This is a class with functional tests (tests that goes through basic functionality and actively communicates with third parties) @@ -603,15 +609,10 @@ def setup_method(self): self.caldav = client(**self.server_params) if self.check_compatibility_flag("rate_limited"): - - def delay_decorator(f): - def foo(*a, **kwa): - time.sleep(60) - return f(*a, **kwa) - - return foo - - self.caldav.request = delay_decorator(self.caldav.request) + self.caldav.request = _delay_decorator(self.caldav.request) + if self.check_compatibility_flag("search_delay"): + Calendar._search = Calendar.search + Calendar.search = _delay_decorator(Calendar.search) if False and self.check_compatibility_flag("no-current-user-principal"): self.principal = Principal( @@ -631,6 +632,8 @@ def foo(*a, **kwa): logging.debug("##############################") def teardown_method(self): + if self.check_compatibility_flag("search_delay"): + Calendar.search = Calendar._search logging.debug("############################") logging.debug("############## test teardown_method") logging.debug("############################")