From 78a33da96704dc6f32d3482524fd4d9880f6a8d1 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 15 Nov 2024 21:10:13 +0100 Subject: [PATCH 01/34] Tests: add a text identifier to each caldav server entry Probably not relevant for many, but I do have a tests/conf_private.py filled up with various servers, so it would be nice to add "friendly" names to identify them --- tests/compatibility_issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index e8d532c..1b209ad 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -230,7 +230,7 @@ ## This one is fixed - but still breaks our test code for python 3.7 ## TODO: remove this when shredding support for python 3.7 - ## https://github.com/jelmer/xandikos/pull/194 + ## https://github.com/jelmer/xandikos/pull/194 'category_search_yields_nothing', ## scheduling is not supported From 232b60c7b125d23080bcebb084a264507250c731 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 15 Nov 2024 21:14:20 +0100 Subject: [PATCH 02/34] fixup! allow strings instead of tuples as input for sort_keys in search. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24d1a2b..cb8a544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0 ### Added -* By now `calendar.search(..., sort_keys=("DTSTART")` will work. Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake. https://github.com/python-caldav/caldav/pull/449 +* By now `calendar.search(..., sort_keys=("DTSTART")` will work. Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake. https://github.com/python-caldav/caldav/issues/448 https://github.com/python-caldav/caldav/pull/449 ## [1.4.0] - 2024-11-05 From 123fa488fa6137bbe5dc9c3803ceb10a7c115132 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 16 Nov 2024 13:08:32 +0100 Subject: [PATCH 03/34] work in progress --- caldav/lib/error.py | 2 +- tests/conf.py | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/caldav/lib/error.py b/caldav/lib/error.py index a09e0e9..1efb7d2 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -14,7 +14,7 @@ ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"] except: - if "dev" in __version__: + if "dev" in __version__ or __version__=='(unknown)': debugmode = "DEVELOPMENT" else: debugmode = "PRODUCTION" diff --git a/tests/conf.py b/tests/conf.py index b777b33..118390c 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -231,30 +231,33 @@ def silly_request(): ################################################################### # Convenience - get a DAVClient object from the caldav_servers list ################################################################### -CONNKEYS = set( - ("url", "proxy", "username", "password", "ssl_verify_cert", "ssl_cert", "auth") -) - +## TODO: this is already declared in davclient.DAVClient.__init__(...) +## TODO: is it possible to reuse the declaration here instead of +## duplicating the list? +## TODO: If not, it's needed to look through and ensure the list is uptodate +CONNKEYS = set(( + "url", "proxy", "username", "password", "timeout", "headers", "huge_tree", "ssl_verify_cert", "ssl_cert", "auth")) def client( idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs ): - ## No parameters given - find the first server in caldav_servers list - if idx is None and not kwargs and caldav_servers: + no_args = not any (x for x in kwargs if kwargs[x] is not None) + if idx is None and no_args and caldav_servers: + ## No parameters given - find the first server in caldav_servers list idx = 0 while idx < len(caldav_servers) and not caldav_servers[idx].get("enable", True): idx += 1 if idx == len(caldav_servers): return None return client(idx=idx) - elif idx is not None and not kwargs and caldav_servers: + elif idx is not None and no_args and caldav_servers: return client(**caldav_servers[idx]) - elif name is not None and not kwargs and caldav_servers: + elif name is not None and no_args and caldav_servers: for s in caldav_servers: if caldav_servers["name"] == s: return s return None - elif not kwargs: + elif noargs: return None for bad_param in ( "incompatibilities", @@ -264,7 +267,7 @@ def client( ): if bad_param in kwargs: kwargs.pop(bad_param) - for kw in kwargs: + for kw in list(kwargs.keys()): if not kw in CONNKEYS: logging.critical( "unknown keyword %s in connection parameters. All compatibility flags should now be sent as a separate list, see conf_private.py.EXAMPLE. Ignoring." From f587e65b8a524a1cadd7b00cc084147b2dfbfd46 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 16 Nov 2024 13:11:55 +0100 Subject: [PATCH 04/34] fixup! work in progress --- caldav/lib/error.py | 2 +- tests/conf.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/caldav/lib/error.py b/caldav/lib/error.py index 1efb7d2..93e5e00 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -14,7 +14,7 @@ ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"] except: - if "dev" in __version__ or __version__=='(unknown)': + if "dev" in __version__ or __version__ == "(unknown)": debugmode = "DEVELOPMENT" else: debugmode = "PRODUCTION" diff --git a/tests/conf.py b/tests/conf.py index 118390c..6d0364e 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -235,8 +235,21 @@ def silly_request(): ## TODO: is it possible to reuse the declaration here instead of ## duplicating the list? ## TODO: If not, it's needed to look through and ensure the list is uptodate -CONNKEYS = set(( - "url", "proxy", "username", "password", "timeout", "headers", "huge_tree", "ssl_verify_cert", "ssl_cert", "auth")) +CONNKEYS = set( + ( + "url", + "proxy", + "username", + "password", + "timeout", + "headers", + "huge_tree", + "ssl_verify_cert", + "ssl_cert", + "auth", + ) +) + def client( idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs From 3887476582a9598970e52e98a07dbc96f0ea0053 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 17 Nov 2024 08:48:13 +0100 Subject: [PATCH 05/34] wip --- tests/conf.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/conf.py b/tests/conf.py index 6d0364e..cbdfb3c 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -254,6 +254,7 @@ def silly_request(): def client( idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs ): + kwargs_ = kwargs.copy() no_args = not any (x for x in kwargs if kwargs[x] is not None) if idx is None and no_args and caldav_servers: ## No parameters given - find the first server in caldav_servers list @@ -270,7 +271,7 @@ def client( if caldav_servers["name"] == s: return s return None - elif noargs: + elif no_args: return None for bad_param in ( "incompatibilities", @@ -278,18 +279,19 @@ def client( "principal_url", "enable", ): - if bad_param in kwargs: - kwargs.pop(bad_param) - for kw in list(kwargs.keys()): + if bad_param in kwargs_: + kwargs_.pop(bad_param) + for kw in list(kwargs_.keys()): if not kw in CONNKEYS: logging.critical( "unknown keyword %s in connection parameters. All compatibility flags should now be sent as a separate list, see conf_private.py.EXAMPLE. Ignoring." % kw ) - kwargs.pop(kw) - conn = DAVClient(**kwargs) + kwargs_.pop(kw) + conn = DAVClient(**kwargs_) setup(conn) conn.teardown = teardown + conn.incompatibilities = kwargs.get('incompatibilities') return conn From fc856c71ec376010a8c935ea0aa47c8abafea10e Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 17 Nov 2024 08:55:40 +0100 Subject: [PATCH 06/34] woot ... the most important file not added to the repository? --- check_server_compatibility.py | 306 ++++++++++++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100755 check_server_compatibility.py diff --git a/check_server_compatibility.py b/check_server_compatibility.py new file mode 100755 index 0000000..a037828 --- /dev/null +++ b/check_server_compatibility.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python + +from tests.conf import client +from tests.conf import CONNKEYS +from tests.compatibility_issues import incompatibility_description +from caldav.lib.error import AuthorizationError, DAVError, NotFoundError +from caldav.lib.python_utilities import to_local +from caldav.elements import dav +import datetime +import click +import time +import json +import uuid + +class ServerQuirkChecker(): + def __init__(self, client_obj): + self.client_obj = client_obj + self.flags_checked = {} + self.other_info = {} + + def _try_make_calendar(self, cal_id, **kwargs): + calmade = False + + ## In case calendar already exists ... wipe it first + try: + self.principal.calendar(cal_id=cal_id).delete() + except: + pass + + ## create the calendar + try: + cal = self.principal.make_calendar(cal_id=cal_id, **kwargs) + ## calendar creation probably went OK + calmade = True + self.flags_checked['no_mkcalendar'] = False + self.flags_checked['read_only'] = False + except Exception as e: + ## calendar creation created an exception - return exception + cal = self.principal.calendar(cal_id=cal_id) + if not cal: + ## cal not made and does not exist, exception thrown. + ## Caller to decide why the calendar was not made + return (False, e) + + assert(cal) + + try: + cal.delete() + try: + cal = self.principal.calendar(cal_id=cal_id) + cal.events() + except NotFoundError: + cal = None + ## Delete throw no exceptions, but was the calendar deleted? + if calmade and (not cal or self.flags_checked.get('non_existing_calendar_found')): + self.flags_checked['no_delete_calendar'] = False + ## Calendar probably deleted OK. + ## (in the case of non_existing_calendar_found, we should add + ## some events t o the calendar, delete the calendar and make + ## sure no events are found on a new calendar with same ID) + else: + ## Calendar not deleted. + ## Perhaps the server needs some time to delete the calendar + time.sleep(10) + try: + cal = self.principal.calendar(cal_id=cal_id) + assert(cal) + cal.events() + ## Calendar not deleted, but no exception thrown. + ## Perhaps it's a "move to thrashbin"-regime on the server + self.flags_checked['no_delete_calendar'] = 'maybe' + except NotFoundError as e: + ## Calendar was deleted, it just took some time. + self.flags_checked['no_delete_calendar'] = False + self.flags_checked['rate_limited'] = True + return (calmade, e) + return (calmade, None) + except Exception as e: + self.flags_checked['no_delete_calendar'] = True + time.sleep(10) + try: + cal.delete() + self.flags_Checked['no_delete_calendar'] = False + self.flags_Checked['rate_limited'] = True + except Exception as e2: + pass + return (calmade, e) + + def check_principal(self): + ## TODO + ## There was a sabre server having this issue. + ## I'm not sure if this will give the right result, as I don't have + ## access to any test servers with this compatibility-quirk. + ## In any case, this test will stop the script on authorization problems + try: + self.principal = self.client_obj.principal() + self.flags_checked['no-current-user-principal'] = False + except AuthorizationError: + raise + except DAVError: + self.flags_checked['no-current-user-principal'] = True + + def check_mkcalendar(self): + try: + cal = self.principal.calendar(cal_id="this_should_not_exist") + cal.events() + self.flags_checked["non_existing_calendar_found"] = True + except NotFoundError: + self.flags_checked["non_existing_calendar_found"] = False + except: + import pdb; pdb.set_trace() + makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") + if makeret[0]: + self._default_calendar = self.principal.make_calendar(name="Yep", cal_id="pythoncaldav-test") + return + makeret = self._try_make_calendar(cal_id="pythoncaldav-test") + if makeret[0]: + self._default_calendar = self.principal.make_calendar(cal_id="pythoncaldav-test") + self.flags_checked['no_displayname'] = True + return + unique_id1 = "testcalendar-" + str(uuid.uuid4()) + unique_id2 = "testcalendar-" + str(uuid.uuid4()) + makeret = self._try_make_calendar(cal_id=unique_id1) + if makeret[0]: + self._default_calendar = self.principal.make_calendar(cal_id=unique_id2) + self.flags_checked['unique_calendar_ids'] = True + unique_id = "testcalendar-" + str(uuid.uuid4()) + makeret = self._try_make_calendar(cal_id=unique_id, name='Yep') + if not makeret[0]: + self.flags_checked['no_displayname'] = True + + def check_support(self): + self.flags_checked['dav_not_supported'] = True + self.flags_checked['no_scheduling'] = True + try: + self.flags_checked['dav_not_supported'] = not self.client_obj.check_dav_support() + self.flags_checked['no_scheduling'] = not self.client_obj.check_scheduling_support() + except: + pass + if not self.flags_checked['no_scheduling']: + try: + inbox = self.principal.schedule_inbox() + outbox = self.principal.schedule_outbox() + self.flags_checked['no_scheduling_mailbox'] = False + except: + self.flags_checked['no_scheduling_mailbox'] = True + + def check_propfind(self): + try: + foo = self.client_obj.propfind( + self.principal.url, + props='' + '' + " " + "", + ) + assert "resourcetype" in to_local(foo.raw) + + # next, the internal _query_properties, returning an xml tree ... + foo2 = self.principal._query_properties( + [ + dav.Status(), + ] + ) + assert "resourcetype" in to_local(foo.raw) + self.flags_checked['propfind_allprop_failure'] = False + except: + self.flags_checked['propfind_allprop_failure'] = True + + def check_event(self): + try: + cal = self._default_calendar + cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], uid='check_event_1') + cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') + try: + try: + objcnt = len(cal.objects()) == 2 + except: + objcnt = 0 + if objcnt != 2: + if len(cal.events()) == 2: + self.flags_checked['search_always_needs_comptype'] = True + else: + import pdb; pdb.set_trace() + ## we should not be here + else: + self.flags_checked['search_always_needs_comptype'] = False + + events = cal.search(summary='Test event 1') + if len(events) == 0: + events = cal.search(summary='Test event 1', event=True) + if len(events)==1: + self.flags_checked['search_needs_comptype'] = True + self.flags_checked['no_search'] = False + self.flags_checked['text_search_not_working'] = False + else: + import pdb; pdb.set_trace() + ## we should not be here ... unless search is not working? + elif len(events) == 1: + self.flags_checked['no_search'] = False + self.flags_checked['text_search_not_working'] = False + self.flags_checked['search_always_needs_comptype'] = False + events = cal.search(summary='test event 1', event=True) + if len(events) == 1: + self.flags_checked['text_search_is_case_insensitive'] = True + elif len(events)==0: + self.flags_checked['text_search_is_case_insensitive'] = False + else: + ## we should not be here + import pdb; pdb.set_trace() + pass + events = cal.search(summary='test event', event=True) + if len(events)==2: + self.flags_checked['text_search_is_exact_match_only'] = False + elif len(events)==0: + self.flags_checked['text_search_is_exact_match_only'] = 'maybe' + ## may also be text_search_is_exact_match_sometimes + events = cal.search(category='foo', event=True) + if len(events) == 1: + self.flags_checked["category_search_yields_nothing"] = False + elif len(events) == 0: + self.flags_checked["category_search_yields_nothing"] = False + else: + ## we should not be here + import pdb; pdb.set_trace() + pass + + except: + import pdb; pdb.set_trace() + ## TODO ... + raise + except: + import pdb; pdb.set_trace() + raise + + def check_all(self): + try: + self.check_principal() + self.check_support() + self.check_propfind() + self.check_mkcalendar() + self.check_event() + finally: + if self._default_calendar: + self._default_calendar.delete() + + def report(self, verbose, json): + if self.client_obj.incompatibilities is not None: + flags_found = set([x for x in self.flags_checked if self.flags_checked[x]]) + self.diff1 = set(self.client_obj.incompatibilities) - flags_found + self.diff2 = flags_found - set(self.client_obj.incompatibilities) + if json: + click.echo(json.dumps({'flags_checked': self.flags_checked, 'diff1': self.diff1, 'diff2': self.diff2})) + click.echo() + return + if verbose is False: + return + else: + for x in self.flags_checked: + if self.flags_checked[x] or verbose: + click.echo(f"{x:28} {self.flags_checked[x]}") + if verbose: + click.echo() + if self.diff1 or self.diff2: + click.echo("differences between configured quirk list and found quirk list:") + for x in self.diff1: + click.echo(f"-{x}") + for x in self.diff2: + click.echo(f"+{x}") + for x in self.flags_checked: + if self.flags_checked[x]: + click.echo(f"## {x}") + click.echo(incompatibility_description[x]) ## todo: format with linebreaks ... and indentation? + click.echo() + for x in self.other_info: + click.echo(f"{x:28} {self.other_info[x]}") + +## click-decorating ... this got messy, perhaps I should have used good, +## old argparse rather than "click" ... +def _set_conn_options(func): + """ + Decorator adding all the caldav connection params + """ + ## TODO: fetch this from the DAVClient.__init__ declaration + types = {"timeout": int, "auth": object, "headers": dict, "huge_tree": bool} + + for foo in CONNKEYS: + footype = types.get(foo, str) + if footype == object: + continue + func = click.option(f"--{foo}", type=footype)(func) + return func +@click.command() +@_set_conn_options +@click.option("--idx", type=int, help="Choose a server from the test config, by index number") +@click.option("--verbose/--quiet", help="More output") +@click.option("--json/--text", help="JSON output. Overrides verbose") +def check_server_compatibility(verbose, json, **kwargs): + conn = client(**kwargs) + obj = ServerQuirkChecker(conn) + obj.check_all() + obj.report(verbose=verbose, json=json) + conn.teardown(conn) + +if __name__ == '__main__': + check_server_compatibility() From dada1bcf23f73a2225790aca9cd65a3e836a2132 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 17 Nov 2024 09:10:09 +0100 Subject: [PATCH 07/34] work in progress --- check_server_compatibility.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index a037828..1793770 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -215,11 +215,14 @@ def check_event(self): elif len(events)==0: self.flags_checked['text_search_is_exact_match_only'] = 'maybe' ## may also be text_search_is_exact_match_sometimes - events = cal.search(category='foo', event=True) + try: + events = cal.search(category='foo', event=True) + except: + events = [] if len(events) == 1: self.flags_checked["category_search_yields_nothing"] = False elif len(events) == 0: - self.flags_checked["category_search_yields_nothing"] = False + self.flags_checked["category_search_yields_nothing"] = True else: ## we should not be here import pdb; pdb.set_trace() From d44b6daa4ebf788d7f0718d7594ad5b1cc9dcb1e Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 17 Nov 2024 12:01:45 +0100 Subject: [PATCH 08/34] work in progress --- check_server_compatibility.py | 134 ++++++++++++++++++---------------- tests/compatibility_issues.py | 6 -- tests/conf.py | 1 + tests/test_caldav.py | 89 +++++++++++----------- 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 1793770..b5b6964 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -21,6 +21,15 @@ def __init__(self, client_obj): def _try_make_calendar(self, cal_id, **kwargs): calmade = False + ## Check on "no_default_calendar" flag + try: + cals = self.principal.calendars() + cals[0].events() + self.flags_checked["no_default_calendar"] = False + except: + import pdb; pdb.set_trace() + self.flags_checked["no_default_calendar"] = True + ## In case calendar already exists ... wipe it first try: self.principal.calendar(cal_id=cal_id).delete() @@ -168,73 +177,63 @@ def check_propfind(self): self.flags_checked['propfind_allprop_failure'] = True def check_event(self): + cal = self._default_calendar + cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], uid='check_event_1') + cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') try: - cal = self._default_calendar - cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], uid='check_event_1') - cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') - try: - try: - objcnt = len(cal.objects()) == 2 - except: - objcnt = 0 - if objcnt != 2: - if len(cal.events()) == 2: - self.flags_checked['search_always_needs_comptype'] = True - else: - import pdb; pdb.set_trace() - ## we should not be here - else: - self.flags_checked['search_always_needs_comptype'] = False + objcnt = len(cal.objects()) + except: + objcnt = 0 + if objcnt != 2: + if len(cal.events()) == 2: + self.flags_checked['search_always_needs_comptype'] = True + else: + import pdb; pdb.set_trace() + ## we should not be here + else: + self.flags_checked['search_always_needs_comptype'] = False - events = cal.search(summary='Test event 1') - if len(events) == 0: - events = cal.search(summary='Test event 1', event=True) - if len(events)==1: - self.flags_checked['search_needs_comptype'] = True - self.flags_checked['no_search'] = False - self.flags_checked['text_search_not_working'] = False - else: - import pdb; pdb.set_trace() - ## we should not be here ... unless search is not working? - elif len(events) == 1: - self.flags_checked['no_search'] = False - self.flags_checked['text_search_not_working'] = False - self.flags_checked['search_always_needs_comptype'] = False - events = cal.search(summary='test event 1', event=True) - if len(events) == 1: - self.flags_checked['text_search_is_case_insensitive'] = True - elif len(events)==0: - self.flags_checked['text_search_is_case_insensitive'] = False - else: - ## we should not be here - import pdb; pdb.set_trace() - pass - events = cal.search(summary='test event', event=True) - if len(events)==2: - self.flags_checked['text_search_is_exact_match_only'] = False - elif len(events)==0: - self.flags_checked['text_search_is_exact_match_only'] = 'maybe' - ## may also be text_search_is_exact_match_sometimes - try: - events = cal.search(category='foo', event=True) - except: - events = [] - if len(events) == 1: - self.flags_checked["category_search_yields_nothing"] = False - elif len(events) == 0: - self.flags_checked["category_search_yields_nothing"] = True - else: - ## we should not be here - import pdb; pdb.set_trace() - pass - - except: + events = cal.search(summary='Test event 1') + if len(events) == 0: + events = cal.search(summary='Test event 1', event=True) + if len(events)==1: + self.flags_checked['search_needs_comptype'] = True + self.flags_checked['no_search'] = False + self.flags_checked['text_search_not_working'] = False + else: import pdb; pdb.set_trace() - ## TODO ... - raise + ## we should not be here ... unless search is not working? + elif len(events) == 1: + self.flags_checked['no_search'] = False + self.flags_checked['text_search_not_working'] = False + self.flags_checked['search_always_needs_comptype'] = False + events = cal.search(summary='test event 1', event=True) + if len(events) == 1: + self.flags_checked['text_search_is_case_insensitive'] = True + elif len(events)==0: + self.flags_checked['text_search_is_case_insensitive'] = False + else: + ## we should not be here + import pdb; pdb.set_trace() + pass + events = cal.search(summary='test event', event=True) + if len(events)==2: + self.flags_checked['text_search_is_exact_match_only'] = False + elif len(events)==0: + self.flags_checked['text_search_is_exact_match_only'] = 'maybe' + ## may also be text_search_is_exact_match_sometimes + try: + events = cal.search(category='foo', event=True) except: + events = [] + if len(events) == 1: + self.flags_checked["category_search_yields_nothing"] = False + elif len(events) == 0: + self.flags_checked["category_search_yields_nothing"] = True + else: + ## we should not be here import pdb; pdb.set_trace() - raise + pass def check_all(self): try: @@ -243,11 +242,20 @@ def check_all(self): self.check_propfind() self.check_mkcalendar() self.check_event() + except: + import pdb; pdb.set_trace() + raise finally: if self._default_calendar: self._default_calendar.delete() def report(self, verbose, json): + if verbose: + if self.client_obj.server_name: + click.echo(f"# {self.client_obj.server_name} - {self.client_obj.url}") + else: + click.echo(f"# {self.client_obj.url}") + click.echo() if self.client_obj.incompatibilities is not None: flags_found = set([x for x in self.flags_checked if self.flags_checked[x]]) self.diff1 = set(self.client_obj.incompatibilities) - flags_found @@ -296,7 +304,7 @@ def _set_conn_options(func): @click.command() @_set_conn_options @click.option("--idx", type=int, help="Choose a server from the test config, by index number") -@click.option("--verbose/--quiet", help="More output") +@click.option("--verbose/--quiet", default=None, help="More output") @click.option("--json/--text", help="JSON output. Overrides verbose") def check_server_compatibility(verbose, json, **kwargs): conn = client(**kwargs) diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 1b209ad..6ed3d84 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -181,9 +181,6 @@ 'text_search_not_working': """Text search is generally broken""", - 'radicale_breaks_on_category_search': - """See https://github.com/Kozea/Radicale/issues/1125""", - 'fastmail_buggy_noexpand_date_search': """The 'blissful anniversary' recurrent example event is returned when asked for a no-expand date search for some timestamps covering a completely different date""", @@ -246,9 +243,6 @@ ## freebusy is not supported yet, but on the long-term road map "no_freebusy_rfc4791", - ## TODO: raise an issue on this one - "radicale_breaks_on_category_search", - 'no_scheduling', 'no_todo_datesearch', diff --git a/tests/conf.py b/tests/conf.py index cbdfb3c..f78f147 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -292,6 +292,7 @@ def client( setup(conn) conn.teardown = teardown conn.incompatibilities = kwargs.get('incompatibilities') + conn.server_name = name return conn diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 2166362..07f255f 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -1413,54 +1413,53 @@ def testSearchEvent(self): self.skip_on_compatibility_flag("text_search_not_working") ## category - if not self.check_compatibility_flag("radicale_breaks_on_category_search"): - some_events = c.search(comp_class=Event, category="PERSONAL") - if not self.check_compatibility_flag("category_search_yields_nothing"): + some_events = c.search(comp_class=Event, category="PERSONAL") + if not self.check_compatibility_flag("category_search_yields_nothing"): + assert len(some_events) == 1 + some_events = c.search(comp_class=Event, category="personal") + if not self.check_compatibility_flag("category_search_yields_nothing"): + if self.check_compatibility_flag("text_search_is_case_insensitive"): assert len(some_events) == 1 - some_events = c.search(comp_class=Event, category="personal") - if not self.check_compatibility_flag("category_search_yields_nothing"): - if self.check_compatibility_flag("text_search_is_case_insensitive"): - assert len(some_events) == 1 - else: - assert len(some_events) == 0 + else: + assert len(some_events) == 0 - ## This is not a very useful search, and it's sort of a client side bug that we allow it at all. - ## It will not match if categories field is set to "PERSONAL,ANNIVERSARY,SPECIAL OCCASION" - ## It may not match since the above is to be considered equivalent to the raw data entered. - some_events = c.search( - comp_class=Event, category="ANNIVERSARY,PERSONAL,SPECIAL OCCASION" - ) + ## This is not a very useful search, and it's sort of a client side bug that we allow it at all. + ## It will not match if categories field is set to "PERSONAL,ANNIVERSARY,SPECIAL OCCASION" + ## It may not match since the above is to be considered equivalent to the raw data entered. + some_events = c.search( + comp_class=Event, category="ANNIVERSARY,PERSONAL,SPECIAL OCCASION" + ) + assert len(some_events) in (0, 1) + ## TODO: This is actually a bug. We need to do client side filtering + some_events = c.search(comp_class=Event, category="PERSON") + if self.check_compatibility_flag("text_search_is_exact_match_sometimes"): assert len(some_events) in (0, 1) - ## TODO: This is actually a bug. We need to do client side filtering - some_events = c.search(comp_class=Event, category="PERSON") - if self.check_compatibility_flag("text_search_is_exact_match_sometimes"): - assert len(some_events) in (0, 1) - if self.check_compatibility_flag("text_search_is_exact_match_only"): - assert len(some_events) == 0 - elif not self.check_compatibility_flag("category_search_yields_nothing"): - assert len(some_events) == 1 + if self.check_compatibility_flag("text_search_is_exact_match_only"): + assert len(some_events) == 0 + elif not self.check_compatibility_flag("category_search_yields_nothing"): + assert len(some_events) == 1 - ## I expect "logical and" when combining category with a date range - no_events = c.search( - comp_class=Event, - category="PERSONAL", - start=datetime(2006, 7, 13, 13, 0), - end=datetime(2006, 7, 15, 13, 0), - ) - if not self.check_compatibility_flag( - "category_search_yields_nothing" - ) and not self.check_compatibility_flag("combined_search_not_working"): - assert len(no_events) == 0 - some_events = c.search( - comp_class=Event, - category="PERSONAL", - start=datetime(1997, 11, 1, 13, 0), - end=datetime(1997, 11, 3, 13, 0), - ) - if not self.check_compatibility_flag( - "category_search_yields_nothing" - ) and not self.check_compatibility_flag("combined_search_not_working"): - assert len(some_events) == 1 + ## I expect "logical and" when combining category with a date range + no_events = c.search( + comp_class=Event, + category="PERSONAL", + start=datetime(2006, 7, 13, 13, 0), + end=datetime(2006, 7, 15, 13, 0), + ) + if not self.check_compatibility_flag( + "category_search_yields_nothing" + ) and not self.check_compatibility_flag("combined_search_not_working"): + assert len(no_events) == 0 + some_events = c.search( + comp_class=Event, + category="PERSONAL", + start=datetime(1997, 11, 1, 13, 0), + end=datetime(1997, 11, 3, 13, 0), + ) + if not self.check_compatibility_flag( + "category_search_yields_nothing" + ) and not self.check_compatibility_flag("combined_search_not_working"): + assert len(some_events) == 1 some_events = c.search(comp_class=Event, summary="Bastille Day Party") assert len(some_events) == 1 @@ -1628,8 +1627,6 @@ def testSearchTodos(self): assert len(some_todos) == 6 ## category - self.skip_on_compatibility_flag("radicale_breaks_on_category_search") - ## Too much copying of the examples ... some_todos = c.search(comp_class=Todo, category="FINANCE") if not self.check_compatibility_flag( From b4f2de4b27d8a814ff330928412a438742469b06 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 18 Nov 2024 12:15:19 +0100 Subject: [PATCH 09/34] event check is done, I think --- caldav/davclient.py | 19 ++- caldav/lib/debug.py | 2 + caldav/lib/error.py | 11 +- caldav/objects.py | 2 +- check_server_compatibility.py | 284 +++++++++++++++++++++++----------- tests/compatibility_issues.py | 11 +- tests/conf.py | 7 +- tests/test_caldav.py | 21 +-- 8 files changed, 241 insertions(+), 116 deletions(-) diff --git a/caldav/davclient.py b/caldav/davclient.py index 4da3e6b..711430a 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -211,6 +211,7 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] status = None href: Optional[str] = None propstats: List[_Element] = [] + check_404 = False ## special for purelymail error.assert_(response.tag == dav.Response.tag) for elem in response: if elem.tag == dav.Status.tag: @@ -223,13 +224,27 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] href = unquote(elem.text) elif elem.tag == dav.PropStat.tag: propstats.append(elem) + elif elem.tag == '{DAV:}error': + ## This happens with purelymail on a 404. + ## This code is mostly moot, but in debug + ## mode I want to be sure we do not toss away any data + children = elem.getchildren() + error.assert_(len(children)==1) + error.assert_(children[0].tag=='{https://purelymail.com}does-not-exist') + check_404 = True else: - error.assert_(False) + ## i.e. purelymail may contain one more tag, ... + ## This is probably not a breach of the standard. It may + ## probably be ignored. But it's something we may want to + ## know. + error.weirdness("unexpected element found in response", elem) error.assert_(href) + if check_404: + error.assert_('404' in status) ## TODO: is this safe/sane? ## Ref https://github.com/python-caldav/caldav/issues/435 the paths returned may be absolute URLs, ## but the caller expects them to be paths. Could we have issues when a server has same path - ## but different URLs for different elements? + ## but different URLs for different elements? Perhaps href should always be made into an URL-object? if ":" in href: href = unquote(URL(href).path) return (cast(str, href), propstats, status) diff --git a/caldav/lib/debug.py b/caldav/lib/debug.py index 9c7bcd2..cdb4951 100644 --- a/caldav/lib/debug.py +++ b/caldav/lib/debug.py @@ -2,6 +2,8 @@ def xmlstring(root): + if isinstance(root, str): + return root if hasattr(root, "xmlelement"): root = root.xmlelement() return etree.tostring(root, pretty_print=True).decode("utf-8") diff --git a/caldav/lib/error.py b/caldav/lib/error.py index 93e5e00..3d795f2 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -25,6 +25,13 @@ else: log.setLevel(logging.WARNING) +def weirdness(*reasons): + from caldav.lib.debug import xmlstring + reason = " : ".join([xmlstring(x) for x in reasons]) + log.warning(f"Deviation from expectations found: {reason}") + if debugmode == "DEBUG_PDB": + log.error(f"Dropping into debugger due to {reason}") + import pdb; pdb.set_trace() def assert_(condition: object) -> None: try: @@ -36,9 +43,7 @@ def assert_(condition: object) -> None: ) elif debugmode == "DEBUG_PDB": log.error("Deviation from expectations found. Dropping into debugger") - import pdb - - pdb.set_trace() + import pdb; pdb.set_trace() else: raise diff --git a/caldav/objects.py b/caldav/objects.py index 13cc740..fcdfed7 100644 --- a/caldav/objects.py +++ b/caldav/objects.py @@ -791,7 +791,7 @@ def _create( if name: try: self.set_properties([display_name]) - except: + except Exception as e: ## TODO: investigate. Those asserts break. error.assert_(False) try: diff --git a/check_server_compatibility.py b/check_server_compatibility.py index b5b6964..d626816 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -6,30 +6,37 @@ from caldav.lib.error import AuthorizationError, DAVError, NotFoundError from caldav.lib.python_utilities import to_local from caldav.elements import dav +from json import dumps +import caldav import datetime import click import time -import json import uuid +def _delay_decorator(f, delay=10): + def foo(*a, **kwa): + time.sleep(delay) + return f(*a, **kwa) + return foo + class ServerQuirkChecker(): def __init__(self, client_obj): self.client_obj = client_obj self.flags_checked = {} self.other_info = {} + def set_flag(self, flag, value=True): + if flag == 'rate_limited': + self.client_obj.request = _delay_decorator(self.client_obj.request) + elif flag == 'search_delay': + caldav.Calendar.search = _delay_decorator(caldav.Calendar.search, 60) + if hasattr(self, '_default_calendar'): + self._default_calendar.search = _delay_decorator(self._default_calendar.search, 60) + self.flags_checked[flag] = value + def _try_make_calendar(self, cal_id, **kwargs): calmade = False - ## Check on "no_default_calendar" flag - try: - cals = self.principal.calendars() - cals[0].events() - self.flags_checked["no_default_calendar"] = False - except: - import pdb; pdb.set_trace() - self.flags_checked["no_default_calendar"] = True - ## In case calendar already exists ... wipe it first try: self.principal.calendar(cal_id=cal_id).delete() @@ -39,13 +46,19 @@ def _try_make_calendar(self, cal_id, **kwargs): ## create the calendar try: cal = self.principal.make_calendar(cal_id=cal_id, **kwargs) - ## calendar creation probably went OK + ## calendar creation probably went OK, but we need to be sure... + cal.events() + ## calendar creation must have gone OK. calmade = True - self.flags_checked['no_mkcalendar'] = False - self.flags_checked['read_only'] = False + self.set_flag('no_mkcalendar', False) + self.set_flag('read_only', False) except Exception as e: ## calendar creation created an exception - return exception cal = self.principal.calendar(cal_id=cal_id) + try: + cal.events() + except: + cal = None if not cal: ## cal not made and does not exist, exception thrown. ## Caller to decide why the calendar was not made @@ -57,12 +70,12 @@ def _try_make_calendar(self, cal_id, **kwargs): cal.delete() try: cal = self.principal.calendar(cal_id=cal_id) - cal.events() + events = cal.events() except NotFoundError: cal = None ## Delete throw no exceptions, but was the calendar deleted? - if calmade and (not cal or self.flags_checked.get('non_existing_calendar_found')): - self.flags_checked['no_delete_calendar'] = False + if (not cal or (self.flags_checked.get('non_existing_calendar_found' and len(events)==0))): + self.set_flag('no_delete_calendar', False) ## Calendar probably deleted OK. ## (in the case of non_existing_calendar_found, we should add ## some events t o the calendar, delete the calendar and make @@ -77,23 +90,22 @@ def _try_make_calendar(self, cal_id, **kwargs): cal.events() ## Calendar not deleted, but no exception thrown. ## Perhaps it's a "move to thrashbin"-regime on the server - self.flags_checked['no_delete_calendar'] = 'maybe' + self.set_flag('no_delete_calendar', 'maybe') except NotFoundError as e: ## Calendar was deleted, it just took some time. - self.flags_checked['no_delete_calendar'] = False - self.flags_checked['rate_limited'] = True + self.set_flag('no_delete_calendar', False) + self.set_flag('rate_limited', True) return (calmade, e) return (calmade, None) except Exception as e: - self.flags_checked['no_delete_calendar'] = True + self.set_flag('no_delete_calendar', True) time.sleep(10) try: cal.delete() - self.flags_Checked['no_delete_calendar'] = False - self.flags_Checked['rate_limited'] = True + self.set_flag('no_delete_calendar', False) + self.set_flag('rate_limited', True) except Exception as e2: pass - return (calmade, e) def check_principal(self): ## TODO @@ -103,21 +115,31 @@ def check_principal(self): ## In any case, this test will stop the script on authorization problems try: self.principal = self.client_obj.principal() - self.flags_checked['no-current-user-principal'] = False + self.set_flag('no-current-user-principal', False) except AuthorizationError: raise except DAVError: - self.flags_checked['no-current-user-principal'] = True + self.set_flag('no-current-user-principal', True) def check_mkcalendar(self): try: cal = self.principal.calendar(cal_id="this_should_not_exist") cal.events() - self.flags_checked["non_existing_calendar_found"] = True + self.set_flag("non_existing_calendar_found", True) except NotFoundError: - self.flags_checked["non_existing_calendar_found"] = False - except: + self.set_flag("non_existing_calendar_found", False) + except Exception as e: import pdb; pdb.set_trace() + pass + ## Check on "no_default_calendar" flag + try: + cals = self.principal.calendars() + cals[0].events() + self.set_flag("no_default_calendar", False) + self._default_calendar = cals[0] + except: + self.set_flag("no_default_calendar", True) + makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: self._default_calendar = self.principal.make_calendar(name="Yep", cal_id="pythoncaldav-test") @@ -125,34 +147,36 @@ def check_mkcalendar(self): makeret = self._try_make_calendar(cal_id="pythoncaldav-test") if makeret[0]: self._default_calendar = self.principal.make_calendar(cal_id="pythoncaldav-test") - self.flags_checked['no_displayname'] = True + self.set_flag('no_displayname', True) return unique_id1 = "testcalendar-" + str(uuid.uuid4()) unique_id2 = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id1) if makeret[0]: self._default_calendar = self.principal.make_calendar(cal_id=unique_id2) - self.flags_checked['unique_calendar_ids'] = True + self.set_flag('unique_calendar_ids', True) unique_id = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id, name='Yep') - if not makeret[0]: + if not makeret[0] and not self.flags_checked.get('no_mkcalendar', True): self.flags_checked['no_displayname'] = True + if not 'no_mkcalendar' in self.flags_checked: + self.set_flag('no_mkcalendar', True) def check_support(self): - self.flags_checked['dav_not_supported'] = True - self.flags_checked['no_scheduling'] = True + self.set_flag('dav_not_supported', True) + self.set_flag('no_scheduling', True) try: - self.flags_checked['dav_not_supported'] = not self.client_obj.check_dav_support() - self.flags_checked['no_scheduling'] = not self.client_obj.check_scheduling_support() + self.set_flag('dav_not_supported', not self.client_obj.check_dav_support()) + self.set_flag('no_scheduling', not self.client_obj.check_scheduling_support()) except: pass if not self.flags_checked['no_scheduling']: try: inbox = self.principal.schedule_inbox() outbox = self.principal.schedule_outbox() - self.flags_checked['no_scheduling_mailbox'] = False + self.set_flag('no_scheduling_mailbox', False) except: - self.flags_checked['no_scheduling_mailbox'] = True + self.set_flag('no_scheduling_mailbox', True) def check_propfind(self): try: @@ -172,68 +196,144 @@ def check_propfind(self): ] ) assert "resourcetype" in to_local(foo.raw) - self.flags_checked['propfind_allprop_failure'] = False + self.set_flag('propfind_allprop_failure', False) except: - self.flags_checked['propfind_allprop_failure'] = True + self.set_flag('propfind_allprop_failure', True) def check_event(self): cal = self._default_calendar - cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], uid='check_event_1') - cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') - try: - objcnt = len(cal.objects()) - except: - objcnt = 0 - if objcnt != 2: - if len(cal.events()) == 2: - self.flags_checked['search_always_needs_comptype'] = True + + ## Two simple events with text fields, dtstart=now and no dtend + obj1 = cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], class_='CONFIDENTIAL', uid='check_event_1') + obj2 = cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') + try: ## try-finally-block covering testing of obj1 and obj2 + try: + foo = cal.event_by_uid('check_event_2') + assert(foo) + self.set_flag('object_by_uid_is_broken', False) + except: + time.sleep(60) + try: + foo = cal.event_by_uid('check_event_2') + assert(foo) + self.set_flag('search_delay') + self.set_flag('object_by_uid_is_broken', False) + except: + self.set_flag('object_by_uid_is_broken', True) + + try: + objcnt = len(cal.objects()) + except: + objcnt = 0 + if objcnt != 2: + if len(cal.events()) == 2: + self.set_flag('search_always_needs_comptype', True) + else: + import pdb; pdb.set_trace() + pass + ## we should not be here else: - import pdb; pdb.set_trace() - ## we should not be here - else: - self.flags_checked['search_always_needs_comptype'] = False + self.set_flag('search_always_needs_comptype', False) - events = cal.search(summary='Test event 1') - if len(events) == 0: + ## purelymail writes things to an index as a background thread + ## and have delayed search. Let's test for that first. events = cal.search(summary='Test event 1', event=True) - if len(events)==1: - self.flags_checked['search_needs_comptype'] = True - self.flags_checked['no_search'] = False - self.flags_checked['text_search_not_working'] = False + if len(events) == 0: + events = cal.search(summary='Test event 1', event=True) + if len(events) == 1: + self.set_flag('rate_limited', True) + if len(events) == 1: + self.set_flag('no_search', False) + self.set_flag('text_search_not_working', False) else: + self.set_flag('text_search_not_working', True) + + objs = cal.search(summary='Test event 1') + if len(objs) == 0 and len(events)==1: + self.set_flag('search_needs_comptype', True) + elif len(objs) == 1: + self.set_flag('search_always_needs_comptype', False) + + if not self.flags_checked['text_search_not_working']: + events = cal.search(summary='test event 1', event=True) + if len(events) == 1: + self.set_flag('text_search_is_case_insensitive', True) + elif len(events)==0: + self.set_flag('text_search_is_case_insensitive', False) + else: + ## we should not be here + import pdb; pdb.set_trace() + pass + events = cal.search(summary='test event', event=True) + if len(events)==2: + self.set_flag('text_search_is_exact_match_only', False) + elif len(events)==0: + self.set_flag('text_search_is_exact_match_only', 'maybe') + ## may also be text_search_is_exact_match_sometimes + events = cal.search(summary='Test event 1', class_='CONFIDENTIAL', event=True) + if len(events)==1: + self.set_flag('combined_search_not_working', False) + elif len(events)==0: + self.set_flag('combined_search_not_working', True) + else: + import pdb; pdb.set_trace() + ## We should not be here + pass + try: + events = cal.search(category='foo', event=True) + except: + events = [] + if len(events) == 1: + self.set_flag("category_search_yields_nothing", False) + elif len(events) == 0: + self.set_flag("category_search_yields_nothing", True) + else: + ## we should not be here import pdb; pdb.set_trace() - ## we should not be here ... unless search is not working? - elif len(events) == 1: - self.flags_checked['no_search'] = False - self.flags_checked['text_search_not_working'] = False - self.flags_checked['search_always_needs_comptype'] = False - events = cal.search(summary='test event 1', event=True) - if len(events) == 1: - self.flags_checked['text_search_is_case_insensitive'] = True - elif len(events)==0: - self.flags_checked['text_search_is_case_insensitive'] = False - else: - ## we should not be here - import pdb; pdb.set_trace() - pass - events = cal.search(summary='test event', event=True) - if len(events)==2: - self.flags_checked['text_search_is_exact_match_only'] = False - elif len(events)==0: - self.flags_checked['text_search_is_exact_match_only'] = 'maybe' - ## may also be text_search_is_exact_match_sometimes + pass + + + events = cal.search(summary='test event', class_='CONFIDENTIAL', event=True) + finally: + obj1.delete() + obj2.delete() + + ## Recurring events try: - events = cal.search(category='foo', event=True) + yearly_time = cal.add_event(dtstart=datetime.datetime(2000, 1, 1, 0, 0), summary="Yearly timed event", uid='firework_event', rrule={'FREQ': 'YEARLY'}) + yearly_day = cal.add_event(dtstart=datetime.date(2000, 5, 1), summary="Yearly day event", uid='full_day_event', rrule={'FREQ': 'YEARLY'}) except: - events = [] - if len(events) == 1: - self.flags_checked["category_search_yields_nothing"] = False - elif len(events) == 0: - self.flags_checked["category_search_yields_nothing"] = True - else: - ## we should not be here + ## should not be here import pdb; pdb.set_trace() - pass + raise + try: + try: + events = cal.search(start=datetime.datetime(2001, 4, 1), end=datetime.datetime(2002,2,2), event=True) + assert len(events)==2 + self.set_flag('no_recurring', False) + except: + self.set_flag('no_recurring', True) + + if not (self.flags_checked['no_recurring']): + events = cal.search(start=datetime.datetime(2001, 4, 1), end=datetime.datetime(2002,2,2), event=True, expand='server') + assert len(events)==2 + if 'RRULE' in events[0].data: + assert 'RRULE' in events[1].data + assert not 'RECURRENCE-ID' in events[0].data + assert not 'RECURRENCE-ID' in events[1].data + self.set_flag('no_expand', True) + else: + assert not 'RRULE' in events[1].data + self.set_flag('no_expand', False) + if 'RECURRENCE-ID' in events[0].data and 'DTSTART:2001' in events[0].data: + assert 'RECURRENCE-ID' in events[1].data + self.set_flag('broken_expand', False) + else: + self.set_flag('broken_expand', True) + + finally: + yearly_time.delete() + yearly_day.delete() def check_all(self): try: @@ -246,8 +346,11 @@ def check_all(self): import pdb; pdb.set_trace() raise finally: - if self._default_calendar: - self._default_calendar.delete() + if self._default_calendar and not self.flags_checked['no_mkcalendar']: + try: + self._default_calendar.delete() + except: + pass def report(self, verbose, json): if verbose: @@ -261,7 +364,7 @@ def report(self, verbose, json): self.diff1 = set(self.client_obj.incompatibilities) - flags_found self.diff2 = flags_found - set(self.client_obj.incompatibilities) if json: - click.echo(json.dumps({'flags_checked': self.flags_checked, 'diff1': self.diff1, 'diff2': self.diff2})) + click.echo(dumps({'name': self.client_obj.server_name, 'url': str(self.client_obj.url), 'flags_checked': self.flags_checked, 'diff1': list(self.diff1), 'diff2': list(self.diff2)}, indent=4)) click.echo() return if verbose is False: @@ -278,6 +381,7 @@ def report(self, verbose, json): click.echo(f"-{x}") for x in self.diff2: click.echo(f"+{x}") + click.echo() for x in self.flags_checked: if self.flags_checked[x]: click.echo(f"## {x}") diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 6ed3d84..8c2015d 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -15,6 +15,9 @@ 'rate_limited': """It may be needed to pause a bit between each request when doing tests""", + 'search_delay': + """Server populates indexes through some background job, so it takes some time from an event is added/edited until it's possible to search for it""", + 'cleanup_calendar': """Remove everything on the calendar for every test""", @@ -229,6 +232,7 @@ ## TODO: remove this when shredding support for python 3.7 ## https://github.com/jelmer/xandikos/pull/194 'category_search_yields_nothing', + "search_needs_comptype", ## scheduling is not supported "no_scheduling", @@ -249,6 +253,7 @@ 'text_search_is_case_insensitive', 'text_search_is_exact_match_sometimes', 'combined_search_not_working', + "search_needs_comptype", ## extra features not specified in RFC5545 "calendar_order", @@ -434,11 +439,7 @@ ## Purelymail claims that the search indexes are "lazily" populated, ## so search works some minutes after the event was created/edited. - ## I tried adding arbitrary delays in commit 5d052b1 but still didn't - ## manage to get search to work. Should eventually do more research - ## into this. (personal email communication with contact@purelymail.com) - 'no_search', - 'object_by_uid_is_broken', + 'search_delay' ] # fmt: on diff --git a/tests/conf.py b/tests/conf.py index f78f147..13226e9 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -258,12 +258,7 @@ def client( no_args = not any (x for x in kwargs if kwargs[x] is not None) if idx is None and no_args and caldav_servers: ## No parameters given - find the first server in caldav_servers list - idx = 0 - while idx < len(caldav_servers) and not caldav_servers[idx].get("enable", True): - idx += 1 - if idx == len(caldav_servers): - return None - return client(idx=idx) + return client(idx=0) elif idx is not None and no_args and caldav_servers: return client(**caldav_servers[idx]) elif name is not None and no_args and caldav_servers: diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 07f255f..8817693 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -554,6 +554,12 @@ def testInviteAndRespond(self): ## inbox/outbox? +def _delay_decorator(f, t=60): + def foo(*a, **kwa): + time.sleep(t) + return f(*a, **kwa) + return foo + class RepeatedFunctionalTestsBaseClass: """This is a class with functional tests (tests that goes through basic functionality and actively communicates with third parties) @@ -601,15 +607,10 @@ def setup_method(self): self.caldav = client(**self.server_params) if self.check_compatibility_flag("rate_limited"): - - def delay_decorator(f): - def foo(*a, **kwa): - time.sleep(60) - return f(*a, **kwa) - - return foo - - self.caldav.request = delay_decorator(self.caldav.request) + self.caldav.request = _delay_decorator(self.caldav.request) + if self.check_compatibility_flag("search_delay"): + Calendar._search = Calendar.search + Calendar.search = _delay_decorator(Calendar.search) if False and self.check_compatibility_flag("no-current-user-principal"): self.principal = Principal( @@ -629,6 +630,8 @@ def foo(*a, **kwa): logging.debug("##############################") def teardown_method(self): + if self.check_compatibility_flag("search_delay"): + Calendar.search = Calendar._search logging.debug("############################") logging.debug("############## test teardown_method") logging.debug("############################") From 6fe61b96945d5db29fb04035a87d14b142492193 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 18 Nov 2024 20:12:41 +0100 Subject: [PATCH 10/34] bugfixing and xandikos compatibility matrix. xandikos now passes with same quirks reported by script --- check_server_compatibility.py | 6 +++++- tests/compatibility_issues.py | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index d626816..b90f497 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -142,7 +142,11 @@ def check_mkcalendar(self): makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: - self._default_calendar = self.principal.make_calendar(name="Yep", cal_id="pythoncaldav-test") + try: + self._default_calendar = self.principal.make_calendar(name="Yep", cal_id="pythoncaldav-test") + except: + self._default_calendar = self.principal.calendar(cal_id="pythoncaldav-test") + self._default_calendar.events() return makeret = self._try_make_calendar(cal_id="pythoncaldav-test") if makeret[0]: diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 8c2015d..85b0afe 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -228,16 +228,25 @@ 'text_search_is_exact_match_only', - ## This one is fixed - but still breaks our test code for python 3.7 - ## TODO: remove this when shredding support for python 3.7 - ## https://github.com/jelmer/xandikos/pull/194 - 'category_search_yields_nothing', "search_needs_comptype", ## scheduling is not supported "no_scheduling", ] +## This can soon be removed (relevant for running tests under python 3.7 and python 3.8) +## https://github.com/jelmer/xandikos/pull/194 +'category_search_yields_nothing', +try: + import xandikos.__version__ as xver + goodver = (0,2,12) + for i in range(0,2): + if xver[i] Date: Mon, 18 Nov 2024 20:44:27 +0100 Subject: [PATCH 11/34] style fix --- caldav/davclient.py | 12 +- caldav/lib/error.py | 11 +- check_server_compatibility.py | 341 +++++++++++++++++++++------------- tests/compatibility_issues.py | 2 +- tests/conf.py | 4 +- tests/test_caldav.py | 1 + 6 files changed, 235 insertions(+), 136 deletions(-) diff --git a/caldav/davclient.py b/caldav/davclient.py index 711430a..ee25f06 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -211,7 +211,7 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] status = None href: Optional[str] = None propstats: List[_Element] = [] - check_404 = False ## special for purelymail + check_404 = False ## special for purelymail error.assert_(response.tag == dav.Response.tag) for elem in response: if elem.tag == dav.Status.tag: @@ -224,13 +224,15 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] href = unquote(elem.text) elif elem.tag == dav.PropStat.tag: propstats.append(elem) - elif elem.tag == '{DAV:}error': + elif elem.tag == "{DAV:}error": ## This happens with purelymail on a 404. ## This code is mostly moot, but in debug ## mode I want to be sure we do not toss away any data children = elem.getchildren() - error.assert_(len(children)==1) - error.assert_(children[0].tag=='{https://purelymail.com}does-not-exist') + error.assert_(len(children) == 1) + error.assert_( + children[0].tag == "{https://purelymail.com}does-not-exist" + ) check_404 = True else: ## i.e. purelymail may contain one more tag, ... @@ -240,7 +242,7 @@ def _parse_response(self, response) -> Tuple[str, List[_Element], Optional[Any]] error.weirdness("unexpected element found in response", elem) error.assert_(href) if check_404: - error.assert_('404' in status) + error.assert_("404" in status) ## TODO: is this safe/sane? ## Ref https://github.com/python-caldav/caldav/issues/435 the paths returned may be absolute URLs, ## but the caller expects them to be paths. Could we have issues when a server has same path diff --git a/caldav/lib/error.py b/caldav/lib/error.py index 3d795f2..65d3640 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -25,13 +25,18 @@ else: log.setLevel(logging.WARNING) + def weirdness(*reasons): from caldav.lib.debug import xmlstring + reason = " : ".join([xmlstring(x) for x in reasons]) log.warning(f"Deviation from expectations found: {reason}") if debugmode == "DEBUG_PDB": log.error(f"Dropping into debugger due to {reason}") - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() + def assert_(condition: object) -> None: try: @@ -43,7 +48,9 @@ def assert_(condition: object) -> None: ) elif debugmode == "DEBUG_PDB": log.error("Deviation from expectations found. Dropping into debugger") - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() else: raise diff --git a/check_server_compatibility.py b/check_server_compatibility.py index b90f497..f0280b4 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -1,37 +1,45 @@ #!/usr/bin/env python - -from tests.conf import client -from tests.conf import CONNKEYS -from tests.compatibility_issues import incompatibility_description -from caldav.lib.error import AuthorizationError, DAVError, NotFoundError -from caldav.lib.python_utilities import to_local -from caldav.elements import dav -from json import dumps -import caldav import datetime -import click import time import uuid +from json import dumps + +import click + +import caldav +from caldav.elements import dav +from caldav.lib.error import AuthorizationError +from caldav.lib.error import DAVError +from caldav.lib.error import NotFoundError +from caldav.lib.python_utilities import to_local +from tests.compatibility_issues import incompatibility_description +from tests.conf import client +from tests.conf import CONNKEYS + def _delay_decorator(f, delay=10): def foo(*a, **kwa): time.sleep(delay) return f(*a, **kwa) + return foo -class ServerQuirkChecker(): + +class ServerQuirkChecker: def __init__(self, client_obj): self.client_obj = client_obj self.flags_checked = {} self.other_info = {} def set_flag(self, flag, value=True): - if flag == 'rate_limited': + if flag == "rate_limited": self.client_obj.request = _delay_decorator(self.client_obj.request) - elif flag == 'search_delay': + elif flag == "search_delay": caldav.Calendar.search = _delay_decorator(caldav.Calendar.search, 60) - if hasattr(self, '_default_calendar'): - self._default_calendar.search = _delay_decorator(self._default_calendar.search, 60) + if hasattr(self, "_default_calendar"): + self._default_calendar.search = _delay_decorator( + self._default_calendar.search, 60 + ) self.flags_checked[flag] = value def _try_make_calendar(self, cal_id, **kwargs): @@ -50,8 +58,8 @@ def _try_make_calendar(self, cal_id, **kwargs): cal.events() ## calendar creation must have gone OK. calmade = True - self.set_flag('no_mkcalendar', False) - self.set_flag('read_only', False) + self.set_flag("no_mkcalendar", False) + self.set_flag("read_only", False) except Exception as e: ## calendar creation created an exception - return exception cal = self.principal.calendar(cal_id=cal_id) @@ -64,8 +72,8 @@ def _try_make_calendar(self, cal_id, **kwargs): ## Caller to decide why the calendar was not made return (False, e) - assert(cal) - + assert cal + try: cal.delete() try: @@ -74,8 +82,12 @@ def _try_make_calendar(self, cal_id, **kwargs): except NotFoundError: cal = None ## Delete throw no exceptions, but was the calendar deleted? - if (not cal or (self.flags_checked.get('non_existing_calendar_found' and len(events)==0))): - self.set_flag('no_delete_calendar', False) + if not cal or ( + self.flags_checked.get( + "non_existing_calendar_found" and len(events) == 0 + ) + ): + self.set_flag("no_delete_calendar", False) ## Calendar probably deleted OK. ## (in the case of non_existing_calendar_found, we should add ## some events t o the calendar, delete the calendar and make @@ -86,24 +98,24 @@ def _try_make_calendar(self, cal_id, **kwargs): time.sleep(10) try: cal = self.principal.calendar(cal_id=cal_id) - assert(cal) + assert cal cal.events() ## Calendar not deleted, but no exception thrown. ## Perhaps it's a "move to thrashbin"-regime on the server - self.set_flag('no_delete_calendar', 'maybe') + self.set_flag("no_delete_calendar", "maybe") except NotFoundError as e: ## Calendar was deleted, it just took some time. - self.set_flag('no_delete_calendar', False) - self.set_flag('rate_limited', True) + self.set_flag("no_delete_calendar", False) + self.set_flag("rate_limited", True) return (calmade, e) return (calmade, None) except Exception as e: - self.set_flag('no_delete_calendar', True) + self.set_flag("no_delete_calendar", True) time.sleep(10) try: cal.delete() - self.set_flag('no_delete_calendar', False) - self.set_flag('rate_limited', True) + self.set_flag("no_delete_calendar", False) + self.set_flag("rate_limited", True) except Exception as e2: pass @@ -115,11 +127,11 @@ def check_principal(self): ## In any case, this test will stop the script on authorization problems try: self.principal = self.client_obj.principal() - self.set_flag('no-current-user-principal', False) + self.set_flag("no-current-user-principal", False) except AuthorizationError: raise except DAVError: - self.set_flag('no-current-user-principal', True) + self.set_flag("no-current-user-principal", True) def check_mkcalendar(self): try: @@ -129,7 +141,9 @@ def check_mkcalendar(self): except NotFoundError: self.set_flag("non_existing_calendar_found", False) except Exception as e: - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() pass ## Check on "no_default_calendar" flag try: @@ -143,44 +157,52 @@ def check_mkcalendar(self): makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: try: - self._default_calendar = self.principal.make_calendar(name="Yep", cal_id="pythoncaldav-test") + self._default_calendar = self.principal.make_calendar( + name="Yep", cal_id="pythoncaldav-test" + ) except: - self._default_calendar = self.principal.calendar(cal_id="pythoncaldav-test") + self._default_calendar = self.principal.calendar( + cal_id="pythoncaldav-test" + ) self._default_calendar.events() return makeret = self._try_make_calendar(cal_id="pythoncaldav-test") if makeret[0]: - self._default_calendar = self.principal.make_calendar(cal_id="pythoncaldav-test") - self.set_flag('no_displayname', True) + self._default_calendar = self.principal.make_calendar( + cal_id="pythoncaldav-test" + ) + self.set_flag("no_displayname", True) return unique_id1 = "testcalendar-" + str(uuid.uuid4()) unique_id2 = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id1) if makeret[0]: self._default_calendar = self.principal.make_calendar(cal_id=unique_id2) - self.set_flag('unique_calendar_ids', True) + self.set_flag("unique_calendar_ids", True) unique_id = "testcalendar-" + str(uuid.uuid4()) - makeret = self._try_make_calendar(cal_id=unique_id, name='Yep') - if not makeret[0] and not self.flags_checked.get('no_mkcalendar', True): - self.flags_checked['no_displayname'] = True - if not 'no_mkcalendar' in self.flags_checked: - self.set_flag('no_mkcalendar', True) + makeret = self._try_make_calendar(cal_id=unique_id, name="Yep") + if not makeret[0] and not self.flags_checked.get("no_mkcalendar", True): + self.flags_checked["no_displayname"] = True + if not "no_mkcalendar" in self.flags_checked: + self.set_flag("no_mkcalendar", True) def check_support(self): - self.set_flag('dav_not_supported', True) - self.set_flag('no_scheduling', True) + self.set_flag("dav_not_supported", True) + self.set_flag("no_scheduling", True) try: - self.set_flag('dav_not_supported', not self.client_obj.check_dav_support()) - self.set_flag('no_scheduling', not self.client_obj.check_scheduling_support()) + self.set_flag("dav_not_supported", not self.client_obj.check_dav_support()) + self.set_flag( + "no_scheduling", not self.client_obj.check_scheduling_support() + ) except: pass - if not self.flags_checked['no_scheduling']: + if not self.flags_checked["no_scheduling"]: try: inbox = self.principal.schedule_inbox() outbox = self.principal.schedule_outbox() - self.set_flag('no_scheduling_mailbox', False) + self.set_flag("no_scheduling_mailbox", False) except: - self.set_flag('no_scheduling_mailbox', True) + self.set_flag("no_scheduling_mailbox", True) def check_propfind(self): try: @@ -200,91 +222,110 @@ def check_propfind(self): ] ) assert "resourcetype" in to_local(foo.raw) - self.set_flag('propfind_allprop_failure', False) + self.set_flag("propfind_allprop_failure", False) except: - self.set_flag('propfind_allprop_failure', True) + self.set_flag("propfind_allprop_failure", True) def check_event(self): cal = self._default_calendar ## Two simple events with text fields, dtstart=now and no dtend - obj1 = cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 1", categories=['foo','bar'], class_='CONFIDENTIAL', uid='check_event_1') - obj2 = cal.add_event(dtstart=datetime.datetime.now(), summary="Test event 2", categories=['zoo','test'], uid='check_event_2') - try: ## try-finally-block covering testing of obj1 and obj2 + obj1 = cal.add_event( + dtstart=datetime.datetime.now(), + summary="Test event 1", + categories=["foo", "bar"], + class_="CONFIDENTIAL", + uid="check_event_1", + ) + obj2 = cal.add_event( + dtstart=datetime.datetime.now(), + summary="Test event 2", + categories=["zoo", "test"], + uid="check_event_2", + ) + try: ## try-finally-block covering testing of obj1 and obj2 try: - foo = cal.event_by_uid('check_event_2') - assert(foo) - self.set_flag('object_by_uid_is_broken', False) + foo = cal.event_by_uid("check_event_2") + assert foo + self.set_flag("object_by_uid_is_broken", False) except: time.sleep(60) try: - foo = cal.event_by_uid('check_event_2') - assert(foo) - self.set_flag('search_delay') - self.set_flag('object_by_uid_is_broken', False) + foo = cal.event_by_uid("check_event_2") + assert foo + self.set_flag("search_delay") + self.set_flag("object_by_uid_is_broken", False) except: - self.set_flag('object_by_uid_is_broken', True) - + self.set_flag("object_by_uid_is_broken", True) + try: objcnt = len(cal.objects()) except: objcnt = 0 if objcnt != 2: if len(cal.events()) == 2: - self.set_flag('search_always_needs_comptype', True) + self.set_flag("search_always_needs_comptype", True) else: - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() pass ## we should not be here else: - self.set_flag('search_always_needs_comptype', False) + self.set_flag("search_always_needs_comptype", False) ## purelymail writes things to an index as a background thread ## and have delayed search. Let's test for that first. - events = cal.search(summary='Test event 1', event=True) + events = cal.search(summary="Test event 1", event=True) if len(events) == 0: - events = cal.search(summary='Test event 1', event=True) + events = cal.search(summary="Test event 1", event=True) if len(events) == 1: - self.set_flag('rate_limited', True) + self.set_flag("rate_limited", True) if len(events) == 1: - self.set_flag('no_search', False) - self.set_flag('text_search_not_working', False) + self.set_flag("no_search", False) + self.set_flag("text_search_not_working", False) else: - self.set_flag('text_search_not_working', True) + self.set_flag("text_search_not_working", True) - objs = cal.search(summary='Test event 1') - if len(objs) == 0 and len(events)==1: - self.set_flag('search_needs_comptype', True) - elif len(objs) == 1: - self.set_flag('search_always_needs_comptype', False) + objs = cal.search(summary="Test event 1") + if len(objs) == 0 and len(events) == 1: + self.set_flag("search_needs_comptype", True) + elif len(objs) == 1: + self.set_flag("search_always_needs_comptype", False) - if not self.flags_checked['text_search_not_working']: - events = cal.search(summary='test event 1', event=True) + if not self.flags_checked["text_search_not_working"]: + events = cal.search(summary="test event 1", event=True) if len(events) == 1: - self.set_flag('text_search_is_case_insensitive', True) - elif len(events)==0: - self.set_flag('text_search_is_case_insensitive', False) + self.set_flag("text_search_is_case_insensitive", True) + elif len(events) == 0: + self.set_flag("text_search_is_case_insensitive", False) else: ## we should not be here - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() pass - events = cal.search(summary='test event', event=True) - if len(events)==2: - self.set_flag('text_search_is_exact_match_only', False) - elif len(events)==0: - self.set_flag('text_search_is_exact_match_only', 'maybe') + events = cal.search(summary="test event", event=True) + if len(events) == 2: + self.set_flag("text_search_is_exact_match_only", False) + elif len(events) == 0: + self.set_flag("text_search_is_exact_match_only", "maybe") ## may also be text_search_is_exact_match_sometimes - events = cal.search(summary='Test event 1', class_='CONFIDENTIAL', event=True) - if len(events)==1: - self.set_flag('combined_search_not_working', False) - elif len(events)==0: - self.set_flag('combined_search_not_working', True) + events = cal.search( + summary="Test event 1", class_="CONFIDENTIAL", event=True + ) + if len(events) == 1: + self.set_flag("combined_search_not_working", False) + elif len(events) == 0: + self.set_flag("combined_search_not_working", True) else: - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() ## We should not be here pass try: - events = cal.search(category='foo', event=True) + events = cal.search(category="foo", event=True) except: events = [] if len(events) == 1: @@ -293,47 +334,72 @@ def check_event(self): self.set_flag("category_search_yields_nothing", True) else: ## we should not be here - import pdb; pdb.set_trace() - pass + import pdb + pdb.set_trace() + pass - events = cal.search(summary='test event', class_='CONFIDENTIAL', event=True) + events = cal.search(summary="test event", class_="CONFIDENTIAL", event=True) finally: obj1.delete() obj2.delete() ## Recurring events try: - yearly_time = cal.add_event(dtstart=datetime.datetime(2000, 1, 1, 0, 0), summary="Yearly timed event", uid='firework_event', rrule={'FREQ': 'YEARLY'}) - yearly_day = cal.add_event(dtstart=datetime.date(2000, 5, 1), summary="Yearly day event", uid='full_day_event', rrule={'FREQ': 'YEARLY'}) + yearly_time = cal.add_event( + dtstart=datetime.datetime(2000, 1, 1, 0, 0), + summary="Yearly timed event", + uid="firework_event", + rrule={"FREQ": "YEARLY"}, + ) + yearly_day = cal.add_event( + dtstart=datetime.date(2000, 5, 1), + summary="Yearly day event", + uid="full_day_event", + rrule={"FREQ": "YEARLY"}, + ) except: ## should not be here - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() raise try: try: - events = cal.search(start=datetime.datetime(2001, 4, 1), end=datetime.datetime(2002,2,2), event=True) - assert len(events)==2 - self.set_flag('no_recurring', False) + events = cal.search( + start=datetime.datetime(2001, 4, 1), + end=datetime.datetime(2002, 2, 2), + event=True, + ) + assert len(events) == 2 + self.set_flag("no_recurring", False) except: - self.set_flag('no_recurring', True) - - if not (self.flags_checked['no_recurring']): - events = cal.search(start=datetime.datetime(2001, 4, 1), end=datetime.datetime(2002,2,2), event=True, expand='server') - assert len(events)==2 - if 'RRULE' in events[0].data: - assert 'RRULE' in events[1].data - assert not 'RECURRENCE-ID' in events[0].data - assert not 'RECURRENCE-ID' in events[1].data - self.set_flag('no_expand', True) + self.set_flag("no_recurring", True) + + if not (self.flags_checked["no_recurring"]): + events = cal.search( + start=datetime.datetime(2001, 4, 1), + end=datetime.datetime(2002, 2, 2), + event=True, + expand="server", + ) + assert len(events) == 2 + if "RRULE" in events[0].data: + assert "RRULE" in events[1].data + assert not "RECURRENCE-ID" in events[0].data + assert not "RECURRENCE-ID" in events[1].data + self.set_flag("no_expand", True) else: - assert not 'RRULE' in events[1].data - self.set_flag('no_expand', False) - if 'RECURRENCE-ID' in events[0].data and 'DTSTART:2001' in events[0].data: - assert 'RECURRENCE-ID' in events[1].data - self.set_flag('broken_expand', False) + assert not "RRULE" in events[1].data + self.set_flag("no_expand", False) + if ( + "RECURRENCE-ID" in events[0].data + and "DTSTART:2001" in events[0].data + ): + assert "RECURRENCE-ID" in events[1].data + self.set_flag("broken_expand", False) else: - self.set_flag('broken_expand', True) + self.set_flag("broken_expand", True) finally: yearly_time.delete() @@ -347,10 +413,12 @@ def check_all(self): self.check_mkcalendar() self.check_event() except: - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() raise finally: - if self._default_calendar and not self.flags_checked['no_mkcalendar']: + if self._default_calendar and not self.flags_checked["no_mkcalendar"]: try: self._default_calendar.delete() except: @@ -368,7 +436,18 @@ def report(self, verbose, json): self.diff1 = set(self.client_obj.incompatibilities) - flags_found self.diff2 = flags_found - set(self.client_obj.incompatibilities) if json: - click.echo(dumps({'name': self.client_obj.server_name, 'url': str(self.client_obj.url), 'flags_checked': self.flags_checked, 'diff1': list(self.diff1), 'diff2': list(self.diff2)}, indent=4)) + click.echo( + dumps( + { + "name": self.client_obj.server_name, + "url": str(self.client_obj.url), + "flags_checked": self.flags_checked, + "diff1": list(self.diff1), + "diff2": list(self.diff2), + }, + indent=4, + ) + ) click.echo() return if verbose is False: @@ -380,7 +459,9 @@ def report(self, verbose, json): if verbose: click.echo() if self.diff1 or self.diff2: - click.echo("differences between configured quirk list and found quirk list:") + click.echo( + "differences between configured quirk list and found quirk list:" + ) for x in self.diff1: click.echo(f"-{x}") for x in self.diff2: @@ -389,11 +470,14 @@ def report(self, verbose, json): for x in self.flags_checked: if self.flags_checked[x]: click.echo(f"## {x}") - click.echo(incompatibility_description[x]) ## todo: format with linebreaks ... and indentation? + click.echo( + incompatibility_description[x] + ) ## todo: format with linebreaks ... and indentation? click.echo() for x in self.other_info: click.echo(f"{x:28} {self.other_info[x]}") + ## click-decorating ... this got messy, perhaps I should have used good, ## old argparse rather than "click" ... def _set_conn_options(func): @@ -409,9 +493,13 @@ def _set_conn_options(func): continue func = click.option(f"--{foo}", type=footype)(func) return func + + @click.command() @_set_conn_options -@click.option("--idx", type=int, help="Choose a server from the test config, by index number") +@click.option( + "--idx", type=int, help="Choose a server from the test config, by index number" +) @click.option("--verbose/--quiet", default=None, help="More output") @click.option("--json/--text", help="JSON output. Overrides verbose") def check_server_compatibility(verbose, json, **kwargs): @@ -421,5 +509,6 @@ def check_server_compatibility(verbose, json, **kwargs): obj.report(verbose=verbose, json=json) conn.teardown(conn) -if __name__ == '__main__': + +if __name__ == "__main__": check_server_compatibility() diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 85b0afe..dc9010b 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -235,7 +235,7 @@ ] ## This can soon be removed (relevant for running tests under python 3.7 and python 3.8) -## https://github.com/jelmer/xandikos/pull/194 +## https://github.com/jelmer/xandikos/pull/194 'category_search_yields_nothing', try: import xandikos.__version__ as xver diff --git a/tests/conf.py b/tests/conf.py index 13226e9..c7452fe 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -255,7 +255,7 @@ def client( idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs ): kwargs_ = kwargs.copy() - no_args = not any (x for x in kwargs if kwargs[x] is not None) + no_args = not any(x for x in kwargs if kwargs[x] is not None) if idx is None and no_args and caldav_servers: ## No parameters given - find the first server in caldav_servers list return client(idx=0) @@ -286,7 +286,7 @@ def client( conn = DAVClient(**kwargs_) setup(conn) conn.teardown = teardown - conn.incompatibilities = kwargs.get('incompatibilities') + conn.incompatibilities = kwargs.get("incompatibilities") conn.server_name = name return conn diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 8817693..4440c9f 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -558,6 +558,7 @@ def _delay_decorator(f, t=60): def foo(*a, **kwa): time.sleep(t) return f(*a, **kwa) + return foo class RepeatedFunctionalTestsBaseClass: From f5a4fd1c3bc48fbf211941c9eeb1199fa418c430 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 18 Nov 2024 20:50:26 +0100 Subject: [PATCH 12/34] bugfix --- check_server_compatibility.py | 14 ++++++++++++++ tests/compatibility_issues.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index f0280b4..96be017 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -405,6 +405,20 @@ def check_event(self): yearly_time.delete() yearly_day.delete() + def check_todo(self): + cal = self._default_calendar + + try: + ## Add a simplest possible todo + todo1 = cal.add_todo( + summary="This is a summary", + uid="check_todo_1", + ) + except: + import pdb; pdb.set_trace() + pass + + def check_all(self): try: self.check_principal() diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index dc9010b..8957e5d 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -242,7 +242,7 @@ goodver = (0,2,12) for i in range(0,2): if xver[i] Date: Mon, 18 Nov 2024 21:01:21 +0100 Subject: [PATCH 13/34] bugfix --- tests/compatibility_issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 8957e5d..c8429c6 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -238,7 +238,7 @@ ## https://github.com/jelmer/xandikos/pull/194 'category_search_yields_nothing', try: - import xandikos.__version__ as xver + from xandikos import __version__ as xver goodver = (0,2,12) for i in range(0,2): if xver[i] Date: Mon, 18 Nov 2024 21:08:17 +0100 Subject: [PATCH 14/34] debug --- check_server_compatibility.py | 5 +++-- tests/compatibility_issues.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 96be017..c40648b 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -414,9 +414,10 @@ def check_todo(self): summary="This is a summary", uid="check_todo_1", ) + self.set_flag('no_todo', False) except: - import pdb; pdb.set_trace() - pass + self.set_flag('no_todo') + return def check_all(self): diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index c8429c6..56c9b04 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -245,7 +245,7 @@ xandikos.append('category_search_yields_nothing') break except Exception: - pass + raise radicale = [ ## calendar listings and calendar creation works a bit From c60160fd519c8c89edf34652069a575f8ebbbf79 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 18 Nov 2024 21:10:34 +0100 Subject: [PATCH 15/34] bufgfix --- check_server_compatibility.py | 2 +- tests/compatibility_issues.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index c40648b..27f3d78 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -416,9 +416,9 @@ def check_todo(self): ) self.set_flag('no_todo', False) except: + import pdb; pdb.set_trace() self.set_flag('no_todo') return - def check_all(self): try: diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 56c9b04..dd66134 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -240,12 +240,12 @@ try: from xandikos import __version__ as xver goodver = (0,2,12) - for i in range(0,2): + for i in range(0,3): if xver[i] Date: Mon, 18 Nov 2024 21:14:22 +0100 Subject: [PATCH 16/34] bgufix --- caldav/lib/vcal.py | 2 ++ check_server_compatibility.py | 1 + 2 files changed, 3 insertions(+) diff --git a/caldav/lib/vcal.py b/caldav/lib/vcal.py index 0c3de5a..5a91899 100644 --- a/caldav/lib/vcal.py +++ b/caldav/lib/vcal.py @@ -154,6 +154,8 @@ def create_ical(ical_fragment=None, objtype=None, language="en_DK", **props): I somehow feel this fits more into the icalendar library than here """ ical_fragment = to_normal_str(ical_fragment) + if 'class_' in props: + props['class'] = props.pop('class_') if not ical_fragment or not re.search("^BEGIN:V", ical_fragment, re.MULTILINE): my_instance = icalendar.Calendar() if objtype is None: diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 27f3d78..adc4f78 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -317,6 +317,7 @@ def check_event(self): if len(events) == 1: self.set_flag("combined_search_not_working", False) elif len(events) == 0: + import pdb; pdb.set_trace() self.set_flag("combined_search_not_working", True) else: import pdb From 3b50b189dbfc6f644b554cf8c88ba3fe1f03cd0b Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Mon, 18 Nov 2024 21:24:19 +0100 Subject: [PATCH 17/34] refactoring a bit --- caldav/lib/vcal.py | 4 +- check_server_compatibility.py | 283 ++++++++++++++++++---------------- 2 files changed, 149 insertions(+), 138 deletions(-) diff --git a/caldav/lib/vcal.py b/caldav/lib/vcal.py index 5a91899..3962abd 100644 --- a/caldav/lib/vcal.py +++ b/caldav/lib/vcal.py @@ -154,8 +154,8 @@ def create_ical(ical_fragment=None, objtype=None, language="en_DK", **props): I somehow feel this fits more into the icalendar library than here """ ical_fragment = to_normal_str(ical_fragment) - if 'class_' in props: - props['class'] = props.pop('class_') + if "class_" in props: + props["class"] = props.pop("class_") if not ical_fragment or not re.search("^BEGIN:V", ical_fragment, re.MULTILINE): my_instance = icalendar.Calendar() if objtype is None: diff --git a/check_server_compatibility.py b/check_server_compatibility.py index adc4f78..79df3b0 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -243,104 +243,9 @@ def check_event(self): categories=["zoo", "test"], uid="check_event_2", ) - try: ## try-finally-block covering testing of obj1 and obj2 - try: - foo = cal.event_by_uid("check_event_2") - assert foo - self.set_flag("object_by_uid_is_broken", False) - except: - time.sleep(60) - try: - foo = cal.event_by_uid("check_event_2") - assert foo - self.set_flag("search_delay") - self.set_flag("object_by_uid_is_broken", False) - except: - self.set_flag("object_by_uid_is_broken", True) - try: - objcnt = len(cal.objects()) - except: - objcnt = 0 - if objcnt != 2: - if len(cal.events()) == 2: - self.set_flag("search_always_needs_comptype", True) - else: - import pdb - - pdb.set_trace() - pass - ## we should not be here - else: - self.set_flag("search_always_needs_comptype", False) - - ## purelymail writes things to an index as a background thread - ## and have delayed search. Let's test for that first. - events = cal.search(summary="Test event 1", event=True) - if len(events) == 0: - events = cal.search(summary="Test event 1", event=True) - if len(events) == 1: - self.set_flag("rate_limited", True) - if len(events) == 1: - self.set_flag("no_search", False) - self.set_flag("text_search_not_working", False) - else: - self.set_flag("text_search_not_working", True) - - objs = cal.search(summary="Test event 1") - if len(objs) == 0 and len(events) == 1: - self.set_flag("search_needs_comptype", True) - elif len(objs) == 1: - self.set_flag("search_always_needs_comptype", False) - - if not self.flags_checked["text_search_not_working"]: - events = cal.search(summary="test event 1", event=True) - if len(events) == 1: - self.set_flag("text_search_is_case_insensitive", True) - elif len(events) == 0: - self.set_flag("text_search_is_case_insensitive", False) - else: - ## we should not be here - import pdb - - pdb.set_trace() - pass - events = cal.search(summary="test event", event=True) - if len(events) == 2: - self.set_flag("text_search_is_exact_match_only", False) - elif len(events) == 0: - self.set_flag("text_search_is_exact_match_only", "maybe") - ## may also be text_search_is_exact_match_sometimes - events = cal.search( - summary="Test event 1", class_="CONFIDENTIAL", event=True - ) - if len(events) == 1: - self.set_flag("combined_search_not_working", False) - elif len(events) == 0: - import pdb; pdb.set_trace() - self.set_flag("combined_search_not_working", True) - else: - import pdb - - pdb.set_trace() - ## We should not be here - pass - try: - events = cal.search(category="foo", event=True) - except: - events = [] - if len(events) == 1: - self.set_flag("category_search_yields_nothing", False) - elif len(events) == 0: - self.set_flag("category_search_yields_nothing", True) - else: - ## we should not be here - import pdb - - pdb.set_trace() - pass - - events = cal.search(summary="test event", class_="CONFIDENTIAL", event=True) + try: ## try-finally-block covering testing of obj1 and obj2 + self._check_simple_events(obj1, obj2) finally: obj1.delete() obj2.delete() @@ -366,46 +271,150 @@ def check_event(self): pdb.set_trace() raise try: - try: - events = cal.search( - start=datetime.datetime(2001, 4, 1), - end=datetime.datetime(2002, 2, 2), - event=True, - ) - assert len(events) == 2 - self.set_flag("no_recurring", False) - except: - self.set_flag("no_recurring", True) - - if not (self.flags_checked["no_recurring"]): - events = cal.search( - start=datetime.datetime(2001, 4, 1), - end=datetime.datetime(2002, 2, 2), - event=True, - expand="server", - ) - assert len(events) == 2 - if "RRULE" in events[0].data: - assert "RRULE" in events[1].data - assert not "RECURRENCE-ID" in events[0].data - assert not "RECURRENCE-ID" in events[1].data - self.set_flag("no_expand", True) - else: - assert not "RRULE" in events[1].data - self.set_flag("no_expand", False) - if ( - "RECURRENCE-ID" in events[0].data - and "DTSTART:2001" in events[0].data - ): - assert "RECURRENCE-ID" in events[1].data - self.set_flag("broken_expand", False) - else: - self.set_flag("broken_expand", True) - + self._check_recurring_events(yearly_time, yearly_day) finally: yearly_time.delete() yearly_day.delete() + def _check_simple_events(self, obj1, obj2): + cal = self._default_calendar + try: + foo = cal.event_by_uid("check_event_2") + assert foo + self.set_flag("object_by_uid_is_broken", False) + except: + time.sleep(60) + try: + foo = cal.event_by_uid("check_event_2") + assert foo + self.set_flag("search_delay") + self.set_flag("object_by_uid_is_broken", False) + except: + self.set_flag("object_by_uid_is_broken", True) + + try: + objcnt = len(cal.objects()) + except: + objcnt = 0 + if objcnt != 2: + if len(cal.events()) == 2: + self.set_flag("search_always_needs_comptype", True) + else: + import pdb + + pdb.set_trace() + pass + ## we should not be here + else: + self.set_flag("search_always_needs_comptype", False) + + ## purelymail writes things to an index as a background thread + ## and have delayed search. Let's test for that first. + events = cal.search(summary="Test event 1", event=True) + if len(events) == 0: + events = cal.search(summary="Test event 1", event=True) + if len(events) == 1: + self.set_flag("rate_limited", True) + if len(events) == 1: + self.set_flag("no_search", False) + self.set_flag("text_search_not_working", False) + else: + self.set_flag("text_search_not_working", True) + + objs = cal.search(summary="Test event 1") + if len(objs) == 0 and len(events) == 1: + self.set_flag("search_needs_comptype", True) + elif len(objs) == 1: + self.set_flag("search_always_needs_comptype", False) + + if not self.flags_checked["text_search_not_working"]: + events = cal.search(summary="test event 1", event=True) + if len(events) == 1: + self.set_flag("text_search_is_case_insensitive", True) + elif len(events) == 0: + self.set_flag("text_search_is_case_insensitive", False) + else: + ## we should not be here + import pdb + + pdb.set_trace() + pass + events = cal.search(summary="test event", event=True) + if len(events) == 2: + self.set_flag("text_search_is_exact_match_only", False) + elif len(events) == 0: + self.set_flag("text_search_is_exact_match_only", "maybe") + ## may also be text_search_is_exact_match_sometimes + events = cal.search( + summary="Test event 1", class_="CONFIDENTIAL", event=True + ) + if len(events) == 1: + self.set_flag("combined_search_not_working", False) + elif len(events) == 0: + import pdb + + pdb.set_trace() + self.set_flag("combined_search_not_working", True) + else: + import pdb + + pdb.set_trace() + ## We should not be here + pass + try: + events = cal.search(category="foo", event=True) + except: + events = [] + if len(events) == 1: + self.set_flag("category_search_yields_nothing", False) + elif len(events) == 0: + self.set_flag("category_search_yields_nothing", True) + else: + ## we should not be here + import pdb + + pdb.set_trace() + pass + + events = cal.search(summary="test event", class_="CONFIDENTIAL", event=True) + + def _check_recurring_events(self, yearly_time, yearly_day): + cal = self._default_calendar + try: + events = cal.search( + start=datetime.datetime(2001, 4, 1), + end=datetime.datetime(2002, 2, 2), + event=True, + ) + assert len(events) == 2 + self.set_flag("no_recurring", False) + except: + self.set_flag("no_recurring", True) + + if self.flags_checked["no_recurring"]: + return + + events = cal.search( + start=datetime.datetime(2001, 4, 1), + end=datetime.datetime(2002, 2, 2), + event=True, + expand="server", + ) + assert len(events) == 2 + if "RRULE" in events[0].data: + assert "RRULE" in events[1].data + assert not "RECURRENCE-ID" in events[0].data + assert not "RECURRENCE-ID" in events[1].data + self.set_flag("no_expand", True) + else: + assert not "RRULE" in events[1].data + self.set_flag("no_expand", False) + if "RECURRENCE-ID" in events[0].data and "DTSTART:2001" in events[0].data: + assert "RECURRENCE-ID" in events[1].data + self.set_flag("broken_expand", False) + else: + self.set_flag("broken_expand", True) + def check_todo(self): cal = self._default_calendar @@ -415,10 +424,12 @@ def check_todo(self): summary="This is a summary", uid="check_todo_1", ) - self.set_flag('no_todo', False) + self.set_flag("no_todo", False) except: - import pdb; pdb.set_trace() - self.set_flag('no_todo') + import pdb + + pdb.set_trace() + self.set_flag("no_todo") return def check_all(self): From d822c9a6485100cc03d47a428970f1db15c706e9 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Tue, 19 Nov 2024 06:33:17 +0100 Subject: [PATCH 18/34] wip --- check_server_compatibility.py | 93 +++++++++++++++++++++++++++++++---- tests/compatibility_issues.py | 5 -- 2 files changed, 83 insertions(+), 15 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 79df3b0..f75f704 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -16,6 +16,11 @@ from tests.conf import client from tests.conf import CONNKEYS +def _debugger(): + import pdb + ## TODO: check some environmental flags + ## only hackers want the debug mode + pdb.set_trace() def _delay_decorator(f, delay=10): def foo(*a, **kwa): @@ -141,9 +146,7 @@ def check_mkcalendar(self): except NotFoundError: self.set_flag("non_existing_calendar_found", False) except Exception as e: - import pdb - - pdb.set_trace() + debugger() pass ## Check on "no_default_calendar" flag try: @@ -267,17 +270,48 @@ def check_event(self): except: ## should not be here import pdb - + pdb.set_trace() raise try: - self._check_recurring_events(yearly_time, yearly_day) + self._check_recurring_events(yearly_time, yearly_day, span1, span2) finally: yearly_time.delete() yearly_day.delete() + ## Finally, a check that searches involving timespans works as intended + try: + span1 = cal.add_event( + dtstart=datetime.datetime(2000, 7, 1, 8), + dtend=datetime.datetime(2000, 7, 1, 16), + summary="Yearly 8 hour event", + uid="eight_hour_event", + ) + span2 = cal.add_event( + dtstart=datetime.datetime(2000, 7, 2, 8), + duration=datetime.timedelta(hours=8), + summary="Yearly 8 hour event", + uid="eight_hour_event", + rrule={"FREQ": "YEARLY"}, + ) + except: + ## should not be here + import pdb + + pdb.set_trace() + raise + def _check_simple_events(self, obj1, obj2): cal = self._default_calendar + try: + obj1_ = cal.event_by_url(obj1.url) + assert obj1.data == obj1_.data + except: + self.set_flag("event_by_url_is_broken") + import pdb + + pdb.set_trace() + try: foo = cal.event_by_uid("check_event_2") assert foo @@ -416,21 +450,60 @@ def _check_recurring_events(self, yearly_time, yearly_day): self.set_flag("broken_expand", True) def check_todo(self): - cal = self._default_calendar - try: ## Add a simplest possible todo - todo1 = cal.add_todo( + todo_simple = cal.add_todo( summary="This is a summary", uid="check_todo_1", ) + todo_with_dtstart = cal.add_todo( + summary="This is a summary", + dtstart=datetime.datetime(2000,1,1) + uid="check_todo_2", + ) + todo_with_due = cal.add_todo( + summary="This is a summary", + due=datetime.datetime(2000,1,1,1,0,0) + uid="check_todo_3", + ) + todo_with_dtstart_due = cal.add_todo( + summary="This is a summary", + dtstart=datetime.datetime(2000,1,1,2,0,0) + due=datetime.datetime(2000,1,1,3,0,0) + uid="check_todo_4", + ) + todo_with_dtstart_dur = cal.add_todo( + summary="This is a summary", + dtstart=datetime.datetime(2000,1,1,4,0,0) + duration=datetime.timedelta(hours=1) + uid="check_todo_2", + ) + if not self.flags['object_by_uid_is_broken']: + assert(len(cal.todo_by_uid('check_todo_1'))==1) self.set_flag("no_todo", False) + except: + self.set_flag("no_todo") + return + try: + self.check_simple_todo(todo1) + finally: + todo1.delete() + + def check_simple_todo(self, todo): + cal = self._default_calendar + + ## search for a simple todo + try: + sr = cal.search(summary="This is a summary", todo=True) + assert(len(sr)==1) except: import pdb pdb.set_trace() - self.set_flag("no_todo") - return + ## simple search for a todo won't work. + ## I haven't seen that before. + ## TODO: add a flag for this + raise def check_all(self): try: diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index dd66134..a96adb0 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -144,9 +144,6 @@ """duration if no dtend is set, and most server implementations seems to """ """treat VTODOs the same""", - 'no_todo_on_standard_calendar': - """Tasklists can be created, but a normal calendar does not support tasks""", - 'unique_calendar_ids': """For every test, generate a new and unique calendar id""", @@ -284,10 +281,8 @@ ## earlier versions of Zimbra display-name could be changed, but ## then the calendar would not be available on the old URL ## anymore) - 'no_displayname', 'duplicate_in_other_calendar_with_same_uid_is_lost', 'event_by_url_is_broken', - 'no_todo_on_standard_calendar', 'no_sync_token', 'vtodo_datesearch_notime_task_is_skipped', 'category_search_yields_nothing', From d5825d130f03cb3de92ab80e7c1f56b307aec36a Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Tue, 19 Nov 2024 06:35:50 +0100 Subject: [PATCH 19/34] debug abstracted away --- check_server_compatibility.py | 40 +++++++++-------------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index f75f704..0a8e960 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -269,9 +269,7 @@ def check_event(self): ) except: ## should not be here - import pdb - - pdb.set_trace() + debugger() raise try: self._check_recurring_events(yearly_time, yearly_day, span1, span2) @@ -296,9 +294,7 @@ def check_event(self): ) except: ## should not be here - import pdb - - pdb.set_trace() + debugger() raise def _check_simple_events(self, obj1, obj2): @@ -308,9 +304,7 @@ def _check_simple_events(self, obj1, obj2): assert obj1.data == obj1_.data except: self.set_flag("event_by_url_is_broken") - import pdb - - pdb.set_trace() + debugger() try: foo = cal.event_by_uid("check_event_2") @@ -334,9 +328,7 @@ def _check_simple_events(self, obj1, obj2): if len(cal.events()) == 2: self.set_flag("search_always_needs_comptype", True) else: - import pdb - - pdb.set_trace() + debugger() pass ## we should not be here else: @@ -369,9 +361,7 @@ def _check_simple_events(self, obj1, obj2): self.set_flag("text_search_is_case_insensitive", False) else: ## we should not be here - import pdb - - pdb.set_trace() + debugger() pass events = cal.search(summary="test event", event=True) if len(events) == 2: @@ -385,14 +375,10 @@ def _check_simple_events(self, obj1, obj2): if len(events) == 1: self.set_flag("combined_search_not_working", False) elif len(events) == 0: - import pdb - - pdb.set_trace() + debugger() self.set_flag("combined_search_not_working", True) else: - import pdb - - pdb.set_trace() + debugger() ## We should not be here pass try: @@ -405,9 +391,7 @@ def _check_simple_events(self, obj1, obj2): self.set_flag("category_search_yields_nothing", True) else: ## we should not be here - import pdb - - pdb.set_trace() + debugger() pass events = cal.search(summary="test event", class_="CONFIDENTIAL", event=True) @@ -497,9 +481,7 @@ def check_simple_todo(self, todo): sr = cal.search(summary="This is a summary", todo=True) assert(len(sr)==1) except: - import pdb - - pdb.set_trace() + debugger() ## simple search for a todo won't work. ## I haven't seen that before. ## TODO: add a flag for this @@ -513,9 +495,7 @@ def check_all(self): self.check_mkcalendar() self.check_event() except: - import pdb - - pdb.set_trace() + debugger() raise finally: if self._default_calendar and not self.flags_checked["no_mkcalendar"]: From 3ef739a2f018667fb15079f198e53484f5055e04 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Tue, 19 Nov 2024 09:12:32 +0100 Subject: [PATCH 20/34] no_todo_on_standard_calendar is out, plus misc other fixes --- check_server_compatibility.py | 151 ++++++++++++++++++++++++---------- tests/compatibility_issues.py | 1 - tests/test_caldav.py | 36 ++------ 3 files changed, 117 insertions(+), 71 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 0a8e960..3a485bd 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -1,7 +1,9 @@ #!/usr/bin/env python -import datetime import time import uuid +from datetime import date +from datetime import datetime +from datetime import timedelta from json import dumps import click @@ -16,12 +18,15 @@ from tests.conf import client from tests.conf import CONNKEYS + def _debugger(): import pdb + ## TODO: check some environmental flags ## only hackers want the debug mode pdb.set_trace() + def _delay_decorator(f, delay=10): def foo(*a, **kwa): time.sleep(delay) @@ -146,7 +151,7 @@ def check_mkcalendar(self): except NotFoundError: self.set_flag("non_existing_calendar_found", False) except Exception as e: - debugger() + _debugger() pass ## Check on "no_default_calendar" flag try: @@ -234,14 +239,14 @@ def check_event(self): ## Two simple events with text fields, dtstart=now and no dtend obj1 = cal.add_event( - dtstart=datetime.datetime.now(), + dtstart=datetime.now(), summary="Test event 1", categories=["foo", "bar"], class_="CONFIDENTIAL", uid="check_event_1", ) obj2 = cal.add_event( - dtstart=datetime.datetime.now(), + dtstart=datetime.now(), summary="Test event 2", categories=["zoo", "test"], uid="check_event_2", @@ -256,23 +261,23 @@ def check_event(self): ## Recurring events try: yearly_time = cal.add_event( - dtstart=datetime.datetime(2000, 1, 1, 0, 0), + dtstart=datetime(2000, 1, 1, 0, 0), summary="Yearly timed event", uid="firework_event", rrule={"FREQ": "YEARLY"}, ) yearly_day = cal.add_event( - dtstart=datetime.date(2000, 5, 1), + dtstart=date(2000, 5, 1), summary="Yearly day event", uid="full_day_event", rrule={"FREQ": "YEARLY"}, ) except: ## should not be here - debugger() + _debugger() raise try: - self._check_recurring_events(yearly_time, yearly_day, span1, span2) + self._check_recurring_events(yearly_time, yearly_day) finally: yearly_time.delete() yearly_day.delete() @@ -280,31 +285,35 @@ def check_event(self): ## Finally, a check that searches involving timespans works as intended try: span1 = cal.add_event( - dtstart=datetime.datetime(2000, 7, 1, 8), - dtend=datetime.datetime(2000, 7, 1, 16), - summary="Yearly 8 hour event", - uid="eight_hour_event", + dtstart=datetime(2000, 7, 1, 8), + dtend=datetime(2000, 7, 1, 16), + summary="An 8 hour event", + uid="eight_hour_event1", ) span2 = cal.add_event( - dtstart=datetime.datetime(2000, 7, 2, 8), - duration=datetime.timedelta(hours=8), - summary="Yearly 8 hour event", - uid="eight_hour_event", - rrule={"FREQ": "YEARLY"}, + dtstart=datetime(2000, 7, 2, 8), + duration=timedelta(hours=8), + summary="Another 8 hour event", + uid="eight_hour_event2", ) except: ## should not be here - debugger() + _debugger() raise - + try: + self._check_spanning_events(span1, span2) + finally: + span1.delete() + span2.delete() + def _check_simple_events(self, obj1, obj2): cal = self._default_calendar try: obj1_ = cal.event_by_url(obj1.url) - assert obj1.data == obj1_.data + assert "SUMMARY:Test event 1" in obj1_.data except: self.set_flag("event_by_url_is_broken") - debugger() + _debugger() try: foo = cal.event_by_uid("check_event_2") @@ -328,7 +337,7 @@ def _check_simple_events(self, obj1, obj2): if len(cal.events()) == 2: self.set_flag("search_always_needs_comptype", True) else: - debugger() + _debugger() pass ## we should not be here else: @@ -361,7 +370,7 @@ def _check_simple_events(self, obj1, obj2): self.set_flag("text_search_is_case_insensitive", False) else: ## we should not be here - debugger() + _debugger() pass events = cal.search(summary="test event", event=True) if len(events) == 2: @@ -375,10 +384,10 @@ def _check_simple_events(self, obj1, obj2): if len(events) == 1: self.set_flag("combined_search_not_working", False) elif len(events) == 0: - debugger() + _debugger() self.set_flag("combined_search_not_working", True) else: - debugger() + _debugger() ## We should not be here pass try: @@ -391,17 +400,15 @@ def _check_simple_events(self, obj1, obj2): self.set_flag("category_search_yields_nothing", True) else: ## we should not be here - debugger() + _debugger() pass - events = cal.search(summary="test event", class_="CONFIDENTIAL", event=True) - def _check_recurring_events(self, yearly_time, yearly_day): cal = self._default_calendar try: events = cal.search( - start=datetime.datetime(2001, 4, 1), - end=datetime.datetime(2002, 2, 2), + start=datetime(2001, 4, 1), + end=datetime(2002, 2, 2), event=True, ) assert len(events) == 2 @@ -413,8 +420,8 @@ def _check_recurring_events(self, yearly_time, yearly_day): return events = cal.search( - start=datetime.datetime(2001, 4, 1), - end=datetime.datetime(2002, 2, 2), + start=datetime(2001, 4, 1), + end=datetime(2002, 2, 2), event=True, expand="server", ) @@ -433,6 +440,64 @@ def _check_recurring_events(self, yearly_time, yearly_day): else: self.set_flag("broken_expand", True) + def _check_spanning_events(self, span1, span2): + cal = self._default_calendar + one_event_lists = [] + try: + ## Any overlapping of an event timespan and search timespan + ## should yield the event, as I remember the RFC + one_event_lists.append(cal.search(event=True, end=datetime(2000, 7, 1, 10))) + one_event_lists.append( + cal.search( + event=True, + start=datetime(2000, 7, 1, 10), + end=datetime(2000, 7, 1, 12), + ) + ) + one_event_lists.append( + cal.search( + event=True, + start=datetime(2000, 7, 1, 4), + end=datetime(2000, 7, 1, 12), + ) + ) + one_event_lists.append( + cal.search( + event=True, + start=datetime(2000, 7, 1, 4), + end=datetime(2000, 7, 1, 22), + ) + ) + one_event_lists.append( + cal.search(event=True, start=datetime(2000, 7, 2, 10)) + ) + one_event_lists.append( + cal.search( + event=True, + start=datetime(2000, 7, 2, 10), + end=datetime(2000, 7, 2, 12), + ) + ) + one_event_lists.append( + cal.search( + event=True, + start=datetime(2000, 7, 2, 4), + end=datetime(2000, 7, 2, 12), + ) + ) + one_event_lists.append( + cal.search( + event=True, + start=datetime(2000, 7, 2, 4), + end=datetime(2000, 7, 2, 22), + ) + ) + for one_event in one_event_lists: + assert len(one_event) == 1 + except: + _debugger() + raise + def check_todo(self): try: ## Add a simplest possible todo @@ -442,28 +507,28 @@ def check_todo(self): ) todo_with_dtstart = cal.add_todo( summary="This is a summary", - dtstart=datetime.datetime(2000,1,1) + dtstart=datetime(2000, 1, 1), uid="check_todo_2", ) todo_with_due = cal.add_todo( summary="This is a summary", - due=datetime.datetime(2000,1,1,1,0,0) + due=datetime(2000, 1, 1, 1, 0, 0), uid="check_todo_3", ) todo_with_dtstart_due = cal.add_todo( summary="This is a summary", - dtstart=datetime.datetime(2000,1,1,2,0,0) - due=datetime.datetime(2000,1,1,3,0,0) + dtstart=datetime(2000, 1, 1, 2, 0, 0), + due=datetime(2000, 1, 1, 3, 0, 0), uid="check_todo_4", ) todo_with_dtstart_dur = cal.add_todo( summary="This is a summary", - dtstart=datetime.datetime(2000,1,1,4,0,0) - duration=datetime.timedelta(hours=1) + dtstart=datetime(2000, 1, 1, 4, 0, 0), + duration=timedelta(hours=1), uid="check_todo_2", ) - if not self.flags['object_by_uid_is_broken']: - assert(len(cal.todo_by_uid('check_todo_1'))==1) + if not self.flags["object_by_uid_is_broken"]: + assert len(cal.todo_by_uid("check_todo_1")) == 1 self.set_flag("no_todo", False) except: self.set_flag("no_todo") @@ -479,9 +544,9 @@ def check_simple_todo(self, todo): ## search for a simple todo try: sr = cal.search(summary="This is a summary", todo=True) - assert(len(sr)==1) + assert len(sr) == 1 except: - debugger() + _debugger() ## simple search for a todo won't work. ## I haven't seen that before. ## TODO: add a flag for this @@ -495,7 +560,7 @@ def check_all(self): self.check_mkcalendar() self.check_event() except: - debugger() + _debugger() raise finally: if self._default_calendar and not self.flags_checked["no_mkcalendar"]: diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index a96adb0..6b467df 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -258,7 +258,6 @@ 'text_search_is_case_insensitive', 'text_search_is_exact_match_sometimes', - 'combined_search_not_working', "search_needs_comptype", ## extra features not specified in RFC5545 diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 4440c9f..43e6541 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -1015,9 +1015,7 @@ def testObjectBySyncToken(self): if not self.check_compatibility_flag("no_recurring"): c.save_event(evr) objcnt += 1 - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): c.save_todo(todo) c.save_todo(todo2) c.save_todo(todo3) @@ -1146,9 +1144,7 @@ def testSync(self): if not self.check_compatibility_flag("no_recurring"): c.save_event(evr) objcnt += 1 - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): c.save_todo(todo) c.save_todo(todo2) c.save_todo(todo3) @@ -2474,14 +2470,10 @@ def testCreateOverwriteDeleteEvent(self): # add event e1 = c.save_event(ev1) - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): t1 = c.save_todo(todo) assert e1.url is not None - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): assert t1.url is not None if not self.check_compatibility_flag("event_by_url_is_broken"): assert c.event_by_url(e1.url).url == e1.url @@ -2495,25 +2487,19 @@ def testCreateOverwriteDeleteEvent(self): ## (but some calendars may throw a "409 Conflict") if not self.check_compatibility_flag("no_overwrite"): e2 = c.save_event(ev1) - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): t2 = c.save_todo(todo) ## add same event with "no_create". Should work like a charm. e2 = c.save_event(ev1, no_create=no_create) - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): t2 = c.save_todo(todo, no_create=no_create) ## this should also work. e2.instance.vevent.summary.value = e2.instance.vevent.summary.value + "!" e2.save(no_create=no_create) - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): t2.instance.vtodo.summary.value = t2.instance.vtodo.summary.value + "!" t2.save(no_create=no_create) @@ -2525,17 +2511,13 @@ def testCreateOverwriteDeleteEvent(self): if not self.check_compatibility_flag("object_by_uid_is_broken"): with pytest.raises(error.ConsistencyError): c.save_event(ev1, no_overwrite=True) - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): with pytest.raises(error.ConsistencyError): c.save_todo(todo, no_overwrite=True) # delete event e1.delete() - if not self.check_compatibility_flag( - "no_todo" - ) and not self.check_compatibility_flag("no_todo_on_standard_calendar"): + if not self.check_compatibility_flag("no_todo"): t1.delete() if self.check_compatibility_flag("non_existing_raises_other"): From c49662346953df7ed21f28c175999a5b702f04b4 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Wed, 20 Nov 2024 20:28:55 +0100 Subject: [PATCH 21/34] wip --- check_server_compatibility.py | 329 ++++++++++++++++++++++------------ tests/compatibility_issues.py | 17 +- 2 files changed, 231 insertions(+), 115 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 3a485bd..e4fce0c 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -40,6 +40,7 @@ def __init__(self, client_obj): self.client_obj = client_obj self.flags_checked = {} self.other_info = {} + self._default_calendar = None def set_flag(self, flag, value=True): if flag == "rate_limited": @@ -141,6 +142,11 @@ def check_principal(self): except AuthorizationError: raise except DAVError: + ## This probably applies to calendar.mail.ru + ## TODO: investigate if there are any quick-fixes + ## TODO: the workaround is to set a calendar path in the config + ## and fix the rest of the check script so that it works even + ## without a self.principal object. self.set_flag("no-current-user-principal", True) def check_mkcalendar(self): @@ -176,16 +182,11 @@ def check_mkcalendar(self): return makeret = self._try_make_calendar(cal_id="pythoncaldav-test") if makeret[0]: - self._default_calendar = self.principal.make_calendar( - cal_id="pythoncaldav-test" - ) self.set_flag("no_displayname", True) return unique_id1 = "testcalendar-" + str(uuid.uuid4()) - unique_id2 = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id1) if makeret[0]: - self._default_calendar = self.principal.make_calendar(cal_id=unique_id2) self.set_flag("unique_calendar_ids", True) unique_id = "testcalendar-" + str(uuid.uuid4()) makeret = self._try_make_calendar(cal_id=unique_id, name="Yep") @@ -193,6 +194,23 @@ def check_mkcalendar(self): self.flags_checked["no_displayname"] = True if not "no_mkcalendar" in self.flags_checked: self.set_flag("no_mkcalendar", True) + self._fix_cal() + + def _fix_cal(): + if self._default_calendar: + self._default_calendar.delete() + if self.flags_checked['unique_calendar_ids']: + cal_id = "testcalendar-" + str(uuid.uuid4()) + else: + cal_id = "pythoncaldav-test" + if self.flags_checked['no_displayname']: + name = None + else: + name = "CalDAV Server Testing" + #cal = self.principal.make_calendar(cal_id=cal_id, name=name) + cal = self.principal.make_calendar(cal_id=cal_id, name=None) + self._default_calendar = cal + return cal def check_support(self): self.set_flag("dav_not_supported", True) @@ -281,39 +299,48 @@ def check_event(self): finally: yearly_time.delete() yearly_day.delete() + + if cal.events(): + ## Zimbra. Probably related to event_by_url_is_broken + self.set_flag('no_delete_event') + cal = self._fix_cal() + else: + self.set_flag('no_delete_event', False) ## Finally, a check that searches involving timespans works as intended - try: - span1 = cal.add_event( - dtstart=datetime(2000, 7, 1, 8), - dtend=datetime(2000, 7, 1, 16), - summary="An 8 hour event", - uid="eight_hour_event1", - ) - span2 = cal.add_event( - dtstart=datetime(2000, 7, 2, 8), - duration=timedelta(hours=8), - summary="Another 8 hour event", - uid="eight_hour_event2", - ) - except: - ## should not be here - _debugger() + span = cal.add_event( + dtstart=datetime(2000, 7, 1, 8), + dtend=datetime(2000, 7, 1, 16), + summary="An 8 hour event", + uid="eight_hour_event1", + ) + + foo = self._date_search(span, assert_found=False, event=True) + if len(foo) != 0: + import pdb; pdb.set_trace() raise - try: - self._check_spanning_events(span1, span2) - finally: - span1.delete() - span2.delete() + + span = cal.add_event( + dtstart=datetime(2000, 7, 1, 8), + duration=timedelta(hours=8), + summary="Another 8 hour event", + uid="eight_hour_event2", + ) + ret = self._date_search(span, assert_found=False, event=True) + if ret == [4, 6, 7]: + self.set_flag('date_search_ignores_duration') + else: + self.set_flag('date_search_ignores_duration', False) + assert len(ret)==0 def _check_simple_events(self, obj1, obj2): cal = self._default_calendar try: obj1_ = cal.event_by_url(obj1.url) assert "SUMMARY:Test event 1" in obj1_.data + self.set_flag("event_by_url_is_broken", False) except: self.set_flag("event_by_url_is_broken") - _debugger() try: foo = cal.event_by_uid("check_event_2") @@ -440,105 +467,135 @@ def _check_recurring_events(self, yearly_time, yearly_day): else: self.set_flag("broken_expand", True) - def _check_spanning_events(self, span1, span2): - cal = self._default_calendar - one_event_lists = [] + def _date_search(self, obj, has_duration=True, **kwargs): try: - ## Any overlapping of an event timespan and search timespan - ## should yield the event, as I remember the RFC - one_event_lists.append(cal.search(event=True, end=datetime(2000, 7, 1, 10))) - one_event_lists.append( - cal.search( - event=True, - start=datetime(2000, 7, 1, 10), - end=datetime(2000, 7, 1, 12), - ) - ) - one_event_lists.append( - cal.search( - event=True, - start=datetime(2000, 7, 1, 4), - end=datetime(2000, 7, 1, 12), - ) - ) - one_event_lists.append( - cal.search( - event=True, - start=datetime(2000, 7, 1, 4), - end=datetime(2000, 7, 1, 22), - ) - ) - one_event_lists.append( - cal.search(event=True, start=datetime(2000, 7, 2, 10)) - ) - one_event_lists.append( - cal.search( - event=True, - start=datetime(2000, 7, 2, 10), - end=datetime(2000, 7, 2, 12), - ) - ) - one_event_lists.append( - cal.search( - event=True, - start=datetime(2000, 7, 2, 4), - end=datetime(2000, 7, 2, 12), - ) - ) - one_event_lists.append( - cal.search( - event=True, - start=datetime(2000, 7, 2, 4), - end=datetime(2000, 7, 2, 22), - ) - ) - for one_event in one_event_lists: - assert len(one_event) == 1 - except: - _debugger() - raise + return self._do_date_search(has_duration=has_duration, **kwargs) + finally: + obj.delete() + + def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): + """ + returns a "disbehavior list". + All those searches should find the event: + 0: open-ended search with end after event + 1: open-ended search with start before event + 2: search interval covers event + 3: open-ended search with end during event + 4: open-ended search with start during event + 5: search with end during event + 6: search with start and end during event + 7: search with start during event + """ + cal = self._default_calendar + longbefore = datetime(2000, 6, 30, 4) + before = datetime(2000, 7, 1, 4) + during1 = datetime(2000, 7, 1, 10) + during2 = datetime(2000, 7, 1, 12) + after = datetime(2000, 7, 1, 22) + longafter = datetime(2000, 7, 2, 10) + one_event_lists = [ + ## open-ended searches, should yield object + cal.search(end=after, **kwargs), + cal.search(start=before, **kwargs), + cal.search(start=before, end=after, **kwargs) + ] + if has_duration: + ## overlapping searches, everything should yield object + one_event_lists.extend([ + cal.search(end=during1, **kwargs), + cal.search(start=during1, **kwargs), + cal.search(start=before, end=during1, **kwargs), + cal.search(start=during1, end=during2, **kwargs), + cal.search(start=during1, end=after, **kwargs) + ]) + ret = [] + for i in range(0, len(one_event_lists)): + if not assert_found and len(one_event_lists[i])==0: + ret.append(i) + else: + assert len(one_event_lists[i])==1 + assert(len(cal.search(start=longbefore, end=before))==0) + if kwargs.get('todo'): + if len(cal.search(end=before, **kwargs))==0: + if not 'vtodo_datesearch_nostart_future_tasks_delivered' in self.flags_checked: + self.set_flag('vtodo_datesearch_nostart_future_tasks_delivered', False) + else: + self.set_flag('vtodo_datesearch_nostart_future_tasks_delivered', True) + assert len(cal.search(end=before, **kwargs))==1 + else: + assert len(cal.search(end=before, **kwargs))==0 + assert(len(cal.search(start=after, end=longafter))==0) + assert(len(cal.search(start=after, **kwargs))==0) + return ret def check_todo(self): + cal = self._default_calendar try: ## Add a simplest possible todo todo_simple = cal.add_todo( summary="This is a summary", uid="check_todo_1", ) - todo_with_dtstart = cal.add_todo( - summary="This is a summary", - dtstart=datetime(2000, 1, 1), - uid="check_todo_2", - ) - todo_with_due = cal.add_todo( - summary="This is a summary", - due=datetime(2000, 1, 1, 1, 0, 0), - uid="check_todo_3", - ) - todo_with_dtstart_due = cal.add_todo( - summary="This is a summary", - dtstart=datetime(2000, 1, 1, 2, 0, 0), - due=datetime(2000, 1, 1, 3, 0, 0), - uid="check_todo_4", - ) - todo_with_dtstart_dur = cal.add_todo( - summary="This is a summary", - dtstart=datetime(2000, 1, 1, 4, 0, 0), - duration=timedelta(hours=1), - uid="check_todo_2", - ) - if not self.flags["object_by_uid_is_broken"]: - assert len(cal.todo_by_uid("check_todo_1")) == 1 + if not self.flags_checked["object_by_uid_is_broken"]: + assert str(cal.todo_by_uid("check_todo_1").icalendar_component['UID']) == 'check_todo_1' self.set_flag("no_todo", False) - except: + except Exception as e: + import pdb; pdb.set_trace() self.set_flag("no_todo") return try: - self.check_simple_todo(todo1) + self._check_simple_todo(todo_simple) finally: - todo1.delete() + todo_simple.delete() + + ## There are more corner cases to consider + ## See RFC 4791, section 9.9 + ## For tasks missing DTSTART and DUE/DURATION, but having + ## CREATED/COMPLETED, those time attributes should be + ## considered. TODO: test that, too. + todo = cal.add_todo( + summary="This has dtstart", + dtstart=datetime(2000, 7, 1, 8), + uid="check_todo_2", + ) + foobar1 = self._date_search(todo, assert_found=False, has_duration=False, todo=True) + + todo = cal.add_todo( + summary="This has due", + due=datetime(2000, 7, 1, 16), + uid="check_todo_3", + ) + foobar2 = self._date_search(todo, assert_found=False, has_duration=False, todo=True) - def check_simple_todo(self, todo): + todo = cal.add_todo( + summary="This has dtstart and due", + dtstart=datetime(2000, 7, 1, 8), + due=datetime(2000, 7, 1, 16), + uid="check_todo_4", + ) + + foobar3 = self._date_search(todo, assert_found=False, todo=True) + + todo = cal.add_todo( + summary="This has dtstart and dur", + dtstart=datetime(2000, 7, 1, 8), + duration=timedelta(hours=1), + uid="check_todo_5", + ) + foobar4 = self._date_search(todo, assert_found=False, todo=True) + + if len(foobar1 + foobar2 + foobar3 + foobar4) == 22: + ## no todos found + self.set_flag('no_todo_datesearch') + else: + self.set_flag('no_todo_datesearch', False) + assert len(foobar1 + foobar2 + foobar3) == 0 + if foobar4 == [4, 6, 7]: + self.set_flag('date_todo_search_ignores_duration') + else: + assert len(foobar4)==0 + + def _check_simple_todo(self, todo): cal = self._default_calendar ## search for a simple todo @@ -551,6 +608,52 @@ def check_simple_todo(self, todo): ## I haven't seen that before. ## TODO: add a flag for this raise + + ## RFC says that a todo without dtstart/due is + ## supposed to span over "infinite time". So itshould always appear + ## in date searches. + try: + todos = cal.search(start=datetime(2020,1,1), end=datetime(2020,1,2), todo=True) + assert len(todos) in (0,1) + self.set_flag("vtodo_datesearch_notime_task_is_skipped", len(todos) == 0) + except Exception as e: + self.set_flag("no_todo_datesearch", True) + + def _check_todo_date_search(self, todo_with_dtstart, todo_with_due, todo_with_dtstart_due, todo_with_dtstart_dur): + if self.flag_checked["no_todo_datesearch"]: + return + ## Every check below should return one and only one task if + ## everything works according to my understanding of the RFC + one_task_lists = [] + for start_end in [ + ## 0 - todo_with_dtstart + ((1999, 12, 31, 22, 22, 22),(2000, 1, 1, 0, 30)), + + ## 1 - todo_with_due + ((2000, 1, 1, 0, 30),(2000, 1, 1, 1, 30)), + + ## 2, 3, 4 - todo_with_dtstart_due + ((2000, 1, 1, 1, 30),(2000, 1, 1, 3, 30)), + ((2000, 1, 1, 1, 30),(2000, 1, 1, 2, 30)), + ((2000, 1, 1, 2, 30),(2000, 1, 1, 3, 30)), + + ## 5, 6, 7 - todo_with_dtstart_dur + ((2000, 1, 1, 3, 30),(2000, 1, 1, 5, 30)), + ((2000, 1, 1, 3, 30),(2000, 1, 1, 4, 30)), + ((2000, 1, 1, 4, 30),(2000, 1, 1, 5, 30)), + ]: + one_task_lists.append(cal.search(start=datetime(*start_end[0]), end=datetime(*start_end[1]))) + + if sum([len(x) for x in one_task_lists]) == 0: + self.set_flag('no_todo_datesearch') + return + else: + self.set_flag('no_todo_datesearch', False) + + for idx in range(0,8): + if not len(one_task_lists[idx])==1: + debugging + pass def check_all(self): try: @@ -559,9 +662,7 @@ def check_all(self): self.check_propfind() self.check_mkcalendar() self.check_event() - except: - _debugger() - raise + self.check_todo() finally: if self._default_calendar and not self.flags_checked["no_mkcalendar"]: try: diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 6b467df..659609e 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -95,6 +95,9 @@ 'event_by_url_is_broken': """A GET towards a valid calendar object resource URL will yield 404 (wtf?)""", + 'no_delete_event': + """Zimbra does not support deleting an event, probably because event_by_url is broken""", + 'no_sync_token': """RFC6578 is not supported, things will break if we try to do a sync-token report""", @@ -137,6 +140,9 @@ """date searches for todo-items will (only) find tasks that has either """ """a dtstart or due set""", + 'vtodo_datesearch_nostart_future_tasks_delivered': + """Future tasks are yielded when doing a date search with some end timestamp and without start timestamp and the task contains both dtstart and due, but not duration (xandikos 0.2.12)""", + 'vtodo_no_due_infinite_duration': """date search will find todo-items without due if dtstart is """ """before the date search interval. I didn't find anything explicit """ @@ -181,6 +187,12 @@ 'text_search_not_working': """Text search is generally broken""", + 'date_search_ignores_duration': + """Date search with search interval overlapping event interval works on events with dtstart and dtend, but not on events with dtstart and due""", + + 'date_todo_search_ignores_duration': + """Same as above, but specifically for tasks""", + 'fastmail_buggy_noexpand_date_search': """The 'blissful anniversary' recurrent example event is returned when asked for a no-expand date search for some timestamps covering a completely different date""", @@ -223,9 +235,10 @@ ## https://github.com/jelmer/xandikos/issues/8 "no_recurring", + 'date_todo_search_ignores_duration', 'text_search_is_exact_match_only', - "search_needs_comptype", + 'vtodo_datesearch_nostart_future_tasks_delivered', ## scheduling is not supported "no_scheduling", @@ -282,6 +295,7 @@ ## anymore) 'duplicate_in_other_calendar_with_same_uid_is_lost', 'event_by_url_is_broken', + 'no_delete_event', 'no_sync_token', 'vtodo_datesearch_notime_task_is_skipped', 'category_search_yields_nothing', @@ -368,6 +382,7 @@ ] nextcloud = [ + 'date_search_ignores_duration', 'sync_breaks_on_delete', 'no_recurring_todo', 'combined_search_not_working', From c112d278a0835b05813bc8b9e4e66170264688d1 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Thu, 21 Nov 2024 23:23:42 +0100 Subject: [PATCH 22/34] wip --- check_server_compatibility.py | 88 +++++++++++++++++++++++------------ tests/compatibility_issues.py | 3 ++ 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index e4fce0c..00d6e9f 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -71,6 +71,22 @@ def _try_make_calendar(self, cal_id, **kwargs): calmade = True self.set_flag("no_mkcalendar", False) self.set_flag("read_only", False) + self.principal.calendar(cal_id=cal_id).events() + import pdb; pdb.set_trace() + if kwargs.get('name'): + try: + name = 'A calendar with this name should not exist' + self.principal.calendar(name=name).events() + except: + ## This is not the exception, this is the normal + try: + cal2 = self.principal.calendar(name=kwargs['name']) + cal2.events() + assert(cal2.id == cal.id) + self.set_flag("no_displayname", False) + except: + self.set_flag("no_displayname", True) + except Exception as e: ## calendar creation created an exception - return exception cal = self.principal.calendar(cal_id=cal_id) @@ -150,6 +166,7 @@ def check_principal(self): self.set_flag("no-current-user-principal", True) def check_mkcalendar(self): + self.set_flag("unique_calendar_ids", False) try: cal = self.principal.calendar(cal_id="this_should_not_exist") cal.events() @@ -162,53 +179,55 @@ def check_mkcalendar(self): ## Check on "no_default_calendar" flag try: cals = self.principal.calendars() - cals[0].events() + events = cals[0].events() + ## We will not do any testing on a calendar that already contains events self.set_flag("no_default_calendar", False) - self._default_calendar = cals[0] except: self.set_flag("no_default_calendar", True) + import pdb; pdb.set_trace() makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: - try: - self._default_calendar = self.principal.make_calendar( - name="Yep", cal_id="pythoncaldav-test" - ) - except: - self._default_calendar = self.principal.calendar( - cal_id="pythoncaldav-test" - ) - self._default_calendar.events() + ## calendar created return makeret = self._try_make_calendar(cal_id="pythoncaldav-test") if makeret[0]: self.set_flag("no_displayname", True) return unique_id1 = "testcalendar-" + str(uuid.uuid4()) - makeret = self._try_make_calendar(cal_id=unique_id1) + makeret = self._try_make_calendar(cal_id=unique_id1, name="Yep") if makeret[0]: self.set_flag("unique_calendar_ids", True) + return unique_id = "testcalendar-" + str(uuid.uuid4()) - makeret = self._try_make_calendar(cal_id=unique_id, name="Yep") - if not makeret[0] and not self.flags_checked.get("no_mkcalendar", True): + makeret = self._try_make_calendar(cal_id=unique_id) + if makeret[0]: self.flags_checked["no_displayname"] = True + return if not "no_mkcalendar" in self.flags_checked: self.set_flag("no_mkcalendar", True) - self._fix_cal() - def _fix_cal(): + def _fix_cal_if_needed(self, todo=False): + if self.flags_checked['no_delete_event']: + return self._fix_cal(todo=todo) + else: + return self._default_calendar + + def _fix_cal(self, todo=False): + kwargs = {} if self._default_calendar: self._default_calendar.delete() + if todo: + kwargs['supported_calendar_component_set'] = ["VTODO"] if self.flags_checked['unique_calendar_ids']: - cal_id = "testcalendar-" + str(uuid.uuid4()) + kwargs['cal_id'] = "testcalendar-" + str(uuid.uuid4()) else: - cal_id = "pythoncaldav-test" + kwargs['cal_id'] = "pythoncaldav-test" if self.flags_checked['no_displayname']: - name = None + kwargs['name'] = None else: - name = "CalDAV Server Testing" - #cal = self.principal.make_calendar(cal_id=cal_id, name=name) - cal = self.principal.make_calendar(cal_id=cal_id, name=None) + kwargs['name'] = "CalDAV Server Testing" + cal = self.principal.make_calendar(**kwargs) self._default_calendar = cal return cal @@ -530,23 +549,33 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): def check_todo(self): cal = self._default_calendar + simple = { + 'summary': "This is a summary", + 'uid': "check_todo_1", + } try: ## Add a simplest possible todo - todo_simple = cal.add_todo( - summary="This is a summary", - uid="check_todo_1", - ) + todo_simple = cal.add_todo(**simple) if not self.flags_checked["object_by_uid_is_broken"]: assert str(cal.todo_by_uid("check_todo_1").icalendar_component['UID']) == 'check_todo_1' self.set_flag("no_todo", False) except Exception as e: - import pdb; pdb.set_trace() - self.set_flag("no_todo") - return + cal = self._fix_cal(todo=True) + try: + ## Add a simplest possible todo + todo_simple = cal.add_todo(**simple) + if not self.flags_checked["object_by_uid_is_broken"]: + assert str(cal.todo_by_uid("check_todo_1").icalendar_component['UID']) == 'check_todo_1' + self.set_flag("no_todo", False) + self.set_flag("no_todo_on_standard_calendar") + except: + self.set_flag("no_todo") + return try: self._check_simple_todo(todo_simple) finally: todo_simple.delete() + self._fix_cal_if_needed() ## There are more corner cases to consider ## See RFC 4791, section 9.9 @@ -661,6 +690,7 @@ def check_all(self): self.check_support() self.check_propfind() self.check_mkcalendar() + self._fix_cal() self.check_event() self.check_todo() finally: diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 659609e..f3d7535 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -127,6 +127,9 @@ 'no_todo': """Support for VTODO (tasks) apparently missing""", + 'no_todo_on_standard_calendar': + """Tasklists can be created, but a normal calendar does not support tasks""", + 'no_todo_datesearch': """Date search on todo items fails""", From 65bb875026f7ecdd806e7a2cb9f36efcf2f9f0cd Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Thu, 21 Nov 2024 23:26:10 +0100 Subject: [PATCH 23/34] style fixup --- check_server_compatibility.py | 186 ++++++++++++++++++++-------------- tests/compatibility_issues.py | 2 +- tests/test_caldav.py | 3 +- 3 files changed, 111 insertions(+), 80 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 00d6e9f..b371c99 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -72,17 +72,19 @@ def _try_make_calendar(self, cal_id, **kwargs): self.set_flag("no_mkcalendar", False) self.set_flag("read_only", False) self.principal.calendar(cal_id=cal_id).events() - import pdb; pdb.set_trace() - if kwargs.get('name'): + import pdb + + pdb.set_trace() + if kwargs.get("name"): try: - name = 'A calendar with this name should not exist' + name = "A calendar with this name should not exist" self.principal.calendar(name=name).events() except: ## This is not the exception, this is the normal try: - cal2 = self.principal.calendar(name=kwargs['name']) + cal2 = self.principal.calendar(name=kwargs["name"]) cal2.events() - assert(cal2.id == cal.id) + assert cal2.id == cal.id self.set_flag("no_displayname", False) except: self.set_flag("no_displayname", True) @@ -185,7 +187,9 @@ def check_mkcalendar(self): except: self.set_flag("no_default_calendar", True) - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: ## calendar created @@ -208,7 +212,7 @@ def check_mkcalendar(self): self.set_flag("no_mkcalendar", True) def _fix_cal_if_needed(self, todo=False): - if self.flags_checked['no_delete_event']: + if self.flags_checked["no_delete_event"]: return self._fix_cal(todo=todo) else: return self._default_calendar @@ -218,15 +222,15 @@ def _fix_cal(self, todo=False): if self._default_calendar: self._default_calendar.delete() if todo: - kwargs['supported_calendar_component_set'] = ["VTODO"] - if self.flags_checked['unique_calendar_ids']: - kwargs['cal_id'] = "testcalendar-" + str(uuid.uuid4()) + kwargs["supported_calendar_component_set"] = ["VTODO"] + if self.flags_checked["unique_calendar_ids"]: + kwargs["cal_id"] = "testcalendar-" + str(uuid.uuid4()) else: - kwargs['cal_id'] = "pythoncaldav-test" - if self.flags_checked['no_displayname']: - kwargs['name'] = None + kwargs["cal_id"] = "pythoncaldav-test" + if self.flags_checked["no_displayname"]: + kwargs["name"] = None else: - kwargs['name'] = "CalDAV Server Testing" + kwargs["name"] = "CalDAV Server Testing" cal = self.principal.make_calendar(**kwargs) self._default_calendar = cal return cal @@ -318,13 +322,13 @@ def check_event(self): finally: yearly_time.delete() yearly_day.delete() - + if cal.events(): ## Zimbra. Probably related to event_by_url_is_broken - self.set_flag('no_delete_event') + self.set_flag("no_delete_event") cal = self._fix_cal() else: - self.set_flag('no_delete_event', False) + self.set_flag("no_delete_event", False) ## Finally, a check that searches involving timespans works as intended span = cal.add_event( @@ -336,7 +340,9 @@ def check_event(self): foo = self._date_search(span, assert_found=False, event=True) if len(foo) != 0: - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() raise span = cal.add_event( @@ -347,10 +353,10 @@ def check_event(self): ) ret = self._date_search(span, assert_found=False, event=True) if ret == [4, 6, 7]: - self.set_flag('date_search_ignores_duration') + self.set_flag("date_search_ignores_duration") else: - self.set_flag('date_search_ignores_duration', False) - assert len(ret)==0 + self.set_flag("date_search_ignores_duration", False) + assert len(ret) == 0 def _check_simple_events(self, obj1, obj2): cal = self._default_calendar @@ -511,53 +517,63 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): during1 = datetime(2000, 7, 1, 10) during2 = datetime(2000, 7, 1, 12) after = datetime(2000, 7, 1, 22) - longafter = datetime(2000, 7, 2, 10) + longafter = datetime(2000, 7, 2, 10) one_event_lists = [ ## open-ended searches, should yield object cal.search(end=after, **kwargs), cal.search(start=before, **kwargs), - cal.search(start=before, end=after, **kwargs) + cal.search(start=before, end=after, **kwargs), ] if has_duration: ## overlapping searches, everything should yield object - one_event_lists.extend([ - cal.search(end=during1, **kwargs), - cal.search(start=during1, **kwargs), - cal.search(start=before, end=during1, **kwargs), - cal.search(start=during1, end=during2, **kwargs), - cal.search(start=during1, end=after, **kwargs) - ]) + one_event_lists.extend( + [ + cal.search(end=during1, **kwargs), + cal.search(start=during1, **kwargs), + cal.search(start=before, end=during1, **kwargs), + cal.search(start=during1, end=during2, **kwargs), + cal.search(start=during1, end=after, **kwargs), + ] + ) ret = [] for i in range(0, len(one_event_lists)): - if not assert_found and len(one_event_lists[i])==0: + if not assert_found and len(one_event_lists[i]) == 0: ret.append(i) else: - assert len(one_event_lists[i])==1 - assert(len(cal.search(start=longbefore, end=before))==0) - if kwargs.get('todo'): - if len(cal.search(end=before, **kwargs))==0: - if not 'vtodo_datesearch_nostart_future_tasks_delivered' in self.flags_checked: - self.set_flag('vtodo_datesearch_nostart_future_tasks_delivered', False) + assert len(one_event_lists[i]) == 1 + assert len(cal.search(start=longbefore, end=before)) == 0 + if kwargs.get("todo"): + if len(cal.search(end=before, **kwargs)) == 0: + if ( + not "vtodo_datesearch_nostart_future_tasks_delivered" + in self.flags_checked + ): + self.set_flag( + "vtodo_datesearch_nostart_future_tasks_delivered", False + ) else: - self.set_flag('vtodo_datesearch_nostart_future_tasks_delivered', True) - assert len(cal.search(end=before, **kwargs))==1 + self.set_flag("vtodo_datesearch_nostart_future_tasks_delivered", True) + assert len(cal.search(end=before, **kwargs)) == 1 else: - assert len(cal.search(end=before, **kwargs))==0 - assert(len(cal.search(start=after, end=longafter))==0) - assert(len(cal.search(start=after, **kwargs))==0) + assert len(cal.search(end=before, **kwargs)) == 0 + assert len(cal.search(start=after, end=longafter)) == 0 + assert len(cal.search(start=after, **kwargs)) == 0 return ret def check_todo(self): cal = self._default_calendar simple = { - 'summary': "This is a summary", - 'uid': "check_todo_1", + "summary": "This is a summary", + "uid": "check_todo_1", } try: ## Add a simplest possible todo todo_simple = cal.add_todo(**simple) if not self.flags_checked["object_by_uid_is_broken"]: - assert str(cal.todo_by_uid("check_todo_1").icalendar_component['UID']) == 'check_todo_1' + assert ( + str(cal.todo_by_uid("check_todo_1").icalendar_component["UID"]) + == "check_todo_1" + ) self.set_flag("no_todo", False) except Exception as e: cal = self._fix_cal(todo=True) @@ -565,7 +581,10 @@ def check_todo(self): ## Add a simplest possible todo todo_simple = cal.add_todo(**simple) if not self.flags_checked["object_by_uid_is_broken"]: - assert str(cal.todo_by_uid("check_todo_1").icalendar_component['UID']) == 'check_todo_1' + assert ( + str(cal.todo_by_uid("check_todo_1").icalendar_component["UID"]) + == "check_todo_1" + ) self.set_flag("no_todo", False) self.set_flag("no_todo_on_standard_calendar") except: @@ -587,14 +606,18 @@ def check_todo(self): dtstart=datetime(2000, 7, 1, 8), uid="check_todo_2", ) - foobar1 = self._date_search(todo, assert_found=False, has_duration=False, todo=True) + foobar1 = self._date_search( + todo, assert_found=False, has_duration=False, todo=True + ) todo = cal.add_todo( summary="This has due", due=datetime(2000, 7, 1, 16), uid="check_todo_3", ) - foobar2 = self._date_search(todo, assert_found=False, has_duration=False, todo=True) + foobar2 = self._date_search( + todo, assert_found=False, has_duration=False, todo=True + ) todo = cal.add_todo( summary="This has dtstart and due", @@ -602,9 +625,9 @@ def check_todo(self): due=datetime(2000, 7, 1, 16), uid="check_todo_4", ) - + foobar3 = self._date_search(todo, assert_found=False, todo=True) - + todo = cal.add_todo( summary="This has dtstart and dur", dtstart=datetime(2000, 7, 1, 8), @@ -615,14 +638,14 @@ def check_todo(self): if len(foobar1 + foobar2 + foobar3 + foobar4) == 22: ## no todos found - self.set_flag('no_todo_datesearch') + self.set_flag("no_todo_datesearch") else: - self.set_flag('no_todo_datesearch', False) + self.set_flag("no_todo_datesearch", False) assert len(foobar1 + foobar2 + foobar3) == 0 if foobar4 == [4, 6, 7]: - self.set_flag('date_todo_search_ignores_duration') + self.set_flag("date_todo_search_ignores_duration") else: - assert len(foobar4)==0 + assert len(foobar4) == 0 def _check_simple_todo(self, todo): cal = self._default_calendar @@ -637,50 +660,57 @@ def _check_simple_todo(self, todo): ## I haven't seen that before. ## TODO: add a flag for this raise - + ## RFC says that a todo without dtstart/due is ## supposed to span over "infinite time". So itshould always appear ## in date searches. try: - todos = cal.search(start=datetime(2020,1,1), end=datetime(2020,1,2), todo=True) - assert len(todos) in (0,1) + todos = cal.search( + start=datetime(2020, 1, 1), end=datetime(2020, 1, 2), todo=True + ) + assert len(todos) in (0, 1) self.set_flag("vtodo_datesearch_notime_task_is_skipped", len(todos) == 0) except Exception as e: self.set_flag("no_todo_datesearch", True) - def _check_todo_date_search(self, todo_with_dtstart, todo_with_due, todo_with_dtstart_due, todo_with_dtstart_dur): + def _check_todo_date_search( + self, + todo_with_dtstart, + todo_with_due, + todo_with_dtstart_due, + todo_with_dtstart_dur, + ): if self.flag_checked["no_todo_datesearch"]: return ## Every check below should return one and only one task if ## everything works according to my understanding of the RFC one_task_lists = [] for start_end in [ - ## 0 - todo_with_dtstart - ((1999, 12, 31, 22, 22, 22),(2000, 1, 1, 0, 30)), - - ## 1 - todo_with_due - ((2000, 1, 1, 0, 30),(2000, 1, 1, 1, 30)), - - ## 2, 3, 4 - todo_with_dtstart_due - ((2000, 1, 1, 1, 30),(2000, 1, 1, 3, 30)), - ((2000, 1, 1, 1, 30),(2000, 1, 1, 2, 30)), - ((2000, 1, 1, 2, 30),(2000, 1, 1, 3, 30)), - - ## 5, 6, 7 - todo_with_dtstart_dur - ((2000, 1, 1, 3, 30),(2000, 1, 1, 5, 30)), - ((2000, 1, 1, 3, 30),(2000, 1, 1, 4, 30)), - ((2000, 1, 1, 4, 30),(2000, 1, 1, 5, 30)), + ## 0 - todo_with_dtstart + ((1999, 12, 31, 22, 22, 22), (2000, 1, 1, 0, 30)), + ## 1 - todo_with_due + ((2000, 1, 1, 0, 30), (2000, 1, 1, 1, 30)), + ## 2, 3, 4 - todo_with_dtstart_due + ((2000, 1, 1, 1, 30), (2000, 1, 1, 3, 30)), + ((2000, 1, 1, 1, 30), (2000, 1, 1, 2, 30)), + ((2000, 1, 1, 2, 30), (2000, 1, 1, 3, 30)), + ## 5, 6, 7 - todo_with_dtstart_dur + ((2000, 1, 1, 3, 30), (2000, 1, 1, 5, 30)), + ((2000, 1, 1, 3, 30), (2000, 1, 1, 4, 30)), + ((2000, 1, 1, 4, 30), (2000, 1, 1, 5, 30)), ]: - one_task_lists.append(cal.search(start=datetime(*start_end[0]), end=datetime(*start_end[1]))) + one_task_lists.append( + cal.search(start=datetime(*start_end[0]), end=datetime(*start_end[1])) + ) if sum([len(x) for x in one_task_lists]) == 0: - self.set_flag('no_todo_datesearch') + self.set_flag("no_todo_datesearch") return else: - self.set_flag('no_todo_datesearch', False) + self.set_flag("no_todo_datesearch", False) - for idx in range(0,8): - if not len(one_task_lists[idx])==1: + for idx in range(0, 8): + if not len(one_task_lists[idx]) == 1: debugging pass diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index f3d7535..8587258 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -192,7 +192,7 @@ 'date_search_ignores_duration': """Date search with search interval overlapping event interval works on events with dtstart and dtend, but not on events with dtstart and due""", - + 'date_todo_search_ignores_duration': """Same as above, but specifically for tasks""", diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 43e6541..39f129b 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -560,7 +560,8 @@ def foo(*a, **kwa): return f(*a, **kwa) return foo - + + class RepeatedFunctionalTestsBaseClass: """This is a class with functional tests (tests that goes through basic functionality and actively communicates with third parties) From 401e88e13079d7093ddf022296d93c573e517a30 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 30 Nov 2024 20:26:50 +0100 Subject: [PATCH 24/34] work in progress --- check_server_compatibility.py | 149 +++++++++++++++++++--------------- tests/compatibility_issues.py | 25 ++++-- 2 files changed, 102 insertions(+), 72 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index b371c99..d1c34a5 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -72,9 +72,6 @@ def _try_make_calendar(self, cal_id, **kwargs): self.set_flag("no_mkcalendar", False) self.set_flag("read_only", False) self.principal.calendar(cal_id=cal_id).events() - import pdb - - pdb.set_trace() if kwargs.get("name"): try: name = "A calendar with this name should not exist" @@ -147,6 +144,7 @@ def _try_make_calendar(self, cal_id, **kwargs): self.set_flag("rate_limited", True) except Exception as e2: pass + return (calmade, None) def check_principal(self): ## TODO @@ -187,9 +185,6 @@ def check_mkcalendar(self): except: self.set_flag("no_default_calendar", True) - import pdb - - pdb.set_trace() makeret = self._try_make_calendar(name="Yep", cal_id="pythoncaldav-test") if makeret[0]: ## calendar created @@ -219,9 +214,18 @@ def _fix_cal_if_needed(self, todo=False): def _fix_cal(self, todo=False): kwargs = {} - if self._default_calendar: - self._default_calendar.delete() - if todo: + try: + if self._default_calendar: + self._default_calendar.delete() + except: + pass + if self.flags_checked['no_mkcalendar']: + cal = self.principal.calendars()[0] + if cal.events() or cal.todos(): + raise "Refusing to run tests on a calendar with content" + self._default_calendar = cal + return cal + if todo and self.flags_checked.get('no_todo_on_standard_calendar'): kwargs["supported_calendar_component_set"] = ["VTODO"] if self.flags_checked["unique_calendar_ids"]: kwargs["cal_id"] = "testcalendar-" + str(uuid.uuid4()) @@ -318,6 +322,7 @@ def check_event(self): _debugger() raise try: + self._check_freebusy() self._check_recurring_events(yearly_time, yearly_day) finally: yearly_time.delete() @@ -358,6 +363,22 @@ def check_event(self): self.set_flag("date_search_ignores_duration", False) assert len(ret) == 0 + def _check_freebusy(self): + cal = self._default_calendar + ## TODO: + ## Surely we should do more tests on freebusy, how does it work wrg + ## of tasks, recurring events, etc, etc. + try: + freebusy = cal.freebusy_request( + datetime(1999, 12, 30, 17, 0, 0), datetime(2000, 1, 1, 12, 30, 0) + ) + # TODO: assert something more complex on the return object + assert isinstance(freebusy, FreeBusy) + assert freebusy.instance.vfreebusy + self.set_flag('no_freebusy_rfc4791', False) + except: + self.set_flag('no_freebusy_rfc4791') + def _check_simple_events(self, obj1, obj2): cal = self._default_calendar try: @@ -520,19 +541,19 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): longafter = datetime(2000, 7, 2, 10) one_event_lists = [ ## open-ended searches, should yield object - cal.search(end=after, **kwargs), - cal.search(start=before, **kwargs), - cal.search(start=before, end=after, **kwargs), + cal.search(end=after, **kwargs), ## 0 + cal.search(start=before, **kwargs), ## 1 + cal.search(start=before, end=after, **kwargs), ## 2 ] if has_duration: ## overlapping searches, everything should yield object one_event_lists.extend( [ - cal.search(end=during1, **kwargs), - cal.search(start=during1, **kwargs), - cal.search(start=before, end=during1, **kwargs), - cal.search(start=during1, end=during2, **kwargs), - cal.search(start=during1, end=after, **kwargs), + cal.search(end=during1, **kwargs), ## 3 + cal.search(start=during1, **kwargs), ## 4 + cal.search(start=before, end=during1, **kwargs), ## 5 + cal.search(start=during1, end=during2, **kwargs), ## 6 + cal.search(start=during1, end=after, **kwargs), ## 7 ] ) ret = [] @@ -541,7 +562,13 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): ret.append(i) else: assert len(one_event_lists[i]) == 1 - assert len(cal.search(start=longbefore, end=before)) == 0 + should_be_empty = cal.search(start=longbefore, end=before) + if should_be_empty: + assert len(should_be_empty) == 1 + ical = should_be_empty[0].icalendar_component + assert('due' in ical and not 'dtstart' in ical) + self.set_flag('vtodo_no_dtstart_infinite_duration') + if kwargs.get("todo"): if len(cal.search(end=before, **kwargs)) == 0: if ( @@ -576,6 +603,7 @@ def check_todo(self): ) self.set_flag("no_todo", False) except Exception as e: + self.set_flag("no_todo_on_standard_calendar") cal = self._fix_cal(todo=True) try: ## Add a simplest possible todo @@ -586,8 +614,8 @@ def check_todo(self): == "check_todo_1" ) self.set_flag("no_todo", False) - self.set_flag("no_todo_on_standard_calendar") except: + self.set_flag("no_todo_on_standard_calendar", False) self.set_flag("no_todo") return try: @@ -619,6 +647,9 @@ def check_todo(self): todo, assert_found=False, has_duration=False, todo=True ) + if not 'vtodo_no_dtstart_infinite_duration' in self.flags_checked: + self.set_flag('vtodo_no_dtstart_infinite_duration', False) + todo = cal.add_todo( summary="This has dtstart and due", dtstart=datetime(2000, 7, 1, 8), @@ -639,13 +670,38 @@ def check_todo(self): if len(foobar1 + foobar2 + foobar3 + foobar4) == 22: ## no todos found self.set_flag("no_todo_datesearch") + assert self.flags_checked.pop("vtodo_datesearch_notime_task_is_skipped") ## redundant + return + + self.set_flag("no_todo_datesearch", False) + if foobar1 == [1,2]: + ## dtstart, but no due. + ## open-ended search with end after event: found + ## open-ended search with start before event: not found + ## search with interval covering due: not found + ## Weird! + self.set_flag("no_dtstart_search_weirdness") else: - self.set_flag("no_todo_datesearch", False) - assert len(foobar1 + foobar2 + foobar3) == 0 - if foobar4 == [4, 6, 7]: - self.set_flag("date_todo_search_ignores_duration") - else: - assert len(foobar4) == 0 + assert not foobar1 + self.set_flag("vtodo_no_dtstart_search_weirdness", False) + + if len(foobar2) == 3: + self.set_flag("vtodo_datesearch_nodtstart_task_is_skipped") + else: + self.set_flag("vtodo_datesearch_nodtstart_task_is_skipped", False) + assert not foobar2 + + assert not foobar3 + + self.set_flag("vtodo_no_duration_search_weirdness", False) + if foobar4 == [4, 6, 7]: + self.set_flag("date_todo_search_ignores_duration") + elif foobar4 == [1, 2, 4, 5, 6, 7]: + ## Zimbra is weird! + self.set_flag("vtodo_no_dtstart_search_weirdness") + else: + self.set_flag("date_todo_search_ignores_duration", False) + assert len(foobar4) == 0 def _check_simple_todo(self, todo): cal = self._default_calendar @@ -673,47 +729,6 @@ def _check_simple_todo(self, todo): except Exception as e: self.set_flag("no_todo_datesearch", True) - def _check_todo_date_search( - self, - todo_with_dtstart, - todo_with_due, - todo_with_dtstart_due, - todo_with_dtstart_dur, - ): - if self.flag_checked["no_todo_datesearch"]: - return - ## Every check below should return one and only one task if - ## everything works according to my understanding of the RFC - one_task_lists = [] - for start_end in [ - ## 0 - todo_with_dtstart - ((1999, 12, 31, 22, 22, 22), (2000, 1, 1, 0, 30)), - ## 1 - todo_with_due - ((2000, 1, 1, 0, 30), (2000, 1, 1, 1, 30)), - ## 2, 3, 4 - todo_with_dtstart_due - ((2000, 1, 1, 1, 30), (2000, 1, 1, 3, 30)), - ((2000, 1, 1, 1, 30), (2000, 1, 1, 2, 30)), - ((2000, 1, 1, 2, 30), (2000, 1, 1, 3, 30)), - ## 5, 6, 7 - todo_with_dtstart_dur - ((2000, 1, 1, 3, 30), (2000, 1, 1, 5, 30)), - ((2000, 1, 1, 3, 30), (2000, 1, 1, 4, 30)), - ((2000, 1, 1, 4, 30), (2000, 1, 1, 5, 30)), - ]: - one_task_lists.append( - cal.search(start=datetime(*start_end[0]), end=datetime(*start_end[1])) - ) - - if sum([len(x) for x in one_task_lists]) == 0: - self.set_flag("no_todo_datesearch") - return - else: - self.set_flag("no_todo_datesearch", False) - - for idx in range(0, 8): - if not len(one_task_lists[idx]) == 1: - debugging - pass - def check_all(self): try: self.check_principal() @@ -745,6 +760,8 @@ def report(self, verbose, json): click.echo( dumps( { + "caldav_version": caldav.__version__, + "ts": time.time(), "name": self.client_obj.server_name, "url": str(self.client_obj.url), "flags_checked": self.flags_checked, diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 8587258..f7b3c33 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -148,10 +148,19 @@ 'vtodo_no_due_infinite_duration': """date search will find todo-items without due if dtstart is """ - """before the date search interval. I didn't find anything explicit """ - """in The RFC on this (), but an event should be considered to have 0 """ - """duration if no dtend is set, and most server implementations seems to """ - """treat VTODOs the same""", + """before the date search interval. This is in breach of rfc4791""" + """section 9.9""", + + 'vtodo_no_dtstart_infinite_duration': + """date search will find todo-items without dtstart if due is """ + """after the date search interval. This is in breach of rfc4791""" + """section 9.9""", + + 'vtodo_no_dtstart_search_weirdness': + """Zimbra is weird""", + + 'vtodo_with_due_weirdness': + """Zimbra is weird""", 'unique_calendar_ids': """For every test, generate a new and unique calendar id""", @@ -245,6 +254,10 @@ ## scheduling is not supported "no_scheduling", + + ## The test in the tests itself passes, but the test in the + ## check_server_compatibility triggers a 500-error + "no_freebusy_rfc4791", ] ## This can soon be removed (relevant for running tests under python 3.7 and python 3.8) @@ -270,10 +283,10 @@ "no_freebusy_rfc4791", 'no_scheduling', - 'no_todo_datesearch', + "no_todo_datesearch", 'text_search_is_case_insensitive', - 'text_search_is_exact_match_sometimes', + #'text_search_is_exact_match_sometimes', "search_needs_comptype", ## extra features not specified in RFC5545 From 347ff5a98032c2aa34c5a8756ccafce74de2b793 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 30 Nov 2024 22:07:40 +0100 Subject: [PATCH 25/34] bugfix --- check_server_compatibility.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index d1c34a5..16a1e5c 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -14,6 +14,7 @@ from caldav.lib.error import DAVError from caldav.lib.error import NotFoundError from caldav.lib.python_utilities import to_local +from caldav.objects import FreeBusy from tests.compatibility_issues import incompatibility_description from tests.conf import client from tests.conf import CONNKEYS @@ -345,9 +346,6 @@ def check_event(self): foo = self._date_search(span, assert_found=False, event=True) if len(foo) != 0: - import pdb - - pdb.set_trace() raise span = cal.add_event( @@ -376,7 +374,7 @@ def _check_freebusy(self): assert isinstance(freebusy, FreeBusy) assert freebusy.instance.vfreebusy self.set_flag('no_freebusy_rfc4791', False) - except: + except Exception as e: self.set_flag('no_freebusy_rfc4791') def _check_simple_events(self, obj1, obj2): From 2f481d623dc639feabfaa56c80375fc28df7321b Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 30 Nov 2024 22:20:58 +0100 Subject: [PATCH 26/34] bugfix - exact match vs case sensitive --- check_server_compatibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 16a1e5c..082c036 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -443,7 +443,7 @@ def _check_simple_events(self, obj1, obj2): ## we should not be here _debugger() pass - events = cal.search(summary="test event", event=True) + events = cal.search(summary="Test event", event=True) if len(events) == 2: self.set_flag("text_search_is_exact_match_only", False) elif len(events) == 0: From 2a44b124d574ffdea1e71d940c978346e4b74ece Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sun, 1 Dec 2024 13:19:00 +0100 Subject: [PATCH 27/34] requires a newer caldav version (TODO: should use pyproject.toml) --- check_server_compatibility.py | 113 +++++++++++++++++++++++++++++++++- tests/compatibility_issues.py | 9 ++- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 082c036..c17d8ec 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -9,7 +9,7 @@ import click import caldav -from caldav.elements import dav +from caldav.elements import dav, ical from caldav.lib.error import AuthorizationError from caldav.lib.error import DAVError from caldav.lib.error import NotFoundError @@ -19,6 +19,62 @@ from tests.conf import client from tests.conf import CONNKEYS +ical_with_exception1="""BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +BEGIN:VEVENT +UID:c26921f4-0653-11ef-b756-58ce2a14e2e5 +DTSTART:20240411T123000Z +DTEND:20240412T123000Z +DTSTAMP:20240429T181103Z +LAST-MODIFIED:20240429T181103Z +RRULE:FREQ=WEEKLY;INTERVAL=2 +SEQUENCE:1 +SUMMARY:Test +X-MOZ-GENERATION:1 +END:VEVENT +BEGIN:VEVENT +UID:c26921f4-0653-11ef-b756-58ce2a14e2e5 +RECURRENCE-ID:20240425T123000Z +DTSTART:20240425T123000Z +DTEND:20240426T123000Z +CREATED:20240429T181031Z +DTSTAMP:20240429T181103Z +LAST-MODIFIED:20240429T181103Z +SEQUENCE:1 +SUMMARY:Test (edited) +X-MOZ-GENERATION:1 +END:VEVENT +END:VCALENDAR""" + +ical_with_exception2="""BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +BEGIN:VEVENT +UID:c26921f4-0653-11ef-b756-58ce2a14e2e5 +DTSTART;VALUE=DATE:20240411 +DTEND;VALUE=DATE:20240412 +DTSTAMP:20240429T181103Z +LAST-MODIFIED:20240429T181103Z +RRULE:FREQ=WEEKLY;INTERVAL=2 +SEQUENCE:1 +SUMMARY:Test +X-MOZ-GENERATION:1 +END:VEVENT +BEGIN:VEVENT +UID:c26921f4-0653-11ef-b756-58ce2a14e2e5 +RECURRENCE-ID;VALUE=DATE:20240425 +DTSTART;VALUE=DATE:20240425 +DTEND;VALUE=DATE:20240426 +CREATED:20240429T181031Z +DTSTAMP:20240429T181103Z +LAST-MODIFIED:20240429T181103Z +SEQUENCE:1 +SUMMARY:Test (edited) +X-MOZ-GENERATION:1 +END:VEVENT +END:VCALENDAR""" + def _debugger(): import pdb @@ -280,6 +336,30 @@ def check_propfind(self): except: self.set_flag("propfind_allprop_failure", True) + def check_calendar_color_and_order(self): + """ + Calendar color and order is not a part of the CalDAV standard, but + many calendar servers supports it + """ + try: + self._check_prop(ical.CalendarColor, 'goldenred', 'blue') + self.set_flag('calendar_color', True) + except Exception as e: + self.set_flag('calendar_color', False) + try: + self._check_prop(ical.CalendarOrder, '-143', '8') + self.set_flag('calendar_order', True) + except: + self.set_flag('calendar_order', False) + + def _check_prop(self, propclass, silly_value, test_value): + cal = self._default_calendar + props = cal.get_properties([(propclass())]) + assert props[propclass.tag] != silly_value + cal.set_properties(propclass(test_value)) + props = cal.get_properties([(propclass())]) + assert props[propclass.tag] == test_value + def check_event(self): cal = self._default_calendar @@ -361,6 +441,34 @@ def check_event(self): self.set_flag("date_search_ignores_duration", False) assert len(ret) == 0 + def check_exception(self): + if self.flags_checked.get('broken_expand'): + return + self._check_exception(ical_with_exception1) + self._check_exception(ical_with_exception2) + + def _check_exception(self, ical): + cal = self._default_calendar + obj = cal.add_event(ical) + try: + results = cal.search( + start=datetime(2024, 3, 31, 0, 0), + end=datetime(2024, 5, 4, 0, 0, 0), + event=True, + expand="server", + ) + assert len(results) == 2 + for r in results: + assert "RRULE" not in r.data + recurrence_id = r.icalendar_component["RECURRENCE-ID"] + assert isinstance(recurrence_id, icalendar.vDDDTypes) + if not "broken_expand_on_exceptions" in self.flags_checked: + self.set_flag("broken_expand_on_exceptions", False) + except Exception as e: + self.set_flag("broken_expand_on_exceptions") + finally: + obj.delete() + def _check_freebusy(self): cal = self._default_calendar ## TODO: @@ -734,7 +842,9 @@ def check_all(self): self.check_propfind() self.check_mkcalendar() self._fix_cal() + self.check_calendar_color_and_order() self.check_event() + self.check_exception() self.check_todo() finally: if self._default_calendar and not self.flags_checked["no_mkcalendar"]: @@ -824,6 +934,7 @@ def _set_conn_options(func): @click.option("--verbose/--quiet", default=None, help="More output") @click.option("--json/--text", help="JSON output. Overrides verbose") def check_server_compatibility(verbose, json, **kwargs): + click.echo("WARNING: this script is not production-ready") conn = client(**kwargs) obj = ServerQuirkChecker(conn) obj.check_all() diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index f7b3c33..3e21fd0 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -258,6 +258,13 @@ ## The test in the tests itself passes, but the test in the ## check_server_compatibility triggers a 500-error "no_freebusy_rfc4791", + + ## The test with an rrule and an overridden event passes as + ## long as it's with timestamps. With dates, xandikos gets + ## into troubles. I've chosen to edit the test to use timestamp + ## rather than date, just to have the test exercised ... but we + ## should report this upstream + #'broken_expand_on_exceptions', ] ## This can soon be removed (relevant for running tests under python 3.7 and python 3.8) @@ -280,7 +287,7 @@ "no_default_calendar", ## freebusy is not supported yet, but on the long-term road map - "no_freebusy_rfc4791", + #"no_freebusy_rfc4791", 'no_scheduling', "no_todo_datesearch", From 5f6dc38856232d2c98c089a677a64ae2c4e32f02 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Thu, 5 Dec 2024 08:28:15 +0100 Subject: [PATCH 28/34] wip --- check_server_compatibility.py | 21 +++++++++++++++++---- tests/compatibility_issues.py | 9 ++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index c17d8ec..7d6e6d5 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -85,6 +85,10 @@ def _debugger(): def _delay_decorator(f, delay=10): + """ + Sometimes we need to pause between each request, i.e. due to servers + that queues up work, rate-limits requests, etc. + """ def foo(*a, **kwa): time.sleep(delay) return f(*a, **kwa) @@ -93,6 +97,12 @@ def foo(*a, **kwa): class ServerQuirkChecker: + """ + This class will ... + * Keep the connection details to the server + * Keep the state of what's already checked + * + """ def __init__(self, client_obj): self.client_obj = client_obj self.flags_checked = {} @@ -557,13 +567,16 @@ def _check_simple_events(self, obj1, obj2): elif len(events) == 0: self.set_flag("text_search_is_exact_match_only", "maybe") ## may also be text_search_is_exact_match_sometimes - events = cal.search( + events1 = cal.search( summary="Test event 1", class_="CONFIDENTIAL", event=True ) - if len(events) == 1: + ## I don't expect this program to be in use by 2055. + events2 = cal.search( + start=datetime(2000,1,1), end=datetime(2055,1,1), class_="CONFIDENTIAL", event=True + ) + if len(events1) == 1 and len(events2) == 1: self.set_flag("combined_search_not_working", False) - elif len(events) == 0: - _debugger() + elif len(events1 + events2) in (0,1,3): self.set_flag("combined_search_not_working", True) else: _debugger() diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 3e21fd0..16a4687 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -380,7 +380,11 @@ #'nofreebusy', ## for old versions 'fragile_sync_tokens', ## no issue raised yet 'vtodo_datesearch_nodtstart_task_is_skipped', ## no issue raised yet - 'broken_expand_on_exceptions', + 'broken_expand_on_exceptions', ## no issue raised yet + 'date_todo_search_ignores_duration' + 'calendar_color', + 'calendar_order', + 'vtodo_datesearch_notime_task_is_skipped' ] google = [ @@ -410,6 +414,9 @@ 'no_recurring_todo', 'combined_search_not_working', 'text_search_is_exact_match_sometimes', + 'search_needs_comptype', + 'calendar_color', + 'calendar_order' ] fastmail = [ From 5705632987b650f1ecaeeccee99d8376b48c4858 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 6 Dec 2024 17:28:02 +0100 Subject: [PATCH 29/34] throw user into python debugger only if an environmental variable is set --- check_server_compatibility.py | 9 ++++----- tests/compatibility_issues.py | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 7d6e6d5..e5f3fd8 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import time import uuid +import os from datetime import date from datetime import datetime from datetime import timedelta @@ -77,11 +78,9 @@ def _debugger(): - import pdb - - ## TODO: check some environmental flags - ## only hackers want the debug mode - pdb.set_trace() + if os.environ.get('PYTHON_CALDAV_DEBUGMODE') == "DEBUG_PDB": + import pdb + pdb.set_trace() def _delay_decorator(f, delay=10): diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index 16a4687..ccd5051 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -411,12 +411,13 @@ nextcloud = [ 'date_search_ignores_duration', 'sync_breaks_on_delete', - 'no_recurring_todo', 'combined_search_not_working', 'text_search_is_exact_match_sometimes', 'search_needs_comptype', 'calendar_color', - 'calendar_order' + 'calendar_order', + 'date_todo_search_ignores_duration', + 'broken_expand_on_exceptions' ] fastmail = [ From fa37f97ee8f982a24077f51a5b7b02fc57f3a1f3 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 6 Dec 2024 18:01:23 +0100 Subject: [PATCH 30/34] don't break if the calendar server denies telling details about events --- caldav/objects.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/caldav/objects.py b/caldav/objects.py index fcdfed7..c120ecc 100644 --- a/caldav/objects.py +++ b/caldav/objects.py @@ -1253,7 +1253,11 @@ def search( for o in objects: ## This would not be needed if the servers would follow the standard ... - o.load(only_if_unloaded=True) + try: + o.load(only_if_unloaded=True) + except: + logging.critical("Server does not want to reveal details about the calendar object", exc_info=True) + pass ## Google sometimes returns empty objects objects = [o for o in objects if o.has_component()] From d8970d265a0e5c3479e258fe95b7f80ca2d43026 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 6 Dec 2024 23:08:35 +0100 Subject: [PATCH 31/34] if we cannot load an object using GET, try using REPORT and multiget --- caldav/objects.py | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/caldav/objects.py b/caldav/objects.py index c120ecc..25d5ea5 100644 --- a/caldav/objects.py +++ b/caldav/objects.py @@ -241,6 +241,7 @@ def _query( expected_return_value is not None and ret.status != expected_return_value ) or ret.status >= 400: ## COMPATIBILITY HACK - see https://github.com/python-caldav/caldav/issues/309 + ## TODO: server quirks! body = to_wire(body) if ( ret.status == 500 @@ -946,7 +947,11 @@ def save(self): self._create(id=self.id, name=self.name, **self.extra_init_options) return self - def calendar_multiget(self, event_urls: Iterable[URL]) -> List["Event"]: + ## TODO: this is missing test code. + ## TODO: needs refactoring: + ## Objects found may be Todo and Journal, not only Event. + ## Replace the last lines with _request_report_build_resultlist method + def calendar_multiget(self, event_urls: Iterable[URL]) -> List[_CC]: """ get multiple events' data @author mtorange@gmail.com @@ -1078,15 +1083,15 @@ def date_search( return objects + ## TODO: this logic has been partly duplicated in calendar_multiget, but + ## the code there is much more readable and condensed than this. + ## Can code below be refactored? def _request_report_build_resultlist( self, xml, comp_class=None, props=None, no_calendardata=False ): """ Takes some input XML, does a report query on a calendar object and returns the resource objects found. - - TODO: similar code is duplicated many places, we ought to do even more code - refactoring """ matches = [] if props is None: @@ -1122,7 +1127,6 @@ def _request_report_build_resultlist( props=pdata, ) ) - return (response, matches) def search( @@ -1163,7 +1167,7 @@ def search( unless the next parameter is set ... * include_completed - include completed tasks * event - sets comp_class to event - * text attribute search parameters: category, uid, summary, omment, + * text attribute search parameters: category, uid, summary, comment, description, location, status * no-category, no-summary, etc ... search for objects that does not have those attributes. TODO: WRITE TEST CODE! @@ -1251,13 +1255,16 @@ def search( ) raise + obj2 = [] for o in objects: ## This would not be needed if the servers would follow the standard ... try: o.load(only_if_unloaded=True) + obj2.append(o) except: - logging.critical("Server does not want to reveal details about the calendar object", exc_info=True) + logging.error("Server does not want to reveal details about the calendar object", exc_info=True) pass + objects = obj2 ## Google sometimes returns empty objects objects = [o for o in objects if o.has_component()] @@ -2342,6 +2349,8 @@ def copy(self, keep_uid: bool = False, new_parent: Optional[Any] = None) -> Self obj.url = self.url return obj + ## TODO: move get-logics to a load_by_get method. + ## The load method should deal with "server quirks". def load(self, only_if_unloaded: bool = False) -> Self: """ (Re)load the object from the caldav server. @@ -2355,7 +2364,10 @@ def load(self, only_if_unloaded: bool = False) -> Self: if self.client is None: raise ValueError("Unexpected value None for self.client") - r = self.client.request(str(self.url)) + try: + r = self.client.request(str(self.url)) + except: + return self.load_by_multiget() if r.status == 404: raise error.NotFoundError(errmsg(r)) self.data = vcal.fix(r.raw) @@ -2365,6 +2377,23 @@ def load(self, only_if_unloaded: bool = False) -> Self: self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] return self + def load_by_multiget(self) -> Self: + """ + Some servers do not accept a GET, but we can still do a REPORT + with a multiget query + """ + error.assert_(self.url) + href = self.url.path + prop = dav.Prop() + cdav.CalendarData() + root = cdav.CalendarMultiGet() + prop + dav.Href(value=href) + response = self.parent._query(root, 1, "report") + results = response.expand_simple_props([cdav.CalendarData()]) + error.assert_(len(results) == 1) + data = results[href][cdav.CalendarData.tag] + error.assert_(data) + self.data = data + return self + ## TODO: self.id should either always be available or never def _find_id_path(self, id=None, path=None) -> None: """ From ac5f2462fae431a884dbe6e822e3ca0d59515e79 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 6 Dec 2024 23:08:51 +0100 Subject: [PATCH 32/34] wip --- check_server_compatibility.py | 3 +++ tests/compatibility_issues.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index e5f3fd8..7fe68bf 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -372,6 +372,8 @@ def _check_prop(self, propclass, silly_value, test_value): def check_event(self): cal = self._default_calendar + import pdb; pdb.set_trace() + ## Two simple events with text fields, dtstart=now and no dtend obj1 = cal.add_event( dtstart=datetime.now(), @@ -700,6 +702,7 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): self.set_flag("vtodo_datesearch_nostart_future_tasks_delivered", True) assert len(cal.search(end=before, **kwargs)) == 1 else: + import pdb; pdb.set_trace() assert len(cal.search(end=before, **kwargs)) == 0 assert len(cal.search(start=after, end=longafter)) == 0 assert len(cal.search(start=after, **kwargs)) == 0 diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index ccd5051..ae19ff0 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -381,7 +381,7 @@ 'fragile_sync_tokens', ## no issue raised yet 'vtodo_datesearch_nodtstart_task_is_skipped', ## no issue raised yet 'broken_expand_on_exceptions', ## no issue raised yet - 'date_todo_search_ignores_duration' + 'date_todo_search_ignores_duration', 'calendar_color', 'calendar_order', 'vtodo_datesearch_notime_task_is_skipped' From 138578d782573fcccf6f6b38b4f5ce8255143d93 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 7 Dec 2024 11:40:42 +0100 Subject: [PATCH 33/34] inaccurate datesearch --- check_server_compatibility.py | 25 +++++++++++++++++++------ tests/compatibility_issues.py | 3 +++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/check_server_compatibility.py b/check_server_compatibility.py index 7fe68bf..029e47f 100755 --- a/check_server_compatibility.py +++ b/check_server_compatibility.py @@ -288,6 +288,7 @@ def _fix_cal(self, todo=False): if self.flags_checked['no_mkcalendar']: cal = self.principal.calendars()[0] if cal.events() or cal.todos(): + import pdb; pdb.set_trace() raise "Refusing to run tests on a calendar with content" self._default_calendar = cal return cal @@ -372,8 +373,6 @@ def _check_prop(self, propclass, silly_value, test_value): def check_event(self): cal = self._default_calendar - import pdb; pdb.set_trace() - ## Two simple events with text fields, dtstart=now and no dtend obj1 = cal.add_event( dtstart=datetime.now(), @@ -653,12 +652,15 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): 7: search with start during event """ cal = self._default_calendar - longbefore = datetime(2000, 6, 30, 4) + longbefore = datetime(2000, 5, 30, 4) before = datetime(2000, 7, 1, 4) during1 = datetime(2000, 7, 1, 10) during2 = datetime(2000, 7, 1, 12) after = datetime(2000, 7, 1, 22) - longafter = datetime(2000, 7, 2, 10) + longafter = datetime(2000, 9, 2, 10) + if self.flags_checked.get('inaccurate_datesearch'): + before = before - timedelta(days=31) + after = after + timedelta(days=31) one_event_lists = [ ## open-ended searches, should yield object cal.search(end=after, **kwargs), ## 0 @@ -702,9 +704,20 @@ def _do_date_search(self, assert_found=True, has_duration=True, **kwargs): self.set_flag("vtodo_datesearch_nostart_future_tasks_delivered", True) assert len(cal.search(end=before, **kwargs)) == 1 else: - import pdb; pdb.set_trace() - assert len(cal.search(end=before, **kwargs)) == 0 + none = cal.search(end=before, **kwargs) + if none: + none = cal.search(end=longbefore, **kwargs) + assert not none + self.set_flag("inaccurate_datesearch") + before = before - timedelta(days=31) + after = after + timedelta(days=31) + else: + if not 'inaccurate_Datesearch' in self.flags_checked: + self.set_flag("inaccurate_datesearch", False) + assert len(cal.search(start=after, end=longafter)) == 0 + if len(cal.search(start=after, **kwargs)): + import pdb; pdb.set_trace() assert len(cal.search(start=after, **kwargs)) == 0 return ret diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index ae19ff0..addcab5 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -124,6 +124,9 @@ """it asserts DAV:allprop response contains the text 'resourcetype', """ """possibly this assert is wrong""", + 'inaccurate_datesearch': + """Searching by datetime seems to have a one-day resolution, so if searching for events over a timespan of hours, all events for the day is found""", + 'no_todo': """Support for VTODO (tasks) apparently missing""", From 77cbe66fcba82376f8ebf5029fda3ccf13d08625 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 7 Dec 2024 11:41:05 +0100 Subject: [PATCH 34/34] inaccurate datesearch --- tests/compatibility_issues.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/compatibility_issues.py b/tests/compatibility_issues.py index addcab5..619d631 100644 --- a/tests/compatibility_issues.py +++ b/tests/compatibility_issues.py @@ -463,6 +463,7 @@ 'no_sync_token', 'combined_search_not_working', 'broken_expand', + 'inaccurate_datesearch' ] calendar_mail_ru = [