Test refactoring

CHANGELOG.md

@@ -12,6 +12,16 @@ This project should more or less adhere to [Semantic Versioning](https://semver.
 
 ## [Unreleased]
 
+### Changed
+
+#### Test framework
+
+* Functional test framework has been refactored - code for setting up and rigging down xandikos/radicale servers have been moved from `tests/test_caldav.py` to `tests/conf.py`.  This allows for:
+    * Adding code (including system calls or remote API calls) for Setting up and tearing down calendar servers in `conf_private.py`
+    * Creating a local xandikos or radicale server in the `tests.client`-method, which is also used in the `examples`-section.
+    * Allows offline testing of my upcoming `check_server_compatibility`-script
+* Also added the possibility to tag test servers with a name
+
 ### Added
 
 * By now `calendar.search(..., sort_keys=("DTSTART")` will work.  Sort keys expects a list or a tuple, but it's easy to send an attribute by mistake.

RELEASE-HOWTO.md

@@ -17,7 +17,7 @@ I have no clue on the proper procedures for doing releases, and I keep on doing
 * Verify that we're on the right branch - `git checkout master`.  (`master` may not always be right - sometimes we may want to use a dedicated branch connected to the release-series, i.e. `v1.3`)
 * Set the variable `VERSION=1.4.0`
 * Commit the changes (typically `CHANGELOG.md`, perhaps documentation): `git commit -am "preparing for releasing v${VERSION}"`
-* Create a tag: `git tag -as v${VERSION}` - use the release notes in the tag message.
+* Create a tag: `git tag -as v${VERSION}` - use the release notes in the tag message.  Don't push it yet.
 * Make a clone: `git clone caldav/ caldav-release ; cd caldav-release ; git checkout v${VERSION}`
 * Run tests (particularly the style check): `pytest` and `tox -e style`.
 * Push the code to github: `cd ~/caldav ; git push ; git push --tags`

tests/compatibility_issues.py

@@ -13,7 +13,7 @@
 ## * Consider how to get this into the documentation
 incompatibility_description = {
     'rate_limited':
-        """Pause a bit between each request""",
+        """It may be needed to pause a bit between each request when doing tests""",
 
     'cleanup_calendar':
         """Remove everything on the calendar for every test""",
@@ -228,8 +228,10 @@
 
     'text_search_is_exact_match_only',
 
-    ## This one is fixed in master branch
-    'category_search_yields_nothing', ## https://github.com/jelmer/xandikos/pull/194
+    ## This one is fixed - but still breaks our test code for python 3.7
+    ## TODO: remove this when shredding support for python 3.7
+    ## https://github.com/jelmer/xandikos/pull/194
+    'category_search_yields_nothing',
 
     ## scheduling is not supported
     "no_scheduling",
@@ -240,7 +242,6 @@
     ## "weird" on radicale
     "broken_expand",
     "no_default_calendar",
-    "non_existing_calendar_found",
 
     ## freebusy is not supported yet, but on the long-term road map
     "no_freebusy_rfc4791",

tests/conf.py

@@ -4,10 +4,14 @@
 ## Make a conf_private.py for personal configuration.
 ## Check conf_private.py.EXAMPLE
 import logging
+import tempfile
+import threading
+import time
 
-from caldav.davclient import DAVClient
+import requests
 
-# from .compatibility_issues import bedework, xandikos
+from . import compatibility_issues
+from caldav.davclient import DAVClient
 
 ####################################
 # Import personal test server config
@@ -79,63 +83,151 @@
 #####################
 # Public test servers
 #####################
-## As of 2019-09, all of those are down.  Will try to fix Real Soon ... possibly before 2029 even.
-if False:
-    # if test_public_test_servers:
 
-    ## TODO: this one is set up on emphemeral storage on OpenShift and
-    ## then configured manually through the webui installer, it will
-    ## most likely only work for some few days until it's down again.
-    ## It's needed to hard-code the configuration into
-    ## https://github.com/python-caldav/baikal
+## Currently I'm not aware of any publically available test servers, and my
+## own attempts on maintaining any has been canned.
 
-    caldav_servers.append(
-        {
-            "url": "http://baikal-caldav-servers.cloudapps.bitbit.net/html/cal.php/",
-            "username": "baikaluser",
-            "password": "asdf",
-        }
-    )
+# if test_public_test_servers:
+# caldav_servers.append( ... )
 
-    # bedework:
-    # * todos and journals are not properly supported -
-    #   ref https://github.com/Bedework/bedework/issues/5
-    # * propfind fails to return resourcetype,
-    #   ref https://github.com/Bedework/bedework/issues/110
-    # * date search on recurrences of recurring events doesn't work
-    #   (not reported yet - TODO)
-    caldav_servers.append(
-        {
-            "url": "http://bedework-caldav-servers.cloudapps.bitbit.net/ucaldav/",
-            "username": "vbede",
-            "password": "bedework",
-            "incompatibilities": compatibility_issues.bedework,
-        }
-    )
+#######################
+# Internal test servers
+#######################
+
+if test_radicale:
+    import radicale.config
+    import radicale
+    import radicale.server
+    import socket
+
+    def setup_radicale(self):
+        self.serverdir = tempfile.TemporaryDirectory()
+        self.serverdir.__enter__()
+        self.configuration = radicale.config.load("")
+        self.configuration.update(
+            {"storage": {"filesystem_folder": self.serverdir.name}}
+        )
+        self.server = radicale.server
+        self.shutdown_socket, self.shutdown_socket_out = socket.socketpair()
+        self.radicale_thread = threading.Thread(
+            target=self.server.serve,
+            args=(self.configuration, self.shutdown_socket_out),
+        )
+        self.radicale_thread.start()
+        i = 0
+        while True:
+            try:
+                requests.get(self.url)
+                break
+            except:
+                time.sleep(0.05)
+                i += 1
+                assert i < 100
+
+    def teardown_radicale(self):
+        self.shutdown_socket.close()
+        i = 0
+        self.serverdir.__exit__(None, None, None)
 
+    url = "http://%s:%i/" % (radicale_host, radicale_port)
     caldav_servers.append(
         {
-            "url": "http://xandikos-caldav-servers.cloudapps.bitbit.net/",
+            "url": url,
+            "name": "LocalRadicale",
             "username": "user1",
-            "password": "password1",
-            "incompatibilities": compatibility_issues.xandikos,
+            "password": "any-password-seems-to-work",
+            "backwards_compatibility_url": url + "user1",
+            "incompatibilities": compatibility_issues.radicale,
+            "setup": setup_radicale,
+            "teardown": teardown_radicale,
         }
     )
 
-    # radicale
+if test_xandikos:
+    import asyncio
+
+    import aiohttp
+    import aiohttp.web
+    from xandikos.web import XandikosApp, XandikosBackend
+
+    def setup_xandikos(self):
+        ## TODO: https://github.com/jelmer/xandikos/issues/131#issuecomment-1054805270 suggests a simpler way to launch the xandikos server
+
+        self.serverdir = tempfile.TemporaryDirectory()
+        self.serverdir.__enter__()
+        ## Most of the stuff below is cargo-cult-copied from xandikos.web.main
+        ## Later jelmer created some API that could be used for this
+        ## Threshold put high due to https://github.com/jelmer/xandikos/issues/235
+        ## index_threshold not supported in latest release yet
+        # self.backend = XandikosBackend(path=self.serverdir.name, index_threshold=0, paranoid=True)
+        # self.backend = XandikosBackend(path=self.serverdir.name, index_threshold=9999, paranoid=True)
+        self.backend = XandikosBackend(path=self.serverdir.name)
+        self.backend._mark_as_principal("/sometestuser/")
+        self.backend.create_principal("/sometestuser/", create_defaults=True)
+        mainapp = XandikosApp(
+            self.backend, current_user_principal="sometestuser", strict=True
+        )
+
+        async def xandikos_handler(request):
+            return await mainapp.aiohttp_handler(request, "/")
+
+        self.xapp = aiohttp.web.Application()
+        self.xapp.router.add_route("*", "/{path_info:.*}", xandikos_handler)
+        ## https://stackoverflow.com/questions/51610074/how-to-run-an-aiohttp-server-in-a-thread
+        self.xapp_loop = asyncio.new_event_loop()
+        self.xapp_runner = aiohttp.web.AppRunner(self.xapp)
+        asyncio.set_event_loop(self.xapp_loop)
+        self.xapp_loop.run_until_complete(self.xapp_runner.setup())
+        self.xapp_site = aiohttp.web.TCPSite(
+            self.xapp_runner, host=xandikos_host, port=xandikos_port
+        )
+        self.xapp_loop.run_until_complete(self.xapp_site.start())
+
+        def aiohttp_server():
+            self.xapp_loop.run_forever()
+
+        self.xandikos_thread = threading.Thread(target=aiohttp_server)
+        self.xandikos_thread.start()
+
+    def teardown_xandikos(self):
+        if not test_xandikos:
+            return
+        self.xapp_loop.stop()
+
+        ## ... but the thread may be stuck waiting for a request ...
+        def silly_request():
+            try:
+                requests.get(self.url)
+            except:
+                pass
+
+        threading.Thread(target=silly_request).start()
+        i = 0
+        while self.xapp_loop.is_running():
+            time.sleep(0.05)
+            i += 1
+            assert i < 100
+        self.xapp_loop.run_until_complete(self.xapp_runner.cleanup())
+        i = 0
+        while self.xandikos_thread.is_alive():
+            time.sleep(0.05)
+            i += 1
+            assert i < 100
+
+        self.serverdir.__exit__(None, None, None)
+
+    url = "http://%s:%i/" % (xandikos_host, xandikos_port)
     caldav_servers.append(
         {
-            "url": "http://radicale-caldav-servers.cloudapps.bitbit.net/",
-            "username": "testuser",
-            "password": "123",
-            "nofreebusy": True,
-            "nodefaultcalendar": True,
-            "noproxy": True,
+            "name": "LocalXandikos",
+            "url": url,
+            "backwards_compatibility_url": url + "sometestuser",
+            "incompatibilities": compatibility_issues.xandikos,
+            "setup": setup_xandikos,
+            "teardown": teardown_xandikos,
         }
     )
 
-caldav_servers = [x for x in caldav_servers if x.get("enable", True)]
-
 ###################################################################
 # Convenience - get a DAVClient object from the caldav_servers list
 ###################################################################
@@ -144,11 +236,24 @@
 )
 
 
-def client(idx=None, **kwargs):
-    if idx is None and not kwargs:
-        return client(0)
+def client(
+    idx=None, name=None, setup=lambda conn: None, teardown=lambda conn: None, **kwargs
+):
+    ## No parameters given - find the first server in caldav_servers list
+    if idx is None and not kwargs and caldav_servers:
+        idx = 0
+        while idx < len(caldav_servers) and not caldav_servers[idx].get("enable", True):
+            idx += 1
+        if idx == len(caldav_servers):
+            return None
+        return client(idx=idx)
     elif idx is not None and not kwargs and caldav_servers:
         return client(**caldav_servers[idx])
+    elif name is not None and not kwargs and caldav_servers:
+        for s in caldav_servers:
+            if caldav_servers["name"] == s:
+                return s
+        return None
     elif not kwargs:
         return None
     for bad_param in (
@@ -166,4 +271,10 @@ def client(idx=None, **kwargs):
                 % kw
             )
             kwargs.pop(kw)
-    return DAVClient(**kwargs)
+    conn = DAVClient(**kwargs)
+    setup(conn)
+    conn.teardown = teardown
+    return conn
+
+
+caldav_servers = [x for x in caldav_servers if x.get("enable", True)]

tests/conf_private.py.EXAMPLE

@@ -10,7 +10,12 @@ from tests import compatibility_issues
 ## Define your primary caldav server here
 caldav_servers = [
   {
-	      ## Set enable to False if you don't want to use a server
+        ## A friendly identifiter for the server.  Should be a CamelCase name
+	## Not needed, but may be nice if you have several servers to test towards.
+	## Should not affect test runs in any other way than improved verbosity.
+        'name': 'MyExampleServer'
+
+        ## Set enable to False if you don't want to use a server
         'enable': True,
 
         ## This is all that is really needed - url, username and
@@ -27,6 +32,11 @@ caldav_servers = [
         ## tests/compatibility_issues.py for premade lists
         #'incompatibilities': compatibility_issues.nextcloud
       	'incompatibilities': [],
+
+        ## You may even add setup and teardown methods to set up
+        ## and rig down the calendar server
+        #setup = lambda self: ...
+        #teardown = lambda self: ...
   }
 ]
 
@@ -71,4 +81,5 @@ test_radicale = True
 
 ## For usage by ../examples/scheduling_examples.py.  Should typically
 ## be three different users on the same caldav server.
+## (beware of dragons - there is some half-done work in the caldav_test that is likely to break if this is set)
 #rfc6638_users = [ caldav_servers[0], caldav_servers[1], caldav_servers[2] ]