Basic outgoing proxy & Tor support, adds config.sys.proxy.*

Make the gravatar downloader require Tor unless configured otherwise.
Make the SMTP client proxy & Tor capable via. the connection broker.

Relates to: #1075, #1131, #721
Fixes: #724, #989, #1281

mailpile/config.py

@@ -1991,6 +1991,14 @@ def _unlocked_prepare_workers(config, session=None,
             config.background.ui = BackgroundInteraction(config,
                                                          log_parent=session.ui)
 
+        # Tell conn broker that we exist
+        from mailpile.conn_brokers import Master as ConnBroker
+        ConnBroker.set_config(config)
+        if 'connbroker' in config.sys.debug:
+            ConnBroker.debug_callback = lambda msg: config.background.ui.debug(msg)
+        else:
+            ConnBroker.debug_callback = None
+
         def start_httpd(sspec=None):
             sspec = sspec or (config.sys.http_host, config.sys.http_port,
                               config.sys.http_path or '')

mailpile/conn_brokers.py

@@ -35,6 +35,16 @@
 import threading
 import traceback
 
+# Import SOCKS proxy support...
+try:
+    import sockschain as socks
+except ImportError:
+    try:
+        import socks
+    except ImportError:
+        socks = None
+
+
 org_cconn = socket.create_connection
 org_sslwrap = ssl.wrap_socket
 
@@ -51,6 +61,7 @@ class Capability(object):
     OUTGOING_RAW = 'o:raw'      # Request this to avoid meddling brokers
     OUTGOING_ENCRYPTED = 'o:e'  # Request this if sending encrypted data
     OUTGOING_CLEARTEXT = 'o:c'  # Request this if sending clear-text data
+    OUTGOING_TRACKABLE = 'o:t'  # Reject this to require anonymity
     OUTGOING_SMTP = 'o:smtp'    # These inform brokers what protocol is being
     OUTGOING_IMAP = 'o:imap'    # .. used, to allow protocol-specific features
     OUTGOING_POP3 = 'o:pop3'    # .. such as enabling STARTTLS or upgrading
@@ -67,6 +78,18 @@ class Capability(object):
     INCOMING_HTTP = 27
     INCOMING_HTTPS = 28
 
+    ALL_OUTGOING = set([OUTGOING_RAW, OUTGOING_ENCRYPTED, OUTGOING_CLEARTEXT,
+                        OUTGOING_TRACKABLE,
+                        OUTGOING_SMTP, OUTGOING_IMAP, OUTGOING_POP3,
+                        OUTGOING_HTTP, OUTGOING_HTTPS])
+
+    ALL_OUTGOING_ENCRYPTED = set([OUTGOING_RAW, OUTGOING_TRACKABLE,
+                                  OUTGOING_ENCRYPTED, OUTGOING_HTTPS])
+
+    ALL_INCOMING = set([INCOMING_RAW, INCOMING_LOCALNET, INCOMING_INTERNET,
+                        INCOMING_DARKNET, INCOMING_SMTP, INCOMING_IMAP,
+                        INCOMING_POP3, INCOMING_HTTP, INCOMING_HTTPS])
+
 
 class CapabilityFailure(IOError):
     """
@@ -150,26 +173,41 @@ class BaseConnectionBroker(Capability):
     SUPPORTS = []
 
     def __init__(self, master=None):
-        self.supports = self.SUPPORTS[:]
+        self.supports = list(self.SUPPORTS)[:]
         self.master = master
-        self._debug = None
+        self._config = None
+        self._debug = master._debug if master else None
+
+    def configure(self):
+        self.supports = list(self.SUPPORTS)[:]
 
-    def _raise_or_none(self, exc):
+    def set_config(self, config):
+        self._config = config
+        self.configure()
+
+    def config(self):
+        if self._config is not None:
+            return self._config
+        if self.master is not None:
+            return self.master.config()
+        return None
+
+    def _raise_or_none(self, exc, why):
         if exc is not None:
-            raise exc()
+            raise exc(why)
         return None
 
     def _check(self, need, reject, _raise=CapabilityFailure):
         for n in need or []:
             if n not in self.supports:
                 if self._debug is not None:
                     self._debug('%s: lacking capabilty %s' % (self, n))
-                return self._raise_or_none(_raise)
+                return self._raise_or_none(_raise, 'Lacking %s' % n)
         for n in reject or []:
             if n in self.supports:
                 if self._debug is not None:
                     self._debug('%s: unwanted capabilty %s' % (self, n))
-                return self._raise_or_none(_raise)
+                return self._raise_or_none(_raise, 'Unwanted %s' % n)
         if self._debug is not None:
             self._debug('%s: checks passed!' % (self, ))
         return self
@@ -225,35 +263,130 @@ class TcpConnectionBroker(BaseConnectionBroker):
     The only clever thing this class does, is to avoid trying to connect
     to .onion addresses, preventing that from leaking over DNS.
     """
-    SUPPORTS = [Capability.OUTGOING_RAW,
-                Capability.OUTGOING_ENCRYPTED,
-                Capability.OUTGOING_CLEARTEXT, # In strict mode, omit?
-                Capability.OUTGOING_SMTP,
-                Capability.OUTGOING_IMAP,
-                Capability.OUTGOING_POP3,
-                Capability.OUTGOING_HTTP,
-                Capability.OUTGOING_HTTPS,
-                Capability.INCOMING_RAW,
-#               Capability.INCOMING_INTERNET,  # Only if we have a public IP!
-                Capability.INCOMING_SMTP,
-                Capability.INCOMING_IMAP,
-                Capability.INCOMING_POP3,
-                Capability.INCOMING_HTTP,
-                Capability.INCOMING_HTTPS,
-                Capability.INCOMING_LOCALNET]
+    SUPPORTS = (
+        # Normal TCP/IP is not anonymous, and we do not have incoming
+        # capability unless we have a public IP.
+        (Capability.ALL_OUTGOING) |
+        (Capability.ALL_INCOMING - set([Capability.INCOMING_INTERNET]))
+    )
+
+    DEBUG_FMT = '%s: Raw TCP conn to: %s'
+
+    def configure(self):
+        BaseConnectionBroker.configure(self)
+        # FIXME: If our config indicates we have a public IP, add the
+        #        INCOMING_INTERNET capability.
+        # FIXME: If our coonfig indicates that the user does not care
+        #        about anonymity at all, remove OUTGOING_TRACKABLE.
+        if (self.config().sys.proxy.protocol != 'none' and
+                not self.config().sys.proxy.fallback):
+            self.supports = []
 
     def _describe(self, context, conn):
         context.encryption = None
         context.is_internet = True
         return conn
 
-    def _create_connection(self, context, address, *args, **kwargs):
-        if self._debug is not None:
-            self._debug('%s: Raw TCP conn to: %s' % (self, address))
+    def _avoid(self, address):
         if address[0].endswith('.onion'):
             raise CapabilityFailure('Cannot connect to .onion addresses')
+
+    def _conn(self, address, *args, **kwargs):
         return org_cconn(address, *args, **kwargs)
 
+    def _create_connection(self, context, address, *args, **kwargs):
+        self._avoid(address)
+        if self._debug is not None:
+            self._debug(self.DEBUG_FMT % (self, address))
+        return self._conn(address, *args, **kwargs)
+
+
+class SocksConnBroker(TcpConnectionBroker):
+    """
+    This broker offers the same services as the TcpConnBroker, but over a
+    SOCKS connection.
+    """
+    SUPPORTS = []
+    CONFIGURED = Capability.ALL_OUTGOING
+    DEBUG_FMT = '%s: Raw SOCKS5 conn to: %s'
+    PROXY_TYPES = ('socks5', 'http', 'socks4')
+
+    def __init__(self, *args, **kwargs):
+        TcpConnectionBroker.__init__(self, *args, **kwargs)
+        self.proxy_config = None
+        self.typemap = {}
+
+    def configure(self):
+        BaseConnectionBroker.configure(self)
+        if self.config().sys.proxy.protocol in self.PROXY_TYPES:
+            self.proxy_config = self.config().sys.proxy
+            self.supports = list(self.CONFIGURED)[:]
+            self.typemap = {
+                'socks5': socks.PROXY_TYPE_SOCKS5,
+                'socks4': socks.PROXY_TYPE_SOCKS4,
+                'http': socks.PROXY_TYPE_HTTP,
+                'tor': socks.PROXY_TYPE_SOCKS5  # For TorConnBrokerk
+            }
+        else:
+            self.proxy_config = None
+            self.supports = []
+
+    def _conn(self, address, timeout=None, source_address=None):
+        sock = socks.socksocket()
+        sock.setproxy(proxytype=self.typemap[self.proxy_config.protocol],
+                      addr=self.proxy_config.host,
+                      port=self.proxy_config.port,
+                      rdns=True,
+                      username=self.proxy_config.username or None,
+                      password=self.proxy_config.password or None)
+        if timeout and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+            sock.settimeout(float(timeout))
+        if source_address:
+            raise IOError('Cannot bind source address')
+        try:
+            address = (str(address[0]), address[1])
+            sock.connect(address)
+        except socks.ProxyError:
+            self._debug(traceback.format_exc())
+            raise IOError('Proxy failed for %s' % address)
+        return sock
+
+
+class TorConnBroker(SocksConnBroker):
+    """
+    This broker offers the same services as the TcpConnBroker, but over Tor.
+
+    This removes the "trackable" capability, so requests that reject it can
+    find their way here safely...
+    """
+    SUPPORTS = []
+    CONFIGURED = (Capability.ALL_OUTGOING_ENCRYPTED
+                  - set([Capability.OUTGOING_TRACKABLE]))
+    REJECTS = None
+    DEBUG_FMT = '%s: Raw Tor conn to: %s'
+    PROXY_TYPES = ('tor', )
+
+    def _avoid(self, address):
+        pass
+
+
+class TorOnionBroker(SocksConnBroker):
+    """
+    This broker offers the same services as the TcpConnBroker, but over Tor.
+    This removes the "trackable" capability, so requests that reject it can
+    find their way here safely...
+    """
+    SUPPORTS = []
+    CONFIGURED = (Capability.ALL_OUTGOING
+                  - set([Capability.OUTGOING_TRACKABLE]))
+    REJECTS = None
+    DEBUG_FMT = '%s: Raw Tor conn to: %s'
+    PROXY_TYPES = ('tor', )
+
+    def _avoid(self, address):
+        if not address[0].endswith('.onion'):
+            raise CapabilityFailure('Can only connect to .onion addresses')
+
 
 class BaseConnectionBrokerProxy(TcpConnectionBroker):
     """
@@ -300,6 +433,8 @@ def _describe(self, context, conn):
         return conn
 
     def _proxy_address(self, address):
+        if address[0].endswith('.onion'):
+            raise CapabilityFailure('I do not like .onion addresses')
         if int(address[1]) == 80:
             return (address[0], 443)
         return address
@@ -332,6 +467,16 @@ class MasterBroker(BaseConnectionBroker):
     def __init__(self, *args, **kwargs):
         BaseConnectionBroker.__init__(self, *args, **kwargs)
         self.brokers = []
+        self._debug = self._debugger
+        self.debug_callback = None
+
+    def configure(self):
+        for prio, cb in self.brokers:
+            cb.configure()
+
+    def _debugger(self, *args, **kwargs):
+        if self.debug_callback is not None:
+            self.debug_callback(*args, **kwargs)
 
     def register_broker(self, priority, cb):
         """
@@ -392,6 +537,11 @@ def CreateConnWarning(*args, **kwargs):
     register(9500, AutoImapStartTLSConnBroker)
     register(9500, AutoPop3StartTLSConnBroker)
 
+    if socks is not None:
+        register(1500, SocksConnBroker)
+        register(3500, TorConnBroker)
+        register(3500, TorOnionBroker)
+
     def SslWrapOnlyOnce(sock, *args, **kwargs):
         """
         Since we like to wrap things our own way, this make ssl.wrap_socket

mailpile/defaults.py

@@ -33,6 +33,8 @@
     'sys': p(_('Technical system settings'), False, {
         'fd_cache_size':  (_('Max files kept open at once'), int,         500),
         'history_length': (_('History length (lines, <0=no save)'), int,  100),
+        'http_host':     p(_('Listening host for web UI'),
+                           'hostname', 'localhost'),
         'http_port':     p(_('Listening port for web UI'), int,         33411),
         'http_path':     p(_('HTTP path of web UI'), 'webroot',            ''),
         'postinglist_kb': (_('Posting list target size in KB'), int,       64),
@@ -43,8 +45,6 @@
                            str, 'pool.sks-keyservers.net'),
         'gpg_home':      p(_('Override the home directory of GnuPG'), 'dir',
                            None),
-        'http_host':     p(_('Listening host for web UI'),
-                           'hostname', 'localhost'),
         'local_mailbox_id': (_('Local read/write Maildir'), 'b36',         ''),
         'mailindex_file': (_('Metadata index file'), 'file',               ''),
         'postinglist_dir': (_('Search index directory'), 'dir',            ''),
@@ -59,6 +59,16 @@
         }],
         'lockdown':       [_('Demo mode, disallow changes'), bool,      False],
         'login_banner':   [_('A custom banner for the login page'), str,   ''],
+        'proxy':          [_('Proxy settings'), False, {
+            'protocol':   (_('Proxy protocol'),
+                           ["tor", "socks5", "socks4", "http", "none"],
+                           'none'),
+            'fallback':   (_('Allow fallback to direct conns'), bool, False),
+            'username':   (_('User name'), str, ''),
+            'password':   (_('Password'), str, ''),
+            'host':       (_('Host'), str, ''),
+            'port':       (_('Port'), int, 8080)
+        }],
     }),
     'prefs': p(_("User preferences"), False, {
         'num_results':     (_('Search results per page'), int,             20),

mailpile/plugins/setup_magic.py

@@ -478,12 +478,16 @@ def _progress(self, message):
             self.event.message = message
             self._update_event_state(self.event.RUNNING, log=True)
         else:
-            session.ui.mark(message)
+            self.session.ui.mark(message)
 
     def _urlget(self, url):
-        with ConnBroker.context(need=[ConnBroker.OUTGOING_HTTP]) as context:
+        if url.lower().startswith('https'):
+            conn_needs = [ConnBroker.OUTGOING_HTTPS]
+        else:
+            conn_needs = [ConnBroker.OUTGOING_HTTP]
+        with ConnBroker.context(need=conn_needs) as context:
             self.session.ui.mark('Getting: %s' % url)
-            return urlopen(url, data=None, timeout=3).read()
+            return urlopen(url, data=None, timeout=10).read()
 
     def _username(self, val, email):
         lpart = email.split('@')[0]

mailpile/plugins/vcard_gravatar.py

@@ -32,6 +32,7 @@ class GravatarImporter(VCardImporter):
     SHORT_NAME = 'gravatar'
     CONFIG_RULES = {
         'active': [_('Enable this importer'), bool, True],
+        'anonymous': [_('Require anonymity for use'), bool, True],
         'interval': [_('Minimum days between refreshing'), 'int', 7],
         'batch': [_('Max batch size per update'), 'int', 30],
         'default': [_('Default thumbnail style'), str, 'retro'],
@@ -66,7 +67,10 @@ def _jittery_time():
         return want
 
     def _get(self, url):
-        with ConnBroker.context(need=[ConnBroker.OUTGOING_HTTP]) as context:
+        conn_need, conn_reject = [ConnBroker.OUTGOING_HTTP], []
+        if self.config.anonymous:
+            conn_reject += [ConnBroker.OUTGOING_TRACKABLE]
+        with ConnBroker.context(need=conn_need, reject=conn_reject) as ctx:
             self.session.ui.mark('Getting: %s' % url)
             return urlopen(url, data=None, timeout=3).read()