Adding custom mirror URL fields to installer (#135)

* Add GUI Field Option for custom mirror/version URL

Add custom mirror and version url handling
to the GUI. Implemented changes to persist
GUI modifications to both phsyical and image
targets. Removed hard coded defaults in GUI.

ister.py

@@ -664,6 +664,38 @@ def set_hostname(template, target_dir):
         file.write(hostname)
 
 
+def set_mirror_url(template, target_dir):
+    """Writes custom mirror url to <target disk>/etc/swupd/mirror_contenturl
+    """
+    target_mirror_url = template.get("MirrorURL")
+    if not target_mirror_url:
+        return
+
+    LOG.info("Setting custom mirror url")
+    path = '{0}/etc/swupd/'.format(target_dir)
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    with open(path + "mirror_contenturl", "w") as file:
+        file.write(target_mirror_url)
+
+
+def set_mirror_version_url(template, target_dir):
+    """Writes custom mirror version url to <target disk>/etc/swupd/mirror_versionurl
+    """
+    target_mirror_version_url = template.get("VersionURL")
+    if not target_mirror_version_url:
+        return
+
+    LOG.info("Setting custom mirror version url")
+    path = '{0}/etc/swupd/'.format(target_dir)
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    with open(path + "mirror_versionurl", "w") as file:
+        file.write(target_mirror_version_url)
+
+
 def set_static_configuration(template, target_dir):
     """Writes the configuration on /etc/systemd/network/10-en-static.network
     """
@@ -1135,6 +1167,26 @@ def validate_proxy_url_template(proxy):
         raise Exception("Invalid proxy url: {}".format(proxy))
 
 
+def validate_mirror_url_template(mirror):
+    """Attempt to verify the mirror setting is valid
+
+    This function will raise an Exception on finding an error.
+    """
+    url = urlparse(mirror)
+    if not (url.scheme and url.netloc):
+        raise Exception("Invalid mirror url: {}".format(mirror))
+
+
+def validate_mirror_version_url_template(version):
+    """Attempt to verify the version setting is valid
+
+    This function will raise an Exception on finding an error.
+    """
+    url = urlparse(version)
+    if not (url.scheme and url.netloc):
+        raise Exception("Invalid version url: {}".format(version))
+
+
 def validate_cmdline_template(cmdline):
     """Attempt to verify the cmdline configuration
 
@@ -1180,6 +1232,10 @@ def validate_template(template):
         validate_proxy_url_template(template["HTTPSProxy"])
     if template.get("HTTPProxy"):
         validate_proxy_url_template(template["HTTPProxy"])
+    if template.get("MirrorURL"):
+        validate_mirror_url_template(template["MirrorURL"])
+    if template.get("VersionURL"):
+        validate_mirror_version_url_template(template["VersionURL"])
     if template.get("cmdline"):
         validate_cmdline_template(template["cmdline"])
     LOG.debug("Configuration is valid:")
@@ -1422,6 +1478,8 @@ def install_os(args, template):
         copy_os(args, template, target_dir)
         add_users(template, target_dir)
         set_hostname(template, target_dir)
+        set_mirror_url(template,target_dir)
+        set_mirror_version_url(template,target_dir)
         set_static_configuration(template, target_dir)
         set_kernel_cmdline_appends(template, target_dir)
         if template.get("IsterCloudInitSvc"):

ister_gui.py

@@ -427,6 +427,40 @@ def interface_list():
     return [ifc for ifc in netifaces.interfaces() if ifc.startswith('e')]
 
 
+def get_swupd_content_url():
+    """
+    Find and return the content url that swupd determines
+    """
+    cmd = ['swupd', 'mirror']
+    try:
+        output = subprocess.check_output(cmd).decode('utf-8')
+    except:
+        return None
+
+    for line in output.split('\n'):
+        if re.match('^Content URL:\s+', line):
+            return (re.split(r'\s+',line)[-1]).strip()
+
+    return None
+
+
+def get_swupd_version_url():
+    """
+    Find and return the version url that swupd determines
+    """
+    cmd = ['swupd', 'mirror']
+    try:
+        output = subprocess.check_output(cmd).decode('utf-8')
+    except:
+        return None
+
+    for line in output.split('\n'):
+        if re.match('^Version URL:\s+', line):
+            return (re.split(r'\s+',line)[-1]).strip()
+
+    return None
+
+
 class Alert(object):
     """Class to display alerts or confirm boxes"""
     # pylint: disable=R0902
@@ -1479,7 +1513,6 @@ def update_widgets(self):
         self.gateway = urwid.Columns([self.gateway_e, gateway])
         self.dns = urwid.Columns([self.dns_e, dns])
         self.widgets = [self.title,
-                        urwid.Divider(),
                         PopUpWidget('Show available interfaces',
                                     '\n'.join(interface_list()),
                                     'close'),
@@ -1633,15 +1666,32 @@ def __init__(self, cur_step, tot_steps, content_url, version_url):
         super(NetworkRequirements, self).__init__()
         self.content_url = content_url
         self.version_url = version_url
+
+        # if not set on command line, find the swupd defaults via the swupd command
+        if self.content_url is None:
+            self.content_url = get_swupd_content_url()
+        if self.version_url is None:
+            self.version_url = get_swupd_version_url()
+
         left = 8  # padding desired left of buttons, modifiable
         # configure proxy section
         self.proxy_header = urwid.Text('Proxy Settings')
         self.proxy_button = ister_button('Set proxy configuration',
                                          on_press=self._set_proxy,
                                          left=left)
+        # configure mirror/version section
+        self.mirror_header = urwid.Text('Mirror Settings (Optional)')
+        self.mirror_button = ister_button('Set mirror URL',
+                                         on_press=self._set_mirror,
+                                         left=left)
+        self.version_button = ister_button('Set version url',
+                                         on_press=self._set_version,
+                                         left=left)
         # initiate necessary instance variables
         self.progress = urwid.Text('Step {} of {}'.format(cur_step, tot_steps))
         self.https_proxy = None
+        self.content_url_alt = None
+        self.version_url_alt = None
         self.config = None
         self.nettime = False
         self.reset = False
@@ -1690,6 +1740,18 @@ def build_ui_widgets(self):
             [self.https_proxy,
              urwid.Text(('ex', 'example: http://proxy.url.com:123'),
                         align='center')])
+
+        self.content_url_alt = urwid.Edit(fmt.format('Mirror URL: '),self.content_url)
+        self.version_url_alt = urwid.Edit(fmt.format('Version URL: '),self.version_url)
+        content_col = urwid.Columns(
+            [self.content_url_alt,
+                urwid.Text(('ex', 'example: http://alt.mirror.com/update/'),
+                        align='center')])
+        version_col = urwid.Columns(
+            [self.version_url_alt,
+                urwid.Text(('ex', 'example: http://alt.version.com/update/'),
+                        align='center')])
+
         if self.timeout:
             wired_req = urwid.Text(('warn', self.timeout))
         else:
@@ -1702,13 +1764,16 @@ def build_ui_widgets(self):
                                         'install will fail'])
 
         self._ui_widgets = [self.progress,
-                            urwid.Divider(),
                             wired_req,
                             urwid.Divider(),
-                            self.proxy_header,
+                            self.mirror_header,
+                            content_col,
+                            self.mirror_button,
+                            version_col,
+                            self.version_button,
                             urwid.Divider(),
+                            self.proxy_header,
                             https_col,
-                            urwid.Divider(),
                             self.proxy_button,
                             urwid.Divider()]
         self._ui_widgets.extend(self.netcontrol.widgets)
@@ -1843,6 +1908,35 @@ def _set_proxy(self, _):
 
         raise urwid.ExitMainLoop()
 
+    def _set_mirror(self, _):
+        """Set the user defined content mirror url for the installer in the template"""
+        content_default = get_swupd_content_url()
+        if self.content_url_alt.get_edit_text():
+            self.content_url = self.content_url_alt.get_edit_text()
+            if content_default == self.content_url:
+                if self.config.get("MirrorURL"):
+                    self.config.pop('MirrorURL', None)
+            else:
+                self.config['MirrorURL'] = self.content_url
+        else:
+            self.config.pop('MirrorURL', None)
+
+        raise urwid.ExitMainLoop()
+
+    def _set_version(self, _):
+        """Set the user defined content mirror version url for the installer in the template"""
+        version_default = get_swupd_version_url()
+        if self.version_url_alt.get_edit_text():
+            self.version_url = self.version_url_alt.get_edit_text()
+            if version_default == self.version_url:
+                if self.config.get("VersionURL"):
+                    self.config.pop('VersionURL', None)
+            else:
+                self.config['VersionURL'] = self.version_url
+        else:
+            self.config.pop('VersionURL', None)
+
+        raise urwid.ExitMainLoop()
 
 class TelemetryDisclosure(ProcessStep):
     # pylint: disable=R0902
@@ -3264,6 +3358,10 @@ def automatic_install(self):
             {'name': 'contenturl', 'out': '--contenturl={0}'},
             {'name': 'versionurl', 'out': '--versionurl={0}'},
             {'name': 'format', 'out': '--format={0}'}]
+        if self.installation_d.get('VersionURL'):
+            self.args['versionurl'] = self.installation_d.get('VersionURL')
+        if self.installation_d.get('MirrorURL'):
+            self.args['contenturl'] = self.installation_d.get('MirrorURL')
         ister_cmd = [item['out'].format(self.args[item['name']])
                  for item in supported if self.args[item['name']] is not None]
         ister_log = '/var/log/ister.log'
@@ -3314,10 +3412,10 @@ def handle_options():
     """Argument parser for the ui"""
     parser = argparse.ArgumentParser()
     parser.add_argument("-V", "--versionurl", action="store",
-                        default="https://cdn.download.clearlinux.org/update",
+                        default=None,
                         help="URL to use for looking for update versions")
     parser.add_argument("-C", "--contenturl", action="store",
-                        default="https://cdn.download.clearlinux.org/update",
+                        default=None,
                         help="URL to use for looking for update content")
     parser.add_argument("-f", "--format", action="store", default=None,
                         help="format to use for looking for update content")