Add endpoint for uploading docker image (autolab#232)

* Add endpoint for building docker containers

* Finish docker endpoint

* Actually increase upload size

* Accept questionable linting changes

* Accept more questionable linting changes

* Add docker to requirements.txt

* Add dockerTmp directory to .gitignore

* Add max docker version supported

* More useful logging

* Downgrade requests from 2.31.0 to 2.28.0 for docker compatibility

* Added check for localDocker when building

* Remove test string

.gitignore

@@ -7,6 +7,7 @@ scripts/*.sh
 # exclude
 vmms/id_rsa*
 courselabs/*
+dockerTmp/*
 # config
 config.py
 

clients/tango-cli.py

@@ -60,6 +60,10 @@
 parser.add_argument("--pool", action="store_true", help=pool_help)
 prealloc_help = "Create a pool of instances spawned from a specific image. Must specify key with -k. Modify defaults with --image (autograding_image), --num (2), --vmms (localDocker), --cores (1), and --memory (512)."
 parser.add_argument("--prealloc", action="store_true", help=prealloc_help)
+build_help = (
+    "Build a docker image. Must specify key with -k and filename with --filename."
+)
+parser.add_argument("--build", action="store_true", help=build_help)
 
 parser.add_argument(
     "--getPartialOutput", action="store_true", help="Get partial output"
@@ -487,6 +491,33 @@ def file_to_dict(file):
         return {"localFile": file, "destFile": file}
 
 
+# build
+
+
+def tango_build():
+    try:
+        res = checkKey() + checkFilename()
+        if res != 0:
+            raise Exception("Invalid usage: [build] " + build_help)
+
+        f = open(args.filename, "rb")
+        response = requests.post(
+            "%s://%s:%d/build/%s/"
+            % (_tango_protocol, args.server, args.port, args.key),
+            data=f.read(),
+        )
+        print("Sent request to %s:%d/build/%s/" % (args.server, args.port, args.key))
+        print(response.text)
+
+    except Exception as err:
+        print(
+            "Failed to send request to %s:%d/build/%s/"
+            % (args.server, args.port, args.key)
+        )
+        print(str(err))
+        sys.exit(0)
+
+
 # runJob
 
 
@@ -547,6 +578,8 @@ def router():
         tango_runJob()
     elif args.getPartialOutput:
         tango_getPartialOutput()
+    elif args.build:
+        tango_build()
 
 
 #
@@ -564,6 +597,7 @@ def router():
     and not args.prealloc
     and not args.runJob
     and not args.getPartialOutput
+    and not args.build
 ):
     parser.print_help()
     sys.exit(0)

config.template.py

@@ -87,7 +87,7 @@ class Config(object):
     DOCKER_HOST_USER = ""
 
     # Maximum size for input files in bytes
-    MAX_INPUT_FILE_SIZE = 250 * 1024 * 1024  # 250MB
+    MAX_INPUT_FILE_SIZE = 10 * 1024 * 1024 * 1024  # 10GB
 
     # Maximum size for output file in bytes
     MAX_OUTPUT_FILE_SIZE = 1000 * 1024

requirements.txt

@@ -2,6 +2,7 @@ backports.ssl-match-hostname==3.7.0.1
 boto==2.49.0 # used only by ec2SSH.py
 pyflakes==2.1.1
 redis==4.4.4
-requests==2.31.0
+requests==2.28.0
 # needed for tashi, rpyc==4.1.4
 tornado==6.3.3
+docker==5.0.3

restful_tango/server.py

@@ -125,6 +125,28 @@ async def post(self, key, image, num):
         self.write(instances)
 
 
+@tornado.web.stream_request_body
+class BuildHandler(tornado.web.RequestHandler):
+    def prepare(self):
+        """set up the temporary file"""
+        tempdir = "dockerTmp"
+        if not os.path.exists(tempdir):
+            os.mkdir(tempdir, 0o700)
+        if os.path.exists(tempdir) and not os.path.isdir(tempdir):
+            tangoREST.log("Cannot process uploads, %s is not a directory" % (tempdir,))
+            return self.send_error()
+        self.tempfile = NamedTemporaryFile(prefix="docker", dir=tempdir, delete=False)
+
+    def data_received(self, chunk):
+        self.tempfile.write(chunk)
+
+    def post(self, key):
+        """post - Handles the post request to build."""
+        name = self.tempfile.name
+        self.tempfile.close()
+        self.write(tangoREST.build(key, name))
+
+
 async def main(port: int):
     # Routes
     application = tornado.web.Application(
@@ -139,6 +161,7 @@ async def main(port: int):
             (r"/jobs/(%s)/(%s)/" % (SHA1_KEY, DEADJOBS), JobsHandler),
             (r"/pool/(%s)/" % (SHA1_KEY), PoolHandler),
             (r"/prealloc/(%s)/(%s)/(%s)/" % (SHA1_KEY, IMAGE, NUM), PreallocHandler),
+            (r"/build/(%s)/" % (SHA1_KEY), BuildHandler),
         ]
     )
     application.listen(port, max_buffer_size=Config.MAX_INPUT_FILE_SIZE)

restful_tango/tangoREST.py

@@ -10,6 +10,7 @@
 import hashlib
 import json
 import logging
+import docker
 
 currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
 parentdir = os.path.dirname(currentdir)
@@ -33,6 +34,7 @@ def __init__(self):
         self.obtained_pool = self.create(0, "Found pool")
         self.obtained_all_pools = self.create(0, "Found all pools")
         self.partial_output_obtained = self.create(0, "Partial output obtained")
+        self.image_built = self.create(0, "Docker image built")
 
         self.wrong_key = self.create(-1, "Key not recognized")
         self.wrong_courselab = self.create(-1, "Courselab not found")
@@ -41,6 +43,7 @@ def __init__(self):
         self.invalid_prealloc_size = self.create(-1, "Invalid prealloc size")
         self.pool_not_found = self.create(-1, "Pool not found")
         self.prealloc_failed = self.create(-1, "Preallocate VM failed")
+        self.image_build_failed = self.create(-1, "Image build failed")
 
     def create(self, id, msg):
         """create - Constructs a dict with the given ID and message"""
@@ -439,3 +442,28 @@ def prealloc(self, key, image, num, vmStr):
         else:
             self.log.info("Key not recognized: %s" % key)
             return self.status.wrong_key
+
+    def build(self, key, tempfile):
+        self.log.debug("Received build request(%s)" % (key))
+        if self.validateKey(key):
+            if Config.VMMS_NAME != "localDocker":
+                self.log.error("Not using Docker backend, so cannot build image")
+                os.unlink(tempfile)
+                return self.status.image_build_failed
+            try:
+                client = docker.from_env()
+                imageTarStr = open(tempfile, "rb").read()
+                images = client.images.load(imageTarStr)
+                id_list = ", ".join(image.short_id for image in images)
+            except Exception as e:
+                self.log.error("Image build failed: " + str(e))
+                os.unlink(tempfile)
+                return self.status.image_build_failed
+
+            self.log.info("Successfully loaded images: %s" % (id_list))
+            os.unlink(tempfile)
+            return self.status.image_built
+        else:
+            self.log.info("Key not recognized: %s" % key)
+            os.unlink(tempfile)
+            return self.status.wrong_key