Parameterize master memory fraction (#53)

* Parameterize master memory fraction * Parameterize master memory fraction * Parameterize master memory fraction
Nealelab · Mar 16, 2018 · 2e03a3b · 2e03a3b
1 parent 105f2e5
commit 2e03a3b
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -156,6 +156,7 @@ $ cluster start -h
 usage: cluster start [-h] [--hash HASH] [--spark {2.0.2,2.2.0}]
                      [--version {0.1,devel}]
                      [--master-machine-type MASTER_MACHINE_TYPE]
+                     [--master-memory-fraction MASTER_MEMORY_FRACTION]
                      [--master-boot-disk-size MASTER_BOOT_DISK_SIZE]
                      [--num-master-local-ssds NUM_MASTER_LOCAL_SSDS]
                      [--num-preemptible-workers NUM_PREEMPTIBLE_WORKERS]
@@ -166,9 +167,8 @@ usage: cluster start [-h] [--hash HASH] [--spark {2.0.2,2.2.0}]
                      [--worker-machine-type WORKER_MACHINE_TYPE] [--zone ZONE]
                      [--properties PROPERTIES] [--metadata METADATA]
                      [--packages PACKAGES] [--jar JAR] [--zip ZIP]
-                     [--init INIT] [--vep]
+                     [--init INIT] [--vep] [--dry-run]
                      name
-
 Start a Dataproc cluster configured for Hail.
 
 positional arguments:
@@ -184,6 +184,10 @@ optional arguments:
                         Hail version to use (default: 0.1).
   --master-machine-type MASTER_MACHINE_TYPE, --master MASTER_MACHINE_TYPE, -m MASTER_MACHINE_TYPE
                         Master machine type (default: n1-highmem-8).
+  --master-memory-fraction MASTER_MEMORY_FRACTION
+                        Fraction of master memory allocated to the JVM. Use a
+                        smaller value to reserve more memory for Python.
+                        (default: 0.8)
   --master-boot-disk-size MASTER_BOOT_DISK_SIZE
                         Disk size of master machine, in GB (default: 100).
   --num-master-local-ssds NUM_MASTER_LOCAL_SSDS
@@ -216,8 +220,7 @@ optional arguments:
   --zip ZIP             Hail zip to use for Jupyter notebook.
   --init INIT           Comma-separated list of init scripts to run.
   --vep                 Configure the cluster to run VEP.
-  --dry-run             Don't start a cluster, just print the gcloud command.
-```
+  --dry-run             Print gcloud dataproc command, but don't run it.```
 
 ```
 $ cluster submit -h

diff --git a/cloudtools/start.py b/cloudtools/start.py
@@ -41,6 +41,10 @@ def init_parser(parser):
                         help='Hail version to use (default: %(default)s).')
     parser.add_argument('--master-machine-type', '--master', '-m', default='n1-highmem-8', type=str,
                         help='Master machine type (default: %(default)s).')
+    parser.add_argument('--master-memory-fraction', default=0.8, type=float,
+                        help='Fraction of master memory allocated to the JVM. '
+                             'Use a smaller value to reserve more memory '
+                             'for Python. (default: %(default)s)')
     parser.add_argument('--master-boot-disk-size', default=100, type=int,
                         help='Disk size of master machine, in GB (default: %(default)s).')
     parser.add_argument('--num-master-local-ssds', default=0, type=int,
@@ -94,7 +98,7 @@ def main(args):
 
     # parse Spark and HDFS configuration parameters, combine into properties argument
     properties = [
-        'spark:spark.driver.memory={}g'.format(str(int(machine_mem[args.master_machine_type] * 0.8))),
+        'spark:spark.driver.memory={}g'.format(str(int(machine_mem[args.master_machine_type] * args.master_memory_fraction))),
         'spark:spark.driver.maxResultSize=0',
         'spark:spark.task.maxFailures=20',
         'spark:spark.kryoserializer.buffer.max=1g',

diff --git a/setup.py b/setup.py
@@ -1,7 +1,7 @@
 from setuptools import setup
 
 setup(name='cloudtools',
-	  version='1.1.9',
+	  version='1.1.10',
 	  description='Collection of utilities for working on the Google Cloud Platform.',
 	  url='https://github.com/Nealelab/cloudtools',
 	  author='Liam Abbott',