diff --git a/.buildkite/common.py b/.buildkite/common.py
index a8913d5d6ef..342bd671ddc 100644
--- a/.buildkite/common.py
+++ b/.buildkite/common.py
@@ -6,6 +6,7 @@
 """
 
 import argparse
+import ast
 import json
 import os
 import random
@@ -138,7 +139,7 @@ def __call__(self, parser, namespace, value, option_string=None):
         res = getattr(namespace, self.dest, {})
         key_str, val = value.split("=", maxsplit=1)
         keys = key_str.split("/")
-        update = {keys[-1]: val}
+        update = {keys[-1]: ast.literal_eval(val)}
         for key in list(reversed(keys))[1:]:
             update = {key: update}
         res = overlay_dict(res, update)