diff --git a/examples/sagemaker-tensorflow/sagemaker-merlin-tensorflow.ipynb b/examples/sagemaker-tensorflow/sagemaker-merlin-tensorflow.ipynb index f08f09199..c912183b4 100644 --- a/examples/sagemaker-tensorflow/sagemaker-merlin-tensorflow.ipynb +++ b/examples/sagemaker-tensorflow/sagemaker-merlin-tensorflow.ipynb @@ -52,7 +52,7 @@ "in this repository or example notebooks in\n", "[Merlin Models](https://github.com/NVIDIA-Merlin/models/tree/stable/examples).\n", "\n", - "To run this notebook, you need to have [Amazon SageMaker Python SDK](https://sagemaker.readthedocs.io/en/stable/) installed." + "To run this notebook, you need to have [Amazon SageMaker Python SDK](https://sagemaker.readthedocs.io/en/stable/) installed. If you are *not* running this notebook in the [merlin-tensorflow](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/merlin/containers/merlin-tensorflow/tags) container (e.g., in a Sagemaker notebook instance or on Sagemaker Studio), you will also need to install the merlin packages by uncommenting below. You do not need to install them again if you run this notebook in [merlin-tensorflow](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/merlin/containers/merlin-tensorflow/tags) container." ] }, { @@ -65,47 +65,103 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: sagemaker==2.167.0 in /usr/local/lib/python3.8/dist-packages (2.167.0)\n", - "Requirement already satisfied: PyYAML==6.0 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (6.0)\n", - "Requirement already satisfied: attrs<24,>=23.1.0 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (23.1.0)\n", - "Requirement already satisfied: boto3<2.0,>=1.26.131 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (1.26.159)\n", - "Requirement already satisfied: cloudpickle==2.2.1 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (2.2.1)\n", - "Requirement already satisfied: google-pasta in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (0.2.0)\n", - "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (4.13.0)\n", - "Requirement already satisfied: jsonschema in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (4.17.3)\n", - "Requirement already satisfied: numpy<2.0,>=1.9.0 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (1.22.4)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (23.1)\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (1.5.2)\n", - "Requirement already satisfied: pathos in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (0.3.0)\n", - "Requirement already satisfied: platformdirs in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (3.8.0)\n", - "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (0.1.5)\n", - "Requirement already satisfied: protobuf<4.0,>=3.1 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (3.20.3)\n", - "Requirement already satisfied: schema in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (0.7.5)\n", - "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (1.0.1)\n", - "Requirement already satisfied: tblib==1.7.0 in /usr/local/lib/python3.8/dist-packages (from sagemaker==2.167.0) (1.7.0)\n", - "Requirement already satisfied: botocore<1.30.0,>=1.29.159 in /usr/local/lib/python3.8/dist-packages (from boto3<2.0,>=1.26.131->sagemaker==2.167.0) (1.29.159)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.8/dist-packages (from boto3<2.0,>=1.26.131->sagemaker==2.167.0) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from boto3<2.0,>=1.26.131->sagemaker==2.167.0) (0.6.1)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker==2.167.0) (3.15.0)\n", - "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker==2.167.0) (1.14.0)\n", - "Requirement already satisfied: importlib-resources>=1.4.0 in /usr/local/lib/python3.8/dist-packages (from jsonschema->sagemaker==2.167.0) (5.12.0)\n", - "Requirement already satisfied: pkgutil-resolve-name>=1.3.10 in /usr/local/lib/python3.8/dist-packages (from jsonschema->sagemaker==2.167.0) (1.3.10)\n", - "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.8/dist-packages (from jsonschema->sagemaker==2.167.0) (0.19.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.8/dist-packages (from pandas->sagemaker==2.167.0) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.8/dist-packages (from pandas->sagemaker==2.167.0) (2023.3)\n", - "Requirement already satisfied: ppft>=1.7.6.6 in /usr/local/lib/python3.8/dist-packages (from pathos->sagemaker==2.167.0) (1.7.6.6)\n", - "Requirement already satisfied: dill>=0.3.6 in /usr/local/lib/python3.8/dist-packages (from pathos->sagemaker==2.167.0) (0.3.6)\n", - "Requirement already satisfied: pox>=0.3.2 in /usr/local/lib/python3.8/dist-packages (from pathos->sagemaker==2.167.0) (0.3.2)\n", - "Requirement already satisfied: multiprocess>=0.70.14 in /usr/local/lib/python3.8/dist-packages (from pathos->sagemaker==2.167.0) (0.70.14)\n", - "Requirement already satisfied: contextlib2>=0.5.5 in /usr/local/lib/python3.8/dist-packages (from schema->sagemaker==2.167.0) (21.6.0)\n", - "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/lib/python3/dist-packages (from botocore<1.30.0,>=1.29.159->boto3<2.0,>=1.26.131->sagemaker==2.167.0) (1.25.8)\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" + "Requirement already satisfied: sagemaker in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (2.188.0)\n", + "Requirement already satisfied: attrs<24,>=23.1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (23.1.0)\n", + "Requirement already satisfied: boto3<2.0,>=1.26.131 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (1.28.57)\n", + "Requirement already satisfied: cloudpickle==2.2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (2.2.1)\n", + "Requirement already satisfied: google-pasta in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (0.2.0)\n", + "Requirement already satisfied: numpy<2.0,>=1.9.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (1.22.3)\n", + "Requirement already satisfied: protobuf<5.0,>=3.12 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (3.20.3)\n", + "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (1.0.1)\n", + "Requirement already satisfied: importlib-metadata<7.0,>=1.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (6.8.0)\n", + "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (21.3)\n", + "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (1.5.3)\n", + "Requirement already satisfied: pathos in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (0.3.1)\n", + "Requirement already satisfied: schema in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (0.7.5)\n", + "Requirement already satisfied: PyYAML~=6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (6.0)\n", + "Requirement already satisfied: jsonschema in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (4.18.4)\n", + "Requirement already satisfied: platformdirs in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (3.9.1)\n", + "Requirement already satisfied: tblib==1.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from sagemaker) (1.7.0)\n", + "Requirement already satisfied: botocore<1.32.0,>=1.31.57 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.31.57)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.8.0,>=0.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (0.7.0)\n", + "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker) (3.16.2)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from packaging>=20.0->sagemaker) (3.0.9)\n", + "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from google-pasta->sagemaker) (1.16.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from jsonschema->sagemaker) (2023.7.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from jsonschema->sagemaker) (0.30.0)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from jsonschema->sagemaker) (0.9.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas->sagemaker) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas->sagemaker) (2023.3)\n", + "Requirement already satisfied: ppft>=1.7.6.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pathos->sagemaker) (1.7.6.7)\n", + "Requirement already satisfied: dill>=0.3.7 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pathos->sagemaker) (0.3.7)\n", + "Requirement already satisfied: pox>=0.3.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pathos->sagemaker) (0.3.3)\n", + "Requirement already satisfied: multiprocess>=0.70.15 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pathos->sagemaker) (0.70.15)\n", + "Requirement already satisfied: contextlib2>=0.5.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from schema->sagemaker) (21.6.0)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from botocore<1.32.0,>=1.31.57->boto3<2.0,>=1.26.131->sagemaker) (1.26.14)\n", + "Requirement already satisfied: merlin-core==23.08 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (23.8.0)\n", + "Requirement already satisfied: merlin-dataloader==23.08 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (23.8.0)\n", + "Requirement already satisfied: nvtabular==23.08 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (23.8.0)\n", + "Requirement already satisfied: merlin-models==23.08 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (23.8.0)\n", + "Requirement already satisfied: merlin-systems==23.08 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (23.8.0)\n", + "Requirement already satisfied: dask>=2022.11.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (2023.9.2)\n", + "Requirement already satisfied: dask-cuda>=22.12.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (23.10.0)\n", + "Requirement already satisfied: distributed>=2022.11.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (2023.9.2)\n", + "Requirement already satisfied: fsspec>=2022.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (2023.6.0)\n", + "Requirement already satisfied: numpy>=1.22.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (1.22.3)\n", + "Requirement already satisfied: pandas<1.6.0dev0,>=1.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (1.5.3)\n", + "Requirement already satisfied: numba>=0.54 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (0.57.1)\n", + "Requirement already satisfied: pyarrow>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (12.0.1)\n", + "Requirement already satisfied: protobuf>=3.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (3.20.3)\n", + "Requirement already satisfied: tqdm>=4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (4.65.0)\n", + "Requirement already satisfied: tensorflow-metadata>=1.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (1.14.0)\n", + "Requirement already satisfied: betterproto<2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (1.2.5)\n", + "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (21.3)\n", + "Requirement already satisfied: npy-append-array in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (0.9.16)\n", + "Requirement already satisfied: pynvml<11.5,>=11.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-core==23.08) (11.4.1)\n", + "Requirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from nvtabular==23.08) (1.11.1)\n", + "Requirement already satisfied: requests<3,>=2.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-systems==23.08) (2.31.0)\n", + "Requirement already satisfied: treelite==2.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-systems==23.08) (2.4.0)\n", + "Requirement already satisfied: treelite-runtime==2.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from merlin-systems==23.08) (2.4.0)\n", + "Requirement already satisfied: grpclib in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from betterproto<2.0.0->merlin-core==23.08) (0.4.6)\n", + "Requirement already satisfied: stringcase in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from betterproto<2.0.0->merlin-core==23.08) (1.2.0)\n", + "Requirement already satisfied: click>=8.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask>=2022.11.1->merlin-core==23.08) (8.1.6)\n", + "Requirement already satisfied: cloudpickle>=1.5.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask>=2022.11.1->merlin-core==23.08) (2.2.1)\n", + "Requirement already satisfied: partd>=1.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask>=2022.11.1->merlin-core==23.08) (1.4.0)\n", + "Requirement already satisfied: pyyaml>=5.3.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask>=2022.11.1->merlin-core==23.08) (6.0)\n", + "Requirement already satisfied: toolz>=0.10.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask>=2022.11.1->merlin-core==23.08) (0.12.0)\n", + "Requirement already satisfied: importlib-metadata>=4.13.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask>=2022.11.1->merlin-core==23.08) (6.8.0)\n", + "Requirement already satisfied: zict>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from dask-cuda>=22.12.0->merlin-core==23.08) (3.0.0)\n", + "Requirement already satisfied: jinja2>=2.10.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (3.1.2)\n", + "Requirement already satisfied: locket>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (1.0.0)\n", + "Requirement already satisfied: msgpack>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (1.0.5)\n", + "Requirement already satisfied: psutil>=5.7.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (5.9.5)\n", + "Requirement already satisfied: sortedcontainers>=2.0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (2.4.0)\n", + "Requirement already satisfied: tblib>=1.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (1.7.0)\n", + "Requirement already satisfied: tornado>=6.0.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (6.3.2)\n", + "Requirement already satisfied: urllib3>=1.24.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from distributed>=2022.11.1->merlin-core==23.08) (1.26.14)\n", + "Requirement already satisfied: llvmlite<0.41,>=0.40.0dev0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from numba>=0.54->merlin-core==23.08) (0.40.1)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from packaging->merlin-core==23.08) (3.0.9)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas<1.6.0dev0,>=1.2.0->merlin-core==23.08) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from pandas<1.6.0dev0,>=1.2.0->merlin-core==23.08) (2023.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from requests<3,>=2.10->merlin-systems==23.08) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from requests<3,>=2.10->merlin-systems==23.08) (3.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from requests<3,>=2.10->merlin-systems==23.08) (2023.5.7)\n", + "Requirement already satisfied: absl-py<2.0.0,>=0.9 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from tensorflow-metadata>=1.2.0->merlin-core==23.08) (1.4.0)\n", + "Requirement already satisfied: googleapis-common-protos<2,>=1.52.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from tensorflow-metadata>=1.2.0->merlin-core==23.08) (1.61.0)\n", + "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from importlib-metadata>=4.13.0->dask>=2022.11.1->merlin-core==23.08) (3.16.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from jinja2>=2.10.3->distributed>=2022.11.1->merlin-core==23.08) (2.1.3)\n", + "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas<1.6.0dev0,>=1.2.0->merlin-core==23.08) (1.16.0)\n", + "Requirement already satisfied: h2<5,>=3.1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from grpclib->betterproto<2.0.0->merlin-core==23.08) (4.1.0)\n", + "Requirement already satisfied: multidict in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from grpclib->betterproto<2.0.0->merlin-core==23.08) (6.0.4)\n", + "Requirement already satisfied: hyperframe<7,>=6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from h2<5,>=3.1.0->grpclib->betterproto<2.0.0->merlin-core==23.08) (6.0.1)\n", + "Requirement already satisfied: hpack<5,>=4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages (from h2<5,>=3.1.0->grpclib->betterproto<2.0.0->merlin-core==23.08) (4.0.0)\n" ] } ], "source": [ - "! python -m pip install sagemaker" + "! python -m pip install sagemaker\n", + "#! python -m pip install merlin-core==23.08 merlin-dataloader==23.08 nvtabular==23.08 merlin-models==23.08 merlin-systems==23.08" ] }, { @@ -138,11 +194,12 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-07-03 03:20:13.448803: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2023-07-03 03:20:13.492697: I tensorflow/core/platform/cpu_feature_guard.cc:183] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "/usr/local/lib/python3.8/dist-packages/merlin/dtypes/mappings/torch.py:43: UserWarning: PyTorch dtype mappings did not load successfully due to an error: No module named 'torch'\n", - " warn(f\"PyTorch dtype mappings did not load successfully due to an error: {exc.msg}\")\n" + "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/merlin/dtypes/mappings/tf.py:52: UserWarning: Tensorflow dtype mappings did not load successfully due to an error: No module named 'tensorflow'\n", + " warn(f\"Tensorflow dtype mappings did not load successfully due to an error: {exc.msg}\")\n", + "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/merlin/dtypes/mappings/torch.py:43: UserWarning: PyTorch dtype mappings did not load successfully due to an error: No module named 'torch'\n", + " warn(f\"PyTorch dtype mappings did not load successfully due to an error: {exc.msg}\")\n", + "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/merlin/io/dataset.py:267: UserWarning: Initializing an NVTabular Dataset in CPU mode.This is an experimental feature with extremely limited support!\n", + " warnings.warn(\n" ] } ], @@ -151,7 +208,7 @@ "\n", "from merlin.datasets.synthetic import generate_data\n", "\n", - "DATA_FOLDER = os.environ.get(\"DATA_FOLDER\", \"/workspace/data/\")\n", + "DATA_FOLDER = os.environ.get(\"DATA_FOLDER\", \"./data/\")\n", "NUM_ROWS = os.environ.get(\"NUM_ROWS\", 1_000_000)\n", "SYNTHETIC_DATA = eval(os.environ.get(\"SYNTHETIC_DATA\", \"True\"))\n", "BATCH_SIZE = int(os.environ.get(\"BATCH_SIZE\", 512))\n", @@ -423,10 +480,11 @@ ], "source": [ "%%writefile container/Dockerfile\n", + "FROM nvcr.io/nvidia/merlin/merlin-tensorflow:23.08\n", "\n", - "FROM nvcr.io/nvidia/merlin/merlin-tensorflow:23.06\n", + "RUN pip3 install sagemaker-training\n", "\n", - "RUN pip3 install sagemaker-training" + "COPY --chown=1000:1000 serve /usr/bin/serve" ] }, { @@ -532,17 +590,16 @@ "id": "2b62f39e-af41-4aec-857f-0541038d9c5c", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Couldn't call 'get_role' to get Role ARN from role name AWSOS-AD-Engineer to get Role path.\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n", "arn:aws:iam::843263297212:role/AWSOS-AD-Engineer\n" ] } @@ -656,6 +713,8 @@ "name": "stdout", "output_type": "stream", "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n", "Using provided s3_resource\n" ] }, @@ -663,33 +722,29 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:sagemaker:Creating training-job with name: sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019\n" + "INFO:sagemaker:Creating training-job with name: sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-07-03 03:20:45 Starting - Starting the training job...\n", - "2023-07-03 03:20:59 Starting - Preparing the instances for training......\n", - "2023-07-03 03:21:59 Downloading - Downloading input data...\n", - "2023-07-03 03:22:19 Training - Downloading the training image........................\n", - "2023-07-03 03:26:41 Training - Training image download completed. Training in progress..\u001b[34m==================================\u001b[0m\n", + "2023-10-26 00:23:35 Starting - Starting the training job...\n", + "2023-10-26 00:23:51 Starting - Preparing the instances for training......\n", + "2023-10-26 00:25:02 Downloading - Downloading input data...\n", + "2023-10-26 00:25:27 Training - Downloading the training image.......................\u001b[34m==================================\u001b[0m\n", "\u001b[34m== Triton Inference Server Base ==\u001b[0m\n", "\u001b[34m==================================\u001b[0m\n", - "\u001b[34mNVIDIA Release 23.04 (build 58408265)\u001b[0m\n", + "\u001b[34mNVIDIA Release 23.06 (build 62878575)\u001b[0m\n", "\u001b[34mCopyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.\u001b[0m\n", "\u001b[34mVarious files include modifications (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.\u001b[0m\n", "\u001b[34mThis container image and its contents are governed by the NVIDIA Deep Learning Container License.\u001b[0m\n", "\u001b[34mBy pulling and using the container, you accept the terms and conditions of this license:\u001b[0m\n", "\u001b[34mhttps://developer.nvidia.com/ngc/nvidia-deep-learning-container-license\u001b[0m\n", - "\u001b[34mNOTE: CUDA Forward Compatibility mode ENABLED.\n", - " Using CUDA 12.1 driver version 530.30.02 with kernel driver version 525.85.12.\n", - " See https://docs.nvidia.com/deploy/cuda-compatibility/ for details.\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53,487 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53,523 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53,559 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53,572 sagemaker-training-toolkit INFO Invoking user script\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:21,913 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:21,947 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:21,979 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:21,992 sagemaker-training-toolkit INFO Invoking user script\u001b[0m\n", "\u001b[34mTraining Env:\u001b[0m\n", "\u001b[34m{\n", " \"additional_framework_parameters\": {},\n", @@ -743,11 +798,11 @@ " \"is_master\": true,\n", " \"is_modelparallel_enabled\": null,\n", " \"is_smddpmprun_installed\": false,\n", - " \"job_name\": \"sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019\",\n", + " \"job_name\": \"sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295\",\n", " \"log_level\": 20,\n", " \"master_hostname\": \"algo-1\",\n", " \"model_dir\": \"/opt/ml/model\",\n", - " \"module_dir\": \"s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019/source/sourcedir.tar.gz\",\n", + " \"module_dir\": \"s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295/source/sourcedir.tar.gz\",\n", " \"module_name\": \"train\",\n", " \"network_interface_name\": \"eth0\",\n", " \"num_cpus\": 4,\n", @@ -804,67 +859,68 @@ "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n", "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n", "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n", - "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019/source/sourcedir.tar.gz\u001b[0m\n", - "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"train\":\"/opt/ml/input/data/train\",\"valid\":\"/opt/ml/input/data/valid\"},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.g4dn.xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":null,\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch_size\":1024,\"epoch\":10},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"valid\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":false,\"job_name\":\"sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019/source/sourcedir.tar.gz\",\"module_name\":\"train\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.py\"}\u001b[0m\n", + "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295/source/sourcedir.tar.gz\u001b[0m\n", + "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"train\":\"/opt/ml/input/data/train\",\"valid\":\"/opt/ml/input/data/valid\"},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.g4dn.xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":null,\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch_size\":1024,\"epoch\":10},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"valid\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":false,\"job_name\":\"sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295/source/sourcedir.tar.gz\",\"module_name\":\"train\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.py\"}\u001b[0m\n", "\u001b[34mSM_USER_ARGS=[\"--batch_size\",\"1024\",\"--epoch\",\"10\"]\u001b[0m\n", "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n", "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n", "\u001b[34mSM_CHANNEL_VALID=/opt/ml/input/data/valid\u001b[0m\n", "\u001b[34mSM_HP_BATCH_SIZE=1024\u001b[0m\n", "\u001b[34mSM_HP_EPOCH=10\u001b[0m\n", - "\u001b[34mPYTHONPATH=/opt/ml/code:/usr/local/bin:/opt/tritonserver:/usr/local/lib/python3.8/dist-packages:/usr/lib/python38.zip:/usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages/faiss-1.7.2-py3.8.egg:/ptx:/usr/local/lib/python3.8/dist-packages/merlin_sok-1.2.0-py3.8-linux-x86_64.egg:/usr/local/lib/python3.8/dist-packages/merlin_hps-1.0.0-py3.8-linux-x86_64.egg:/usr/lib/python3/dist-packages:/usr/lib/python3.8/dist-packages\u001b[0m\n", + "\u001b[34mPYTHONPATH=/opt/ml/code:/usr/local/bin:/opt/tritonserver:/usr/local/lib/python3.10/dist-packages:/usr/lib/python310.zip:/usr/lib/python3.10:/usr/lib/python3.10/lib-dynload:/usr/local/lib/python3.10/dist-packages/faiss-1.7.2-py3.10.egg:/ptx:/usr/local/lib/python3.10/dist-packages/merlin_sok-1.2.0-py3.10-linux-x86_64.egg:/usr/local/lib/python3.10/dist-packages/merlin_hps-1.0.0-py3.10-linux-x86_64.egg:/usr/lib/python3/dist-packages:/usr/lib/python3.10/dist-packages\u001b[0m\n", "\u001b[34mInvoking script with the following command:\u001b[0m\n", "\u001b[34m/usr/bin/python3 train.py --batch_size 1024 --epoch 10\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53,573 sagemaker-training-toolkit INFO Exceptions not imported for SageMaker Debugger as it is not installed.\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53.751603: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\u001b[0m\n", - "\u001b[34m2023-07-03 03:26:53.808472: I tensorflow/core/platform/cpu_feature_guard.cc:183] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:21,993 sagemaker-training-toolkit INFO Exceptions not imported for SageMaker Debugger as it is not installed.\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:22.172037: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:22.226772: I tensorflow/core/platform/cpu_feature_guard.cc:183] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\u001b[0m\n", "\u001b[34mTo enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX, in other operations, rebuild TensorFlow with the appropriate compiler flags.\u001b[0m\n", - "\u001b[34m/usr/local/lib/python3.8/dist-packages/merlin/dtypes/mappings/torch.py:43: UserWarning: PyTorch dtype mappings did not load successfully due to an error: No module named 'torch'\n", + "\n", + "2023-10-26 00:29:13 Training - Training image download completed. Training in progress.\u001b[34m/usr/local/lib/python3.10/dist-packages/merlin/dtypes/mappings/torch.py:43: UserWarning: PyTorch dtype mappings did not load successfully due to an error: No module named 'torch'\n", " warn(f\"PyTorch dtype mappings did not load successfully due to an error: {exc.msg}\")\u001b[0m\n", "\u001b[34mWARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.data_structures has been moved to tensorflow.python.trackable.data_structures. The old module will be deleted in version 2.11.\u001b[0m\n", "\u001b[34mWARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11.\u001b[0m\n", - "\u001b[34m2023-07-03 03:27:05.042291: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:47] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\u001b[0m\n", + "\u001b[34m2023-10-26 00:29:30.531929: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:47] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\u001b[0m\n", "\u001b[34m[INFO]: sparse_operation_kit is imported\u001b[0m\n", - "\u001b[34m[SOK INFO] Import /usr/local/lib/python3.8/dist-packages/merlin_sok-1.2.0-py3.8-linux-x86_64.egg/sparse_operation_kit/lib/libsok_experiment.so\u001b[0m\n", - "\u001b[34m[SOK INFO] Import /usr/local/lib/python3.8/dist-packages/merlin_sok-1.2.0-py3.8-linux-x86_64.egg/sparse_operation_kit/lib/libsok_experiment.so\u001b[0m\n", + "\u001b[34m[SOK INFO] Import /usr/local/lib/python3.10/dist-packages/merlin_sok-1.2.0-py3.10-linux-x86_64.egg/sparse_operation_kit/lib/libsok_experiment.so\u001b[0m\n", + "\u001b[34m[SOK INFO] Import /usr/local/lib/python3.10/dist-packages/merlin_sok-1.2.0-py3.10-linux-x86_64.egg/sparse_operation_kit/lib/libsok_experiment.so\u001b[0m\n", "\u001b[34m[SOK INFO] Initialize finished, communication tool: horovod\u001b[0m\n", - "\u001b[34mWorkflow saved to /tmp/tmphhafd6ia/workflow.\u001b[0m\n", + "\u001b[34mWorkflow saved to /tmp/tmpo0dgd1_j/workflow.\u001b[0m\n", "\u001b[34mbatch_size = 1024, epochs = 10\u001b[0m\n", "\u001b[34mEpoch 1/10\u001b[0m\n", - "\u001b[34m684/684 - 15s - loss: 0.6932 - auc: 0.4993 - regularization_loss: 0.0000e+00 - loss_batch: 0.6931 - val_loss: 0.6931 - val_auc: 0.5000 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6931 - 15s/epoch - 22ms/step\u001b[0m\n", + "\u001b[34m684/684 - 15s - loss: 0.6932 - auc: 0.5000 - regularization_loss: 0.0000e+00 - loss_batch: 0.6931 - val_loss: 0.6931 - val_auc: 0.5000 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6932 - 15s/epoch - 22ms/step\u001b[0m\n", "\u001b[34mEpoch 2/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6932 - auc: 0.5019 - regularization_loss: 0.0000e+00 - loss_batch: 0.6932 - val_loss: 0.6932 - val_auc: 0.5000 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6929 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6932 - auc: 0.4992 - regularization_loss: 0.0000e+00 - loss_batch: 0.6932 - val_loss: 0.6932 - val_auc: 0.5007 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6930 - 8s/epoch - 12ms/step\u001b[0m\n", "\u001b[34mEpoch 3/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6923 - auc: 0.5212 - regularization_loss: 0.0000e+00 - loss_batch: 0.6918 - val_loss: 0.6939 - val_auc: 0.4998 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6946 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6931 - auc: 0.5043 - regularization_loss: 0.0000e+00 - loss_batch: 0.6930 - val_loss: 0.6932 - val_auc: 0.4992 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6928 - 8s/epoch - 12ms/step\u001b[0m\n", "\u001b[34mEpoch 4/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6858 - auc: 0.5491 - regularization_loss: 0.0000e+00 - loss_batch: 0.6895 - val_loss: 0.6988 - val_auc: 0.4992 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7044 - 8s/epoch - 11ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6916 - auc: 0.5279 - regularization_loss: 0.0000e+00 - loss_batch: 0.6920 - val_loss: 0.6945 - val_auc: 0.4992 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6923 - 8s/epoch - 11ms/step\u001b[0m\n", "\u001b[34mEpoch 5/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6791 - auc: 0.5649 - regularization_loss: 0.0000e+00 - loss_batch: 0.6835 - val_loss: 0.7026 - val_auc: 0.5000 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7072 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6843 - auc: 0.5551 - regularization_loss: 0.0000e+00 - loss_batch: 0.6859 - val_loss: 0.7006 - val_auc: 0.4997 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6933 - 8s/epoch - 12ms/step\u001b[0m\n", "\u001b[34mEpoch 6/10\u001b[0m\n", - "\u001b[34m684/684 - 9s - loss: 0.6753 - auc: 0.5713 - regularization_loss: 0.0000e+00 - loss_batch: 0.6780 - val_loss: 0.7089 - val_auc: 0.5001 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7231 - 9s/epoch - 13ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6780 - auc: 0.5685 - regularization_loss: 0.0000e+00 - loss_batch: 0.6825 - val_loss: 0.7065 - val_auc: 0.4995 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.6988 - 8s/epoch - 12ms/step\u001b[0m\n", "\u001b[34mEpoch 7/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6724 - auc: 0.5746 - regularization_loss: 0.0000e+00 - loss_batch: 0.6717 - val_loss: 0.7175 - val_auc: 0.4998 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7370 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6748 - auc: 0.5739 - regularization_loss: 0.0000e+00 - loss_batch: 0.6718 - val_loss: 0.7130 - val_auc: 0.4990 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7103 - 8s/epoch - 11ms/step\u001b[0m\n", "\u001b[34mEpoch 8/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6702 - auc: 0.5771 - regularization_loss: 0.0000e+00 - loss_batch: 0.6703 - val_loss: 0.7386 - val_auc: 0.5002 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7747 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6723 - auc: 0.5769 - regularization_loss: 0.0000e+00 - loss_batch: 0.6648 - val_loss: 0.7166 - val_auc: 0.4989 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7082 - 8s/epoch - 11ms/step\u001b[0m\n", "\u001b[34mEpoch 9/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6689 - auc: 0.5783 - regularization_loss: 0.0000e+00 - loss_batch: 0.6691 - val_loss: 0.7423 - val_auc: 0.4999 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7735 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 8s - loss: 0.6702 - auc: 0.5789 - regularization_loss: 0.0000e+00 - loss_batch: 0.6652 - val_loss: 0.7227 - val_auc: 0.4987 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7124 - 8s/epoch - 12ms/step\u001b[0m\n", "\u001b[34mEpoch 10/10\u001b[0m\n", - "\u001b[34m684/684 - 8s - loss: 0.6681 - auc: 0.5796 - regularization_loss: 0.0000e+00 - loss_batch: 0.6654 - val_loss: 0.7554 - val_auc: 0.5002 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.8033 - 8s/epoch - 12ms/step\u001b[0m\n", + "\u001b[34m684/684 - 9s - loss: 0.6688 - auc: 0.5802 - regularization_loss: 0.0000e+00 - loss_batch: 0.6631 - val_loss: 0.7403 - val_auc: 0.4993 - val_regularization_loss: 0.0000e+00 - val_loss_batch: 0.7173 - 9s/epoch - 13ms/step\u001b[0m\n", "\u001b[34mWARNING:absl:Found untraced functions such as model_context_layer_call_fn, model_context_layer_call_and_return_conditional_losses, prepare_list_features_layer_call_fn, prepare_list_features_layer_call_and_return_conditional_losses, output_layer_layer_call_fn while saving (showing 5 of 98). These functions will not be directly callable after loading.\u001b[0m\n", - "\u001b[34mINFO:__main__:Model saved to /tmp/tmphhafd6ia/dlrm.\u001b[0m\n", - "\u001b[34mModel saved to /tmp/tmphhafd6ia/dlrm.\u001b[0m\n", + "\u001b[34mModel saved to /tmp/tmpo0dgd1_j/dlrm.\u001b[0m\n", + "\u001b[34mINFO:__main__:Model saved to /tmp/tmpo0dgd1_j/dlrm.\u001b[0m\n", "\u001b[34mWARNING:absl:Found untraced functions such as model_context_layer_call_fn, model_context_layer_call_and_return_conditional_losses, prepare_list_features_layer_call_fn, prepare_list_features_layer_call_and_return_conditional_losses, output_layer_layer_call_fn while saving (showing 5 of 98). These functions will not be directly callable after loading.\u001b[0m\n", "\u001b[34mWARNING:absl:Found untraced functions such as model_context_layer_call_fn, model_context_layer_call_and_return_conditional_losses, prepare_list_features_layer_call_fn, prepare_list_features_layer_call_and_return_conditional_losses, output_layer_layer_call_fn while saving (showing 5 of 98). These functions will not be directly callable after loading.\u001b[0m\n", "\u001b[34mWARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\u001b[0m\n", "\u001b[34mWARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\u001b[0m\n", "\u001b[34mEnsemble graph saved to /opt/ml/model.\u001b[0m\n", "\u001b[34mINFO:__main__:Ensemble graph saved to /opt/ml/model.\u001b[0m\n", - "\u001b[34m2023-07-03 03:29:29,126 sagemaker-training-toolkit INFO Reporting training SUCCESS\u001b[0m\n", + "\u001b[34m2023-10-26 00:31:48,854 sagemaker-training-toolkit INFO Reporting training SUCCESS\u001b[0m\n", "\n", - "2023-07-03 03:29:37 Uploading - Uploading generated training model\n", - "2023-07-03 03:29:48 Completed - Training job completed\n", - "Training seconds: 468\n", - "Billable seconds: 468\n" + "2023-10-26 00:32:05 Uploading - Uploading generated training model\n", + "2023-10-26 00:32:05 Completed - Training job completed\n", + "Training seconds: 423\n", + "Billable seconds: 423\n" ] } ], @@ -905,7 +961,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-07-03-03-20-43-019/output/model.tar.gz\n" + "s3://sagemaker-us-east-1-843263297212/sagemaker-merlin-tensorflow-2023-10-26-00-23-35-295/output/model.tar.gz\n" ] } ], @@ -919,6 +975,14 @@ "id": "c5d6d979-1976-46dd-bf88-669bbad39ede", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n" + ] + }, { "data": { "text/plain": [ @@ -946,51 +1010,96 @@ "name": "stdout", "output_type": "stream", "text": [ + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/config.pbtxt\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/variables/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/variables/variables.data-00000-of-00001\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/variables/variables.index\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/saved_model.pb\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/keras_metadata.pb\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/assets/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/.merlin/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/.merlin/input_schema.json\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/.merlin/output_schema.json\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "1_predicttensorflowtriton/1/model.savedmodel/fingerprint.pb\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/config.pbtxt\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/\n", - "0_transformworkflowtriton/1/model.py\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/\n", - "0_transformworkflowtriton/1/workflow/workflow.pkl\n", - "0_transformworkflowtriton/1/workflow/metadata.json\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_is_occupied.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.item_shop.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.user_consumption_2.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.user_intentions.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_geography.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_group.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_shops.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_gender.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_categories.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_profile.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.item_id.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_shops.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.item_brand.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.user_id.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_brands.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.item_shop.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_group.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.item_category.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "0_transformworkflowtriton/1/workflow/categories/unique.user_age.parquet\n", - "0_transformworkflowtriton/1/workflow/categories/unique.user_profile.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_gender.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_geography.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_is_occupied.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_brands.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/categories/unique.user_categories.parquet\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/workflow.pkl\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/workflow/metadata.json\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "0_transformworkflowtriton/1/model.py\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "executor_model/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "executor_model/config.pbtxt\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "executor_model/1/\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "executor_model/1/model.py\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "executor_model/1/ensemble/\n", - "executor_model/1/ensemble/metadata.json\n", + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", "executor_model/1/ensemble/ensemble.pkl\n", - "1_predicttensorflowtriton/\n", - "1_predicttensorflowtriton/config.pbtxt\n", - "1_predicttensorflowtriton/1/\n", - "1_predicttensorflowtriton/1/model.savedmodel/\n", - "1_predicttensorflowtriton/1/model.savedmodel/variables/\n", - "1_predicttensorflowtriton/1/model.savedmodel/variables/variables.data-00000-of-00001\n", - "1_predicttensorflowtriton/1/model.savedmodel/variables/variables.index\n", - "1_predicttensorflowtriton/1/model.savedmodel/fingerprint.pb\n", - "1_predicttensorflowtriton/1/model.savedmodel/keras_metadata.pb\n", - "1_predicttensorflowtriton/1/model.savedmodel/saved_model.pb\n", - "1_predicttensorflowtriton/1/model.savedmodel/assets/\n", - "1_predicttensorflowtriton/1/model.savedmodel/.merlin/\n", - "1_predicttensorflowtriton/1/model.savedmodel/.merlin/input_schema.json\n", - "1_predicttensorflowtriton/1/model.savedmodel/.merlin/output_schema.json\n" + "tar: Ignoring unknown extended header keyword `LIBARCHIVE.creationtime'\n", + "executor_model/1/ensemble/metadata.json\n" ] } ], @@ -1007,48 +1116,52 @@ "\n", "Although we use the Sagemaker Python SDK to train our model, here we will use `boto3` to launch our inference endpoint as it offers more low-level control than the Python SDK.\n", "\n", - "The model artificat `model.tar.gz` uploaded to S3 from the Sagemaker training job contained three directories: `0_transformworkflow` for the NVTabular workflow, `1_predicttensorflow` for the Tensorflow model, and `executor_model` for the ensemble graph that we can use in Triton.\n", + "The model artificat `model.tar.gz` uploaded to S3 from the Sagemaker training job contained three directories: `0_transformworkflowtriton` for the NVTabular workflow, `1_predicttensorflowtriton` for the Tensorflow model, and `executor_model` for the ensemble graph that we can use in Triton.\n", "\n", "```shell\n", "/tmp/ensemble/\n", - "├── 0_transformworkflow\n", - "│ ├── 1\n", - "│ │ ├── model.py\n", - "│ │ └── workflow\n", - "│ │ ├── categories\n", - "│ │ │ ├── unique.item_brand.parquet\n", - "│ │ │ ├── unique.item_category.parquet\n", - "│ │ │ ├── unique.item_id.parquet\n", - "│ │ │ ├── unique.item_shop.parquet\n", - "│ │ │ ├── unique.user_age.parquet\n", - "│ │ │ ├── unique.user_brands.parquet\n", - "│ │ │ ├── unique.user_categories.parquet\n", - "│ │ │ ├── unique.user_consumption_2.parquet\n", - "│ │ │ ├── unique.user_gender.parquet\n", - "│ │ │ ├── unique.user_geography.parquet\n", - "│ │ │ ├── unique.user_group.parquet\n", - "│ │ │ ├── unique.user_id.parquet\n", - "│ │ │ ├── unique.user_intentions.parquet\n", - "│ │ │ ├── unique.user_is_occupied.parquet\n", - "│ │ │ ├── unique.user_profile.parquet\n", - "│ │ │ └── unique.user_shops.parquet\n", - "│ │ ├── metadata.json\n", - "│ │ └── workflow.pkl\n", - "│ └── config.pbtxt\n", - "├── 1_predicttensorflow\n", - "│ ├── 1\n", - "│ │ └── model.savedmodel\n", - "│ │ ├── assets\n", - "│ │ ├── keras_metadata.pb\n", - "│ │ ├── saved_model.pb\n", - "│ │ └── variables\n", - "│ │ ├── variables.data-00000-of-00001\n", - "│ │ └── variables.index\n", - "│ └── config.pbtxt\n", - "├── executor_model\n", - "│ ├── 1\n", - "│ └── config.pbtxt\n", - "└── model.tar.gz\n", + "├── 0_transformworkflowtriton\n", + "│   ├── 1\n", + "│   │   ├── model.py\n", + "│   │   └── workflow\n", + "│   │   ├── categories\n", + "│   │   │   ├── unique.item_brand.parquet\n", + "│   │   │   ├── unique.item_category.parquet\n", + "│   │   │   ├── unique.item_id.parquet\n", + "│   │   │   ├── unique.item_shop.parquet\n", + "│   │   │   ├── unique.user_age.parquet\n", + "│   │   │   ├── unique.user_brands.parquet\n", + "│   │   │   ├── unique.user_categories.parquet\n", + "│   │   │   ├── unique.user_consumption_2.parquet\n", + "│   │   │   ├── unique.user_gender.parquet\n", + "│   │   │   ├── unique.user_geography.parquet\n", + "│   │   │   ├── unique.user_group.parquet\n", + "│   │   │   ├── unique.user_id.parquet\n", + "│   │   │   ├── unique.user_intentions.parquet\n", + "│   │   │   ├── unique.user_is_occupied.parquet\n", + "│   │   │   ├── unique.user_profile.parquet\n", + "│   │   │   └── unique.user_shops.parquet\n", + "│   │   ├── metadata.json\n", + "│   │   └── workflow.pkl\n", + "│   └── config.pbtxt\n", + "├── 1_predicttensorflowtriton\n", + "│   ├── 1\n", + "│   │   └── model.savedmodel\n", + "│   │   ├── assets\n", + "│   │   ├── fingerprint.pb\n", + "│   │   ├── keras_metadata.pb\n", + "│   │   ├── saved_model.pb\n", + "│   │   └── variables\n", + "│   │   ├── variables.data-00000-of-00001\n", + "│   │   └── variables.index\n", + "│   └── config.pbtxt\n", + "└── executor_model\n", + " ├── 1\n", + " │   ├── ensemble\n", + " │   │   ├── ensemble.pkl\n", + " │   │   └── metadata.json\n", + " │   └── model.py\n", + " └── config.pbtxt\n", "```\n", "\n", "We specify that we only want to use `executor_model` in Triton by passing the environment variable `SAGEMAKER_TRITON_DEFAULT_MODEL_NAME`." @@ -1064,7 +1177,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Model Arn: arn:aws:sagemaker:us-east-1:843263297212:model/model-triton-merlin-ensemble-2023-07-03-03-30-31\n" + "Model Arn: arn:aws:sagemaker:us-east-1:843263297212:model/model-triton-merlin-ensemble-2023-10-26-00-32-25\n" ] } ], @@ -1115,7 +1228,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Endpoint Config Arn: arn:aws:sagemaker:us-east-1:843263297212:endpoint-config/endpoint-config-triton-merlin-ensemble-2023-07-03-03-30-33\n" + "Endpoint Config Arn: arn:aws:sagemaker:us-east-1:843263297212:endpoint-config/endpoint-config-triton-merlin-ensemble-2023-10-26-00-32-26\n" ] } ], @@ -1154,7 +1267,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Endpoint Arn: arn:aws:sagemaker:us-east-1:843263297212:endpoint/endpoint-triton-merlin-ensemble-2023-07-03-03-30-33\n" + "Endpoint Arn: arn:aws:sagemaker:us-east-1:843263297212:endpoint/endpoint-triton-merlin-ensemble-2023-10-26-00-32-27\n" ] } ], @@ -1190,7 +1303,7 @@ "Endpoint Creation Status: Creating\n", "Endpoint Creation Status: Creating\n", "Endpoint Creation Status: InService\n", - "Endpoint Arn: arn:aws:sagemaker:us-east-1:843263297212:endpoint/endpoint-triton-merlin-ensemble-2023-07-03-03-30-33\n", + "Endpoint Arn: arn:aws:sagemaker:us-east-1:843263297212:endpoint/endpoint-triton-merlin-ensemble-2023-10-26-00-32-27\n", "Endpoint Status: InService\n" ] } @@ -1234,29 +1347,29 @@ "text": [ " item_category item_shop item_brand user_shops \\\n", "__null_dask_index__ \n", - "700000 54 3772 1299 153 \n", - "700001 108 7543 2598 2394 \n", - "700002 11 719 248 2904 \n", - "700003 8 539 186 1834 \n", - "700004 67 4670 1609 306 \n", - "700005 102 7184 2474 3006 \n", - "700006 74 5209 1794 561 \n", - "700007 67 4670 1609 1885 \n", - "700008 41 2874 990 561 \n", - "700009 34 2335 805 2496 \n", + "700000 147 10342 3562 235 \n", + "700001 84 5876 2024 141 \n", + "700002 147 10342 3562 1361 \n", + "700003 244 17158 5909 1033 \n", + "700004 134 9402 3238 71 \n", + "700005 64 4466 1538 681 \n", + "700006 24 1646 567 165 \n", + "700007 14 941 324 188 \n", + "700008 14 941 324 1596 \n", + "700009 64 4466 1538 962 \n", "\n", " user_profile user_group user_gender user_age \\\n", "__null_dask_index__ \n", "700000 1 1 1 1 \n", - "700001 2 1 1 1 \n", - "700002 3 1 1 1 \n", - "700003 2 1 1 1 \n", + "700001 1 1 1 1 \n", + "700002 2 1 1 1 \n", + "700003 1 1 1 1 \n", "700004 1 1 1 1 \n", - "700005 3 1 1 1 \n", + "700005 1 1 1 1 \n", "700006 1 1 1 1 \n", - "700007 2 1 1 1 \n", - "700008 1 1 1 1 \n", - "700009 3 1 1 1 \n", + "700007 1 1 1 1 \n", + "700008 2 1 1 1 \n", + "700009 1 1 1 1 \n", "\n", " user_consumption_2 user_is_occupied user_geography \\\n", "__null_dask_index__ \n", @@ -1273,29 +1386,37 @@ "\n", " user_intentions user_brands user_categories user_id \\\n", "__null_dask_index__ \n", - "700000 45 76 8 4 \n", - "700001 693 1190 125 48 \n", - "700002 841 1443 152 58 \n", - "700003 531 912 96 37 \n", - "700004 89 152 16 7 \n", - "700005 870 1494 157 60 \n", - "700006 163 279 30 12 \n", - "700007 546 937 99 38 \n", - "700008 163 279 30 12 \n", - "700009 723 1241 131 50 \n", + "700000 68 117 13 11 \n", + "700001 41 70 8 7 \n", + "700002 394 677 71 59 \n", + "700003 299 513 54 45 \n", + "700004 21 35 4 4 \n", + "700005 197 339 36 30 \n", + "700006 48 82 9 8 \n", + "700007 55 94 10 9 \n", + "700008 462 793 84 69 \n", + "700009 279 479 51 42 \n", "\n", " item_id \n", "__null_dask_index__ \n", - "700000 22 \n", - "700001 43 \n", - "700002 5 \n", - "700003 4 \n", - "700004 27 \n", - "700005 41 \n", - "700006 30 \n", - "700007 27 \n", - "700008 17 \n", - "700009 14 \n" + "700000 45 \n", + "700001 26 \n", + "700002 45 \n", + "700003 74 \n", + "700004 41 \n", + "700005 20 \n", + "700006 8 \n", + "700007 5 \n", + "700008 5 \n", + "700009 20 \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/nvtabular/workflow/workflow.py:445: UserWarning: Loading workflow generated on GPU\n", + " warnings.warn(f\"Loading workflow generated on {expected}\")\n" ] } ], @@ -1339,7 +1460,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "b'{\"inputs\":[{\"name\":\"item_category\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"item_shop\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"item_brand\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_shops\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_profile\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_group\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_gender\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_age\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_consumption_2\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_is_occupied\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_geography\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_intentions\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_brands\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_categories\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_id\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"item_id\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}}],\"parameters\":{\"binary_data_output\":true}}6\\x00\\x00\\x00l\\x00\\x00\\x00\\x0b\\x00\\x00\\x00\\x08\\x00\\x00\\x00C\\x00\\x00\\x00f\\x00\\x00\\x00J\\x00\\x00\\x00C\\x00\\x00\\x00)\\x00\\x00\\x00\"\\x00\\x00\\x00\\xbc\\x0e\\x00\\x00w\\x1d\\x00\\x00\\xcf\\x02\\x00\\x00\\x1b\\x02\\x00\\x00>\\x12\\x00\\x00\\x10\\x1c\\x00\\x00Y\\x14\\x00\\x00>\\x12\\x00\\x00:\\x0b\\x00\\x00\\x1f\\t\\x00\\x00\\x13\\x05\\x00\\x00&\\n\\x00\\x00\\xf8\\x00\\x00\\x00\\xba\\x00\\x00\\x00I\\x06\\x00\\x00\\xaa\\t\\x00\\x00\\x02\\x07\\x00\\x00I\\x06\\x00\\x00\\xde\\x03\\x00\\x00%\\x03\\x00\\x00\\x99\\x00\\x00\\x00Z\\t\\x00\\x00X\\x0b\\x00\\x00*\\x07\\x00\\x002\\x01\\x00\\x00\\xbe\\x0b\\x00\\x001\\x02\\x00\\x00]\\x07\\x00\\x001\\x02\\x00\\x00\\xc0\\t\\x00\\x00\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00-\\x00\\x00\\x00\\xb5\\x02\\x00\\x00I\\x03\\x00\\x00\\x13\\x02\\x00\\x00Y\\x00\\x00\\x00f\\x03\\x00\\x00\\xa3\\x00\\x00\\x00\"\\x02\\x00\\x00\\xa3\\x00\\x00\\x00\\xd3\\x02\\x00\\x00L\\x00\\x00\\x00\\xa6\\x04\\x00\\x00\\xa3\\x05\\x00\\x00\\x90\\x03\\x00\\x00\\x98\\x00\\x00\\x00\\xd6\\x05\\x00\\x00\\x17\\x01\\x00\\x00\\xa9\\x03\\x00\\x00\\x17\\x01\\x00\\x00\\xd9\\x04\\x00\\x00\\x08\\x00\\x00\\x00}\\x00\\x00\\x00\\x98\\x00\\x00\\x00`\\x00\\x00\\x00\\x10\\x00\\x00\\x00\\x9d\\x00\\x00\\x00\\x1e\\x00\\x00\\x00c\\x00\\x00\\x00\\x1e\\x00\\x00\\x00\\x83\\x00\\x00\\x00\\x04\\x00\\x00\\x000\\x00\\x00\\x00:\\x00\\x00\\x00%\\x00\\x00\\x00\\x07\\x00\\x00\\x00<\\x00\\x00\\x00\\x0c\\x00\\x00\\x00&\\x00\\x00\\x00\\x0c\\x00\\x00\\x002\\x00\\x00\\x00\\x16\\x00\\x00\\x00+\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x1b\\x00\\x00\\x00)\\x00\\x00\\x00\\x1e\\x00\\x00\\x00\\x1b\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x0e\\x00\\x00\\x00'\n", + "b'{\"inputs\":[{\"name\":\"item_category\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"item_shop\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"item_brand\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_shops\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_profile\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_group\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_gender\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_age\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_consumption_2\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_is_occupied\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_geography\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_intentions\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_brands\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_categories\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"user_id\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}},{\"name\":\"item_id\",\"shape\":[10],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":40}}],\"parameters\":{\"binary_data_output\":true}}\\x93\\x00\\x00\\x00T\\x00\\x00\\x00\\x93\\x00\\x00\\x00\\xf4\\x00\\x00\\x00\\x86\\x00\\x00\\x00@\\x00\\x00\\x00\\x18\\x00\\x00\\x00\\x0e\\x00\\x00\\x00\\x0e\\x00\\x00\\x00@\\x00\\x00\\x00f(\\x00\\x00\\xf4\\x16\\x00\\x00f(\\x00\\x00\\x06C\\x00\\x00\\xba$\\x00\\x00r\\x11\\x00\\x00n\\x06\\x00\\x00\\xad\\x03\\x00\\x00\\xad\\x03\\x00\\x00r\\x11\\x00\\x00\\xea\\r\\x00\\x00\\xe8\\x07\\x00\\x00\\xea\\r\\x00\\x00\\x15\\x17\\x00\\x00\\xa6\\x0c\\x00\\x00\\x02\\x06\\x00\\x007\\x02\\x00\\x00D\\x01\\x00\\x00D\\x01\\x00\\x00\\x02\\x06\\x00\\x00\\xeb\\x00\\x00\\x00\\x8d\\x00\\x00\\x00Q\\x05\\x00\\x00\\t\\x04\\x00\\x00G\\x00\\x00\\x00\\xa9\\x02\\x00\\x00\\xa5\\x00\\x00\\x00\\xbc\\x00\\x00\\x00<\\x06\\x00\\x00\\xc2\\x03\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x00\\x00D\\x00\\x00\\x00)\\x00\\x00\\x00\\x8a\\x01\\x00\\x00+\\x01\\x00\\x00\\x15\\x00\\x00\\x00\\xc5\\x00\\x00\\x000\\x00\\x00\\x007\\x00\\x00\\x00\\xce\\x01\\x00\\x00\\x17\\x01\\x00\\x00u\\x00\\x00\\x00F\\x00\\x00\\x00\\xa5\\x02\\x00\\x00\\x01\\x02\\x00\\x00#\\x00\\x00\\x00S\\x01\\x00\\x00R\\x00\\x00\\x00^\\x00\\x00\\x00\\x19\\x03\\x00\\x00\\xdf\\x01\\x00\\x00\\r\\x00\\x00\\x00\\x08\\x00\\x00\\x00G\\x00\\x00\\x006\\x00\\x00\\x00\\x04\\x00\\x00\\x00$\\x00\\x00\\x00\\t\\x00\\x00\\x00\\n\\x00\\x00\\x00T\\x00\\x00\\x003\\x00\\x00\\x00\\x0b\\x00\\x00\\x00\\x07\\x00\\x00\\x00;\\x00\\x00\\x00-\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x1e\\x00\\x00\\x00\\x08\\x00\\x00\\x00\\t\\x00\\x00\\x00E\\x00\\x00\\x00*\\x00\\x00\\x00-\\x00\\x00\\x00\\x1a\\x00\\x00\\x00-\\x00\\x00\\x00J\\x00\\x00\\x00)\\x00\\x00\\x00\\x14\\x00\\x00\\x00\\x08\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x14\\x00\\x00\\x00'\n", "1535\n" ] } @@ -1381,16 +1502,20 @@ "metadata": {}, "outputs": [ { - "ename": "ModelError", - "evalue": "An error occurred (ModelError) when calling the InvokeEndpoint operation: Received client error (400) from primary and could not load the entire response body. See https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/aws/sagemaker/Endpoints/endpoint-triton-merlin-ensemble-2023-07-03-03-30-33 in account 843263297212 for more information.", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModelError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[21], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m runtime_sm_client \u001b[38;5;241m=\u001b[39m boto3\u001b[38;5;241m.\u001b[39mclient(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msagemaker-runtime\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mruntime_sm_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke_endpoint\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mEndpointName\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mendpoint_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mContentType\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mapplication/vnd.sagemaker-triton.binary+json;json-header-size=\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mheader_length\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mBody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_body\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;66;03m# Parse json header size length from the response\u001b[39;00m\n\u001b[1;32m 10\u001b[0m header_length_prefix \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/vnd.sagemaker-triton.binary+json;json-header-size=\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", - "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/botocore/client.py:530\u001b[0m, in \u001b[0;36mClientCreator._create_api_method.._api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 527\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpy_operation_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m() only accepts keyword arguments.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 528\u001b[0m )\n\u001b[1;32m 529\u001b[0m \u001b[38;5;66;03m# The \"self\" in this scope is referring to the BaseClient.\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_api_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43moperation_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/botocore/client.py:964\u001b[0m, in \u001b[0;36mBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 962\u001b[0m error_code \u001b[38;5;241m=\u001b[39m parsed_response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCode\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 963\u001b[0m error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 964\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 966\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parsed_response\n", - "\u001b[0;31mModelError\u001b[0m: An error occurred (ModelError) when calling the InvokeEndpoint operation: Received client error (400) from primary and could not load the entire response body. See https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/aws/sagemaker/Endpoints/endpoint-triton-merlin-ensemble-2023-07-03-03-30-33 in account 843263297212 for more information." + "name": "stdout", + "output_type": "stream", + "text": [ + "predicted sigmoid result:\n", + " [[0.48622972]\n", + " [0.52633965]\n", + " [0.46211788]\n", + " [0.6060424 ]\n", + " [0.46786073]\n", + " [0.47899282]\n", + " [0.48058835]\n", + " [0.5024603 ]\n", + " [0.565173 ]\n", + " [0.46900135]]\n" ] } ], @@ -1411,7 +1536,7 @@ "result = httpclient.InferenceServerClient.parse_response_body(\n", " response[\"Body\"].read(), header_length=int(header_length_str)\n", ")\n", - "print(result)\n", + "\n", "output_data = result.as_numpy(\"click/binary_classification_task\")\n", "print(\"predicted sigmoid result:\\n\", output_data)" ] @@ -1428,30 +1553,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "2ce1630c-96aa-49ee-ac79-cf44120adb78", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'ResponseMetadata': {'RequestId': 'c12a4eca-a162-48cd-9787-cf5f3525bc7e',\n", + " 'HTTPStatusCode': 200,\n", + " 'HTTPHeaders': {'x-amzn-requestid': 'c12a4eca-a162-48cd-9787-cf5f3525bc7e',\n", + " 'content-type': 'application/x-amz-json-1.1',\n", + " 'content-length': '0',\n", + " 'date': 'Thu, 26 Oct 2023 00:39:31 GMT'},\n", + " 'RetryAttempts': 0}}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sm_client.delete_model(ModelName=model_name)\n", "sm_client.delete_endpoint_config(EndpointConfigName=endpoint_config_name)\n", "sm_client.delete_endpoint(EndpointName=endpoint_name)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e2ef06b3-fa31-4148-97e7-3805c3a7bf91", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "conda_python3", "language": "python", - "name": "python3" + "name": "conda_python3" }, "language_info": { "codemirror_mode": { @@ -1463,7 +1597,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.12" } }, "nbformat": 4,