From c64970bd677afddb983f82f607016bbc16c3e3cc Mon Sep 17 00:00:00 2001 From: "Santiago L. Valdarrama" Date: Tue, 21 May 2024 20:37:46 -0400 Subject: [PATCH] Fixed issue with libraries in system path --- program/cohort.ipynb | 1032 ++++++++++++++++-------------------------- 1 file changed, 386 insertions(+), 646 deletions(-) diff --git a/program/cohort.ipynb b/program/cohort.ipynb index 11d6902..ff48d83 100644 --- a/program/cohort.ipynb +++ b/program/cohort.ipynb @@ -68,6 +68,7 @@ "import ipytest\n", "\n", "CODE_FOLDER = Path(\"code\")\n", + "CODE_FOLDER.mkdir(exist_ok=True)\n", "sys.path.extend([f\"./{CODE_FOLDER}\"])\n", "\n", "DATA_FILEPATH = \"penguins.csv\"\n", @@ -1187,13 +1188,12 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "78fe069e", "metadata": {}, "outputs": [], "source": [ - "(CODE_FOLDER / \"processing\").mkdir(parents=True, exist_ok=True)\n", - "sys.path.extend([f\"./{CODE_FOLDER}/processing\"])" + "(CODE_FOLDER / \"processing\").mkdir(parents=True, exist_ok=True)" ] }, { @@ -1206,7 +1206,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "id": "fb6ba7c0-1bd6-4fe5-8b7f-f6cbdfd3846c", "metadata": { "tags": [ @@ -1218,7 +1218,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/processing/script.py\n" + "Writing code/processing/script.py\n" ] } ], @@ -1441,7 +1441,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "d1f122a4-acff-4687-91b9-bfef13567d88", "metadata": { "tags": [ @@ -1454,7 +1454,7 @@ "output_type": "stream", "text": [ "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.13s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.14s\u001b[0m\u001b[0m\n" ] } ], @@ -1578,7 +1578,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "id": "d88e9ccf", "metadata": {}, "outputs": [], @@ -1600,7 +1600,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "id": "331fe373", "metadata": {}, "outputs": [], @@ -1636,7 +1636,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "id": "3aa4471a", "metadata": { "tags": [ @@ -1675,7 +1675,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "id": "cdbd9303", "metadata": { "tags": [ @@ -1755,7 +1755,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "id": "e140642a", "metadata": { "tags": [ @@ -1769,7 +1769,7 @@ "{'PipelineArn': 'session3-pipeline'}" ] }, - "execution_count": 27, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1824,13 +1824,12 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "id": "ac0b891c", "metadata": {}, "outputs": [], "source": [ - "(CODE_FOLDER / \"training\").mkdir(parents=True, exist_ok=True)\n", - "sys.path.extend([f\"./{CODE_FOLDER}/training\"])" + "(CODE_FOLDER / \"training\").mkdir(parents=True, exist_ok=True)" ] }, { @@ -1843,7 +1842,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "id": "d92b121d-dcb9-43e8-9ee3-3ececb583e7e", "metadata": { "tags": [ @@ -1855,7 +1854,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/training/script.py\n" + "Writing code/training/script.py\n" ] } ], @@ -2020,7 +2019,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "id": "14ea27ce-c453-4cb0-b309-dbecd732957e", "metadata": { "tags": [ @@ -2040,16 +2039,16 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.0608 - accuracy: 0.3389 - val_loss: 0.9358 - val_accuracy: 0.4510 - 285ms/epoch - 36ms/step\n", - "2/2 [==============================] - 0s 1ms/step\n", - "Validation accuracy: 0.45098039215686275\n" + "8/8 - 0s - loss: 1.2194 - accuracy: 0.4435 - val_loss: 1.1648 - val_accuracy: 0.4314 - 184ms/epoch - 23ms/step\n", + "2/2 [==============================] - 0s 913us/step\n", + "Validation accuracy: 0.43137254901960786\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp_0t_ovx_/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp98oimg1w/model/001/assets\n" ] }, { @@ -2071,24 +2070,24 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.1533 - accuracy: 0.1172 - val_loss: 1.1482 - val_accuracy: 0.0588 - 220ms/epoch - 27ms/step\n", - "2/2 [==============================] - 0s 1ms/step\n", - "Validation accuracy: 0.058823529411764705\n" + "8/8 - 0s - loss: 1.2077 - accuracy: 0.4142 - val_loss: 1.1445 - val_accuracy: 0.4706 - 168ms/epoch - 21ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpcerqj9uj/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmplqxil_9g/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ + "Validation accuracy: 0.47058823529411764\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.31s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 0.85s\u001b[0m\u001b[0m\n" ] } ], @@ -2168,7 +2167,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "id": "d00eda86", "metadata": { "tags": [ @@ -2180,7 +2179,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/training/requirements.txt\n" + "Writing code/training/requirements.txt\n" ] } ], @@ -2212,7 +2211,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "id": "90fe82ae-6a2c-4461-bc83-bb52d8871e3b", "metadata": { "tags": [] @@ -2278,7 +2277,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "id": "99e4850c-83d6-4f4e-a813-d5a3f4bb7486", "metadata": { "tags": [ @@ -2345,7 +2344,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "id": "d92cac3d", "metadata": { "tags": [ @@ -2359,7 +2358,7 @@ "{'PipelineArn': 'session4-pipeline'}" ] }, - "execution_count": 34, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -2403,7 +2402,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "id": "57c2915c", "metadata": {}, "outputs": [ @@ -2413,7 +2412,7 @@ "PosixPath('code/containers/training/train.py')" ] }, - "execution_count": 35, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -2440,7 +2439,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "id": "3ffa9250", "metadata": { "tags": [ @@ -2452,7 +2451,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/containers/training/requirements.txt\n" + "Writing code/containers/training/requirements.txt\n" ] } ], @@ -2482,7 +2481,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "id": "de5d473c", "metadata": { "tags": [ @@ -2494,7 +2493,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/containers/training/Dockerfile\n" + "Writing code/containers/training/Dockerfile\n" ] } ], @@ -2536,7 +2535,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "id": "e37a06ac", "metadata": { "tags": [ @@ -2549,57 +2548,7 @@ "output_type": "stream", "text": [ "Building Docker image for arm64 architecture...\n", - "\u001b[1A\u001b[1B\u001b[0G\u001b[?25l[+] Building 0.0s (0/1) \n", - "\u001b[?25h\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.1s (2/3) \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.1s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.3s (2/3) \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.2s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.5s (2/3) \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.4s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.6s (2/3) \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.5s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.7s (3/3) \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load metadata for docker.io/library/python:3.10-slim 0.6s\n", - "\u001b[0m\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.8s (11/11) FINISHED \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load metadata for docker.io/library/python:3.10-slim 0.6s\n", - "\u001b[0m\u001b[34m => [1/6] FROM docker.io/library/python:3.10-slim@sha256:f80e619e067eebe4 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load build context 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 4.94kB 0.0s\n", - "\u001b[0m\u001b[34m => CACHED [2/6] RUN apt-get -y update && apt-get install -y --no-install 0.0s\n", - "\u001b[0m\u001b[34m => CACHED [3/6] COPY requirements.txt . 0.0s\n", - "\u001b[0m\u001b[34m => CACHED [4/6] RUN pip install --user --upgrade pip 0.0s\n", - "\u001b[0m\u001b[34m => CACHED [5/6] RUN pip3 install -r requirements.txt 0.0s\n", - "\u001b[0m\u001b[34m => [6/6] COPY train.py /opt/ml/code/train.py 0.0s\n", - "\u001b[0m\u001b[34m => exporting to image 0.0s\n", - "\u001b[0m\u001b[34m => => exporting layers 0.0s\n", - "\u001b[0m\u001b[34m => => writing image sha256:b610ac09ff5e496e19e1d172b544150c5d88d6d3f7480 0.0s\n", - "\u001b[0m\u001b[34m => => naming to docker.io/library/keras-custom-training-container 0.0s\n", - "\u001b[0m\u001b[?25h\n", - "Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them\n" + "ERROR: Cannot connect to the Docker daemon at unix:///Users/svpino/.docker/run/docker.sock. Is the docker daemon running?\n" ] } ], @@ -2635,7 +2584,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "id": "ceede5e2", "metadata": { "tags": [ @@ -2696,7 +2645,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "id": "49d9ab0e", "metadata": {}, "outputs": [ @@ -2706,7 +2655,7 @@ "'keras-custom-training-container'" ] }, - "execution_count": 40, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -2738,7 +2687,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "id": "f4e17204", "metadata": { "tags": [ @@ -2781,7 +2730,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "id": "86056197", "metadata": { "tags": [ @@ -2795,7 +2744,7 @@ "{'PipelineArn': 'session5-pipeline'}" ] }, - "execution_count": 42, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -2841,7 +2790,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "id": "f367d0e3", "metadata": {}, "outputs": [], @@ -2876,7 +2825,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "id": "c8c82750", "metadata": {}, "outputs": [], @@ -2909,7 +2858,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "id": "038ff2e5-ed28-445b-bc03-4e996ec2286f", "metadata": { "tags": [] @@ -2958,7 +2907,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "id": "9799ab39-fcae-41f4-a68b-85ab71b3ba9a", "metadata": { "tags": [ @@ -2972,7 +2921,7 @@ "{'PipelineArn': 'session6-pipeline'}" ] }, - "execution_count": 46, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -3028,13 +2977,12 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "id": "0590fcfa", "metadata": {}, "outputs": [], "source": [ - "(CODE_FOLDER / \"evaluation\").mkdir(parents=True, exist_ok=True)\n", - "sys.path.extend([f\"./{CODE_FOLDER}/evaluation\"])" + "(CODE_FOLDER / \"evaluation\").mkdir(parents=True, exist_ok=True)" ] }, { @@ -3047,7 +2995,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 47, "id": "3ee3ab26-afa5-4ceb-9f7a-005d5fdea646", "metadata": { "tags": [ @@ -3059,7 +3007,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/evaluation/script.py\n" + "Writing code/evaluation/script.py\n" ] } ], @@ -3124,7 +3072,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 48, "id": "9a2540d8-278a-4953-bc54-0469d154427d", "metadata": { "tags": [ @@ -3144,16 +3092,28 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.2319 - accuracy: 0.0335 - val_loss: 1.2054 - val_accuracy: 0.0980 - 243ms/epoch - 30ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n", - "Validation accuracy: 0.09803921568627451\n" + "8/8 - 0s - loss: 1.0641 - accuracy: 0.3264 - val_loss: 0.9960 - val_accuracy: 0.4314 - 198ms/epoch - 25ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp4jp7oim1/model/001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Validation accuracy: 0.43137254901960786\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpcmeaj3zh/model/001/assets\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n" ] @@ -3162,8 +3122,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "2/2 [==============================] - 0s 2ms/step\n", - "Test accuracy: 0.058823529411764705\n", + "2/2 [==============================] - 0s 1ms/step\n", + "Test accuracy: 0.4117647058823529\n", "\u001b[32m.\u001b[0m" ] }, @@ -3179,42 +3139,54 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.2704 - accuracy: 0.4351 - val_loss: 1.1779 - val_accuracy: 0.5098 - 222ms/epoch - 28ms/step\n" + "8/8 - 0s - loss: 1.2814 - accuracy: 0.2929 - val_loss: 1.2008 - val_accuracy: 0.3922 - 164ms/epoch - 20ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "WARNING:tensorflow:5 out of the last 9 calls to .predict_function at 0x2b62ac940> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:5 out of the last 9 calls to .predict_function at 0x38051e710> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2/2 [==============================] - 0s 1ms/step\n", - "Validation accuracy: 0.5098039215686274\n" + "2/2 [==============================] - 0s 851us/step\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp3xilsfww/model/001/assets\n", + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Validation accuracy: 0.39215686274509803\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpc93ppxgz/model/001/assets\n", - "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n", - "WARNING:tensorflow:6 out of the last 11 calls to .predict_function at 0x2b62af250> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:6 out of the last 11 calls to .predict_function at 0x381ac12d0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2/2 [==============================] - 0s 2ms/step\n", - "Test accuracy: 0.39215686274509803\n", + "2/2 [==============================] - 0s 1ms/step\n", + "Test accuracy: 0.2549019607843137\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.38s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.00s\u001b[0m\u001b[0m\n" ] } ], @@ -3297,7 +3269,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 49, "id": "4f19e15b", "metadata": {}, "outputs": [], @@ -3325,7 +3297,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 50, "id": "1f27b2ef", "metadata": {}, "outputs": [], @@ -3351,7 +3323,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 51, "id": "2fdff07f", "metadata": { "tags": [ @@ -3384,7 +3356,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 52, "id": "48139a07-5c8e-4bc6-b666-bf9531f7f520", "metadata": { "tags": [ @@ -3449,7 +3421,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 53, "id": "21a1ab1f", "metadata": { "tags": [ @@ -3463,7 +3435,7 @@ "{'PipelineArn': 'session7-pipeline'}" ] }, - "execution_count": 54, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -3508,7 +3480,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 54, "id": "bb70f907", "metadata": {}, "outputs": [], @@ -3528,7 +3500,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 55, "id": "4ca4cb61", "metadata": {}, "outputs": [], @@ -3555,7 +3527,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 56, "id": "8c05a7e1", "metadata": {}, "outputs": [], @@ -3591,7 +3563,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "id": "c9773a4a", "metadata": { "tags": [ @@ -3659,7 +3631,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 58, "id": "4d09cc85", "metadata": { "tags": [ @@ -3673,7 +3645,7 @@ "{'PipelineArn': 'session8-pipeline'}" ] }, - "execution_count": 59, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -3721,7 +3693,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 59, "id": "745486b5", "metadata": {}, "outputs": [], @@ -3743,7 +3715,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 60, "id": "c4431bbf", "metadata": {}, "outputs": [], @@ -3774,7 +3746,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 61, "id": "bebeecab", "metadata": {}, "outputs": [], @@ -3804,7 +3776,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 62, "id": "36e2a2b1-6711-4266-95d8-d2aebd52e199", "metadata": { "tags": [] @@ -3833,7 +3805,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 63, "id": "f70bcd33-b499-4e2b-953e-94d1ed96c10a", "metadata": { "tags": [ @@ -3847,7 +3819,7 @@ "{'PipelineArn': 'session9-pipeline'}" ] }, - "execution_count": 64, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -3895,7 +3867,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 64, "id": "a67726d7", "metadata": {}, "outputs": [ @@ -3910,7 +3882,7 @@ " 'ModelApprovalStatus': 'Approved'}" ] }, - "execution_count": 65, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -3952,7 +3924,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 65, "id": "f2e94adc", "metadata": {}, "outputs": [], @@ -3970,7 +3942,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 66, "id": "bd12fbe9", "metadata": {}, "outputs": [], @@ -4000,7 +3972,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 67, "id": "c79a0e74", "metadata": { "tags": [ @@ -4012,7 +3984,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/serving/app.py\n" + "Writing code/serving/app.py\n" ] } ], @@ -4125,7 +4097,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 68, "id": "befd5ad3", "metadata": {}, "outputs": [], @@ -4147,7 +4119,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 69, "id": "dee516e9", "metadata": {}, "outputs": [ @@ -4196,7 +4168,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 70, "id": "7c8852d5-818a-406c-944d-30bf6de90288", "metadata": { "tags": [] @@ -4229,7 +4201,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 71, "id": "ba7da291", "metadata": {}, "outputs": [], @@ -4251,7 +4223,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 72, "id": "0817a25e-8224-4911-830b-d659e7458b4a", "metadata": { "tags": [] @@ -4309,7 +4281,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 73, "id": "e1dbadb0", "metadata": {}, "outputs": [], @@ -4342,7 +4314,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 74, "id": "ac7630ca", "metadata": {}, "outputs": [], @@ -4360,7 +4332,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 75, "id": "44b651af", "metadata": { "tags": [ @@ -4372,7 +4344,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/lambda/lambda.py\n" + "Writing code/lambda/lambda.py\n" ] } ], @@ -4489,7 +4461,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 76, "id": "200d0370", "metadata": { "tags": [ @@ -4521,7 +4493,7 @@ " \"Principal\": {\n", " \"Service\": [\"lambda.amazonaws.com\", \"events.amazonaws.com\"],\n", " },\n", - " \"Acti,on\": \"sts:AssumeRole\",\n", + " \"Action\": \"sts:AssumeRole\",\n", " },\n", " ],\n", " },\n", @@ -4560,7 +4532,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 77, "id": "4a742974", "metadata": { "tags": [ @@ -4571,13 +4543,13 @@ { "data": { "text/plain": [ - "{'ResponseMetadata': {'RequestId': '113fc003-bba4-470a-be79-36588ac79f6d',\n", + "{'ResponseMetadata': {'RequestId': 'b35a622f-fa7e-400b-9558-33911681827e',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'date': 'Sun, 31 Mar 2024 15:33:21 GMT',\n", + " 'HTTPHeaders': {'date': 'Wed, 22 May 2024 00:35:30 GMT',\n", " 'content-type': 'application/json',\n", " 'content-length': '1593',\n", " 'connection': 'keep-alive',\n", - " 'x-amzn-requestid': '113fc003-bba4-470a-be79-36588ac79f6d'},\n", + " 'x-amzn-requestid': 'b35a622f-fa7e-400b-9558-33911681827e'},\n", " 'RetryAttempts': 0},\n", " 'FunctionName': 'deployment_fn',\n", " 'FunctionArn': 'arn:aws:lambda:us-east-1:325223348818:function:deployment_fn',\n", @@ -4588,15 +4560,15 @@ " 'Description': '',\n", " 'Timeout': 600,\n", " 'MemorySize': 128,\n", - " 'LastModified': '2024-03-31T15:33:21.000+0000',\n", - " 'CodeSha256': 'TBaeM5KkfChf4nvbY+qFGsICzi0L3/cjgbb1bu75UAs=',\n", + " 'LastModified': '2024-05-22T00:35:30.000+0000',\n", + " 'CodeSha256': 'mVZJaWMX6GbPPLP05/zbUdn+cGyNMgCJaFEAoVS0eEA=',\n", " 'Version': '$LATEST',\n", " 'Environment': {'Variables': {'ROLE': 'arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501',\n", " 'DATA_CAPTURE_PERCENTAGE': '100',\n", " 'DATA_CAPTURE_DESTINATION': 's3://mlschool/penguins/monitoring/data-capture',\n", " 'ENDPOINT': 'penguins-endpoint'}},\n", " 'TracingConfig': {'Mode': 'PassThrough'},\n", - " 'RevisionId': 'f72c34d8-9f36-4ff2-9f61-2e0b8ebb3ebf',\n", + " 'RevisionId': '0fd37072-7535-47f1-81a7-436d48a2398b',\n", " 'Layers': [],\n", " 'State': 'Active',\n", " 'LastUpdateStatus': 'InProgress',\n", @@ -4606,12 +4578,12 @@ " 'Architectures': ['x86_64'],\n", " 'EphemeralStorage': {'Size': 512},\n", " 'SnapStart': {'ApplyOn': 'None', 'OptimizationStatus': 'Off'},\n", - " 'RuntimeVersionConfig': {'RuntimeVersionArn': 'arn:aws:lambda:us-east-1::runtime:2d26a57600db1f6cf7cc34a4f11185001f7bae5ca901b445c13723efea1130a6'},\n", + " 'RuntimeVersionConfig': {'RuntimeVersionArn': 'arn:aws:lambda:us-east-1::runtime:1bebe65bb4f19dd8c37baeb0be18a0b278155b42bfe12dd28480199436d18229'},\n", " 'LoggingConfig': {'LogFormat': 'Text',\n", " 'LogGroup': '/aws/lambda/deployment_fn'}}" ] }, - "execution_count": 78, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } @@ -4655,7 +4627,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 78, "id": "1f86b997", "metadata": {}, "outputs": [], @@ -4689,7 +4661,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 79, "id": "5cc3e44b", "metadata": {}, "outputs": [], @@ -4714,7 +4686,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 80, "id": "3d86577a", "metadata": { "tags": [ @@ -4728,7 +4700,7 @@ "{'PipelineArn': 'session12-pipeline'}" ] }, - "execution_count": 81, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -4764,7 +4736,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 81, "id": "1dd5bb33", "metadata": {}, "outputs": [ @@ -4825,7 +4797,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 82, "id": "96751e71", "metadata": {}, "outputs": [], @@ -4845,7 +4817,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 83, "id": "1ee4b9d5", "metadata": {}, "outputs": [], @@ -4872,7 +4844,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 84, "id": "63f294bc", "metadata": {}, "outputs": [], @@ -4898,7 +4870,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 85, "id": "9985bd49", "metadata": {}, "outputs": [], @@ -4926,7 +4898,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 86, "id": "32302639", "metadata": {}, "outputs": [ @@ -4966,7 +4938,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 87, "id": "18adeb93", "metadata": { "tags": [ @@ -5004,7 +4976,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 88, "id": "fafb5514", "metadata": {}, "outputs": [], @@ -5029,7 +5001,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 89, "id": "b95c22b2", "metadata": { "tags": [ @@ -5043,7 +5015,7 @@ "{'PipelineArn': 'session13-pipeline'}" ] }, - "execution_count": 90, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -5132,13 +5104,12 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 90, "id": "a1f25973", "metadata": {}, "outputs": [], "source": [ - "(CODE_FOLDER / \"pipeline\").mkdir(parents=True, exist_ok=True)\n", - "sys.path.extend([f\"./{CODE_FOLDER}/pipeline\"])" + "(CODE_FOLDER / \"pipeline\").mkdir(parents=True, exist_ok=True)" ] }, { @@ -5151,7 +5122,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 91, "id": "c480ae02", "metadata": { "tags": [ @@ -5163,7 +5134,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/pipeline/preprocessing_component.py\n" + "Writing code/pipeline/preprocessing_component.py\n" ] } ], @@ -5281,7 +5252,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 92, "id": "34721e63", "metadata": { "tags": [ @@ -5294,7 +5265,7 @@ "output_type": "stream", "text": [ "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.07s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.06s\u001b[0m\u001b[0m\n" ] } ], @@ -5429,7 +5400,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 93, "id": "d23798aa", "metadata": { "tags": [ @@ -5441,7 +5412,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/pipeline/postprocessing_component.py\n" + "Writing code/pipeline/postprocessing_component.py\n" ] } ], @@ -5529,7 +5500,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 94, "id": "965a6be5", "metadata": { "tags": [ @@ -5611,7 +5582,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 95, "id": "82dafcd0", "metadata": {}, "outputs": [], @@ -5637,7 +5608,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 96, "id": "8c7dda7a", "metadata": {}, "outputs": [], @@ -5664,7 +5635,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 97, "id": "6c31090e", "metadata": {}, "outputs": [], @@ -5689,7 +5660,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 98, "id": "9fa95ab8", "metadata": {}, "outputs": [], @@ -5716,7 +5687,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 99, "id": "7c052332", "metadata": {}, "outputs": [], @@ -5736,7 +5707,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 100, "id": "3cb0ce9e", "metadata": { "tags": [ @@ -5775,7 +5746,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 101, "id": "8ec7b7d8", "metadata": {}, "outputs": [], @@ -5795,7 +5766,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 102, "id": "b9712905-9fe3-4148-ae6d-05b0a48e742e", "metadata": { "tags": [] @@ -5822,7 +5793,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 103, "id": "bad9f51d", "metadata": { "tags": [ @@ -5836,7 +5807,7 @@ "{'PipelineArn': 'session14-pipeline'}" ] }, - "execution_count": 104, + "execution_count": 103, "metadata": {}, "output_type": "execute_result" } @@ -5870,7 +5841,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 104, "id": "3cc966fb-b611-417f-a8b8-0c5d2f95252c", "metadata": { "tags": [] @@ -5924,7 +5895,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 105, "id": "ce3d33f0", "metadata": {}, "outputs": [ @@ -5973,7 +5944,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 106, "id": "240aaff2", "metadata": {}, "outputs": [ @@ -6034,7 +6005,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 107, "id": "97b3cd80", "metadata": { "tags": [ @@ -6046,7 +6017,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/pipeline/inference.py\n" + "Writing code/pipeline/inference.py\n" ] } ], @@ -6173,7 +6144,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 108, "id": "91786cc7", "metadata": { "tags": [ @@ -6206,29 +6177,29 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:tensorflow:5 out of the last 9 calls to .test_function at 0x2b474e950> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:5 out of the last 9 calls to .test_function at 0x37e4637f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8/8 - 0s - loss: 1.5890 - accuracy: 0.4351 - val_loss: 1.3773 - val_accuracy: 0.3333 - 221ms/epoch - 28ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n", - "Validation accuracy: 0.3333333333333333\n" + "8/8 - 0s - loss: 1.1710 - accuracy: 0.5690 - val_loss: 1.0926 - val_accuracy: 0.5686 - 182ms/epoch - 23ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpoum_3699/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp6wvjg7zi/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ + "Validation accuracy: 0.5686274509803921\n", "Handling endpoint request\n", "Processing input data...\n", "Sending input data to model to make a prediction...\n" @@ -6246,10 +6217,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 28ms/step\n", - "Response: {'predictions': [[0.35001689195632935, 0.25480717420578003, 0.3951759338378906]]}\n", + "1/1 [==============================] - 0s 19ms/step\n", + "Response: {'predictions': [[0.2550545930862427, 0.310475617647171, 0.43446972966194153]]}\n", "Processing prediction received from the model...\n", - "{'prediction': 'Gentoo', 'confidence': 0.3951759338378906}\n", + "{'prediction': 'Gentoo', 'confidence': 0.43446972966194153}\n", "\u001b[32m.\u001b[0m" ] }, @@ -6271,29 +6242,30 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:tensorflow:6 out of the last 11 calls to .test_function at 0x2b47a7c70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:6 out of the last 11 calls to .test_function at 0x380bb32e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "8/8 - 0s - loss: 1.1736 - accuracy: 0.3138 - val_loss: 1.2214 - val_accuracy: 0.2745 - 224ms/epoch - 28ms/step\n", - "2/2 [==============================] - 0s 1ms/step\n", - "Validation accuracy: 0.27450980392156865\n" + "8/8 - 0s - loss: 1.0871 - accuracy: 0.4100 - val_loss: 1.0737 - val_accuracy: 0.3725 - 179ms/epoch - 22ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpm8jumn5k/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp48cud_do/model/001/assets\n", + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ + "Validation accuracy: 0.37254901960784315\n", "Handling endpoint request\n", "Processing input data...\n", "Sending input data to model to make a prediction...\n" @@ -6303,7 +6275,6 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n" ] }, @@ -6311,10 +6282,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 27ms/step\n", - "Response: {'predictions': [[0.2764297425746918, 0.32800644636154175, 0.3955638110637665]]}\n", + "1/1 [==============================] - 0s 19ms/step\n", + "Response: {'predictions': [[0.3279760181903839, 0.3261311650276184, 0.3458927571773529]]}\n", "Processing prediction received from the model...\n", - "{'prediction': 'Gentoo', 'confidence': 0.3955638110637665}\n", + "{'prediction': 'Gentoo', 'confidence': 0.3458927571773529}\n", "\u001b[32m.\u001b[0m" ] }, @@ -6330,7 +6301,7 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.2106 - accuracy: 0.2762 - val_loss: 1.2037 - val_accuracy: 0.2745 - 218ms/epoch - 27ms/step\n", + "8/8 - 0s - loss: 1.1961 - accuracy: 0.1464 - val_loss: 1.2074 - val_accuracy: 0.1569 - 177ms/epoch - 22ms/step\n", "2/2 [==============================] - 0s 1ms/step\n" ] }, @@ -6338,14 +6309,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp2u9k20ic/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpflb8od78/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.27450980392156865\n", + "Validation accuracy: 0.1568627450980392\n", "Handling endpoint request\n", "Processing input data...\n", "Sending input data to model to make a prediction...\n" @@ -6363,10 +6334,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 24ms/step\n", - "Response: {'predictions': [[0.5782711505889893, 0.2070244550704956, 0.21470439434051514]]}\n", + "1/1 [==============================] - 0s 21ms/step\n", + "Response: {'predictions': [[0.3537094295024872, 0.3050204813480377, 0.3412701487541199]]}\n", "Processing prediction received from the model...\n", - "{'prediction': 'Adelie', 'confidence': 0.5782711505889893}\n", + "{'prediction': 'Adelie', 'confidence': 0.3537094295024872}\n", "\u001b[32m.\u001b[0m" ] }, @@ -6382,7 +6353,7 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.1899 - accuracy: 0.2218 - val_loss: 1.1516 - val_accuracy: 0.2157 - 218ms/epoch - 27ms/step\n", + "8/8 - 0s - loss: 1.3418 - accuracy: 0.1130 - val_loss: 1.3069 - val_accuracy: 0.1569 - 162ms/epoch - 20ms/step\n", "2/2 [==============================] - 0s 1ms/step\n" ] }, @@ -6390,21 +6361,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp39xu_z41/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpwgsfuyg6/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.21568627450980393\n", + "Validation accuracy: 0.1568627450980392\n", "Handling endpoint request\n", "Processing input data...\n", "There was an error processing the input data. Expecting value: line 1 column 1 (char 0)\n", "Processing prediction received from the model...\n", "{'prediction': None}\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m4 passed\u001b[0m\u001b[32m in 2.34s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m4 passed\u001b[0m\u001b[32m in 1.86s\u001b[0m\u001b[0m\n" ] } ], @@ -6550,7 +6521,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 109, "id": "603cb8f5", "metadata": { "tags": [ @@ -6562,7 +6533,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting code/pipeline/requirements.txt\n" + "Writing code/pipeline/requirements.txt\n" ] } ], @@ -6586,7 +6557,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 110, "id": "b5015c75", "metadata": {}, "outputs": [], @@ -6614,7 +6585,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 111, "id": "7ee3cdb6", "metadata": {}, "outputs": [], @@ -6634,7 +6605,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 112, "id": "21fb3e2c", "metadata": { "tags": [ @@ -6674,7 +6645,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 113, "id": "9e8162b7", "metadata": {}, "outputs": [], @@ -6694,7 +6665,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 114, "id": "849b379f", "metadata": {}, "outputs": [], @@ -6719,7 +6690,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 115, "id": "f64921ee", "metadata": { "tags": [ @@ -6733,7 +6704,7 @@ "{'PipelineArn': 'session15-pipeline'}" ] }, - "execution_count": 116, + "execution_count": 115, "metadata": {}, "output_type": "execute_result" } @@ -6767,7 +6738,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 116, "id": "1d63ce59", "metadata": {}, "outputs": [ @@ -6835,7 +6806,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 117, "id": "f4747fc7", "metadata": {}, "outputs": [], @@ -6865,7 +6836,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 118, "id": "3fc728fa", "metadata": { "tags": [ @@ -6925,7 +6896,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 119, "id": "2d5d8133", "metadata": {}, "outputs": [], @@ -6967,7 +6938,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 120, "id": "fc5c4325", "metadata": { "tags": [ @@ -6998,7 +6969,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 121, "id": "be9d6de7", "metadata": {}, "outputs": [], @@ -7023,7 +6994,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 122, "id": "fb9b8d1e", "metadata": { "tags": [ @@ -7037,7 +7008,7 @@ "{'PipelineArn': 'session16-pipeline'}" ] }, - "execution_count": 123, + "execution_count": 122, "metadata": {}, "output_type": "execute_result" } @@ -7080,7 +7051,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 123, "id": "c2b95d03", "metadata": {}, "outputs": [ @@ -7140,7 +7111,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 124, "id": "63908598", "metadata": {}, "outputs": [ @@ -7265,7 +7236,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 125, "id": "69e115c3", "metadata": {}, "outputs": [], @@ -7285,7 +7256,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 126, "id": "f93f2751", "metadata": { "tags": [ @@ -7322,7 +7293,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 127, "id": "2a530d12", "metadata": {}, "outputs": [], @@ -7356,7 +7327,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 128, "id": "653c6ec8", "metadata": { "tags": [ @@ -7414,7 +7385,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 129, "id": "b50a9ed5", "metadata": { "tags": [ @@ -7476,7 +7447,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 130, "id": "3aecc022", "metadata": {}, "outputs": [], @@ -7534,7 +7505,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 131, "id": "20154a1a", "metadata": { "tags": [ @@ -7565,7 +7536,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 132, "id": "cd84e567", "metadata": {}, "outputs": [], @@ -7597,7 +7568,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 133, "id": "c244e206", "metadata": { "tags": [ @@ -7611,7 +7582,7 @@ "{'PipelineArn': 'session17-pipeline'}" ] }, - "execution_count": 134, + "execution_count": 133, "metadata": {}, "output_type": "execute_result" } @@ -7646,7 +7617,7 @@ }, { "cell_type": "code", - "execution_count": 341, + "execution_count": 134, "id": "dc31aa28", "metadata": {}, "outputs": [ @@ -7722,7 +7693,7 @@ }, { "cell_type": "code", - "execution_count": 342, + "execution_count": 135, "id": "57cc8903", "metadata": {}, "outputs": [], @@ -7782,7 +7753,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 136, "id": "4966c9c2", "metadata": { "tags": [ @@ -7809,14 +7780,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 137, "id": "cea757af", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" + ] + } + ], "source": [ "from sagemaker.serializers import JSONSerializer\n", "\n", @@ -7852,7 +7831,7 @@ }, { "cell_type": "code", - "execution_count": 349, + "execution_count": 138, "id": "caee992c", "metadata": {}, "outputs": [ @@ -7860,7 +7839,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "File: s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2024/03/30/17/32-02-242-191b135d-085a-484d-a119-45b26c51554c.jsonl\n", + "File: s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2024/03/30/18/40-45-068-0f144be9-ac73-4c4e-a0c7-82b1ba7db88b.jsonl\n", "{\n", " \"captureData\": {\n", " \"endpointInput\": {\n", @@ -7877,9 +7856,9 @@ " }\n", " },\n", " \"eventMetadata\": {\n", - " \"eventId\": \"3211434d-0db6-4ee2-8848-95ce11f6d5e6\",\n", + " \"eventId\": \"08a239af-c98c-4984-b9bf-4ea049d88617\",\n", " \"inferenceId\": \"0\",\n", - " \"inferenceTime\": \"2024-03-30T17:32:02Z\"\n", + " \"inferenceTime\": \"2024-03-30T18:40:45Z\"\n", " },\n", " \"eventVersion\": \"0\"\n", "}\n" @@ -7910,7 +7889,7 @@ }, { "cell_type": "code", - "execution_count": 350, + "execution_count": 139, "id": "7b48cd49", "metadata": {}, "outputs": [], @@ -7930,7 +7909,7 @@ }, { "cell_type": "code", - "execution_count": 351, + "execution_count": 140, "id": "0114a817", "metadata": { "tags": [ @@ -7971,7 +7950,7 @@ }, { "cell_type": "code", - "execution_count": 374, + "execution_count": 141, "id": "21240214", "metadata": { "tags": [ @@ -7992,7 +7971,7 @@ "'s3://mlschool/penguins/monitoring/data_quality_preprocessor.py'" ] }, - "execution_count": 374, + "execution_count": 141, "metadata": {}, "output_type": "execute_result" } @@ -8019,7 +7998,7 @@ }, { "cell_type": "code", - "execution_count": 353, + "execution_count": 142, "id": "b9df3d16", "metadata": {}, "outputs": [ @@ -8068,23 +8047,14 @@ }, { "cell_type": "code", - "execution_count": 355, + "execution_count": 143, "id": "c2b3fe69", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.model_monitor.model_monitoring:Creating Monitoring Schedule with name: penguins-data-monitoring-schedule\n", - "INFO:sagemaker:Starting Monitoring Schedule with name: penguins-data-monitoring-schedule\n" - ] - } - ], + "outputs": [], "source": [ "%%script false --no-raise-error\n", "# | eval: false\n", @@ -8121,7 +8091,7 @@ }, { "cell_type": "code", - "execution_count": 356, + "execution_count": 144, "id": "5cedff08", "metadata": {}, "outputs": [ @@ -8129,127 +8099,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Processing Job Status: Completed\n", - "Exit Message: \"Completed: Job completed successfully with no violations.\"\n", - "Last Modified Time: 2024-03-30 14:15:49.146000-04:00\n", - "\n", - "Execution:\n", - "{\n", - " \"ProcessingInputs\": [\n", - " {\n", - " \"InputName\": \"baseline\",\n", - " \"AppManaged\": false,\n", - " \"S3Input\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-quality/statistics.json\",\n", - " \"LocalPath\": \"/opt/ml/processing/baseline/stats\",\n", - " \"S3DataType\": \"S3Prefix\",\n", - " \"S3InputMode\": \"File\",\n", - " \"S3DataDistributionType\": \"FullyReplicated\"\n", - " }\n", - " },\n", - " {\n", - " \"InputName\": \"constraints\",\n", - " \"AppManaged\": false,\n", - " \"S3Input\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-quality/constraints.json\",\n", - " \"LocalPath\": \"/opt/ml/processing/baseline/constraints\",\n", - " \"S3DataType\": \"S3Prefix\",\n", - " \"S3InputMode\": \"File\",\n", - " \"S3DataDistributionType\": \"FullyReplicated\"\n", - " }\n", - " },\n", - " {\n", - " \"InputName\": \"pre_processor_script\",\n", - " \"AppManaged\": false,\n", - " \"S3Input\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data_quality_preprocessor.py\",\n", - " \"LocalPath\": \"/opt/ml/processing/code/preprocessing\",\n", - " \"S3DataType\": \"S3Prefix\",\n", - " \"S3InputMode\": \"File\",\n", - " \"S3DataDistributionType\": \"FullyReplicated\"\n", - " }\n", - " },\n", - " {\n", - " \"InputName\": \"endpoint_input_1\",\n", - " \"AppManaged\": false,\n", - " \"S3Input\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2024/03/30/17\",\n", - " \"LocalPath\": \"/opt/ml/processing/input/endpoint/penguins-endpoint/AllTraffic/2024/03/30/17\",\n", - " \"S3DataType\": \"S3Prefix\",\n", - " \"S3InputMode\": \"File\",\n", - " \"S3DataDistributionType\": \"FullyReplicated\",\n", - " \"S3CompressionType\": \"None\"\n", - " }\n", - " }\n", - " ],\n", - " \"ProcessingOutputConfig\": {\n", - " \"Outputs\": [\n", - " {\n", - " \"OutputName\": \"result\",\n", - " \"S3Output\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-quality/penguins-endpoint/penguins-data-monitoring-schedule/2024/03/30/18\",\n", - " \"LocalPath\": \"/opt/ml/processing/output\",\n", - " \"S3UploadMode\": \"Continuous\"\n", - " },\n", - " \"AppManaged\": false\n", - " }\n", - " ]\n", - " },\n", - " \"ProcessingJobName\": \"model-monitoring-202403301800-17aa1fca873fac795ffba24a\",\n", - " \"ProcessingResources\": {\n", - " \"ClusterConfig\": {\n", - " \"InstanceCount\": 1,\n", - " \"InstanceType\": \"ml.m5.xlarge\",\n", - " \"VolumeSizeInGB\": 20\n", - " }\n", - " },\n", - " \"StoppingCondition\": {\n", - " \"MaxRuntimeInSeconds\": 1800\n", - " },\n", - " \"AppSpecification\": {\n", - " \"ImageUri\": \"156813124566.dkr.ecr.us-east-1.amazonaws.com/sagemaker-model-monitor-analyzer\"\n", - " },\n", - " \"Environment\": {\n", - " \"baseline_constraints\": \"/opt/ml/processing/baseline/constraints/constraints.json\",\n", - " \"baseline_statistics\": \"/opt/ml/processing/baseline/stats/statistics.json\",\n", - " \"dataset_format\": \"{\\\"sagemakerCaptureJson\\\":{\\\"captureIndexNames\\\":[\\\"endpointInput\\\",\\\"endpointOutput\\\"]}}\",\n", - " \"dataset_source\": \"/opt/ml/processing/input/endpoint\",\n", - " \"end_time\": \"2024-03-30T18:00:00Z\",\n", - " \"metric_time\": \"2024-03-30T17:00:00Z\",\n", - " \"monitoring_input_type\": \"ENDPOINT_INPUT\",\n", - " \"output_path\": \"/opt/ml/processing/output\",\n", - " \"publish_cloudwatch_metrics\": \"Enabled\",\n", - " \"record_preprocessor_script\": \"/opt/ml/processing/code/preprocessing/data_quality_preprocessor.py\",\n", - " \"sagemaker_endpoint_name\": \"penguins-endpoint\",\n", - " \"sagemaker_monitoring_schedule_name\": \"penguins-data-monitoring-schedule\",\n", - " \"start_time\": \"2024-03-30T17:00:00Z\"\n", - " },\n", - " \"RoleArn\": \"arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501\",\n", - " \"ProcessingJobArn\": \"arn:aws:sagemaker:us-east-1:325223348818:processing-job/model-monitoring-202403301800-17aa1fca873fac795ffba24a\",\n", - " \"ProcessingJobStatus\": \"Completed\",\n", - " \"ExitMessage\": \"Completed: Job completed successfully with no violations.\",\n", - " \"ProcessingEndTime\": \"2024-03-30 14:15:48.732000-04:00\",\n", - " \"ProcessingStartTime\": \"2024-03-30 14:14:14.760000-04:00\",\n", - " \"LastModifiedTime\": \"2024-03-30 14:15:49.146000-04:00\",\n", - " \"CreationTime\": \"2024-03-30 14:09:54.896000-04:00\",\n", - " \"MonitoringScheduleArn\": \"arn:aws:sagemaker:us-east-1:325223348818:monitoring-schedule/penguins-data-monitoring-schedule\",\n", - " \"ResponseMetadata\": {\n", - " \"RequestId\": \"4e348652-7dff-4c40-96fb-b944aa6ed83b\",\n", - " \"HTTPStatusCode\": 200,\n", - " \"HTTPHeaders\": {\n", - " \"x-amzn-requestid\": \"4e348652-7dff-4c40-96fb-b944aa6ed83b\",\n", - " \"content-type\": \"application/x-amz-json-1.1\",\n", - " \"content-length\": \"3233\",\n", - " \"date\": \"Sat, 30 Mar 2024 18:34:16 GMT\"\n", - " },\n", - " \"RetryAttempts\": 0\n", - " }\n", - "}\n", - "\n", - "Violations:\n", - "{\n", - " \"violations\": []\n", - "}\n" + "Parameter validation failed:\n", + "Invalid type for parameter MonitoringScheduleName, value: None, type: , valid types: \n" ] } ], @@ -8304,10 +8155,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 145, "id": "7ff8bfc3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker:Deleting Monitoring Schedule with name: None\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parameter validation failed:\n", + "Invalid type for parameter MonitoringScheduleName, value: None, type: , valid types: \n" + ] + } + ], "source": [ "try:\n", " data_monitor.delete_monitoring_schedule()\n", @@ -8339,7 +8206,7 @@ }, { "cell_type": "code", - "execution_count": 357, + "execution_count": 146, "id": "5ddca53e", "metadata": {}, "outputs": [], @@ -8373,7 +8240,7 @@ }, { "cell_type": "code", - "execution_count": 358, + "execution_count": 147, "id": "c2089ba4", "metadata": { "tags": [ @@ -8400,14 +8267,22 @@ }, { "cell_type": "code", - "execution_count": 359, + "execution_count": 148, "id": "c4545041", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" + ] + } + ], "source": [ "generate_fake_traffic()" ] @@ -8424,7 +8299,7 @@ }, { "cell_type": "code", - "execution_count": 361, + "execution_count": 149, "id": "49bc662a", "metadata": {}, "outputs": [ @@ -8478,7 +8353,7 @@ }, { "cell_type": "code", - "execution_count": 366, + "execution_count": 150, "id": "a4a32d8c", "metadata": { "tags": [ @@ -8489,10 +8364,10 @@ { "data": { "text/plain": [ - "'s3://mlschool/penguins/monitoring/groundtruth/2024/03/30/18/5226.jsonl'" + "'s3://mlschool/penguins/monitoring/groundtruth/2024/05/22/00/3542.jsonl'" ] }, - "execution_count": 366, + "execution_count": 150, "metadata": {}, "output_type": "execute_result" } @@ -8546,7 +8421,7 @@ }, { "cell_type": "code", - "execution_count": 367, + "execution_count": 151, "id": "54762759", "metadata": { "tags": [ @@ -8599,23 +8474,14 @@ }, { "cell_type": "code", - "execution_count": 368, + "execution_count": 152, "id": "9665c61a", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.model_monitor.model_monitoring:Creating Monitoring Schedule with name: penguins-model-monitoring-schedule\n", - "INFO:sagemaker:Starting Monitoring Schedule with name: penguins-model-monitoring-schedule\n" - ] - } - ], + "outputs": [], "source": [ "%%script false --no-raise-error\n", "# | eval: false\n", @@ -8661,7 +8527,7 @@ }, { "cell_type": "code", - "execution_count": 380, + "execution_count": 153, "id": "34ba56d9", "metadata": {}, "outputs": [ @@ -8669,132 +8535,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Processing Job Status: Completed\n", - "Exit Message: \"CompletedWithViolations: Job completed successfully with 5 violations.\"\n", - "Last Modified Time: 2024-03-30 15:18:36.431000-04:00\n", - "\n", - "Execution:\n", - "{\n", - " \"ProcessingInputs\": [\n", - " {\n", - " \"InputName\": \"constraints\",\n", - " \"AppManaged\": false,\n", - " \"S3Input\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/model-quality/constraints.json\",\n", - " \"LocalPath\": \"/opt/ml/processing/baseline/constraints\",\n", - " \"S3DataType\": \"S3Prefix\",\n", - " \"S3InputMode\": \"File\",\n", - " \"S3DataDistributionType\": \"FullyReplicated\"\n", - " }\n", - " },\n", - " {\n", - " \"InputName\": \"endpoint_input_1\",\n", - " \"AppManaged\": false,\n", - " \"S3Input\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/model-quality/merge/penguins-endpoint/AllTraffic/2024/03/30/18\",\n", - " \"LocalPath\": \"/opt/ml/processing/input_data/penguins-endpoint/AllTraffic/2024/03/30/18\",\n", - " \"S3DataType\": \"S3Prefix\",\n", - " \"S3InputMode\": \"File\",\n", - " \"S3DataDistributionType\": \"FullyReplicated\",\n", - " \"S3CompressionType\": \"None\"\n", - " }\n", - " }\n", - " ],\n", - " \"ProcessingOutputConfig\": {\n", - " \"Outputs\": [\n", - " {\n", - " \"OutputName\": \"result\",\n", - " \"S3Output\": {\n", - " \"S3Uri\": \"s3://mlschool/penguins/monitoring/model-quality/penguins-endpoint/penguins-model-monitoring-schedule/2024/03/30/19\",\n", - " \"LocalPath\": \"/opt/ml/processing/output\",\n", - " \"S3UploadMode\": \"Continuous\"\n", - " },\n", - " \"AppManaged\": false\n", - " }\n", - " ]\n", - " },\n", - " \"ProcessingJobName\": \"model-quality-monitoring-202403301900-896e874cc3a809cdf37d6cc2\",\n", - " \"ProcessingResources\": {\n", - " \"ClusterConfig\": {\n", - " \"InstanceCount\": 1,\n", - " \"InstanceType\": \"ml.m5.xlarge\",\n", - " \"VolumeSizeInGB\": 20\n", - " }\n", - " },\n", - " \"StoppingCondition\": {\n", - " \"MaxRuntimeInSeconds\": 1800\n", - " },\n", - " \"AppSpecification\": {\n", - " \"ImageUri\": \"156813124566.dkr.ecr.us-east-1.amazonaws.com/sagemaker-model-monitor-analyzer\"\n", - " },\n", - " \"Environment\": {\n", - " \"analysis_type\": \"MODEL_QUALITY\",\n", - " \"baseline_constraints\": \"/opt/ml/processing/baseline/constraints/constraints.json\",\n", - " \"dataset_format\": \"{\\\"sagemakerMergeJson\\\":{\\\"captureIndexNames\\\":[\\\"endpointOutput\\\"],\\\"originalDatasetFormat\\\":null}}\",\n", - " \"dataset_source\": \"/opt/ml/processing/input_data\",\n", - " \"end_time\": \"2024-03-30T19:00:00Z\",\n", - " \"inference_attribute\": \"0\",\n", - " \"metric_time\": \"2024-03-30T18:00:00Z\",\n", - " \"monitoring_input_type\": \"ENDPOINT_INPUT\",\n", - " \"output_path\": \"/opt/ml/processing/output\",\n", - " \"problem_type\": \"MulticlassClassification\",\n", - " \"publish_cloudwatch_metrics\": \"Enabled\",\n", - " \"sagemaker_endpoint_name\": \"penguins-endpoint\",\n", - " \"sagemaker_monitoring_schedule_name\": \"penguins-model-monitoring-schedule\",\n", - " \"start_time\": \"2024-03-30T18:00:00Z\"\n", - " },\n", - " \"RoleArn\": \"arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501\",\n", - " \"ProcessingJobArn\": \"arn:aws:sagemaker:us-east-1:325223348818:processing-job/model-quality-monitoring-202403301900-896e874cc3a809cdf37d6cc2\",\n", - " \"ProcessingJobStatus\": \"Completed\",\n", - " \"ExitMessage\": \"CompletedWithViolations: Job completed successfully with 5 violations.\",\n", - " \"ProcessingEndTime\": \"2024-03-30 15:18:35.908000-04:00\",\n", - " \"ProcessingStartTime\": \"2024-03-30 15:16:52.922000-04:00\",\n", - " \"LastModifiedTime\": \"2024-03-30 15:18:36.431000-04:00\",\n", - " \"CreationTime\": \"2024-03-30 15:12:22.569000-04:00\",\n", - " \"MonitoringScheduleArn\": \"arn:aws:sagemaker:us-east-1:325223348818:monitoring-schedule/penguins-model-monitoring-schedule\",\n", - " \"ResponseMetadata\": {\n", - " \"RequestId\": \"85abb737-543a-4c92-928b-4a293c599f18\",\n", - " \"HTTPStatusCode\": 200,\n", - " \"HTTPHeaders\": {\n", - " \"x-amzn-requestid\": \"85abb737-543a-4c92-928b-4a293c599f18\",\n", - " \"content-type\": \"application/x-amz-json-1.1\",\n", - " \"content-length\": \"2660\",\n", - " \"date\": \"Sat, 30 Mar 2024 19:33:23 GMT\"\n", - " },\n", - " \"RetryAttempts\": 0\n", - " }\n", - "}\n", - "\n", - "Violations:\n", - "{\n", - " \"violations\": [\n", - " {\n", - " \"constraint_check_type\": \"LessThanThreshold\",\n", - " \"description\": \"Metric weightedF2 with 0.3518870011147463 +/- 0.006730551075118943 was LessThanThreshold '1.0'\",\n", - " \"metric_name\": \"weightedF2\"\n", - " },\n", - " {\n", - " \"constraint_check_type\": \"LessThanThreshold\",\n", - " \"description\": \"Metric accuracy with 0.35755813953488375 +/- 0.007228798319401767 was LessThanThreshold '1.0'\",\n", - " \"metric_name\": \"accuracy\"\n", - " },\n", - " {\n", - " \"constraint_check_type\": \"LessThanThreshold\",\n", - " \"description\": \"Metric weightedRecall with 0.35755813953488375 +/- 0.007228798319401765 was LessThanThreshold '1.0'\",\n", - " \"metric_name\": \"weightedRecall\"\n", - " },\n", - " {\n", - " \"constraint_check_type\": \"LessThanThreshold\",\n", - " \"description\": \"Metric weightedPrecision with 0.35624627310673823 +/- 0.008910206698382583 was LessThanThreshold '1.0'\",\n", - " \"metric_name\": \"weightedPrecision\"\n", - " },\n", - " {\n", - " \"constraint_check_type\": \"LessThanThreshold\",\n", - " \"description\": \"Metric weightedF1 with 0.34769539574160063 +/- 0.006655863903356062 was LessThanThreshold '1.0'\",\n", - " \"metric_name\": \"weightedF1\"\n", - " }\n", - " ]\n", - "}\n" + "Parameter validation failed:\n", + "Invalid type for parameter MonitoringScheduleName, value: None, type: , valid types: \n" ] } ], @@ -8814,10 +8556,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 154, "id": "7f250115", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker:Deleting Monitoring Schedule with name: None\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parameter validation failed:\n", + "Invalid type for parameter MonitoringScheduleName, value: None, type: , valid types: \n" + ] + } + ], "source": [ "try:\n", " model_monitor.delete_monitoring_schedule()\n", @@ -8849,7 +8607,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 155, "id": "d322d415", "metadata": {}, "outputs": [ @@ -8895,7 +8653,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 156, "id": "6c8aea09", "metadata": {}, "outputs": [], @@ -8920,7 +8678,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 157, "id": "dd20b354", "metadata": { "tags": [ @@ -8931,17 +8689,17 @@ { "data": { "text/plain": [ - "{'ModelArn': 'arn:aws:sagemaker:us-east-1:325223348818:model/shadow-penguins-endpoint-production-0331125310',\n", - " 'ResponseMetadata': {'RequestId': '15e570ab-106c-4b58-b536-1146770164d7',\n", + "{'ModelArn': 'arn:aws:sagemaker:us-east-1:325223348818:model/shadow-penguins-endpoint-production-0521203542',\n", + " 'ResponseMetadata': {'RequestId': '99d46cfc-2877-46e4-9e76-b105220e20a4',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '15e570ab-106c-4b58-b536-1146770164d7',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '99d46cfc-2877-46e4-9e76-b105220e20a4',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '108',\n", - " 'date': 'Sun, 31 Mar 2024 16:53:12 GMT'},\n", + " 'date': 'Wed, 22 May 2024 00:35:42 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 147, + "execution_count": 157, "metadata": {}, "output_type": "execute_result" } @@ -8966,24 +8724,24 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 158, "id": "d2024f35", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'ModelArn': 'arn:aws:sagemaker:us-east-1:325223348818:model/shadow-penguins-endpoint-shadow-0331125310',\n", - " 'ResponseMetadata': {'RequestId': '21aaeb87-98e5-49c3-8912-1143ef75f86c',\n", + "{'ModelArn': 'arn:aws:sagemaker:us-east-1:325223348818:model/shadow-penguins-endpoint-shadow-0521203542',\n", + " 'ResponseMetadata': {'RequestId': 'c9905dba-09bf-4e77-b3d9-7f09ad308c81',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '21aaeb87-98e5-49c3-8912-1143ef75f86c',\n", + " 'HTTPHeaders': {'x-amzn-requestid': 'c9905dba-09bf-4e77-b3d9-7f09ad308c81',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '104',\n", - " 'date': 'Sun, 31 Mar 2024 16:53:13 GMT'},\n", - " 'RetryAttempts': 0}}" + " 'date': 'Wed, 22 May 2024 00:35:48 GMT'},\n", + " 'RetryAttempts': 1}}" ] }, - "execution_count": 148, + "execution_count": 158, "metadata": {}, "output_type": "execute_result" } @@ -9018,7 +8776,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 159, "id": "85769b3e", "metadata": {}, "outputs": [], @@ -9036,24 +8794,24 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 160, "id": "969df4f8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'EndpointConfigArn': 'arn:aws:sagemaker:us-east-1:325223348818:endpoint-config/shadow-penguins-endpoint-config-0331125310',\n", - " 'ResponseMetadata': {'RequestId': '24973c88-6726-4737-ae91-1138b77f5775',\n", + "{'EndpointConfigArn': 'arn:aws:sagemaker:us-east-1:325223348818:endpoint-config/shadow-penguins-endpoint-config-0521203542',\n", + " 'ResponseMetadata': {'RequestId': '721699dd-8e93-4e9f-b784-ce06f1dd0fed',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '24973c88-6726-4737-ae91-1138b77f5775',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '721699dd-8e93-4e9f-b784-ce06f1dd0fed',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '123',\n", - " 'date': 'Sun, 31 Mar 2024 16:53:17 GMT'},\n", + " 'date': 'Wed, 22 May 2024 00:35:48 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 150, + "execution_count": 160, "metadata": {}, "output_type": "execute_result" } @@ -9107,31 +8865,26 @@ "Finally, we can create the Endpoint using the Endpoint Configuration we created before.\n" ] }, + { + "cell_type": "markdown", + "id": "e0bb879c", + "metadata": {}, + "source": [ + "#| hide\n", + "\n", + "
Note: \n", + " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", + "
\n" + ] + }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 161, "id": "33e8d886", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'EndpointArn': 'arn:aws:sagemaker:us-east-1:325223348818:endpoint/shadow-penguins-endpoint',\n", - " 'ResponseMetadata': {'RequestId': 'df5ebd20-f59f-4895-96f4-18da3beb0cc4',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': 'df5ebd20-f59f-4895-96f4-18da3beb0cc4',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '92',\n", - " 'date': 'Sun, 31 Mar 2024 16:53:21 GMT'},\n", - " 'RetryAttempts': 0}}" - ] - }, - "execution_count": 151, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ + "%%script false --no-raise-error\n", "# | eval: false\n", "\n", "sagemaker_client.create_endpoint(\n", @@ -9152,7 +8905,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 162, "id": "069afdae", "metadata": {}, "outputs": [ @@ -9160,25 +8913,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"predictions\": [\n", - " [\n", - " 0.0403208546,\n", - " 0.0210227184,\n", - " 0.93865639\n", - " ],\n", - " [\n", - " 0.689678669,\n", - " 0.17514421,\n", - " 0.135177106\n", - " ],\n", - " [\n", - " 0.960919619,\n", - " 0.0248175282,\n", - " 0.0142629147\n", - " ]\n", - " ]\n", - "}\n" + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint shadow-penguins-endpoint of account 325223348818 not found.\n" ] } ], @@ -9216,7 +8951,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 163, "id": "9ffadf06", "metadata": {}, "outputs": [ @@ -9224,7 +8959,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "File: s3://mlschool/penguins/endpoint/shadow-penguins-endpoint/ShadowTraffic/2024/03/30/21/28-43-624-8f47e605-6bd2-44dd-bd91-293f29fd227e.jsonl\n", + "File: s3://mlschool/penguins/endpoint/shadow-penguins-endpoint/ShadowTraffic/2024/03/31/16/57-29-141-b62d2c9e-bf7b-4660-8086-e2f28a7c3181.jsonl\n", "{\n", " \"captureData\": {\n", " \"endpointInput\": {\n", @@ -9241,9 +8976,9 @@ " }\n", " },\n", " \"eventMetadata\": {\n", - " \"eventId\": \"98c3c22e-20af-401c-9ca6-6d67d734a83f\",\n", + " \"eventId\": \"706af7e2-4746-4c81-82f9-3b34120142b8\",\n", " \"invocationSource\": \"ShadowExperiment\",\n", - " \"inferenceTime\": \"2024-03-30T21:28:43Z\"\n", + " \"inferenceTime\": \"2024-03-31T16:57:29Z\"\n", " },\n", " \"eventVersion\": \"0\"\n", "}\n" @@ -9272,10 +9007,18 @@ }, { "cell_type": "code", - "execution_count": 451, + "execution_count": 164, "id": "2ee1ce12", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "An error occurred (ValidationException) when calling the DeleteEndpoint operation: Could not find endpoint \"shadow-penguins-endpoint\".\n" + ] + } + ], "source": [ "try:\n", " sagemaker_client.delete_endpoint(EndpointName=SHADOW_DEPLOYMENT_ENDPOINT)\n", @@ -9307,25 +9050,14 @@ }, { "cell_type": "code", - "execution_count": 340, + "execution_count": 165, "id": "59d1e634", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "data": { - "text/plain": [ - "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session17-pipeline/execution/o2a1if9pise8', sagemaker_session=)" - ] - }, - "execution_count": 340, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "%%script false --no-raise-error\n", "# | eval: false\n", @@ -9345,14 +9077,22 @@ }, { "cell_type": "code", - "execution_count": 406, + "execution_count": 166, "id": "6b32c3a4-312e-473c-a217-33606f77d1e9", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "An error occurred (ValidationException) when calling the DeleteEndpoint operation: Could not find endpoint \"penguins-endpoint\".\n" + ] + } + ], "source": [ "try:\n", " sagemaker_client.delete_endpoint(EndpointName=ENDPOINT)\n",