diff --git a/program/cohort.ipynb b/program/cohort.ipynb index 1cc7146..286a919 100644 --- a/program/cohort.ipynb +++ b/program/cohort.ipynb @@ -46,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "4b2265b0", "metadata": { "tags": [ @@ -93,14 +93,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "32c4d764", "metadata": { "tags": [] }, "outputs": [], "source": [ - "LOCAL_MODE = False" + "LOCAL_MODE = True" ] }, { @@ -113,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "3164a3af", "metadata": {}, "outputs": [], @@ -137,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "7bc40d28", "metadata": {}, "outputs": [], @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "3b3f17e5", "metadata": {}, "outputs": [], @@ -202,7 +202,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "942a01b5", "metadata": {}, "outputs": [], @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "f1cd2f0e-446d-48a9-a008-b4f1cc593bfc", "metadata": { "tags": [] @@ -344,7 +344,7 @@ "4 3450.0 FEMALE " ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -387,7 +387,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "f2107c25-e730-4e22-a1b8-5bda53e61124", "metadata": { "tags": [] @@ -566,7 +566,7 @@ "max 6300.000000 NaN " ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -585,7 +585,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "1242122a-726e-4c37-a718-dd8e873d1612", "metadata": { "tags": [] @@ -641,7 +641,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "cf1cf582-8831-4f83-bb17-2175afb193e8", "metadata": { "tags": [] @@ -656,7 +656,7 @@ "Name: count, dtype: int64" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -679,7 +679,7 @@ }, { "cell_type": "code", - "execution_count": 369, + "execution_count": 12, "id": "cc42cb08-275c-4b05-9d2b-77052da2f336", "metadata": { "tags": [] @@ -688,17 +688,17 @@ { "data": { "text/plain": [ - "species 0\n", - "island 0\n", - "culmen_length_mm 0\n", - "culmen_depth_mm 0\n", - "flipper_length_mm 0\n", - "body_mass_g 0\n", - "sex 0\n", + "species 0\n", + "island 0\n", + "culmen_length_mm 2\n", + "culmen_depth_mm 2\n", + "flipper_length_mm 2\n", + "body_mass_g 2\n", + "sex 11\n", "dtype: int64" ] }, - "execution_count": 369, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -717,7 +717,7 @@ }, { "cell_type": "code", - "execution_count": 370, + "execution_count": 13, "id": "3c57d55d-afd6-467a-a7a8-ff04132770ed", "metadata": { "tags": [] @@ -736,7 +736,7 @@ "dtype: int64" ] }, - "execution_count": 370, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -761,7 +761,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "2852c740", "metadata": {}, "outputs": [ @@ -807,7 +807,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "707cc972", "metadata": {}, "outputs": [ @@ -855,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "id": "3daf3ba1-d218-4ad4-b862-af679b91273f", "metadata": { "tags": [] @@ -935,7 +935,7 @@ "body_mass_g 640316.716388 " ] }, - "execution_count": 15, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -966,7 +966,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "id": "1d793e09-2cb9-47ff-a0e6-199a0f4fc1b3", "metadata": { "tags": [] @@ -1046,7 +1046,7 @@ "body_mass_g 1.000000 " ] }, - "execution_count": 16, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -1077,7 +1077,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "id": "1258c99d", "metadata": {}, "outputs": [ @@ -1117,7 +1117,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "id": "45b0a87f-028d-477f-9b65-199728c0b7ee", "metadata": { "tags": [] @@ -1187,7 +1187,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "id": "78fe069e", "metadata": {}, "outputs": [], @@ -1206,7 +1206,7 @@ }, { "cell_type": "code", - "execution_count": 378, + "execution_count": 21, "id": "fb6ba7c0-1bd6-4fe5-8b7f-f6cbdfd3846c", "metadata": { "tags": [ @@ -1441,7 +1441,7 @@ }, { "cell_type": "code", - "execution_count": 379, + "execution_count": 22, "id": "d1f122a4-acff-4687-91b9-bfef13567d88", "metadata": { "tags": [ @@ -1453,15 +1453,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.18s\u001b[0m\u001b[0m\n" + "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n", + "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.15s\u001b[0m\u001b[0m\n" ] } ], @@ -1585,7 +1578,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "id": "d88e9ccf", "metadata": {}, "outputs": [], @@ -1607,7 +1600,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "id": "331fe373", "metadata": {}, "outputs": [], @@ -1643,7 +1636,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "id": "3aa4471a", "metadata": { "tags": [ @@ -1682,7 +1675,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "id": "cdbd9303", "metadata": { "tags": [ @@ -1762,7 +1755,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "id": "e140642a", "metadata": { "tags": [ @@ -1773,17 +1766,10 @@ { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session3-pipeline',\n", - " 'ResponseMetadata': {'RequestId': 'ae1f315d-f268-483b-a670-6e7491cd3a13',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': 'ae1f315d-f268-483b-a670-6e7491cd3a13',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:45:47 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session3-pipeline'}" ] }, - "execution_count": 26, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1838,7 +1824,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "id": "ac0b891c", "metadata": {}, "outputs": [], @@ -1857,7 +1843,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "id": "d92b121d-dcb9-43e8-9ee3-3ececb583e7e", "metadata": { "tags": [ @@ -2033,7 +2019,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "id": "14ea27ce-c453-4cb0-b309-dbecd732957e", "metadata": { "tags": [ @@ -2053,16 +2039,16 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.1231 - accuracy: 0.4937 - val_loss: 1.0338 - val_accuracy: 0.5490 - 255ms/epoch - 32ms/step\n", + "8/8 - 0s - loss: 1.0608 - accuracy: 0.3389 - val_loss: 0.9358 - val_accuracy: 0.4510 - 285ms/epoch - 36ms/step\n", "2/2 [==============================] - 0s 1ms/step\n", - "Validation accuracy: 0.5490196078431373\n" + "Validation accuracy: 0.45098039215686275\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpn2jk6ol0/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp_0t_ovx_/model/001/assets\n" ] }, { @@ -2084,24 +2070,24 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.0947 - accuracy: 0.3473 - val_loss: 1.0409 - val_accuracy: 0.3725 - 207ms/epoch - 26ms/step\n", - "2/2 [==============================] - 0s 1ms/step\n" + "8/8 - 0s - loss: 1.1533 - accuracy: 0.1172 - val_loss: 1.1482 - val_accuracy: 0.0588 - 220ms/epoch - 27ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "Validation accuracy: 0.058823529411764705\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpdmda78an/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpcerqj9uj/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.37254901960784315\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.19s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.31s\u001b[0m\u001b[0m\n" ] } ], @@ -2181,7 +2167,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "id": "d00eda86", "metadata": { "tags": [ @@ -2225,7 +2211,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "id": "90fe82ae-6a2c-4461-bc83-bb52d8871e3b", "metadata": { "tags": [] @@ -2291,7 +2277,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "id": "99e4850c-83d6-4f4e-a813-d5a3f4bb7486", "metadata": { "tags": [ @@ -2358,7 +2344,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "id": "d92cac3d", "metadata": { "tags": [ @@ -2366,28 +2352,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session4-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '42ceacf9-3b68-4727-a8be-82da3aac0a64',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '42ceacf9-3b68-4727-a8be-82da3aac0a64',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:45:52 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session4-pipeline'}" ] }, - "execution_count": 33, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -2431,7 +2402,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "id": "57c2915c", "metadata": {}, "outputs": [ @@ -2441,7 +2412,7 @@ "PosixPath('code/containers/training/train.py')" ] }, - "execution_count": 34, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -2468,7 +2439,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 36, "id": "3ffa9250", "metadata": { "tags": [ @@ -2510,7 +2481,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 37, "id": "de5d473c", "metadata": { "tags": [ @@ -2565,7 +2536,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 38, "id": "e37a06ac", "metadata": { "tags": [ @@ -2577,69 +2548,55 @@ "name": "stdout", "output_type": "stream", "text": [ - "Building Docker image for linux/amd64 architecture...\n", + "Building Docker image for arm64 architecture...\n", "\u001b[1A\u001b[1B\u001b[0G\u001b[?25l[+] Building 0.0s (0/1) \n", "\u001b[?25h\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.1s (2/3) \n", "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.0s\n", + "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.1s\n", "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.3s (2/3) \n", "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.2s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.4s (2/3) \n", + "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.5s (2/3) \n", "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.3s\n", + "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.4s\n", "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.6s (2/3) \n", "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.5s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.7s (2/3) \n", + "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.7s (3/3) \n", "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.6s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.9s (2/3) \n", + "\u001b[0m\u001b[34m => [internal] load metadata for docker.io/library/python:3.10-slim 0.6s\n", + "\u001b[0m\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 0.8s (11/11) FINISHED \n", "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m => [internal] load metadata for docker.io/library/python:3.10-slim 0.8s\n", - "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 1.0s (4/10) \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load metadata for docker.io/library/python:3.10-slim 0.8s\n", - "\u001b[0m => [internal] load build context 0.0s\n", - "\u001b[34m => [1/6] FROM docker.io/library/python:3.10-slim@sha256:f80e619e067eebe4 0.0s\n", - "\u001b[0m\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l[+] Building 1.0s (11/11) FINISHED \n", - "\u001b[34m => [internal] load build definition from Dockerfile 0.0s\n", - "\u001b[0m\u001b[34m => => transferring dockerfile: 647B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load .dockerignore 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 2B 0.0s\n", - "\u001b[0m\u001b[34m => [internal] load metadata for docker.io/library/python:3.10-slim 0.8s\n", - "\u001b[0m\u001b[34m => [internal] load build context 0.0s\n", - "\u001b[0m\u001b[34m => => transferring context: 4.95kB 0.0s\n", + "\u001b[0m\u001b[34m => [internal] load metadata for docker.io/library/python:3.10-slim 0.6s\n", "\u001b[0m\u001b[34m => [1/6] FROM docker.io/library/python:3.10-slim@sha256:f80e619e067eebe4 0.0s\n", + "\u001b[0m\u001b[34m => [internal] load build context 0.0s\n", + "\u001b[0m\u001b[34m => => transferring context: 4.94kB 0.0s\n", "\u001b[0m\u001b[34m => CACHED [2/6] RUN apt-get -y update && apt-get install -y --no-install 0.0s\n", "\u001b[0m\u001b[34m => CACHED [3/6] COPY requirements.txt . 0.0s\n", "\u001b[0m\u001b[34m => CACHED [4/6] RUN pip install --user --upgrade pip 0.0s\n", "\u001b[0m\u001b[34m => CACHED [5/6] RUN pip3 install -r requirements.txt 0.0s\n", - "\u001b[0m\u001b[34m => CACHED [6/6] COPY train.py /opt/ml/code/train.py 0.0s\n", + "\u001b[0m\u001b[34m => [6/6] COPY train.py /opt/ml/code/train.py 0.0s\n", "\u001b[0m\u001b[34m => exporting to image 0.0s\n", "\u001b[0m\u001b[34m => => exporting layers 0.0s\n", - "\u001b[0m\u001b[34m => => writing image sha256:952fdf459612946969bb2cdf46ca507616c57dd316232 0.0s\n", + "\u001b[0m\u001b[34m => => writing image sha256:b610ac09ff5e496e19e1d172b544150c5d88d6d3f7480 0.0s\n", "\u001b[0m\u001b[34m => => naming to docker.io/library/keras-custom-training-container 0.0s\n", "\u001b[0m\u001b[?25h\n", "Use 'docker scan' to run Snyk tests against images to find vulnerabilities and learn how to fix them\n" @@ -2647,8 +2604,6 @@ } ], "source": [ - "# | eval: false\n", - "\n", "IMAGE_NAME = \"keras-custom-training-container\"\n", "\n", "if not LOCAL_MODE:\n", @@ -2680,49 +2635,14 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 39, "id": "ceede5e2", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Login Succeeded\n", - "The push refers to repository [325223348818.dkr.ecr.us-east-1.amazonaws.com/keras-custom-training-container]\n", - "dc9f227d801d: Preparing\n", - "56cff974c816: Preparing\n", - "25f8ff0f81d0: Preparing\n", - "78ea0a62e625: Preparing\n", - "d2e84e969834: Preparing\n", - "9362be696331: Preparing\n", - "23cef6a8f8d1: Preparing\n", - "884af60a304e: Preparing\n", - "c8f253aef560: Preparing\n", - "a483da8ab3e9: Preparing\n", - "23cef6a8f8d1: Waiting\n", - "884af60a304e: Waiting\n", - "9362be696331: Waiting\n", - "a483da8ab3e9: Waiting\n", - "c8f253aef560: Waiting\n", - "d2e84e969834: Layer already exists\n", - "78ea0a62e625: Layer already exists\n", - "dc9f227d801d: Layer already exists\n", - "56cff974c816: Layer already exists\n", - "25f8ff0f81d0: Layer already exists\n", - "884af60a304e: Layer already exists\n", - "23cef6a8f8d1: Layer already exists\n", - "9362be696331: Layer already exists\n", - "c8f253aef560: Layer already exists\n", - "a483da8ab3e9: Layer already exists\n", - "latest: digest: sha256:66434da50d99b87f7c1956ad680873e2921b23938a4090d7df8977446bccddeb size: 2422\n" - ] - } - ], + "outputs": [], "source": [ "%%bash -s \"$LOCAL_MODE\" \"$IMAGE_NAME\"\n", "# | eval: false\n", @@ -2776,17 +2696,17 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 40, "id": "49d9ab0e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'325223348818.dkr.ecr.us-east-1.amazonaws.com/keras-custom-training-container:latest'" + "'keras-custom-training-container'" ] }, - "execution_count": 39, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -2818,7 +2738,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 41, "id": "f4e17204", "metadata": { "tags": [ @@ -2861,7 +2781,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 42, "id": "86056197", "metadata": { "tags": [ @@ -2872,17 +2792,10 @@ { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session5-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '0ab27679-26ed-4319-b1c4-ed92c6183540',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '0ab27679-26ed-4319-b1c4-ed92c6183540',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:46:01 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session5-pipeline'}" ] }, - "execution_count": 41, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -2928,7 +2841,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 43, "id": "f367d0e3", "metadata": {}, "outputs": [], @@ -2963,7 +2876,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 44, "id": "c8c82750", "metadata": {}, "outputs": [], @@ -2996,7 +2909,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 45, "id": "038ff2e5-ed28-445b-bc03-4e996ec2286f", "metadata": { "tags": [] @@ -3045,7 +2958,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 46, "id": "9799ab39-fcae-41f4-a68b-85ab71b3ba9a", "metadata": { "tags": [ @@ -3053,34 +2966,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "WARNING:sagemaker.estimator:No finished training job found associated with this estimator. Please make sure this estimator is only used for building workflow config\n", - "WARNING:sagemaker.estimator:No finished training job found associated with this estimator. Please make sure this estimator is only used for building workflow config\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "WARNING:sagemaker.estimator:No finished training job found associated with this estimator. Please make sure this estimator is only used for building workflow config\n", - "WARNING:sagemaker.estimator:No finished training job found associated with this estimator. Please make sure this estimator is only used for building workflow config\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session6-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '465ede44-428a-4bf9-bd30-16a224949c72',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '465ede44-428a-4bf9-bd30-16a224949c72',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:46:03 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session6-pipeline'}" ] }, - "execution_count": 45, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -3136,7 +3028,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 47, "id": "0590fcfa", "metadata": {}, "outputs": [], @@ -3155,7 +3047,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 48, "id": "3ee3ab26-afa5-4ceb-9f7a-005d5fdea646", "metadata": { "tags": [ @@ -3232,7 +3124,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 49, "id": "9a2540d8-278a-4953-bc54-0469d154427d", "metadata": { "tags": [ @@ -3252,28 +3144,16 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 0.9825 - accuracy: 0.5272 - val_loss: 0.9976 - val_accuracy: 0.4706 - 221ms/epoch - 28ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpsozh1nwx/model/001/assets\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Validation accuracy: 0.47058823529411764\n" + "8/8 - 0s - loss: 1.2319 - accuracy: 0.0335 - val_loss: 1.2054 - val_accuracy: 0.0980 - 243ms/epoch - 30ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "Validation accuracy: 0.09803921568627451\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpcmeaj3zh/model/001/assets\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n" ] @@ -3282,8 +3162,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "2/2 [==============================] - 0s 1ms/step\n", - "Test accuracy: 0.5882352941176471\n", + "2/2 [==============================] - 0s 2ms/step\n", + "Test accuracy: 0.058823529411764705\n", "\u001b[32m.\u001b[0m" ] }, @@ -3299,44 +3179,32 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.3414 - accuracy: 0.2343 - val_loss: 1.1537 - val_accuracy: 0.3333 - 214ms/epoch - 27ms/step\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:5 out of the last 9 calls to .predict_function at 0x2b372b0a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2/2 [==============================] - 0s 2ms/step\n" + "8/8 - 0s - loss: 1.2704 - accuracy: 0.4351 - val_loss: 1.1779 - val_accuracy: 0.5098 - 222ms/epoch - 28ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmps71h92cf/model/001/assets\n" + "WARNING:tensorflow:5 out of the last 9 calls to .predict_function at 0x2b62ac940> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.3333333333333333\n" + "2/2 [==============================] - 0s 1ms/step\n", + "Validation accuracy: 0.5098039215686274\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpc93ppxgz/model/001/assets\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n", - "WARNING:tensorflow:6 out of the last 11 calls to .predict_function at 0x2b372beb0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:6 out of the last 11 calls to .predict_function at 0x2b62af250> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -3346,7 +3214,7 @@ "2/2 [==============================] - 0s 2ms/step\n", "Test accuracy: 0.39215686274509803\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.25s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.38s\u001b[0m\u001b[0m\n" ] } ], @@ -3429,7 +3297,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 50, "id": "4f19e15b", "metadata": {}, "outputs": [], @@ -3457,7 +3325,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 51, "id": "1f27b2ef", "metadata": {}, "outputs": [], @@ -3483,22 +3351,14 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 52, "id": "2fdff07f", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" - ] - } - ], + "outputs": [], "source": [ "from sagemaker.tensorflow import TensorFlowProcessor\n", "\n", @@ -3524,7 +3384,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 53, "id": "48139a07-5c8e-4bc6-b666-bf9531f7f520", "metadata": { "tags": [ @@ -3589,7 +3449,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 54, "id": "21a1ab1f", "metadata": { "tags": [ @@ -3597,32 +3457,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session7-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session7-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session7-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session7-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session7-pipeline',\n", - " 'ResponseMetadata': {'RequestId': 'c9cd2981-f5f4-4732-8f79-9640c4433902',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': 'c9cd2981-f5f4-4732-8f79-9640c4433902',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:46:06 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session7-pipeline'}" ] }, - "execution_count": 53, + "execution_count": 54, "metadata": {}, "output_type": "execute_result" } @@ -3667,7 +3508,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 55, "id": "bb70f907", "metadata": {}, "outputs": [], @@ -3687,7 +3528,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 56, "id": "4ca4cb61", "metadata": {}, "outputs": [], @@ -3714,7 +3555,7 @@ }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 57, "id": "8c05a7e1", "metadata": {}, "outputs": [], @@ -3750,7 +3591,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 58, "id": "c9773a4a", "metadata": { "tags": [ @@ -3818,7 +3659,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 59, "id": "4d09cc85", "metadata": { "tags": [ @@ -3826,33 +3667,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session8-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session8-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session8-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session8-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session8-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '09e144e1-2dba-4d07-9cc0-e4f048eb19b0',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '09e144e1-2dba-4d07-9cc0-e4f048eb19b0',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:46:08 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session8-pipeline'}" ] }, - "execution_count": 58, + "execution_count": 59, "metadata": {}, "output_type": "execute_result" } @@ -3868,7 +3689,7 @@ " register_model_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session8_pipeline.upsert(role_arn=role)" @@ -3900,7 +3721,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 60, "id": "745486b5", "metadata": {}, "outputs": [], @@ -3922,7 +3743,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 61, "id": "c4431bbf", "metadata": {}, "outputs": [], @@ -3953,7 +3774,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 62, "id": "bebeecab", "metadata": {}, "outputs": [], @@ -3983,7 +3804,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 63, "id": "36e2a2b1-6711-4266-95d8-d2aebd52e199", "metadata": { "tags": [] @@ -4012,7 +3833,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 64, "id": "f70bcd33-b499-4e2b-953e-94d1ed96c10a", "metadata": { "tags": [ @@ -4020,32 +3841,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session9-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session9-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session9-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session9-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session9-pipeline',\n", - " 'ResponseMetadata': {'RequestId': 'a78e446d-619f-42b1-b69b-89cf2827ce6c',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': 'a78e446d-619f-42b1-b69b-89cf2827ce6c',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '85',\n", - " 'date': 'Thu, 28 Mar 2024 15:46:10 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session9-pipeline'}" ] }, - "execution_count": 63, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -4061,7 +3863,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session9_pipeline.upsert(role_arn=role)" @@ -4093,7 +3895,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 65, "id": "a67726d7", "metadata": {}, "outputs": [ @@ -4101,14 +3903,14 @@ "data": { "text/plain": [ "{'ModelPackageGroupName': 'basic-penguins',\n", - " 'ModelPackageVersion': 5,\n", - " 'ModelPackageArn': 'arn:aws:sagemaker:us-east-1:325223348818:model-package/basic-penguins/5',\n", - " 'CreationTime': datetime.datetime(2024, 3, 27, 16, 41, 23, 867000, tzinfo=tzlocal()),\n", + " 'ModelPackageVersion': 6,\n", + " 'ModelPackageArn': 'arn:aws:sagemaker:us-east-1:325223348818:model-package/basic-penguins/6',\n", + " 'CreationTime': datetime.datetime(2024, 3, 29, 11, 19, 48, 782000, tzinfo=tzlocal()),\n", " 'ModelPackageStatus': 'Completed',\n", " 'ModelApprovalStatus': 'Approved'}" ] }, - "execution_count": 64, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -4150,7 +3952,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 66, "id": "f2e94adc", "metadata": {}, "outputs": [], @@ -4168,7 +3970,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 67, "id": "bd12fbe9", "metadata": {}, "outputs": [], @@ -4198,7 +4000,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 68, "id": "c79a0e74", "metadata": { "tags": [ @@ -4323,7 +4125,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 69, "id": "befd5ad3", "metadata": {}, "outputs": [], @@ -4345,7 +4147,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 70, "id": "dee516e9", "metadata": {}, "outputs": [ @@ -4353,7 +4155,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "arn:aws:sagemaker:us-east-1:325223348818:model-package/basic-penguins/5\n" + "arn:aws:sagemaker:us-east-1:325223348818:model-package/basic-penguins/6\n" ] } ], @@ -4394,7 +4196,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 71, "id": "7c8852d5-818a-406c-944d-30bf6de90288", "metadata": { "tags": [] @@ -4427,7 +4229,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 72, "id": "ba7da291", "metadata": {}, "outputs": [], @@ -4449,7 +4251,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 73, "id": "0817a25e-8224-4911-830b-d659e7458b4a", "metadata": { "tags": [] @@ -4459,27 +4261,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"predictions\": [\n", - " [\n", - " 0.124825425,\n", - " 0.0847824216,\n", - " 0.79039216\n", - " ],\n", - " [\n", - " 0.766525269,\n", - " 0.220783874,\n", - " 0.0126908608\n", - " ],\n", - " [\n", - " 0.944253445,\n", - " 0.0292692278,\n", - " 0.0264772158\n", - " ]\n", - " ]\n", - "}\n", - "\n", - "Species: [2 0 0]\n" + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" ] } ], @@ -4527,7 +4309,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 74, "id": "e1dbadb0", "metadata": {}, "outputs": [], @@ -4560,7 +4342,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 75, "id": "ac7630ca", "metadata": {}, "outputs": [], @@ -4578,7 +4360,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 76, "id": "44b651af", "metadata": { "tags": [ @@ -4707,7 +4489,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 77, "id": "200d0370", "metadata": { "tags": [ @@ -4778,7 +4560,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 78, "id": "4a742974", "metadata": { "tags": [ @@ -4789,13 +4571,13 @@ { "data": { "text/plain": [ - "{'ResponseMetadata': {'RequestId': 'd4486435-ef58-44e5-a4ec-0fe5c5e80aed',\n", + "{'ResponseMetadata': {'RequestId': '113fc003-bba4-470a-be79-36588ac79f6d',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'date': 'Fri, 29 Mar 2024 16:05:03 GMT',\n", + " 'HTTPHeaders': {'date': 'Sun, 31 Mar 2024 15:33:21 GMT',\n", " 'content-type': 'application/json',\n", " 'content-length': '1593',\n", " 'connection': 'keep-alive',\n", - " 'x-amzn-requestid': 'd4486435-ef58-44e5-a4ec-0fe5c5e80aed'},\n", + " 'x-amzn-requestid': '113fc003-bba4-470a-be79-36588ac79f6d'},\n", " 'RetryAttempts': 0},\n", " 'FunctionName': 'deployment_fn',\n", " 'FunctionArn': 'arn:aws:lambda:us-east-1:325223348818:function:deployment_fn',\n", @@ -4806,15 +4588,15 @@ " 'Description': '',\n", " 'Timeout': 600,\n", " 'MemorySize': 128,\n", - " 'LastModified': '2024-03-29T16:05:03.000+0000',\n", - " 'CodeSha256': 'l4kFNMlylzvLEXPozhpRDuhWg63kP9XVkBfx+Ib9ZrY=',\n", + " 'LastModified': '2024-03-31T15:33:21.000+0000',\n", + " 'CodeSha256': 'TBaeM5KkfChf4nvbY+qFGsICzi0L3/cjgbb1bu75UAs=',\n", " 'Version': '$LATEST',\n", " 'Environment': {'Variables': {'ROLE': 'arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501',\n", " 'DATA_CAPTURE_PERCENTAGE': '100',\n", " 'DATA_CAPTURE_DESTINATION': 's3://mlschool/penguins/monitoring/data-capture',\n", " 'ENDPOINT': 'penguins-endpoint'}},\n", " 'TracingConfig': {'Mode': 'PassThrough'},\n", - " 'RevisionId': '0d400944-0930-4ddf-a318-fa53bf087756',\n", + " 'RevisionId': 'f72c34d8-9f36-4ff2-9f61-2e0b8ebb3ebf',\n", " 'Layers': [],\n", " 'State': 'Active',\n", " 'LastUpdateStatus': 'InProgress',\n", @@ -4829,7 +4611,7 @@ " 'LogGroup': '/aws/lambda/deployment_fn'}}" ] }, - "execution_count": 154, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -4873,7 +4655,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 79, "id": "1f86b997", "metadata": {}, "outputs": [], @@ -4907,7 +4689,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 80, "id": "5cc3e44b", "metadata": {}, "outputs": [], @@ -4932,7 +4714,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 81, "id": "3d86577a", "metadata": { "tags": [ @@ -4940,34 +4722,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session12-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session12-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n", - " warnings.warn(\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session12-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session12-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session12-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '68ba9372-19f6-4f97-9a88-a7a0d582cadf',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '68ba9372-19f6-4f97-9a88-a7a0d582cadf',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '86',\n", - " 'date': 'Thu, 28 Mar 2024 15:46:16 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session12-pipeline'}" ] }, - "execution_count": 80, + "execution_count": 81, "metadata": {}, "output_type": "execute_result" } @@ -4983,7 +4744,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session12_pipeline.upsert(role_arn=role)" @@ -5003,7 +4764,7 @@ }, { "cell_type": "code", - "execution_count": 163, + "execution_count": 82, "id": "1dd5bb33", "metadata": {}, "outputs": [ @@ -5011,27 +4772,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"predictions\": [\n", - " [\n", - " 0.124825425,\n", - " 0.0847824216,\n", - " 0.79039216\n", - " ],\n", - " [\n", - " 0.766525269,\n", - " 0.220783874,\n", - " 0.0126908608\n", - " ],\n", - " [\n", - " 0.944253445,\n", - " 0.0292692278,\n", - " 0.0264772158\n", - " ]\n", - " ]\n", - "}\n", - "\n", - "Species: [2 0 0]\n" + "Waiter EndpointInService failed: Waiter encountered a terminal failure state: Matched expected service error code: ValidationException\n" ] } ], @@ -5084,7 +4825,7 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 83, "id": "96751e71", "metadata": {}, "outputs": [], @@ -5104,7 +4845,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 84, "id": "1ee4b9d5", "metadata": {}, "outputs": [], @@ -5131,7 +4872,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 85, "id": "63f294bc", "metadata": {}, "outputs": [], @@ -5157,7 +4898,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 86, "id": "9985bd49", "metadata": {}, "outputs": [], @@ -5185,7 +4926,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 87, "id": "32302639", "metadata": {}, "outputs": [ @@ -5225,7 +4966,7 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 88, "id": "18adeb93", "metadata": { "tags": [ @@ -5263,7 +5004,7 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 89, "id": "fafb5514", "metadata": {}, "outputs": [], @@ -5288,7 +5029,7 @@ }, { "cell_type": "code", - "execution_count": 202, + "execution_count": 90, "id": "b95c22b2", "metadata": { "tags": [ @@ -5296,33 +5037,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session13-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session13-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session13-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session13-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session13-pipeline',\n", - " 'ResponseMetadata': {'RequestId': 'd5f2ca63-6e11-40f2-a49e-84210bf01848',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': 'd5f2ca63-6e11-40f2-a49e-84210bf01848',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '86',\n", - " 'date': 'Fri, 29 Mar 2024 16:38:40 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session13-pipeline'}" ] }, - "execution_count": 202, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" } @@ -5338,7 +5059,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session13_pipeline.upsert(role_arn=role)" @@ -5411,7 +5132,7 @@ }, { "cell_type": "code", - "execution_count": 289, + "execution_count": 91, "id": "a1f25973", "metadata": {}, "outputs": [], @@ -5430,7 +5151,7 @@ }, { "cell_type": "code", - "execution_count": 290, + "execution_count": 92, "id": "c480ae02", "metadata": { "tags": [ @@ -5560,7 +5281,7 @@ }, { "cell_type": "code", - "execution_count": 291, + "execution_count": 93, "id": "34721e63", "metadata": { "tags": [ @@ -5572,15 +5293,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.06s\u001b[0m\u001b[0m\n" + "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", + "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.07s\u001b[0m\u001b[0m\n" ] } ], @@ -5715,7 +5429,7 @@ }, { "cell_type": "code", - "execution_count": 292, + "execution_count": 94, "id": "d23798aa", "metadata": { "tags": [ @@ -5815,7 +5529,7 @@ }, { "cell_type": "code", - "execution_count": 293, + "execution_count": 95, "id": "965a6be5", "metadata": { "tags": [ @@ -5897,7 +5611,7 @@ }, { "cell_type": "code", - "execution_count": 294, + "execution_count": 96, "id": "82dafcd0", "metadata": {}, "outputs": [], @@ -5923,7 +5637,7 @@ }, { "cell_type": "code", - "execution_count": 295, + "execution_count": 97, "id": "8c7dda7a", "metadata": {}, "outputs": [], @@ -5935,7 +5649,7 @@ " entry_point=\"preprocessing_component.py\",\n", " source_dir=(CODE_FOLDER / \"pipeline\").as_posix(),\n", " framework_version=\"1.2-1\",\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", " role=role,\n", ")" ] @@ -5950,7 +5664,7 @@ }, { "cell_type": "code", - "execution_count": 296, + "execution_count": 98, "id": "6c31090e", "metadata": {}, "outputs": [], @@ -5960,7 +5674,7 @@ " entry_point=\"postprocessing_component.py\",\n", " source_dir=(CODE_FOLDER / \"pipeline\").as_posix(),\n", " framework_version=\"1.2-1\",\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", " role=role,\n", ")" ] @@ -5975,7 +5689,7 @@ }, { "cell_type": "code", - "execution_count": 297, + "execution_count": 99, "id": "9fa95ab8", "metadata": {}, "outputs": [], @@ -5985,7 +5699,7 @@ "pipeline_model = PipelineModel(\n", " name=\"inference-model\",\n", " models=[preprocessing_model, tensorflow_model, postprocessing_model],\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", " role=role,\n", ")" ] @@ -6002,7 +5716,7 @@ }, { "cell_type": "code", - "execution_count": 298, + "execution_count": 100, "id": "7c052332", "metadata": {}, "outputs": [], @@ -6022,7 +5736,7 @@ }, { "cell_type": "code", - "execution_count": 251, + "execution_count": 101, "id": "3cb0ce9e", "metadata": { "tags": [ @@ -6061,7 +5775,7 @@ }, { "cell_type": "code", - "execution_count": 252, + "execution_count": 102, "id": "8ec7b7d8", "metadata": {}, "outputs": [], @@ -6081,7 +5795,7 @@ }, { "cell_type": "code", - "execution_count": 253, + "execution_count": 103, "id": "b9712905-9fe3-4148-ae6d-05b0a48e742e", "metadata": { "tags": [] @@ -6108,7 +5822,7 @@ }, { "cell_type": "code", - "execution_count": 254, + "execution_count": 104, "id": "bad9f51d", "metadata": { "tags": [ @@ -6116,33 +5830,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session14-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session14-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session14-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session14-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session14-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '0f8391bd-02ab-4928-a5f7-7f5e38bc6fbf',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '0f8391bd-02ab-4928-a5f7-7f5e38bc6fbf',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '86',\n", - " 'date': 'Fri, 29 Mar 2024 18:23:16 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session14-pipeline'}" ] }, - "execution_count": 254, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } @@ -6158,7 +5852,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session14_pipeline.upsert(role_arn=role)" @@ -6176,7 +5870,7 @@ }, { "cell_type": "code", - "execution_count": 300, + "execution_count": 105, "id": "3cc966fb-b611-417f-a8b8-0c5d2f95252c", "metadata": { "tags": [] @@ -6191,20 +5885,7 @@ "Torgersen,39.5,17.4,186.0,3800.0,FEMALE\n", "Torgersen,40.3,18.0,195.0,3250.0,FEMALE\n", "\n", - "[\n", - " {\n", - " \"prediction\": \"Adelie\",\n", - " \"confidence\": 0.98598367\n", - " },\n", - " {\n", - " \"prediction\": \"Adelie\",\n", - " \"confidence\": 0.978753448\n", - " },\n", - " {\n", - " \"prediction\": \"Adelie\",\n", - " \"confidence\": 0.947375655\n", - " }\n", - "]\n" + "Waiter EndpointInService failed: Waiter encountered a terminal failure state: Matched expected service error code: ValidationException\n" ] } ], @@ -6243,7 +5924,7 @@ }, { "cell_type": "code", - "execution_count": 301, + "execution_count": 106, "id": "ce3d33f0", "metadata": {}, "outputs": [ @@ -6251,7 +5932,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'prediction': 'Gentoo', 'confidence': 0.989867449}\n" + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" ] } ], @@ -6292,7 +5973,7 @@ }, { "cell_type": "code", - "execution_count": 303, + "execution_count": 107, "id": "240aaff2", "metadata": {}, "outputs": [ @@ -6300,7 +5981,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[['Gentoo', '0.989867449']]\n" + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" ] } ], @@ -6353,7 +6034,7 @@ }, { "cell_type": "code", - "execution_count": 305, + "execution_count": 108, "id": "97b3cd80", "metadata": { "tags": [ @@ -6492,7 +6173,7 @@ }, { "cell_type": "code", - "execution_count": 306, + "execution_count": 109, "id": "91786cc7", "metadata": { "tags": [ @@ -6500,6 +6181,13 @@ ] }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Keras version: 2.14.0" + ] + }, { "name": "stderr", "output_type": "stream", @@ -6511,23 +6199,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 0.9664 - accuracy: 0.5607 - val_loss: 0.9293 - val_accuracy: 0.5490 - 215ms/epoch - 27ms/step\n", - "2/2 [==============================] - 0s 1ms/step\n" + "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpzbvqezsx/model/001/assets\n" + "WARNING:tensorflow:5 out of the last 9 calls to .test_function at 0x2b474e950> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8/8 - 0s - loss: 1.5890 - accuracy: 0.4351 - val_loss: 1.3773 - val_accuracy: 0.3333 - 221ms/epoch - 28ms/step\n", + "2/2 [==============================] - 0s 2ms/step\n", + "Validation accuracy: 0.3333333333333333\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpoum_3699/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.5490196078431373\n", "Handling endpoint request\n", "Processing input data...\n", "Sending input data to model to make a prediction...\n" @@ -6546,10 +6247,9 @@ "output_type": "stream", "text": [ "1/1 [==============================] - 0s 28ms/step\n", - "Response: {'predictions': [[0.4287213087081909, 0.2709770202636719, 0.300301730632782]]}\n", + "Response: {'predictions': [[0.35001689195632935, 0.25480717420578003, 0.3951759338378906]]}\n", "Processing prediction received from the model...\n", - "HERE {'predictions': [[0.4287213087081909, 0.2709770202636719, 0.300301730632782]]}\n", - "{'prediction': 'Adelie', 'confidence': 0.4287213087081909}\n", + "{'prediction': 'Gentoo', 'confidence': 0.3951759338378906}\n", "\u001b[32m.\u001b[0m" ] }, @@ -6564,23 +6264,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.1627 - accuracy: 0.1883 - val_loss: 1.1280 - val_accuracy: 0.1961 - 215ms/epoch - 27ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n" + "Keras version: 2.14.0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpdalhfozy/model/001/assets\n" + "WARNING:tensorflow:6 out of the last 11 calls to .test_function at 0x2b47a7c70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8/8 - 0s - loss: 1.1736 - accuracy: 0.3138 - val_loss: 1.2214 - val_accuracy: 0.2745 - 224ms/epoch - 28ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "Validation accuracy: 0.27450980392156865\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpm8jumn5k/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.19607843137254902\n", "Handling endpoint request\n", "Processing input data...\n", "Sending input data to model to make a prediction...\n" @@ -6598,11 +6311,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 26ms/step\n", - "Response: {'predictions': [[0.2188926637172699, 0.49149057269096375, 0.28961673378944397]]}\n", + "1/1 [==============================] - 0s 27ms/step\n", + "Response: {'predictions': [[0.2764297425746918, 0.32800644636154175, 0.3955638110637665]]}\n", "Processing prediction received from the model...\n", - "HERE {'predictions': [[0.2188926637172699, 0.49149057269096375, 0.28961673378944397]]}\n", - "{'prediction': 'Chinstrap', 'confidence': 0.49149057269096375}\n", + "{'prediction': 'Gentoo', 'confidence': 0.3955638110637665}\n", "\u001b[32m.\u001b[0m" ] }, @@ -6618,22 +6330,22 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.1131 - accuracy: 0.2887 - val_loss: 1.0818 - val_accuracy: 0.3922 - 218ms/epoch - 27ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n" + "8/8 - 0s - loss: 1.2106 - accuracy: 0.2762 - val_loss: 1.2037 - val_accuracy: 0.2745 - 218ms/epoch - 27ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpfstpzt4b/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp2u9k20ic/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.39215686274509803\n", + "Validation accuracy: 0.27450980392156865\n", "Handling endpoint request\n", "Processing input data...\n", "Sending input data to model to make a prediction...\n" @@ -6651,11 +6363,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 25ms/step\n", - "Response: {'predictions': [[0.3396219313144684, 0.33004486560821533, 0.3303331732749939]]}\n", + "1/1 [==============================] - 0s 24ms/step\n", + "Response: {'predictions': [[0.5782711505889893, 0.2070244550704956, 0.21470439434051514]]}\n", "Processing prediction received from the model...\n", - "HERE {'predictions': [[0.3396219313144684, 0.33004486560821533, 0.3303331732749939]]}\n", - "{'prediction': 'Adelie', 'confidence': 0.3396219313144684}\n", + "{'prediction': 'Adelie', 'confidence': 0.5782711505889893}\n", "\u001b[32m.\u001b[0m" ] }, @@ -6671,29 +6382,29 @@ "output_type": "stream", "text": [ "Keras version: 2.14.0\n", - "8/8 - 0s - loss: 1.1813 - accuracy: 0.1506 - val_loss: 1.1372 - val_accuracy: 0.1176 - 221ms/epoch - 28ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n" + "8/8 - 0s - loss: 1.1899 - accuracy: 0.2218 - val_loss: 1.1516 - val_accuracy: 0.2157 - 218ms/epoch - 27ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp4q5229yr/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp39xu_z41/model/001/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Validation accuracy: 0.11764705882352941\n", + "Validation accuracy: 0.21568627450980393\n", "Handling endpoint request\n", "Processing input data...\n", "There was an error processing the input data. Expecting value: line 1 column 1 (char 0)\n", "Processing prediction received from the model...\n", "{'prediction': None}\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m4 passed\u001b[0m\u001b[32m in 2.37s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m4 passed\u001b[0m\u001b[32m in 2.34s\u001b[0m\u001b[0m\n" ] } ], @@ -6839,7 +6550,7 @@ }, { "cell_type": "code", - "execution_count": 307, + "execution_count": 110, "id": "603cb8f5", "metadata": { "tags": [ @@ -6875,7 +6586,7 @@ }, { "cell_type": "code", - "execution_count": 308, + "execution_count": 111, "id": "b5015c75", "metadata": {}, "outputs": [], @@ -6886,7 +6597,7 @@ " entry_point=\"inference.py\",\n", " source_dir=(CODE_FOLDER / \"pipeline\").as_posix(),\n", " framework_version=config[\"framework_version\"],\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", " role=role,\n", ")" ] @@ -6903,7 +6614,7 @@ }, { "cell_type": "code", - "execution_count": 309, + "execution_count": 112, "id": "7ee3cdb6", "metadata": {}, "outputs": [], @@ -6923,7 +6634,7 @@ }, { "cell_type": "code", - "execution_count": 310, + "execution_count": 113, "id": "21fb3e2c", "metadata": { "tags": [ @@ -6963,7 +6674,7 @@ }, { "cell_type": "code", - "execution_count": 311, + "execution_count": 114, "id": "9e8162b7", "metadata": {}, "outputs": [], @@ -6983,7 +6694,7 @@ }, { "cell_type": "code", - "execution_count": 312, + "execution_count": 115, "id": "849b379f", "metadata": {}, "outputs": [], @@ -7008,7 +6719,7 @@ }, { "cell_type": "code", - "execution_count": 313, + "execution_count": 116, "id": "f64921ee", "metadata": { "tags": [ @@ -7016,33 +6727,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session15-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session15-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session15-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session15-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session15-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '7c77830b-b272-416a-9ff9-dcf1b39ea3d2',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '7c77830b-b272-416a-9ff9-dcf1b39ea3d2',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '86',\n", - " 'date': 'Sat, 30 Mar 2024 15:24:30 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session15-pipeline'}" ] }, - "execution_count": 313, + "execution_count": 116, "metadata": {}, "output_type": "execute_result" } @@ -7058,7 +6749,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session15_pipeline.upsert(role_arn=role)" @@ -7076,7 +6767,7 @@ }, { "cell_type": "code", - "execution_count": 316, + "execution_count": 117, "id": "1d63ce59", "metadata": {}, "outputs": [ @@ -7084,7 +6775,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'prediction': 'Adelie', 'confidence': 0.629927337}\n" + "Waiter EndpointInService failed: Waiter encountered a terminal failure state: Matched expected service error code: ValidationException\n" ] } ], @@ -7144,7 +6835,7 @@ }, { "cell_type": "code", - "execution_count": 317, + "execution_count": 118, "id": "f4747fc7", "metadata": {}, "outputs": [], @@ -7174,7 +6865,7 @@ }, { "cell_type": "code", - "execution_count": 318, + "execution_count": 119, "id": "3fc728fa", "metadata": { "tags": [ @@ -7186,13 +6877,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n", "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n" ] } @@ -7211,7 +6896,7 @@ " instance_type=\"ml.c5.xlarge\",\n", " instance_count=1,\n", " volume_size_in_gb=20,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", " role=role,\n", " ),\n", " quality_check_config=DataQualityCheckConfig(\n", @@ -7240,7 +6925,7 @@ }, { "cell_type": "code", - "execution_count": 319, + "execution_count": 120, "id": "2d5d8133", "metadata": {}, "outputs": [], @@ -7282,7 +6967,7 @@ }, { "cell_type": "code", - "execution_count": 320, + "execution_count": 121, "id": "fc5c4325", "metadata": { "tags": [ @@ -7313,7 +6998,7 @@ }, { "cell_type": "code", - "execution_count": 321, + "execution_count": 122, "id": "be9d6de7", "metadata": {}, "outputs": [], @@ -7338,7 +7023,7 @@ }, { "cell_type": "code", - "execution_count": 322, + "execution_count": 123, "id": "fb9b8d1e", "metadata": { "tags": [ @@ -7346,33 +7031,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session16-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session16-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session16-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session16-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session16-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '49b051bc-cb53-4bf6-8fa8-b9f64033f0e3',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '49b051bc-cb53-4bf6-8fa8-b9f64033f0e3',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '86',\n", - " 'date': 'Sat, 30 Mar 2024 16:14:21 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session16-pipeline'}" ] }, - "execution_count": 322, + "execution_count": 123, "metadata": {}, "output_type": "execute_result" } @@ -7389,7 +7054,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session16_pipeline.upsert(role_arn=role)" @@ -7415,7 +7080,7 @@ }, { "cell_type": "code", - "execution_count": 327, + "execution_count": 124, "id": "c2b95d03", "metadata": {}, "outputs": [ @@ -7475,7 +7140,7 @@ }, { "cell_type": "code", - "execution_count": 325, + "execution_count": 125, "id": "63908598", "metadata": {}, "outputs": [ @@ -7600,7 +7265,7 @@ }, { "cell_type": "code", - "execution_count": 328, + "execution_count": 126, "id": "69e115c3", "metadata": {}, "outputs": [], @@ -7620,23 +7285,14 @@ }, { "cell_type": "code", - "execution_count": 329, + "execution_count": 127, "id": "f93f2751", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n", - " warnings.warn(\n" - ] - } - ], + "outputs": [], "source": [ "create_model_step = ModelStep(\n", " name=\"create-model\",\n", @@ -7666,7 +7322,7 @@ }, { "cell_type": "code", - "execution_count": 332, + "execution_count": 128, "id": "2a530d12", "metadata": {}, "outputs": [], @@ -7681,7 +7337,7 @@ " accept=\"text/csv\",\n", " assemble_with=\"Line\",\n", " output_path=f\"{S3_LOCATION}/transform\",\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")" ] }, @@ -7700,23 +7356,14 @@ }, { "cell_type": "code", - "execution_count": 333, + "execution_count": 129, "id": "653c6ec8", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n", - " warnings.warn(\n" - ] - } - ], + "outputs": [], "source": [ "from sagemaker.workflow.steps import TransformStep\n", "\n", @@ -7767,7 +7414,7 @@ }, { "cell_type": "code", - "execution_count": 334, + "execution_count": 130, "id": "b50a9ed5", "metadata": { "tags": [ @@ -7793,7 +7440,7 @@ " instance_type=\"ml.c5.xlarge\",\n", " instance_count=1,\n", " volume_size_in_gb=20,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", " role=role,\n", " ),\n", " quality_check_config=ModelQualityCheckConfig(\n", @@ -7829,7 +7476,7 @@ }, { "cell_type": "code", - "execution_count": 335, + "execution_count": 131, "id": "3aecc022", "metadata": {}, "outputs": [], @@ -7887,7 +7534,7 @@ }, { "cell_type": "code", - "execution_count": 336, + "execution_count": 132, "id": "20154a1a", "metadata": { "tags": [ @@ -7918,7 +7565,7 @@ }, { "cell_type": "code", - "execution_count": 338, + "execution_count": 133, "id": "cd84e567", "metadata": {}, "outputs": [], @@ -7950,7 +7597,7 @@ }, { "cell_type": "code", - "execution_count": 339, + "execution_count": 134, "id": "c244e206", "metadata": { "tags": [ @@ -7958,33 +7605,13 @@ ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session17-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session17-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session17-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session17-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" - ] - }, { "data": { "text/plain": [ - "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session17-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '60fbd90a-5eac-4df5-874c-f1cb865452ae',\n", - " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '60fbd90a-5eac-4df5-874c-f1cb865452ae',\n", - " 'content-type': 'application/x-amz-json-1.1',\n", - " 'content-length': '86',\n", - " 'date': 'Sat, 30 Mar 2024 16:40:57 GMT'},\n", - " 'RetryAttempts': 0}}" + "{'PipelineArn': 'session17-pipeline'}" ] }, - "execution_count": 339, + "execution_count": 134, "metadata": {}, "output_type": "execute_result" } @@ -8001,7 +7628,7 @@ " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=pipeline_session,\n", + " sagemaker_session=config[\"session\"],\n", ")\n", "\n", "session17_pipeline.upsert(role_arn=role)" @@ -9488,6 +9115,8 @@ } ], "source": [ + "# | eval: false\n", + "\n", "sagemaker_client.create_endpoint(\n", " EndpointName=SHADOW_DEPLOYMENT_ENDPOINT,\n", " EndpointConfigName=endpoint_config_name,\n",