diff --git a/program/cohort.ipynb b/program/cohort.ipynb index f3843dc..b988556 100644 --- a/program/cohort.ipynb +++ b/program/cohort.ipynb @@ -42,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 249, + "execution_count": 277, "id": "4b2265b0", "metadata": {}, "outputs": [ @@ -68,6 +68,7 @@ "import sys\n", "import logging\n", "import ipytest\n", + "import json\n", "from pathlib import Path\n", "\n", "\n", @@ -100,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 250, + "execution_count": 278, "id": "32c4d764", "metadata": {}, "outputs": [], @@ -118,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": 251, + "execution_count": 279, "id": "3164a3af", "metadata": {}, "outputs": [], @@ -141,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 252, + "execution_count": 280, "id": "7bc40d28", "metadata": {}, "outputs": [], @@ -160,7 +161,7 @@ }, { "cell_type": "code", - "execution_count": 253, + "execution_count": 281, "id": "3b3f17e5", "metadata": {}, "outputs": [], @@ -200,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 254, + "execution_count": 282, "id": "942a01b5", "metadata": {}, "outputs": [], @@ -237,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 255, + "execution_count": 283, "id": "f1cd2f0e-446d-48a9-a008-b4f1cc593bfc", "metadata": { "tags": [] @@ -344,7 +345,7 @@ "4 3450.0 FEMALE " ] }, - "execution_count": 255, + "execution_count": 283, "metadata": {}, "output_type": "execute_result" } @@ -381,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 256, + "execution_count": 284, "id": "f2107c25-e730-4e22-a1b8-5bda53e61124", "metadata": { "tags": [] @@ -560,7 +561,7 @@ "max 6300.000000 NaN " ] }, - "execution_count": 256, + "execution_count": 284, "metadata": {}, "output_type": "execute_result" } @@ -579,7 +580,7 @@ }, { "cell_type": "code", - "execution_count": 257, + "execution_count": 285, "id": "1242122a-726e-4c37-a718-dd8e873d1612", "metadata": { "tags": [] @@ -637,7 +638,7 @@ }, { "cell_type": "code", - "execution_count": 258, + "execution_count": 286, "id": "cf1cf582-8831-4f83-bb17-2175afb193e8", "metadata": { "tags": [] @@ -652,7 +653,7 @@ "Name: count, dtype: int64" ] }, - "execution_count": 258, + "execution_count": 286, "metadata": {}, "output_type": "execute_result" } @@ -672,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 259, + "execution_count": 287, "id": "cc42cb08-275c-4b05-9d2b-77052da2f336", "metadata": { "tags": [] @@ -691,7 +692,7 @@ "dtype: int64" ] }, - "execution_count": 259, + "execution_count": 287, "metadata": {}, "output_type": "execute_result" } @@ -710,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 260, + "execution_count": 288, "id": "3c57d55d-afd6-467a-a7a8-ff04132770ed", "metadata": { "tags": [] @@ -729,7 +730,7 @@ "dtype: int64" ] }, - "execution_count": 260, + "execution_count": 288, "metadata": {}, "output_type": "execute_result" } @@ -752,7 +753,7 @@ }, { "cell_type": "code", - "execution_count": 261, + "execution_count": 289, "id": "2852c740", "metadata": {}, "outputs": [ @@ -798,7 +799,7 @@ }, { "cell_type": "code", - "execution_count": 262, + "execution_count": 290, "id": "707cc972", "metadata": {}, "outputs": [ @@ -846,7 +847,7 @@ }, { "cell_type": "code", - "execution_count": 263, + "execution_count": 291, "id": "3daf3ba1-d218-4ad4-b862-af679b91273f", "metadata": { "tags": [] @@ -926,7 +927,7 @@ "body_mass_g 640316.716388 " ] }, - "execution_count": 263, + "execution_count": 291, "metadata": {}, "output_type": "execute_result" } @@ -951,7 +952,7 @@ }, { "cell_type": "code", - "execution_count": 264, + "execution_count": 292, "id": "1d793e09-2cb9-47ff-a0e6-199a0f4fc1b3", "metadata": { "tags": [] @@ -1031,7 +1032,7 @@ "body_mass_g 1.000000 " ] }, - "execution_count": 264, + "execution_count": 292, "metadata": {}, "output_type": "execute_result" } @@ -1056,7 +1057,7 @@ }, { "cell_type": "code", - "execution_count": 265, + "execution_count": 293, "id": "1258c99d", "metadata": {}, "outputs": [ @@ -1096,7 +1097,7 @@ }, { "cell_type": "code", - "execution_count": 266, + "execution_count": 294, "id": "45b0a87f-028d-477f-9b65-199728c0b7ee", "metadata": { "tags": [] @@ -1150,7 +1151,7 @@ }, { "cell_type": "code", - "execution_count": 267, + "execution_count": 295, "id": "fb6ba7c0-1bd6-4fe5-8b7f-f6cbdfd3846c", "metadata": { "tags": [] @@ -1222,7 +1223,7 @@ "\n", " df_train, df_validation, df_test = _split_data(df)\n", "\n", - " _save_baseline(base_directory, df_test)\n", + " _save_baselines(base_directory, df_train, df_test)\n", "\n", " y_train = target_transformer.fit_transform(np.array(df_train.species.values).reshape(-1, 1))\n", " y_validation = target_transformer.transform(np.array(df_validation.species.values).reshape(-1, 1))\n", @@ -1270,15 +1271,25 @@ " return df_train, df_validation, df_test\n", "\n", "\n", - "def _save_baseline(base_directory, df_test):\n", + "def _save_baselines(base_directory, df_train, df_test):\n", " \"\"\"\n", - " This function saves the untransformed test split to disk. This file will\n", - " be used later as a baseline to monitor the performance of the model.\n", + " During the data and quality monitoring steps, we will need baselines\n", + " to compute constraints and statistics. This function saves the \n", + " untransformed data to disk so we can use them as baselines later.\n", " \"\"\"\n", "\n", - " baseline_path = Path(base_directory) / f\"baseline\"\n", - " baseline_path.mkdir(parents=True, exist_ok=True)\n", - " df_test.to_csv(baseline_path / \"baseline.csv\", header=False, index=False)\n", + " for split, data in [(\"train\", df_train), (\"test\", df_test)]:\n", + " baseline_path = Path(base_directory) / f\"{split}-baseline\"\n", + " baseline_path.mkdir(parents=True, exist_ok=True)\n", + "\n", + " df = data.copy().dropna()\n", + "\n", + " # We want to save the header only for the train baseline\n", + " # but not for the test baseline. We'll use the test baseline\n", + " # to generate predictions later, and we can't have a header line\n", + " # because the model won't be able to make a prediction for it.\n", + " header = split == \"train\"\n", + " df.to_csv(baseline_path / f\"{split}-baseline.csv\", header=header, index=False)\n", "\n", "\n", "def _save_splits(base_directory, X_train, y_train, X_validation, y_validation, X_test, y_test):\n", @@ -1336,7 +1347,7 @@ }, { "cell_type": "code", - "execution_count": 268, + "execution_count": 296, "id": "d1f122a4-acff-4687-91b9-bfef13567d88", "metadata": { "tags": [] @@ -1346,8 +1357,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m5 passed\u001b[0m\u001b[32m in 0.10s\u001b[0m\u001b[0m\n" + "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n", + "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.17s\u001b[0m\u001b[0m\n" ] } ], @@ -1389,10 +1400,11 @@ " assert \"test\" in output_directories\n", "\n", "\n", - "def test_preprocess_generates_baseline(directory):\n", + "def test_preprocess_generates_baselines(directory):\n", " output_directories = os.listdir(directory)\n", "\n", - " assert \"baseline\" in output_directories\n", + " assert \"train-baseline\" in output_directories\n", + " assert \"test-baseline\" in output_directories\n", "\n", "\n", "def test_preprocess_creates_two_models(directory):\n", @@ -1423,14 +1435,34 @@ " assert test.shape[1] == number_of_features + 1\n", "\n", "\n", - "def test_baseline_is_not_transformed(directory):\n", - " baseline = pd.read_csv(directory / \"baseline\" / \"baseline.csv\", header=None)\n", + "def test_train_baseline_is_not_transformed(directory):\n", + " baseline = pd.read_csv(directory / \"train-baseline\" / \"train-baseline.csv\", header=None)\n", + "\n", + " island = baseline.iloc[:, 1].unique()\n", + "\n", + " assert \"Biscoe\" in island\n", + " assert \"Torgersen\" in island\n", + " assert \"Dream\" in island\n", + "\n", + "\n", + "def test_test_baseline_is_not_transformed(directory):\n", + " baseline = pd.read_csv(directory / \"test-baseline\" / \"test-baseline.csv\", header=None)\n", "\n", " island = baseline.iloc[:, 1].unique()\n", "\n", " assert \"Biscoe\" in island\n", " assert \"Torgersen\" in island\n", - " assert \"Dream\" in island" + " assert \"Dream\" in island\n", + "\n", + "\n", + "def test_train_baseline_includes_header(directory):\n", + " baseline = pd.read_csv(directory / \"train-baseline\" / \"train-baseline.csv\")\n", + " assert baseline.columns[0] == \"species\"\n", + "\n", + "\n", + "def test_test_baseline_does_not_include_header(directory):\n", + " baseline = pd.read_csv(directory / \"test-baseline\" / \"test-baseline.csv\")\n", + " assert baseline.columns[0] != \"species\"" ] }, { @@ -1453,7 +1485,7 @@ }, { "cell_type": "code", - "execution_count": 269, + "execution_count": 297, "id": "d88e9ccf", "metadata": {}, "outputs": [], @@ -1473,7 +1505,7 @@ }, { "cell_type": "code", - "execution_count": 270, + "execution_count": 298, "id": "331fe373", "metadata": {}, "outputs": [], @@ -1496,7 +1528,7 @@ }, { "cell_type": "code", - "execution_count": 271, + "execution_count": 299, "id": "3aa4471a", "metadata": {}, "outputs": [ @@ -1537,7 +1569,7 @@ }, { "cell_type": "code", - "execution_count": 272, + "execution_count": 300, "id": "cdbd9303", "metadata": { "tags": [] @@ -1590,12 +1622,15 @@ " source=\"/opt/ml/processing/model\",\n", " destination=f\"{S3_LOCATION}/preprocessing/model\",\n", " ),\n", - " # The baseline output points to the test set before transforming the data. This set\n", - " # will be helpful to generate a quality baseline for the model performance.\n", " ProcessingOutput(\n", - " output_name=\"baseline\",\n", - " source=\"/opt/ml/processing/baseline\",\n", - " destination=f\"{S3_LOCATION}/preprocessing/baseline\",\n", + " output_name=\"train-baseline\",\n", + " source=\"/opt/ml/processing/train-baseline\",\n", + " destination=f\"{S3_LOCATION}/preprocessing/train-baseline\",\n", + " ),\n", + " ProcessingOutput(\n", + " output_name=\"test-baseline\",\n", + " source=\"/opt/ml/processing/test-baseline\",\n", + " destination=f\"{S3_LOCATION}/preprocessing/test-baseline\",\n", " ),\n", " ],\n", " ),\n", @@ -1615,7 +1650,7 @@ }, { "cell_type": "code", - "execution_count": 273, + "execution_count": 301, "id": "e140642a", "metadata": { "tags": [] @@ -1625,16 +1660,16 @@ "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session1-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '6919902a-84dc-468f-b035-b7546e66d149',\n", + " 'ResponseMetadata': {'RequestId': '42674725-c31d-4b50-b8e3-79aa7f3bc160',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '6919902a-84dc-468f-b035-b7546e66d149',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '42674725-c31d-4b50-b8e3-79aa7f3bc160',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '85',\n", - " 'date': 'Mon, 23 Oct 2023 19:12:00 GMT'},\n", + " 'date': 'Tue, 24 Oct 2023 18:28:24 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 273, + "execution_count": 301, "metadata": {}, "output_type": "execute_result" } @@ -1683,7 +1718,7 @@ }, { "cell_type": "code", - "execution_count": 274, + "execution_count": 302, "id": "59d1e634", "metadata": {}, "outputs": [], @@ -1712,7 +1747,7 @@ "\n", "- Assignment 1.4 For this assignment, we want to run a distributed Processing Job across multiple instances to capitalize the `island` column of the dataset. Your dataset will consist of 10 different files stored in S3. Set up a Processing Job using two instances. When specifying the input to the Processing Job, you must set the `ProcessingInput.s3_data_distribution_type` attribute to `ShardedByS3Key`. By doing this, SageMaker will run a cluster with two instances simultaneously, each with access to half the files.\n", "\n", - "- Assignment 1.5 Pipeline steps can encounter exceptions. In some cases, retrying can resolve these issues. For this assignment, configure the Processing Step so it automatically retries the step a maximum of 5 times if it encounters an `InternalServerError`. Check the [Retry Policy for Pipeline Steps](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-retry-policy.html) documentation for more information.\n" + "- Assignment 1.5 You can use [Amazon SageMaker Data Wrangler](https://aws.amazon.com/sagemaker/data-wrangler/) to complete each step of the data preparation workflow (including data selection, cleansing, exploration, visualization, and processing at scale) from a single visual interface. For this assignment, load the Data Wrangler interface and use it to build the same transformations we implemented using the Scikit-Learn Pipeline. If you have questions, open the [Penguins Data Flow](penguins.flow) included in this repository.\n" ] }, { @@ -1739,7 +1774,7 @@ }, { "cell_type": "code", - "execution_count": 275, + "execution_count": 303, "id": "d92b121d-dcb9-43e8-9ee3-3ececb583e7e", "metadata": { "tags": [] @@ -1848,7 +1883,7 @@ }, { "cell_type": "code", - "execution_count": 276, + "execution_count": 304, "id": "14ea27ce-c453-4cb0-b309-dbecd732957e", "metadata": { "tags": [] @@ -1865,16 +1900,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "8/8 - 0s - loss: 0.8738 - accuracy: 0.7992 - val_loss: 0.8929 - val_accuracy: 0.7647 - 236ms/epoch - 29ms/step\n", - "2/2 [==============================] - 0s 2ms/step\n", - "Validation accuracy: 0.7647058823529411\n" + "8/8 - 0s - loss: 1.0366 - accuracy: 0.4226 - val_loss: 1.0048 - val_accuracy: 0.4314 - 234ms/epoch - 29ms/step\n", + "2/2 [==============================] - 0s 1ms/step\n", + "Validation accuracy: 0.43137254901960786\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp65a47aey/model/001/assets\n" + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpf0he4ftj/model/001/assets\n" ] }, { @@ -1951,7 +1986,7 @@ }, { "cell_type": "code", - "execution_count": 277, + "execution_count": 305, "id": "90fe82ae-6a2c-4461-bc83-bb52d8871e3b", "metadata": { "tags": [] @@ -2007,7 +2042,7 @@ }, { "cell_type": "code", - "execution_count": 278, + "execution_count": 306, "id": "99e4850c-83d6-4f4e-a813-d5a3f4bb7486", "metadata": { "tags": [] @@ -2072,7 +2107,7 @@ }, { "cell_type": "code", - "execution_count": 279, + "execution_count": 307, "id": "f367d0e3", "metadata": {}, "outputs": [], @@ -2103,7 +2138,7 @@ }, { "cell_type": "code", - "execution_count": 280, + "execution_count": 308, "id": "c8c82750", "metadata": {}, "outputs": [], @@ -2134,7 +2169,7 @@ }, { "cell_type": "code", - "execution_count": 281, + "execution_count": 309, "id": "038ff2e5-ed28-445b-bc03-4e996ec2286f", "metadata": { "tags": [] @@ -2177,7 +2212,7 @@ }, { "cell_type": "code", - "execution_count": 282, + "execution_count": 310, "id": "9799ab39-fcae-41f4-a68b-85ab71b3ba9a", "metadata": { "tags": [] @@ -2194,6 +2229,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Using provided s3_resource\n", "Using provided s3_resource\n" ] }, @@ -2204,27 +2240,20 @@ "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" ] }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using provided s3_resource\n" - ] - }, { "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session2-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '5f3aa5bd-4c3c-490f-b216-41e952d718b8',\n", + " 'ResponseMetadata': {'RequestId': 'e06d0053-459e-4151-b9d6-79253eb517c9',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '5f3aa5bd-4c3c-490f-b216-41e952d718b8',\n", + " 'HTTPHeaders': {'x-amzn-requestid': 'e06d0053-459e-4151-b9d6-79253eb517c9',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '85',\n", - " 'date': 'Mon, 23 Oct 2023 19:12:03 GMT'},\n", + " 'date': 'Tue, 24 Oct 2023 18:28:26 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 282, + "execution_count": 310, "metadata": {}, "output_type": "execute_result" } @@ -2269,7 +2298,7 @@ }, { "cell_type": "code", - "execution_count": 283, + "execution_count": 311, "id": "274a9b1e", "metadata": {}, "outputs": [], @@ -2327,7 +2356,7 @@ }, { "cell_type": "code", - "execution_count": 284, + "execution_count": 312, "id": "3ee3ab26-afa5-4ceb-9f7a-005d5fdea646", "metadata": { "tags": [] @@ -2413,7 +2442,7 @@ }, { "cell_type": "code", - "execution_count": 285, + "execution_count": 313, "id": "9a2540d8-278a-4953-bc54-0469d154427d", "metadata": { "tags": [] @@ -2430,16 +2459,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "8/8 - 0s - loss: 1.0332 - accuracy: 0.5272 - val_loss: 1.0559 - val_accuracy: 0.5294 - 232ms/epoch - 29ms/step\n", + "8/8 - 0s - loss: 1.0416 - accuracy: 0.3640 - val_loss: 1.0164 - val_accuracy: 0.3333 - 234ms/epoch - 29ms/step\n", "2/2 [==============================] - 0s 2ms/step\n", - "Validation accuracy: 0.5294117647058824\n" + "Validation accuracy: 0.3333333333333333\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpwl94wrqn/model/001/assets\n", + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpgy49w09u/model/001/assets\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n" ] @@ -2449,7 +2478,7 @@ "output_type": "stream", "text": [ "2/2 [==============================] - 0s 2ms/step\n", - "Test accuracy: 0.5686274509803921\n", + "Test accuracy: 0.49019607843137253\n", "\u001b[32m.\u001b[0m" ] }, @@ -2464,16 +2493,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "8/8 - 0s - loss: 1.1213 - accuracy: 0.2887 - val_loss: 1.1126 - val_accuracy: 0.3137 - 234ms/epoch - 29ms/step\n", + "8/8 - 0s - loss: 1.0747 - accuracy: 0.1841 - val_loss: 1.0577 - val_accuracy: 0.2549 - 227ms/epoch - 28ms/step\n", "2/2 [==============================] - 0s 2ms/step\n", - "Validation accuracy: 0.3137254901960784\n" + "Validation accuracy: 0.2549019607843137\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp44v9gj0_/model/001/assets\n", + "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpk_wecsvh/model/001/assets\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RestoredOptimizer` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RestoredOptimizer`.\n", "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.SGD` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.SGD`.\n" ] @@ -2482,10 +2511,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "2/2 [==============================] - 0s 2ms/step\n", - "Test accuracy: 0.37254901960784315\n", + "2/2 [==============================] - 0s 1ms/step\n", + "Test accuracy: 0.1568627450980392\n", "\u001b[32m.\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.34s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.35s\u001b[0m\u001b[0m\n" ] } ], @@ -2501,7 +2530,6 @@ "import pytest\n", "import tempfile\n", "import joblib\n", - "import json\n", "\n", "from preprocessor import preprocess\n", "from train import train\n", @@ -2569,7 +2597,7 @@ }, { "cell_type": "code", - "execution_count": 286, + "execution_count": 314, "id": "2fdff07f", "metadata": {}, "outputs": [ @@ -2609,7 +2637,7 @@ }, { "cell_type": "code", - "execution_count": 287, + "execution_count": 315, "id": "4f19e15b", "metadata": {}, "outputs": [], @@ -2630,7 +2658,7 @@ }, { "cell_type": "code", - "execution_count": 288, + "execution_count": 316, "id": "1f27b2ef", "metadata": {}, "outputs": [], @@ -2652,7 +2680,7 @@ }, { "cell_type": "code", - "execution_count": 289, + "execution_count": 317, "id": "48139a07-5c8e-4bc6-b666-bf9531f7f520", "metadata": { "tags": [] @@ -2719,7 +2747,7 @@ }, { "cell_type": "code", - "execution_count": 290, + "execution_count": 318, "id": "bb70f907", "metadata": {}, "outputs": [], @@ -2737,7 +2765,7 @@ }, { "cell_type": "code", - "execution_count": 291, + "execution_count": 319, "id": "4ca4cb61", "metadata": {}, "outputs": [], @@ -2763,7 +2791,7 @@ }, { "cell_type": "code", - "execution_count": 292, + "execution_count": 320, "id": "8c05a7e1", "metadata": {}, "outputs": [], @@ -2797,7 +2825,7 @@ }, { "cell_type": "code", - "execution_count": 293, + "execution_count": 321, "id": "c9773a4a", "metadata": { "tags": [] @@ -2858,7 +2886,7 @@ }, { "cell_type": "code", - "execution_count": 294, + "execution_count": 322, "id": "745486b5", "metadata": {}, "outputs": [], @@ -2878,7 +2906,7 @@ }, { "cell_type": "code", - "execution_count": 295, + "execution_count": 323, "id": "c4431bbf", "metadata": {}, "outputs": [], @@ -2907,7 +2935,7 @@ }, { "cell_type": "code", - "execution_count": 296, + "execution_count": 324, "id": "bebeecab", "metadata": {}, "outputs": [], @@ -2935,7 +2963,7 @@ }, { "cell_type": "code", - "execution_count": 297, + "execution_count": 325, "id": "36e2a2b1-6711-4266-95d8-d2aebd52e199", "metadata": { "tags": [] @@ -2964,7 +2992,7 @@ }, { "cell_type": "code", - "execution_count": 298, + "execution_count": 326, "id": "f70bcd33-b499-4e2b-953e-94d1ed96c10a", "metadata": { "tags": [] @@ -2991,13 +3019,7 @@ "text": [ "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session3-pipeline/code/09fea667a5ab7c37a068f22c00762d0b/sourcedir.tar.gz\n", "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session3-pipeline/code/2c207c809cb0e0e9a1d77e5247f961f9/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" ] }, @@ -3021,16 +3043,16 @@ "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session3-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '18bcbbca-b314-44d5-8bd4-70ac1f63c2cf',\n", + " 'ResponseMetadata': {'RequestId': '25850e57-1cdc-40f7-acf3-2b793dc77474',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '18bcbbca-b314-44d5-8bd4-70ac1f63c2cf',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '25850e57-1cdc-40f7-acf3-2b793dc77474',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '85',\n", - " 'date': 'Mon, 23 Oct 2023 19:12:06 GMT'},\n", + " 'date': 'Tue, 24 Oct 2023 18:28:31 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 298, + "execution_count": 326, "metadata": {}, "output_type": "execute_result" } @@ -3077,7 +3099,7 @@ }, { "cell_type": "code", - "execution_count": 299, + "execution_count": 327, "id": "f3b4126e", "metadata": {}, "outputs": [], @@ -3106,7 +3128,7 @@ "\n", "- Assignment 3.4 The current pipeline uses either a Training Step or a Tuning Step to build a model. Modify the pipeline to use both steps at the same time. The evaluation script should evaluate the model coming from the Training Step and the best model coming from the Tuning Step and output the accuracy and location in S3 of the best model. You should modify the code to register the model assets specified in the evaluation report.\n", "\n", - "- Assignment 3.5 Instead of running the entire pipeline from start to finish, sometimes you may only need to iterate over particular steps. SageMaker Pipelines supports [Selective Execution for Pipeline Steps](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-selective-ex.html). In this assignment you will use Selective Execution to only run one specific step of the pipeline. [Unlocking efficiency: Harnessing the power of Selective Execution in Amazon SageMaker Pipelines](https://aws.amazon.com/blogs/machine-learning/unlocking-efficiency-harnessing-the-power-of-selective-execution-in-amazon-sagemaker-pipelines/) is a great article that explains this feature.\n" + "- Assignment 3.5 Pipeline steps can encounter exceptions. In some cases, retrying can resolve these issues. For this assignment, configure the Processing Step so it automatically retries the step a maximum of 5 times if it encounters an `InternalServerError`. Check the [Retry Policy for Pipeline Steps](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-retry-policy.html) documentation for more information." ] }, { @@ -3118,44 +3140,41 @@ "source": [ "## Session 4 - Deploying Models and Serving Predictions\n", "\n", - "In this session we'll explore how to deploy a model to a SageMaker Endpoint and how to use a SageMaker Inference Pipeline to control the data that goes in and comes out of the endpoint.\n" - ] - }, - { - "cell_type": "markdown", - "id": "93727425-fac6-44ec-91ed-130a50fdd18a", - "metadata": {}, - "source": [ - "### Step 1 - Deploying Model From Registry\n", + "In this session we'll explore how to deploy a model to a SageMaker Endpoint and how to use a SageMaker Inference Pipeline to control the data that goes in and comes out of the endpoint.\n", "\n", - "Let's manually deploy the latest model from the Model Registry to an endpoint.\n", + "Deployment Pipeline\n", "\n", - "Let's start by defining the name of the endpoint where we'll deploy the model:\n" + "Let's start by defining the name of the endpoint where we'll deploy the model and creating a constant pointing to the location where we'll store the data that the endpoint will capture:" ] }, { "cell_type": "code", - "execution_count": 300, - "id": "2a116f93", + "execution_count": 328, + "id": "befd5ad3", "metadata": {}, "outputs": [], "source": [ "from sagemaker.predictor import Predictor\n", "\n", - "ENDPOINT = \"penguins-endpoint\"" + "ENDPOINT = \"penguins-endpoint\"\n", + "DATA_CAPTURE_DESTINATION = f\"{S3_LOCATION}/monitoring/data-capture\"" ] }, { "cell_type": "markdown", - "id": "ae95f1d6", + "id": "93727425-fac6-44ec-91ed-130a50fdd18a", "metadata": {}, "source": [ - "We want to query the list of approved models from the Model Registry and get the last one:\n" + "### Step 1 - Deploying Model From Registry\n", + "\n", + "Let's manually deploy the latest model from the Model Registry to an endpoint.\n", + "\n", + "We want to query the list of approved models from the Model Registry and get the last one:" ] }, { "cell_type": "code", - "execution_count": 301, + "execution_count": 329, "id": "87437a26-e9ea-4866-9dc3-630444c0fb46", "metadata": { "tags": [] @@ -3172,7 +3191,7 @@ " 'ModelApprovalStatus': 'Approved'}" ] }, - "execution_count": 301, + "execution_count": 329, "metadata": {}, "output_type": "execute_result" } @@ -3203,7 +3222,7 @@ }, { "cell_type": "code", - "execution_count": 302, + "execution_count": 330, "id": "dee516e9", "metadata": {}, "outputs": [], @@ -3239,29 +3258,12 @@ }, { "cell_type": "code", - "execution_count": 303, + "execution_count": 331, "id": "7c8852d5-818a-406c-944d-30bf6de90288", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker:Creating model with name: penguins-2023-10-23-19-14-04-732\n", - "INFO:sagemaker:Creating endpoint-config with name penguins-endpoint\n", - "INFO:sagemaker:Creating endpoint with name penguins-endpoint\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---!" - ] - } - ], + "outputs": [], "source": [ "%%script false --no-raise-error\n", "#| eval: false\n", @@ -3287,15 +3289,15 @@ }, { "cell_type": "code", - "execution_count": 304, + "execution_count": 332, "id": "ba7da291", "metadata": {}, "outputs": [], "source": [ "payload = \"\"\"\n", - "0.6569590202313976, -1.0813829646495108, 1.2097102831892812, 0.9226343641317372, 1.0, 0.0, 0.0\n", - "-0.7751048801481084, 0.8822689351285553, -1.2168066120762704, 0.9226343641317372, 0.0, 1.0, 0.0\n", - "-0.837387834894918, 0.3386660813829646, -0.26237731892812, -1.92351941317372, 0.0, 0.0, 1.0\n", + "0.6569590202313976,-1.0813829646495108,1.2097102831892812,0.9226343641317372,1.0,0.0,0.0\n", + "-0.7751048801481084,0.8822689351285553,-1.2168066120762704,0.9226343641317372,0.0,1.0,0.0\n", + "-0.837387834894918,0.3386660813829646,-0.26237731892812,-1.92351941317372,0.0,0.0,1.0\n", "\"\"\"" ] }, @@ -3307,21 +3309,9 @@ "Let's send the payload to the endpoint and print its response:\n" ] }, - { - "cell_type": "markdown", - "id": "65389b75", - "metadata": {}, - "source": [ - "#| hide\n", - "\n", - "
Note: \n", - " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", - "
\n" - ] - }, { "cell_type": "code", - "execution_count": 305, + "execution_count": 333, "id": "0817a25e-8224-4911-830b-d659e7458b4a", "metadata": { "tags": [] @@ -3331,40 +3321,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"predictions\": [\n", - " [\n", - " 0.00874690246,\n", - " 0.0456074178,\n", - " 0.94564569\n", - " ],\n", - " [\n", - " 0.77917105,\n", - " 0.0874738544,\n", - " 0.133355066\n", - " ],\n", - " [\n", - " 0.933188081,\n", - " 0.0207268894,\n", - " 0.0460849963\n", - " ]\n", - " ]\n", - "}\n", - "\n", - "Species: [2 0 0]\n" + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" ] } ], "source": [ - "%%script false --no-raise-error\n", - "#| eval: false\n", - "\n", "predictor = Predictor(endpoint_name=ENDPOINT)\n", - "response = predictor.predict(payload, initial_args={\"ContentType\": \"text/csv\"})\n", - "response = json.loads(response.decode(\"utf-8\"))\n", "\n", - "print(json.dumps(response, indent=2))\n", - "print(f\"\\nSpecies: {np.argmax(response['predictions'], axis=1)}\")" + "try:\n", + " response = predictor.predict(payload, initial_args={\"ContentType\": \"text/csv\"})\n", + " response = json.loads(response.decode(\"utf-8\"))\n", + "\n", + " print(json.dumps(response, indent=2))\n", + " print(f\"\\nSpecies: {np.argmax(response['predictions'], axis=1)}\")\n", + "except Exception as e:\n", + " print(e)" ] }, { @@ -3377,36 +3348,29 @@ }, { "cell_type": "markdown", - "id": "dc56ac34", + "id": "d9ec7eeb", "metadata": {}, "source": [ "#| hide\n", "\n", "
Note: \n", " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", - "
\n" + "" ] }, { "cell_type": "code", - "execution_count": 306, + "execution_count": 334, "id": "6b32c3a4-312e-473c-a217-33606f77d1e9", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker:Deleting endpoint configuration with name: penguins-endpoint\n", - "INFO:sagemaker:Deleting endpoint with name: penguins-endpoint\n" - ] - } - ], + "outputs": [], "source": [ "%%script false --no-raise-error\n", "#| eval: false\n", + "#| code: true\n", + "#| output: false\n", "\n", "predictor.delete_endpoint()" ] @@ -3458,7 +3422,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 335, "id": "e2d61d5c", "metadata": { "tags": [] @@ -3591,7 +3555,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 336, "id": "33893ef2", "metadata": { "tags": [] @@ -3602,7 +3566,7 @@ "output_type": "stream", "text": [ "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m10 passed\u001b[0m\u001b[32m in 0.06s\u001b[0m\u001b[0m\n" + "\u001b[32m\u001b[32m\u001b[1m10 passed\u001b[0m\u001b[32m in 0.07s\u001b[0m\u001b[0m\n" ] } ], @@ -3611,8 +3575,6 @@ "#| code-fold: true\n", "#| output: false\n", "\n", - "import json\n", - "\n", "from preprocessing_component import input_fn, predict_fn, output_fn, model_fn\n", "\n", "\n", @@ -3755,7 +3717,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 337, "id": "48c69002", "metadata": { "tags": [] @@ -3824,6 +3786,11 @@ " \"prediction\": p,\n", " \"confidence\": c\n", " })\n", + "\n", + " # If there's only one prediction, we'll return it\n", + " # as a single object.\n", + " if len(response) == 1:\n", + " response = response[0]\n", " \n", " return worker.Response(json.dumps(response), mimetype=accept) if worker else (response, accept)\n", " \n", @@ -3834,7 +3801,7 @@ " \"\"\"\n", " Transforms the prediction into its corresponding category.\n", " \"\"\"\n", - " \n", + "\n", " predictions = np.argmax(input_data, axis=-1)\n", " confidence = np.max(input_data, axis=-1)\n", " return [(model[prediction], confidence) for confidence, prediction in zip(confidence, predictions)]\n", @@ -3859,7 +3826,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 338, "id": "741b8402", "metadata": { "tags": [] @@ -3869,8 +3836,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", - "\u001b[32m\u001b[32m\u001b[1m1 passed\u001b[0m\u001b[32m in 0.01s\u001b[0m\u001b[0m\n" + "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", + "\u001b[32m\u001b[32m\u001b[1m3 passed\u001b[0m\u001b[32m in 0.01s\u001b[0m\u001b[0m\n" ] } ], @@ -3879,13 +3846,12 @@ "#| code-fold: true\n", "#| output: false\n", "\n", - "import json\n", "import numpy as np\n", "\n", - "from postprocessing_component import predict_fn\n", + "from postprocessing_component import predict_fn, output_fn\n", "\n", "\n", - "def test_predict_returns_prediction_as_last_column():\n", + "def test_predict_returns_prediction_as_first_column():\n", " input_data = [\n", " [0.6, 0.2, 0.2], \n", " [0.1, 0.8, 0.1],\n", @@ -3897,10 +3863,26 @@ " response = predict_fn(input_data, categories)\n", " \n", " assert response == [\n", - " (0.6, \"Adelie\"),\n", - " (0.8, \"Gentoo\"),\n", - " (0.7, \"Chinstrap\")\n", - " ]" + " (\"Adelie\", 0.6),\n", + " (\"Gentoo\", 0.8),\n", + " (\"Chinstrap\", 0.7)\n", + " ]\n", + "\n", + "\n", + "def test_output_does_not_return_array_if_single_prediction():\n", + " prediction = [(\"Adelie\", 0.6)]\n", + " response, _ = output_fn(prediction, \"application/json\")\n", + "\n", + " assert response[\"prediction\"] == \"Adelie\"\n", + "\n", + "\n", + "def test_output_returns_array_if_multiple_predictions():\n", + " prediction = [(\"Adelie\", 0.6), (\"Gentoo\", 0.8)]\n", + " response, _ = output_fn(prediction, \"application/json\")\n", + "\n", + " assert len(response) == 2\n", + " assert response[0][\"prediction\"] == \"Adelie\"\n", + " assert response[1][\"prediction\"] == \"Gentoo\"\n" ] }, { @@ -3923,7 +3905,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 339, "id": "53ea0ccf", "metadata": {}, "outputs": [], @@ -3949,7 +3931,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 340, "id": "11a0effd", "metadata": {}, "outputs": [], @@ -3976,7 +3958,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 341, "id": "5d7a5926", "metadata": {}, "outputs": [], @@ -4001,7 +3983,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 342, "id": "157b8858", "metadata": { "tags": [] @@ -4030,7 +4012,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 343, "id": "aefe580a", "metadata": {}, "outputs": [], @@ -4048,7 +4030,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 344, "id": "f84d2cd5", "metadata": { "tags": [] @@ -4103,7 +4085,7 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 345, "id": "b9712905-9fe3-4148-ae6d-05b0a48e742e", "metadata": { "tags": [] @@ -4130,7 +4112,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 346, "id": "bad9f51d", "metadata": { "tags": [] @@ -4147,6 +4129,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Using provided s3_resource\n", "Using provided s3_resource\n" ] }, @@ -4154,22 +4137,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session4-pipeline/code/09fea667a5ab7c37a068f22c00762d0b/sourcedir.tar.gz\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using provided s3_resource\n" + "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session4-pipeline/code/09fea667a5ab7c37a068f22c00762d0b/sourcedir.tar.gz\n", + "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session4-pipeline/code/2c207c809cb0e0e9a1d77e5247f961f9/runproc.sh\n", + "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session4-pipeline/code/2c207c809cb0e0e9a1d77e5247f961f9/runproc.sh\n", - "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" ] }, @@ -4193,16 +4169,16 @@ "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session4-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '14c33b8d-0697-4b6c-a839-aa3c73acb35f',\n", + " 'ResponseMetadata': {'RequestId': '6b81bad9-1f68-4b36-bf30-b194f5097a55',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '14c33b8d-0697-4b6c-a839-aa3c73acb35f',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '6b81bad9-1f68-4b36-bf30-b194f5097a55',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '85',\n", - " 'date': 'Mon, 23 Oct 2023 15:47:23 GMT'},\n", + " 'date': 'Tue, 24 Oct 2023 18:28:34 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 149, + "execution_count": 346, "metadata": {}, "output_type": "execute_result" } @@ -4249,7 +4225,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 347, "id": "20dfbd97", "metadata": {}, "outputs": [], @@ -4280,7 +4256,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 348, "id": "998314a3", "metadata": {}, "outputs": [ @@ -4397,24 +4373,6 @@ " }" ] }, - { - "cell_type": "markdown", - "id": "b3374868", - "metadata": {}, - "source": [ - "Let's create a constant pointing to the location where we'll store the data that the endpoint will capture:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 152, - "id": "c51f421f", - "metadata": {}, - "outputs": [], - "source": [ - "DATA_CAPTURE_DESTINATION = f\"{S3_LOCATION}/monitoring/data-capture\"" - ] - }, { "cell_type": "markdown", "id": "5b582ace", @@ -4425,7 +4383,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 349, "id": "4ad4f1f2", "metadata": { "tags": [] @@ -4492,7 +4450,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 350, "id": "ad8c8019", "metadata": { "tags": [] @@ -4501,13 +4459,13 @@ { "data": { "text/plain": [ - "{'ResponseMetadata': {'RequestId': '21713bbd-59e2-478d-995f-129a5503b310',\n", + "{'ResponseMetadata': {'RequestId': 'a6e915cb-e440-4ecd-94bb-458139388602',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'date': 'Mon, 23 Oct 2023 15:47:24 GMT',\n", + " 'HTTPHeaders': {'date': 'Tue, 24 Oct 2023 18:28:36 GMT',\n", " 'content-type': 'application/json',\n", " 'content-length': '1428',\n", " 'connection': 'keep-alive',\n", - " 'x-amzn-requestid': '21713bbd-59e2-478d-995f-129a5503b310'},\n", + " 'x-amzn-requestid': 'a6e915cb-e440-4ecd-94bb-458139388602'},\n", " 'RetryAttempts': 0},\n", " 'FunctionName': 'deploy_fn',\n", " 'FunctionArn': 'arn:aws:lambda:us-east-1:325223348818:function:deploy_fn',\n", @@ -4518,14 +4476,14 @@ " 'Description': '',\n", " 'Timeout': 600,\n", " 'MemorySize': 128,\n", - " 'LastModified': '2023-10-23T15:47:24.000+0000',\n", - " 'CodeSha256': 'ueBRU1NVemrHDSO1q2iAskrlzV83Ha77uojYSsRFZVQ=',\n", + " 'LastModified': '2023-10-24T18:28:36.000+0000',\n", + " 'CodeSha256': 'gTB7D5GxQS4xUk99eaZAfIFv2GPHZ6s2D+aNyzOy19Q=',\n", " 'Version': '$LATEST',\n", " 'Environment': {'Variables': {'ROLE': 'arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501',\n", " 'DATA_CAPTURE_DESTINATION': 's3://mlschool/penguins/monitoring/data-capture',\n", " 'ENDPOINT': 'penguins-endpoint'}},\n", " 'TracingConfig': {'Mode': 'PassThrough'},\n", - " 'RevisionId': '46956778-abb2-4e7f-b332-84cdd1bb9772',\n", + " 'RevisionId': '175878c6-9ff3-47b1-b1a3-b5df361b9fc9',\n", " 'Layers': [],\n", " 'State': 'Active',\n", " 'LastUpdateStatus': 'InProgress',\n", @@ -4538,7 +4496,7 @@ " 'RuntimeVersionConfig': {'RuntimeVersionArn': 'arn:aws:lambda:us-east-1::runtime:6cf63f1a78b5c5e19617d6b4b111370fdbda415ea91bdfdc5aacef9fee76b64a'}}" ] }, - "execution_count": 154, + "execution_count": 350, "metadata": {}, "output_type": "execute_result" } @@ -4586,7 +4544,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 351, "id": "27ce7cc5", "metadata": {}, "outputs": [], @@ -4613,7 +4571,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 352, "id": "2a878179", "metadata": {}, "outputs": [], @@ -4637,7 +4595,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 353, "id": "dc714a97", "metadata": { "tags": [] @@ -4665,7 +4623,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 354, "id": "d74be86b", "metadata": {}, "outputs": [ @@ -4701,21 +4659,9 @@ "Let's now test the endpoint we deployed automatically with the pipeline. We will use the function to create a predictor with a JSON encoder and decoder.\n" ] }, - { - "cell_type": "markdown", - "id": "384de56d", - "metadata": {}, - "source": [ - "#| hide\n", - "\n", - "
Note: \n", - " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", - "
\n" - ] - }, { "cell_type": "code", - "execution_count": 308, + "execution_count": 355, "id": "3cc966fb-b611-417f-a8b8-0c5d2f95252c", "metadata": { "tags": [] @@ -4725,26 +4671,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "[{\"prediction\": 0.971010566, \"confidence\": \"Adelie\"}, {\"prediction\": 0.931603611, \"confidence\": \"Adelie\"}, {\"prediction\": 0.933561683, \"confidence\": \"Adelie\"}]\n" + "Payload:\n", + "Torgersen,39.1,18.7,181.0,3750.0,MALE\n", + "Torgersen,39.5,17.4,186.0,3800.0,FEMALE\n", + "Torgersen,40.3,18.0,195.0,3250.0,FEMALE\n", + "\n", + "An error occurred (ValidationError) when calling the InvokeEndpoint operation: Endpoint penguins-endpoint of account 325223348818 not found.\n" ] } ], "source": [ - "%%script false --no-raise-error\n", - "#| eval: false\n", - "\n", "from sagemaker.serializers import CSVSerializer\n", "\n", - "\n", - "waiter = sagemaker_client.get_waiter(\"endpoint_in_service\")\n", - "waiter.wait(\n", - " EndpointName=ENDPOINT,\n", - " WaiterConfig={\n", - " \"Delay\": 10,\n", - " \"MaxAttempts\": 30\n", - " }\n", - ")\n", - "\n", "predictor = Predictor(\n", " endpoint_name=ENDPOINT, \n", " serializer=CSVSerializer(),\n", @@ -4755,42 +4693,48 @@ "data = data.drop(\"species\", axis=1)\n", "\n", "payload = data.iloc[:3].to_csv(header=False, index=False)\n", - "response = predictor.predict(payload, initial_args={\"ContentType\": \"text/csv\"})\n", - "print(response.decode(\"utf-8\"))" + "print(f\"Payload:\\n{payload}\")\n", + "\n", + "try:\n", + " response = predictor.predict(payload, initial_args={\"ContentType\": \"text/csv\"})\n", + " print(response.decode(\"utf-8\"))\n", + "except Exception as e:\n", + " print(e)" ] }, { "cell_type": "markdown", "id": "67e883b0", "metadata": {}, + "source": [ + "Let's delete the endpoint:" + ] + }, + { + "cell_type": "markdown", + "id": "6cffc2b5", + "metadata": {}, "source": [ "#| hide\n", "\n", "
Note: \n", " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", - "
\n" + "" ] }, { "cell_type": "code", - "execution_count": 309, + "execution_count": 356, "id": "8c3e851a-2416-4a0b-b8a1-c483cde3d776", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker:Deleting endpoint configuration with name: penguins-endpoint-config-1023194943\n", - "INFO:sagemaker:Deleting endpoint with name: penguins-endpoint\n" - ] - } - ], + "outputs": [], "source": [ "%%script false --no-raise-error\n", "#| eval: false\n", + "#| code: true\n", + "#| output: false\n", "\n", "predictor.delete_endpoint()" ] @@ -4802,15 +4746,15 @@ "source": [ "### Assignments\n", "\n", - "- Assignment 4.1 Every Endpoint has an invocation URL you can use to generate predictions with the model from outside AWS. As part of this assignment, write a simple Python script that will run on your local computer and run a few samples through the Endpoint. You will need your AWS access key and secret to connect to the Endpoint.\n", + "* Assignment 4.1 Every Endpoint has an invocation URL you can use to generate predictions with the model from outside AWS. As part of this assignment, write a simple Python script that will run on your local computer and run a few samples through the Endpoint. You will need your AWS access key and secret to connect to the Endpoint.\n", "\n", - "- Assignment 4.2 We can use model variants to perform A/B testing between a new model and an old model. Create a function that given the ARN of two models in the Model Registry deploys them to an Endpoint as separate variants. Each variant should receive 50% of the traffic. Write another function that invokes the endpoint by default, but allows the caller to invoke a specific variant if they want to.\n", + "* Assignment 4.2 We can use model variants to perform A/B testing between a new model and an old model. Create a function that given the ARN of two models in the Model Registry deploys them to an Endpoint as separate variants. Each variant should receive 50% of the traffic. Write another function that invokes the endpoint by default, but allows the caller to invoke a specific variant if they want to.\n", "\n", - "- Assignment 4.3 We can use SageMaker Model Shadow Deployments to create shadow variants to validate a new model version before promoting it to production. Write a function that given the ARN of a model in the Model Registry, updates an Endpoint and deploys the model as a shadow variant. Check [Shadow variants](https://docs.aws.amazon.com/sagemaker/latest/dg/model-shadow-deployment.html) for more information about this topic. Send some traffic to the Endpoint and compare the results from the main model with its shadow variant.\n", + "* Assignment 4.3 We can use SageMaker Model Shadow Deployments to create shadow variants to validate a new model version before promoting it to production. Write a function that given the ARN of a model in the Model Registry, updates an Endpoint and deploys the model as a shadow variant. Check [Shadow variants](https://docs.aws.amazon.com/sagemaker/latest/dg/model-shadow-deployment.html) for more information about this topic. Send some traffic to the Endpoint and compare the results from the main model with its shadow variant.\n", "\n", - "- Assignment 4.4 SageMaker supports auto scaling your models. Auto scaling dynamically adjusts the number of instances provisioned for a model in response to changes in the workload. For this assignment, define a target-tracking scaling policy for a variant of your Endpoint and use the `SageMakerVariantInvocationsPerInstance` metric. `SageMakerVariantInvocationsPerInstance` is the average number of times per minute that the variant is invoked. Check [Automatically Scale Amazon SageMaker Models](https://docs.aws.amazon.com/sagemaker/latest/dg/endpoint-auto-scaling.html) for more information about auto scaling models.\n", + "* Assignment 4.4 SageMaker supports auto scaling your models. Auto scaling dynamically adjusts the number of instances provisioned for a model in response to changes in the workload. For this assignment, define a target-tracking scaling policy for a variant of your Endpoint and use the `SageMakerVariantInvocationsPerInstance` metric. `SageMakerVariantInvocationsPerInstance` is the average number of times per minute that the variant is invoked. Check [Automatically Scale Amazon SageMaker Models](https://docs.aws.amazon.com/sagemaker/latest/dg/endpoint-auto-scaling.html) for more information about auto scaling models.\n", "\n", - "- Assignment 4.5 TBD\n" + "* Assignment 4.5 Modify the SageMaker Pipeline by adding a Lambda Step that will deploy the model directly as part of the pipeline. You won't need to set up Event Bridge anymore because your pipeline will automatically deploy the model.\n" ] }, { @@ -4822,6 +4766,8 @@ "\n", "In this session we'll set up a monitoring process to analyze the quality of the data our endpoint receives and the endpoint predictions. For this, we need to check the data received by the endpoint, generate ground truth labels, and compare them with a baseline performance.\n", "\n", + "\"Data\n", + "\n", "To enable this functionality, we need a couple of steps:\n", "\n", "1. Create baselines we can use to compare against real-time traffic.\n", @@ -4840,7 +4786,7 @@ }, { "cell_type": "code", - "execution_count": 174, + "execution_count": 357, "id": "2bb846d0", "metadata": {}, "outputs": [], @@ -4855,7 +4801,7 @@ "id": "24c26ac4-5d30-41e9-8952-e4deb39de819", "metadata": {}, "source": [ - "### Step 1 - Generating the Data Baseline\n", + "### Step 1 - Generating Data Quality Baseline\n", "\n", "Let's start by configuring a [Quality Check Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-quality-check) to compute the general statistics of the data we used to build our model.\n", "\n", @@ -4864,7 +4810,7 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 358, "id": "0b80bcab-d2c5-437c-a1c8-8eea208c0e29", "metadata": { "tags": [] @@ -4900,8 +4846,10 @@ " role=role,\n", " ),\n", " quality_check_config=DataQualityCheckConfig(\n", - " baseline_dataset=f\"{S3_LOCATION}/data\",\n", - " dataset_format=DatasetFormat.csv(header=True, output_columns_position=\"END\"),\n", + " baseline_dataset=split_and_transform_data_step.properties.ProcessingOutputConfig.Outputs[\n", + " \"train-baseline\"\n", + " ].S3Output.S3Uri,\n", + " dataset_format=DatasetFormat.csv(header=True, output_columns_position=\"START\"),\n", " output_s3_uri=DATA_QUALITY_LOCATION,\n", " ),\n", " skip_check=True,\n", @@ -4917,7 +4865,7 @@ "tags": [] }, "source": [ - "### Step 2 - Creating Test Predictions\n", + "### Step 2 - Generating Test Predictions\n", "\n", "To create a baseline to compare the model performance, we must create predictions for the test set and compare the model's metrics with the model performance on production data. We can do this by running a [Batch Transform Job](https://docs.aws.amazon.com/sagemaker/latest/dg/batch-transform.html) to predict every sample from the test set. We can use a [Transform Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-transform) as part of the pipeline to run this job. This Batch Transform Job will run every sample from the training dataset through the model so we can compute the baseline metrics.\n", "\n", @@ -4926,20 +4874,29 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 359, "id": "8194b462", "metadata": {}, - "outputs": [], - "source": [ - "# | code: true\n", - "# | output: false\n", - "\n", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/svpino/dev/ml.school/.venv/lib/python3.9/site-packages/sagemaker/workflow/pipeline_context.py:297: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "# | code: true\n", + "# | output: false\n", + "\n", "from sagemaker.workflow.model_step import ModelStep\n", "\n", "create_model_step = ModelStep(\n", " name=\"create\",\n", " display_name=\"create-model\",\n", - " step_args=pipeline_model.create(instance_type=\"ml.m5.xlarge\"),\n", + " step_args=pipeline_model.create(instance_type=config[\"instance_type\"]),\n", ")" ] }, @@ -4953,7 +4910,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 360, "id": "bf6aa4f0", "metadata": {}, "outputs": [], @@ -4965,10 +4922,10 @@ " instance_type=config[\"instance_type\"],\n", " instance_count=1,\n", " strategy=\"MultiRecord\",\n", - " accept=\"/csv\",\n", + " accept=\"text/csv\",\n", " assemble_with=\"Line\",\n", " output_path=f\"{S3_LOCATION}/transform\",\n", - " sagemaker_session=sagemaker_session,\n", + " sagemaker_session=pipeline_session,\n", ")" ] }, @@ -4987,7 +4944,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 361, "id": "1987a788-de7a-4f60-ac8d-819d9ffcdf8e", "metadata": { "tags": [] @@ -5005,12 +4962,13 @@ " # We will use the baseline set we generated when we split the data.\n", " # This set corresponds to the test split before the transformation step.\n", " data=split_and_transform_data_step.properties.ProcessingOutputConfig.Outputs[\n", - " \"baseline\"\n", + " \"test-baseline\"\n", " ].S3Output.S3Uri,\n", + "\n", " join_source=\"Input\",\n", " split_type=\"Line\",\n", " content_type=\"text/csv\",\n", - " input_filter=\"$\",\n", + " \n", " # We want to output the first and the last field from the joint set.\n", " # The first field corresponds to the groundtruth, and the last field\n", " # corresponds to the prediction.\n", @@ -5025,14 +4983,14 @@ "id": "2fafc7c4-6fef-4832-8b99-8c45d078fdd2", "metadata": {}, "source": [ - "### Step 3 - Generating a Model Drift Baseline\n", + "### Step 3 - Generating Model Quality Baseline\n", "\n", "Let's now configure the [Quality Check Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-quality-check) and feed it the data we generated in the Transform Step. This step will automatically compute the performance metrics of the model on the test set:\n" ] }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 362, "id": "9aa3a284-8763-4000-a263-70314b530652", "metadata": { "tags": [] @@ -5042,13 +5000,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n", "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n" ] } @@ -5073,9 +5025,11 @@ " # the model quality baseline.\n", " baseline_dataset=generate_test_predictions_step.properties.TransformOutput.S3OutputPath,\n", " dataset_format=DatasetFormat.csv(header=False),\n", + "\n", " # We need to specify the problem type and the fields where the prediction\n", " # and groundtruth are so the process knows how to interpret the results.\n", " problem_type=\"MulticlassClassification\",\n", + " \n", " # Since the data doesn't have headers, SageMaker will autocreate headers for it.\n", " # _c0 corresponds to the first column, and _c1 corresponds to the second column.\n", " ground_truth_attribute=\"_c0\",\n", @@ -5100,7 +5054,7 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 363, "id": "a773f134-ac2f-4dba-976e-9b7f0b384b6e", "metadata": { "tags": [] @@ -5160,7 +5114,7 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 364, "id": "7056a009-91c0-4955-90dd-b90ef8cab149", "metadata": { "tags": [] @@ -5176,6 +5130,7 @@ " step_args=pipeline_model.register(\n", " model_package_group_name=PIPELINE_MODEL_PACKAGE_GROUP,\n", " model_metrics=model_metrics,\n", + " drift_check_baselines=drift_check_baselines,\n", " approval_status=\"PendingManualApproval\",\n", " content_types=[\"text/csv\", \"application/json\"],\n", " response_types=[\"text/csv\", \"application/json\"],\n", @@ -5201,7 +5156,7 @@ }, { "cell_type": "code", - "execution_count": 203, + "execution_count": 365, "id": "bacaa9c6-22b0-48df-b138-95b6422fe834", "metadata": { "tags": [] @@ -5235,7 +5190,7 @@ }, { "cell_type": "code", - "execution_count": 204, + "execution_count": 366, "id": "4da5e453-acd8-47a0-a39f-264d05dd93d0", "metadata": { "tags": [] @@ -5276,7 +5231,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n" + "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", + "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session5-pipeline/code/09fea667a5ab7c37a068f22c00762d0b/sourcedir.tar.gz\n", + "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session5-pipeline/code/2c207c809cb0e0e9a1d77e5247f961f9/runproc.sh\n" ] }, { @@ -5286,28 +5243,20 @@ "Using provided s3_resource\n" ] }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session5-pipeline/code/09fea667a5ab7c37a068f22c00762d0b/sourcedir.tar.gz\n", - "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session5-pipeline/code/2c207c809cb0e0e9a1d77e5247f961f9/runproc.sh\n" - ] - }, { "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session5-pipeline',\n", - " 'ResponseMetadata': {'RequestId': '571c3344-c2f1-4c39-a6f7-7bb71a799eaa',\n", + " 'ResponseMetadata': {'RequestId': '450e9e6e-9ec9-40de-bda6-be8564981011',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '571c3344-c2f1-4c39-a6f7-7bb71a799eaa',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '450e9e6e-9ec9-40de-bda6-be8564981011',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '85',\n", - " 'date': 'Mon, 23 Oct 2023 17:19:13 GMT'},\n", + " 'date': 'Tue, 24 Oct 2023 18:28:40 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 204, + "execution_count": 366, "metadata": {}, "output_type": "execute_result" } @@ -5355,17 +5304,17 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 367, "id": "10ba9909", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session5-pipeline/execution/yh29jzycbsp2', sagemaker_session=)" + "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session5-pipeline/execution/a8jrffhsgbcm', sagemaker_session=)" ] }, - "execution_count": 190, + "execution_count": 367, "metadata": {}, "output_type": "execute_result" } @@ -5382,170 +5331,225 @@ }, { "cell_type": "markdown", - "id": "b948aa92-8064-4f03-af08-0f6a8fc329cf", + "id": "6fd182a9", "metadata": {}, "source": [ - "#| hide\n", + "### Step 8 - Checking Constraints and Statistics\n", "\n", - "### Step 8 - Generating Fake Traffic\n", + "Our pipeline generated data baseline statistics and constraints. We can take a look at what these values look like by downloading them from S3. You need to wait for the pipeline to finish running before these files are available.\n", "\n", - "To test the monitoring functionality, we need to generate traffic to the endpoint. To generate traffic, we will repeatedly send every sample from the dataset to the endpoint to simulate real prediction requests:\n" + "Here are the data quality statistics:" ] }, { "cell_type": "code", - "execution_count": 205, - "id": "87a3c4ce-aff7-4f48-9d1b-be98eb746e66", + "execution_count": 368, + "id": "42daa82b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"name\": \"species\",\n", + " \"inferred_type\": \"String\",\n", + " \"string_statistics\": {\n", + " \"common\": {\n", + " \"num_present\": 232,\n", + " \"num_missing\": 0\n", + " },\n", + " \"distinct_count\": 3.0,\n", + " \"distribution\": {\n", + " \"categorical\": {\n", + " \"buckets\": [\n", + " {\n", + " \"value\": \"Adelie\",\n", + " \"count\": 94\n", + " },\n", + " {\n", + " \"value\": \"Chinstrap\",\n", + " \"count\": 48\n", + " },\n", + " {\n", + " \"value\": \"Gentoo\",\n", + " \"count\": 90\n", + " }\n", + " ]\n", + " }\n", + " }\n", + " }\n", + "}\n" + ] + } + ], "source": [ - "from time import sleep\n", - "from threading import Thread, Event\n", - "\n", - "\n", - "def generate_traffic(predictor):\n", - " def _predict(data, predictor, stop_traffic_thread):\n", - " for index, row in data.iterrows():\n", - " data = row.tolist()\n", - " data = \",\".join(map(str, data))\n", - " predictor.predict(\n", - " data, inference_id=str(index), initial_args={\"ContentType\": \"text/csv\"}\n", - " )\n", - "\n", - " sleep(1)\n", - "\n", - " if stop_traffic_thread.is_set():\n", - " break\n", - "\n", - " def _generate_prediction_data(data, predictor, stop_traffic_thread):\n", - " while True:\n", - " print(f\"Generating {data.shape[0]} predictions...\")\n", - " _predict(data, predictor, stop_traffic_thread)\n", - "\n", - " if stop_traffic_thread.is_set():\n", - " break\n", - "\n", - " stop_traffic_thread = Event()\n", - "\n", - " data = pd.read_csv(DATA_FILEPATH, header=0).dropna()\n", - " data.drop([\"species\"], axis=1, inplace=True)\n", + "from sagemaker.s3 import S3Downloader\n", "\n", - " traffic_thread = Thread(\n", - " target=_generate_prediction_data,\n", - " args=(\n", - " data,\n", - " predictor,\n", - " stop_traffic_thread,\n", - " ),\n", + "try:\n", + " response = json.loads(\n", + " S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/statistics.json\")\n", " )\n", - "\n", - " traffic_thread.start()\n", - "\n", - " return stop_traffic_thread, traffic_thread" + " print(json.dumps(response[\"features\"][0], indent=2))\n", + "except Exception as e:\n", + " pass" ] }, { "cell_type": "markdown", - "id": "10652fb1", + "id": "8104ad3c", "metadata": {}, "source": [ - "Let's now start generating traffic to the endpoint:\n" + "Here are the data quality constraints:" ] }, { - "cell_type": "markdown", - "id": "f2fd7307", + "cell_type": "code", + "execution_count": 369, + "id": "898d9626", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"name\": \"species\",\n", + " \"inferred_type\": \"String\",\n", + " \"completeness\": 1.0,\n", + " \"string_constraints\": {\n", + " \"domains\": [\n", + " \"Adelie\",\n", + " \"Chinstrap\",\n", + " \"Gentoo\"\n", + " ]\n", + " }\n", + "}\n" + ] + } + ], "source": [ - "#| hide\n", - "\n", - "
Note: \n", - " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", - "
\n" + "try:\n", + " response = json.loads(S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/constraints.json\"))\n", + " print(json.dumps(response[\"features\"][0], indent=2))\n", + "except Exception as e:\n", + " pass" ] }, { - "cell_type": "code", - "execution_count": 207, - "id": "3d68cbd8", + "cell_type": "markdown", + "id": "35eaf9af", "metadata": {}, - "outputs": [], "source": [ - "%%script false --no-raise-error\n", - "#| eval: false\n", - "#| code: true\n", - "#| output: false\n", - "\n", - "predictor = Predictor(\n", - " endpoint_name=ENDPOINT, \n", - " serializer=CSVSerializer(),\n", - " sagemaker_session=sagemaker_session\n", - ")\n", - "\n", - "stop_traffic_thread, traffic_thread = generate_traffic(predictor)" + "And here are the model quality constraints:" ] }, { - "cell_type": "markdown", - "id": "5754a314-3bc0-4b41-8767-e9f06d96d250", + "cell_type": "code", + "execution_count": 370, + "id": "2df52332", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"version\": 0.0,\n", + " \"multiclass_classification_constraints\": {\n", + " \"accuracy\": {\n", + " \"threshold\": 0.9259259259259259,\n", + " \"comparison_operator\": \"LessThanThreshold\"\n", + " },\n", + " \"weighted_recall\": {\n", + " \"threshold\": 0.9259259259259259,\n", + " \"comparison_operator\": \"LessThanThreshold\"\n", + " },\n", + " \"weighted_precision\": {\n", + " \"threshold\": 0.933862433862434,\n", + " \"comparison_operator\": \"LessThanThreshold\"\n", + " },\n", + " \"weighted_f0_5\": {\n", + " \"threshold\": 0.928855833521148,\n", + " \"comparison_operator\": \"LessThanThreshold\"\n", + " },\n", + " \"weighted_f1\": {\n", + " \"threshold\": 0.9247293447293448,\n", + " \"comparison_operator\": \"LessThanThreshold\"\n", + " },\n", + " \"weighted_f2\": {\n", + " \"threshold\": 0.9242942991137502,\n", + " \"comparison_operator\": \"LessThanThreshold\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], "source": [ - "### Step 9 - Generating Fake Labels\n", - "\n", - "To test the performance of the model, we need to label the samples captured by the endpoint.\n" + "try:\n", + " response = json.loads(S3Downloader.read_file(f\"{MODEL_QUALITY_LOCATION}/constraints.json\"))\n", + " print(json.dumps(response, indent=2))\n", + "except Exception as e:\n", + " pass" ] }, { "cell_type": "markdown", - "id": "02a1e7af-933e-492d-948e-aa16cc67c3db", - "metadata": { - "tags": [] - }, + "id": "b948aa92-8064-4f03-af08-0f6a8fc329cf", + "metadata": {}, "source": [ - "Let's start by checking the location where the endpoint stores the captured data. It make take a few minutes for the first few files to show up in S3:\n" + "### Step 9 - Generating Fake Traffic\n", + "\n", + "To test the monitoring functionality, we need to generate traffic to the endpoint. To generate traffic, we will send every sample from the dataset to the endpoint to simulate real prediction requests:\n" ] }, { "cell_type": "code", - "execution_count": 210, - "id": "3f35e8db-24d7-4d4b-9264-78ee5070cf27", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "['s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2023/09/25/13/15-40-735-9cc3750d-ba42-472c-903d-969695d2096d.jsonl',\n", - " 's3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2023/09/27/15/45-31-289-001fc69f-c352-4da2-b57a-a3a69fe3fecf.jsonl',\n", - " 's3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2023/10/05/16/50-04-992-e16242d1-925c-4b07-9289-dffa0e026679.jsonl']" - ] - }, - "execution_count": 210, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 393, + "id": "c658bad0", + "metadata": {}, + "outputs": [], "source": [ - "from sagemaker.s3 import S3Downloader\n", + "# | code: true\n", + "# | output: false\n", "\n", - "files = S3Downloader.list(DATA_CAPTURE_DESTINATION)[:3]\n", - "files" + "from sagemaker.serializers import JSONSerializer\n", + "\n", + "data = penguins.drop([\"species\"], axis=1)\n", + "data = data.dropna()\n", + "\n", + "predictor = Predictor(\n", + " endpoint_name=ENDPOINT,\n", + " serializer=JSONSerializer(),\n", + " sagemaker_session=sagemaker_session,\n", + ")\n", + "\n", + "for index, row in data.iterrows():\n", + " try:\n", + " predictor.predict(row.to_dict(), inference_id=str(index))\n", + " except Exception as e:\n", + " print(e)\n", + " break" ] }, { "cell_type": "markdown", - "id": "74c28f66", + "id": "0d3f61b9", "metadata": {}, "source": [ - "These files contain the data captured by the endpoint in a SageMaker-specific JSON-line format. Each inference request is captured in a single line in the `jsonl` file. The line contains both the input and output merged together:\n" + "We can check the location where the endpoint stores the captured data, download a file, and display its content. It may take a few minutes for the first few files to show up in S3.\n", + "\n", + "These files contain the data captured by the endpoint in a SageMaker-specific JSON-line format. Each inference request is captured in a single line in the `jsonl` file. The line contains both the input and output merged together:" ] }, { "cell_type": "code", - "execution_count": 211, - "id": "6305949f", - "metadata": {}, + "execution_count": 394, + "id": "3f35e8db-24d7-4d4b-9264-78ee5070cf27", + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stdout", @@ -5554,21 +5558,22 @@ "{\n", " \"captureData\": {\n", " \"endpointInput\": {\n", - " \"observedContentType\": \"text/csv\",\n", + " \"observedContentType\": \"application/json\",\n", " \"mode\": \"INPUT\",\n", - " \"data\": \"Torgersen,39.1,18.7,181.0,3750.0,MALE\\nTorgersen,39.5,17.4,186.0,3800.0,FEMALE\\nTorgersen,40.3,18.0,195.0,3250.0,FEMALE\\n\",\n", - " \"encoding\": \"CSV\"\n", + " \"data\": \"{\\\"island\\\": \\\"Torgersen\\\", \\\"culmen_length_mm\\\": 39.1, \\\"culmen_depth_mm\\\": 18.7, \\\"flipper_length_mm\\\": 181.0, \\\"body_mass_g\\\": 3750.0, \\\"sex\\\": \\\"MALE\\\"}\",\n", + " \"encoding\": \"JSON\"\n", " },\n", " \"endpointOutput\": {\n", " \"observedContentType\": \"application/json\",\n", " \"mode\": \"OUTPUT\",\n", - " \"data\": \"[{\\\"prediction\\\": \\\"Adelie\\\", \\\"confidence\\\": 0.775418103}, {\\\"prediction\\\": \\\"Adelie\\\", \\\"confidence\\\": 0.775709867}, {\\\"prediction\\\": \\\"Adelie\\\", \\\"confidence\\\": 0.67967391}]\",\n", + " \"data\": \"{\\\"prediction\\\": \\\"Adelie\\\", \\\"confidence\\\": 0.953110516}\",\n", " \"encoding\": \"JSON\"\n", " }\n", " },\n", " \"eventMetadata\": {\n", - " \"eventId\": \"d33f9a23-5ae3-4403-9aa1-3759d7fa8015\",\n", - " \"inferenceTime\": \"2023-09-25T13:15:40Z\"\n", + " \"eventId\": \"ddf80c99-e582-4243-9309-4bc9085c01ec\",\n", + " \"inferenceId\": \"0\",\n", + " \"inferenceTime\": \"2023-10-24T19:10:30Z\"\n", " },\n", " \"eventVersion\": \"0\"\n", "}\n" @@ -5576,6 +5581,7 @@ } ], "source": [ + "files = S3Downloader.list(DATA_CAPTURE_DESTINATION)[:3]\n", "if len(files):\n", " lines = S3Downloader.read_file(files[0])\n", " print(json.dumps(json.loads(lines.split(\"\\n\")[0]), indent=2))" @@ -5583,150 +5589,66 @@ }, { "cell_type": "markdown", - "id": "c1d4904f", - "metadata": {}, - "source": [ - "Let's now define the function that will generate random labels. We can simulate the labeling process by generating a random label for every sample. Check [Ingest Ground Truth Labels and Merge Them With Predictions](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality-merge.html) for more information about this.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 212, - "id": "cb649a6e-fabe-4103-b1db-7c6a01fe959a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import random\n", - "from datetime import datetime\n", - "\n", - "\n", - "def generate_ground_truth_data(ground_truth_location):\n", - " def _generate_ground_truth_record(inference_id):\n", - " random.seed(inference_id)\n", - "\n", - " return {\n", - " \"groundTruthData\": {\n", - " \"data\": random.choice([\"Adelie\", \"Chinstrap\", \"Gentoo\"]),\n", - " \"encoding\": \"CSV\",\n", - " },\n", - " \"eventMetadata\": {\n", - " \"eventId\": str(inference_id),\n", - " },\n", - " \"eventVersion\": \"0\",\n", - " }\n", - "\n", - " def _upload_ground_truth(records, upload_time):\n", - " records = [json.dumps(r) for r in records]\n", - " data = \"\\n\".join(records)\n", - " uri = f\"{ground_truth_location}/{upload_time:%Y/%m/%d/%H/%M%S}.jsonl\"\n", - "\n", - " print(f\"Uploading ground truth data to {uri}...\")\n", - "\n", - " S3Uploader.upload_string_as_file_body(data, uri)\n", - "\n", - " def _generate_ground_truth_data(max_records, stop_ground_truth_thread):\n", - " while True:\n", - " records = [_generate_ground_truth_record(i) for i in range(max_records)]\n", - " _upload_ground_truth(records, datetime.utcnow())\n", - "\n", - " if stop_ground_truth_thread.is_set():\n", - " break\n", - "\n", - " sleep(30)\n", - "\n", - " stop_ground_truth_thread = Event()\n", - " data = pd.read_csv(DATA_FILEPATH).dropna()\n", - "\n", - " groundtruth_thread = Thread(\n", - " target=_generate_ground_truth_data,\n", - " args=(\n", - " len(data),\n", - " stop_ground_truth_thread,\n", - " ),\n", - " )\n", - "\n", - " groundtruth_thread.start()\n", - "\n", - " return stop_ground_truth_thread, groundtruth_thread" - ] - }, - { - "cell_type": "markdown", - "id": "bca6ebbc", + "id": "59e53138", "metadata": {}, "source": [ - "Let's now start generating fake labels:\n" + "These files contain the data captured by the endpoint in a SageMaker-specific JSON-line format. Each inference request is captured in a single line in the `jsonl` file. The line contains both the input and output merged together:" ] }, { "cell_type": "markdown", - "id": "5278b3e1", + "id": "5754a314-3bc0-4b41-8767-e9f06d96d250", "metadata": {}, "source": [ - "#| hide\n", + "### Step 10 - Generating Fake Labels\n", "\n", - "
Note: \n", - " The %%script cell magic is a convenient way to prevent the notebook from executing a specific cell. If you want to run the cell, comment out the line containing the %%script cell magic.\n", - "
\n" + "To test the performance of the model, we need to label the samples captured by the endpoint. We can simulate the labeling process by generating a random label for every sample. Check [Ingest Ground Truth Labels and Merge Them With Predictions](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality-merge.html) for more information about this.\n" ] }, { "cell_type": "code", - "execution_count": 214, - "id": "1f516993", + "execution_count": 395, + "id": "bb999995", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'s3://mlschool/penguins/monitoring/groundtruth/2023/10/24/19/3952.jsonl'" + ] + }, + "execution_count": 395, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "%%script false --no-raise-error\n", - "#| eval: false\n", "#| code: true\n", "#| output: false\n", "\n", - "stop_ground_truth_thread, groundtruth_thread = generate_ground_truth_data(\n", - " GROUND_TRUTH_LOCATION\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "1813bd51-df2d-4ef7-9a37-098d230e4450", - "metadata": {}, - "source": [ - "#| hide\n", - "\n", - "Let's make a prediction for a penguin and include extra fields in the request. This should be flagged by the monitoring job.\n", + "import random\n", + "from datetime import datetime\n", + "from sagemaker.s3 import S3Uploader\n", "\n", - "
Uncomment the %%script cell magic line to execute this cell.
\n" - ] - }, - { - "cell_type": "code", - "execution_count": 475, - "id": "83157778-dab9-4c4d-9127-bd74e1b35b93", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "#| hide\n", + "records = []\n", + "for inference_id in range(len(data)):\n", + " random.seed(inference_id)\n", "\n", - "%%script false --no-raise-error\n", + " records.append(json.dumps({\n", + " \"groundTruthData\": {\n", + " \"data\": random.choice([\"Adelie\", \"Chinstrap\", \"Gentoo\"]),\n", + " \"encoding\": \"CSV\",\n", + " },\n", + " \"eventMetadata\": {\n", + " \"eventId\": str(inference_id),\n", + " },\n", + " \"eventVersion\": \"0\",\n", + " }))\n", "\n", - "predictor.predict({\n", - " \"island\": \"Dream\",\n", - " \"culmen_length_mm\": 46.4,\n", - " \"culmen_depth_mm\": 18.6,\n", - " \"flipper_length_mm\": 190.0,\n", - " \"body_mass_g\": 5608.0,\n", - " \n", - " # These two columns are not in the baseline data,\n", - " # so they will be reported by the monitoring job\n", - " # as a violation.\n", - " \"name\": \"Johnny\",\n", - " \"height\": 28.0\n", - "})" + "groundtruth_payload = \"\\n\".join(records)\n", + "upload_time = datetime.utcnow()\n", + "uri = f\"{GROUND_TRUTH_LOCATION}/{upload_time:%Y/%m/%d/%H/%M%S}.jsonl\"\n", + "S3Uploader.upload_string_as_file_body(groundtruth_payload, uri)" ] }, { @@ -5734,14 +5656,14 @@ "id": "a65bd669", "metadata": {}, "source": [ - "### Step 10 - Preparing Monitoring Functions\n", + "### Step 11 - Preparing Monitoring Functions\n", "\n", "Let's create a few functions that will help us work with monitoring schedules later on:" ] }, { "cell_type": "code", - "execution_count": 216, + "execution_count": 380, "id": "da145ba1-4966-4dab-8a73-281db364cbc7", "metadata": { "tags": [] @@ -5869,7 +5791,7 @@ "id": "d936df76-e0b8-4dad-a04f-ef77ce2a2df1", "metadata": {}, "source": [ - "### Step 11 - Setting Up Data Monitoring Job\n", + "### Step 12 - Setting Up Data Monitoring Job\n", "\n", "SageMaker looks for violations in the data captured by the endpoint. By default, it combines the input data with the endpoint output and compares the result with the baseline we generated. If we let SageMaker do this, we will get a few violations, for example an \"extra column check\" violation because the field `confidence` doesn't exist in the baseline data.\n", "\n", @@ -5880,7 +5802,7 @@ }, { "cell_type": "code", - "execution_count": 237, + "execution_count": 381, "id": "cc119422-2e85-4e8c-86cd-6d59e353d09d", "metadata": { "tags": [] @@ -5900,7 +5822,7 @@ }, { "cell_type": "code", - "execution_count": 238, + "execution_count": 382, "id": "083b0bd0-4035-43fe-9b2c-946b12a5e266", "metadata": { "tags": [] @@ -5924,7 +5846,7 @@ "def preprocess_handler(inference_record):\n", " input_data = inference_record.endpoint_input.data\n", " output_data = json.loads(inference_record.endpoint_output.data)\n", - " \n", + "\n", " response = json.loads(input_data)\n", " response[\"species\"] = output_data[\"prediction\"]\n", "\n", @@ -5943,7 +5865,7 @@ }, { "cell_type": "code", - "execution_count": 240, + "execution_count": 383, "id": "96e5c0c1-7e40-47df-8f40-1d891db13875", "metadata": { "tags": [] @@ -5971,107 +5893,6 @@ ")" ] }, - { - "cell_type": "markdown", - "id": "062fb443", - "metadata": {}, - "source": [ - "Our pipeline generated data baseline statistics and constraints using our train set. We can take a look at what these values look like by downloading them from S3:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3fa3d73", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"name\": \"species\",\n", - " \"inferred_type\": \"String\",\n", - " \"string_statistics\": {\n", - " \"common\": {\n", - " \"num_present\": 344,\n", - " \"num_missing\": 0\n", - " },\n", - " \"distinct_count\": 3.0,\n", - " \"distribution\": {\n", - " \"categorical\": {\n", - " \"buckets\": [\n", - " {\n", - " \"value\": \"Adelie\",\n", - " \"count\": 152\n", - " },\n", - " {\n", - " \"value\": \"Chinstrap\",\n", - " \"count\": 68\n", - " },\n", - " {\n", - " \"value\": \"Gentoo\",\n", - " \"count\": 124\n", - " }\n", - " ]\n", - " }\n", - " }\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "try:\n", - " response = json.loads(\n", - " S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/statistics.json\")\n", - " )\n", - " print(json.dumps(response[\"features\"][0], indent=2))\n", - "except Exception as e:\n", - " pass" - ] - }, - { - "cell_type": "markdown", - "id": "20067cda", - "metadata": {}, - "source": [ - "And here are the constraints:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e940197", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"name\": \"species\",\n", - " \"inferred_type\": \"String\",\n", - " \"completeness\": 1.0,\n", - " \"string_constraints\": {\n", - " \"domains\": [\n", - " \"Adelie\",\n", - " \"Chinstrap\",\n", - " \"Gentoo\"\n", - " ]\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "try:\n", - " response = json.loads(S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/constraints.json\"))\n", - " print(json.dumps(response[\"features\"][0], indent=2))\n", - "except Exception as e:\n", - " pass" - ] - }, { "cell_type": "markdown", "id": "56e107eb-546d-431c-b74d-1bfd412711b7", @@ -6094,7 +5915,7 @@ }, { "cell_type": "code", - "execution_count": 242, + "execution_count": 385, "id": "15caf9e1-97fc-4379-893b-6062d4bd876e", "metadata": { "tags": [] @@ -6122,6 +5943,7 @@ " statistics=f\"{DATA_QUALITY_LOCATION}/statistics.json\",\n", " constraints=f\"{DATA_QUALITY_LOCATION}/constraints.json\",\n", " schedule_cron_expression=CronExpressionGenerator.hourly(),\n", + " output_s3_uri=DATA_QUALITY_LOCATION,\n", " enable_cloudwatch_metrics=True,\n", ")" ] @@ -6136,7 +5958,7 @@ }, { "cell_type": "code", - "execution_count": 243, + "execution_count": 401, "id": "2c04fdd4-cc03-496c-a0a1-405854505c46", "metadata": { "tags": [] @@ -6146,7 +5968,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "There's no DataQuality Monitoring Schedule.\n" + "{\n", + " \"MonitoringScheduleName\": \"penguins-data-monitoring-schedule\",\n", + " \"MonitoringType\": \"DataQuality\",\n", + " \"Status\": \"Failed\",\n", + " \"FailureReason\": \"Job inputs had no data\"\n", + "}\n" ] } ], @@ -6159,71 +5986,13 @@ "id": "3a9d201d-f60f-49f2-b4e9-eb0a0159ecfd", "metadata": {}, "source": [ - "### Step 12 - Setting up Model Monitoring Job\n", + "### Step 13 - Setting up Model Monitoring Job\n", "\n", "To set up a Model Quality Monitoring Job, we can use the [ModelQualityMonitor](https://sagemaker.readthedocs.io/en/stable/api/inference/model_monitor.html#sagemaker.model_monitor.model_monitoring.ModelQualityMonitor) class. The [EndpointInput](https://sagemaker.readthedocs.io/en/v2.24.2/api/inference/model_monitor.html#sagemaker.model_monitor.model_monitoring.EndpointInput) instance configures the attribute the monitoring job should use to determine the prediction from the model.\n", "\n", "Check [Amazon SageMaker Model Quality Monitor](https://sagemaker-examples.readthedocs.io/en/latest/sagemaker_model_monitor/model_quality/model_quality_churn_sdk.html) for a complete tutorial on how to run a Model Monitoring Job in SageMaker." ] }, - { - "cell_type": "markdown", - "id": "664a2f3a", - "metadata": {}, - "source": [ - "Let's check the baseline performance that we generated using the test set:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ecd37e48", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"version\": 0.0,\n", - " \"multiclass_classification_constraints\": {\n", - " \"accuracy\": {\n", - " \"threshold\": 0.9259259259259259,\n", - " \"comparison_operator\": \"LessThanThreshold\"\n", - " },\n", - " \"weighted_recall\": {\n", - " \"threshold\": 0.9259259259259259,\n", - " \"comparison_operator\": \"LessThanThreshold\"\n", - " },\n", - " \"weighted_precision\": {\n", - " \"threshold\": 0.933862433862434,\n", - " \"comparison_operator\": \"LessThanThreshold\"\n", - " },\n", - " \"weighted_f0_5\": {\n", - " \"threshold\": 0.928855833521148,\n", - " \"comparison_operator\": \"LessThanThreshold\"\n", - " },\n", - " \"weighted_f1\": {\n", - " \"threshold\": 0.9247293447293448,\n", - " \"comparison_operator\": \"LessThanThreshold\"\n", - " },\n", - " \"weighted_f2\": {\n", - " \"threshold\": 0.9242942991137502,\n", - " \"comparison_operator\": \"LessThanThreshold\"\n", - " }\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "try:\n", - " response = json.loads(S3Downloader.read_file(f\"{MODEL_QUALITY_LOCATION}/constraints.json\"))\n", - " print(json.dumps(response, indent=2))\n", - "except Exception as e:\n", - " pass" - ] - }, { "cell_type": "markdown", "id": "9d217afd", @@ -6246,7 +6015,7 @@ }, { "cell_type": "code", - "execution_count": 485, + "execution_count": 388, "id": "070e0d73-5375-4fc3-b94c-da0574600c05", "metadata": {}, "outputs": [], @@ -6255,6 +6024,7 @@ "#| code: true\n", "#| output: false\n", "#| eval: false\n", + "from sagemaker.model_monitor import ModelQualityMonitor, EndpointInput\n", "\n", "model_monitor = ModelQualityMonitor(\n", " instance_type=\"ml.m5.xlarge\",\n", @@ -6274,11 +6044,9 @@ " \n", " problem_type=\"MulticlassClassification\",\n", " ground_truth_input=GROUND_TRUTH_LOCATION,\n", - " \n", " constraints=f\"{MODEL_QUALITY_LOCATION}/constraints.json\",\n", - " \n", " schedule_cron_expression=CronExpressionGenerator.hourly(),\n", - " output_s3_uri=f\"{S3_LOCATION}/monitoring/model-quality\",\n", + " output_s3_uri=MODEL_QUALITY_LOCATION,\n", " enable_cloudwatch_metrics=True,\n", ")" ] @@ -6293,7 +6061,7 @@ }, { "cell_type": "code", - "execution_count": 245, + "execution_count": 402, "id": "347de298-16f2-42e0-85c4-dfc916080020", "metadata": { "tags": [] @@ -6303,7 +6071,38 @@ "name": "stdout", "output_type": "stream", "text": [ - "There's no ModelQuality Monitoring Schedule.\n" + "{\n", + " \"MonitoringScheduleName\": \"penguins-model-monitoring-schedule\",\n", + " \"MonitoringType\": \"ModelQuality\",\n", + " \"Status\": \"CompletedWithViolations\",\n", + " \"Violations\": [\n", + " {\n", + " \"constraint_check_type\": \"LessThanThreshold\",\n", + " \"description\": \"Metric weightedF2 with 0.3505018546481581 +/- 0.004778110439777429 was LessThanThreshold '0.9242942991137502'\",\n", + " \"metric_name\": \"weightedF2\"\n", + " },\n", + " {\n", + " \"constraint_check_type\": \"LessThanThreshold\",\n", + " \"description\": \"Metric accuracy with 0.35755813953488375 +/- 0.004625699974871179 was LessThanThreshold '0.9259259259259259'\",\n", + " \"metric_name\": \"accuracy\"\n", + " },\n", + " {\n", + " \"constraint_check_type\": \"LessThanThreshold\",\n", + " \"description\": \"Metric weightedRecall with 0.3575581395348837 +/- 0.004625699974871179 was LessThanThreshold '0.9259259259259259'\",\n", + " \"metric_name\": \"weightedRecall\"\n", + " },\n", + " {\n", + " \"constraint_check_type\": \"LessThanThreshold\",\n", + " \"description\": \"Metric weightedPrecision with 0.35662633279042494 +/- 0.005592963346101618 was LessThanThreshold '0.933862433862434'\",\n", + " \"metric_name\": \"weightedPrecision\"\n", + " },\n", + " {\n", + " \"constraint_check_type\": \"LessThanThreshold\",\n", + " \"description\": \"Metric weightedF1 with 0.34519661584972283 +/- 0.004997774377359799 was LessThanThreshold '0.9247293447293448'\",\n", + " \"metric_name\": \"weightedF1\"\n", + " }\n", + " ]\n", + "}\n" ] } ], @@ -6316,14 +6115,47 @@ "id": "38c3d9f6", "metadata": {}, "source": [ - "### Step 13 - Stopping Monitoring Jobs\n", + "### Step 14 - Tearing Down Resources\n", "\n", - "The following code will stop the generation of traffic and labels, delete the monitoring jobs, and delete the endpoint." + "The following code will stop the monitoring jobs and delete the endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": 403, + "id": "bb74dc04-54a1-4a3f-854f-4877f7f0b4a1", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Monitoring schedule deleted.\n", + "Monitoring schedule deleted.\n" + ] + } + ], + "source": [ + "#| code: true\n", + "#| output: false\n", + "\n", + "delete_data_monitoring_schedule(ENDPOINT)\n", + "delete_model_monitoring_schedule(ENDPOINT)" + ] + }, + { + "cell_type": "markdown", + "id": "c97e5419", + "metadata": {}, + "source": [ + "Let's delete the endpoint:" ] }, { "cell_type": "markdown", - "id": "2c267ea0-f9c0-4bf2-8281-8d21edebb2a0", + "id": "307f5062", "metadata": {}, "source": [ "#| hide\n", @@ -6335,26 +6167,24 @@ }, { "cell_type": "code", - "execution_count": 247, - "id": "bb74dc04-54a1-4a3f-854f-4877f7f0b4a1", - "metadata": { - "tags": [] - }, - "outputs": [], + "execution_count": 404, + "id": "9eabe84e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker:Deleting endpoint configuration with name: penguins-endpoint-config-1024190416\n", + "INFO:sagemaker:Deleting endpoint with name: penguins-endpoint\n" + ] + } + ], "source": [ "%%script false --no-raise-error\n", + "#| eval: false\n", "#| code: true\n", "#| output: false\n", - "#| eval: false\n", - "\n", - "stop_traffic_thread.set()\n", - "traffic_thread.join()\n", - "\n", - "stop_ground_truth_thread.set()\n", - "groundtruth_thread.join()\n", - "\n", - "delete_data_monitoring_schedule(ENDPOINT)\n", - "delete_model_monitoring_schedule(ENDPOINT)\n", "\n", "predictor.delete_endpoint()" ] @@ -6366,15 +6196,15 @@ "source": [ "### Assignments\n", "\n", - "* Assignment 5.1 We built a custom inference script to handle the input and output of our endpoint. However, this custom code doesn't support processing more than one sample simultaneously. Modify the inference script to allow the processing of multiple samples in a single request. The output should be a JSON containing an array of objects with the prediction and the confidence corresponding to each input sample.\n", + "* Assignment 5.1 You can visualize the results of your monitoring jobs in Amazon SageMaker Studio. Go to your endpoint, and visit the Data quality and Model quality tabs. View the details of your monitoring jobs, and create a few charts to explore the baseline and the captured values for any metric that the monitoring job calculates.\n", "\n", - "* Assignment 5.2 You can visualize the results of your monitoring jobs in Amazon SageMaker Studio. Go to your endpoint, and visit the Data quality and Model quality tabs. View the details of your monitoring jobs, and create a few charts to explore the baseline and the captured values for any metric that the monitoring job calculates.\n", + "* Assignment 5.2 The QualityCheck Step runs a processing job to compute baseline statistics and constraints from the input dataset. We configured the pipeline to generate the initial baselines every time it runs. Modify the code to prevent the pipeline from registering a new version of the model if the dataset violates the baseline of the previous model version. You can configure the QualityCheck Step to accomplish this.\n", "\n", - "* Assignment 5.3 The QualityCheck Step runs a processing job to compute baseline statistics and constraints from the input dataset. We configured the pipeline to generate the initial baselines every time it runs. Modify the code to prevent the pipeline from registering a new version of the model if the dataset violates the baseline of the previous model version. You can configure the QualityCheck Step to accomplish this.\n", + "* Assignment 5.3 We are generating predictions for the test set twice during the execution of our pipeline. First, during the Evaluation Step, and then using a Transform Step in anticipation of generating the baseline to monitor the model. Modify the Evaluation Step so it reuses the model performance computed by the QualityCheck Step instead of generating predictions again.\n", "\n", - "* Assignment 5.4 We are generating predictions for the test set twice during the execution of our pipeline. First, in the Evaluation step, and then using a Transform Step in anticipation of generating the baseline to monitor the model. Modify the pipeline to remove the Evaluation step and reuse the metrics computed by the QualityCheck Step to determine whether we should register the model.\n", + "* Assignment 5.4 [Evidently AI](https://evidentlyai.com/) is an open-source Machine Learning observability platform that you can use to evaluate, test, and monitor models. For this assignment, integrate the endpoint we built with Evidently AI to use its capabilities to monitor the model.\n", "\n", - "* Assignment 5.5 Modify the SageMaker Pipeline you created for the \"Pipeline of Digits\" project and add the necessary steps to generate a model quality baseline. Schedule a Model Monitoring Job that reports any violations if there's model drift.\n" + "* Assignment 5.5 Instead of running the entire pipeline from start to finish, sometimes you may only need to iterate over particular steps. SageMaker Pipelines supports [Selective Execution for Pipeline Steps](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-selective-ex.html). In this assignment you will use Selective Execution to only run one specific step of the pipeline. [Unlocking efficiency: Harnessing the power of Selective Execution in Amazon SageMaker Pipelines](https://aws.amazon.com/blogs/machine-learning/unlocking-efficiency-harnessing-the-power-of-selective-execution-in-amazon-sagemaker-pipelines/) is a great article that explains this feature." ] } ], diff --git a/program/images/deployment.png b/program/images/deployment.png new file mode 100644 index 0000000..aa02665 Binary files /dev/null and b/program/images/deployment.png differ diff --git a/program/images/monitoring.png b/program/images/monitoring.png new file mode 100644 index 0000000..6691eb6 Binary files /dev/null and b/program/images/monitoring.png differ