diff --git a/program/cohort.ipynb b/program/cohort.ipynb
index 2096e56..1cc7146 100644
--- a/program/cohort.ipynb
+++ b/program/cohort.ipynb
@@ -60,12 +60,12 @@
"%load_ext dotenv\n",
"%dotenv\n",
"\n",
- "import sys\n",
- "import logging\n",
- "import ipytest\n",
"import json\n",
+ "import logging\n",
+ "import sys\n",
"from pathlib import Path\n",
"\n",
+ "import ipytest\n",
"\n",
"CODE_FOLDER = Path(\"code\")\n",
"sys.path.extend([f\"./{CODE_FOLDER}\"])\n",
@@ -165,7 +165,7 @@
"outputs": [],
"source": [
"import sagemaker\n",
- "from sagemaker.workflow.pipeline_context import PipelineSession, LocalPipelineSession\n",
+ "from sagemaker.workflow.pipeline_context import LocalPipelineSession, PipelineSession\n",
"\n",
"pipeline_session = PipelineSession(default_bucket=bucket) if not LOCAL_MODE else None\n",
"\n",
@@ -350,8 +350,8 @@
}
],
"source": [
- "import pandas as pd\n",
"import numpy as np\n",
+ "import pandas as pd\n",
"\n",
"penguins = pd.read_csv(DATA_FILEPATH)\n",
"penguins.head()"
@@ -679,7 +679,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 369,
"id": "cc42cb08-275c-4b05-9d2b-77052da2f336",
"metadata": {
"tags": []
@@ -688,23 +688,23 @@
{
"data": {
"text/plain": [
- "species 0\n",
- "island 0\n",
- "culmen_length_mm 2\n",
- "culmen_depth_mm 2\n",
- "flipper_length_mm 2\n",
- "body_mass_g 2\n",
- "sex 11\n",
+ "species 0\n",
+ "island 0\n",
+ "culmen_length_mm 0\n",
+ "culmen_depth_mm 0\n",
+ "flipper_length_mm 0\n",
+ "body_mass_g 0\n",
+ "sex 0\n",
"dtype: int64"
]
},
- "execution_count": 11,
+ "execution_count": 369,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "penguins.isnull().sum()"
+ "penguins.isna().sum()"
]
},
{
@@ -717,7 +717,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 370,
"id": "3c57d55d-afd6-467a-a7a8-ff04132770ed",
"metadata": {
"tags": []
@@ -736,7 +736,7 @@
"dtype: int64"
]
},
- "execution_count": 12,
+ "execution_count": 370,
"metadata": {},
"output_type": "execute_result"
}
@@ -748,7 +748,7 @@
"penguins.iloc[:, :] = imputer.fit_transform(penguins)\n",
"\n",
"# Let's display again the number of missing values:\n",
- "penguins.isnull().sum()"
+ "penguins.isna().sum()"
]
},
{
@@ -1206,7 +1206,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 378,
"id": "fb6ba7c0-1bd6-4fe5-8b7f-f6cbdfd3846c",
"metadata": {
"tags": [
@@ -1230,36 +1230,34 @@
"import os\n",
"import tarfile\n",
"import tempfile\n",
+ "from pathlib import Path\n",
+ "\n",
"import joblib\n",
"import numpy as np\n",
"import pandas as pd\n",
- "\n",
- "from pathlib import Path\n",
"from sklearn.compose import ColumnTransformer, make_column_selector\n",
"from sklearn.impute import SimpleImputer\n",
- "from sklearn.pipeline import make_pipeline\n",
"from sklearn.model_selection import train_test_split\n",
- "from sklearn.preprocessing import OneHotEncoder, StandardScaler, OrdinalEncoder\n",
+ "from sklearn.pipeline import make_pipeline\n",
+ "from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler\n",
"\n",
"\n",
"def preprocess(base_directory):\n",
- " \"\"\"\n",
- " This function loads the supplied data, splits it\n",
- " and transforms it.\n",
- " \"\"\"\n",
- "\n",
+ " \"\"\"Load the supplied data, split it and transform it.\"\"\"\n",
" df = _read_data_from_input_csv_files(base_directory)\n",
"\n",
" target_transformer = ColumnTransformer(\n",
- " transformers=[(\"species\", OrdinalEncoder(), [0])]\n",
+ " transformers=[(\"species\", OrdinalEncoder(), [0])],\n",
" )\n",
"\n",
" numeric_transformer = make_pipeline(\n",
- " SimpleImputer(strategy=\"mean\"), StandardScaler()\n",
+ " SimpleImputer(strategy=\"mean\"),\n",
+ " StandardScaler(),\n",
" )\n",
"\n",
" categorical_transformer = make_pipeline(\n",
- " SimpleImputer(strategy=\"most_frequent\"), OneHotEncoder()\n",
+ " SimpleImputer(strategy=\"most_frequent\"),\n",
+ " OneHotEncoder(),\n",
" )\n",
"\n",
" features_transformer = ColumnTransformer(\n",
@@ -1270,7 +1268,7 @@
" make_column_selector(dtype_exclude=\"object\"),\n",
" ),\n",
" (\"categorical\", categorical_transformer, [\"island\"]),\n",
- " ]\n",
+ " ],\n",
" )\n",
"\n",
" df_train, df_validation, df_test = _split_data(df)\n",
@@ -1279,40 +1277,47 @@
" _save_test_baseline(base_directory, df_test)\n",
"\n",
" y_train = target_transformer.fit_transform(\n",
- " np.array(df_train.species.values).reshape(-1, 1)\n",
+ " np.array(df_train.species.values).reshape(-1, 1),\n",
" )\n",
" y_validation = target_transformer.transform(\n",
- " np.array(df_validation.species.values).reshape(-1, 1)\n",
+ " np.array(df_validation.species.values).reshape(-1, 1),\n",
" )\n",
" y_test = target_transformer.transform(\n",
- " np.array(df_test.species.values).reshape(-1, 1)\n",
+ " np.array(df_test.species.values).reshape(-1, 1),\n",
" )\n",
"\n",
" df_train = df_train.drop(\"species\", axis=1)\n",
" df_validation = df_validation.drop(\"species\", axis=1)\n",
" df_test = df_test.drop(\"species\", axis=1)\n",
"\n",
- " X_train = features_transformer.fit_transform(df_train)\n",
- " X_validation = features_transformer.transform(df_validation)\n",
- " X_test = features_transformer.transform(df_test)\n",
+ " X_train = features_transformer.fit_transform(df_train) # noqa: N806\n",
+ " X_validation = features_transformer.transform(df_validation) # noqa: N806\n",
+ " X_test = features_transformer.transform(df_test) # noqa: N806\n",
"\n",
" _save_splits(\n",
- " base_directory, X_train, y_train, X_validation, y_validation, X_test, y_test\n",
+ " base_directory,\n",
+ " X_train,\n",
+ " y_train,\n",
+ " X_validation,\n",
+ " y_validation,\n",
+ " X_test,\n",
+ " y_test,\n",
" )\n",
" _save_model(base_directory, target_transformer, features_transformer)\n",
"\n",
"\n",
"def _read_data_from_input_csv_files(base_directory):\n",
- " \"\"\"\n",
+ " \"\"\"Read the data from the input CSV files.\n",
+ "\n",
" This function reads every CSV file available and\n",
" concatenates them into a single dataframe.\n",
" \"\"\"\n",
- "\n",
" input_directory = Path(base_directory) / \"input\"\n",
- " files = [file for file in input_directory.glob(\"*.csv\")]\n",
+ " files = list(input_directory.glob(\"*.csv\"))\n",
"\n",
" if len(files) == 0:\n",
- " raise ValueError(f\"The are no CSV files in {input_directory.as_posix()}/\")\n",
+ " message = f\"The are no CSV files in {input_directory.as_posix()}/\"\n",
+ " raise ValueError(message)\n",
"\n",
" raw_data = [pd.read_csv(file) for file in files]\n",
" df = pd.concat(raw_data)\n",
@@ -1322,10 +1327,7 @@
"\n",
"\n",
"def _split_data(df):\n",
- " \"\"\"\n",
- " Splits the data into three sets: train, validation and test.\n",
- " \"\"\"\n",
- "\n",
+ " \"\"\"Split the data into train, validation, and test.\"\"\"\n",
" df_train, temp = train_test_split(df, test_size=0.3)\n",
" df_validation, df_test = train_test_split(temp, test_size=0.5)\n",
"\n",
@@ -1333,13 +1335,12 @@
"\n",
"\n",
"def _save_train_baseline(base_directory, df_train):\n",
- " \"\"\"\n",
+ " \"\"\"Save the untransformed training data to disk.\n",
+ "\n",
" We will need the training data to compute a baseline to\n",
" determine the quality of the data that the model receives\n",
- " when deployed. This function saves the untransformed training\n",
- " data to disk.\n",
+ " when deployed.\n",
" \"\"\"\n",
- "\n",
" baseline_path = Path(base_directory) / \"train-baseline\"\n",
" baseline_path.mkdir(parents=True, exist_ok=True)\n",
"\n",
@@ -1347,18 +1348,17 @@
"\n",
" # To compute the data quality baseline, we don't need the\n",
" # target variable, so we'll drop it from the dataframe.\n",
- " df.drop(\"species\", axis=1, inplace=True)\n",
+ " df = df.drop(\"species\", axis=1)\n",
"\n",
" df.to_csv(baseline_path / \"train-baseline.csv\", header=True, index=False)\n",
"\n",
"\n",
"def _save_test_baseline(base_directory, df_test):\n",
- " \"\"\"\n",
+ " \"\"\"Save the untransformed test data to disk.\n",
+ "\n",
" We will need the test data to compute a baseline to\n",
" determine the quality of the model predictions when deployed.\n",
- " This function saves the untransformed test data to disk.\n",
" \"\"\"\n",
- "\n",
" baseline_path = Path(base_directory) / \"test-baseline\"\n",
" baseline_path.mkdir(parents=True, exist_ok=True)\n",
"\n",
@@ -1371,14 +1371,20 @@
"\n",
"\n",
"def _save_splits(\n",
- " base_directory, X_train, y_train, X_validation, y_validation, X_test, y_test\n",
+ " base_directory,\n",
+ " X_train, # noqa: N803\n",
+ " y_train,\n",
+ " X_validation, # noqa: N803\n",
+ " y_validation,\n",
+ " X_test, # noqa: N803\n",
+ " y_test,\n",
"):\n",
- " \"\"\"\n",
+ " \"\"\"Save data splits to disk.\n",
+ "\n",
" This function concatenates the transformed features\n",
" and the target variable, and saves each one of the split\n",
" sets to disk.\n",
" \"\"\"\n",
- "\n",
" train = np.concatenate((X_train, y_train), axis=1)\n",
" validation = np.concatenate((X_validation, y_validation), axis=1)\n",
" test = np.concatenate((X_test, y_test), axis=1)\n",
@@ -1393,29 +1399,31 @@
"\n",
" pd.DataFrame(train).to_csv(train_path / \"train.csv\", header=False, index=False)\n",
" pd.DataFrame(validation).to_csv(\n",
- " validation_path / \"validation.csv\", header=False, index=False\n",
+ " validation_path / \"validation.csv\",\n",
+ " header=False,\n",
+ " index=False,\n",
" )\n",
" pd.DataFrame(test).to_csv(test_path / \"test.csv\", header=False, index=False)\n",
"\n",
"\n",
"def _save_model(base_directory, target_transformer, features_transformer):\n",
- " \"\"\"\n",
+ " \"\"\"Save the Scikit-Learn transformation pipelines.\n",
+ "\n",
" This function creates a model.tar.gz file that\n",
" contains the two transformation pipelines we built\n",
" to transform the data.\n",
" \"\"\"\n",
- "\n",
" with tempfile.TemporaryDirectory() as directory:\n",
- " joblib.dump(target_transformer, os.path.join(directory, \"target.joblib\"))\n",
- " joblib.dump(features_transformer, os.path.join(directory, \"features.joblib\"))\n",
+ " joblib.dump(target_transformer, Path(directory) / \"target.joblib\")\n",
+ " joblib.dump(features_transformer, Path(directory) / \"features.joblib\")\n",
"\n",
" model_path = Path(base_directory) / \"model\"\n",
" model_path.mkdir(parents=True, exist_ok=True)\n",
"\n",
" with tarfile.open(f\"{(model_path / 'model.tar.gz').as_posix()}\", \"w:gz\") as tar:\n",
- " tar.add(os.path.join(directory, \"target.joblib\"), arcname=\"target.joblib\")\n",
+ " tar.add(Path(directory) / \"target.joblib\", arcname=\"target.joblib\")\n",
" tar.add(\n",
- " os.path.join(directory, \"features.joblib\"), arcname=\"features.joblib\"\n",
+ " Path(directory) / \"features.joblib\", arcname=\"features.joblib\",\n",
" )\n",
"\n",
"\n",
@@ -1433,7 +1441,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 379,
"id": "d1f122a4-acff-4687-91b9-bfef13567d88",
"metadata": {
"tags": [
@@ -1445,25 +1453,32 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n",
- "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.16s\u001b[0m\u001b[0m\n"
+ "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\n",
+ "\u001b[32m\u001b[32m\u001b[1m8 passed\u001b[0m\u001b[32m in 0.18s\u001b[0m\u001b[0m\n"
]
}
],
"source": [
"%%ipytest -s\n",
- "#| code-fold: true\n",
+ "# | code-fold: true\n",
"\n",
"import os\n",
- "import shutil \n",
+ "import shutil\n",
"import tarfile\n",
- "import pytest\n",
"import tempfile\n",
"\n",
+ "import pytest\n",
"from processing.script import preprocess\n",
"\n",
"\n",
- "@pytest.fixture(scope=\"function\", autouse=False)\n",
+ "@pytest.fixture(autouse=False)\n",
"def directory():\n",
" directory = tempfile.mkdtemp()\n",
" input_directory = Path(directory) / \"input\"\n",
@@ -1523,7 +1538,8 @@
"\n",
"def test_train_baseline_is_not_transformed(directory):\n",
" baseline = pd.read_csv(\n",
- " directory / \"train-baseline\" / \"train-baseline.csv\", header=None\n",
+ " directory / \"train-baseline\" / \"train-baseline.csv\",\n",
+ " header=None,\n",
" )\n",
"\n",
" island = baseline.iloc[:, 0].unique()\n",
@@ -1684,9 +1700,8 @@
}
],
"source": [
- "from sagemaker.workflow.steps import ProcessingStep\n",
"from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
- "\n",
+ "from sagemaker.workflow.steps import ProcessingStep\n",
"\n",
"preprocessing_step = ProcessingStep(\n",
" name=\"preprocess-data\",\n",
@@ -1694,7 +1709,8 @@
" code=f\"{(CODE_FOLDER / 'processing' / 'script.py').as_posix()}\",\n",
" inputs=[\n",
" ProcessingInput(\n",
- " source=dataset_location, destination=\"/opt/ml/processing/input\"\n",
+ " source=dataset_location,\n",
+ " destination=\"/opt/ml/processing/input\",\n",
" ),\n",
" ],\n",
" outputs=[\n",
@@ -1862,25 +1878,22 @@
"# | filename: script.py\n",
"# | code-line-numbers: true\n",
"\n",
- "import os\n",
"import argparse\n",
- "import tarfile\n",
"import json\n",
+ "import os\n",
+ "import tarfile\n",
+ "from pathlib import Path\n",
+ "\n",
+ "import keras\n",
"import numpy as np\n",
"import pandas as pd\n",
- "\n",
"from comet_ml import Experiment\n",
- "\n",
- "import keras\n",
- "\n",
- "from pathlib import Path\n",
- "from packaging import version\n",
- "from sklearn.metrics import accuracy_score\n",
- "\n",
"from keras import Input\n",
- "from keras.models import Sequential\n",
"from keras.layers import Dense\n",
+ "from keras.models import Sequential\n",
"from keras.optimizers import SGD\n",
+ "from packaging import version\n",
+ "from sklearn.metrics import accuracy_score\n",
"\n",
"\n",
"def train(\n",
@@ -1896,11 +1909,11 @@
"\n",
" X_train = pd.read_csv(Path(train_path) / \"train.csv\")\n",
" y_train = X_train[X_train.columns[-1]]\n",
- " X_train.drop(X_train.columns[-1], axis=1, inplace=True)\n",
+ " X_train = X_train.drop(X_train.columns[-1], axis=1)\n",
"\n",
" X_validation = pd.read_csv(Path(validation_path) / \"validation.csv\")\n",
" y_validation = X_validation[X_validation.columns[-1]]\n",
- " X_validation.drop(X_validation.columns[-1], axis=1, inplace=True)\n",
+ " X_validation = X_validation.drop(X_validation.columns[-1], axis=1)\n",
"\n",
" model = Sequential(\n",
" [\n",
@@ -2296,14 +2309,12 @@
}
],
"source": [
- "from sagemaker.workflow.steps import TrainingStep\n",
"from sagemaker.inputs import TrainingInput\n",
+ "from sagemaker.workflow.steps import TrainingStep\n",
"\n",
"\n",
"def create_training_step(estimator):\n",
- " \"\"\"\n",
- " Creates a SageMaker TrainingStep using the provided estimator.\n",
- " \"\"\"\n",
+ " \"\"\"Create a SageMaker TrainingStep using the provided estimator.\"\"\"\n",
" return TrainingStep(\n",
" name=\"train-model\",\n",
" step_args=estimator.fit(\n",
@@ -2326,7 +2337,7 @@
" ].S3Output.S3Uri,\n",
" content_type=\"application/tar+gzip\",\n",
" ),\n",
- " }\n",
+ " },\n",
" ),\n",
" cache_config=cache_config,\n",
" )\n",
@@ -2922,7 +2933,7 @@
"metadata": {},
"outputs": [],
"source": [
- "USE_TUNING_STEP = False and not LOCAL_MODE"
+ "USE_TUNING_STEP = False"
]
},
{
@@ -2957,8 +2968,8 @@
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.tuner import HyperparameterTuner\n",
"from sagemaker.parameter import IntegerParameter\n",
+ "from sagemaker.tuner import HyperparameterTuner\n",
"\n",
"tuner = HyperparameterTuner(\n",
" estimator,\n",
@@ -3167,18 +3178,18 @@
"\n",
"import json\n",
"import tarfile\n",
+ "from pathlib import Path\n",
+ "\n",
"import numpy as np\n",
"import pandas as pd\n",
- "\n",
- "from pathlib import Path\n",
- "from tensorflow import keras\n",
"from sklearn.metrics import accuracy_score\n",
+ "from tensorflow import keras\n",
"\n",
"\n",
"def evaluate(model_path, test_path, output_path):\n",
" X_test = pd.read_csv(Path(test_path) / \"test.csv\")\n",
" y_test = X_test[X_test.columns[-1]]\n",
- " X_test.drop(X_test.columns[-1], axis=1, inplace=True)\n",
+ " X_test = X_test.drop(X_test.columns[-1], axis=1)\n",
"\n",
" # Let's now extract the model package so we can load\n",
" # it in memory.\n",
@@ -3427,7 +3438,8 @@
"\n",
"if USE_TUNING_STEP:\n",
" model_assets = tune_model_step.get_top_model_s3_uri(\n",
- " top_k=0, s3_bucket=config[\"session\"].default_bucket()\n",
+ " top_k=0,\n",
+ " s3_bucket=config[\"session\"].default_bucket(),\n",
" )"
]
},
@@ -3555,7 +3567,8 @@
" # The output is the evaluation report that we generated\n",
" # in the evaluation script.\n",
" ProcessingOutput(\n",
- " output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\"\n",
+ " output_name=\"evaluation\",\n",
+ " source=\"/opt/ml/processing/evaluation\",\n",
" ),\n",
" ],\n",
" ),\n",
@@ -3706,7 +3719,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.model_metrics import ModelMetrics, MetricsSource\n",
+ "from sagemaker.model_metrics import MetricsSource, ModelMetrics\n",
"from sagemaker.workflow.functions import Join\n",
"\n",
"model_metrics = ModelMetrics(\n",
@@ -3721,7 +3734,7 @@
" ],\n",
" ),\n",
" content_type=\"application/json\",\n",
- " )\n",
+ " ),\n",
")"
]
},
@@ -3766,6 +3779,7 @@
" model_metrics=None,\n",
" drift_check_baselines=None,\n",
"):\n",
+ " \"\"\"Create a Registration Step using the supplied parameters.\"\"\"\n",
" return ModelStep(\n",
" name=\"register\",\n",
" step_args=model.register(\n",
@@ -3944,8 +3958,8 @@
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.workflow.functions import JsonGet\n",
"from sagemaker.workflow.conditions import ConditionGreaterThanOrEqualTo\n",
+ "from sagemaker.workflow.functions import JsonGet\n",
"\n",
"condition = ConditionGreaterThanOrEqualTo(\n",
" left=JsonGet(\n",
@@ -4163,7 +4177,7 @@
"\n",
"if package:\n",
" response = sagemaker_client.describe_model_package(\n",
- " ModelPackageName=package[\"ModelPackageArn\"]\n",
+ " ModelPackageName=package[\"ModelPackageArn\"],\n",
" )\n",
"\n",
" model_data = response[\"InferenceSpecification\"][\"Containers\"][0][\"ModelDataUrl\"]\n",
@@ -4723,12 +4737,12 @@
" {\n",
" \"Effect\": \"Allow\",\n",
" \"Principal\": {\n",
- " \"Service\": [\"lambda.amazonaws.com\", \"events.amazonaws.com\"]\n",
+ " \"Service\": [\"lambda.amazonaws.com\", \"events.amazonaws.com\"],\n",
" },\n",
- " \"Action\": \"sts:AssumeRole\",\n",
- " }\n",
+ " \"Acti,on\": \"sts:AssumeRole\",\n",
+ " },\n",
" ],\n",
- " }\n",
+ " },\n",
" ),\n",
" Description=\"Lambda Endpoint Deployment\",\n",
" )\n",
@@ -4823,7 +4837,6 @@
"source": [
"from sagemaker.lambda_helper import Lambda\n",
"\n",
- "\n",
"deploy_lambda_fn = Lambda(\n",
" function_name=\"deployment_fn\",\n",
" execution_role_arn=lambda_role_arn,\n",
@@ -4838,7 +4851,7 @@
" \"DATA_CAPTURE_DESTINATION\": DATA_CAPTURE_DESTINATION,\n",
" \"DATA_CAPTURE_PERCENTAGE\": str(DATA_CAPTURE_PERCENTAGE),\n",
" \"ROLE\": role,\n",
- " }\n",
+ " },\n",
" },\n",
")\n",
"\n",
@@ -4869,6 +4882,7 @@
"\n",
"\n",
"def create_deployment_step(register_model_step):\n",
+ " \"\"\"Create a Deploy Step using the supplied parameters.\"\"\"\n",
" return LambdaStep(\n",
" name=\"deploy\",\n",
" lambda_func=deploy_lambda_fn,\n",
@@ -5023,11 +5037,12 @@
],
"source": [
"def wait_for_endpoint():\n",
+ " \"\"\"Wait for the endpoint to come in service.\"\"\"\n",
" waiter = sagemaker_client.get_waiter(\"endpoint_in_service\")\n",
" waiter.wait(EndpointName=ENDPOINT, WaiterConfig={\"Delay\": 10, \"MaxAttempts\": 30})\n",
"\n",
"\n",
- "payload = \"0.6569590202313976,-1.0813829646495108,1.2097102831892812,0.9226343641317372,1.0,0.0,0.0\"\n",
+ "payload = \"0.6569590202313976,-1.0813829646495108,1.2097102831892812,0.9226343641317372,1.0,0.0,0.0\" # noqa: E501\n",
"\n",
"\n",
"try:\n",
@@ -5153,7 +5168,7 @@
" {\n",
" \"Id\": \"1\",\n",
" \"Arn\": deploy_lambda_fn_response[\"FunctionArn\"],\n",
- " }\n",
+ " },\n",
" ],\n",
")"
]
@@ -6223,7 +6238,7 @@
"id": "83fd85c2",
"metadata": {},
"source": [
- "We can also test the endpoint by sending a JSON payload. Notice how you can use a deserealizer to automatically decode the response from the model."
+ "We can also test the endpoint by sending a JSON payload. Notice how you can use a deserealizer to automatically decode the response from the model.\n"
]
},
{
@@ -6241,9 +6256,8 @@
}
],
"source": [
- "from sagemaker.serializers import JSONSerializer\n",
"from sagemaker.deserializers import JSONDeserializer\n",
- "\n",
+ "from sagemaker.serializers import JSONSerializer\n",
"\n",
"sample = {\n",
" \"island\": \"Biscoe\",\n",
@@ -6339,7 +6353,7 @@
},
{
"cell_type": "code",
- "execution_count": 266,
+ "execution_count": 305,
"id": "97b3cd80",
"metadata": {
"tags": [
@@ -6447,8 +6461,6 @@
" print(\"Processing prediction received from the model...\")\n",
"\n",
" if output:\n",
- " print(\"HERE\", output)\n",
- "\n",
" prediction = np.argmax(output[\"predictions\"][0])\n",
" confidence = output[\"predictions\"][0][prediction]\n",
"\n",
@@ -6480,7 +6492,7 @@
},
{
"cell_type": "code",
- "execution_count": 267,
+ "execution_count": 306,
"id": "91786cc7",
"metadata": {
"tags": [
@@ -6500,7 +6512,7 @@
"output_type": "stream",
"text": [
"Keras version: 2.14.0\n",
- "8/8 - 0s - loss: 1.3752 - accuracy: 0.0251 - val_loss: 1.3031 - val_accuracy: 0.0196 - 217ms/epoch - 27ms/step\n",
+ "8/8 - 0s - loss: 0.9664 - accuracy: 0.5607 - val_loss: 0.9293 - val_accuracy: 0.5490 - 215ms/epoch - 27ms/step\n",
"2/2 [==============================] - 0s 1ms/step\n"
]
},
@@ -6508,14 +6520,14 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpebq8p3w6/model/001/assets\n"
+ "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpzbvqezsx/model/001/assets\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Validation accuracy: 0.0196078431372549\n",
+ "Validation accuracy: 0.5490196078431373\n",
"Handling endpoint request\n",
"Processing input data...\n",
"Sending input data to model to make a prediction...\n"
@@ -6533,11 +6545,11 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "1/1 [==============================] - 0s 25ms/step\n",
- "Response: {'predictions': [[0.08315584808588028, 0.5809251666069031, 0.3359190821647644]]}\n",
+ "1/1 [==============================] - 0s 28ms/step\n",
+ "Response: {'predictions': [[0.4287213087081909, 0.2709770202636719, 0.300301730632782]]}\n",
"Processing prediction received from the model...\n",
- "HERE {'predictions': [[0.08315584808588028, 0.5809251666069031, 0.3359190821647644]]}\n",
- "{'prediction': 'Chinstrap', 'confidence': 0.5809251666069031}\n",
+ "HERE {'predictions': [[0.4287213087081909, 0.2709770202636719, 0.300301730632782]]}\n",
+ "{'prediction': 'Adelie', 'confidence': 0.4287213087081909}\n",
"\u001b[32m.\u001b[0m"
]
},
@@ -6553,22 +6565,22 @@
"output_type": "stream",
"text": [
"Keras version: 2.14.0\n",
- "8/8 - 0s - loss: 1.0557 - accuracy: 0.3849 - val_loss: 1.0517 - val_accuracy: 0.2941 - 215ms/epoch - 27ms/step\n",
- "2/2 [==============================] - 0s 1ms/step\n"
+ "8/8 - 0s - loss: 1.1627 - accuracy: 0.1883 - val_loss: 1.1280 - val_accuracy: 0.1961 - 215ms/epoch - 27ms/step\n",
+ "2/2 [==============================] - 0s 2ms/step\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
- "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpu3pzk9nz/model/001/assets\n"
+ "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpdalhfozy/model/001/assets\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Validation accuracy: 0.29411764705882354\n",
+ "Validation accuracy: 0.19607843137254902\n",
"Handling endpoint request\n",
"Processing input data...\n",
"Sending input data to model to make a prediction...\n"
@@ -6587,10 +6599,10 @@
"output_type": "stream",
"text": [
"1/1 [==============================] - 0s 26ms/step\n",
- "Response: {'predictions': [[0.3565749526023865, 0.21773585677146912, 0.4256891906261444]]}\n",
+ "Response: {'predictions': [[0.2188926637172699, 0.49149057269096375, 0.28961673378944397]]}\n",
"Processing prediction received from the model...\n",
- "HERE {'predictions': [[0.3565749526023865, 0.21773585677146912, 0.4256891906261444]]}\n",
- "{'prediction': 'Gentoo', 'confidence': 0.4256891906261444}\n",
+ "HERE {'predictions': [[0.2188926637172699, 0.49149057269096375, 0.28961673378944397]]}\n",
+ "{'prediction': 'Chinstrap', 'confidence': 0.49149057269096375}\n",
"\u001b[32m.\u001b[0m"
]
},
@@ -6606,7 +6618,7 @@
"output_type": "stream",
"text": [
"Keras version: 2.14.0\n",
- "8/8 - 0s - loss: 1.5010 - accuracy: 0.1883 - val_loss: 1.4162 - val_accuracy: 0.1373 - 226ms/epoch - 28ms/step\n",
+ "8/8 - 0s - loss: 1.1131 - accuracy: 0.2887 - val_loss: 1.0818 - val_accuracy: 0.3922 - 218ms/epoch - 27ms/step\n",
"2/2 [==============================] - 0s 2ms/step\n"
]
},
@@ -6614,14 +6626,14 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpj0f8xfh9/model/001/assets\n"
+ "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmpfstpzt4b/model/001/assets\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Validation accuracy: 0.13725490196078433\n",
+ "Validation accuracy: 0.39215686274509803\n",
"Handling endpoint request\n",
"Processing input data...\n",
"Sending input data to model to make a prediction...\n"
@@ -6639,11 +6651,11 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "1/1 [==============================] - 0s 27ms/step\n",
- "Response: {'predictions': [[0.7790030837059021, 0.13591648638248444, 0.08508043736219406]]}\n",
+ "1/1 [==============================] - 0s 25ms/step\n",
+ "Response: {'predictions': [[0.3396219313144684, 0.33004486560821533, 0.3303331732749939]]}\n",
"Processing prediction received from the model...\n",
- "HERE {'predictions': [[0.7790030837059021, 0.13591648638248444, 0.08508043736219406]]}\n",
- "{'prediction': 'Adelie', 'confidence': 0.7790030837059021}\n",
+ "HERE {'predictions': [[0.3396219313144684, 0.33004486560821533, 0.3303331732749939]]}\n",
+ "{'prediction': 'Adelie', 'confidence': 0.3396219313144684}\n",
"\u001b[32m.\u001b[0m"
]
},
@@ -6659,7 +6671,7 @@
"output_type": "stream",
"text": [
"Keras version: 2.14.0\n",
- "8/8 - 0s - loss: 1.3193 - accuracy: 0.1674 - val_loss: 1.3469 - val_accuracy: 0.1569 - 217ms/epoch - 27ms/step\n",
+ "8/8 - 0s - loss: 1.1813 - accuracy: 0.1506 - val_loss: 1.1372 - val_accuracy: 0.1176 - 221ms/epoch - 28ms/step\n",
"2/2 [==============================] - 0s 2ms/step\n"
]
},
@@ -6667,21 +6679,21 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp6oxxyllo/model/001/assets\n"
+ "INFO:tensorflow:Assets written to: /var/folders/4c/v1q3hy1x4mb5w0wpc72zl3_w0000gp/T/tmp4q5229yr/model/001/assets\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Validation accuracy: 0.1568627450980392\n",
+ "Validation accuracy: 0.11764705882352941\n",
"Handling endpoint request\n",
"Processing input data...\n",
"There was an error processing the input data. Expecting value: line 1 column 1 (char 0)\n",
"Processing prediction received from the model...\n",
"{'prediction': None}\n",
"\u001b[32m.\u001b[0m\n",
- "\u001b[32m\u001b[32m\u001b[1m4 passed\u001b[0m\u001b[32m in 2.36s\u001b[0m\u001b[0m\n"
+ "\u001b[32m\u001b[32m\u001b[1m4 passed\u001b[0m\u001b[32m in 2.37s\u001b[0m\u001b[0m\n"
]
}
],
@@ -6827,7 +6839,7 @@
},
{
"cell_type": "code",
- "execution_count": 268,
+ "execution_count": 307,
"id": "603cb8f5",
"metadata": {
"tags": [
@@ -6863,7 +6875,7 @@
},
{
"cell_type": "code",
- "execution_count": 269,
+ "execution_count": 308,
"id": "b5015c75",
"metadata": {},
"outputs": [],
@@ -6891,7 +6903,7 @@
},
{
"cell_type": "code",
- "execution_count": 270,
+ "execution_count": 309,
"id": "7ee3cdb6",
"metadata": {},
"outputs": [],
@@ -6911,7 +6923,7 @@
},
{
"cell_type": "code",
- "execution_count": 271,
+ "execution_count": 310,
"id": "21fb3e2c",
"metadata": {
"tags": [
@@ -6951,7 +6963,7 @@
},
{
"cell_type": "code",
- "execution_count": 272,
+ "execution_count": 311,
"id": "9e8162b7",
"metadata": {},
"outputs": [],
@@ -6971,7 +6983,7 @@
},
{
"cell_type": "code",
- "execution_count": 273,
+ "execution_count": 312,
"id": "849b379f",
"metadata": {},
"outputs": [],
@@ -6996,7 +7008,7 @@
},
{
"cell_type": "code",
- "execution_count": 274,
+ "execution_count": 313,
"id": "f64921ee",
"metadata": {
"tags": [
@@ -7021,16 +7033,16 @@
"data": {
"text/plain": [
"{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session15-pipeline',\n",
- " 'ResponseMetadata': {'RequestId': 'a7b456b4-ac91-4100-9f8e-d6fdcb370b55',\n",
+ " 'ResponseMetadata': {'RequestId': '7c77830b-b272-416a-9ff9-dcf1b39ea3d2',\n",
" 'HTTPStatusCode': 200,\n",
- " 'HTTPHeaders': {'x-amzn-requestid': 'a7b456b4-ac91-4100-9f8e-d6fdcb370b55',\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': '7c77830b-b272-416a-9ff9-dcf1b39ea3d2',\n",
" 'content-type': 'application/x-amz-json-1.1',\n",
" 'content-length': '86',\n",
- " 'date': 'Fri, 29 Mar 2024 19:19:55 GMT'},\n",
+ " 'date': 'Sat, 30 Mar 2024 15:24:30 GMT'},\n",
" 'RetryAttempts': 0}}"
]
},
- "execution_count": 274,
+ "execution_count": 313,
"metadata": {},
"output_type": "execute_result"
}
@@ -7064,7 +7076,7 @@
},
{
"cell_type": "code",
- "execution_count": 277,
+ "execution_count": 316,
"id": "1d63ce59",
"metadata": {},
"outputs": [
@@ -7072,7 +7084,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "b'{\"prediction\": \"Adelie\", \"confidence\": 0.629927337}'\n"
+ "{'prediction': 'Adelie', 'confidence': 0.629927337}\n"
]
}
],
@@ -7095,7 +7107,7 @@
" \"culmen_depth_mm\": 18.6,\n",
" \"flipper_length_mm\": 190.0,\n",
" \"body_mass_g\": 3450.0,\n",
- " }\n",
+ " },\n",
" )\n",
"\n",
" print(response)\n",
@@ -7132,7 +7144,7 @@
},
{
"cell_type": "code",
- "execution_count": 130,
+ "execution_count": 317,
"id": "f4747fc7",
"metadata": {},
"outputs": [],
@@ -7162,7 +7174,7 @@
},
{
"cell_type": "code",
- "execution_count": 131,
+ "execution_count": 318,
"id": "3fc728fa",
"metadata": {
"tags": [
@@ -7174,19 +7186,24 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n",
+ "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
"INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n"
]
}
],
"source": [
+ "from sagemaker.model_monitor.dataset_format import DatasetFormat\n",
+ "from sagemaker.workflow.check_job_config import CheckJobConfig\n",
"from sagemaker.workflow.quality_check_step import (\n",
- " QualityCheckStep,\n",
" DataQualityCheckConfig,\n",
+ " QualityCheckStep,\n",
")\n",
- "from sagemaker.workflow.check_job_config import CheckJobConfig\n",
- "from sagemaker.model_monitor.dataset_format import DatasetFormat\n",
- "\n",
"\n",
"data_quality_baseline_step = QualityCheckStep(\n",
" name=\"generate-data-quality-baseline\",\n",
@@ -7201,7 +7218,7 @@
" baseline_dataset=preprocessing_step.properties.ProcessingOutputConfig.Outputs[\n",
" \"train-baseline\"\n",
" ].S3Output.S3Uri,\n",
- " dataset_format=DatasetFormat.csv(header=True, output_columns_position=\"START\"),\n",
+ " dataset_format=DatasetFormat.csv(header=True),\n",
" output_s3_uri=DATA_QUALITY_LOCATION,\n",
" ),\n",
" model_package_group_name=PIPELINE_MODEL_PACKAGE_GROUP,\n",
@@ -7223,7 +7240,7 @@
},
{
"cell_type": "code",
- "execution_count": 132,
+ "execution_count": 319,
"id": "2d5d8133",
"metadata": {},
"outputs": [],
@@ -7265,7 +7282,7 @@
},
{
"cell_type": "code",
- "execution_count": 133,
+ "execution_count": 320,
"id": "fc5c4325",
"metadata": {
"tags": [
@@ -7296,7 +7313,7 @@
},
{
"cell_type": "code",
- "execution_count": 134,
+ "execution_count": 321,
"id": "be9d6de7",
"metadata": {},
"outputs": [],
@@ -7321,7 +7338,7 @@
},
{
"cell_type": "code",
- "execution_count": 135,
+ "execution_count": 322,
"id": "fb9b8d1e",
"metadata": {
"tags": [
@@ -7336,8 +7353,6 @@
"INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
"INFO:sagemaker.processing:Uploaded None to s3://mlschool/session16-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n",
"INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session16-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n",
- "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n",
- " warnings.warn(\n",
"WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n",
"INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
"INFO:sagemaker.processing:Uploaded None to s3://mlschool/session16-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n",
@@ -7348,16 +7363,16 @@
"data": {
"text/plain": [
"{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session16-pipeline',\n",
- " 'ResponseMetadata': {'RequestId': '07a1218c-b06c-4f38-87b5-27c741a397b4',\n",
+ " 'ResponseMetadata': {'RequestId': '49b051bc-cb53-4bf6-8fa8-b9f64033f0e3',\n",
" 'HTTPStatusCode': 200,\n",
- " 'HTTPHeaders': {'x-amzn-requestid': '07a1218c-b06c-4f38-87b5-27c741a397b4',\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': '49b051bc-cb53-4bf6-8fa8-b9f64033f0e3',\n",
" 'content-type': 'application/x-amz-json-1.1',\n",
" 'content-length': '86',\n",
- " 'date': 'Thu, 28 Mar 2024 18:15:21 GMT'},\n",
+ " 'date': 'Sat, 30 Mar 2024 16:14:21 GMT'},\n",
" 'RetryAttempts': 0}}"
]
},
- "execution_count": 135,
+ "execution_count": 322,
"metadata": {},
"output_type": "execute_result"
}
@@ -7400,17 +7415,53 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 327,
"id": "c2b95d03",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"name\": \"island\",\n",
+ " \"inferred_type\": \"String\",\n",
+ " \"string_statistics\": {\n",
+ " \"common\": {\n",
+ " \"num_present\": 236,\n",
+ " \"num_missing\": 0\n",
+ " },\n",
+ " \"distinct_count\": 3.0,\n",
+ " \"distribution\": {\n",
+ " \"categorical\": {\n",
+ " \"buckets\": [\n",
+ " {\n",
+ " \"value\": \"Dream\",\n",
+ " \"count\": 84\n",
+ " },\n",
+ " {\n",
+ " \"value\": \"Torgersen\",\n",
+ " \"count\": 32\n",
+ " },\n",
+ " {\n",
+ " \"value\": \"Biscoe\",\n",
+ " \"count\": 120\n",
+ " }\n",
+ " ]\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"try:\n",
" response = json.loads(\n",
- " S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/statistics.json\")\n",
+ " S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/statistics.json\"),\n",
" )\n",
" print(json.dumps(response[\"features\"][0], indent=2))\n",
- "except Exception:\n",
+ "except Exception: # noqa: S110\n",
" pass"
]
},
@@ -7424,17 +7475,98 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 325,
"id": "63908598",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"version\": 0.0,\n",
+ " \"features\": [\n",
+ " {\n",
+ " \"name\": \"island\",\n",
+ " \"inferred_type\": \"String\",\n",
+ " \"completeness\": 1.0,\n",
+ " \"string_constraints\": {\n",
+ " \"domains\": [\n",
+ " \"Dream\",\n",
+ " \"Torgersen\",\n",
+ " \"Biscoe\"\n",
+ " ]\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"name\": \"culmen_length_mm\",\n",
+ " \"inferred_type\": \"Fractional\",\n",
+ " \"completeness\": 1.0,\n",
+ " \"num_constraints\": {\n",
+ " \"is_non_negative\": true\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"name\": \"culmen_depth_mm\",\n",
+ " \"inferred_type\": \"Fractional\",\n",
+ " \"completeness\": 1.0,\n",
+ " \"num_constraints\": {\n",
+ " \"is_non_negative\": true\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"name\": \"flipper_length_mm\",\n",
+ " \"inferred_type\": \"Fractional\",\n",
+ " \"completeness\": 1.0,\n",
+ " \"num_constraints\": {\n",
+ " \"is_non_negative\": true\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"name\": \"body_mass_g\",\n",
+ " \"inferred_type\": \"Fractional\",\n",
+ " \"completeness\": 1.0,\n",
+ " \"num_constraints\": {\n",
+ " \"is_non_negative\": true\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"name\": \"sex\",\n",
+ " \"inferred_type\": \"String\",\n",
+ " \"completeness\": 1.0,\n",
+ " \"string_constraints\": {\n",
+ " \"domains\": [\n",
+ " \"FEMALE\",\n",
+ " \".\",\n",
+ " \"MALE\"\n",
+ " ]\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"monitoring_config\": {\n",
+ " \"evaluate_constraints\": \"Enabled\",\n",
+ " \"emit_metrics\": \"Enabled\",\n",
+ " \"datatype_check_threshold\": 1.0,\n",
+ " \"domain_content_threshold\": 1.0,\n",
+ " \"distribution_constraints\": {\n",
+ " \"perform_comparison\": \"Enabled\",\n",
+ " \"comparison_threshold\": 0.1,\n",
+ " \"comparison_method\": \"Robust\",\n",
+ " \"categorical_comparison_threshold\": 0.1,\n",
+ " \"categorical_drift_method\": \"LInfinity\"\n",
+ " }\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"try:\n",
" response = json.loads(\n",
- " S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/constraints.json\")\n",
+ " S3Downloader.read_file(f\"{DATA_QUALITY_LOCATION}/constraints.json\"),\n",
" )\n",
" print(json.dumps(response, indent=2))\n",
- "except Exception:\n",
+ "except Exception: # noqa: S110\n",
" pass"
]
},
@@ -7468,7 +7600,7 @@
},
{
"cell_type": "code",
- "execution_count": 137,
+ "execution_count": 328,
"id": "69e115c3",
"metadata": {},
"outputs": [],
@@ -7488,12 +7620,23 @@
},
{
"cell_type": "code",
- "execution_count": 139,
+ "execution_count": 329,
"id": "f93f2751",
"metadata": {
- "tags": []
+ "tags": [
+ "hide-output"
+ ]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n",
+ " warnings.warn(\n"
+ ]
+ }
+ ],
"source": [
"create_model_step = ModelStep(\n",
" name=\"create-model\",\n",
@@ -7523,7 +7666,7 @@
},
{
"cell_type": "code",
- "execution_count": 140,
+ "execution_count": 332,
"id": "2a530d12",
"metadata": {},
"outputs": [],
@@ -7538,7 +7681,7 @@
" accept=\"text/csv\",\n",
" assemble_with=\"Line\",\n",
" output_path=f\"{S3_LOCATION}/transform\",\n",
- " sagemaker_session=config[\"session\"],\n",
+ " sagemaker_session=pipeline_session,\n",
")"
]
},
@@ -7557,14 +7700,23 @@
},
{
"cell_type": "code",
- "execution_count": 141,
+ "execution_count": 333,
"id": "653c6ec8",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n",
+ " warnings.warn(\n"
+ ]
+ }
+ ],
"source": [
"from sagemaker.workflow.steps import TransformStep\n",
"\n",
@@ -7615,7 +7767,7 @@
},
{
"cell_type": "code",
- "execution_count": 142,
+ "execution_count": 334,
"id": "b50a9ed5",
"metadata": {
"tags": [
@@ -7677,7 +7829,7 @@
},
{
"cell_type": "code",
- "execution_count": 143,
+ "execution_count": 335,
"id": "3aecc022",
"metadata": {},
"outputs": [],
@@ -7735,7 +7887,7 @@
},
{
"cell_type": "code",
- "execution_count": 144,
+ "execution_count": 336,
"id": "20154a1a",
"metadata": {
"tags": [
@@ -7766,7 +7918,7 @@
},
{
"cell_type": "code",
- "execution_count": 145,
+ "execution_count": 338,
"id": "cd84e567",
"metadata": {},
"outputs": [],
@@ -7798,7 +7950,7 @@
},
{
"cell_type": "code",
- "execution_count": 146,
+ "execution_count": 339,
"id": "c244e206",
"metadata": {
"tags": [
@@ -7813,8 +7965,6 @@
"INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
"INFO:sagemaker.processing:Uploaded None to s3://mlschool/session17-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n",
"INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session17-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n",
- "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n",
- " warnings.warn(\n",
"WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n",
"INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
"INFO:sagemaker.processing:Uploaded None to s3://mlschool/session17-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n",
@@ -7825,16 +7975,16 @@
"data": {
"text/plain": [
"{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session17-pipeline',\n",
- " 'ResponseMetadata': {'RequestId': '84349ce8-4da7-46e2-9223-f1569f391299',\n",
+ " 'ResponseMetadata': {'RequestId': '60fbd90a-5eac-4df5-874c-f1cb865452ae',\n",
" 'HTTPStatusCode': 200,\n",
- " 'HTTPHeaders': {'x-amzn-requestid': '84349ce8-4da7-46e2-9223-f1569f391299',\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': '60fbd90a-5eac-4df5-874c-f1cb865452ae',\n",
" 'content-type': 'application/x-amz-json-1.1',\n",
" 'content-length': '86',\n",
- " 'date': 'Thu, 28 Mar 2024 18:27:25 GMT'},\n",
+ " 'date': 'Sat, 30 Mar 2024 16:40:57 GMT'},\n",
" 'RetryAttempts': 0}}"
]
},
- "execution_count": 146,
+ "execution_count": 339,
"metadata": {},
"output_type": "execute_result"
}
@@ -7869,17 +8019,53 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 341,
"id": "dc31aa28",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"version\": 0.0,\n",
+ " \"multiclass_classification_constraints\": {\n",
+ " \"accuracy\": {\n",
+ " \"threshold\": 1.0,\n",
+ " \"comparison_operator\": \"LessThanThreshold\"\n",
+ " },\n",
+ " \"weighted_recall\": {\n",
+ " \"threshold\": 1.0,\n",
+ " \"comparison_operator\": \"LessThanThreshold\"\n",
+ " },\n",
+ " \"weighted_precision\": {\n",
+ " \"threshold\": 1.0,\n",
+ " \"comparison_operator\": \"LessThanThreshold\"\n",
+ " },\n",
+ " \"weighted_f0_5\": {\n",
+ " \"threshold\": 1.0,\n",
+ " \"comparison_operator\": \"LessThanThreshold\"\n",
+ " },\n",
+ " \"weighted_f1\": {\n",
+ " \"threshold\": 1.0,\n",
+ " \"comparison_operator\": \"LessThanThreshold\"\n",
+ " },\n",
+ " \"weighted_f2\": {\n",
+ " \"threshold\": 1.0,\n",
+ " \"comparison_operator\": \"LessThanThreshold\"\n",
+ " }\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"try:\n",
" response = json.loads(\n",
- " S3Downloader.read_file(f\"{MODEL_QUALITY_LOCATION}/constraints.json\")\n",
+ " S3Downloader.read_file(f\"{MODEL_QUALITY_LOCATION}/constraints.json\"),\n",
" )\n",
" print(json.dumps(response, indent=2))\n",
- "except Exception:\n",
+ "except Exception: # noqa: S110\n",
" pass"
]
},
@@ -7890,7 +8076,7 @@
"source": [
"## Session 18 - Data Monitoring\n",
"\n",
- "In this section, we'll setup a monitoring job to monitor the quality of the data received by the endpoint. This schedule will run periodically and check the data that goes into the endpoint against the baseline we generated before.\n",
+ "This session creates a Monitoring Job to monitor the quality of the data received by the endpoint. This schedule will run periodically and check the data that goes into the endpoint against the baseline we generated before.\n",
"\n",
"Check [Amazon SageMaker Model Monitor](https://sagemaker.readthedocs.io/en/stable/amazon_sagemaker_model_monitoring.html) for an explanation of how to use SageMaker's Model Monitoring functionality. [Monitor models for data and model quality, bias, and explainability](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor.html) is a much more extensive guide to monitoring in Amazon SageMaker.\n"
]
@@ -7909,7 +8095,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 342,
"id": "57cc8903",
"metadata": {},
"outputs": [],
@@ -7918,6 +8104,7 @@
"\n",
"\n",
"def deploy_model():\n",
+ " \"\"\"Deploy the latest model registered in the Model Registry.\"\"\"\n",
" response = sagemaker_client.list_model_packages(\n",
" ModelPackageGroupName=PIPELINE_MODEL_PACKAGE_GROUP,\n",
" ModelApprovalStatus=\"Approved\",\n",
@@ -7968,14 +8155,31 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 344,
"id": "4966c9c2",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker:Creating model with name: pipeline-penguins-2024-03-30-17-25-24-537\n",
+ "INFO:sagemaker:Creating endpoint-config with name penguins-endpoint\n",
+ "INFO:sagemaker:Creating endpoint with name penguins-endpoint\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "-------!"
+ ]
+ }
+ ],
"source": [
"%%script false --no-raise-error\n",
"#| eval: false\n",
@@ -7995,32 +8199,34 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 371,
"id": "cea757af",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "An error occurred (ModelError) when calling the InvokeEndpoint operation: Received server error (500) from container-1 with message \"\n",
+ "500 Internal Server Error\n",
+ "Internal Server Error
\n",
+ "The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.
\n",
+ "\". See https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=/aws/sagemaker/Endpoints/penguins-endpoint in account 325223348818 for more information.\n"
+ ]
+ }
+ ],
"source": [
"from sagemaker.serializers import JSONSerializer\n",
"\n",
"\n",
"def generate_fake_traffic():\n",
- " # We don't want to send the target column to the endpoint.\n",
- " # We also don't want to send any missing fields.\n",
- " data = penguins.drop([\"species\"], axis=1)\n",
- " data = data.dropna()\n",
- "\n",
- " predictor = Predictor(\n",
- " endpoint_name=ENDPOINT,\n",
- " serializer=CSVSerializer(),\n",
- " sagemaker_session=sagemaker_session,\n",
- " )\n",
- "\n",
- " for index, row in data.iterrows():\n",
- " try:\n",
+ " \"\"\"Generate fake traffic to the endpoint.\"\"\"\n",
+ " try:\n",
+ " for index, row in data.iterrows():\n",
" payload = \",\".join([str(x) for x in row.to_list()])\n",
" predictor.predict(\n",
" payload,\n",
@@ -8029,9 +8235,8 @@
" # it later with a corresponding ground-truth label.\n",
" inference_id=str(index),\n",
" )\n",
- " except Exception as e:\n",
- " print(e)\n",
- " break\n",
+ " except Exception as e:\n",
+ " print(e)\n",
"\n",
"\n",
"generate_fake_traffic()"
@@ -8049,10 +8254,40 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 349,
"id": "caee992c",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "File: s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2024/03/30/17/32-02-242-191b135d-085a-484d-a119-45b26c51554c.jsonl\n",
+ "{\n",
+ " \"captureData\": {\n",
+ " \"endpointInput\": {\n",
+ " \"observedContentType\": \"text/csv\",\n",
+ " \"mode\": \"INPUT\",\n",
+ " \"data\": \"Torgersen,39.1,18.7,181.0,3750.0,MALE\",\n",
+ " \"encoding\": \"CSV\"\n",
+ " },\n",
+ " \"endpointOutput\": {\n",
+ " \"observedContentType\": \"text/csv; charset=utf-8\",\n",
+ " \"mode\": \"OUTPUT\",\n",
+ " \"data\": \"Adelie,0.964408875\\n\",\n",
+ " \"encoding\": \"CSV\"\n",
+ " }\n",
+ " },\n",
+ " \"eventMetadata\": {\n",
+ " \"eventId\": \"3211434d-0db6-4ee2-8848-95ce11f6d5e6\",\n",
+ " \"inferenceId\": \"0\",\n",
+ " \"inferenceTime\": \"2024-03-30T17:32:02Z\"\n",
+ " },\n",
+ " \"eventVersion\": \"0\"\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"files = S3Downloader.list(DATA_CAPTURE_DESTINATION)\n",
"if len(files):\n",
@@ -8077,7 +8312,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 350,
"id": "7b48cd49",
"metadata": {},
"outputs": [],
@@ -8097,25 +8332,33 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 351,
"id": "0114a817",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
- "source": [
- "%%writefile {CODE_FOLDER}/monitoring/{DATA_QUALITY_PREPROCESSOR}\n",
- "#| filename: data_quality_preprocessor.py\n",
- "#| code-line-numbers: true\n",
- "\n",
- "import json\n",
- "\n",
- "def preprocess_handler(inference_record, logger):\n",
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Writing code/monitoring/data_quality_preprocessor.py\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%writefile {CODE_FOLDER}/monitoring/{DATA_QUALITY_PREPROCESSOR}\n",
+ "# | filename: data_quality_preprocessor.py\n",
+ "# | code-line-numbers: true\n",
+ "\n",
+ "import json\n",
+ "\n",
+ "\n",
+ "def preprocess_handler(inference_record, logger):\n",
" input_data = inference_record.endpoint_input.data\n",
- " return { str(i).zfill(2) : d for i, d in enumerate(input_data.split(\",\")) }\n",
- " "
+ " return {str(i).zfill(2): d for i, d in enumerate(input_data.split(\",\"))}"
]
},
{
@@ -8130,23 +8373,40 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 374,
"id": "21240214",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:botocore.credentials:Found credentials in shared credentials file: ~/.aws/credentials\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "'s3://mlschool/penguins/monitoring/data_quality_preprocessor.py'"
+ ]
+ },
+ "execution_count": 374,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"bucket = boto3.Session().resource(\"s3\").Bucket(config[\"session\"].default_bucket())\n",
- "prefix = \"penguins/monitoring\"\n",
- "bucket.Object(os.path.join(prefix, DATA_QUALITY_PREPROCESSOR)).upload_file(\n",
- " (CODE_FOLDER / \"monitoring\" / DATA_QUALITY_PREPROCESSOR).as_posix()\n",
+ "prefix = Path(\"penguins/monitoring\")\n",
+ "bucket.Object((prefix / DATA_QUALITY_PREPROCESSOR).as_posix()).upload_file(\n",
+ " (CODE_FOLDER / \"monitoring\" / DATA_QUALITY_PREPROCESSOR).as_posix(),\n",
")\n",
- "data_quality_preprocessor = (\n",
- " f\"s3://{os.path.join(bucket.name, prefix, DATA_QUALITY_PREPROCESSOR)}\"\n",
- ")"
+ "data_quality_preprocessor = f\"s3://{(bucket.name / prefix / DATA_QUALITY_PREPROCESSOR)}\"\n",
+ "data_quality_preprocessor"
]
},
{
@@ -8161,10 +8421,19 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 353,
"id": "b9df3d16",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n",
+ "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n"
+ ]
+ }
+ ],
"source": [
"from sagemaker.model_monitor import DefaultModelMonitor\n",
"\n",
@@ -8201,17 +8470,26 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 355,
"id": "c2b3fe69",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker.model_monitor.model_monitoring:Creating Monitoring Schedule with name: penguins-data-monitoring-schedule\n",
+ "INFO:sagemaker:Starting Monitoring Schedule with name: penguins-data-monitoring-schedule\n"
+ ]
+ }
+ ],
"source": [
"%%script false --no-raise-error\n",
- "#| eval: false\n",
+ "# | eval: false\n",
"\n",
"import time\n",
"from sagemaker.model_monitor import CronExpressionGenerator\n",
@@ -8227,7 +8505,7 @@
" enable_cloudwatch_metrics=True,\n",
")\n",
"\n",
- "# Let's give SageMaker some time to process the \n",
+ "# Let's give SageMaker some time to process the\n",
"# monitoring job before we start it.\n",
"time.sleep(10)\n",
"data_monitor.start_monitoring_schedule()"
@@ -8245,15 +8523,145 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 356,
"id": "5cedff08",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Processing Job Status: Completed\n",
+ "Exit Message: \"Completed: Job completed successfully with no violations.\"\n",
+ "Last Modified Time: 2024-03-30 14:15:49.146000-04:00\n",
+ "\n",
+ "Execution:\n",
+ "{\n",
+ " \"ProcessingInputs\": [\n",
+ " {\n",
+ " \"InputName\": \"baseline\",\n",
+ " \"AppManaged\": false,\n",
+ " \"S3Input\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-quality/statistics.json\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/baseline/stats\",\n",
+ " \"S3DataType\": \"S3Prefix\",\n",
+ " \"S3InputMode\": \"File\",\n",
+ " \"S3DataDistributionType\": \"FullyReplicated\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"InputName\": \"constraints\",\n",
+ " \"AppManaged\": false,\n",
+ " \"S3Input\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-quality/constraints.json\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/baseline/constraints\",\n",
+ " \"S3DataType\": \"S3Prefix\",\n",
+ " \"S3InputMode\": \"File\",\n",
+ " \"S3DataDistributionType\": \"FullyReplicated\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"InputName\": \"pre_processor_script\",\n",
+ " \"AppManaged\": false,\n",
+ " \"S3Input\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data_quality_preprocessor.py\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/code/preprocessing\",\n",
+ " \"S3DataType\": \"S3Prefix\",\n",
+ " \"S3InputMode\": \"File\",\n",
+ " \"S3DataDistributionType\": \"FullyReplicated\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"InputName\": \"endpoint_input_1\",\n",
+ " \"AppManaged\": false,\n",
+ " \"S3Input\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2024/03/30/17\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/input/endpoint/penguins-endpoint/AllTraffic/2024/03/30/17\",\n",
+ " \"S3DataType\": \"S3Prefix\",\n",
+ " \"S3InputMode\": \"File\",\n",
+ " \"S3DataDistributionType\": \"FullyReplicated\",\n",
+ " \"S3CompressionType\": \"None\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"ProcessingOutputConfig\": {\n",
+ " \"Outputs\": [\n",
+ " {\n",
+ " \"OutputName\": \"result\",\n",
+ " \"S3Output\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/data-quality/penguins-endpoint/penguins-data-monitoring-schedule/2024/03/30/18\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/output\",\n",
+ " \"S3UploadMode\": \"Continuous\"\n",
+ " },\n",
+ " \"AppManaged\": false\n",
+ " }\n",
+ " ]\n",
+ " },\n",
+ " \"ProcessingJobName\": \"model-monitoring-202403301800-17aa1fca873fac795ffba24a\",\n",
+ " \"ProcessingResources\": {\n",
+ " \"ClusterConfig\": {\n",
+ " \"InstanceCount\": 1,\n",
+ " \"InstanceType\": \"ml.m5.xlarge\",\n",
+ " \"VolumeSizeInGB\": 20\n",
+ " }\n",
+ " },\n",
+ " \"StoppingCondition\": {\n",
+ " \"MaxRuntimeInSeconds\": 1800\n",
+ " },\n",
+ " \"AppSpecification\": {\n",
+ " \"ImageUri\": \"156813124566.dkr.ecr.us-east-1.amazonaws.com/sagemaker-model-monitor-analyzer\"\n",
+ " },\n",
+ " \"Environment\": {\n",
+ " \"baseline_constraints\": \"/opt/ml/processing/baseline/constraints/constraints.json\",\n",
+ " \"baseline_statistics\": \"/opt/ml/processing/baseline/stats/statistics.json\",\n",
+ " \"dataset_format\": \"{\\\"sagemakerCaptureJson\\\":{\\\"captureIndexNames\\\":[\\\"endpointInput\\\",\\\"endpointOutput\\\"]}}\",\n",
+ " \"dataset_source\": \"/opt/ml/processing/input/endpoint\",\n",
+ " \"end_time\": \"2024-03-30T18:00:00Z\",\n",
+ " \"metric_time\": \"2024-03-30T17:00:00Z\",\n",
+ " \"monitoring_input_type\": \"ENDPOINT_INPUT\",\n",
+ " \"output_path\": \"/opt/ml/processing/output\",\n",
+ " \"publish_cloudwatch_metrics\": \"Enabled\",\n",
+ " \"record_preprocessor_script\": \"/opt/ml/processing/code/preprocessing/data_quality_preprocessor.py\",\n",
+ " \"sagemaker_endpoint_name\": \"penguins-endpoint\",\n",
+ " \"sagemaker_monitoring_schedule_name\": \"penguins-data-monitoring-schedule\",\n",
+ " \"start_time\": \"2024-03-30T17:00:00Z\"\n",
+ " },\n",
+ " \"RoleArn\": \"arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501\",\n",
+ " \"ProcessingJobArn\": \"arn:aws:sagemaker:us-east-1:325223348818:processing-job/model-monitoring-202403301800-17aa1fca873fac795ffba24a\",\n",
+ " \"ProcessingJobStatus\": \"Completed\",\n",
+ " \"ExitMessage\": \"Completed: Job completed successfully with no violations.\",\n",
+ " \"ProcessingEndTime\": \"2024-03-30 14:15:48.732000-04:00\",\n",
+ " \"ProcessingStartTime\": \"2024-03-30 14:14:14.760000-04:00\",\n",
+ " \"LastModifiedTime\": \"2024-03-30 14:15:49.146000-04:00\",\n",
+ " \"CreationTime\": \"2024-03-30 14:09:54.896000-04:00\",\n",
+ " \"MonitoringScheduleArn\": \"arn:aws:sagemaker:us-east-1:325223348818:monitoring-schedule/penguins-data-monitoring-schedule\",\n",
+ " \"ResponseMetadata\": {\n",
+ " \"RequestId\": \"4e348652-7dff-4c40-96fb-b944aa6ed83b\",\n",
+ " \"HTTPStatusCode\": 200,\n",
+ " \"HTTPHeaders\": {\n",
+ " \"x-amzn-requestid\": \"4e348652-7dff-4c40-96fb-b944aa6ed83b\",\n",
+ " \"content-type\": \"application/x-amz-json-1.1\",\n",
+ " \"content-length\": \"3233\",\n",
+ " \"date\": \"Sat, 30 Mar 2024 18:34:16 GMT\"\n",
+ " },\n",
+ " \"RetryAttempts\": 0\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "Violations:\n",
+ "{\n",
+ " \"violations\": []\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"def check_execution(monitoring_schedule):\n",
- " \"\"\"\n",
- " This function checks the execution of a monitoring job and\n",
- " prints out the list of violations if the job completed.\n",
+ " \"\"\"Check the execution of the Monitoring Job.\n",
+ "\n",
+ " This function checks the execution of the Monitoring\n",
+ " Job and prints out the list of violations if the job\n",
+ " completed.\n",
" \"\"\"\n",
" try:\n",
" executions = monitoring_schedule.list_executions()\n",
@@ -8265,7 +8673,8 @@
" if execution[\"ProcessingJobStatus\"] == \"Completed\":\n",
" print(f\"Exit Message: \\\"{execution['ExitMessage']}\\\"\")\n",
" print(\n",
- " f\"Last Modified Time: {execution['LastModifiedTime']}\", end=\"\\n\\n\"\n",
+ " f\"Last Modified Time: {execution['LastModifiedTime']}\",\n",
+ " end=\"\\n\\n\",\n",
" )\n",
" print(\"Execution:\")\n",
" print(json.dumps(execution, default=str, indent=2), end=\"\\n\\n\")\n",
@@ -8274,7 +8683,7 @@
" monitoring_schedule.latest_monitoring_constraint_violations()\n",
" )\n",
" response = json.loads(\n",
- " S3Downloader.read_file(latest_monitoring_violations.file_s3_uri)\n",
+ " S3Downloader.read_file(latest_monitoring_violations.file_s3_uri),\n",
" )\n",
" print(\"Violations:\")\n",
" print(json.dumps(response, indent=2))\n",
@@ -8292,7 +8701,7 @@
"source": [
"## Session 19 - Model Monitoring\n",
"\n",
- "In this section, we'll setup a monitoring job to monitor the quality of the model outputs. This schedule will run periodically and check the data that goes into the endpoint against the baseline we generated before.\n",
+ "This session creates a Monitoring Job to monitor the quality of the model outputs. This schedule will run periodically and check the data that goes into the endpoint against the baseline we generated before.\n",
"\n",
"Check [Amazon SageMaker Model Monitor](https://sagemaker.readthedocs.io/en/stable/amazon_sagemaker_model_monitoring.html) for an explanation of how to use SageMaker's Model Monitoring functionality. [Monitor models for data and model quality, bias, and explainability](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor.html) is a much more extensive guide to monitoring in Amazon SageMaker.\n"
]
@@ -8309,7 +8718,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 357,
"id": "5ddca53e",
"metadata": {},
"outputs": [],
@@ -8343,7 +8752,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 358,
"id": "c2089ba4",
"metadata": {
"tags": [
@@ -8370,7 +8779,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 359,
"id": "c4545041",
"metadata": {
"tags": [
@@ -8394,10 +8803,40 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 361,
"id": "49bc662a",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "File: s3://mlschool/penguins/monitoring/data-capture/penguins-endpoint/AllTraffic/2024/03/30/18/40-45-068-0f144be9-ac73-4c4e-a0c7-82b1ba7db88b.jsonl\n",
+ "{\n",
+ " \"captureData\": {\n",
+ " \"endpointInput\": {\n",
+ " \"observedContentType\": \"text/csv\",\n",
+ " \"mode\": \"INPUT\",\n",
+ " \"data\": \"Torgersen,39.1,18.7,181.0,3750.0,MALE\",\n",
+ " \"encoding\": \"CSV\"\n",
+ " },\n",
+ " \"endpointOutput\": {\n",
+ " \"observedContentType\": \"text/csv; charset=utf-8\",\n",
+ " \"mode\": \"OUTPUT\",\n",
+ " \"data\": \"Adelie,0.964408875\\n\",\n",
+ " \"encoding\": \"CSV\"\n",
+ " }\n",
+ " },\n",
+ " \"eventMetadata\": {\n",
+ " \"eventId\": \"08a239af-c98c-4984-b9bf-4ea049d88617\",\n",
+ " \"inferenceId\": \"0\",\n",
+ " \"inferenceTime\": \"2024-03-30T18:40:45Z\"\n",
+ " },\n",
+ " \"eventVersion\": \"0\"\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"files = S3Downloader.list(DATA_CAPTURE_DESTINATION)\n",
"if len(files):\n",
@@ -8418,17 +8857,29 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 366,
"id": "a4a32d8c",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'s3://mlschool/penguins/monitoring/groundtruth/2024/03/30/18/5226.jsonl'"
+ ]
+ },
+ "execution_count": 366,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"import random\n",
- "from datetime import datetime\n",
+ "from datetime import datetime, timezone\n",
+ "\n",
"from sagemaker.s3 import S3Uploader\n",
"\n",
"records = []\n",
@@ -8450,12 +8901,12 @@
" \"eventId\": str(inference_id),\n",
" },\n",
" \"eventVersion\": \"0\",\n",
- " }\n",
- " )\n",
+ " },\n",
+ " ),\n",
" )\n",
"\n",
"groundtruth_payload = \"\\n\".join(records)\n",
- "upload_time = datetime.now(datetime.UTC)\n",
+ "upload_time = datetime.now(tz=timezone.utc)\n",
"uri = f\"{GROUND_TRUTH_LOCATION}/{upload_time:%Y/%m/%d/%H/%M%S}.jsonl\"\n",
"S3Uploader.upload_string_as_file_body(groundtruth_payload, uri)"
]
@@ -8474,14 +8925,23 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 367,
"id": "54762759",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n",
+ "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n"
+ ]
+ }
+ ],
"source": [
"from sagemaker.model_monitor import ModelQualityMonitor\n",
"\n",
@@ -8518,36 +8978,42 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 368,
"id": "9665c61a",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker.model_monitor.model_monitoring:Creating Monitoring Schedule with name: penguins-model-monitoring-schedule\n",
+ "INFO:sagemaker:Starting Monitoring Schedule with name: penguins-model-monitoring-schedule\n"
+ ]
+ }
+ ],
"source": [
"%%script false --no-raise-error\n",
- "#| eval: false\n",
+ "# | eval: false\n",
"\n",
"import time\n",
- "from sagemaker.model_monitor import EndpointInput\n",
"\n",
+ "from sagemaker.model_monitor import CronExpressionGenerator, EndpointInput\n",
"\n",
"model_monitor.create_monitoring_schedule(\n",
" monitor_schedule_name=\"penguins-model-monitoring-schedule\",\n",
- " \n",
- " endpoint_input = EndpointInput(\n",
+ " endpoint_input=EndpointInput(\n",
" endpoint_name=ENDPOINT,\n",
- " \n",
" # The first attribute is the prediction made\n",
- " # by the model. For example, here is a \n",
+ " # by the model. For example, here is a\n",
" # potential output from the model:\n",
" # [Adelie,0.977324724\\n]\n",
" inference_attribute=\"0\",\n",
" destination=\"/opt/ml/processing/input_data\",\n",
" ),\n",
- " \n",
" problem_type=\"MulticlassClassification\",\n",
" ground_truth_input=GROUND_TRUTH_LOCATION,\n",
" constraints=f\"{MODEL_QUALITY_LOCATION}/constraints.json\",\n",
@@ -8556,7 +9022,7 @@
" enable_cloudwatch_metrics=True,\n",
")\n",
"\n",
- "# Let's give SageMaker some time to process the \n",
+ "# Let's give SageMaker some time to process the\n",
"# monitoring job before we start it.\n",
"time.sleep(10)\n",
"model_monitor.start_monitoring_schedule()"
@@ -8574,10 +9040,143 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 380,
"id": "34ba56d9",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Processing Job Status: Completed\n",
+ "Exit Message: \"CompletedWithViolations: Job completed successfully with 5 violations.\"\n",
+ "Last Modified Time: 2024-03-30 15:18:36.431000-04:00\n",
+ "\n",
+ "Execution:\n",
+ "{\n",
+ " \"ProcessingInputs\": [\n",
+ " {\n",
+ " \"InputName\": \"constraints\",\n",
+ " \"AppManaged\": false,\n",
+ " \"S3Input\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/model-quality/constraints.json\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/baseline/constraints\",\n",
+ " \"S3DataType\": \"S3Prefix\",\n",
+ " \"S3InputMode\": \"File\",\n",
+ " \"S3DataDistributionType\": \"FullyReplicated\"\n",
+ " }\n",
+ " },\n",
+ " {\n",
+ " \"InputName\": \"endpoint_input_1\",\n",
+ " \"AppManaged\": false,\n",
+ " \"S3Input\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/model-quality/merge/penguins-endpoint/AllTraffic/2024/03/30/18\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/input_data/penguins-endpoint/AllTraffic/2024/03/30/18\",\n",
+ " \"S3DataType\": \"S3Prefix\",\n",
+ " \"S3InputMode\": \"File\",\n",
+ " \"S3DataDistributionType\": \"FullyReplicated\",\n",
+ " \"S3CompressionType\": \"None\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"ProcessingOutputConfig\": {\n",
+ " \"Outputs\": [\n",
+ " {\n",
+ " \"OutputName\": \"result\",\n",
+ " \"S3Output\": {\n",
+ " \"S3Uri\": \"s3://mlschool/penguins/monitoring/model-quality/penguins-endpoint/penguins-model-monitoring-schedule/2024/03/30/19\",\n",
+ " \"LocalPath\": \"/opt/ml/processing/output\",\n",
+ " \"S3UploadMode\": \"Continuous\"\n",
+ " },\n",
+ " \"AppManaged\": false\n",
+ " }\n",
+ " ]\n",
+ " },\n",
+ " \"ProcessingJobName\": \"model-quality-monitoring-202403301900-896e874cc3a809cdf37d6cc2\",\n",
+ " \"ProcessingResources\": {\n",
+ " \"ClusterConfig\": {\n",
+ " \"InstanceCount\": 1,\n",
+ " \"InstanceType\": \"ml.m5.xlarge\",\n",
+ " \"VolumeSizeInGB\": 20\n",
+ " }\n",
+ " },\n",
+ " \"StoppingCondition\": {\n",
+ " \"MaxRuntimeInSeconds\": 1800\n",
+ " },\n",
+ " \"AppSpecification\": {\n",
+ " \"ImageUri\": \"156813124566.dkr.ecr.us-east-1.amazonaws.com/sagemaker-model-monitor-analyzer\"\n",
+ " },\n",
+ " \"Environment\": {\n",
+ " \"analysis_type\": \"MODEL_QUALITY\",\n",
+ " \"baseline_constraints\": \"/opt/ml/processing/baseline/constraints/constraints.json\",\n",
+ " \"dataset_format\": \"{\\\"sagemakerMergeJson\\\":{\\\"captureIndexNames\\\":[\\\"endpointOutput\\\"],\\\"originalDatasetFormat\\\":null}}\",\n",
+ " \"dataset_source\": \"/opt/ml/processing/input_data\",\n",
+ " \"end_time\": \"2024-03-30T19:00:00Z\",\n",
+ " \"inference_attribute\": \"0\",\n",
+ " \"metric_time\": \"2024-03-30T18:00:00Z\",\n",
+ " \"monitoring_input_type\": \"ENDPOINT_INPUT\",\n",
+ " \"output_path\": \"/opt/ml/processing/output\",\n",
+ " \"problem_type\": \"MulticlassClassification\",\n",
+ " \"publish_cloudwatch_metrics\": \"Enabled\",\n",
+ " \"sagemaker_endpoint_name\": \"penguins-endpoint\",\n",
+ " \"sagemaker_monitoring_schedule_name\": \"penguins-model-monitoring-schedule\",\n",
+ " \"start_time\": \"2024-03-30T18:00:00Z\"\n",
+ " },\n",
+ " \"RoleArn\": \"arn:aws:iam::325223348818:role/service-role/AmazonSageMaker-ExecutionRole-20230312T160501\",\n",
+ " \"ProcessingJobArn\": \"arn:aws:sagemaker:us-east-1:325223348818:processing-job/model-quality-monitoring-202403301900-896e874cc3a809cdf37d6cc2\",\n",
+ " \"ProcessingJobStatus\": \"Completed\",\n",
+ " \"ExitMessage\": \"CompletedWithViolations: Job completed successfully with 5 violations.\",\n",
+ " \"ProcessingEndTime\": \"2024-03-30 15:18:35.908000-04:00\",\n",
+ " \"ProcessingStartTime\": \"2024-03-30 15:16:52.922000-04:00\",\n",
+ " \"LastModifiedTime\": \"2024-03-30 15:18:36.431000-04:00\",\n",
+ " \"CreationTime\": \"2024-03-30 15:12:22.569000-04:00\",\n",
+ " \"MonitoringScheduleArn\": \"arn:aws:sagemaker:us-east-1:325223348818:monitoring-schedule/penguins-model-monitoring-schedule\",\n",
+ " \"ResponseMetadata\": {\n",
+ " \"RequestId\": \"85abb737-543a-4c92-928b-4a293c599f18\",\n",
+ " \"HTTPStatusCode\": 200,\n",
+ " \"HTTPHeaders\": {\n",
+ " \"x-amzn-requestid\": \"85abb737-543a-4c92-928b-4a293c599f18\",\n",
+ " \"content-type\": \"application/x-amz-json-1.1\",\n",
+ " \"content-length\": \"2660\",\n",
+ " \"date\": \"Sat, 30 Mar 2024 19:33:23 GMT\"\n",
+ " },\n",
+ " \"RetryAttempts\": 0\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "Violations:\n",
+ "{\n",
+ " \"violations\": [\n",
+ " {\n",
+ " \"constraint_check_type\": \"LessThanThreshold\",\n",
+ " \"description\": \"Metric weightedF2 with 0.3518870011147463 +/- 0.006730551075118943 was LessThanThreshold '1.0'\",\n",
+ " \"metric_name\": \"weightedF2\"\n",
+ " },\n",
+ " {\n",
+ " \"constraint_check_type\": \"LessThanThreshold\",\n",
+ " \"description\": \"Metric accuracy with 0.35755813953488375 +/- 0.007228798319401767 was LessThanThreshold '1.0'\",\n",
+ " \"metric_name\": \"accuracy\"\n",
+ " },\n",
+ " {\n",
+ " \"constraint_check_type\": \"LessThanThreshold\",\n",
+ " \"description\": \"Metric weightedRecall with 0.35755813953488375 +/- 0.007228798319401765 was LessThanThreshold '1.0'\",\n",
+ " \"metric_name\": \"weightedRecall\"\n",
+ " },\n",
+ " {\n",
+ " \"constraint_check_type\": \"LessThanThreshold\",\n",
+ " \"description\": \"Metric weightedPrecision with 0.35624627310673823 +/- 0.008910206698382583 was LessThanThreshold '1.0'\",\n",
+ " \"metric_name\": \"weightedPrecision\"\n",
+ " },\n",
+ " {\n",
+ " \"constraint_check_type\": \"LessThanThreshold\",\n",
+ " \"description\": \"Metric weightedF1 with 0.34769539574160063 +/- 0.006655863903356062 was LessThanThreshold '1.0'\",\n",
+ " \"metric_name\": \"weightedF1\"\n",
+ " }\n",
+ " ]\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"check_execution(model_monitor)"
]
@@ -8589,111 +9188,232 @@
"source": [
"## Session 20 - Shadow Deployments\n",
"\n",
- "Shadow tests: https://docs.aws.amazon.com/sagemaker/latest/dg/shadow-tests-view-monitor-edit.html\n"
+ "This session configures an endpoint running a production and a shadow variant. Check [Safely validate models in production](https://docs.aws.amazon.com/sagemaker/latest/dg/model-validation.html) for more information.\n",
+ "\n",
+ " ![\"Shadow](\"images/shadow-deployment.png\")
\n"
]
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "02437a4a",
+ "cell_type": "markdown",
+ "id": "19314442",
"metadata": {},
- "outputs": [],
"source": [
- "def get_model_package(model_package_group_name):\n",
- " response = sagemaker_client.list_model_packages(\n",
- " ModelPackageGroupName=model_package_group_name,\n",
- " ModelApprovalStatus=\"Approved\",\n",
- " SortBy=\"CreationTime\",\n",
- " MaxResults=1,\n",
- " )\n",
+ "### Step 1 - Getting The Latest Models\n",
"\n",
- " return (\n",
- " response[\"ModelPackageSummaryList\"][0][\"ModelPackageArn\"]\n",
- " if response[\"ModelPackageSummaryList\"]\n",
- " else None\n",
- " )"
+ "We want to deploy the two latest approved models from the Model Registry to the same endpoint. The latest version of the model will act as the Shadow variant, and the previous version will act as the Production variant.\n"
]
},
{
"cell_type": "code",
- "execution_count": null,
- "id": "1df9cae4",
+ "execution_count": 393,
+ "id": "d322d415",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Production package: arn:aws:sagemaker:us-east-1:325223348818:model-package/basic-penguins/5\n",
+ "Shadow package: arn:aws:sagemaker:us-east-1:325223348818:model-package/basic-penguins/6\n"
+ ]
+ }
+ ],
+ "source": [
+ "response = sagemaker_client.list_model_packages(\n",
+ " ModelPackageGroupName=BASIC_MODEL_PACKAGE_GROUP,\n",
+ " ModelApprovalStatus=\"Approved\",\n",
+ " SortBy=\"CreationTime\",\n",
+ " MaxResults=2,\n",
+ ")\n",
+ "\n",
+ "if response[\"ModelPackageSummaryList\"]:\n",
+ " production_package = response[\"ModelPackageSummaryList\"][1][\"ModelPackageArn\"]\n",
+ " shadow_package = response[\"ModelPackageSummaryList\"][0][\"ModelPackageArn\"]\n",
+ "else:\n",
+ " production_package = None\n",
+ " shadow_package = None\n",
+ "\n",
+ "print(f\"Production package: {production_package}\")\n",
+ "print(f\"Shadow package: {shadow_package}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "015b7995",
"metadata": {},
- "outputs": [],
"source": [
- "package1 = get_model_package(BASIC_MODEL_PACKAGE_GROUP)\n",
- "package2 = get_model_package(CUSTOM_MODEL_PACKAGE_GROUP)\n",
+ "### Step 2 - Creating the Models\n",
"\n",
- "print(package1)\n",
- "print(package2)"
+ "We want to deploy the two packages to a new endpoint. We'll use the boto3 API to deploy these models.\n",
+ "\n",
+ "Let's start by creating the SageMaker Models.\n"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 398,
"id": "6c8aea09",
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
+ "# We'll use a different name for this endpoint.\n",
"SHADOW_DEPLOYMENT_ENDPOINT = \"shadow-penguins-endpoint\"\n",
"\n",
+ "# The timestamp will help us create unique name for the\n",
+ "# name of the models.\n",
"timestamp = time.strftime(\"%m%d%H%M%S\", time.localtime())"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "9b29826d",
+ "metadata": {},
+ "source": [
+ "Let's now create the Production model.\n"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 399,
"id": "dd20b354",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'ModelArn': 'arn:aws:sagemaker:us-east-1:325223348818:model/shadow-penguins-endpoint-production-0330155951',\n",
+ " 'ResponseMetadata': {'RequestId': '9b117d81-8cb6-4477-bc8b-ed43dc3089ff',\n",
+ " 'HTTPStatusCode': 200,\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': '9b117d81-8cb6-4477-bc8b-ed43dc3089ff',\n",
+ " 'content-type': 'application/x-amz-json-1.1',\n",
+ " 'content-length': '108',\n",
+ " 'date': 'Sat, 30 Mar 2024 19:59:52 GMT'},\n",
+ " 'RetryAttempts': 0}}"
+ ]
+ },
+ "execution_count": 399,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "model1_name = f\"{SHADOW_DEPLOYMENT_ENDPOINT}-model1-{timestamp}\"\n",
+ "production_model_name = f\"{SHADOW_DEPLOYMENT_ENDPOINT}-production-{timestamp}\"\n",
"\n",
"sagemaker_client.create_model(\n",
- " ModelName=model1_name,\n",
+ " ModelName=production_model_name,\n",
" ExecutionRoleArn=role,\n",
- " Containers=[{\"ModelPackageName\": package1}],\n",
+ " Containers=[{\"ModelPackageName\": production_package}],\n",
")"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "af41dac0",
+ "metadata": {},
+ "source": [
+ "And now we can create the second model.\n"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 400,
"id": "d2024f35",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'ModelArn': 'arn:aws:sagemaker:us-east-1:325223348818:model/shadow-penguins-endpoint-shadow-0330155951',\n",
+ " 'ResponseMetadata': {'RequestId': 'b64e8acb-80f7-45c5-9562-adb49a729f68',\n",
+ " 'HTTPStatusCode': 200,\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': 'b64e8acb-80f7-45c5-9562-adb49a729f68',\n",
+ " 'content-type': 'application/x-amz-json-1.1',\n",
+ " 'content-length': '104',\n",
+ " 'date': 'Sat, 30 Mar 2024 19:59:53 GMT'},\n",
+ " 'RetryAttempts': 0}}"
+ ]
+ },
+ "execution_count": 400,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "model2_name = f\"{SHADOW_DEPLOYMENT_ENDPOINT}-model2-{timestamp}\"\n",
+ "shadow_model_name = f\"{SHADOW_DEPLOYMENT_ENDPOINT}-shadow-{timestamp}\"\n",
"\n",
"sagemaker_client.create_model(\n",
- " ModelName=model2_name,\n",
+ " ModelName=shadow_model_name,\n",
" ExecutionRoleArn=role,\n",
- " Containers=[{\"ModelPackageName\": package2}],\n",
+ " Containers=[{\"ModelPackageName\": shadow_package}],\n",
")"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "f1215beb",
+ "metadata": {},
+ "source": [
+ "### Step 3 - Creating the Endpoint Configuration\n",
+ "\n",
+ "We can now create the Endpoint Configuration using the two models\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b15d35d1",
+ "metadata": {},
+ "source": [
+ "Let's define the location where SageMaker will output the information captured by the Shadow variant.\n"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 437,
"id": "85769b3e",
"metadata": {},
"outputs": [],
"source": [
- "SHADOW_CORE_DUMP_DESTINATION = f\"{S3_LOCATION}/endpoint/\""
+ "SHADOW_DATA_DESTINATION = f\"{S3_LOCATION}/endpoint/\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c97fe3bf",
+ "metadata": {},
+ "source": [
+ "We can create the Endpoint Configuration now.\n"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 439,
"id": "969df4f8",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'EndpointConfigArn': 'arn:aws:sagemaker:us-east-1:325223348818:endpoint-config/shadow-penguins-endpoint-config-0330155951',\n",
+ " 'ResponseMetadata': {'RequestId': '7b3f1a13-9ecf-4228-994a-aef801fd31a6',\n",
+ " 'HTTPStatusCode': 200,\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': '7b3f1a13-9ecf-4228-994a-aef801fd31a6',\n",
+ " 'content-type': 'application/x-amz-json-1.1',\n",
+ " 'content-length': '123',\n",
+ " 'date': 'Sat, 30 Mar 2024 21:25:47 GMT'},\n",
+ " 'RetryAttempts': 0}}"
+ ]
+ },
+ "execution_count": 439,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"endpoint_config_name = f\"{SHADOW_DEPLOYMENT_ENDPOINT}-config-{timestamp}\"\n",
"\n",
@@ -8701,32 +9421,72 @@
" EndpointConfigName=endpoint_config_name,\n",
" ProductionVariants=[\n",
" {\n",
- " \"ModelName\": model1_name,\n",
+ " \"ModelName\": production_model_name,\n",
" \"InstanceType\": \"ml.m5.xlarge\",\n",
" \"InitialVariantWeight\": 1,\n",
" \"InitialInstanceCount\": 1,\n",
- " \"VariantName\": \"AllTraffic\",\n",
- " }\n",
+ " \"VariantName\": \"ProductionTraffic\",\n",
+ " },\n",
" ],\n",
" ShadowProductionVariants=[\n",
" {\n",
- " \"ModelName\": model2_name,\n",
+ " \"ModelName\": shadow_model_name,\n",
" \"InstanceType\": \"ml.m5.xlarge\",\n",
" \"InitialVariantWeight\": 1,\n",
" \"InitialInstanceCount\": 1,\n",
" \"VariantName\": \"ShadowTraffic\",\n",
- " \"CoreDumpConfig\": {\"DestinationS3Uri\": SHADOW_CORE_DUMP_DESTINATION},\n",
- " }\n",
+ " },\n",
" ],\n",
+ " DataCaptureConfig={\n",
+ " \"EnableCapture\": True,\n",
+ " \"InitialSamplingPercentage\": 100,\n",
+ " \"DestinationS3Uri\": SHADOW_DATA_DESTINATION,\n",
+ " \"CaptureOptions\": [\n",
+ " {\"CaptureMode\": \"Input\"},\n",
+ " {\"CaptureMode\": \"Output\"},\n",
+ " ],\n",
+ " \"CaptureContentTypeHeader\": {\n",
+ " \"CsvContentTypes\": [\"text/csv\", \"application/octect-stream\"],\n",
+ " \"JsonContentTypes\": [\"application/json\", \"application/octect-stream\"],\n",
+ " },\n",
+ " },\n",
")"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "3d8baad3",
+ "metadata": {},
+ "source": [
+ "### Step 4 - Creating the Endpoint\n",
+ "\n",
+ "Finally, we can create the Endpoint using the Endpoint Configuration we created before.\n"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 440,
"id": "33e8d886",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'EndpointArn': 'arn:aws:sagemaker:us-east-1:325223348818:endpoint/shadow-penguins-endpoint',\n",
+ " 'ResponseMetadata': {'RequestId': '65f7c1b4-06ae-4ace-ad66-83a3bacf8511',\n",
+ " 'HTTPStatusCode': 200,\n",
+ " 'HTTPHeaders': {'x-amzn-requestid': '65f7c1b4-06ae-4ace-ad66-83a3bacf8511',\n",
+ " 'content-type': 'application/x-amz-json-1.1',\n",
+ " 'content-length': '92',\n",
+ " 'date': 'Sat, 30 Mar 2024 21:25:49 GMT'},\n",
+ " 'RetryAttempts': 0}}"
+ ]
+ },
+ "execution_count": 440,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"sagemaker_client.create_endpoint(\n",
" EndpointName=SHADOW_DEPLOYMENT_ENDPOINT,\n",
@@ -8734,12 +9494,48 @@
")"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "248702a3",
+ "metadata": {},
+ "source": [
+ "### Step 5 - Generating Traffic\n",
+ "\n",
+ "Let's generate some traffic to the endpoint so we can test the Shadow variant.\n"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 442,
"id": "069afdae",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"predictions\": [\n",
+ " [\n",
+ " 0.0403208546,\n",
+ " 0.0210227184,\n",
+ " 0.93865639\n",
+ " ],\n",
+ " [\n",
+ " 0.689678669,\n",
+ " 0.17514421,\n",
+ " 0.135177106\n",
+ " ],\n",
+ " [\n",
+ " 0.960919619,\n",
+ " 0.0248175282,\n",
+ " 0.0142629147\n",
+ " ]\n",
+ " ]\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"payload = \"\"\"\n",
"0.6569590202313976,-1.0813829646495108,1.2097102831892812,0.9226343641317372,1.0,0.0,0.0\n",
@@ -8747,21 +9543,86 @@
"-0.837387834894918,0.3386660813829646,-0.26237731892812,-1.92351941317372,0.0,0.0,1.0\n",
"\"\"\"\n",
"\n",
- "predictor = Predictor(endpoint_name=SHADOW_DEPLOYMENT_ENDPOINT)\n",
+ "predictor = Predictor(\n",
+ " endpoint_name=SHADOW_DEPLOYMENT_ENDPOINT,\n",
+ " serializer=CSVSerializer(),\n",
+ " deserializer=JSONDeserializer(),\n",
+ ")\n",
"\n",
"try:\n",
- " response = predictor.predict(payload, initial_args={\"ContentType\": \"text/csv\"})\n",
- " response = json.loads(response.decode(\"utf-8\"))\n",
- "\n",
+ " response = predictor.predict(payload)\n",
" print(json.dumps(response, indent=2))\n",
- " print(f\"\\nSpecies: {np.argmax(response['predictions'], axis=1)}\")\n",
"except Exception as e:\n",
" print(e)"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "04a5f044",
+ "metadata": {},
+ "source": [
+ "### Step 6 - Checking Captured Data\n",
+ "\n",
+ "Let's check the location where the endpoint stores the captured data, download a file, and display its content. It may take a few minutes for the first few files to show up in S3.\n"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 450,
+ "id": "9ffadf06",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "File: s3://mlschool/penguins/endpoint/shadow-penguins-endpoint/ShadowTraffic/2024/03/30/21/28-43-624-8f47e605-6bd2-44dd-bd91-293f29fd227e.jsonl\n",
+ "{\n",
+ " \"captureData\": {\n",
+ " \"endpointInput\": {\n",
+ " \"observedContentType\": \"text/csv\",\n",
+ " \"mode\": \"INPUT\",\n",
+ " \"data\": \"\\n0.6569590202313976,-1.0813829646495108,1.2097102831892812,0.9226343641317372,1.0,0.0,0.0\\n-0.7751048801481084,0.8822689351285553,-1.2168066120762704,0.9226343641317372,0.0,1.0,0.0\\n-0.837387834894918,0.3386660813829646,-0.26237731892812,-1.92351941317372,0.0,0.0,1.0\\n\",\n",
+ " \"encoding\": \"CSV\"\n",
+ " },\n",
+ " \"endpointOutput\": {\n",
+ " \"observedContentType\": \"application/json\",\n",
+ " \"mode\": \"OUTPUT\",\n",
+ " \"data\": \"{ \\\"predictions\\\": [[0.124825425, 0.0847824216, 0.79039216], [0.766525269, 0.220783874, 0.0126908608], [0.944253445, 0.0292692278, 0.0264772158] ]}\",\n",
+ " \"encoding\": \"JSON\"\n",
+ " }\n",
+ " },\n",
+ " \"eventMetadata\": {\n",
+ " \"eventId\": \"98c3c22e-20af-401c-9ca6-6d67d734a83f\",\n",
+ " \"invocationSource\": \"ShadowExperiment\",\n",
+ " \"inferenceTime\": \"2024-03-30T21:28:43Z\"\n",
+ " },\n",
+ " \"eventVersion\": \"0\"\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "files = S3Downloader.list(SHADOW_DATA_DESTINATION)\n",
+ "if len(files):\n",
+ " lines = S3Downloader.read_file(files[-1])\n",
+ " print(f\"File: {files[-1]}\")\n",
+ " print(json.dumps(json.loads(lines.split(\"\\n\")[0]), indent=2))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b8535163",
+ "metadata": {},
+ "source": [
+ "### Step 7 - Deleting the Endpoint\n",
+ "\n",
+ "Let's now delete the endpoint.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 451,
"id": "2ee1ce12",
"metadata": {},
"outputs": [],
@@ -8796,7 +9657,7 @@
},
{
"cell_type": "code",
- "execution_count": 299,
+ "execution_count": 340,
"id": "59d1e634",
"metadata": {
"tags": [
@@ -8807,19 +9668,19 @@
{
"data": {
"text/plain": [
- "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session14-pipeline/execution/zma7heuo2ood', sagemaker_session=)"
+ "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session17-pipeline/execution/o2a1if9pise8', sagemaker_session=)"
]
},
- "execution_count": 299,
+ "execution_count": 340,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "# %%script false --no-raise-error\n",
+ "%%script false --no-raise-error\n",
"# | eval: false\n",
"\n",
- "session14_pipeline.start()"
+ "session3_pipeline.start()"
]
},
{
@@ -8842,14 +9703,23 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 404,
"id": "0df2cbb1",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker:Deleting Monitoring Schedule with name: penguins-data-monitoring-schedule\n",
+ "INFO:sagemaker.model_monitor.model_monitoring:Deleting Data Quality Job Definition with name: data-quality-job-definition-2024-03-30-17-36-05-840\n"
+ ]
+ }
+ ],
"source": [
"try:\n",
" data_monitor.delete_monitoring_schedule()\n",
@@ -8867,14 +9737,23 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 405,
"id": "dee9a67f",
"metadata": {
"tags": [
"hide-output"
]
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:sagemaker:Deleting Monitoring Schedule with name: penguins-model-monitoring-schedule\n",
+ "INFO:sagemaker.model_monitor.model_monitoring:Deleting Model Quality Job Definition with name: model-quality-job-definition-2024-03-30-18-54-07-230\n"
+ ]
+ }
+ ],
"source": [
"try:\n",
" model_monitor.delete_monitoring_schedule()\n",
@@ -8892,7 +9771,7 @@
},
{
"cell_type": "code",
- "execution_count": 304,
+ "execution_count": 406,
"id": "6b32c3a4-312e-473c-a217-33606f77d1e9",
"metadata": {
"tags": [
diff --git a/program/images/shadow-deployment.png b/program/images/shadow-deployment.png
new file mode 100644
index 0000000..4a6f56b
Binary files /dev/null and b/program/images/shadow-deployment.png differ
diff --git a/pyproject.toml b/pyproject.toml
index 6e40426..79ce93b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,4 +11,5 @@ skip-magic-trailing-comma = false
line-ending = "auto"
[tool.ruff.lint]
-ignore = ["E402"]
\ No newline at end of file
+select = ["ALL"]
+ignore = ["ANN001", "ANN201", "ANN202", "B006", "BLE001", "E402", "PD901", "PLR0913", "S311", "T201"]
\ No newline at end of file