diff --git a/program/cohort.ipynb b/program/cohort.ipynb index 3781cf4..86418ef 100644 --- a/program/cohort.ipynb +++ b/program/cohort.ipynb @@ -7036,7 +7036,11 @@ "source": [ "## Session 16 - Data Quality Baseline\n", "\n", - "In this section, we'll extend the [SageMaker Pipeline](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-sdk.html) with a [Quality Check Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-quality-check) to compute a baseline for the data the endpoint expects. This step will compute statistics and constraints from the data. We will later use this information as the standard to detect data drift and other data quality issues.\n", + "This session extends the [SageMaker Pipeline](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-sdk.html) with a [Quality Check Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-quality-check) to compute a baseline for the data the endpoint expects.\n", + "\n", + "This step will compute statistics and constraints from the data. We'll' later use this information as the baseline to detect data drift and other data quality issues.\n", + "\n", + " \"Data\n", "\n", "Check [Monitor data quality](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-data-quality.html) for more information about monitoring data quality in SageMaker.\n" ] @@ -7053,7 +7057,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 130, "id": "f4747fc7", "metadata": {}, "outputs": [], @@ -7083,14 +7087,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 131, "id": "3fc728fa", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n", + "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n" + ] + } + ], "source": [ "from sagemaker.workflow.quality_check_step import (\n", " QualityCheckStep,\n", @@ -7099,6 +7112,7 @@ "from sagemaker.workflow.check_job_config import CheckJobConfig\n", "from sagemaker.model_monitor.dataset_format import DatasetFormat\n", "\n", + "\n", "data_quality_baseline_step = QualityCheckStep(\n", " name=\"generate-data-quality-baseline\",\n", " check_job_config=CheckJobConfig(\n", @@ -7134,7 +7148,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 132, "id": "2d5d8133", "metadata": {}, "outputs": [], @@ -7175,7 +7189,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 133, "id": "fc5c4325", "metadata": { "tags": [ @@ -7201,12 +7215,12 @@ "source": [ "### Step 5 - Modifying the Condition Step\n", "\n", - "Since we modified the registration step, we also need to modify the Condition Step to use the new registration:\n" + "Since we modified the Registration Step, we also need to modify the Condition Step to use the new registration:\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 134, "id": "be9d6de7", "metadata": {}, "outputs": [], @@ -7214,7 +7228,7 @@ "condition_step = ConditionStep(\n", " name=\"check-model-accuracy\",\n", " conditions=[condition],\n", - " if_steps=[register_model_step] if not LOCAL_MODE else [],\n", + " if_steps=[register_model_step],\n", " else_steps=[fail_step],\n", ")" ] @@ -7231,27 +7245,60 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 135, "id": "fb9b8d1e", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", + "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session16-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", + "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session16-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", + "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n", + " warnings.warn(\n", + "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", + "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", + "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session16-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", + "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session16-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" + ] + }, + { + "data": { + "text/plain": [ + "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session16-pipeline',\n", + " 'ResponseMetadata': {'RequestId': '07a1218c-b06c-4f38-87b5-27c741a397b4',\n", + " 'HTTPStatusCode': 200,\n", + " 'HTTPHeaders': {'x-amzn-requestid': '07a1218c-b06c-4f38-87b5-27c741a397b4',\n", + " 'content-type': 'application/x-amz-json-1.1',\n", + " 'content-length': '86',\n", + " 'date': 'Thu, 28 Mar 2024 18:15:21 GMT'},\n", + " 'RetryAttempts': 0}}" + ] + }, + "execution_count": 135, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "session16_pipeline = Pipeline(\n", " name=\"session16-pipeline\",\n", " parameters=[dataset_location, accuracy_threshold],\n", " steps=[\n", " preprocessing_step,\n", - " tune_model_step if USE_TUNING_STEP else train_model_step,\n", + " train_model_step,\n", " evaluate_model_step,\n", " data_quality_baseline_step,\n", " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=config[\"session\"],\n", + " sagemaker_session=pipeline_session,\n", ")\n", "\n", "session16_pipeline.upsert(role_arn=role)" @@ -7322,9 +7369,13 @@ "source": [ "## Session 17 - Model Quality Baseline\n", "\n", - "In this section, we'll extend the [SageMaker Pipeline](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-sdk.html) with a [Quality Check Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-quality-check) to compute a baseline for the model performance. This step will compute the baseline metrics we will later use as the standard to detect model drift.\n", + "This session extends the [SageMaker Pipeline](https://docs.aws.amazon.com/sagemaker/latest/dg/pipelines-sdk.html) with a [Quality Check Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-quality-check) to compute a baseline for the model performance.\n", "\n", - "To create a baseline to compare the model performance, we must create predictions for the test set and compare the model's metrics with the model performance on production data. We can do this by running a [Batch Transform Job](https://docs.aws.amazon.com/sagemaker/latest/dg/batch-transform.html) to predict every sample from the test set. We can use a [Transform Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-transform) as part of the pipeline to run this job. This Batch Transform Job will run every sample from the training dataset through the model so we can compute the baseline metrics. Check [Run a Batch Transform Job](https://sagemaker.readthedocs.io/en/stable/frameworks/tensorflow/using_tf.html#run-a-batch-transform-job) for more information about running a Batch Transform Job.\n", + "This step will compute the baseline metrics we will later use as the baseline to detect model drift.\n", + "\n", + "To create a baseline to compare the model performance, we must create predictions for the test set and compare the model's metrics with the model performance on production data. We can do this by running a [Batch Transform Job](https://docs.aws.amazon.com/sagemaker/latest/dg/batch-transform.html) to predict every sample from the test set. We can use a [Transform Step](https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-steps.html#step-type-transform) as part of the pipeline to run this job.\n", + "\n", + " \"Model\n", "\n", "Check [Monitor model quality](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality.html) for more information about monitoring model quality in SageMaker.\n" ] @@ -7341,7 +7392,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 137, "id": "69e115c3", "metadata": {}, "outputs": [], @@ -7361,17 +7412,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 139, "id": "f93f2751", "metadata": { - "tags": [ - "hide-output" - ] + "tags": [] }, "outputs": [], "source": [ - "from sagemaker.workflow.model_step import ModelStep\n", - "\n", "create_model_step = ModelStep(\n", " name=\"create-model\",\n", " step_args=pipeline_model.create(instance_type=config[\"instance_type\"]),\n", @@ -7387,12 +7434,20 @@ "\n", "We are going to use a [Batch Transform Job](https://docs.aws.amazon.com/sagemaker/latest/dg/batch-transform.html) to generate predictions for every sample from the test set.\n", "\n", + "This Batch Transform Job will run every sample from the training dataset through the model so we can compute the baseline metrics. Check [Run a Batch Transform Job](https://sagemaker.readthedocs.io/en/stable/frameworks/tensorflow/using_tf.html#run-a-batch-transform-job) for more information about running a Batch Transform Job.\n" + ] + }, + { + "cell_type": "markdown", + "id": "4227e304", + "metadata": {}, + "source": [ "Let's start by configuring a [Transformer](https://sagemaker.readthedocs.io/en/stable/api/inference/transformer.html) instance:\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 140, "id": "2a530d12", "metadata": {}, "outputs": [], @@ -7421,12 +7476,12 @@ "Notice the following:\n", "\n", "- We'll generate predictions for the baseline test data that we generated when we split and transformed the data. This baseline is the same data we used to test the model, but it's in raw format.\n", - "- The output of this Batch Transform Job will have two fields: the first one will be the ground truth label, and the second one will be the prediction of the model.\n" + "- The output of this Batch Transform Job will have two fields. The first one will be the ground truth label, and the second one will be the prediction of the model.\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 141, "id": "653c6ec8", "metadata": { "tags": [ @@ -7484,14 +7539,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 142, "id": "b50a9ed5", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: .\n", + "INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.\n" + ] + } + ], "source": [ "from sagemaker.workflow.quality_check_step import ModelQualityCheckConfig\n", "\n", @@ -7537,7 +7601,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 143, "id": "3aecc022", "metadata": {}, "outputs": [], @@ -7577,7 +7641,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 144, "id": "20154a1a", "metadata": { "tags": [ @@ -7603,12 +7667,12 @@ "source": [ "### Step 7 - Modifying the Condition Step\n", "\n", - "We need to modify the Condition Step to include the Transform Step and the Quality Step:\n" + "We need to modify the Condition Step to include the new Registration Step and the Transform and Quality Check Steps.\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 145, "id": "cd84e567", "metadata": {}, "outputs": [], @@ -7623,8 +7687,6 @@ " model_quality_baseline_step,\n", " register_model_step,\n", " ]\n", - " if not LOCAL_MODE\n", - " else []\n", " ),\n", " else_steps=[fail_step],\n", ")" @@ -7642,27 +7704,60 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 146, "id": "c244e206", "metadata": { "tags": [ "hide-output" ] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", + "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session17-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", + "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session17-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n", + "/Users/svpino/dev/ml.school/.venv/lib/python3.10/site-packages/sagemaker/workflow/pipeline_context.py:332: UserWarning: Running within a PipelineSession, there will be No Wait, No Logs, and No Job being started.\n", + " warnings.warn(\n", + "WARNING:sagemaker.workflow._utils:Popping out 'CertifyForMarketplace' from the pipeline definition since it will be overridden in pipeline execution time.\n", + "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n", + "INFO:sagemaker.processing:Uploaded None to s3://mlschool/session17-pipeline/code/c90fe2477ad62c58fbc0c004cc5ab07f/sourcedir.tar.gz\n", + "INFO:sagemaker.processing:runproc.sh uploaded to s3://mlschool/session17-pipeline/code/f3b7867d7495763812a03744135acb08/runproc.sh\n" + ] + }, + { + "data": { + "text/plain": [ + "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:325223348818:pipeline/session17-pipeline',\n", + " 'ResponseMetadata': {'RequestId': '84349ce8-4da7-46e2-9223-f1569f391299',\n", + " 'HTTPStatusCode': 200,\n", + " 'HTTPHeaders': {'x-amzn-requestid': '84349ce8-4da7-46e2-9223-f1569f391299',\n", + " 'content-type': 'application/x-amz-json-1.1',\n", + " 'content-length': '86',\n", + " 'date': 'Thu, 28 Mar 2024 18:27:25 GMT'},\n", + " 'RetryAttempts': 0}}" + ] + }, + "execution_count": 146, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "session17_pipeline = Pipeline(\n", " name=\"session17-pipeline\",\n", " parameters=[dataset_location, accuracy_threshold],\n", " steps=[\n", " preprocessing_step,\n", - " tune_model_step if USE_TUNING_STEP else train_model_step,\n", + " train_model_step,\n", " evaluate_model_step,\n", " data_quality_baseline_step,\n", " condition_step,\n", " ],\n", " pipeline_definition_config=pipeline_definition_config,\n", - " sagemaker_session=config[\"session\"],\n", + " sagemaker_session=pipeline_session,\n", ")\n", "\n", "session17_pipeline.upsert(role_arn=role)" @@ -8607,7 +8702,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 147, "id": "59d1e634", "metadata": { "tags": [ @@ -8618,10 +8713,10 @@ { "data": { "text/plain": [ - "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session15-pipeline/execution/89f4c9uyn8hf', sagemaker_session=)" + "_PipelineExecution(arn='arn:aws:sagemaker:us-east-1:325223348818:pipeline/session17-pipeline/execution/69zulyehahnk', sagemaker_session=)" ] }, - "execution_count": 125, + "execution_count": 147, "metadata": {}, "output_type": "execute_result" } @@ -8630,7 +8725,7 @@ "# %%script false --no-raise-error\n", "# | eval: false\n", "\n", - "session15_pipeline.start()" + "session17_pipeline.start()" ] }, { @@ -8703,7 +8798,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 129, "id": "6b32c3a4-312e-473c-a217-33606f77d1e9", "metadata": { "tags": [ diff --git a/program/images/data-quality-baseline.png b/program/images/data-quality-baseline.png new file mode 100644 index 0000000..8afa3b3 Binary files /dev/null and b/program/images/data-quality-baseline.png differ diff --git a/program/images/model-quality-baseline.png b/program/images/model-quality-baseline.png new file mode 100644 index 0000000..6f1a387 Binary files /dev/null and b/program/images/model-quality-baseline.png differ