From cff0c5e6c003b96884ff8a2ea38a6a66df689481 Mon Sep 17 00:00:00 2001
From: AmyNickollse <amy.nickolls@googlemail.com>
Date: Thu, 21 Mar 2024 14:52:03 +0000
Subject: [PATCH 1/2] historic view in navbar and historic chart added

---
 dm_regional_app/charts.py                          | 14 ++++++++++++++
 dm_regional_app/management/commands/sandbox.py     | 10 +++++-----
 .../templates/dm_regional_app/includes/navbar.html |  1 +
 .../templates/dm_regional_app/views/historic.html  |  2 +-
 dm_regional_app/urls.py                            |  1 +
 dm_regional_app/views.py                           | 11 +++++++++--
 6 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/dm_regional_app/charts.py b/dm_regional_app/charts.py
index 6edcf26..78404f9 100644
--- a/dm_regional_app/charts.py
+++ b/dm_regional_app/charts.py
@@ -1,6 +1,9 @@
 import plotly.express as px
 from demand_model.multinomial.predictor import Prediction
 
+from ssda903.datacontainer import DemandModellingDataContainer
+from ssda903.population_stats import PopulationStats
+
 
 def prediction_chart(prediction: Prediction):
     df_pp = prediction.population.unstack().reset_index()
@@ -11,3 +14,14 @@ def prediction_chart(prediction: Prediction):
     fig.update_layout(title="Prediction")
     fig_html = fig.to_html(full_html=False)
     return fig_html
+
+
+def historic_chart(data: PopulationStats):
+    df_pp = data.stock.unstack().reset_index()
+    df_pp.columns = ["from", "date", "value"]
+
+    # visualise prediction using unstacked dataframe
+    fig = px.line(df_pp, y="value", x="date", color="from")
+    fig.update_layout(title="Prediction")
+    fig_html = fig.to_html(full_html=False)
+    return fig_html
diff --git a/dm_regional_app/management/commands/sandbox.py b/dm_regional_app/management/commands/sandbox.py
index cf18af1..b6b0ff6 100644
--- a/dm_regional_app/management/commands/sandbox.py
+++ b/dm_regional_app/management/commands/sandbox.py
@@ -15,13 +15,13 @@
 
 class Command(BaseCommand):
     def handle(self, *args, **kwargs):
-        thing = User.objects.all()
+        """thing = User.objects.all()
         for t in thing:
-            print(t.profile.la)
+            print(t.profile.la)"""
 
-        """datastore = fs_datastore("sample://v1.zip")
+        datastore = fs_datastore("sample://v1.zip")
         config = Config()
         dc = DemandModellingDataContainer(datastore, config)
+        pop = PopulationStats(dc.enriched_view, config)
 
-        
-        print(dc.enriched_view)"""
+        print(pop.stock)
diff --git a/dm_regional_app/templates/dm_regional_app/includes/navbar.html b/dm_regional_app/templates/dm_regional_app/includes/navbar.html
index 38ddd0c..e5fcc2f 100644
--- a/dm_regional_app/templates/dm_regional_app/includes/navbar.html
+++ b/dm_regional_app/templates/dm_regional_app/includes/navbar.html
@@ -14,6 +14,7 @@
       <ul class="navbar-nav me-auto mb-2 mb-lg-0">
         {% if user.is_authenticated %}
           {% include 'dm_regional_app/includes/navbar-item.html' with url='dashboard' name='Dashboard' url_names="['dashboard']" %}
+          {% include 'dm_regional_app/includes/navbar-item.html' with url='historic_data' name='Historic Data' url_names="['historic_data']" %}
           {% include 'dm_regional_app/includes/navbar-item.html' with url='prediction' name='Prediction' url_names="['prediction']" %}
           {% include 'dm_regional_app/includes/navbar-item.html' with url='scenarios' name='Scenarios' url_names="['scenarios', 'scenario_detail']" %}
         {% else %}
diff --git a/dm_regional_app/templates/dm_regional_app/views/historic.html b/dm_regional_app/templates/dm_regional_app/views/historic.html
index f40f6cb..5162c2a 100644
--- a/dm_regional_app/templates/dm_regional_app/views/historic.html
+++ b/dm_regional_app/templates/dm_regional_app/views/historic.html
@@ -17,7 +17,7 @@ <h1>Total # of exiting care</h1>
     </div>
 </div>
 <div class="alert alert-primary" role="alert">
-    TODO - add the relevant charts here!
+    {{chart|safe}}
 
 </div>
 {% endblock %}
diff --git a/dm_regional_app/urls.py b/dm_regional_app/urls.py
index e8dee19..c6e23e2 100644
--- a/dm_regional_app/urls.py
+++ b/dm_regional_app/urls.py
@@ -8,4 +8,5 @@
     path("predict/", views.prediction, name="prediction"),
     path("scenario/<int:pk>", views.scenario_detail, name="scenario_detail"),
     path("scenarios", views.scenarios, name="scenarios"),
+    path("historic_data", views.historic_data, name="historic_data"),
 ]
diff --git a/dm_regional_app/views.py b/dm_regional_app/views.py
index 01bec6c..590ec05 100644
--- a/dm_regional_app/views.py
+++ b/dm_regional_app/views.py
@@ -3,9 +3,11 @@
 from django.contrib.auth.decorators import login_required
 from django.shortcuts import get_object_or_404, render
 
-from dm_regional_app.charts import prediction_chart
+from dm_regional_app.charts import historic_chart, prediction_chart
 from dm_regional_app.forms import HistoricDataFilter, PredictFilter
 from dm_regional_app.models import Scenario
+from ssda903 import Config
+from ssda903.population_stats import PopulationStats
 from ssda903.predictor import predict
 from ssda903.reader import read_data
 
@@ -98,7 +100,6 @@ def historic_data(request):
         )
         data = datacontainer.enriched_view
 
-    # TODO AMY - create the relevant charts from this dataset, according to the designs
     entry_into_care_count = data.loc[
         data.placement_type_before
         == datacontainer.config.PlacementCategories.NOT_IN_CARE
@@ -108,6 +109,11 @@ def historic_data(request):
         == datacontainer.config.PlacementCategories.NOT_IN_CARE
     ]["CHILD"].nunique()
 
+    config = Config()
+    stats = PopulationStats(data, config)
+
+    chart = historic_chart(stats)
+
     return render(
         request,
         "dm_regional_app/views/historic.html",
@@ -115,6 +121,7 @@ def historic_data(request):
             "form": form,
             "entry_into_care_count": entry_into_care_count,
             "exiting_care_count": exiting_care_count,
+            "chart": chart,
         },
     )
 

From 2091595778c97ed9f948a881fe059ec2cefa2984 Mon Sep 17 00:00:00 2001
From: AmyNickollse <amy.nickolls@googlemail.com>
Date: Wed, 10 Apr 2024 18:09:46 +0100
Subject: [PATCH 2/2] historic chart and filter form updated

---
 dm_regional_app/charts.py                     | 14 ++++-
 dm_regional_app/forms.py                      | 60 +++++++++++++++----
 .../management/commands/sandbox.py            |  6 +-
 .../dm_regional_app/views/historic.html       | 15 ++++-
 dm_regional_app/views.py                      |  2 +
 dm_regional_site/settings.py                  |  1 +
 poetry.lock                                   | 39 +++++++++++-
 pyproject.toml                                |  1 +
 ssda903/datacontainer.py                      |  7 ++-
 9 files changed, 125 insertions(+), 20 deletions(-)

diff --git a/dm_regional_app/charts.py b/dm_regional_app/charts.py
index 78404f9..df6f3f3 100644
--- a/dm_regional_app/charts.py
+++ b/dm_regional_app/charts.py
@@ -19,9 +19,19 @@ def prediction_chart(prediction: Prediction):
 def historic_chart(data: PopulationStats):
     df_pp = data.stock.unstack().reset_index()
     df_pp.columns = ["from", "date", "value"]
+    df = df_pp[["date", "value"]].groupby(by="date").sum().reset_index()
 
     # visualise prediction using unstacked dataframe
-    fig = px.line(df_pp, y="value", x="date", color="from")
-    fig.update_layout(title="Prediction")
+    fig = px.line(
+        df,
+        y="value",
+        x="date",
+        labels={
+            "value": "Number of children",
+            "date": "Date",
+        },
+    )
+    fig.update_layout(title="Historic data")
+    fig.update_yaxes(rangemode="tozero")
     fig_html = fig.to_html(full_html=False)
     return fig_html
diff --git a/dm_regional_app/forms.py b/dm_regional_app/forms.py
index 22453cd..8d333c8 100644
--- a/dm_regional_app/forms.py
+++ b/dm_regional_app/forms.py
@@ -3,6 +3,7 @@
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Column, Layout, Row, Submit
 from django import forms
+from django_select2 import forms as s2forms
 
 
 class PredictFilter(forms.Form):
@@ -29,35 +30,58 @@ class HistoricDataFilter(forms.Form):
         label="End Date",
         required=True,
     )
-    la = forms.ChoiceField(label="Local Authority", required=False, choices=[])
+    la = forms.MultipleChoiceField(
+        widget=s2forms.Select2MultipleWidget,
+        label="Local Authority",
+        required=False,
+        choices=[],
+    )
     placement_types = forms.MultipleChoiceField(
-        widget=forms.CheckboxSelectMultiple,
+        widget=s2forms.Select2MultipleWidget,
         label="Placement Type",
         required=False,
         choices=[],
     )
+    age_bins = forms.MultipleChoiceField(
+        widget=s2forms.Select2MultipleWidget,
+        label="Age",
+        required=False,
+        choices=[],
+    )
+    uasc = forms.ChoiceField(
+        label="UASC",
+        required=False,
+        choices=[("all", "All"), (True, "UASC only"), (False, "Exclude UASC")],
+        initial="all",
+    )
 
     def __init__(self, *args, **kwargs):
         la_choices = kwargs.pop("la_choices")
         placement_type_choices = kwargs.pop("placement_type_choices")
+        age_bin_choices = kwargs.pop("age_bin_choices")
 
         super().__init__(*args, **kwargs)
-        self.fields["la"].choices = [("all", "All")] + [(la, la) for la in la_choices]
-        self.fields["la"].initial = "all"
+        self.fields["la"].choices = [(la, la) for la in la_choices]
         self.fields["placement_types"].choices = [
             (placement_type, placement_type)
             for placement_type in placement_type_choices
         ]
+        self.fields["age_bins"].choices = [
+            (age_bin, age_bin) for age_bin in age_bin_choices
+        ]
+
         self.helper = FormHelper()
         self.helper.layout = Layout(
             Row(
-                Column("start_date", css_class="form-group col-md-6 mb-0"),
-                Column("end_date", css_class="form-group col-md-6 mb-0"),
+                Column("start_date", css_class="form-group col-md-3 mb-0"),
+                Column("end_date", css_class="form-group col-md-3 mb-0"),
                 css_class="form-row",
             ),
             Row(
-                Column("la", css_class="form-group col-md-6 mb-0"),
-                Column("placement_types", css_class="form-group col-md-4 mb-0"),
+                Column("la", css_class="form-group col-md-3 mb-0"),
+                Column("placement_types", css_class="form-group col-md-3 mb-0"),
+                Column("age_bins", css_class="form-group col-md-3 mb-0"),
+                Column("uasc", css_class="form-group col-md-3 mb-0"),
                 css_class="form-row",
             ),
             Submit("submit", "Filter"),
@@ -72,8 +96,9 @@ def filter_by_end_date(self, data: pd.DataFrame):
         return data
 
     def filter_by_la(self, data: pd.DataFrame):
-        if self.cleaned_data["la"] != "all":
-            data = data.loc[data.LA == self.cleaned_data["la"]]
+        if self.cleaned_data["la"] != []:
+            loc = data.LA.astype(str).isin(self.cleaned_data["la"])
+            data = data.loc[loc]
         return data
 
     def filter_by_placement_type(self, data: pd.DataFrame):
@@ -84,9 +109,24 @@ def filter_by_placement_type(self, data: pd.DataFrame):
             data = data.loc[loc]
         return data
 
+    def filter_by_age_bin(self, data: pd.DataFrame):
+        if self.cleaned_data["age_bins"] != []:
+            loc = data.age_bin.astype(str).isin(self.cleaned_data["age_bins"])
+            data = data.loc[loc]
+        return data
+
+    def filter_by_uasc(self, data: pd.DataFrame):
+        if self.cleaned_data["uasc"] == "True":
+            data = data.loc[data.UASC == True]
+        elif self.cleaned_data["uasc"] == "False":
+            data = data.loc[data.UASC == True]
+        return data
+
     def apply_filters(self, data: pd.DataFrame):
         data = self.filter_by_start_date(data)
         data = self.filter_by_end_date(data)
         data = self.filter_by_la(data)
         data = self.filter_by_placement_type(data)
+        data = self.filter_by_age_bin(data)
+        data = self.filter_by_uasc(data)
         return data
diff --git a/dm_regional_app/management/commands/sandbox.py b/dm_regional_app/management/commands/sandbox.py
index b6b0ff6..92f998c 100644
--- a/dm_regional_app/management/commands/sandbox.py
+++ b/dm_regional_app/management/commands/sandbox.py
@@ -24,4 +24,8 @@ def handle(self, *args, **kwargs):
         dc = DemandModellingDataContainer(datastore, config)
         pop = PopulationStats(dc.enriched_view, config)
 
-        print(pop.stock)
+        data = dc.enriched_view
+        print(data.loc[data.UASC == True])
+
+        # print(pop.stock)
+        # print(dc.enriched_view)
diff --git a/dm_regional_app/templates/dm_regional_app/views/historic.html b/dm_regional_app/templates/dm_regional_app/views/historic.html
index 5162c2a..bb56624 100644
--- a/dm_regional_app/templates/dm_regional_app/views/historic.html
+++ b/dm_regional_app/templates/dm_regional_app/views/historic.html
@@ -1,23 +1,32 @@
 {% extends "dm_regional_app/base.html" %}
 {% load crispy_forms_tags %}
 {% block content %}
+    <br>
+    <h1>Viewing the SSDA903 data</h1>
+    <br>
+    <p>Explore the make-up of the care population as represented in the SSDA903 returns. You can filter the data to see the pattern of new care episodes over time across dimensions such as placement type, provider type, child age, child ethnicity, child gender, and whether the new placement followed a breakdown.</p>
+    <br>
+    <p><i>Select the period of data in the SSDA903 dataset you want to explore</i></p>
     {{ form.media }}
     <form method="POST">
         {% csrf_token %}
         {% crispy form %}
     </form>
+    <br>
     <div class="row">
     <div class="col">
         <h1>Total # of entry into care</h1>
-        <p>{{ entry_into_care_count }}</p>
+        <p><h1 class="display-2">{{ entry_into_care_count }}</h1></p>
     </div>
     <div class="col">
         <h1>Total # of exiting care</h1>
-        <p>{{ exiting_care_count }}</p>
+        <p><h1 class="display-2">{{ exiting_care_count }}</h1></p>
     </div>
 </div>
 <div class="alert alert-primary" role="alert">
     {{chart|safe}}
-
 </div>
+<div class="d-flex bd-highlight mb-3">
+    <div class="ms-auto p-2 bd-highlight"><a class="btn btn-primary" href="{% url 'prediction' %}">Next</a></div>
+  </div>
 {% endblock %}
diff --git a/dm_regional_app/views.py b/dm_regional_app/views.py
index 590ec05..5b494fa 100644
--- a/dm_regional_app/views.py
+++ b/dm_regional_app/views.py
@@ -80,6 +80,7 @@ def historic_data(request):
             request.POST,
             la_choices=datacontainer.unique_las,
             placement_type_choices=datacontainer.unique_placement_types,
+            age_bin_choices=datacontainer.unique_age_bins,
         )
         if form.is_valid():
             data = form.apply_filters(datacontainer.enriched_view)
@@ -97,6 +98,7 @@ def historic_data(request):
             },
             la_choices=datacontainer.unique_las,
             placement_type_choices=datacontainer.unique_placement_types,
+            age_bin_choices=datacontainer.unique_age_bins,
         )
         data = datacontainer.enriched_view
 
diff --git a/dm_regional_site/settings.py b/dm_regional_site/settings.py
index e4c023f..57933ea 100644
--- a/dm_regional_site/settings.py
+++ b/dm_regional_site/settings.py
@@ -42,6 +42,7 @@
     "crispy_forms",
     "crispy_bootstrap5",
     "bootstrap_datepicker_plus",
+    "django_select2",
 ]
 
 MIDDLEWARE = [
diff --git a/poetry.lock b/poetry.lock
index 424f9bb..52b7c94 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
 
 [[package]]
 name = "annotated-types"
@@ -209,6 +209,20 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""}
 argon2 = ["argon2-cffi (>=19.1.0)"]
 bcrypt = ["bcrypt"]
 
+[[package]]
+name = "django-appconf"
+version = "1.0.6"
+description = "A helper class for handling configuration defaults of packaged apps gracefully."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "django-appconf-1.0.6.tar.gz", hash = "sha256:cfe87ea827c4ee04b9a70fab90b86d704cb02f2981f89da8423cb0fabf88efbf"},
+    {file = "django_appconf-1.0.6-py3-none-any.whl", hash = "sha256:c3ae442fba1ff7ec830412c5184b17169a7a1e71cf0864a4c3f93cf4c98a1993"},
+]
+
+[package.dependencies]
+django = "*"
+
 [[package]]
 name = "django-bootstrap-datepicker-plus"
 version = "5.0.5"
@@ -239,6 +253,26 @@ files = [
 [package.dependencies]
 django = ">=4.2"
 
+[[package]]
+name = "django-select2"
+version = "8.1.2"
+description = "This is a Django_ integration of Select2_."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "django_select2-8.1.2-py3-none-any.whl", hash = "sha256:dc09e09299988dd7cc1f31c27582976377f0e1479101be1f3bb0d1f24216477f"},
+    {file = "django_select2-8.1.2.tar.gz", hash = "sha256:f44685ee1c39090aade01e3ebc256702f05620f3c78a3c268440ad9a66070876"},
+]
+
+[package.dependencies]
+django = ">=3.2"
+django-appconf = ">=0.6.0"
+
+[package.extras]
+docs = ["sphinx"]
+selenium = ["selenium"]
+test = ["pytest", "pytest-cov", "pytest-django", "selenium"]
+
 [[package]]
 name = "djlint"
 version = "1.34.1"
@@ -759,6 +793,7 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -1028,4 +1063,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "5d425972cf892e0f95bd0a0c57932c4f95633a7475075cefc5b39d70949b0b62"
+content-hash = "b3854e905ee165969fb2bbe11e87948c7faacf6b244ae7883988ad497ae07e9e"
diff --git a/pyproject.toml b/pyproject.toml
index 4bb9184..eb763d8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,6 +16,7 @@ crispy-bootstrap5 = "^2024.2"
 python-decouple = "^3.8"
 faker = "^23.3.0"
 django-bootstrap-datepicker-plus = "^5.0.5"
+django-select2 = "^8.1.2"
 
 
 [tool.poetry.group.dev.dependencies]
diff --git a/ssda903/datacontainer.py b/ssda903/datacontainer.py
index 85cc0c3..dff14ce 100644
--- a/ssda903/datacontainer.py
+++ b/ssda903/datacontainer.py
@@ -4,7 +4,6 @@
 from functools import cached_property
 from typing import Optional
 
-
 import numpy as np
 import pandas as pd
 
@@ -117,7 +116,7 @@ def combined_datasets(self) -> pd.DataFrame:
         merged = merged.merge(uasc[["CHILD", "DUC"]], how="left", on="CHILD")
 
         # create UASC flag if DECOM is less than DUC
-        merged["UASC"] = np.where(merged["DECOM"] < merged["DUC"], 1, 0)
+        merged["UASC"] = np.where(merged["DECOM"] < merged["DUC"], True, False)
 
         return merged
 
@@ -203,6 +202,10 @@ def unique_las(self) -> pd.Series:
     def unique_placement_types(self) -> pd.Series:
         return self.enriched_view.placement_type.unique()
 
+    @cached_property
+    def unique_age_bins(self) -> pd.Series:
+        return self.enriched_view.age_bin.unique()
+
     def _add_ages(self, combined: pd.DataFrame) -> pd.DataFrame:
         """
         Calculates the age of the child at the start and end of the episode and adds them as columns