130 model results loading improvements

130 model results loading improvements

backend/controller/models.py

@@ -10,6 +10,17 @@
 #############################################################################
 
 
+def get_model_id_list(dataProviderId, projectId, modelId):
+ """
+ Get the list of models for a project
+ """
+ try:
+ data_provider = data_provider_manager.get_single_data_provider(dataProviderId)
+ return list(data_provider.get_model_results_id_list(projectId, modelId)), 200
+ except DataProviderException as e:
+ return e.message, e.status_code
+
+
 def get_results(dataProviderId, projectId, modelId, data):
  """
  Get the model results from a sample list

backend/controller/projects.py

@@ -55,27 +55,6 @@ def get_projects():
  return projectOverviews, 200
 
 
-def get_projects():
- # return a list of project overviews
- data_providers_list = data_provider_manager.get_data_provider_list()
- projectOverviews = []
- for data_provider in data_providers_list:
- try:
- projects = data_provider.get_projects()
-
- if projects is not None:
- # Adding data provider id to projects
- for project in projects:
- project["dataProviderId"] = data_provider.name
-
- projectOverviews.extend(projects)
-
- except DataProviderException as e:
- print("Warning get DP projects : " + e.message)
-
- return projectOverviews, 200
-
-
 def get_project(dataProviderId, projectId):
  # return the info about datasets, models, selections & tags
  try:
@@ -91,6 +70,23 @@ def get_project(dataProviderId, projectId):
  return e.message, e.status_code
 
 
+def get_data_id_list(dataProviderId, projectId, requestParameters):
+ # return the list of data ids
+ try:
+ data_provider = data_provider_manager.get_single_data_provider(dataProviderId)
+
+ data_id_list = data_provider.get_id_list(
+ projectId,
+ requestParameters["analysis"],
+ requestParameters["from"],
+ requestParameters["to"],
+ )
+
+ return data_id_list, 200
+ except DataProviderException as e:
+ return e.message, e.status_code
+
+
 def delete_project(dataProviderId, projectId):
  # Delete a project
  try:

backend/controller/selection.py

@@ -18,6 +18,14 @@ def get_selections(dataProviderId, projectId):
  return e.message, e.status_code
 
 
+def get_selection_id_list(dataProviderId, projectId, selectionId):
+ try:
+ data_provider = data_provider_manager.get_single_data_provider(dataProviderId)
+ return data_provider.get_selection_id_list(projectId, selectionId), 200
+ except DataProviderException as e:
+ return e.message, e.status_code
+
+
 def post_selection(dataProviderId, projectId, data):
  try:
  data_provider = data_provider_manager.get_single_data_provider(dataProviderId)

backend/modules/dataProviders/webDataProvider/cache/cache.py

@@ -3,42 +3,13 @@
 # It will mainly save the id list of samples, selections and models results
 # The ability to cache and the time to live are configurable in the config file
 
-import time
 from config.init_config import get_config
+from cacheout import Cache as CacheoutCache
 
 
 class Cache:
  def __init__(self):
- self.cache = {
- "project_id_list": {
- # <id_project>: {
- # <from>_<to>: {
- # id_list: [...],
- # timestamp: <timestamp>
- # },
- # total: {
- # id_list: [...],
- # timestamp: <timestamp>
- # }
- # }
- },
- "selection_id_list": {
- # <id_project>: {
- # <id_selection>: {
- # id_list: [...],
- # timestamp: <timestamp>
- # }
- # }
- },
- "model_result_id_list": {
- # <id_project>: {
- # <id_model>: {
- # id_list: [...],
- # timestamp: <timestamp>
- # }
- # }
- },
- }
+ # Get config
  self.config = get_config()
 
  self.cache_enabled = self.config["DATA_PROVIDERS_CONFIG"][
@@ -48,137 +19,76 @@ def __init__(self):
  "web_data_provider_cache_duration"
  ]
 
- # Project id list
- def get_id_list(self, id_project, _from=None, _to=None):
- if not self.cache_enabled:
-  return None
+  # Init cache
+  self.project_id_list_cache = CacheoutCache(maxsize=256, ttl=self.cache_ttl)
+ # <id_project>_<from>_<to>: [...]
+ # <id_project>_total: [...]
 
- if id_project not in self.cache["project_id_list"]:
-  return None
+ self.selection_id_list_cache = CacheoutCache(maxsize=256, ttl=self.cache_ttl)
+ # <id_project>_<id_selection>: [...]
 
- project_cache = self.cache["project_id_list"][id_project]
+ self.model_result_id_list_cache = CacheoutCache(maxsize=256, ttl=self.cache_ttl)
+ # <id_project>_<id_model>: [...]
 
+ # Project id list
+ def get_key(self, id_project, _from=None, _to=None):
  if _from is None or _to is None:
- if "total" not in project_cache:
- return None
-
- # Check if cache is still valid
- timestamp = project_cache["total"]["timestamp"]
- if time.time() - timestamp > self.cache_ttl:
- # Cache is not valid anymore
- # Delete cache
- del project_cache["total"]
- return None
-
- return project_cache["total"]["id_list"]
-
+ return "{}_total".format(id_project)
  else:
- key = "{}_{}".format(_from, _to)
- if key not in project_cache:
- return None
+ return "{}_{}_{}".format(id_project, _from, _to)
 
- # Check if cache is still valid
- timestamp = project_cache[key]["timestamp"]
- if time.time() - timestamp > self.cache_ttl:
- # Cache is not valid anymore
- # Delete cache
- del project_cache[key]
- return None
+ def get_id_list(self, id_project, _from=None, _to=None):
+ if not self.cache_enabled:
+ return None
+
+ key = self.get_key(id_project, _from, _to)
 
-  return project_cache[key]["id_list"]
+ return self.project_id_list_cache.get(key)
 
  def set_id_list(self, id_project, id_list, _from=None, _to=None):
  if not self.cache_enabled:
  return
 
- if id_project not in self.cache["project_id_list"]:
- self.cache["project_id_list"][id_project] = {}
+ key = self.get_key(id_project, _from, _to)
 
- project_cache = self.cache["project_id_list"][id_project]
-
- if _from is None or _to is None:
- project_cache["total"] = {
- "id_list": id_list,
- "timestamp": time.time(),
- }
- else:
- key = "{}_{}".format(_from, _to)
- project_cache[key] = {
- "id_list": id_list,
- "timestamp": time.time(),
- }
+ self.project_id_list_cache.set(key, id_list)
 
  # Selection id list
+ def get_selection_key(self, id_project, id_selection):
+ return "{}_{}".format(id_project, id_selection)
+
  def get_selection_id_list(self, id_project, id_selection):
  if not self.cache_enabled:
  return None
 
- if id_project not in self.cache["selection_id_list"]:
- return None
-
- project_cache = self.cache["selection_id_list"][id_project]
-
- if id_selection not in project_cache:
- return None
+ key = self.get_selection_key(id_project, id_selection)
 
- # Check if cache is still valid
- timestamp = project_cache[id_selection]["timestamp"]
-
- if time.time() - timestamp > self.cache_ttl:
- # Cache is not valid anymore
- # Delete cache
- del project_cache[id_selection]
- return None
-
- return project_cache[id_selection]["id_list"]
+ return self.selection_id_list_cache.get(key)
 
  def set_selection_id_list(self, id_project, id_selection, id_list):
  if not self.cache_enabled:
  return
 
- if id_project not in self.cache["selection_id_list"]:
- self.cache["selection_id_list"][id_project] = {}
-
- project_cache = self.cache["selection_id_list"][id_project]
+ key = self.get_selection_key(id_project, id_selection)
 
- project_cache[id_selection] = {
- "id_list": id_list,
- "timestamp": time.time(),
- }
+ self.selection_id_list_cache.set(key, id_list)
 
  # Model result id list
+ def get_model_result_key(self, id_project, id_model):
+ return "{}_{}".format(id_project, id_model)
+
  def get_model_result_id_list(self, id_project, id_model):
  if not self.cache_enabled:
  return None
 
- if id_project not in self.cache["model_result_id_list"]:
- return None
-
- project_cache = self.cache["model_result_id_list"][id_project]
-
- if id_model not in project_cache:
- return None
-
- # Check if cache is still valid
- timestamp = project_cache[id_model]["timestamp"]
- if time.time() - timestamp > self.cache_ttl:
- # Cache is not valid anymore
- # Delete cache
- del project_cache[id_model]
- return None
+ key = self.get_model_result_key(id_project, id_model)
 
- return project_cache[id_model]["id_list"]
+ return self.model_result_id_list_cache.get(key)
 
  def set_model_result_id_list(self, id_project, id_model, id_list):
  if not self.cache_enabled:
  return
 
- if id_project not in self.cache["model_result_id_list"]:
- self.cache["model_result_id_list"][id_project] = {}
-
- project_cache = self.cache["model_result_id_list"][id_project]
+ key = self.get_model_result_key(id_project, id_model)
 
- project_cache[id_model] = {
- "id_list": id_list,
- "timestamp": time.time(),
- }
+ self.model_result_id_list_cache.set(key, id_list)

backend/requirements.txt

@@ -9,4 +9,5 @@ ujson == 4.0.2
 sklearn == 0.0
 kafka-python == 2.0.2
 openapi_spec_validator == 0.2.8
-PyYAML == 6.0
+PyYAML == 6.0
+cacheout == 0.14.1