health check + filer logs

routers/general.py

@@ -15,7 +15,7 @@
 from .lib.api import popular_ciks_request, top_ciks_request
 from .lib.backup import save_collections
 
-from .filer import create_filer_try, create_filer_replace
+from .filer import query_filer, create_filer_try, create_filer_replace
 
 environment = os.environ["ENVIRONMENT"]
 
@@ -40,6 +40,34 @@ async def info_undefined():
  return {"message": "Hello World!"}
 
 
+@router.get("/health", status_code=200)
+async def health():
+ health_checks = []
+
+ pipeline = [
+ {"$match": {}},
+ {"$addFields": {"randInt": {"$randInt": {}}}},
+ {"$sort": {"randInt": 1}},
+ {"$limit": 5},
+ ]
+ random_filers = database.search_filers(pipeline)
+ for filer in random_filers:
+ cik = filer["cik"]
+ try:
+ query_filer(cik)
+ health_checks.append(True)
+ except Exception as e:
+ create_error(cik, e)
+ health_checks.append(False)
+ continue
+
+ health_passed = sum(health_checks) / len(health_checks)
+ if health_passed < 0.8:
+ raise HTTPException(status_code=500, detail="The server doesn't seem healthy.")
+
+ return {"message": "The server is healthy."}
+
+
 def background_query(query_type, cik_list, background, query_function):
  query = cm.get_key(query_type)
  if query and query == "running":

routers/utils.py

@@ -211,13 +211,15 @@ def insert_search(document_list):
 
  print("Deleting In-Progress Filers ...")
  in_progress_logs = logs.find({"status": {"$gt": 0}}, {"cik": 1})
- in_progress = [log["cik"] for log in in_progress_logs]
+ in_progress = [log.get("cik", None) for log in in_progress_logs]
 
  logs.delete_many({"cik": {"$in": in_progress}})
  filers.delete_many({"cik": {"$in": in_progress}})
 
  print("Deleting Empty Logs ...")
- log_ciks = [log["cik"] for log in logs.find({}, {"cik": 1})]
+ log_ciks = list(
+ filter(lambda x: x, [log.get("cik", None) for log in logs.find({}, {"cik": 1})])
+ )
  log_filers = [
  filer["cik"] for filer in filers.find({"cik": {"$in": log_ciks}}, {"cik": 1})
  ]